diff --git a/.dockerignore b/.dockerignore index 840f292e..411edd75 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,7 @@ npm-debug.log app/tests local-only + +# Local developer logs +/TODO.md +/FRONTEND_TODO.md diff --git a/.gitignore b/.gitignore index 91ad882e..f5cff9f4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,13 @@ # A place to store artifacts during local development (scripts, datasets, dotenv files, etc.) .nocommit/**/* +# Local planning and handoff logs; never commit or force-add these files. +/TODO.md +/FRONTEND_TODO.md +# Keep the retired documentation paths from being recreated. +/docs/developer/TODO.md +/docs/developer/FRONTEND_TODO.md + # Machine-specific agent configuration (copy AGENTS.local.example.md to create) AGENTS.local.md diff --git a/AGENTS.md b/AGENTS.md index 5f75a714..e0333265 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,11 +15,11 @@ before searching the filesystem for any of the resources below. API), `attack-workbench-deployment` (Docker Compose configs), `attack-workbench-taxii-server`. Use them when you need consumer or deployment context. -- **ADM (ATT&CK Data Model)** — this API validates STIX objects against the - published `@mitre-attack/attack-data-model` package. A local checkout of the - ADM source (`src/schemas/{sdo,sro,smo,common}`) is the authoritative - reference for STIX shapes: valid enum values, required fields, refinements. - Consult it when authoring payloads, especially for regression tests. +- **ADM (ATT&CK Data Model)** — the installed + `@mitre-attack/attack-data-model` package determines runtime validation. + Consult its Zod schemas when authoring payloads and regression tests. + A local source checkout is useful for investigation; check its version + against the installed package before treating its shapes as current. - **Bruno API collection** — manual smoke-test requests maintained outside this repo (see Bruno section below). @@ -95,10 +95,12 @@ Tests use `mongodb-memory-server` — no external MongoDB or env setup needed. ## Task workflow -1. **Plan in a committable scratchpad**: track multi-step work as checkboxes in - `docs/developer/TODO.md` so progress survives context-window resets and - sessions. Check items off as they complete. Throwaway artifacts (notes, - datasets, one-off scripts) go in `.nocommit/` (gitignored). +1. **Keep working notes local**: use the gitignored root `TODO.md` for plans + and `FRONTEND_TODO.md` for temporary frontend handoffs. Create them only + when needed; a fresh clone need not contain either file. Never stage or + force-add them, recreate planning logs under `docs/`, or link published + documentation to local notes. Other scratch artifacts (scripts, datasets, + session notes) belong in `.nocommit/` (gitignored). 2. **Definition of done** — a task is complete only when it includes: - implementation, - regression tests (see below), @@ -106,8 +108,10 @@ Tests use `mongodb-memory-server` — no external MongoDB or env setup needed. with `npm run test:file -- ` while iterating, then run the **full** `npm test` suite — all of it must pass before the task is done, - OpenAPI spec updates for any API-surface change, - - documentation updates (`docs/user/**` = what the behavior *is*; - `docs/developer/**` = why/how, including how behavior evolved), + - documentation of current behavior in `docs/user/**` and `docs/admin/**`, + and contracts/design decisions in `docs/developer/**`; keep task checklists, + execution transcripts, test-run tallies, and proposed commit messages in + local notes or the final response, not tracked documentation, - Bruno collection updates for any API-surface change, - a proposed conventional commit message. 3. **Commits**: conventional commits are enforced (commitlint + diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 841d80ef..225f862a 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -52,6 +52,18 @@ components: type: object description: 'A snapshot document for a release track, containing versioned member objects and workflow tiers' properties: + draft_cleanup: + $ref: '#/components/schemas/draft-cleanup-result' + snapshot_count: + type: integer + minimum: 0 + readOnly: true + description: Current registry total, not the size of a filtered history page. + tagged_release_count: + type: integer + minimum: 0 + readOnly: true + description: Current registry release total for virtual Workbench responses. alias: type: string nullable: true @@ -153,7 +165,7 @@ components: composition: nullable: true description: 'Component track references (virtual tracks only)' - $ref: '#/components/schemas/composition' + $ref: '#/components/schemas/stored-composition' composition_resolution: $ref: '#/components/schemas/composition-resolution' snapshot_schedule: @@ -347,8 +359,8 @@ components: readOnly: true description: | Immutable provenance for a materialized virtual snapshot. Each - component_snapshots entry names the exact tagged standard-track - snapshot used, including its version and creation timestamp. Null or + component_snapshots entry names the exact standard-track snapshot used, + including its version (null for a draft) and creation timestamp. Null or absent means the composition was configured but not materialized. Duplicate counts include STIX IDs contributed by more than one component, while conflicts include only genuinely different revisions. @@ -392,13 +404,17 @@ components: description: 'Exact component snapshot timestamp; also identifies when that snapshot was created' resolved_version: type: string - description: 'Tagged version of the exact component snapshot' + nullable: true + description: 'Tagged version of the exact component snapshot, or null for a draft' strategy_used: type: string enum: - latest_tagged + - latest_preview + - latest_draft - specific_version - specific_snapshot + description: 'latest_draft appears only in historical members-only provenance; it is not an active resolution strategy.' filters_applied: type: object additionalProperties: false @@ -414,6 +430,7 @@ components: total_objects_in_source: type: integer minimum: 0 + description: 'Prospective release member count for a preview draft, otherwise the published member count; before filters and cross-component deduplication.' objects_after_filter: type: integer minimum: 0 @@ -620,6 +637,61 @@ components: description: 'How to handle conflicts when tagging (promoting staged to members)' default: 'abort' + stored-composition: + type: object + readOnly: true + description: | + Persisted composition rules. Historical latest_draft rules are returned + unchanged, but must be explicitly replaced with latest_preview or + latest_tagged before another materialization. + properties: + component_tracks: + type: array + items: + oneOf: + - $ref: '#/components/schemas/component-track' + - $ref: '#/components/schemas/retired-draft-component' + deduplication: + type: object + properties: + strategy: + type: string + enum: + - prioritize_latest_object + - prioritize_latest_snapshot + - prioritize_higher_priority + - quarantine + + retired-draft-component: + type: object + readOnly: true + description: 'Read-only historical members-only rule; rejected in composition requests.' + required: + - track_id + - resolution_strategy + - priority + properties: + track_id: + type: string + resolution_strategy: + type: string + enum: + - latest_draft + priority: + type: integer + minimum: 0 + filters: + type: object + properties: + object_types: + type: array + items: + type: string + domains: + type: array + items: + type: string + composition: type: object description: 'Virtual track composition (references to component standard tracks)' @@ -652,9 +724,13 @@ components: type: object description: | Reference to a component standard track. Selector fields are determined - by resolution_strategy: latest_tagged rejects version and snapshot; - specific_version requires only version; specific_snapshot requires only - snapshot. Unknown properties are rejected. + by resolution_strategy: latest_tagged and latest_preview reject version and + snapshot; specific_version requires only version; specific_snapshot requires + only snapshot and remains tagged-only. latest_preview selects the newest + standard snapshot: published members when tagged, otherwise prospective + release members including staged changes under the source conflict policy. + Candidates are excluded. Filters and deduplication follow membership planning. + latest_draft is retired and rejected. Unknown properties are rejected. additionalProperties: false required: - track_id @@ -673,6 +749,7 @@ components: type: string enum: - latest_tagged + - latest_preview - specific_version - specific_snapshot description: 'How to resolve which snapshot to use from this component' @@ -732,6 +809,17 @@ components: - version - required: - snapshot + - title: 'Latest preview' + properties: + resolution_strategy: + enum: + - latest_preview + not: + anyOf: + - required: + - version + - required: + - snapshot - title: 'Specific release version' required: - version @@ -795,11 +883,12 @@ components: type: object description: | Virtual releases only. Immutable provenance keyed by component - release-track ID; each value is the tagged component version frozen - in the released draft's composition_resolution. Standard release - history entries omit this property. + release-track ID; each value is the tagged component version or null + for a draft source frozen in the released draft's composition_resolution. + Standard release history entries omit this property. additionalProperties: type: string + nullable: true pattern: '^\d+\.\d+$' example: release-track--a1b2c3d4-e5f6-7890-abcd-ef1234567890: '5.2' @@ -928,6 +1017,146 @@ components: type: string format: date-time + draft-retention: + type: object + additionalProperties: false + required: [max_drafts] + description: | + Virtual tracks only. A manual materialization may supply this policy + for that operation only; recurring policy belongs inside the cron schedule. + Missing policy or max_drafts null disables cleanup. All untagged creation + causes count across the track. Tags and protected sources always survive. + Saving a cron policy creates no draft and deletes nothing. + properties: + max_drafts: + type: integer + nullable: true + minimum: 1 + maximum: 9007199254740991 + + draft-squash: + type: object + readOnly: true + required: [lower_bound, upper_bound, eligible_count, protected_count, fingerprint] + description: | + Read-only preview. Bounds are exclusive snapshot modified timestamps, + not tagging times. Null lower_bound means the first release. + Submit the fingerprint to the exact upper_bound snapshot endpoint. + properties: + lower_bound: + type: string + format: date-time + nullable: true + upper_bound: + type: string + format: date-time + eligible_count: + type: integer + minimum: 0 + protected_count: + type: integer + minimum: 0 + fingerprint: + type: string + + draft-cleanup-result: + type: object + readOnly: true + required: [operation_id, status, kind, eligible_count, deleted_count, protected_count] + description: | + Nonpersistent operation response, backed by a durable audit intent. + A successful snapshot or release remains committed when cleanup is + pending or failed. Retry this operation, never repeat release POST. + properties: + operation_id: + type: string + format: uuid + status: + type: string + enum: [pending, completed, failed] + kind: + type: string + enum: [retention, squash] + eligible_count: + type: integer + minimum: 0 + deleted_count: + type: integer + minimum: 0 + protected_count: + type: integer + minimum: 0 + target_modified: + type: string + format: date-time + release_committed: + type: boolean + error: + type: string + + release-request: + type: object + additionalProperties: false + properties: + increment: + type: string + enum: [major, minor] + description: Mutually exclusive with version. + version: + type: string + pattern: '^\d+\.\d+$' + description: + type: string + maxLength: 4000 + squash_drafts: + type: boolean + default: false + description: | + Administrator-only virtual-track opt-in. Finish publication before + deleting earlier eligible drafts in the reviewed interval. + squash_fingerprint: + type: string + minLength: 1 + maxLength: 128 + description: Required when squash_drafts is true; stale previews return 409 before tagging. + + release-summary: + type: object + required: + [track_id, type, source_snapshot_modified, release_snapshot_modified, version, releasable] + properties: + track_id: + type: string + type: + type: string + enum: [standard, virtual] + source_snapshot_modified: + type: string + format: date-time + release_snapshot_modified: + type: string + format: date-time + version: + type: string + releasable: + type: boolean + draft_squash: + $ref: '#/components/schemas/draft-squash' + + draft-cleanup-error: + type: object + description: Release publication did not complete. Inspect release_committed and retry the existing cleanup operation to repair partial publication. + properties: + message: + type: string + operation_id: + type: string + format: uuid + release_committed: + type: boolean + draft_cleanup: + $ref: '#/components/schemas/draft-cleanup-result' + snapshot-schedule: description: | Virtual snapshot-creation schedule. Cron and explicit dates are @@ -957,6 +1186,8 @@ components: type: string description: 'Five-field UTC cron expression' example: '0 0 1 1,7 *' + draft_retention: + $ref: '#/components/schemas/draft-retention' - type: object additionalProperties: false required: diff --git a/app/api/definitions/components/system-configuration.yml b/app/api/definitions/components/system-configuration.yml index 791a0e85..2a39ab10 100644 --- a/app/api/definitions/components/system-configuration.yml +++ b/app/api/definitions/components/system-configuration.yml @@ -63,6 +63,165 @@ components: type: string example: 'Linux' + allowed-value-option: + type: object + additionalProperties: false + required: [value, enabled, objectTypes] + properties: + value: + type: string + minLength: 1 + description: Trimmed, case-sensitive value accepted by the installed backend ADM for every selected object type and domain, even when disabled. + enabled: + type: boolean + objectTypes: + type: array + minItems: 1 + uniqueItems: true + description: Supported object types to which this option applies. + items: + type: string + minLength: 1 + + allowed-value-rule: + type: object + additionalProperties: false + required: [propertyName, domainName, objectTypes, values, invalidValues] + properties: + propertyName: + type: string + example: x_mitre_platforms + domainName: + type: string + example: enterprise-attack + objectTypes: + type: array + uniqueItems: true + description: All ADM-supported object types for this property/domain, including those with no options. + items: + type: string + values: + type: array + description: Equal value and enabled pairs are grouped across object types. + items: + $ref: '#/components/schemas/allowed-value-option' + invalidValues: + type: array + description: Persisted legacy options rejected by the installed ADM. Never offered as dropdown choices; saving replaces these along with the rule. + items: + $ref: '#/components/schemas/allowed-value-invalid-option' + + allowed-value-rule-replace: + type: object + additionalProperties: false + required: [values] + properties: + values: + type: array + description: Complete replacement option set. An empty array clears the rule. + items: + $ref: '#/components/schemas/allowed-value-option' + + allowed-value-invalid-option: + type: object + additionalProperties: false + required: [value, enabled, objectTypes, reason] + properties: + value: + type: string + enabled: + type: boolean + objectTypes: + type: array + items: + type: string + reason: + type: string + + allowed-value-definition: + type: object + additionalProperties: false + required: [propertyName, domainName, objectTypes, valueType, choices, description] + properties: + propertyName: + type: string + domainName: + type: string + objectTypes: + type: array + items: + type: string + valueType: + type: string + enum: [enum, formatted] + choices: + type: array + description: ADM enum choices, or validated seed suggestions for formatted fields. Each choice lists its valid object types. + items: + type: object + required: [value, objectTypes] + additionalProperties: false + properties: + value: + type: string + objectTypes: + type: array + items: + type: string + description: + type: string + + allowed-value-catalog: + type: object + additionalProperties: false + required: [admVersion, rules] + properties: + admVersion: + type: string + rules: + type: array + items: + $ref: '#/components/schemas/allowed-value-definition' + + allowed-value-rule-create: + type: object + additionalProperties: false + required: [propertyName, domainName, values] + properties: + propertyName: + type: string + minLength: 1 + domainName: + type: string + minLength: 1 + values: + type: array + description: Initial ADM-valid options. Empty arrays create a persistent empty rule. + items: + $ref: '#/components/schemas/allowed-value-option' + + allowed-value-validate: + type: object + additionalProperties: false + required: [propertyName, domainName, objectTypes, value] + properties: + propertyName: + type: string + minLength: 1 + domainName: + type: string + minLength: 1 + objectTypes: + type: array + minItems: 1 + uniqueItems: true + items: + type: string + minLength: 1 + value: + type: string + minLength: 1 + organization-identity: type: object properties: diff --git a/app/api/definitions/openapi.yml b/app/api/definitions/openapi.yml index 6f70ed80..1f7f41ff 100644 --- a/app/api/definitions/openapi.yml +++ b/app/api/definitions/openapi.yml @@ -388,6 +388,12 @@ paths: /api/release-tracks/{id}/virtual/schedule: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1virtual~1schedule' + /api/release-tracks/{id}/virtual/draft-cleanup: + $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1virtual~1draft-cleanup' + + /api/release-tracks/{id}/virtual/draft-cleanup/{operationId}/retry: + $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1virtual~1draft-cleanup~1{operationId}~1retry' + /api/release-tracks/{id}/virtual/snapshots/create: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1virtual~1snapshots~1create' @@ -434,6 +440,18 @@ paths: /api/config/allowed-values: $ref: 'paths/system-configuration-paths.yml#/paths/~1api~1config~1allowed-values' + /api/config/allowed-values/catalog: + $ref: 'paths/system-configuration-paths.yml#/paths/~1api~1config~1allowed-values~1catalog' + + /api/config/allowed-values/validate: + $ref: 'paths/system-configuration-paths.yml#/paths/~1api~1config~1allowed-values~1validate' + + /api/config/allowed-values/rules: + $ref: 'paths/system-configuration-paths.yml#/paths/~1api~1config~1allowed-values~1rules' + + /api/config/allowed-values/rules/{propertyName}/{domainName}: + $ref: 'paths/system-configuration-paths.yml#/paths/~1api~1config~1allowed-values~1rules~1{propertyName}~1{domainName}' + /api/config/organization-identity: $ref: 'paths/system-configuration-paths.yml#/paths/~1api~1config~1organization-identity' diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 4f2d6a18..7aea0753 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -241,9 +241,22 @@ paths: draft snapshot and is limited to 4000 characters. `alias` is an optional URL-safe slug (2-64 lowercase letters, digits, and hyphens, unique across tracks) accepted wherever the track ID is. + Administrators may provide snapshot_schedule.draft_retention in the + cron variant only, with max_drafts a positive safe integer or null. + This applies only to future recurring materializations. Initial track + creation never deletes history; no top-level retention policy is accepted. tags: - 'Release Tracks' # Request body validation moved to Zod in controller + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + snapshot_schedule: + $ref: '../components/release-tracks.yml#/components/schemas/snapshot-schedule' responses: '201': description: 'Release track created successfully' @@ -253,6 +266,8 @@ paths: $ref: '../components/release-tracks.yml#/components/schemas/release-track-snapshot' '400': description: 'Invalid request parameters' + '403': + description: 'Only administrators may supply a retention policy' /api/release-tracks/new-from-bundle: post: @@ -402,6 +417,9 @@ paths: Relative increments use the latest tagged release. A historical standard draft is published as a new release at the current time and must follow the current version lineage. + Administrators may opt into virtual-only squash_drafts with the exact + preview squash_fingerprint. Stale previews fail before tagging. Successful + publication may return pending/failed draft_cleanup; use cleanup retry. tags: - 'Release Tracks' parameters: @@ -420,21 +438,28 @@ paths: content: application/json: schema: - type: object - additionalProperties: true + $ref: '../components/release-tracks.yml#/components/schemas/release-request' responses: '200': description: 'Snapshot released successfully' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/release-track-snapshot' + '403': + description: 'Only administrators may opt into draft squash' '400': description: 'Invalid release request' '409': description: 'Already released, another release is in progress, conflicting snapshot, or missing persisted primary revisions' '500': - description: 'The release may be tagged, but durable membership-protection reconciliation failed' + description: 'Publication or reconciliation failed; squash errors include operation_id and release_committed for cleanup-only recovery' content: application/json: schema: - $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' + anyOf: + - $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' + - $ref: '../components/release-tracks.yml#/components/schemas/draft-cleanup-error' /api/release-tracks/{id}/snapshots/latest/release/preview: get: @@ -497,6 +522,12 @@ paths: responses: '200': description: 'Release preview generated' + content: + application/json: + schema: + anyOf: + - $ref: '../components/release-tracks.yml#/components/schemas/release-summary' + - $ref: '../components/release-tracks.yml#/components/schemas/release-track-snapshot' '409': description: 'Virtual draft is unmaterialized, release is blocked by a conflict, or persisted primary revisions are missing' '501': @@ -763,6 +794,10 @@ paths: responses: '200': description: 'Configuration updated successfully' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/release-track-snapshot' # ============================================================================= # Object version history @@ -836,8 +871,8 @@ paths: The new pending draft has empty members and quarantine tiers and a null composition_resolution. Materialize it before release preview or commit. Request body is strictly validated via Zod. Unknown composition, - component, filter, and deduplication keys are rejected. latest_tagged - rejects selector fields; specific_version requires version; + component, filter, and deduplication keys are rejected. latest_tagged and + latest_preview reject selector fields; specific_version requires version; specific_snapshot requires snapshot. Component IDs and required non-negative integer priorities must each be unique. An optional strict scheduled_materialization object is persisted on the @@ -859,6 +894,10 @@ paths: responses: '200': description: 'Composition updated successfully' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/release-track-snapshot' '400': description: 'Track is not virtual or composition is invalid' @@ -871,8 +910,12 @@ paths: track without creating or mutating a content snapshot. Request bodies are strictly validated via Zod: manual accepts only mode, cron requires one five-field UTC expression, and dates requires at least one ISO UTC - timestamp. Scheduler reconciliation observes the replacement on its - next configured pass. + timestamp. Only cron accepts draft_retention, with max_drafts a positive + safe integer or null. Changing that threshold while remaining in cron + requires an administrator, including disabling retention. Editors may + preserve its value while editing cron timing, or switch away from cron. + Policy changes delete nothing. Scheduler reconciliation observes the + replacement on its next configured pass. tags: - 'Release Tracks' parameters: @@ -901,9 +944,80 @@ paths: $ref: '../components/release-tracks.yml#/components/schemas/snapshot-schedule' '400': description: 'Track is not virtual or schedule is invalid' + '403': + description: 'Changing cron retention requires an administrator' '404': description: 'Release track not found' + /api/release-tracks/{id}/virtual/draft-cleanup: + get: + summary: Discover pending or failed virtual draft cleanup + operationId: release-tracks-draft-cleanup-list + description: Returns the most recent 25 pending/failed audit-bound operations, including operations whose HTTP response was lost. + tags: ['Release Tracks'] + parameters: + - name: id + in: path + required: true + schema: + type: string + responses: + '200': + description: Recent incomplete cleanup operations + content: + application/json: + schema: + type: object + required: [data] + properties: + data: + type: array + items: + $ref: '../components/release-tracks.yml#/components/schemas/draft-cleanup-result' + '400': + description: Nonvirtual track + '404': + description: Track not found + + /api/release-tracks/{id}/virtual/draft-cleanup/{operationId}/retry: + post: + summary: Repair an existing draft cleanup operation + operationId: release-tracks-draft-cleanup-retry + description: | + Administrator-only cleanup recovery. Accepts no fresh selector or release + request. Repairs partial publication for the original release event and + never tags again. Rechecks protected sources, new tags, and live retention + policy without widening the recorded interval. Disabling retention permits + orphan/counter repair but no additional draft deletion. + tags: ['Release Tracks'] + parameters: + - name: id + in: path + required: true + schema: + type: string + - name: operationId + in: path + required: true + schema: + type: string + format: uuid + responses: + '200': + description: Current operation result, including any repair failure + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/draft-cleanup-result' + '400': + description: Nonvirtual track + '403': + description: Administrator required + '404': + description: Track or existing cleanup intent not found + '409': + description: Another lifecycle operation holds the track lock + /api/release-tracks/{id}/virtual/snapshots/create: post: summary: 'Create a virtual track snapshot' @@ -924,6 +1038,10 @@ paths: Component release locks are held from resolution through persistence. A concurrent release, rollback, retag, or materialization sharing a component may return 409; retry after the competing operation finishes. + Administrators may supply draft_retention for this operation only. + Omitted policy, null, or max_drafts null means no cleanup. Manual + requests never inherit or persist recurring policy, and supplied + scheduled_materialization metadata cannot invoke cron retention. tags: - 'Release Tracks' parameters: @@ -943,13 +1061,30 @@ paths: type: string maxLength: 4000 description: 'Optional notes for this snapshot' + draft_retention: + type: object + nullable: true + additionalProperties: false + required: [max_drafts] + properties: + max_drafts: + type: integer + nullable: true + minimum: 1 + maximum: 9007199254740991 scheduled_materialization: $ref: '../components/release-tracks.yml#/components/schemas/scheduled-materialization' responses: '201': description: 'Virtual snapshot created successfully' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/release-track-snapshot' '400': description: 'Track is not virtual or cannot resolve its composition' + '403': + description: 'Supplying manual draft retention requires an administrator' '409': description: 'A component release lock is busy or a resolved snapshot references missing primary revisions' content: @@ -1022,6 +1157,11 @@ paths: summaries also expose bundle_id and bundle_hashes. Standard releases with a preserved source draft also expose release_source_modified, allowing clients to hide that retained draft. + Counts describe every snapshot matching the tagged filter before + pagination, including when the requested page is empty. Latest snapshot + identities are unfiltered live track metadata, independent of the page + and tagged filter. The latest tagged identity is chronological, not + the highest semantic version. tags: - 'Release Tracks' parameters: @@ -1063,6 +1203,9 @@ paths: required: - data - pagination + - counts + - latest_snapshot_modified + - latest_tagged_snapshot_modified properties: data: type: array @@ -1070,6 +1213,34 @@ paths: oneOf: - $ref: '../components/release-tracks.yml#/components/schemas/standard-snapshot-summary' - $ref: '../components/release-tracks.yml#/components/schemas/virtual-snapshot-summary' + counts: + type: object + description: 'Counts across all snapshots matching the tagged filter, before pagination' + required: + - tagged + - drafts + - total + properties: + tagged: + type: integer + minimum: 0 + drafts: + type: integer + minimum: 0 + total: + type: integer + minimum: 0 + description: 'Sum of tagged and drafts; equals pagination.total' + latest_snapshot_modified: + type: string + format: date-time + nullable: true + description: 'Latest snapshot identity for the track, independent of filtering and pagination; null when absent' + latest_tagged_snapshot_modified: + type: string + format: date-time + nullable: true + description: 'Chronologically latest tagged snapshot identity for the track, independent of filtering and pagination; null when absent' pagination: type: object required: @@ -1463,6 +1634,10 @@ paths: selected draft is historical. An optional `description` is stored as snapshot-local release notes. Relative increments use the latest tagged release. + Explicit virtual squash requires an administrator and the preview + squash_fingerprint. Its exclusive chronological interval never includes + the target, a newer draft, or a tagged release. Publication completes + before cleanup; failures expose an existing cleanup operation for repair. tags: - 'Release Tracks' parameters: @@ -1486,21 +1661,28 @@ paths: content: application/json: schema: - type: object - additionalProperties: true + $ref: '../components/release-tracks.yml#/components/schemas/release-request' responses: '200': description: 'Snapshot released successfully' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/release-track-snapshot' + '403': + description: 'Only administrators may opt into draft squash' '400': description: 'Invalid release request' '409': description: 'Already released, another release is in progress, conflicting snapshot, or missing persisted primary revisions' '500': - description: 'The release may be tagged, but durable membership-protection reconciliation failed' + description: 'Publication or reconciliation failed; squash errors include operation_id and release_committed for cleanup-only recovery' content: application/json: schema: - $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' + anyOf: + - $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' + - $ref: '../components/release-tracks.yml#/components/schemas/draft-cleanup-error' put: summary: 'Change a release version' operationId: 'release-tracks-release-retag' @@ -1615,6 +1797,12 @@ paths: responses: '200': description: 'Release preview generated' + content: + application/json: + schema: + anyOf: + - $ref: '../components/release-tracks.yml#/components/schemas/release-summary' + - $ref: '../components/release-tracks.yml#/components/schemas/release-track-snapshot' '409': description: 'Release is blocked because the draft is unmaterialized, conflicting, or references missing primary revisions' '501': diff --git a/app/api/definitions/paths/system-configuration-paths.yml b/app/api/definitions/paths/system-configuration-paths.yml index 4afa3b33..5b68b01e 100644 --- a/app/api/definitions/paths/system-configuration-paths.yml +++ b/app/api/definitions/paths/system-configuration-paths.yml @@ -23,7 +23,11 @@ paths: summary: 'Get the list of domain-specific allowed values' operationId: 'config-get-allowed-values' description: | - This endpoint gets a list of domain-specific allowed values for certain properties. + Returns enabled, ADM-compliant runtime values in the existing nested shape. + Configured field/domain tuples remain present even when empty. Invalid legacy + settings are excluded regardless of validation flags or bypass rules. + Changes are visible without restarting the API; existing STIX objects are not changed. + Existing visitor-or-higher and read-only service access is unchanged. tags: - 'System Configuration' responses: @@ -36,6 +40,162 @@ paths: items: $ref: '../components/system-configuration.yml#/components/schemas/allowed-values' + /api/config/allowed-values/rules: + get: + summary: 'List runtime allowed-value rules' + operationId: 'config-list-allowed-value-rules' + description: | + Administrator only. Returns one rule per configured propertyName/domainName, + across object types, including persistent empty rules. Object types come from + the backend ADM catalog. Equal value and enabled pairs are grouped; differing + enabled states remain disjoint variants. Invalid legacy options appear only + in invalidValues with reasons, not in selectable values. + tags: + - 'System Configuration' + responses: + '200': + description: 'Configured rules with ADM-compliant enabled and disabled options, plus quarantined legacy options.' + content: + application/json: + schema: + type: array + items: + $ref: '../components/system-configuration.yml#/components/schemas/allowed-value-rule' + '401': + description: 'Authentication or administrator role required.' + post: + summary: 'Create an allowed-value property/domain rule' + operationId: 'config-create-allowed-value-rule' + description: | + Administrator only. Creates a previously unconfigured catalog-supported group. + Enabled and disabled options must conform to the installed backend ADM. + Empty rules persist. Concurrent creation of the same group has one winner. + General validation flags and validation bypass rules do not disable these checks. + tags: + - 'System Configuration' + requestBody: + required: true + content: + application/json: + schema: + $ref: '../components/system-configuration.yml#/components/schemas/allowed-value-rule-create' + responses: + '201': + description: 'Created configured rule.' + content: + application/json: + schema: + $ref: '../components/system-configuration.yml#/components/schemas/allowed-value-rule' + '400': + description: 'Invalid ADM property, domain, scope, or value.' + '401': + description: 'Authentication or administrator role required.' + '409': + description: 'Rule already exists, or overlapping duplicate option.' + + /api/config/allowed-values/catalog: + get: + summary: 'Get the authoritative backend ADM allowed-values catalog' + operationId: 'config-get-allowed-value-catalog' + description: | + Administrator only. Lists all ADM-permitted domains for the original supported + object/property pairs, not arbitrary ADM fields. Enum choices come directly + from installed schemas and formatted suggestions are validated seed values. + Identity uses the stix domain. Includes installed ADM version. + tags: + - 'System Configuration' + responses: + '200': + description: 'ADM-supported definitions, including unconfigured groups.' + content: + application/json: + schema: + $ref: '../components/system-configuration.yml#/components/schemas/allowed-value-catalog' + '401': + description: 'Authentication or administrator role required.' + + /api/config/allowed-values/validate: + post: + summary: 'Validate an allowed value without changing configuration' + operationId: 'config-validate-allowed-value' + description: | + Administrator only. Applies the same unconditional ADM scope and value checks + as final writes and returns the trimmed value. The group need not be configured. + tags: + - 'System Configuration' + requestBody: + required: true + content: + application/json: + schema: + $ref: '../components/system-configuration.yml#/components/schemas/allowed-value-validate' + responses: + '200': + description: 'Validated, trimmed value.' + content: + application/json: + schema: + type: object + additionalProperties: false + required: [value] + properties: + value: + type: string + '400': + description: 'Invalid ADM property, domain, scope, or value.' + '401': + description: 'Authentication or administrator role required.' + + /api/config/allowed-values/rules/{propertyName}/{domainName}: + parameters: + - name: propertyName + in: path + required: true + schema: + type: string + minLength: 1 + - name: domainName + in: path + required: true + schema: + type: string + minLength: 1 + put: + summary: 'Replace the complete option set for an allowed-value rule' + operationId: 'config-replace-allowed-value-rule' + description: | + Administrator only. Atomically replaces only this property/domain group's + values, retaining unrelated rules even during concurrent saves. The rule + must already be configured; each enabled or disabled option requires a nonempty + ADM-valid trimmed value, a boolean enabled state, and distinct supported objectTypes. + The same value cannot occur more than once for an object type, including disabled + options. Disjoint variants are allowed. Empty values clear the rule without + removing it. Saving also replaces any quarantined legacy options. + ADM checks cannot be disabled by general validation flags or bypass rules. + tags: + - 'System Configuration' + requestBody: + required: true + content: + application/json: + schema: + $ref: '../components/system-configuration.yml#/components/schemas/allowed-value-rule-replace' + responses: + '200': + description: 'Full updated rule, including all supported object types.' + content: + application/json: + schema: + $ref: '../components/system-configuration.yml#/components/schemas/allowed-value-rule' + '400': + description: 'Invalid request body, unsupported rule, or ADM-invalid value or scope.' + '401': + description: 'Authentication or administrator role required.' + '404': + description: 'The catalog-supported rule has not been configured.' + '409': + description: 'A value is repeated for the same object type within this rule.' + /api/config/organization-identity: get: summary: 'Get the designated organization identity' diff --git a/app/config/allowed-values.json b/app/config/allowed-values.json index a731a2fc..9d0cba74 100644 --- a/app/config/allowed-values.json +++ b/app/config/allowed-values.json @@ -112,7 +112,7 @@ "domains": [ { "domainName": "enterprise-attack", - "allowedValues": ["Administrator", "root", "SYSTEM", "User", "Remote Desktop Users"] + "allowedValues": ["Administrator", "root", "SYSTEM", "User"] } ] }, diff --git a/app/controllers/allowed-values-controller.js b/app/controllers/allowed-values-controller.js new file mode 100644 index 00000000..abe50b05 --- /dev/null +++ b/app/controllers/allowed-values-controller.js @@ -0,0 +1,75 @@ +'use strict'; + +const { z } = require('zod'); +const service = require('../services/system/allowed-values-service'); +const { BadRequestError } = require('../exceptions'); + +const optionSchema = z + .object({ + value: z.string().trim().min(1), + enabled: z.boolean(), + objectTypes: z + .array(z.string().min(1)) + .min(1) + .refine((types) => new Set(types).size === types.length, { + message: 'Object types must be distinct.', + }), + }) + .strict(); +const replaceSchema = z.object({ values: z.array(optionSchema) }).strict(); +const ruleSchema = z + .object({ propertyName: z.string().min(1), domainName: z.string().min(1) }) + .strict(); +const createSchema = ruleSchema.extend({ values: z.array(optionSchema) }).strict(); +const validateSchema = ruleSchema + .extend({ value: optionSchema.shape.value, objectTypes: optionSchema.shape.objectTypes }) + .strict(); + +function parse(schema, value) { + const result = schema.safeParse(value); + if (!result.success) throw new BadRequestError({ details: result.error.issues }); + return result.data; +} + +exports.retrieveRules = async function (req, res, next) { + try { + return res.status(200).send(await service.retrieveRules()); + } catch (err) { + return next(err); + } +}; + +exports.replaceRule = async function (req, res, next) { + try { + const { propertyName, domainName } = parse(ruleSchema, req.params); + const { values } = parse(replaceSchema, req.body); + return res.status(200).send(await service.replaceRule(propertyName, domainName, values)); + } catch (err) { + return next(err); + } +}; + +exports.retrieveCatalog = async function (req, res, next) { + try { + return res.status(200).send(service.retrieveCatalog()); + } catch (err) { + return next(err); + } +}; + +exports.createRule = async function (req, res, next) { + try { + const { propertyName, domainName, values } = parse(createSchema, req.body); + return res.status(201).send(await service.createRule(propertyName, domainName, values)); + } catch (err) { + return next(err); + } +}; + +exports.validateValue = async function (req, res, next) { + try { + return res.status(200).send(service.validateValue(parse(validateSchema, req.body))); + } catch (err) { + return next(err); + } +}; diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index ee9b1b41..83ccd372 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -373,6 +373,7 @@ exports.createReleaseTrack = async function createReleaseTrack(req, res, next) { const result = await releaseTracksService.createTrack({ ...bodyResult.data, userAccountId: req.user?.userAccountId, + actor: destructiveActor(req), }); logger.debug(`Success: Created release track "${bodyResult.data.name}"`); return res.status(201).send(result); @@ -527,6 +528,7 @@ exports.releaseLatest = async function releaseLatest(req, res, next) { const result = await releaseTracksService.releaseLatest(req.params.id, { ...bodyResult.data, userAccountId: req.user?.userAccountId, + actor: destructiveActor(req), }); logger.debug(`Success: Released latest snapshot for track ${req.params.id}`); return res.status(200).send(result); @@ -626,6 +628,7 @@ exports.releaseByModified = async function releaseByModified(req, res, next) { { ...bodyResult.data, userAccountId: req.user?.userAccountId, + actor: destructiveActor(req), }, ); logger.debug(`Success: Released snapshot ${req.params.modified}`); @@ -1085,7 +1088,11 @@ exports.updateSchedule = async function updateSchedule(req, res, next) { ); } - const result = await releaseTracksService.updateSchedule(req.params.id, bodyResult.data); + const result = await releaseTracksService.updateSchedule( + req.params.id, + bodyResult.data, + destructiveActor(req), + ); logger.debug(`Success: Updated snapshot schedule for track ${req.params.id}`); return res.status(200).send(result); } catch (err) { @@ -1094,6 +1101,30 @@ exports.updateSchedule = async function updateSchedule(req, res, next) { } }; +exports.listDraftCleanup = async function listDraftCleanup(req, res, next) { + try { + return res.status(200).send(await releaseTracksService.listDraftCleanup(req.params.id)); + } catch (error) { + return next(error); + } +}; + +exports.retryDraftCleanup = async function retryDraftCleanup(req, res, next) { + try { + return res + .status(200) + .send( + await releaseTracksService.retryDraftCleanup( + req.params.id, + req.params.operationId, + destructiveActor(req), + ), + ); + } catch (error) { + return next(error); + } +}; + /** POST /api/release-tracks/:id/virtual/snapshots/create */ exports.createVirtualSnapshot = async function createVirtualSnapshot(req, res, next) { try { @@ -1113,6 +1144,7 @@ exports.createVirtualSnapshot = async function createVirtualSnapshot(req, res, n ...snapshotOptions, scheduledMaterialization, userAccountId: req.user?.userAccountId, + actor: destructiveActor(req), }); logger.debug(`Success: Created virtual snapshot for track ${req.params.id}`); return res.status(201).send(result); diff --git a/app/exceptions/index.js b/app/exceptions/index.js index 2f672542..e9fceb52 100644 --- a/app/exceptions/index.js +++ b/app/exceptions/index.js @@ -335,6 +335,15 @@ class ReleaseTrackReconciliationError extends CustomError { } } +class ReleasePublicationError extends CustomError { + constructor(options) { + super( + 'Release publication did not complete; inspect or retry the existing cleanup operation', + options, + ); + } +} + class ReleaseTrackAuditError extends CustomError { constructor(trackId, auditEventId, options = {}) { super('Release-track audit recording could not be finalized', { @@ -487,6 +496,7 @@ module.exports = { ReleaseContentIntegrityError, ReleaseTrackReconciliationError, ReleaseTrackAuditError, + ReleasePublicationError, NoTaggedSnapshotsError, InvalidComponentTypeError, VirtualSnapshotNotMaterializedError, diff --git a/app/lib/allowed-values-catalog.js b/app/lib/allowed-values-catalog.js new file mode 100644 index 00000000..2918b40c --- /dev/null +++ b/app/lib/allowed-values-catalog.js @@ -0,0 +1,161 @@ +'use strict'; + +const { attackDomainSchema } = require('@mitre-attack/attack-data-model/dist/index.cjs'); +const { version: admVersion } = require('@mitre-attack/attack-data-model/package.json'); +const seed = require('../config/allowed-values.json'); +const { getSchema } = require('./validation-schemas'); +const { BadRequestError } = require('../exceptions'); + +const ruleKey = ({ propertyName, domainName }) => JSON.stringify([propertyName, domainName]); +const pairKey = (objectType, propertyName) => JSON.stringify([objectType, propertyName]); +const stixTypes = { + analytic: ['x-mitre-analytic'], + technique: ['attack-pattern'], + software: ['tool', 'malware'], + 'data-source': ['x-mitre-data-source'], + asset: ['x-mitre-asset'], + identity: ['identity'], +}; + +function unwrap(schema) { + while (schema.type === 'optional') schema = schema.unwrap(); + return schema; +} + +// Only the original object/property pairs are configurable. Domains and enum +// choices come from the installed ADM, never from copied vocabulary constants. +const pairs = new Map(); +const initialRules = new Map(); +for (const { objectType, properties } of seed) { + for (const { propertyName, domains } of properties) { + const schemas = stixTypes[objectType].map((type) => { + const objectSchema = getSchema(type, 'work-in-progress'); + const propertySchema = + propertyName === 'related_asset_sectors' + ? unwrap(objectSchema.shape.x_mitre_related_assets).element.shape.related_asset_sectors + : unwrap(objectSchema.shape[propertyName]); + return { + objectSchema, + propertySchema, + valueSchema: propertySchema.type === 'array' ? propertySchema.element : propertySchema, + }; + }); + pairs.set(pairKey(objectType, propertyName), { + objectType, + propertyName, + schemas, + suggestions: [...new Set(domains.flatMap(({ allowedValues }) => allowedValues))], + }); + for (const { domainName } of domains) { + const rule = { propertyName, domainName }; + initialRules.set(ruleKey(rule), rule); + } + } +} + +function valueIssue(pair, domainName, value) { + if (!pair) return 'Unsupported object type/property pair.'; + if ( + pair.objectType === 'identity' + ? domainName !== 'stix' + : !attackDomainSchema.options.includes(domainName) + ) { + return 'Unsupported domain for this object type.'; + } + for (const { objectSchema, propertySchema, valueSchema } of pair.schemas) { + const parsed = valueSchema.safeParse(value); + if (!parsed.success) return parsed.error.issues.map(({ message }) => message).join('; '); + const fieldValue = propertySchema.type === 'array' ? [value] : value; + const object = + pair.propertyName === 'related_asset_sectors' + ? { + x_mitre_related_assets: [ + { + name: 'Related asset', + description: 'Related asset sector', + related_asset_sectors: fieldValue, + }, + ], + } + : { [pair.propertyName]: fieldValue }; + if (pair.objectType !== 'identity') object.x_mitre_domains = [domainName]; + const contextual = objectSchema.safeParse(object); + if (!contextual.success) { + return contextual.error.issues.map(({ message }) => message).join('; '); + } + } + return null; +} + +const definitions = new Map(); +for (const pair of pairs.values()) { + const valueSchema = pair.schemas[0].valueSchema; + const candidates = valueSchema.options || pair.suggestions; + const domains = pair.objectType === 'identity' ? ['stix'] : attackDomainSchema.options; + for (const domainName of domains) { + const choices = candidates.filter((value) => !valueIssue(pair, domainName, value)); + if (!choices.length) continue; + const key = ruleKey({ propertyName: pair.propertyName, domainName }); + if (!definitions.has(key)) { + definitions.set(key, { + propertyName: pair.propertyName, + domainName, + objectTypes: [], + valueType: valueSchema.options ? 'enum' : 'formatted', + choices: [], + description: pair.schemas[0].propertySchema.description || valueSchema.description || '', + }); + } + const definition = definitions.get(key); + definition.objectTypes.push(pair.objectType); + for (const value of choices) { + let choice = definition.choices.find((candidate) => candidate.value === value); + if (!choice) { + choice = { value, objectTypes: [] }; + definition.choices.push(choice); + } + choice.objectTypes.push(pair.objectType); + } + } +} + +function freeze(value) { + for (const child of Object.values(value)) { + if (child && typeof child === 'object') freeze(child); + } + return Object.freeze(value); +} + +const catalog = freeze({ admVersion, rules: [...definitions.values()] }); +const seededRules = freeze([...initialRules.values()]); + +function requireDefinition(propertyName, domainName) { + const definition = definitions.get(ruleKey({ propertyName, domainName })); + if (!definition) throw new BadRequestError({ details: 'Unsupported allowed-values rule.' }); + return definition; +} + +function getValueIssue({ objectType, propertyName, domainName, value }) { + if (typeof value !== 'string' || !value.trim()) return 'Value must be a nonempty string.'; + return valueIssue(pairs.get(pairKey(objectType, propertyName)), domainName, value); +} + +function validateValue({ propertyName, domainName, objectTypes, value }) { + const definition = requireDefinition(propertyName, domainName); + if ( + !Array.isArray(objectTypes) || + !objectTypes.length || + new Set(objectTypes).size !== objectTypes.length || + objectTypes.some((type) => !definition.objectTypes.includes(type)) + ) { + throw new BadRequestError({ details: 'Invalid object-type scope.' }); + } + const trimmed = typeof value === 'string' ? value.trim() : value; + for (const objectType of objectTypes) { + const reason = getValueIssue({ objectType, propertyName, domainName, value: trimmed }); + if (reason) throw new BadRequestError({ details: reason }); + } + return { value: trimmed }; +} + +module.exports = { catalog, seededRules, ruleKey, requireDefinition, getValueIssue, validateValue }; diff --git a/app/lib/database-configuration.js b/app/lib/database-configuration.js index 78b611fe..29fafc66 100644 --- a/app/lib/database-configuration.js +++ b/app/lib/database-configuration.js @@ -272,6 +272,7 @@ async function checkForStaticBypassRules() { exports.checkSystemConfiguration = async function () { logger.info(`Performing system configuration check...`); + await require('../services/system/allowed-values-service').initialize(); await checkForOrganizationIdentity(); await checkForAnonymousUserAccount(); await checkForInvalidEnterpriseCollectionId(); diff --git a/app/lib/error-handler.js b/app/lib/error-handler.js index 0d644433..3f98cfed 100644 --- a/app/lib/error-handler.js +++ b/app/lib/error-handler.js @@ -46,6 +46,7 @@ const { ReleaseContentIntegrityError, ReleaseTrackReconciliationError, ReleaseTrackAuditError, + ReleasePublicationError, NoTaggedSnapshotsError, InvalidComponentTypeError, VirtualSnapshotNotMaterializedError, @@ -172,7 +173,8 @@ exports.serviceExceptions = function (err, req, res, next) { err instanceof GenericServiceError || err instanceof DatabaseError || err instanceof ReleaseTrackReconciliationError || - err instanceof ReleaseTrackAuditError + err instanceof ReleaseTrackAuditError || + err instanceof ReleasePublicationError ) { logger.error('Service error: %s', JSON.stringify(buildErrorResponse(err))); return res.status(500).send(buildErrorResponse(err)); diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index 59858552..0080b2df 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -230,7 +230,12 @@ const deduplicationStrategySchema = z.enum([ 'quarantine', ]); -const resolutionStrategySchema = z.enum(['latest_tagged', 'specific_version', 'specific_snapshot']); +const resolutionStrategySchema = z.enum([ + 'latest_tagged', + 'latest_preview', + 'specific_version', + 'specific_snapshot', +]); const conflictPolicySchema = z.enum([ 'prefer_latest', @@ -303,6 +308,10 @@ const updateConfigBodySchema = z.object({ // Request body schemas (used inline by controller handlers) // ============================================================================= +const draftRetentionSchema = z + .object({ max_drafts: z.number().int().positive().max(Number.MAX_SAFE_INTEGER).nullable() }) + .strict(); + /** POST /release-tracks/new */ const snapshotScheduleSchema = z.discriminatedUnion('mode', [ z @@ -314,6 +323,7 @@ const snapshotScheduleSchema = z.discriminatedUnion('mode', [ .object({ mode: z.literal('cron'), cron: cronSchema, + draft_retention: draftRetentionSchema.optional(), }) .strict(), z @@ -365,6 +375,12 @@ const componentTrackSchema = z.discriminatedUnion('resolution_strategy', [ resolution_strategy: z.literal('latest_tagged'), }) .strict(), + z + .object({ + ...componentTrackBaseShape, + resolution_strategy: z.literal('latest_preview'), + }) + .strict(), z .object({ ...componentTrackBaseShape, @@ -489,6 +505,8 @@ const releaseBodySchema = z increment: releaseIncrementSchema.optional(), version: xMitreVersionSchema.optional(), description: snapshotDescriptionSchema.optional(), + squash_drafts: z.boolean().optional(), + squash_fingerprint: z.string().min(1).max(128).optional(), }) .strict() .refine((value) => !(value.increment && value.version), { @@ -572,6 +590,7 @@ const updateCompositionBodySchema = z const createVirtualSnapshotBodySchema = z .object({ description: snapshotDescriptionSchema.optional(), + draft_retention: draftRetentionSchema.nullable().optional(), scheduled_materialization: scheduledMaterializationSchema.optional(), }) .strict() @@ -700,6 +719,7 @@ module.exports = { componentTrackSchema, compositionSchema, snapshotScheduleSchema, + draftRetentionSchema, scheduledMaterializationSchema, objectRefEntrySchema, promotionConflictsSchema, diff --git a/app/models/allowed-values-configuration-model.js b/app/models/allowed-values-configuration-model.js new file mode 100644 index 00000000..c6a7dbfd --- /dev/null +++ b/app/models/allowed-values-configuration-model.js @@ -0,0 +1,35 @@ +'use strict'; + +const mongoose = require('mongoose'); + +const entrySchema = new mongoose.Schema({ + objectType: { type: String, required: true }, + propertyName: { type: String, required: true }, + domainName: { type: String, required: true }, + value: { type: String, required: true }, + enabled: { type: Boolean, required: true }, +}); + +// A singleton makes first-time seeding and each administrator mutation atomic, +// including on standalone MongoDB deployments without transaction support. +const configurationSchema = new mongoose.Schema( + { + _id: { type: String, required: true }, + values: { type: [entrySchema], required: true }, + rules: { + type: [ + new mongoose.Schema( + { + propertyName: { type: String, required: true }, + domainName: { type: String, required: true }, + }, + { _id: false }, + ), + ], + default: undefined, + }, + }, + { bufferCommands: false, versionKey: false }, +); + +module.exports = mongoose.model('AllowedValuesConfiguration', configurationSchema); diff --git a/app/models/release-tracks/release-track-audit-event-model.js b/app/models/release-tracks/release-track-audit-event-model.js index dad63313..57b635fa 100644 --- a/app/models/release-tracks/release-track-audit-event-model.js +++ b/app/models/release-tracks/release-track-audit-event-model.js @@ -9,7 +9,14 @@ const releaseTrackAuditEventSchema = new mongoose.Schema( action: { type: String, required: true, - enum: ['delete_track', 'delete_release', 'retag_release', 'convert_release_to_draft'], + enum: [ + 'delete_track', + 'delete_release', + 'retag_release', + 'convert_release_to_draft', + 'draft_retention', + 'draft_squash', + ], }, track_id: { type: String, required: true, validate: validateTrackId }, status: { @@ -22,6 +29,7 @@ const releaseTrackAuditEventSchema = new mongoose.Schema( confirmation: { type: String, required: true }, request: { type: mongoose.Schema.Types.Mixed, default: {} }, result: { type: mongoose.Schema.Types.Mixed, default: null }, + cleanup: { type: mongoose.Schema.Types.Mixed, default: undefined }, error: { name: String, message: String, diff --git a/app/models/release-tracks/release-track-registry-model.js b/app/models/release-tracks/release-track-registry-model.js index 219b95ea..d5936681 100644 --- a/app/models/release-tracks/release-track-registry-model.js +++ b/app/models/release-tracks/release-track-registry-model.js @@ -22,6 +22,19 @@ const snapshotScheduleDefinition = { type: String, validate: validateCron, }, + draft_retention: { + type: new mongoose.Schema( + { + max_drafts: { + type: Number, + default: null, + validate: (value) => value == null || (Number.isSafeInteger(value) && value > 0), + }, + }, + { _id: false }, + ), + default: undefined, + }, dates: { type: [Date], default: undefined }, }; const snapshotScheduleSchema = new mongoose.Schema(snapshotScheduleDefinition, { _id: false }); diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index 09bf7ecf..c75b51b6 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -122,7 +122,7 @@ const componentTrackDefinition = { }, resolution_strategy: { type: String, - enum: ['latest_tagged', 'specific_version', 'specific_snapshot'], + enum: ['latest_tagged', 'latest_preview', 'specific_version', 'specific_snapshot'], required: true, }, priority: { @@ -172,10 +172,24 @@ const componentSnapshotResolutionDefinition = { resolved_snapshot_id: { type: Date, required: true }, resolved_version: { type: String, - required: true, + required: function () { + return !['latest_preview', 'latest_draft'].includes(this.strategy_used); + }, + default: null, validate: validateVersion, }, - strategy_used: { type: String, required: true }, + strategy_used: { + type: String, + // Historical members-only draft resolutions retain their original strategy. + enum: [ + 'latest_tagged', + 'latest_preview', + 'latest_draft', + 'specific_version', + 'specific_snapshot', + ], + required: true, + }, filters_applied: { type: componentTrackFiltersSchema, default: undefined }, total_objects_in_source: { type: Number, required: true }, objects_after_filter: { type: Number, required: true }, @@ -360,12 +374,11 @@ const versionHistoryEntryDefinition = { candidates_count: { type: Number }, quarantine_count: { type: Number }, }, - // Virtual tracks only: immutable component track ID → tagged version. + // Virtual tracks only: immutable component track ID → version (null for drafts). component_versions: { type: Map, of: { type: String, - required: true, validate: validateVersion, }, default: undefined, @@ -428,6 +441,8 @@ const releaseTrackSnapshotDefinition = { publication: { type: frozenPublicationSchema }, bundle_id: { type: String }, bundle_hashes: { type: bundleHashesSchema }, + // Identifies the original release event across retags, but not rollback/release cycles. + release_event_id: { type: String }, creation_actor: { type: new mongoose.Schema( { diff --git a/app/models/release-tracks/virtual-track-schedule-occurrence-model.js b/app/models/release-tracks/virtual-track-schedule-occurrence-model.js index ce1f9f35..657e1b5d 100644 --- a/app/models/release-tracks/virtual-track-schedule-occurrence-model.js +++ b/app/models/release-tracks/virtual-track-schedule-occurrence-model.js @@ -24,9 +24,11 @@ const virtualTrackScheduleOccurrenceSchema = new mongoose.Schema( }, attempt_count: { type: Number, required: true, default: 0 }, claimed_at: { type: Date, default: null }, + claim_token: { type: String, default: null }, claim_expires_at: { type: Date, default: null }, next_retry_at: { type: Date, default: null }, finished_at: { type: Date, default: null }, + // Durable, write-once receipt: retained even when the snapshot is removed. snapshot_modified: { type: Date, default: null }, last_error: { type: mongoose.Schema.Types.Mixed, default: null }, }, diff --git a/app/repository/allowed-values-repository.js b/app/repository/allowed-values-repository.js new file mode 100644 index 00000000..817111db --- /dev/null +++ b/app/repository/allowed-values-repository.js @@ -0,0 +1,127 @@ +'use strict'; + +const mongoose = require('mongoose'); +const AllowedValuesConfiguration = require('../models/allowed-values-configuration-model'); +const { + DatabaseError, + SystemConfigurationNotFound, + DuplicateIdError, + NotFoundError, +} = require('../exceptions'); + +const configurationId = 'allowed-values'; + +class AllowedValuesRepository { + constructor(model) { + this.model = model; + } + + async initialize(values, rules) { + try { + await this.model.updateOne( + { _id: configurationId }, + { $setOnInsert: { values, rules } }, + { upsert: true, runValidators: true }, + ); + } catch (err) { + // The built-in _id index arbitrates concurrent first-time initialization. + if (err.code !== 11000) throw new DatabaseError(err); + } + } + + async migrateRules(seededRules) { + try { + // Derive keys from the current persisted rows inside MongoDB. The predicate + // makes the upgrade idempotent without resurrecting deleted options. + await this.model.updateOne({ _id: configurationId, rules: { $exists: false } }, [ + { + $set: { + rules: { + $setUnion: [ + { $literal: seededRules }, + { + $map: { + input: '$values', + as: 'row', + in: { propertyName: '$$row.propertyName', domainName: '$$row.domainName' }, + }, + }, + ], + }, + }, + }, + ]); + } catch (err) { + throw new DatabaseError(err); + } + } + + async retrieveConfiguration({ optional = false } = {}) { + let document; + try { + document = await this.model.findById(configurationId).lean().exec(); + } catch (err) { + throw new DatabaseError(err); + } + if (!document && !optional) throw new SystemConfigurationNotFound(); + return document; + } + + async saveRule(propertyName, domainName, rows, { create = false } = {}) { + const rule = { propertyName, domainName }; + const values = rows.map((row) => ({ _id: new mongoose.Types.ObjectId(), ...row })); + let document; + try { + // Both membership and replacement are evaluated against the current + // document, preserving different-group saves and arbitrating creation. + document = await this.model + .findOneAndUpdate( + { + _id: configurationId, + rules: create ? { $not: { $elemMatch: rule } } : { $elemMatch: rule }, + }, + [ + { + $set: { + ...(create ? { rules: { $concatArrays: ['$rules', { $literal: [rule] }] } } : {}), + values: { + $concatArrays: [ + { + $filter: { + input: '$values', + as: 'row', + cond: { + $not: [ + { + $and: [ + { $eq: ['$$row.propertyName', { $literal: propertyName }] }, + { $eq: ['$$row.domainName', { $literal: domainName }] }, + ], + }, + ], + }, + }, + }, + { $literal: values }, + ], + }, + }, + }, + ], + { new: true }, + ) + .lean() + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + if (!document) { + if (create) + throw new DuplicateIdError({ details: 'This allowed-values rule already exists.' }); + throw new NotFoundError({ details: 'This allowed-values rule has not been configured.' }); + } + return document; + } +} + +module.exports = new AllowedValuesRepository(AllowedValuesConfiguration); diff --git a/app/repository/release-tracks/release-track-audit-event.repository.js b/app/repository/release-tracks/release-track-audit-event.repository.js index 884e1679..ab57be78 100644 --- a/app/repository/release-tracks/release-track-audit-event.repository.js +++ b/app/repository/release-tracks/release-track-audit-event.repository.js @@ -70,6 +70,46 @@ class ReleaseTrackAuditEventRepository { throw new DatabaseError(repositoryError); } } + async getCleanup(trackId, eventId) { + return ReleaseTrackAuditEvent.findOne({ + track_id: trackId, + event_id: eventId, + action: { $in: ['draft_retention', 'draft_squash'] }, + }) + .lean() + .exec(); + } + + async listCleanup(trackId) { + return ReleaseTrackAuditEvent.find({ + track_id: trackId, + action: { $in: ['draft_retention', 'draft_squash'] }, + status: { $in: ['pending', 'failed'] }, + }) + .sort({ started_at: -1 }) + .limit(25) + .lean() + .exec(); + } + + async saveCleanup(eventId, cleanup, status = 'pending', error = null) { + const event = await ReleaseTrackAuditEvent.findOneAndUpdate( + { event_id: eventId, action: { $in: ['draft_retention', 'draft_squash'] } }, + { + $set: { + cleanup, + status, + error: error + ? { name: error.name || 'Error', message: error.message || String(error) } + : null, + finished_at: status === 'pending' ? null : new Date(), + }, + }, + { new: true, lean: true }, + ).exec(); + if (!event) throw new Error(`Cleanup intent ${eventId} no longer exists`); + return event; + } } module.exports = new ReleaseTrackAuditEventRepository(); diff --git a/app/repository/release-tracks/release-track-dynamic.repository.js b/app/repository/release-tracks/release-track-dynamic.repository.js index 0358ad00..495519cb 100644 --- a/app/repository/release-tracks/release-track-dynamic.repository.js +++ b/app/repository/release-tracks/release-track-dynamic.repository.js @@ -1,6 +1,7 @@ 'use strict'; const modelFactory = require('../../models/release-tracks/model-factory'); +const versionUtils = require('../../lib/release-tracks/version-utils'); const { DatabaseError, DuplicateIdError, @@ -23,10 +24,14 @@ class ReleaseTrackDynamicRepository { return this.modelFactory.getModel(trackId); } - async getLatestSnapshot(trackId) { + async getLatestSnapshot(trackId, projection) { try { const Model = this._getModel(trackId); - return await Model.findOne({ id: trackId }).sort({ modified: -1 }).lean().exec(); + return await Model.findOne({ id: trackId }) + .select(projection) + .sort({ modified: -1 }) + .lean() + .exec(); } catch (err) { throw new DatabaseError(err); } @@ -71,10 +76,10 @@ class ReleaseTrackDynamicRepository { } } - async getSnapshotByModified(trackId, modified) { + async getSnapshotByModified(trackId, modified, projection) { try { const Model = this._getModel(trackId); - return await Model.findOne({ id: trackId, modified }).lean().exec(); + return await Model.findOne({ id: trackId, modified }).select(projection).lean().exec(); } catch (err) { if (err.name === 'CastError') { throw new BadlyFormattedParameterError({ parameterName: 'modified' }); @@ -126,7 +131,7 @@ class ReleaseTrackDynamicRepository { } } - async getReleaseBySourceModified(trackId, sourceModified) { + async getReleaseBySourceModified(trackId, sourceModified, projection) { try { const Model = this._getModel(trackId); return await Model.findOne({ @@ -134,6 +139,7 @@ class ReleaseTrackDynamicRepository { version: { $type: 'string' }, release_source_modified: sourceModified, }) + .select(projection) .lean() .exec(); } catch (err) { @@ -285,7 +291,20 @@ class ReleaseTrackDynamicRepository { query.version = null; } - const totalCount = await Model.countDocuments(query).exec(); + const [summaryCounts] = await Model.aggregate([ + { $match: query }, + { + $group: { + _id: null, + total: { $sum: 1 }, + tagged: { + $sum: { $cond: [{ $eq: [{ $type: '$version' }, 'string'] }, 1, 0] }, + }, + }, + }, + ]).exec(); + const total = summaryCounts?.total || 0; + const tagged = summaryCounts?.tagged || 0; const aggregation = [ { $match: query }, { $sort: { modified: -1 } }, @@ -325,8 +344,9 @@ class ReleaseTrackDynamicRepository { return { data: documents, + counts: { tagged, drafts: total - tagged, total }, pagination: { - total: totalCount, + total, offset: options.offset || 0, limit: options.limit || 0, }, @@ -560,7 +580,7 @@ class ReleaseTrackDynamicRepository { } } - async deleteOlderDrafts(trackId, modified) { + async deleteOlderDrafts(trackId, modified, referencedDrafts = []) { try { const Model = this._getModel(trackId); const retainedDrafts = await Model.distinct('release_source_modified', { @@ -568,6 +588,7 @@ class ReleaseTrackDynamicRepository { version: { $type: 'string' }, release_source_modified: { $type: 'date' }, }).exec(); + retainedDrafts.push(...referencedDrafts); const query = { id: trackId, version: null, @@ -586,6 +607,106 @@ class ReleaseTrackDynamicRepository { } } + async getDraftRetentionBoundary(trackId, count) { + return this._getModel(trackId) + .findOne({ id: trackId, version: null }) + .sort({ modified: -1 }) + .skip(count - 1) + .select('modified') + .lean() + .exec(); + } + + async getHistoricalDraftBatch(trackId, { lower_bound, upper_bound, cursor }, limit = 100) { + const lower = [lower_bound, cursor].filter(Boolean).map((value) => new Date(value)); + const modified = { $lt: new Date(upper_bound) }; + if (lower.length) modified.$gt = new Date(Math.max(...lower.map(Number))); + return this._getModel(trackId) + .find({ id: trackId, version: null, modified }) + .select('id modified content_manifest_id scheduled_materialization') + .sort({ modified: 1 }) + .limit(limit) + .lean() + .exec(); + } + + async deleteHistoricalDraft(trackId, snapshot, bounds, latestModified) { + const modified = { + $eq: new Date(snapshot.modified), + $lt: new Date(bounds.upper_bound), + $ne: new Date(latestModified), + }; + if (bounds.lower_bound) modified.$gt = new Date(bounds.lower_bound); + return this._getModel(trackId) + .findOneAndDelete({ + id: trackId, + _id: snapshot._id, + version: null, + modified, + content_manifest_id: snapshot.content_manifest_id, + }) + .lean() + .exec(); + } + + async getSnapshotCounters(trackId) { + const Model = this._getModel(trackId); + const [counts] = await Model.aggregate([ + { $match: { id: trackId } }, + { + $group: { + _id: null, + snapshot_count: { $sum: 1 }, + tagged_release_count: { + $sum: { $cond: [{ $ne: [{ $ifNull: ['$version', null] }, null] }, 1, 0] }, + }, + latest_snapshot_modified: { $max: '$modified' }, + }, + }, + ]).exec(); + let highest = null; + const tagged = Model.find({ id: trackId, version: { $type: 'string' } }) + .select('version') + .lean() + .cursor({ batchSize: 100 }); + for await (const snapshot of tagged) { + if (!highest || versionUtils.compareVersions(snapshot.version, highest) > 0) { + highest = snapshot.version; + } + } + return { + snapshot_count: counts?.snapshot_count || 0, + tagged_release_count: counts?.tagged_release_count || 0, + latest_snapshot_modified: counts?.latest_snapshot_modified || null, + latest_tagged_version: highest, + }; + } + + async hasSnapshotResolvingComponent(trackId, componentTrackId, modified) { + return Boolean( + await this._getModel(trackId).exists({ + id: trackId, + 'composition_resolution.component_snapshots': { + $elemMatch: { track_id: componentTrackId, resolved_snapshot_id: new Date(modified) }, + }, + }), + ); + } + + async getScheduledSnapshotBatch(trackId, afterModified) { + return this._getModel(trackId) + .find({ + id: trackId, + 'scheduled_materialization.scheduled_for': { $type: 'date' }, + ...(afterModified ? { modified: { $gt: new Date(afterModified) } } : {}), + }) + .select('id modified scheduled_materialization') + .sort({ modified: 1 }) + .limit(100) + .lean() + .exec(); + } + async deleteSnapshot(trackId, modified) { try { const Model = this._getModel(trackId); @@ -618,6 +739,26 @@ class ReleaseTrackDynamicRepository { } } + async findResolvedComponentSnapshotIds(trackId, componentTrackId) { + try { + const Model = this._getModel(trackId); + const snapshots = await Model.aggregate([ + { + $match: { + id: trackId, + 'composition_resolution.component_snapshots.track_id': componentTrackId, + }, + }, + { $unwind: '$composition_resolution.component_snapshots' }, + { $match: { 'composition_resolution.component_snapshots.track_id': componentTrackId } }, + { $group: { _id: '$composition_resolution.component_snapshots.resolved_snapshot_id' } }, + ]).exec(); + return snapshots.map((snapshot) => snapshot._id); + } catch (err) { + throw new DatabaseError(err); + } + } + async deleteAllSnapshots(trackId) { try { const Model = this._getModel(trackId); diff --git a/app/repository/release-tracks/release-track-registry.repository.js b/app/repository/release-tracks/release-track-registry.repository.js index 0d3f4135..6b79b9e3 100644 --- a/app/repository/release-tracks/release-track-registry.repository.js +++ b/app/repository/release-tracks/release-track-registry.repository.js @@ -68,9 +68,9 @@ class ReleaseTrackRegistryRepository { } } - async findByTrackId(trackId) { + async findByTrackId(trackId, projection) { try { - return await this.model.findOne({ track_id: trackId }).lean().exec(); + return await this.model.findOne({ track_id: trackId }).select(projection).lean().exec(); } catch (err) { if (err.name === 'CastError') { throw new BadlyFormattedParameterError({ parameterName: 'trackId' }); @@ -101,7 +101,7 @@ class ReleaseTrackRegistryRepository { }); } - aggregation.push({ $project: { release_lock: 0 } }); + aggregation.push({ $project: { release_lock: 0, draft_retention: 0 } }); // Total count before pagination const totalCountResult = await this.model.aggregate(aggregation).count('totalCount').exec(); @@ -128,6 +128,19 @@ class ReleaseTrackRegistryRepository { } } + async findVirtualTrackBatch(afterTrackId) { + return this.model + .find({ + type: 'virtual', + ...(afterTrackId ? { track_id: { $gt: afterTrackId } } : {}), + }) + .select('track_id') + .sort({ track_id: 1 }) + .limit(100) + .lean() + .exec(); + } + async findWithTaggedReleases(options = {}) { try { const query = { 'tagged_releases.0': { $exists: true } }; @@ -222,6 +235,24 @@ class ReleaseTrackRegistryRepository { } } + async renewReleaseLock(trackId, token, staleBefore) { + try { + return await this.model + .findOneAndUpdate( + { + track_id: trackId, + 'release_lock.token': token, + 'release_lock.acquired_at': { $gte: staleBefore }, + }, + { $set: { 'release_lock.acquired_at': new Date() } }, + { new: true, lean: true, projection: { _id: 1 } }, + ) + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + async releaseReleaseLock(trackId, token) { try { return await this.model diff --git a/app/repository/release-tracks/virtual-track-schedule-occurrence.repository.js b/app/repository/release-tracks/virtual-track-schedule-occurrence.repository.js index b470f00b..4a9996f7 100644 --- a/app/repository/release-tracks/virtual-track-schedule-occurrence.repository.js +++ b/app/repository/release-tracks/virtual-track-schedule-occurrence.repository.js @@ -1,7 +1,8 @@ 'use strict'; +const { randomUUID } = require('node:crypto'); const VirtualTrackScheduleOccurrence = require('../../models/release-tracks/virtual-track-schedule-occurrence-model'); -const { DatabaseError } = require('../../exceptions'); +const { DatabaseError, ReleaseConflictError } = require('../../exceptions'); class VirtualTrackScheduleOccurrenceRepository { async register(trackId, scheduleMode, scheduledFor) { @@ -24,6 +25,49 @@ class VirtualTrackScheduleOccurrenceRepository { } } + async getMaterializationReceipt(trackId, scheduledFor) { + try { + return await VirtualTrackScheduleOccurrence.findOne({ + track_id: trackId, + scheduled_for: scheduledFor, + }) + .lean() + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + + async recordMaterialization(trackId, scheduledMaterialization, snapshotModified) { + const { schedule_mode: scheduleMode, scheduled_for: scheduledFor } = scheduledMaterialization; + await this.register(trackId, scheduleMode, scheduledFor); + let receipt; + try { + receipt = await VirtualTrackScheduleOccurrence.findOneAndUpdate( + { + track_id: trackId, + scheduled_for: scheduledFor, + $or: [{ snapshot_modified: null }, { snapshot_modified: snapshotModified }], + }, + { $set: { snapshot_modified: snapshotModified } }, + { new: true, lean: true }, + ).exec(); + } catch (err) { + throw new DatabaseError(err); + } + if (receipt) return receipt; + + const existing = await this.getMaterializationReceipt(trackId, scheduledFor); + throw new ReleaseConflictError('Scheduled occurrence already has a different materialization', { + details: { + track_id: trackId, + scheduled_for: scheduledFor, + snapshot_modified: existing?.snapshot_modified, + requested_snapshot_modified: snapshotModified, + }, + }); + } + async findDue(now) { try { return await VirtualTrackScheduleOccurrence.find({ @@ -57,6 +101,7 @@ class VirtualTrackScheduleOccurrenceRepository { $set: { status: 'running', claimed_at: now, + claim_token: randomUUID(), claim_expires_at: claimExpiresAt, next_retry_at: null, finished_at: null, @@ -71,42 +116,61 @@ class VirtualTrackScheduleOccurrenceRepository { } } - async complete(trackId, scheduledFor, snapshotModified) { - return this._finish(trackId, scheduledFor, { - status: 'completed', - snapshot_modified: snapshotModified, - finished_at: new Date(), - claim_expires_at: null, - next_retry_at: null, - last_error: null, - }); + async complete(claimed) { + return this._finish( + claimed, + { snapshot_modified: { $ne: null } }, + { + status: 'completed', + finished_at: new Date(), + claim_expires_at: null, + next_retry_at: null, + last_error: null, + }, + ); } - async fail(trackId, scheduledFor, error, nextRetryAt) { - return this._finish(trackId, scheduledFor, { - status: 'failed', - finished_at: new Date(), - claim_expires_at: null, - next_retry_at: nextRetryAt, - last_error: error, - }); + async fail(claimed, error, nextRetryAt) { + // A saved result remains a receipt even when its audit needs retrying. + return this._finish( + claimed, + {}, + { + status: 'failed', + finished_at: new Date(), + claim_expires_at: null, + next_retry_at: nextRetryAt, + last_error: error, + }, + ); } - async skip(trackId, scheduledFor, reason) { - return this._finish(trackId, scheduledFor, { - status: 'skipped', - finished_at: new Date(), - claim_expires_at: null, - next_retry_at: null, - last_error: { message: reason }, - }); + async skip(claimed, reason) { + return this._finish( + claimed, + { snapshot_modified: null }, + { + status: 'skipped', + finished_at: new Date(), + claim_expires_at: null, + next_retry_at: null, + last_error: { message: reason }, + }, + ); } - async _finish(trackId, scheduledFor, updates) { + async _finish(claimed, conditions, updates) { + if (!claimed?.claim_token) return null; try { return await VirtualTrackScheduleOccurrence.findOneAndUpdate( - { track_id: trackId, scheduled_for: scheduledFor }, - { $set: updates }, + { + track_id: claimed.track_id, + scheduled_for: claimed.scheduled_for, + status: 'running', + claim_token: claimed.claim_token, + ...conditions, + }, + { $set: { ...updates, claim_token: null } }, { new: true, lean: true }, ).exec(); } catch (err) { diff --git a/app/routes/allowed-values-routes.js b/app/routes/allowed-values-routes.js new file mode 100644 index 00000000..8aeddfc4 --- /dev/null +++ b/app/routes/allowed-values-routes.js @@ -0,0 +1,27 @@ +'use strict'; + +const express = require('express'); +const controller = require('../controllers/allowed-values-controller'); +const authn = require('../lib/authn-middleware'); +const authz = require('../lib/authz-middleware'); + +const router = express.Router(); + +router + .route('/config/allowed-values/rules') + .get(authn.authenticate, authz.requireRole(authz.admin), controller.retrieveRules) + .post(authn.authenticate, authz.requireRole(authz.admin), controller.createRule); + +router + .route('/config/allowed-values/rules/:propertyName/:domainName') + .put(authn.authenticate, authz.requireRole(authz.admin), controller.replaceRule); + +router + .route('/config/allowed-values/catalog') + .get(authn.authenticate, authz.requireRole(authz.admin), controller.retrieveCatalog); + +router + .route('/config/allowed-values/validate') + .post(authn.authenticate, authz.requireRole(authz.admin), controller.validateValue); + +module.exports = router; diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index 2872c192..b3e6177e 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -330,6 +330,22 @@ router releaseTracksController.updateSchedule, ); +router + .route('/release-tracks/:id/virtual/draft-cleanup') + .get( + authn.authenticate, + authz.requireRole(authz.visitorOrHigher, authz.readOnlyService), + releaseTracksController.listDraftCleanup, + ); + +router + .route('/release-tracks/:id/virtual/draft-cleanup/:operationId/retry') + .post( + authn.authenticate, + authz.requireRole(authz.admin), + releaseTracksController.retryDraftCleanup, + ); + // ============================================================================= // Delete release track (must be last -- :id is a catch-all param) // ============================================================================= diff --git a/app/scheduler/virtual-track-snapshots-task.js b/app/scheduler/virtual-track-snapshots-task.js index 0d6c930e..705d825c 100644 --- a/app/scheduler/virtual-track-snapshots-task.js +++ b/app/scheduler/virtual-track-snapshots-task.js @@ -49,7 +49,8 @@ async function auditAttempt(occurrence, execute) { }); try { - const { snapshot, recovered } = await execute(); + const result = await execute(); + const { snapshot, receipt, recovered } = result; await recorder.recordItem({ status: recovered ? 'unchanged' : 'changed', action: recovered ? 'recover_scheduled_virtual_snapshot' : 'materialize_virtual_snapshot', @@ -59,9 +60,13 @@ async function auditAttempt(occurrence, execute) { }, details: { scheduled_for: scheduledFor, - snapshot_modified: snapshot.modified, - members_count: snapshot.members?.length || 0, - quarantine_count: snapshot.quarantine?.length || 0, + snapshot_modified: receipt.snapshot_modified, + ...(snapshot + ? { + members_count: snapshot.members?.length || 0, + quarantine_count: snapshot.quarantine?.length || 0, + } + : { materialized_and_removed: true }), recovered, }, }); @@ -76,7 +81,7 @@ async function auditAttempt(occurrence, execute) { : `Materialized scheduled virtual snapshot for ${occurrence.track_id}`, }, }); - return snapshot; + return result; } catch (err) { const serialized = serializeError(err); await recorder.recordItem({ @@ -101,6 +106,27 @@ async function auditAttempt(occurrence, execute) { } } +async function recoverMaterialization(claimed) { + const receipt = await occurrenceRepo.getMaterializationReceipt( + claimed.track_id, + claimed.scheduled_for, + ); + const snapshot = await dynamicRepo.getSnapshotByScheduledMaterialization( + claimed.track_id, + claimed.scheduled_for, + ); + if (receipt?.snapshot_modified) return { snapshot, receipt, recovered: true }; + if (!snapshot) return null; + + // Backfill legacy saved results before audit completion or any later pruning. + const recorded = await occurrenceRepo.recordMaterialization( + claimed.track_id, + snapshot.scheduled_materialization, + snapshot.modified, + ); + return { snapshot, receipt: recorded, recovered: true }; +} + async function executeOccurrence(occurrence, now = new Date()) { const scheduledFor = new Date(occurrence.scheduled_for); const claimed = await occurrenceRepo.claim( @@ -111,43 +137,56 @@ async function executeOccurrence(occurrence, now = new Date()) { ); if (!claimed) return null; - const track = await registryRepo.findByTrackId(claimed.track_id); - if (!isConfiguredOccurrence(track, claimed)) { - await occurrenceRepo.skip( - claimed.track_id, - scheduledFor, - 'Track was deleted or no longer has the schedule that produced this occurrence', - ); - return null; - } - try { - const snapshot = await auditAttempt(claimed, async () => { - // A worker may have persisted the snapshot and exited before completing - // the occurrence ledger. Recover that durable result without recomputing - // composition, which may no longer be resolvable after the crash. - const existing = await dynamicRepo.getSnapshotByScheduledMaterialization( - claimed.track_id, - scheduledFor, - ); - if (existing) { - return { snapshot: existing, recovered: true }; + const recovered = await recoverMaterialization(claimed); + if (!recovered) { + const track = await registryRepo.findByTrackId(claimed.track_id); + if (!isConfiguredOccurrence(track, claimed)) { + await occurrenceRepo.skip( + claimed, + 'Track was deleted or no longer has the schedule that produced this occurrence', + ); + return null; } + } + + const result = await auditAttempt(claimed, async () => { + if (recovered) return recovered; - const materialized = await virtualTrackService.createVirtualSnapshot(claimed.track_id, { - scheduledMaterialization: { - schedule_mode: claimed.schedule_mode, - scheduled_for: scheduledFor, - }, - }); - return { snapshot: materialized, recovered: false }; + const scheduledMaterialization = { + schedule_mode: claimed.schedule_mode, + scheduled_for: scheduledFor, + }; + let materialized; + try { + materialized = await virtualTrackService.createVirtualSnapshot(claimed.track_id, { + scheduledMaterialization, + useRecurringRetention: claimed.schedule_mode === 'cron', + }); + } catch (err) { + // Another worker may have committed and pruned the result after our + // initial read. The locked service rejects replay; recover the receipt. + const committed = await recoverMaterialization(claimed); + if (committed) return committed; + throw err; + } + const receipt = await occurrenceRepo.recordMaterialization( + claimed.track_id, + scheduledMaterialization, + materialized.modified, + ); + return { snapshot: materialized, receipt, recovered: false }; }); - await occurrenceRepo.complete(claimed.track_id, scheduledFor, snapshot.modified); - return snapshot; + await occurrenceRepo.complete(claimed); + return ( + result.snapshot || { + snapshot_modified: result.receipt.snapshot_modified, + materialized_and_removed: true, + } + ); } catch (err) { await occurrenceRepo.fail( - claimed.track_id, - scheduledFor, + claimed, serializeError(err), new Date(now.getTime() + RETRY_DELAY_MS), ); diff --git a/app/services/release-tracks/content-manifest-service.js b/app/services/release-tracks/content-manifest-service.js index 7deae832..3bcf6c9e 100644 --- a/app/services/release-tracks/content-manifest-service.js +++ b/app/services/release-tracks/content-manifest-service.js @@ -347,10 +347,10 @@ async function activate(manifestId) { async function discard(manifestId) { if (!manifestId) return; - await Promise.all([ - ReleaseTrackContentManifestEntry.deleteMany({ manifest_id: manifestId }).exec(), - ReleaseTrackContentManifest.deleteOne({ manifest_id: manifestId }).exec(), - ]); + // Keep the discoverable header until every entry is gone. A failed entry + // deletion can then be resumed by orphan repair instead of stranding data. + await ReleaseTrackContentManifestEntry.deleteMany({ manifest_id: manifestId }).exec(); + await ReleaseTrackContentManifest.deleteOne({ manifest_id: manifestId }).exec(); } /** @@ -367,34 +367,47 @@ async function discardUnreferenced(trackId, manifestIds) { return unreferenced; } -async function discardTrack(trackId) { - const manifests = await ReleaseTrackContentManifest.find({ track_id: trackId }) - .select({ manifest_id: 1, _id: 0 }) - .lean() - .exec(); - const manifestIds = manifests.map((manifest) => manifest.manifest_id); - - await Promise.all([ - manifestIds.length > 0 - ? ReleaseTrackContentManifestEntry.deleteMany({ manifest_id: { $in: manifestIds } }).exec() - : Promise.resolve(), - ReleaseTrackContentManifest.deleteMany({ track_id: trackId }).exec(), - ]); +async function discardTrack(trackId, assertOwned = async () => {}) { + for (;;) { + await assertOwned(); + const manifests = await ReleaseTrackContentManifest.find({ track_id: trackId }) + .select({ manifest_id: 1, _id: 0 }) + .limit(100) + .lean() + .exec(); + if (!manifests.length) return; + for (const manifest of manifests) { + await assertOwned(); + await discard(manifest.manifest_id); + } + } } /** * Remove manifests owned by a track that no surviving snapshot references. * Used by deletion recovery paths. */ -async function discardOrphans(trackId) { - const manifests = await ReleaseTrackContentManifest.find({ track_id: trackId }) - .select({ manifest_id: 1, _id: 0 }) - .lean() - .exec(); - return discardUnreferenced( - trackId, - manifests.map((manifest) => manifest.manifest_id), - ); +async function discardOrphans(trackId, assertOwned = async () => {}) { + let cursor; + for (;;) { + await assertOwned(); + const manifests = await ReleaseTrackContentManifest.find({ + track_id: trackId, + ...(cursor ? { _id: { $gt: cursor } } : {}), + }) + .select({ manifest_id: 1 }) + .sort({ _id: 1 }) + .limit(100) + .lean() + .exec(); + if (!manifests.length) return; + await assertOwned(); + await discardUnreferenced( + trackId, + manifests.map((manifest) => manifest.manifest_id), + ); + cursor = manifests.at(-1)._id; + } } // ============================================================================= diff --git a/app/services/release-tracks/draft-cleanup-service.js b/app/services/release-tracks/draft-cleanup-service.js new file mode 100644 index 00000000..a5d6a0f1 --- /dev/null +++ b/app/services/release-tracks/draft-cleanup-service.js @@ -0,0 +1,518 @@ +'use strict'; + +const { createHash } = require('crypto'); +const authz = require('../../lib/authz-middleware'); +const { draftRetentionSchema } = require('../../lib/release-tracks/release-track-schemas'); +const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); +const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); +const auditRepo = require('../../repository/release-tracks/release-track-audit-event.repository'); +const occurrenceRepo = require('../../repository/release-tracks/virtual-track-schedule-occurrence.repository'); +const manifests = require('./content-manifest-service'); +const { + BadRequestError, + InsufficientRoleError, + NotFoundError, + ReleaseConflictError, + ReleasePublicationError, + TrackNotFoundError, +} = require('../../exceptions'); + +const BATCH_SIZE = 100; +const MAX_BATCHES = 10; +const iso = (value) => new Date(value).toISOString(); +const IDENTITY_FIELDS = + 'id modified version content_manifest_id scheduled_materialization release_event_id'; + +function assertAdmin(actor) { + if (actor?.role !== authz.userRoles.admin) { + throw new InsufficientRoleError('administrator', { + details: + 'Configuring draft retention, squashing drafts, and retrying cleanup require an administrator.', + }); + } +} +exports.assertAdmin = assertAdmin; + +function validatePolicy(policy, actor, type) { + assertAdmin(actor); + if (type !== 'virtual') + throw new BadRequestError({ message: 'Draft retention is only available for virtual tracks' }); + const parsed = draftRetentionSchema.nullable().safeParse(policy); + if (!parsed.success) + throw new BadRequestError({ + message: 'Invalid draft retention policy', + details: parsed.error.errors, + }); + return parsed.data; +} +exports.validatePolicy = validatePolicy; + +async function virtualRegistry(trackId) { + const registry = await registryRepo.findByTrackId(trackId, 'track_id type snapshot_schedule'); + if (!registry) throw new TrackNotFoundError(trackId); + if (registry.type !== 'virtual') + throw new BadRequestError({ message: 'Draft cleanup is only available for virtual tracks' }); + return registry; +} + +// Called under the target lock before any write carrying scheduled metadata. +// A durable receipt outlives its snapshot; its absence is never permission to replay. +exports.findScheduledResult = async function findScheduledResult(trackId, scheduled) { + if (!scheduled) return null; + const receipt = await occurrenceRepo.getMaterializationReceipt(trackId, scheduled.scheduled_for); + const existing = receipt?.snapshot_modified + ? await dynamicRepo.getSnapshotByModified(trackId, receipt.snapshot_modified) + : await dynamicRepo.getSnapshotByScheduledMaterialization(trackId, scheduled.scheduled_for); + if (receipt?.snapshot_modified && !existing) { + throw new ReleaseConflictError( + 'This scheduled occurrence was already materialized and its snapshot was removed', + { + track_id: trackId, + scheduled_for: iso(scheduled.scheduled_for), + snapshot_modified: iso(receipt.snapshot_modified), + already_materialized: true, + snapshot_removed: true, + }, + ); + } + if (existing) await occurrenceRepo.recordMaterialization(trackId, scheduled, existing.modified); + return existing; +}; + +exports.recordScheduledResult = async function recordScheduledResult(snapshot) { + if (snapshot.scheduled_materialization) { + await occurrenceRepo.recordMaterialization( + snapshot.id, + snapshot.scheduled_materialization, + snapshot.modified, + ); + } +}; + +async function protection(trackId, snapshot, latest) { + if (iso(snapshot.modified) === iso(latest.modified)) return 'latest'; + if (await dynamicRepo.getReleaseBySourceModified(trackId, snapshot.modified, '_id')) + return 'release_source'; + let afterTrackId; + for (;;) { + const tracks = await registryRepo.findVirtualTrackBatch(afterTrackId); + if (!tracks.length) break; + for (const track of tracks) { + if ( + await dynamicRepo.hasSnapshotResolvingComponent(track.track_id, trackId, snapshot.modified) + ) + return 'dependency'; + } + afterTrackId = tracks.at(-1).track_id; + } + const scheduled = snapshot.scheduled_materialization; + if (scheduled) { + const receipt = await occurrenceRepo.getMaterializationReceipt( + trackId, + scheduled.scheduled_for, + ); + if (receipt?.snapshot_modified && iso(receipt.snapshot_modified) !== iso(snapshot.modified)) + return 'scheduled_receipt_conflict'; + } + return null; +} + +exports.previewSquash = async function previewSquash(trackId, target) { + const previous = await dynamicRepo.getLatestTaggedSnapshotBefore(trackId, target.modified); + const bounds = { + lower_bound: previous ? iso(previous.modified) : null, + upper_bound: iso(target.modified), + }; + const latest = await dynamicRepo.getLatestSnapshot(trackId, 'modified'); + const hash = createHash('sha256'); + hash.update( + JSON.stringify([ + trackId, + bounds, + target.content_manifest_id, + previous?.release_event_id || null, + ]), + ); + let eligible = 0; + let protectedCount = 0; + let cursor; + for (;;) { + const batch = await dynamicRepo.getHistoricalDraftBatch( + trackId, + { ...bounds, cursor }, + BATCH_SIZE, + ); + if (!batch.length) break; + for (const snapshot of batch) { + const reason = await protection(trackId, snapshot, latest); + if (reason) protectedCount += 1; + else eligible += 1; + hash.update( + JSON.stringify([ + iso(snapshot.modified), + snapshot.content_manifest_id, + reason, + snapshot.scheduled_materialization || null, + ]), + ); + } + cursor = iso(batch.at(-1).modified); + } + return { + ...bounds, + eligible_count: eligible, + protected_count: protectedCount, + fingerprint: hash.digest('hex'), + }; +}; + +function stateFor(event) { + return ( + event.cleanup || { + kind: event.request.kind, + eligible_count: event.request.eligible_count || 0, + deleted_count: 0, + protected_count: 0, + cursor: null, + pending_batch: [], + release_committed: false, + publication_complete: false, + } + ); +} + +function resultFor( + event, + state = stateFor(event), + status = event.status, + error = event.error?.message, +) { + return { + operation_id: event.event_id, + status, + kind: state.kind, + eligible_count: state.eligible_count, + deleted_count: state.deleted_count, + protected_count: state.protected_count, + ...(event.request.target_modified ? { target_modified: event.request.target_modified } : {}), + ...(state.kind === 'squash' ? { release_committed: state.release_committed } : {}), + ...(error ? { error } : {}), + }; +} +exports.resultFor = resultFor; + +async function createIntent(trackId, actor, request) { + return auditRepo.create({ + action: request.kind === 'squash' ? 'draft_squash' : 'draft_retention', + trackId, + actor, + confirmation: request.fingerprint || trackId, + request, + }); +} + +exports.beginSquash = async function beginSquash(plan, options, lease) { + assertAdmin(options.actor); + if (plan.sourceSnapshot.type !== 'virtual') + throw new BadRequestError({ message: 'Draft squash is only available for virtual tracks' }); + const preview = await exports.previewSquash(plan.trackId, plan.sourceSnapshot); + if (!options.squash_fingerprint || options.squash_fingerprint !== preview.fingerprint) { + throw new ReleaseConflictError( + 'The draft squash preview changed; review a fresh preview before releasing', + { draft_squash: preview }, + ); + } + await lease.assertOwned(); + return createIntent(plan.trackId, options.actor, { + ...preview, + kind: 'squash', + target_modified: iso(plan.sourceSnapshot.modified), + version: plan.version, + }); +}; + +exports.failDraft = async function failDraft(snapshot, event, error) { + const state = stateFor(event); + try { + await auditRepo.saveCleanup(event.event_id, state, 'failed', error); + } catch { + /* The pending intent remains discoverable. */ + } + return { ...snapshot, draft_cleanup: resultFor(event, state, 'failed', error.message) }; +}; + +exports.failRelease = async function failRelease(event, error) { + const state = stateFor(event); + try { + const snapshot = await dynamicRepo.getSnapshotByModified( + event.track_id, + event.request.target_modified, + IDENTITY_FIELDS, + ); + state.release_committed = + snapshot?.version != null && snapshot.release_event_id === event.event_id; + } catch { + // An unavailable store cannot establish whether an unacknowledged tag + // persisted. Omit the outcome rather than reporting an uncommitted release. + state.release_committed = undefined; + } + try { + await auditRepo.saveCleanup(event.event_id, state, 'failed', error); + } catch { + /* The pending intent remains discoverable. */ + } + return new ReleasePublicationError({ + cause: error, + details: error.message, + operation_id: event.event_id, + release_committed: state.release_committed, + draft_cleanup: resultFor(event, state, 'failed', error.message), + }); +}; + +async function boundsFor(event, registry) { + const original = event.request; + if (original.kind === 'retention') { + // Legacy intents did not identify an approved trigger/policy. They may + // repair already-deleted storage, but must never select more history. + if ( + !['manual', 'recurring'].includes(original.source) || + !Number.isSafeInteger(original.max_drafts) || + original.max_drafts < 1 || + !original.upper_bound + ) + return null; + const maximum = + original.source === 'manual' + ? original.max_drafts + : registry.snapshot_schedule?.mode === 'cron' + ? registry.snapshot_schedule.draft_retention?.max_drafts + : null; + if (!Number.isSafeInteger(maximum) || maximum < 1) return null; + const boundary = await dynamicRepo.getDraftRetentionBoundary(event.track_id, maximum); + if (!boundary) return null; + return { + lower_bound: original.lower_bound, + upper_bound: iso( + Math.min(new Date(original.upper_bound).getTime(), new Date(boundary.modified).getTime()), + ), + }; + } + // A newly inserted tag narrows the original interval. Rollback never widens it. + const previous = await dynamicRepo.getLatestTaggedSnapshotBefore( + event.track_id, + original.upper_bound, + ); + const lower = [original.lower_bound, previous?.modified] + .filter(Boolean) + .map((value) => new Date(value).getTime()); + return { + lower_bound: lower.length ? iso(Math.max(...lower)) : null, + upper_bound: original.upper_bound, + }; +} + +function inBounds(snapshot, bounds) { + return ( + bounds && + new Date(snapshot.modified) < new Date(bounds.upper_bound) && + (!bounds.lower_bound || new Date(snapshot.modified) > new Date(bounds.lower_bound)) + ); +} + +// One bounded, durable page is the write-ahead log. A crash after a snapshot +// delete but before manifest/count updates replays that page, not a fresh selector. +async function runCleanup(event, lease) { + const state = stateFor(event); + try { + await lease.assertOwned(); + let registry = await virtualRegistry(event.track_id); + // Repair already removed storage even after rollback or policy disabling. + await manifests.discardOrphans(event.track_id, () => lease.assertOwned()); + await require('./snapshot-service').syncRegistryCounters(event.track_id); + if (state.kind === 'retention') { + const created = await dynamicRepo.getSnapshotByModified( + event.track_id, + event.request.target_modified, + IDENTITY_FIELDS, + ); + if (!created && !state.creation_complete) + throw new ReleaseConflictError( + 'The draft creation associated with this cleanup did not persist or no longer exists', + ); + if (created) await exports.recordScheduledResult(created); + if (created) await manifests.activate(created.content_manifest_id); + if (!state.creation_complete) { + const latest = await dynamicRepo.getLatestSnapshot(event.track_id); + await require('./snapshot-service').emitContentsChanged(event.track_id, latest); + state.creation_complete = true; + } + } + if (state.kind === 'squash') { + const release = await dynamicRepo.getSnapshotByModified( + event.track_id, + event.request.target_modified, + ); + if (!release?.version || release.release_event_id !== event.event_id) { + state.release_committed = false; + throw new ReleaseConflictError( + 'The original release event is no longer present; cleanup cannot resume', + { + operation_id: event.event_id, + release_committed: false, + }, + ); + } + state.release_committed = true; + if (!state.publication_complete) { + const versioning = require('./versioning-service'); + await lease.assertOwned(); + await versioning.refreshReleaseArtifacts(release); + await require('./release-history-service').reconcileTaggedReleases(event.track_id); + const latest = await dynamicRepo.getLatestSnapshot(event.track_id); + await require('./snapshot-service').emitContentsChanged(event.track_id, latest); + state.publication_complete = true; + await auditRepo.saveCleanup(event.event_id, state); + } + } + + for (let page = 0; page < MAX_BATCHES; page += 1) { + await lease.assertOwned(); + registry = await virtualRegistry(event.track_id); + const bounds = await boundsFor(event, registry); + if (!state.pending_batch.length && bounds) { + state.pending_batch = await dynamicRepo.getHistoricalDraftBatch( + event.track_id, + { ...bounds, cursor: state.cursor }, + BATCH_SIZE, + ); + if (state.kind === 'retention') state.eligible_count += state.pending_batch.length; + await auditRepo.saveCleanup(event.event_id, state); + } + if (!state.pending_batch.length) { + // Also repair manifests from an interrupted older cleanup. No additional + // drafts are chosen if a retention policy has since been disabled. + await manifests.discardOrphans(event.track_id, () => lease.assertOwned()); + await require('./snapshot-service').syncRegistryCounters(event.track_id); + const completed = await auditRepo.saveCleanup(event.event_id, state, 'completed'); + return resultFor(completed); + } + + while (state.pending_batch.length) { + const candidate = state.pending_batch[0]; + await lease.assertOwned(); + const snapshot = await dynamicRepo.getSnapshotByModified( + event.track_id, + candidate.modified, + IDENTITY_FIELDS, + ); + if (!snapshot) { + await manifests.discardUnreferenced(event.track_id, [candidate.content_manifest_id]); + state.deleted_count += 1; + } else { + const latest = await dynamicRepo.getLatestSnapshot(event.track_id, 'modified'); + let reason = + snapshot.version != null || !inBounds(snapshot, bounds) + ? 'outside_current_policy' + : await protection(event.track_id, snapshot, latest); + if (!reason && snapshot.scheduled_materialization) { + try { + await exports.recordScheduledResult(snapshot); + } catch { + reason = 'scheduled_receipt_unavailable'; + } + } + if (reason) state.protected_count += 1; + else { + await lease.assertOwned(); + const deleted = await dynamicRepo.deleteHistoricalDraft( + event.track_id, + candidate, + bounds, + latest.modified, + ); + if (deleted) { + await manifests.discardUnreferenced(event.track_id, [candidate.content_manifest_id]); + state.deleted_count += 1; + } else state.protected_count += 1; + } + } + state.cursor = iso(candidate.modified); + state.pending_batch.shift(); + await auditRepo.saveCleanup(event.event_id, state); + } + await require('./snapshot-service').syncRegistryCounters(event.track_id); + } + return resultFor(await auditRepo.saveCleanup(event.event_id, state)); + } catch (error) { + // Failure to record completion must still be visible, while the persisted + // intent/page makes repair possible even if this response is lost. + try { + await auditRepo.saveCleanup(event.event_id, state, 'failed', error); + } catch { + /* Preserve the last durable page. */ + } + return resultFor(event, state, 'failed', error.message); + } +} +exports.runCleanup = runCleanup; + +exports.prepareRetention = async function prepareRetention(snapshot, retention, lease) { + const maximum = retention.policy.max_drafts; + // Account for the new draft before it is saved. Preserve this original cutoff + // durably so retries never expand into drafts retained by the approved run. + const boundary = + maximum === 1 + ? snapshot + : await dynamicRepo.getDraftRetentionBoundary(snapshot.id, maximum - 1); + await lease.assertOwned(); + // Insert before saving the new draft: an audit outage cannot produce a + // successful creation with an undiscoverable/fabricated cleanup operation. + return createIntent(snapshot.id, retention.actor, { + kind: 'retention', + source: retention.source, + max_drafts: maximum, + lower_bound: null, + upper_bound: boundary ? iso(boundary.modified) : null, + target_modified: iso(snapshot.modified), + }); +}; + +exports.afterDraft = async function afterDraft(snapshot, lease, event) { + if (!event) return snapshot; + event.cleanup = { ...stateFor(event), creation_complete: true }; + return { ...snapshot, draft_cleanup: await runCleanup(event, lease) }; +}; + +exports.list = async function list(trackId) { + await virtualRegistry(trackId); + const events = await auditRepo.listCleanup(trackId); + const data = await Promise.all( + events.map(async (event) => { + const state = { ...stateFor(event) }; + if (state.kind === 'squash') { + const release = await dynamicRepo.getSnapshotByModified( + trackId, + event.request.target_modified, + IDENTITY_FIELDS, + ); + state.release_committed = + release?.version != null && release.release_event_id === event.event_id; + } + return resultFor(event, state); + }), + ); + return { data }; +}; + +exports.retry = async function retry(trackId, operationId, actor) { + assertAdmin(actor); + return require('./versioning-service').withReleaseLock(trackId, async (lease) => { + await virtualRegistry(trackId); + const event = await auditRepo.getCleanup(trackId, operationId); + if (!event) + throw new NotFoundError({ details: 'No cleanup intent exists for this track and operation' }); + if (event.status === 'completed') return resultFor(event); + return runCleanup(event, lease); + }); +}; diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index 1e935e0c..93b6d5ab 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -37,6 +37,7 @@ const bundleImportService = require('./bundle-import-service'); const memberSyncService = require('./member-sync-service'); const releaseHistoryService = require('./release-history-service'); const destructiveAuditService = require('./destructive-audit-service'); +const draftCleanupService = require('./draft-cleanup-service'); const attackObjectsService = require('../stix/attack-objects-service'); const userAccountsService = require('../system/user-accounts-service'); const revisionReference = require('../../lib/release-tracks/revision-reference'); @@ -246,10 +247,12 @@ async function formatWorkbenchSnapshot(snapshot, options) { const [attributed] = await addCreationActors([enriched]); // Registry-derived, read-only metadata used alongside snapshot content. const metadata = await snapshotService.getTrackMetadata(snapshot.id); - enriched.alias = metadata.alias; - enriched.creation_cause = snapshot.creation_cause || 'unknown'; + attributed.alias = metadata.alias; + attributed.creation_cause = snapshot.creation_cause || 'unknown'; if (snapshot.type === 'virtual') { - enriched.snapshot_schedule = metadata.snapshot_schedule || { mode: 'manual' }; + attributed.snapshot_schedule = metadata.snapshot_schedule || { mode: 'manual' }; + attributed.snapshot_count = metadata.snapshot_count; + attributed.tagged_release_count = metadata.tagged_release_count; } return filterSnapshotTiers(attributed, options?.include); } @@ -272,7 +275,6 @@ exports.getReleasesByObject = function getReleasesByObject(objectRef, options) { exports.createTrack = async function createTrack(data) { let validatedData = data; - if (data.scheduled_materialization !== undefined) { if (data.type !== 'virtual') { throw new BadRequestError({ @@ -309,7 +311,7 @@ exports.createTrack = async function createTrack(data) { details: scheduleResult.error.errors, }); } - validatedData = { ...data, snapshot_schedule: scheduleResult.data }; + validatedData = { ...validatedData, snapshot_schedule: scheduleResult.data }; } if (validatedData.composition !== undefined) { @@ -392,15 +394,17 @@ exports.cloneFromSnapshot = function cloneFromSnapshot(trackId, modified, option }; exports.deleteTrack = function deleteTrack(trackId, actor, confirmation) { - return destructiveAuditService.execute( - { - action: 'delete_track', - trackId, - ...destructiveIdentity(trackId, actor, confirmation), - request: {}, - result: () => ({ deleted: true }), - }, - () => snapshotService.deleteTrack(trackId), + return versioningService.withReleaseLock(trackId, (lease) => + destructiveAuditService.execute( + { + action: 'delete_track', + trackId, + ...destructiveIdentity(trackId, actor, confirmation), + request: {}, + result: () => ({ deleted: true }), + }, + () => snapshotService.deleteTrack(trackId, lease), + ), ); }; @@ -617,7 +621,7 @@ exports.updateComposition = function updateComposition(trackId, composition, use }); }; -exports.updateSchedule = function updateSchedule(trackId, schedule) { +exports.updateSchedule = function updateSchedule(trackId, schedule, actor) { const scheduleResult = snapshotScheduleSchema.safeParse(schedule); if (!scheduleResult.success) { throw new BadRequestError({ @@ -625,9 +629,12 @@ exports.updateSchedule = function updateSchedule(trackId, schedule) { details: scheduleResult.error.errors, }); } - return virtualTrackService.updateSchedule(trackId, scheduleResult.data); + return virtualTrackService.updateSchedule(trackId, scheduleResult.data, actor); }; +exports.listDraftCleanup = draftCleanupService.list; +exports.retryDraftCleanup = draftCleanupService.retry; + exports.createVirtualSnapshot = function createVirtualSnapshot(trackId, options) { let validatedOptions = options; if (options?.scheduledMaterialization !== undefined) { diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index ca66f216..d3ab9a47 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -25,7 +25,6 @@ const registryRepo = require('../../repository/release-tracks/release-track-regi const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); const modelFactory = require('../../models/release-tracks/model-factory'); const logger = require('../../lib/logger'); -const versionUtils = require('../../lib/release-tracks/version-utils'); const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); const primaryRevisionService = require('./primary-revision-service'); const reconciliationService = require('./reconciliation-service'); @@ -86,32 +85,8 @@ async function mapWithConcurrency(items, concurrency, mapper) { * @param {string} trackId */ async function syncRegistryCounters(trackId) { - const { data: snapshots } = await dynamicRepo.getAllSnapshots(trackId, { - projection: 'modified version', - }); - - const snapshotCount = snapshots.length; - const tagged = snapshots.filter((s) => s.version != null); - const taggedReleaseCount = tagged.length; - - // Latest snapshot is first (sorted desc by modified) - const latestSnapshotModified = snapshots.length > 0 ? snapshots[0].modified : null; - - const latestTaggedVersion = tagged.reduce( - (highest, snapshot) => - !highest || versionUtils.compareVersions(snapshot.version, highest) > 0 - ? snapshot.version - : highest, - null, - ); - - await registryRepo.updateByTrackId(trackId, { - snapshot_count: snapshotCount, - tagged_release_count: taggedReleaseCount, - latest_snapshot_modified: latestSnapshotModified, - latest_tagged_version: latestTaggedVersion, - updated_at: new Date(), - }); + const counters = await dynamicRepo.getSnapshotCounters(trackId); + await registryRepo.updateByTrackId(trackId, { ...counters, updated_at: new Date() }); } exports.syncRegistryCounters = syncRegistryCounters; @@ -159,10 +134,11 @@ exports.findVirtualSnapshotDependents = findVirtualSnapshotDependents; * @param {string} reason - seal_reason enum value * @returns {Promise} The saved snapshot */ -async function saveSealedSnapshot(trackId, snapshotData, reason) { +async function saveSealedSnapshot(trackId, snapshotData, reason, lease) { const manifestId = await contentManifestService.seal(snapshotData, { reason }); let saved; try { + if (lease) await lease.assertOwned(); saved = await dynamicRepo.saveSnapshot(trackId, { ...snapshotData, content_manifest_id: manifestId, @@ -214,6 +190,13 @@ exports.createTrack = async function createTrack(data, options = {}) { const trackId = `release-track--${uuidv4()}`; const now = new Date(); const trackType = data.type || 'standard'; + if (data.snapshot_schedule?.draft_retention !== undefined) { + require('./draft-cleanup-service').validatePolicy( + data.snapshot_schedule.draft_retention, + data.actor, + trackType, + ); + } if (data.alias) await assertAliasAvailable(data.alias); const initialSnapshot = { @@ -238,11 +221,12 @@ exports.createTrack = async function createTrack(data, options = {}) { version_history: [], }; - // Create collection + indexes, then persist the initial sealed snapshot await modelFactory.ensureIndexes(trackId); - const snapshot = await saveSealedSnapshot(trackId, initialSnapshot, 'track_creation'); + let snapshot; + if (trackType !== 'virtual') { + snapshot = await saveSealedSnapshot(trackId, initialSnapshot, 'track_creation'); + } - // Register in the central registry await registryRepo.create({ track_id: trackId, type: trackType, @@ -250,12 +234,26 @@ exports.createTrack = async function createTrack(data, options = {}) { alias: data.alias || undefined, description: data.description, latest_snapshot_modified: now, - snapshot_count: 1, + snapshot_count: snapshot ? 1 : 0, tagged_release_count: 0, created_at: now, updated_at: now, snapshot_schedule: trackType === 'virtual' ? data.snapshot_schedule : undefined, }); + if (trackType === 'virtual') { + const { withReleaseLock } = require('./versioning-service'); + snapshot = await withReleaseLock(trackId, async (lease) => { + await require('./draft-cleanup-service').findScheduledResult( + trackId, + data.scheduled_materialization, + ); + await lease.assertOwned(); + const created = await saveSealedSnapshot(trackId, initialSnapshot, 'track_creation', lease); + await require('./draft-cleanup-service').recordScheduledResult(created); + await syncRegistryCounters(trackId); + return created; + }); + } logger.verbose(`SnapshotService: Created ${trackType} track "${data.name}" (${trackId})`); return snapshot; @@ -290,6 +288,8 @@ exports.getTrackMetadata = async function getTrackMetadata(trackId) { return { alias: entry?.alias ?? null, snapshot_schedule: entry?.snapshot_schedule, + snapshot_count: entry?.snapshot_count, + tagged_release_count: entry?.tagged_release_count, }; }; @@ -307,7 +307,8 @@ exports.getTrackMetadata = async function getTrackMetadata(trackId) { * * @param {string} trackId * @param {Object} options - { tagged?, limit, offset } - * @returns {Promise<{data: Object[], pagination: Object}>} + * @returns {Promise<{data: Object[], pagination: Object, counts: Object, + * latest_snapshot_modified: Date|null, latest_tagged_snapshot_modified: Date|null}>} * @throws {TrackNotFoundError} If the release track does not exist */ exports.listSnapshots = async function listSnapshots(trackId, options) { @@ -320,8 +321,15 @@ exports.listSnapshots = async function listSnapshots(trackId, options) { const statisticsByManifestId = await contentManifestService.getStatisticsByManifestIds( result.data.map((snapshot) => snapshot.content_manifest_id), ); + const latestTaggedModified = (track.tagged_releases || []).reduce( + (latest, release) => + !latest || release.snapshot_modified > latest ? release.snapshot_modified : latest, + null, + ); return { ...result, + latest_snapshot_modified: track.latest_snapshot_modified ?? null, + latest_tagged_snapshot_modified: latestTaggedModified, data: result.data.map((snapshot) => { const common = { id: snapshot.id, @@ -430,6 +438,24 @@ exports.cloneSnapshot = async function cloneSnapshot( overrides, options = {}, ) { + if (options.lease) return cloneSnapshotUnlocked(trackId, sourceSnapshot, overrides, options); + const { withReleaseLock } = require('./versioning-service'); + return withReleaseLock(trackId, (lease) => + cloneSnapshotUnlocked(trackId, sourceSnapshot, overrides, { ...options, lease }), + ); +}; + +async function cloneSnapshotUnlocked(trackId, sourceSnapshot, overrides, options) { + await options.lease.assertOwned(); + if (sourceSnapshot.type === 'virtual') { + const existing = await require('./draft-cleanup-service').findScheduledResult( + trackId, + overrides?.scheduled_materialization, + ); + if (existing) return existing; + // Direct callers must not resurrect a source removed while acquiring the lock. + await exports.getSnapshotByModified(trackId, sourceSnapshot.modified); + } const clone = deepClone(sourceSnapshot); const hasSnapshotDescriptionOverride = Object.prototype.hasOwnProperty.call( overrides || {}, @@ -439,7 +465,10 @@ exports.cloneSnapshot = async function cloneSnapshot( delete clone.publication; delete clone.bundle_id; delete clone.bundle_hashes; - clone.modified = new Date(); + delete clone.release_event_id; + delete clone.draft_retention; + delete clone.draft_cleanup; + clone.modified = new Date(Math.max(Date.now(), new Date(sourceSnapshot.modified).getTime() + 1)); clone.version = null; // clones are always drafts delete clone.scheduled_materialization; @@ -467,28 +496,63 @@ exports.cloneSnapshot = async function cloneSnapshot( clone.creation_cause = options.creationCause || CreationCause.Unknown; clone.creation_actor = creationActor(options.userAccountId); const normalized = tierRevisionInvariant.normalizeSnapshot(clone); + await options.lease.assertOwned(); + const cleanupIntent = + normalized.snapshot.type === 'virtual' && options.retention + ? await require('./draft-cleanup-service').prepareRetention( + normalized.snapshot, + options.retention, + options.lease, + ) + : null; let saved; - if (rewritesMembers || !normalized.snapshot.content_manifest_id) { - saved = await saveSealedSnapshot( - trackId, - normalized.snapshot, - options.sealReason || 'members_written', - ); - } else { - saved = await dynamicRepo.saveSnapshot(trackId, normalized.snapshot); - } + try { + if (rewritesMembers || !normalized.snapshot.content_manifest_id) { + saved = await saveSealedSnapshot( + trackId, + normalized.snapshot, + options.sealReason || 'members_written', + options.lease, + ); + } else { + saved = await dynamicRepo.saveSnapshot(trackId, normalized.snapshot); + } - if (saved.type === 'standard') { - const prunedDrafts = await dynamicRepo.deleteOlderDrafts(trackId, saved.modified); - await contentManifestService.discardUnreferenced( - trackId, - prunedDrafts.map((snapshot) => snapshot.content_manifest_id), - ); + if (saved.type === 'standard') { + // Materialization holds this same track's release lock until provenance + // is persisted, so this scan cannot miss a concurrently created dependent. + const virtualTracks = (await registryRepo.findAll({ type: 'virtual' })).data; + const referencedDrafts = await mapWithConcurrency(virtualTracks, 12, (track) => + dynamicRepo.findResolvedComponentSnapshotIds(track.track_id, trackId), + ); + const prunedDrafts = await dynamicRepo.deleteOlderDrafts( + trackId, + saved.modified, + referencedDrafts.flat(), + ); + await contentManifestService.discardUnreferenced( + trackId, + prunedDrafts.map((snapshot) => snapshot.content_manifest_id), + ); + } + await syncRegistryCounters(trackId); + + // The clone (modified = now) is the track's new latest snapshot + await emitContentsChanged(trackId, saved); + if (saved.type === 'virtual') { + await require('./draft-cleanup-service').recordScheduledResult(saved); + saved = await require('./draft-cleanup-service').afterDraft( + saved, + options.lease, + cleanupIntent, + ); + } + } catch (error) { + if (!cleanupIntent) throw error; + const persisted = saved || (await dynamicRepo.getSnapshotByModified(trackId, clone.modified)); + if (!persisted) throw error; + return require('./draft-cleanup-service').failDraft(persisted, cleanupIntent, error); } - await syncRegistryCounters(trackId); - - // The clone (modified = now) is the track's new latest snapshot - await emitContentsChanged(trackId, saved); if (normalized.removed.length > 0) { logger.warn( @@ -498,7 +562,7 @@ exports.cloneSnapshot = async function cloneSnapshot( } logger.verbose(`SnapshotService: Cloned snapshot for track "${trackId}"`); return saved; -}; +} // ============================================================================= // Track cloning @@ -512,8 +576,11 @@ exports.cloneSnapshot = async function cloneSnapshot( * @returns {Promise} The initial snapshot of the new track */ exports.cloneTrack = async function cloneTrack(trackId, options) { - const source = await exports.getLatestSnapshot(trackId); - return _cloneToNewTrack(source, options); + return require('./versioning-service').withReleaseLock(trackId, async (lease) => { + const source = await exports.getLatestSnapshot(trackId); + await lease.assertOwned(); + return _cloneToNewTrack(source, { ...options, lease }); + }); }; /** @@ -525,8 +592,11 @@ exports.cloneTrack = async function cloneTrack(trackId, options) { * @returns {Promise} The initial snapshot of the new track */ exports.cloneFromSnapshot = async function cloneFromSnapshot(trackId, modified, options) { - const source = await exports.getSnapshotByModified(trackId, modified); - return _cloneToNewTrack(source, options); + return require('./versioning-service').withReleaseLock(trackId, async (lease) => { + const source = await exports.getSnapshotByModified(trackId, modified); + await lease.assertOwned(); + return _cloneToNewTrack(source, { ...options, lease }); + }); }; /** @@ -541,6 +611,9 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { delete clone.publication; delete clone.bundle_id; delete clone.bundle_hashes; + delete clone.release_event_id; + delete clone.draft_retention; + delete clone.draft_cleanup; clone.id = newTrackId; clone.creation_cause = CreationCause.TrackCloned; clone.creation_actor = creationActor(options.userAccountId); @@ -565,7 +638,12 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { ); await modelFactory.ensureIndexes(newTrackId); - const saved = await saveSealedSnapshot(newTrackId, normalized.snapshot, 'track_clone'); + const saved = await saveSealedSnapshot( + newTrackId, + normalized.snapshot, + 'track_clone', + options.lease, + ); await registryRepo.create({ track_id: newTrackId, @@ -611,29 +689,32 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { */ exports.updateMetadata = async function updateMetadata(trackId, updates, userId) { - const source = await exports.getLatestSnapshot(trackId); - const overrides = {}; - if (updates.name !== undefined) overrides.name = updates.name; - if (updates.description !== undefined) overrides.description = updates.description; - - if (updates.alias !== undefined) { - if (updates.alias) await assertAliasAvailable(updates.alias, trackId); - await registryRepo.setAlias(trackId, updates.alias); - } + return require('./versioning-service').withReleaseLock(trackId, async (lease) => { + const source = await exports.getLatestSnapshot(trackId); + const overrides = {}; + if (updates.name !== undefined) overrides.name = updates.name; + if (updates.description !== undefined) overrides.description = updates.description; + + if (updates.alias !== undefined) { + if (updates.alias) await assertAliasAvailable(updates.alias, trackId); + await registryRepo.setAlias(trackId, updates.alias); + } - // Also update the registry name/description if changed - const registryUpdates = {}; - if (updates.name !== undefined) registryUpdates.name = updates.name; - if (updates.description !== undefined) registryUpdates.description = updates.description; - if (Object.keys(registryUpdates).length > 0) { - registryUpdates.updated_at = new Date(); - await registryRepo.updateByTrackId(trackId, registryUpdates); - } + // Also update the registry name/description if changed + const registryUpdates = {}; + if (updates.name !== undefined) registryUpdates.name = updates.name; + if (updates.description !== undefined) registryUpdates.description = updates.description; + if (Object.keys(registryUpdates).length > 0) { + registryUpdates.updated_at = new Date(); + await registryRepo.updateByTrackId(trackId, registryUpdates); + } - if (Object.keys(overrides).length === 0) return source; - return exports.cloneSnapshot(trackId, source, overrides, { - creationCause: CreationCause.MetadataUpdated, - userAccountId: userId, + if (Object.keys(overrides).length === 0) return source; + return exports.cloneSnapshot(trackId, source, overrides, { + creationCause: CreationCause.MetadataUpdated, + userAccountId: userId, + lease, + }); }); }; @@ -723,49 +804,51 @@ exports.getConfig = async function getConfig(trackId) { */ exports.updateConfig = async function updateConfig(trackId, config, userId) { - const source = await exports.getLatestSnapshot(trackId); - const existing = source.config || {}; - - const mergedConfig = { ...existing }; - - if (config.candidacy_threshold !== undefined) - mergedConfig.candidacy_threshold = config.candidacy_threshold; - if (config.auto_promote !== undefined) mergedConfig.auto_promote = config.auto_promote; - if (config.promotion_conflicts !== undefined) { - mergedConfig.promotion_conflicts = { - ...(existing.promotion_conflicts || {}), - ...config.promotion_conflicts, - }; - } - if (config.member_sync !== undefined) { - const existingMemberSync = existing.member_sync || {}; - mergedConfig.member_sync = { - ...existingMemberSync, - ...config.member_sync, - }; - // Nested merge for supplant sub-object - if (config.member_sync.supplant !== undefined) { - mergedConfig.member_sync.supplant = { - ...(existingMemberSync.supplant || {}), - ...config.member_sync.supplant, + return require('./versioning-service').withReleaseLock(trackId, async (lease) => { + const source = await exports.getLatestSnapshot(trackId); + const existing = source.config || {}; + + const mergedConfig = { ...existing }; + + if (config.candidacy_threshold !== undefined) + mergedConfig.candidacy_threshold = config.candidacy_threshold; + if (config.auto_promote !== undefined) mergedConfig.auto_promote = config.auto_promote; + if (config.promotion_conflicts !== undefined) { + mergedConfig.promotion_conflicts = { + ...(existing.promotion_conflicts || {}), + ...config.promotion_conflicts, }; } - } - if (config.publication !== undefined) { - const hasReleases = (source.version_history || []).length > 0; - mergedConfig.publication = publicationService.mergePublicationConfig( - existing.publication, - config.publication, - hasReleases, - ); - } + if (config.member_sync !== undefined) { + const existingMemberSync = existing.member_sync || {}; + mergedConfig.member_sync = { + ...existingMemberSync, + ...config.member_sync, + }; + // Nested merge for supplant sub-object + if (config.member_sync.supplant !== undefined) { + mergedConfig.member_sync.supplant = { + ...(existingMemberSync.supplant || {}), + ...config.member_sync.supplant, + }; + } + } + if (config.publication !== undefined) { + const hasReleases = (source.version_history || []).length > 0; + mergedConfig.publication = publicationService.mergePublicationConfig( + existing.publication, + config.publication, + hasReleases, + ); + } - return exports.cloneSnapshot( - trackId, - source, - { config: mergedConfig }, - { creationCause: CreationCause.ConfigurationUpdated, userAccountId: userId }, - ); + return exports.cloneSnapshot( + trackId, + source, + { config: mergedConfig }, + { creationCause: CreationCause.ConfigurationUpdated, userAccountId: userId, lease }, + ); + }); }; // ============================================================================= @@ -852,7 +935,12 @@ exports.reconstructManifest = async function reconstructManifest(trackId, modifi * @param {string} trackId * @throws {TrackNotFoundError} If the track does not exist in the registry */ -exports.deleteTrack = async function deleteTrack(trackId) { +exports.deleteTrack = async function deleteTrack(trackId, lease) { + if (!lease) { + return require('./versioning-service').withReleaseLock(trackId, (heldLease) => + exports.deleteTrack(trackId, heldLease), + ); + } const registry = await registryRepo.findByTrackId(trackId); if (!registry) { // A previous delete may have removed the registry only after dropping the @@ -861,8 +949,21 @@ exports.deleteTrack = async function deleteTrack(trackId) { throw new TrackNotFoundError(trackId); } + if (registry.type === 'virtual') { + let cursor; + for (;;) { + await lease.assertOwned(); + const scheduled = await dynamicRepo.getScheduledSnapshotBatch(trackId, cursor); + if (!scheduled.length) break; + for (const snapshot of scheduled) { + await require('./draft-cleanup-service').recordScheduledResult(snapshot); + } + cursor = scheduled.at(-1).modified; + } + } + await lease.assertOwned(); await dynamicRepo.dropCollection(trackId); - await contentManifestService.discardTrack(trackId); + await contentManifestService.discardTrack(trackId, () => lease.assertOwned()); await registryRepo.deleteByTrackId(trackId); // Remove all backrefs to the deleted track @@ -939,7 +1040,7 @@ exports.convertReleaseToDraft = async function convertReleaseToDraft(trackId, mo // composition resolution, notes, and immutable creation provenance. await dynamicRepo.updateSnapshot(trackId, snapshot.modified, { $set: { version: null }, - $unset: { publication: '', bundle_id: '', bundle_hashes: '' }, + $unset: { publication: '', bundle_id: '', bundle_hashes: '', release_event_id: '' }, }); draft = snapshot; } @@ -1011,6 +1112,7 @@ exports.deleteSnapshot = async function deleteSnapshot(trackId, modified) { snapshot_modified: new Date(snapshot.modified).toISOString(), }); } + await require('./draft-cleanup-service').recordScheduledResult(snapshot); await dynamicRepo.deleteSnapshot(trackId, modified); await contentManifestService.discardUnreferenced(trackId, [snapshot.content_manifest_id]); diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index 1d67ba69..ca41f1a7 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -18,6 +18,7 @@ const contentManifestService = require('./content-manifest-service'); const publicationService = require('./publication-service'); const bundleHashService = require('./bundle-hash-service'); const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); +const draftCleanupService = require('./draft-cleanup-service'); const uuid = require('uuid'); const logger = require('../../lib/logger'); const { @@ -95,19 +96,86 @@ function virtualReleaseChanges(previousSnapshot, draftSnapshot) { } /** - * Capture the tagged component versions frozen into a materialized virtual - * draft. Track IDs are stable provenance keys; component names are descriptive - * metadata and may change or collide. + * Capture the component versions frozen into a materialized virtual draft, + * using null for draft sources. Track IDs are stable provenance keys; + * component names are descriptive metadata and may change or collide. */ function virtualComponentVersions(snapshot) { return Object.fromEntries( (snapshot.composition_resolution?.component_snapshots || []).map((component) => [ component.track_id, - component.resolved_version, + component.resolved_version ?? null, ]), ); } +/** + * Calculate prospective membership independently of version allocation and + * publication. Callers must resolve staged selectors before planning. + */ +function planMembership(sourceSnapshot) { + if ( + sourceSnapshot.type === 'standard' && + (sourceSnapshot.staged || []).some((entry) => revisionReference.isLatest(entry.object_modified)) + ) { + throw new TypeError('Standard release planning requires resolved staged revisions'); + } + + const normalized = tierRevisionInvariant.normalizeSnapshot(sourceSnapshot); + const snapshot = normalized.snapshot; + const staged = snapshot.type === 'standard' ? snapshot.staged || [] : []; + const existingMembers = snapshot.members || []; + let mergedMembers = existingMembers; + let blockingError; + + if (staged.length > 0) { + const incoming = staged.map(({ object_ref, object_modified }) => ({ + object_ref, + object_modified, + })); + const policy = snapshot.config?.promotion_conflicts?.staged_to_members || 'abort'; + + try { + mergedMembers = conflictResolution.applyConflictPolicy( + existingMembers, + incoming, + policy, + ).merged; + } catch (err) { + if (!(err instanceof ReleaseConflictError)) throw err; + blockingError = err; + } + } + + return { normalized, snapshot, staged, mergedMembers, blockingError }; +} + +async function resolveReleaseInput(snapshot, latestByObjectRef) { + const releaseInput = + snapshot.type === 'standard' + ? { + ...snapshot, + staged: await revisionReference.resolveEntries(snapshot.staged || [], latestByObjectRef), + } + : snapshot; + await primaryRevisionService.assertStoredEntries([ + ...(releaseInput.members || []), + ...(releaseInput.staged || []), + ]); + return releaseInput; +} + +/** + * Return the exact members a standard draft would release, without allocating + * a version, checking prior publication, or changing the source snapshot. + */ +exports.planPreviewMembers = async function planPreviewMembers(snapshot, latestByObjectRef) { + const input = await resolveReleaseInput(snapshot, latestByObjectRef); + const plan = planMembership(input); + if (plan.blockingError) throw plan.blockingError; + return plan.mergedMembers; +}; + /** * Build the complete release plan without reading or writing external state. * @@ -136,15 +204,8 @@ function planRelease( 'Create a persisted draft with POST /api/release-tracks/:id/virtual/snapshots/create before previewing or releasing it', }); } - if ( - sourceSnapshot.type === 'standard' && - (sourceSnapshot.staged || []).some((entry) => revisionReference.isLatest(entry.object_modified)) - ) { - throw new TypeError('Standard release planning requires resolved staged revisions'); - } - - const normalized = tierRevisionInvariant.normalizeSnapshot(sourceSnapshot); - const snapshot = normalized.snapshot; + const { normalized, snapshot, staged, mergedMembers, blockingError } = + planMembership(sourceSnapshot); const releaseModified = sourceSnapshot.type === 'standard' ? new Date(Math.max(now.getTime(), new Date(sourceSnapshot.modified).getTime() + 1)) @@ -164,29 +225,6 @@ function planRelease( ? tierCounts(previousTaggedSnapshot) : { members_count: 0, quarantine_count: 0 } : tierCounts(snapshot); - const staged = snapshot.type === 'standard' ? snapshot.staged || [] : []; - const existingMembers = snapshot.members || []; - let mergedMembers = existingMembers; - let blockingError; - - if (staged.length > 0) { - const incoming = staged.map(({ object_ref, object_modified }) => ({ - object_ref, - object_modified, - })); - const policy = snapshot.config?.promotion_conflicts?.staged_to_members || 'abort'; - - try { - mergedMembers = conflictResolution.applyConflictPolicy( - existingMembers, - incoming, - policy, - ).merged; - } catch (err) { - if (!(err instanceof ReleaseConflictError)) throw err; - blockingError = err; - } - } const additionalOps = {}; for (const tier of normalized.changedTiers) { @@ -304,26 +342,12 @@ async function planLoadedSnapshot(trackId, snapshot, options) { throw new AlreadyReleasedError(existingRelease.version); } } - const [versionHistory, previousTaggedSnapshot, resolvedStaged] = await Promise.all([ + const [versionHistory, previousTaggedSnapshot, releaseInput] = await Promise.all([ releaseHistoryService.getTrackWideVersionHistory(trackId), snapshot.type === 'virtual' ? dynamicRepo.getLatestTaggedSnapshotBefore(trackId, snapshot.modified) : Promise.resolve(null), - snapshot.type === 'standard' - ? revisionReference.resolveEntries(snapshot.staged || []) - : Promise.resolve(snapshot.staged || []), - ]); - const releaseInput = - snapshot.type === 'standard' - ? { - ...snapshot, - staged: resolvedStaged, - } - : snapshot; - - await primaryRevisionService.assertStoredEntries([ - ...(releaseInput.members || []), - ...(releaseInput.staged || []), + resolveReleaseInput(snapshot), ]); const plan = planRelease( @@ -345,6 +369,9 @@ async function planLoadedSnapshot(trackId, snapshot, options) { plan.plannedSnapshot.members, ); } + if (snapshot.type === 'virtual') { + plan.summary.draft_squash = await draftCleanupService.previewSquash(trackId, snapshot); + } return plan; } @@ -379,7 +406,7 @@ async function refreshReleaseArtifacts(tagged) { } exports.refreshReleaseArtifacts = refreshReleaseArtifacts; -async function commitPlan(plan) { +async function commitPlan(plan, lease) { if (plan.blockingError) throw plan.blockingError; const source = plan.sourceSnapshot; @@ -402,9 +429,11 @@ async function commitPlan(plan) { } setOps.publication = await publicationService.freezePublication(source); setOps.bundle_id = `bundle--${uuid.v4()}`; + if (source.type === 'virtual') setOps.release_event_id = plan.releaseEventId || uuid.v4(); let tagged; try { + await lease.assertOwned(); if (source.type === 'standard') { const releaseSnapshot = { ...plan.plannedSnapshot, ...setOps }; delete releaseSnapshot._id; @@ -436,6 +465,7 @@ async function commitPlan(plan) { } const withArtifacts = await refreshReleaseArtifacts(tagged); + await lease.assertOwned(); await releaseHistoryService.reconcileTaggedReleases(plan.trackId); if (source.type === 'standard') { @@ -472,8 +502,22 @@ async function withReleaseLock(trackId, operation) { }); } + const lease = { + async assertOwned() { + const renewed = await registryRepo.renewReleaseLock( + trackId, + token, + new Date(Date.now() - RELEASE_LOCK_TIMEOUT_MS), + ); + if (!renewed) { + throw new ReleaseConflictError('The release-track lifecycle lease was lost', { + track_id: trackId, + }); + } + }, + }; try { - return await operation(); + return await operation(lease); } finally { try { await registryRepo.releaseReleaseLock(trackId, token); @@ -508,15 +552,45 @@ exports.planReleaseByModified = async function planReleaseByModified( return planLoadedSnapshot(trackId, snapshot, options); }; +async function releaseLocked(trackId, loadPlan, options) { + if (options.squash_drafts) draftCleanupService.assertAdmin(options.actor); + return withReleaseLock(trackId, async (lease) => { + const plan = await loadPlan(); + const intent = options.squash_drafts + ? await draftCleanupService.beginSquash(plan, options, lease) + : null; + if (intent) plan.releaseEventId = intent.event_id; + let released; + try { + released = await commitPlan(plan, lease); + } catch (error) { + if (intent) throw await draftCleanupService.failRelease(intent, error); + throw error; + } + if (!intent) return released; + intent.cleanup = { + kind: 'squash', + eligible_count: intent.request.eligible_count, + deleted_count: 0, + protected_count: 0, + cursor: null, + pending_batch: [], + release_committed: true, + publication_complete: true, + }; + return { ...released, draft_cleanup: await draftCleanupService.runCleanup(intent, lease) }; + }); +} + exports.releaseLatest = async function releaseLatest(trackId, options = {}) { - return withReleaseLock(trackId, async () => - commitPlan(await exports.planLatestRelease(trackId, options)), - ); + return releaseLocked(trackId, () => exports.planLatestRelease(trackId, options), options); }; exports.releaseByModified = async function releaseByModified(trackId, modified, options = {}) { - return withReleaseLock(trackId, async () => - commitPlan(await exports.planReleaseByModified(trackId, modified, options)), + return releaseLocked( + trackId, + () => exports.planReleaseByModified(trackId, modified, options), + options, ); }; diff --git a/app/services/release-tracks/virtual-track-service.js b/app/services/release-tracks/virtual-track-service.js index 838d9957..5fe4e940 100644 --- a/app/services/release-tracks/virtual-track-service.js +++ b/app/services/release-tracks/virtual-track-service.js @@ -9,8 +9,8 @@ const CreationCause = require('../../lib/release-tracks/snapshot-creation-causes // snapshot creation via resolution of component tracks. // // Virtual tracks aggregate content from multiple standard tracks by: -// 1. Resolving each component track to a specific tagged snapshot -// 2. Collecting members from each resolved snapshot +// 1. Resolving each component track to its selected standard snapshot +// 2. Planning draft preview members or collecting published members // 3. Applying per-component filters (object_types and domains) // 4. Deduplicating across all components // 5. Persisting the result as a new draft snapshot @@ -19,6 +19,7 @@ const CreationCause = require('../../lib/release-tracks/snapshot-creation-causes // ============================================================================= const snapshotService = require('./snapshot-service'); +const draftCleanup = require('./draft-cleanup-service'); const primaryRevisionService = require('./primary-revision-service'); const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); @@ -33,6 +34,7 @@ const { TrackNotFoundError, NoTaggedSnapshotsError, InvalidComponentTypeError, + ReleaseConflictError, NotFoundError, } = require('../../exceptions'); @@ -127,8 +129,7 @@ exports.validateComposition = async function validateComposition(composition) { }; /** - * Resolve a component track to a specific tagged snapshot based on its - * resolution strategy. + * Resolve a component track to a snapshot based on its resolution strategy. * * @param {Object} component - A component_tracks entry * @returns {Promise} The resolved snapshot document @@ -142,6 +143,10 @@ async function resolveComponentSnapshot(component) { snapshot = await dynamicRepo.getLatestTaggedSnapshot(component.track_id); break; + case 'latest_preview': + snapshot = await dynamicRepo.getLatestSnapshot(component.track_id); + break; + case 'specific_version': snapshot = await dynamicRepo.getSnapshotByVersion(component.track_id, component.version); break; @@ -150,6 +155,12 @@ async function resolveComponentSnapshot(component) { snapshot = await dynamicRepo.getSnapshotByModified(component.track_id, component.snapshot); break; + case 'latest_draft': + throw new BadRequestError({ + message: `Component track '${component.track_id}' uses retired strategy latest_draft`, + details: 'Update composition to explicitly choose latest_preview or latest_tagged', + }); + default: throw new BadRequestError({ message: `Unknown resolution strategy: ${component.resolution_strategy}`, @@ -160,8 +171,8 @@ async function resolveComponentSnapshot(component) { throw new NoTaggedSnapshotsError(component.track_id); } - // For specific_snapshot strategy, the snapshot may be a draft — validate it's tagged - if (snapshot.version == null) { + // Explicit snapshot selection remains tagged-only. + if (component.resolution_strategy !== 'latest_preview' && snapshot.version == null) { throw new NoTaggedSnapshotsError(component.track_id); } @@ -278,11 +289,10 @@ async function hydrateDomains(componentTracks, resolutions) { * component data. The virtual snapshot itself never persists a moving ref. * * @param {Array} resolutions + * @param {Map>} latestByObjectRef * @returns {Promise>} */ -async function lockComponentMemberRevisions(resolutions) { - const latestByObjectRef = new Map(); - +async function lockComponentMemberRevisions(resolutions, latestByObjectRef) { const resolveLatest = (objectRef) => { if (!latestByObjectRef.has(objectRef)) { latestByObjectRef.set(objectRef, objectResolver.resolveLatestModified(objectRef)); @@ -340,7 +350,29 @@ async function resolveComposition(snapshot, registryMap) { const resolvedComponentSnapshots = await Promise.all( componentTracks.map((component) => resolveComponentSnapshot(component)), ); - const resolutions = await lockComponentMemberRevisions(resolvedComponentSnapshots); + const latestByObjectRef = new Map(); + const resolutions = await lockComponentMemberRevisions( + resolvedComponentSnapshots, + latestByObjectRef, + ); + const { planPreviewMembers } = require('./versioning-service'); + for (let i = 0; i < componentTracks.length; i++) { + if ( + componentTracks[i].resolution_strategy !== 'latest_preview' || + resolutions[i].version != null + ) { + continue; + } + try { + resolutions[i].members = await planPreviewMembers(resolutions[i], latestByObjectRef); + } catch (err) { + if (!(err instanceof ReleaseConflictError)) throw err; + throw new ReleaseConflictError( + `Component track '${componentTracks[i].track_id}' cannot preview staged membership: ${err.message}`, + { track_id: componentTracks[i].track_id, conflicts: err.conflicts }, + ); + } + } const domainsByVersion = await hydrateDomains(componentTracks, resolutions); for (let i = 0; i < componentTracks.length; i++) { @@ -375,7 +407,7 @@ async function resolveComposition(snapshot, registryMap) { track_name: registry.name, track_type: registry.type, resolved_snapshot_id: resolvedSnapshot.modified, - resolved_version: resolvedSnapshot.version, + resolved_version: resolvedSnapshot.version ?? null, strategy_used: component.resolution_strategy, filters_applied: component.filters || undefined, total_objects_in_source: totalObjectsInSource, @@ -439,30 +471,37 @@ exports.updateComposition = async function updateComposition( userId, options = {}, ) { - const source = await snapshotService.getLatestSnapshot(trackId); - assertVirtualTrack(source); - - // Validate all component tracks - await validateComponentTracks(composition.component_tracks); - - const snapshot = await snapshotService.cloneSnapshot( - trackId, - source, - { - composition, - members: [], - quarantine: [], - composition_resolution: null, - scheduled_materialization: options.scheduledMaterialization, - }, - { creationCause: CreationCause.CompositionUpdated, userAccountId: userId }, - ); + return require('./versioning-service').withReleaseLock(trackId, async (lease) => { + const existing = await require('./draft-cleanup-service').findScheduledResult( + trackId, + options.scheduledMaterialization, + ); + if (existing) return existing; + const source = await snapshotService.getLatestSnapshot(trackId); + assertVirtualTrack(source); - logger.verbose( - `VirtualTrackService: Updated composition for track "${trackId}" ` + - `(${composition.component_tracks.length} component track(s))`, - ); - return snapshot; + // Validate all component tracks + await validateComponentTracks(composition.component_tracks); + + const snapshot = await snapshotService.cloneSnapshot( + trackId, + source, + { + composition, + members: [], + quarantine: [], + composition_resolution: null, + scheduled_materialization: options.scheduledMaterialization, + }, + { creationCause: CreationCause.CompositionUpdated, userAccountId: userId, lease }, + ); + + logger.verbose( + `VirtualTrackService: Updated composition for track "${trackId}" ` + + `(${composition.component_tracks.length} component track(s))`, + ); + return snapshot; + }); }; /** @@ -472,33 +511,46 @@ exports.updateComposition = async function updateComposition( * * @param {string} trackId * @param {Object} schedule + * @param {Object} actor * @returns {Promise<{snapshot_schedule: Object}>} */ -exports.updateSchedule = async function updateSchedule(trackId, schedule) { - const registry = await registryRepo.findByTrackId(trackId); - if (!registry) { - throw new TrackNotFoundError(trackId); - } - if (registry.type !== 'virtual') { - throw new BadRequestError({ - message: 'This operation is only available for virtual release tracks', - details: `Track ${trackId} is a ${registry.type} track`, - }); - } - - const updated = await registryRepo.setSnapshotSchedule(trackId, schedule); - logger.verbose( - `VirtualTrackService: Updated snapshot schedule for track "${trackId}" to ${schedule.mode}`, - ); - return { snapshot_schedule: updated.snapshot_schedule }; +exports.updateSchedule = async function updateSchedule(trackId, schedule, actor) { + return require('./versioning-service').withReleaseLock(trackId, async (lease) => { + const registry = await registryRepo.findByTrackId(trackId); + if (!registry) { + throw new TrackNotFoundError(trackId); + } + if (registry.type !== 'virtual') { + throw new BadRequestError({ + message: 'This operation is only available for virtual release tracks', + details: `Track ${trackId} is a ${registry.type} track`, + }); + } + const previousMaximum = + registry.snapshot_schedule?.mode === 'cron' + ? (registry.snapshot_schedule.draft_retention?.max_drafts ?? null) + : null; + if ( + schedule.mode === 'cron' && + (schedule.draft_retention?.max_drafts ?? null) !== previousMaximum + ) { + draftCleanup.assertAdmin(actor); + } + await lease.assertOwned(); + const updated = await registryRepo.setSnapshotSchedule(trackId, schedule); + logger.verbose( + `VirtualTrackService: Updated snapshot schedule for track "${trackId}" to ${schedule.mode}`, + ); + return { snapshot_schedule: updated.snapshot_schedule }; + }); }; /** * Create a new virtual snapshot by resolving the composition rules. * * For each component track: - * 1. Resolve to a tagged snapshot via the configured strategy - * 2. Extract and filter members + * 1. Resolve the selected standard snapshot via the configured strategy + * 2. Plan preview membership for drafts, then filter the resolved members * Then deduplicate across all components and persist a new draft snapshot. * * @param {string} trackId @@ -506,82 +558,121 @@ exports.updateSchedule = async function updateSchedule(trackId, schedule) { * @returns {Promise} The new snapshot with composition_resolution metadata */ exports.createVirtualSnapshot = async function createVirtualSnapshot(trackId, options = {}) { - const scheduledFor = options.scheduledMaterialization?.scheduled_for; - if (scheduledFor) { - const existing = await dynamicRepo.getSnapshotByScheduledMaterialization(trackId, scheduledFor); + return require('./versioning-service').withReleaseLock(trackId, async (lease) => { + // Only an internal scheduler option can select recurring policy. Public + // occurrence metadata is an idempotency receipt, never cleanup authority. + const manualPolicy = + options.draft_retention !== undefined + ? draftCleanup.validatePolicy(options.draft_retention, options.actor, 'virtual') + : null; + let retention = null; + if (options.useRecurringRetention === true) { + const registry = await registryRepo.findByTrackId(trackId, 'snapshot_schedule'); + const schedule = registry?.snapshot_schedule; + if (schedule?.mode === 'cron' && schedule.draft_retention?.max_drafts != null) { + retention = { + source: 'recurring', + policy: schedule.draft_retention, + actor: { kind: 'system' }, + }; + } + } else if (manualPolicy?.max_drafts != null) { + retention = { source: 'manual', policy: manualPolicy, actor: options.actor }; + } + const scheduledFor = options.scheduledMaterialization?.scheduled_for; + const existing = await require('./draft-cleanup-service').findScheduledResult( + trackId, + options.scheduledMaterialization, + ); if (existing) return existing; - } - const source = await snapshotService.getLatestSnapshot(trackId); - assertVirtualTrack(source); + const source = await snapshotService.getLatestSnapshot(trackId); + assertVirtualTrack(source); - const composition = source.composition; - if (!composition || !composition.component_tracks || composition.component_tracks.length === 0) { - throw new BadRequestError({ - message: 'Cannot create virtual snapshot: no component tracks configured', - details: 'Update the composition before creating a snapshot', - }); - } - - // Hold component release locks from resolution through persistence. Rollback - // cannot pass its dependency scan while a new dependent is being created. - // Sorted acquisition and fail-fast conflicts also release partial lock sets. - const componentIds = [ - ...new Set(composition.component_tracks.map((entry) => entry.track_id)), - ].sort(); - const { withReleaseLock } = require('./versioning-service'); - async function withComponentLocks(index) { - if (index === componentIds.length) return materialize(); - return withReleaseLock(componentIds[index], () => withComponentLocks(index + 1)); - } - return withComponentLocks(0); - - async function materialize() { - // Validate component tracks - const registryMap = await validateComponentTracks(composition.component_tracks); + const composition = source.composition; + if ( + !composition || + !composition.component_tracks || + composition.component_tracks.length === 0 + ) { + throw new BadRequestError({ + message: 'Cannot create virtual snapshot: no component tracks configured', + details: 'Update the composition before creating a snapshot', + }); + } - // Resolve composition - const { members, quarantined, compositionResolution } = await resolveComposition( - source, - registryMap, - ); - await primaryRevisionService.assertStoredEntries([...members, ...quarantined]); - - // Build overrides for the new snapshot - const overrides = { - members, - quarantine: quarantined, - composition_resolution: compositionResolution, - scheduled_materialization: options.scheduledMaterialization, - snapshot_description: options.description, - }; - - let snapshot; - try { - snapshot = await snapshotService.cloneSnapshot(trackId, source, overrides, { - creationCause: options.scheduledMaterialization - ? CreationCause.ScheduledSnapshot - : CreationCause.ManualSnapshot, - userAccountId: - options.userAccountId || (options.scheduledMaterialization ? 'system' : undefined), + // Hold component release locks from resolution through persistence. Rollback + // cannot pass its dependency scan while a new dependent is being created. + // Sorted acquisition and fail-fast conflicts also release partial lock sets. + const componentIds = [ + ...new Set(composition.component_tracks.map((entry) => entry.track_id)), + ].sort(); + const { withReleaseLock } = require('./versioning-service'); + const componentLeases = []; + async function withComponentLocks(index) { + if (index === componentIds.length) return materialize(); + return withReleaseLock(componentIds[index], (componentLease) => { + componentLeases.push(componentLease); + return withComponentLocks(index + 1); }); - } catch (err) { - if (!scheduledFor || !(err instanceof DuplicateIdError)) throw err; + } + return withComponentLocks(0); - const existing = await dynamicRepo.getSnapshotByScheduledMaterialization( - trackId, - scheduledFor, + async function materialize() { + // Validate component tracks + const registryMap = await validateComponentTracks(composition.component_tracks); + + // Resolve composition + const { members, quarantined, compositionResolution } = await resolveComposition( + source, + registryMap, + ); + await primaryRevisionService.assertStoredEntries([...members, ...quarantined]); + const materializationLease = { + async assertOwned() { + await lease.assertOwned(); + for (const componentLease of componentLeases) await componentLease.assertOwned(); + }, + }; + + // Build overrides for the new snapshot + const overrides = { + members, + quarantine: quarantined, + composition_resolution: compositionResolution, + scheduled_materialization: options.scheduledMaterialization, + snapshot_description: options.description, + }; + + let snapshot; + try { + snapshot = await snapshotService.cloneSnapshot(trackId, source, overrides, { + lease: materializationLease, + retention, + creationCause: options.scheduledMaterialization + ? CreationCause.ScheduledSnapshot + : CreationCause.ManualSnapshot, + userAccountId: + options.userAccountId || (options.scheduledMaterialization ? 'system' : undefined), + }); + } catch (err) { + if (!scheduledFor || !(err instanceof DuplicateIdError)) throw err; + + const existing = await dynamicRepo.getSnapshotByScheduledMaterialization( + trackId, + scheduledFor, + ); + if (!existing) throw err; + snapshot = existing; + } + + logger.verbose( + `VirtualTrackService: Created virtual snapshot for track "${trackId}" ` + + `(${members.length} members, ${quarantined.length} quarantined)`, ); - if (!existing) throw err; - snapshot = existing; + return snapshot; } - - logger.verbose( - `VirtualTrackService: Created virtual snapshot for track "${trackId}" ` + - `(${members.length} members, ${quarantined.length} quarantined)`, - ); - return snapshot; - } + }); }; /** @@ -601,48 +692,50 @@ exports.promoteQuarantinedObject = async function promoteQuarantinedObject( selection, userId, ) { - const source = await snapshotService.getLatestSnapshot(trackId); - assertVirtualTrack(source); - - const selectedTime = new Date(selection.object_modified).getTime(); - const selected = (source.quarantine || []).find( - (entry) => - entry.object_ref === selection.object_ref && - new Date(entry.object_modified).getTime() === selectedTime, - ); + return require('./versioning-service').withReleaseLock(trackId, async (lease) => { + const source = await snapshotService.getLatestSnapshot(trackId); + assertVirtualTrack(source); + + const selectedTime = new Date(selection.object_modified).getTime(); + const selected = (source.quarantine || []).find( + (entry) => + entry.object_ref === selection.object_ref && + new Date(entry.object_modified).getTime() === selectedTime, + ); - if (!selected) { - throw new NotFoundError({ - details: - `Revision '${selection.object_modified}' of '${selection.object_ref}' ` + - `was not found in the latest snapshot's quarantine tier`, - }); - } + if (!selected) { + throw new NotFoundError({ + details: + `Revision '${selection.object_modified}' of '${selection.object_ref}' ` + + `was not found in the latest snapshot's quarantine tier`, + }); + } - const members = (source.members || []) - .filter((entry) => entry.object_ref !== selected.object_ref) - .concat({ - object_ref: selected.object_ref, - object_modified: selected.object_modified, - }); - const quarantine = (source.quarantine || []).filter( - (entry) => entry.object_ref !== selected.object_ref, - ); - await primaryRevisionService.assertStoredEntries([...members, ...quarantine]); - - const snapshot = await snapshotService.cloneSnapshot( - trackId, - source, - { - members, - quarantine, - }, - { creationCause: CreationCause.QuarantinePromoted, userAccountId: userId }, - ); + const members = (source.members || []) + .filter((entry) => entry.object_ref !== selected.object_ref) + .concat({ + object_ref: selected.object_ref, + object_modified: selected.object_modified, + }); + const quarantine = (source.quarantine || []).filter( + (entry) => entry.object_ref !== selected.object_ref, + ); + await primaryRevisionService.assertStoredEntries([...members, ...quarantine]); - logger.verbose( - `VirtualTrackService: Promoted quarantined revision "${selected.object_ref}" ` + - `at ${new Date(selected.object_modified).toISOString()} in track "${trackId}"`, - ); - return snapshot; + const snapshot = await snapshotService.cloneSnapshot( + trackId, + source, + { + members, + quarantine, + }, + { creationCause: CreationCause.QuarantinePromoted, userAccountId: userId, lease }, + ); + + logger.verbose( + `VirtualTrackService: Promoted quarantined revision "${selected.object_ref}" ` + + `at ${new Date(selected.object_modified).toISOString()} in track "${trackId}"`, + ); + return snapshot; + }); }; diff --git a/app/services/system/allowed-values-service.js b/app/services/system/allowed-values-service.js new file mode 100644 index 00000000..3330135a --- /dev/null +++ b/app/services/system/allowed-values-service.js @@ -0,0 +1,176 @@ +'use strict'; + +const fs = require('fs/promises'); +const config = require('../../config/config'); +const seed = require('../../config/allowed-values.json'); +const repository = require('../../repository/allowed-values-repository'); +const { BadRequestError, DuplicateIdError } = require('../../exceptions'); +const { + catalog, + seededRules, + ruleKey, + requireDefinition, + getValueIssue, + validateValue, +} = require('../../lib/allowed-values-catalog'); + +function projectRules({ rules, values }) { + const grouped = new Map( + rules.map((rule) => { + const definition = catalog.rules.find((candidate) => ruleKey(candidate) === ruleKey(rule)); + return [ + ruleKey(rule), + { + ...rule, + objectTypes: definition?.objectTypes || [], + options: new Map(), + invalid: new Map(), + }, + ]; + }), + ); + for (const { objectType, propertyName, domainName, value, enabled } of values) { + const rule = grouped.get(ruleKey({ propertyName, domainName })); + if (!rule) continue; + const reason = getValueIssue({ objectType, propertyName, domainName, value }); + const options = reason ? rule.invalid : rule.options; + const key = JSON.stringify([value, enabled, reason]); + if (!options.has(key)) { + options.set(key, { value, enabled, objectTypes: [], ...(reason ? { reason } : {}) }); + } + const option = options.get(key); + if (!option.objectTypes.includes(objectType)) option.objectTypes.push(objectType); + } + return [...grouped.values()].map(({ options, invalid, ...rule }) => ({ + ...rule, + values: [...options.values()], + invalidValues: [...invalid.values()], + })); +} + +class AllowedValuesService { + async initialize() { + if (await repository.retrieveConfiguration({ optional: true })) { + // Legacy values are quarantined on reads, not deleted or allowed to prevent + // startup. The seed file is relevant only when initializing a new database. + await repository.migrateRules(seededRules); + return; + } + const configuredSeed = JSON.parse( + await fs.readFile(config.configurationFiles.allowedValues, 'utf8'), + ); + const values = []; + const seen = new Set(); + const rules = new Map(seededRules.map((rule) => [ruleKey(rule), rule])); + if (!Array.isArray(configuredSeed)) + throw new BadRequestError({ details: 'Invalid allowed-values seed.' }); + for (const { objectType, properties } of configuredSeed) { + for (const { propertyName, domains } of properties) { + for (const { domainName, allowedValues } of domains) { + const definition = requireDefinition(propertyName, domainName); + if (!definition.objectTypes.includes(objectType) || !Array.isArray(allowedValues)) { + throw new BadRequestError({ details: 'Invalid allowed-values seed scope.' }); + } + const field = { objectType, propertyName, domainName }; + rules.set(ruleKey(field), { propertyName, domainName }); + for (const rawValue of allowedValues) { + const { value } = validateValue({ + propertyName, + domainName, + objectTypes: [objectType], + value: rawValue, + }); + const key = JSON.stringify([objectType, propertyName, domainName, value]); + if (!seen.has(key)) { + values.push({ ...field, value, enabled: true }); + seen.add(key); + } + } + } + } + } + await repository.initialize(values, [...rules.values()]); + await repository.migrateRules(seededRules); + } + + retrieveCatalog() { + return catalog; + } + + validateValue(input) { + return validateValue(input); + } + + async retrieveRules() { + return projectRules(await repository.retrieveConfiguration()); + } + + async retrieveAllowedValues() { + const configuration = await repository.retrieveConfiguration(); + const configuredKeys = new Set(configuration.rules.map(ruleKey)); + const rules = catalog.rules.filter((rule) => configuredKeys.has(ruleKey(rule))); + const valuesByField = new Map(); + for (const row of configuration.values) { + if (!row.enabled || !configuredKeys.has(ruleKey(row)) || getValueIssue(row)) continue; + const key = JSON.stringify([row.objectType, row.propertyName, row.domainName]); + if (!valuesByField.has(key)) valuesByField.set(key, []); + valuesByField.get(key).push(row.value); + } + // Preserve the original nested ordering and empty scopes, while including + // ADM-supported domains explicitly added by administrators. + return seed.map(({ objectType, properties }) => ({ + objectType, + properties: properties.map(({ propertyName, domains }) => { + const domainNames = new Set(domains.map(({ domainName }) => domainName)); + for (const rule of rules) { + if (rule.propertyName === propertyName && rule.objectTypes.includes(objectType)) { + domainNames.add(rule.domainName); + } + } + return { + propertyName, + domains: [...domainNames].map((domainName) => ({ + domainName, + allowedValues: + valuesByField.get(JSON.stringify([objectType, propertyName, domainName])) || [], + })), + }; + }), + })); + } + + async saveRule(propertyName, domainName, values, options) { + requireDefinition(propertyName, domainName); + const seen = new Set(); + const rows = []; + for (const option of values) { + const { value } = validateValue({ propertyName, domainName, ...option }); + if (typeof option.enabled !== 'boolean') + throw new BadRequestError({ details: 'Enabled must be boolean.' }); + for (const objectType of option.objectTypes) { + const tupleValue = JSON.stringify([objectType, value]); + if (seen.has(tupleValue)) { + throw new DuplicateIdError({ + details: 'A value may appear only once per object type within a rule.', + }); + } + seen.add(tupleValue); + rows.push({ objectType, propertyName, domainName, value, enabled: option.enabled }); + } + } + const updated = await repository.saveRule(propertyName, domainName, rows, options); + return projectRules(updated).find( + (rule) => ruleKey(rule) === ruleKey({ propertyName, domainName }), + ); + } + + createRule(propertyName, domainName, values) { + return this.saveRule(propertyName, domainName, values, { create: true }); + } + + replaceRule(propertyName, domainName, values) { + return this.saveRule(propertyName, domainName, values); + } +} + +module.exports = new AllowedValuesService(); diff --git a/app/services/system/system-configuration-service.js b/app/services/system/system-configuration-service.js index 371fe0b3..b1897c7c 100644 --- a/app/services/system/system-configuration-service.js +++ b/app/services/system/system-configuration-service.js @@ -1,6 +1,6 @@ 'use strict'; -const fs = require('fs'); +const allowedValuesService = require('./allowed-values-service'); const config = require('../../config/config'); const systemConfigurationRepository = require('../../repository/system-configurations-repository'); const userAccountsService = require('./user-accounts-service'); @@ -22,7 +22,6 @@ const { class SystemConfigurationService extends BaseService { constructor() { super(null, systemConfigurationRepository); - this._allowedValues = null; } /** @@ -46,12 +45,7 @@ class SystemConfigurationService extends BaseService { * Returns allowed values for system configuration */ async retrieveAllowedValues() { - if (this._allowedValues) { - return this._allowedValues; - } - const data = await fs.promises.readFile(config.configurationFiles.allowedValues); - this._allowedValues = JSON.parse(data); - return this._allowedValues; + return allowedValuesService.retrieveAllowedValues(); } /** diff --git a/app/tests/api/release-tracks/draft-lifecycle.spec.js b/app/tests/api/release-tracks/draft-lifecycle.spec.js new file mode 100644 index 00000000..39b71233 --- /dev/null +++ b/app/tests/api/release-tracks/draft-lifecycle.spec.js @@ -0,0 +1,1041 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const service = require('../../../services/release-tracks/release-tracks-service'); +const snapshotService = require('../../../services/release-tracks/snapshot-service'); +const virtual = require('../../../services/release-tracks/virtual-track-service'); +const versioning = require('../../../services/release-tracks/versioning-service'); +const cleanup = require('../../../services/release-tracks/draft-cleanup-service'); +const manifests = require('../../../services/release-tracks/content-manifest-service'); +const bundleHashes = require('../../../services/release-tracks/bundle-hash-service'); +const publication = require('../../../services/release-tracks/publication-service'); +const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); +const registryRepo = require('../../../repository/release-tracks/release-track-registry.repository'); +const auditRepo = require('../../../repository/release-tracks/release-track-audit-event.repository'); +const occurrenceRepo = require('../../../repository/release-tracks/virtual-track-schedule-occurrence.repository'); +const modelFactory = require('../../../models/release-tracks/model-factory'); +const { + ReleaseTrackContentManifest, + ReleaseTrackContentManifestEntry, +} = require('../../../models/release-tracks/release-track-content-manifest-model'); +const { ReleaseConflictError, InsufficientRoleError } = require('../../../exceptions'); +const admin = { role: 'admin', user_account_id: 'identity--00000000-0000-4000-8000-000000000901' }; +const editor = { ...admin, role: 'editor' }; +const iso = (value) => new Date(value).toISOString(); + +// These tests exercise observable history, exports, recovery, and lock conflicts. +// Faults are restored in finally blocks so the shared API process remains usable. +describe('Virtual draft lifecycle API', function () { + let app; + let cookie; + let fixtureNumber = 0; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + cookie = await login.loginAnonymous(app); + }); + + after(async function () { + await database.closeConnection(); + }); + + async function api(method, path, body, status = 200) { + const call = request(app) + [method](path) + .set('Accept', 'application/json') + .set('Cookie', `${cookie.name}=${cookie.value}`); + if (body !== undefined) call.send(body); + return (await call.expect(status)).body; + } + + function base(track) { + return `/api/release-tracks/${track.id}`; + } + function selected(track, snapshot) { + return `${base(track)}/snapshots/${encodeURIComponent(iso(snapshot.modified))}`; + } + async function history(track) { + return api('get', `${base(track)}/snapshots?limit=200&offset=0`); + } + async function draft(track, description = 'Draft') { + return api('post', `${base(track)}/meta`, { description }); + } + async function policy(track, maximum) { + return api('put', `${base(track)}/virtual/schedule`, { + mode: 'cron', + cron: '0 0 * * *', + draft_retention: { max_drafts: maximum }, + }); + } + async function materialize(track, maximum, options = {}) { + return api( + 'post', + `${base(track)}/virtual/snapshots/create`, + { + ...(maximum === undefined ? {} : { draft_retention: { max_drafts: maximum } }), + ...options, + }, + 201, + ); + } + async function recurringDraft(track) { + return JSON.parse( + JSON.stringify( + await virtual.createVirtualSnapshot(track.id, { useRecurringRetention: true }), + ), + ); + } + async function preview(track, snapshot) { + return api('get', `${selected(track, snapshot)}/release/preview`); + } + async function squash(track, target, version = '1.0') { + const reviewed = await preview(track, target); + return api('post', `${selected(track, target)}/release`, { + version, + squash_drafts: true, + squash_fingerprint: reviewed.draft_squash.fingerprint, + }); + } + + async function fixture() { + fixtureNumber += 1; + const component = await api( + 'post', + '/api/release-tracks/new', + { name: `Lifecycle Component ${fixtureNumber}`, type: 'standard' }, + 201, + ); + const composition = { + component_tracks: [ + { track_id: component.id, resolution_strategy: 'latest_preview', priority: 0 }, + ], + }; + const track = await api( + 'post', + '/api/release-tracks/new', + { name: `Lifecycle Virtual ${fixtureNumber}`, type: 'virtual', composition }, + 201, + ); + const materialized = await api('post', `${base(track)}/virtual/snapshots/create`, {}, 201); + return { track, materialized, component, composition }; + } + + it('saves cron policy without a content snapshot and never projects a global policy', async function () { + const { track } = await fixture(); + await draft(track, 'Unlimited first'); + const latest = await draft(track, 'Unlimited second'); + const before = await history(track); + expect(before.pagination.total).toBe(4); + const saved = await policy(track, 1); + expect(saved.snapshot_schedule.draft_retention).toEqual({ max_drafts: 1 }); + expect((await history(track)).data.map((entry) => entry.modified)).toEqual( + before.data.map((entry) => entry.modified), + ); + const oldView = await api('get', selected(track, track)); + expect(oldView.snapshot_schedule).toEqual(saved.snapshot_schedule); + expect(oldView).not.toHaveProperty('draft_retention'); + expect(oldView.snapshot_count).toBe(4); + expect(oldView.tagged_release_count).toBe(0); + expect(await api('get', `${base(track)}/config`)).not.toHaveProperty('draft_retention'); + const clone = await api( + 'post', + `${selected(track, latest)}/clone`, + { name: `Lifecycle Clone ${fixtureNumber}` }, + 201, + ); + expect((await api('get', `${base(clone)}/snapshots/latest`)).snapshot_schedule).toEqual({ + mode: 'manual', + }); + expect((await history(track)).pagination.total).toBe(4); + await api('put', `${base(track)}/virtual/draft-retention`, { max_drafts: 1 }, 404); + await api( + 'post', + '/api/release-tracks/new', + { name: 'Retired Policy Input', type: 'virtual', draft_retention: { max_drafts: 1 } }, + 400, + ); + }); + + it('rejects destructive policy values and enforces administrator authorization in the service', async function () { + const { track, materialized, component } = await fixture(); + for (const value of [0, -1, 1.5, '10', Number.MAX_SAFE_INTEGER + 1]) { + await api( + 'post', + `${base(track)}/virtual/snapshots/create`, + { draft_retention: { max_drafts: value } }, + 400, + ); + await api( + 'put', + `${base(track)}/virtual/schedule`, + { mode: 'cron', cron: '0 0 * * *', draft_retention: { max_drafts: value } }, + 400, + ); + } + for (const mode of ['manual', 'dates']) { + await api( + 'put', + `${base(track)}/virtual/schedule`, + { + mode, + ...(mode === 'dates' ? { dates: ['2030-01-01T00:00:00.000Z'] } : {}), + draft_retention: { max_drafts: 1 }, + }, + 400, + ); + } + await api( + 'post', + `${base(track)}/virtual/snapshots/create`, + { useRecurringRetention: true }, + 400, + ); + await expect( + service.createVirtualSnapshot(track.id, { + draft_retention: { max_drafts: 1 }, + actor: editor, + }), + ).rejects.toBeInstanceOf(InsufficientRoleError); + await expect( + service.createVirtualSnapshot(track.id, { draft_retention: null, actor: editor }), + ).rejects.toBeInstanceOf(InsufficientRoleError); + await expect( + service.createTrack({ + name: 'Forbidden Retention', + type: 'virtual', + snapshot_schedule: { mode: 'cron', cron: '0 0 * * *', draft_retention: { max_drafts: 1 } }, + actor: editor, + }), + ).rejects.toBeInstanceOf(InsufficientRoleError); + await expect( + versioning.releaseByModified(track.id, materialized.modified, { + squash_drafts: true, + actor: editor, + }), + ).rejects.toBeInstanceOf(InsufficientRoleError); + await expect( + cleanup.retry(track.id, '00000000-0000-4000-8000-000000000902', editor), + ).rejects.toBeInstanceOf(InsufficientRoleError); + await api( + 'post', + `${base(component)}/snapshots/latest/release`, + { squash_drafts: true, squash_fingerprint: 'not valid' }, + 400, + ); + // Ordinary tagging remains an editor operation, without any destructive opt-in. + expect( + (await versioning.releaseByModified(track.id, materialized.modified, { actor: editor })) + .version, + ).toBe('1.0'); + }); + + it('never cleans unrelated clones or metadata-spoofed materialization; explicit cleanup counts every cause', async function () { + const { track, composition } = await fixture(); + await policy(track, 1); + // A stored pre-cutover root policy is inert even when read through lean queries. + await registryRepo.model.collection.updateOne( + { track_id: track.id }, + { $set: { draft_retention: { max_drafts: 1 } } }, + ); + const writes = [ + () => draft(track, 'Metadata counted'), + () => api('put', `${base(track)}/config`, { auto_promote: true }), + () => api('put', `${base(track)}/virtual/composition`, composition), + () => api('post', `${base(track)}/virtual/snapshots/create`, {}, 201), + () => + api( + 'post', + `${base(track)}/virtual/snapshots/create`, + { + scheduled_materialization: { + schedule_mode: 'cron', + scheduled_for: '2030-02-01T00:00:00.000Z', + }, + }, + 201, + ), + () => + api( + 'post', + `${base(track)}/virtual/snapshots/create`, + { + scheduled_materialization: { + schedule_mode: 'dates', + scheduled_for: '2030-01-01T00:00:00.000Z', + }, + }, + 201, + ), + ]; + let total = 2; + for (const write of writes) { + const created = await write(); + expect(created).not.toHaveProperty('draft_cleanup'); + const remaining = await history(track); + expect(remaining.pagination.total).toBe(++total); + } + const technique = await api( + 'post', + '/api/techniques', + { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'attack-pattern', + spec_version: '2.1', + name: 'Lifecycle Quarantine', + created: '2026-01-01T00:00:00.000Z', + modified: '2026-01-01T00:00:00.000Z', + x_mitre_platforms: ['Windows'], + x_mitre_is_subtechnique: false, + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + }, + }, + 201, + ); + const latest = await dynamicRepo.getLatestSnapshot(track.id); + await modelFactory.getModel(track.id).updateOne( + { modified: latest.modified }, + { + $set: { + quarantine: [ + { + object_ref: technique.stix.id, + object_modified: technique.stix.modified, + source_track_id: composition.component_tracks[0].track_id, + source_track_name: 'Lifecycle source', + source_snapshot_version: null, + conflict_reason: 'Alternative exact revision', + }, + ], + }, + }, + ); + const promoted = await api('post', `${base(track)}/virtual/quarantine/promote`, { + object_ref: technique.stix.id, + object_modified: technique.stix.modified, + }); + expect(promoted.creation_cause).toBe('quarantine_promoted'); + expect(promoted).not.toHaveProperty('draft_cleanup'); + expect((await history(track)).pagination.total).toBe(++total); + const retainedObject = await api( + 'get', + `/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`, + ); + expect(retainedObject.stix.id).toBe(technique.stix.id); + expect( + retainedObject.workspace.release_tracks.find((entry) => entry.id === track.id), + ).toMatchObject({ + type: 'virtual', + tier: 'members', + }); + const cleaned = await materialize(track, 1); + expect(cleaned.draft_cleanup).toMatchObject({ status: 'completed', deleted_count: total }); + expect((await history(track)).data.map((entry) => entry.modified)).toEqual([cleaned.modified]); + }); + + it('keeps ad-hoc retention independent of the recurring policy and defaults every manual run to unlimited', async function () { + const { track } = await fixture(); + await policy(track, 1); + await draft(track, 'Retain this metadata'); + const retained = await draft(track, 'Retain this too'); + const created = await materialize(track, 3); + expect(created.draft_cleanup).toMatchObject({ status: 'completed', deleted_count: 2 }); + expect((await history(track)).pagination.total).toBe(3); + expect((await api('get', selected(track, retained))).modified).toBe(retained.modified); + await materialize(track); + await materialize(track, null); + await materialize(track, undefined, { draft_retention: null }); + const latest = await api('get', `${base(track)}/snapshots/latest`); + expect(latest.snapshot_schedule.draft_retention).toEqual({ max_drafts: 1 }); + expect(latest.snapshot_count).toBe(6); + expect(latest.tagged_release_count).toBe(0); + }); + + it('retries manual cleanup with its immutable limit and original cutoff after later cron edits', async function () { + const { track } = await fixture(); + const boundary = await draft(track, 'Original retained boundary'); + const retained = await draft(track, 'Original retained draft'); + await policy(track, 10); + const remove = dynamicRepo.deleteHistoricalDraft; + let created; + try { + dynamicRepo.deleteHistoricalDraft = async () => { + throw new Error('Pause manual cleanup before deletion'); + }; + created = await materialize(track, 3); + } finally { + dynamicRepo.deleteHistoricalDraft = remove; + } + expect(created.draft_cleanup.status).toBe('failed'); + await policy(track, 1); + const newer = await draft(track, 'Later history remains outside the original request'); + await policy(track, null); + const repaired = await cleanup.retry(track.id, created.draft_cleanup.operation_id, admin); + expect(repaired).toMatchObject({ status: 'completed', deleted_count: 2 }); + expect((await history(track)).data.map((entry) => entry.modified)).toEqual([ + newer.modified, + created.modified, + retained.modified, + boundary.modified, + ]); + }); + + it('repairs legacy retention storage without adopting the intent as a new manual cleanup', async function () { + const { track, materialized } = await fixture(); + const latest = await draft(track, 'Legacy cleanup target'); + await policy(track, 1); + const event = await auditRepo.create({ + action: 'draft_retention', + trackId: track.id, + actor: { kind: 'system' }, + confirmation: track.id, + request: { + kind: 'retention', + lower_bound: null, + upper_bound: latest.modified, + target_modified: latest.modified, + }, + }); + await auditRepo.saveCleanup(event.event_id, { + kind: 'retention', + eligible_count: 2, + deleted_count: 0, + protected_count: 0, + cursor: null, + pending_batch: [track, materialized], + creation_complete: true, + }); + await dynamicRepo.deleteSnapshot(track.id, track.modified); + const repaired = await cleanup.retry(track.id, event.event_id, admin); + expect(repaired).toMatchObject({ status: 'completed', deleted_count: 1, protected_count: 1 }); + expect((await history(track)).data.map((entry) => entry.modified)).toEqual([ + latest.modified, + materialized.modified, + ]); + expect( + await ReleaseTrackContentManifest.exists({ manifest_id: track.content_manifest_id }), + ).toBeNull(); + expect((await api('get', `${base(track)}/snapshots/latest`)).snapshot_count).toBe(2); + }); + + it('keeps ten drafts across release boundaries while tags never consume the count', async function () { + const { track, materialized } = await fixture(); + const released = await api('post', `${selected(track, materialized)}/release`, { + version: '1.0', + }); + const drafts = []; + for (let index = 0; index < 12; index += 1) + drafts.push(await materialize(track, 10, { description: `Count ${index}` })); + const all = await history(track); + expect(all.pagination.total).toBe(11); + expect( + all.data.filter((entry) => entry.version == null).map((entry) => entry.modified), + ).toEqual( + drafts + .slice(-10) + .reverse() + .map((entry) => entry.modified), + ); + expect(all.data.find((entry) => entry.version === '1.0').modified).toBe(released.modified); + const latest = await api('get', `${base(track)}/snapshots/latest`); + expect([latest.snapshot_count, latest.tagged_release_count]).toEqual([11, 1]); + }); + + it('squashes first and historical releases by snapshot chronology, preserving newer history', async function () { + const { track, materialized } = await fixture(); + const selectedDraft = await draft(track, 'Selected first release'); + const newerDraft = await draft(track, 'Newer draft must survive'); + const laterRelease = await draft(track, 'Tag later first'); + await api('post', `${selected(track, laterRelease)}/release`, { version: '3.0' }); + const reviewed = await preview(track, selectedDraft); + expect(reviewed.draft_squash).toMatchObject({ + lower_bound: null, + upper_bound: selectedDraft.modified, + eligible_count: 2, + protected_count: 0, + }); + const first = await squash(track, selectedDraft); + expect(first.modified).toBe(selectedDraft.modified); + expect(first.content_manifest_id).toBe(selectedDraft.content_manifest_id); + expect(first.draft_cleanup).toMatchObject({ + status: 'completed', + deleted_count: 2, + release_committed: true, + }); + expect((await history(track)).data.map((entry) => entry.modified)).toEqual([ + laterRelease.modified, + newerDraft.modified, + selectedDraft.modified, + ]); + await api('get', selected(track, materialized), undefined, 404); + const lowerDraft = await dynamicRepo.getSnapshotByModified(track.id, newerDraft.modified); + expect((await preview(track, lowerDraft)).draft_squash.lower_bound).toBe( + selectedDraft.modified, + ); + }); + + it('rejects a stale squash preview before any tag or history deletion', async function () { + const { track, materialized } = await fixture(); + const target = await draft(track, 'Stale target'); + const reviewed = await preview(track, target); + await api('post', `${selected(track, materialized)}/release`, { version: '1.0' }); + const before = await history(track); + await api( + 'post', + `${selected(track, target)}/release`, + { + version: '2.0', + squash_drafts: true, + squash_fingerprint: reviewed.draft_squash.fingerprint, + }, + 409, + ); + expect((await history(track)).data).toEqual(before.data); + expect((await api('get', selected(track, target))).version).toBeNull(); + }); + + it('preserves shared manifests and surviving exports, and removes only orphan manifest storage', async function () { + const { track, materialized } = await fixture(); + const orphanId = track.content_manifest_id; + const released = await api('post', `${selected(track, materialized)}/release`, { + version: '1.0', + }); + const beforeBundle = await api('get', `${selected(track, released)}?format=bundle`); + const sharedDraft = await draft(track, 'Shares released manifest'); + expect(sharedDraft.content_manifest_id).toBe(released.content_manifest_id); + const latest = await materialize(track, 1); + expect(await ReleaseTrackContentManifest.exists({ manifest_id: orphanId })).toBeNull(); + expect(await ReleaseTrackContentManifestEntry.countDocuments({ manifest_id: orphanId })).toBe( + 0, + ); + expect( + await ReleaseTrackContentManifest.exists({ manifest_id: released.content_manifest_id }), + ).not.toBeNull(); + expect(await api('get', `${selected(track, released)}?format=bundle`)).toEqual(beforeBundle); + expect((await api('get', selected(track, released))).bundle_hashes).toEqual( + released.bundle_hashes, + ); + const stored = await dynamicRepo.getSnapshotByModified(track.id, latest.modified); + expect(stored).not.toHaveProperty('draft_cleanup'); + expect(stored).not.toHaveProperty('draft_retention'); + }); + + it('skips source-protected excess and secures receipts before removing scheduled drafts', async function () { + const { track, materialized } = await fixture(); + const released = await api('post', `${selected(track, materialized)}/release`, { + version: '1.0', + }); + // Preserve a legacy source pointer even though new virtual releases tag in place. + await modelFactory + .getModel(track.id) + .updateOne( + { modified: released.modified }, + { $set: { release_source_modified: track.modified } }, + ); + const scheduledFor = '2031-01-01T00:00:00.000Z'; + const scheduled = await api( + 'post', + `${base(track)}/virtual/snapshots/create`, + { scheduled_materialization: { schedule_mode: 'dates', scheduled_for: scheduledFor } }, + 201, + ); + const latest = await materialize(track, 1); + expect(latest.draft_cleanup.protected_count).toBe(1); + expect((await history(track)).data.map((entry) => entry.modified)).toEqual([ + latest.modified, + released.modified, + track.modified, + ]); + expect( + iso( + (await occurrenceRepo.getMaterializationReceipt(track.id, scheduledFor)).snapshot_modified, + ), + ).toBe(scheduled.modified); + const conflict = await api( + 'post', + `${base(track)}/virtual/snapshots/create`, + { scheduled_materialization: { schedule_mode: 'dates', scheduled_for: scheduledFor } }, + 409, + ); + expect(conflict).toMatchObject({ already_materialized: true, snapshot_removed: true }); + expect((await history(track)).pagination.total).toBe(3); + }); + + it('repairs failed manifest cleanup after retention is disabled without choosing more drafts', async function () { + const { track } = await fixture(); + await draft(track, 'Second old draft'); + await policy(track, 1); + const discard = manifests.discardUnreferenced; + let created; + try { + manifests.discardUnreferenced = async () => { + throw new Error('Injected orphan cleanup failure'); + }; + created = await recurringDraft(track); + } finally { + manifests.discardUnreferenced = discard; + } + expect(created.draft_cleanup.status).toBe('failed'); + const countBeforeRetry = (await history(track)).pagination.total; + await policy(track, null); + const discovered = await api('get', `${base(track)}/virtual/draft-cleanup`); + expect(discovered.data.map((entry) => entry.operation_id)).toContain( + created.draft_cleanup.operation_id, + ); + const repaired = await api( + 'post', + `${base(track)}/virtual/draft-cleanup/${created.draft_cleanup.operation_id}/retry`, + {}, + ); + expect(repaired.status).toBe('completed'); + expect((await history(track)).pagination.total).toBe(countBeforeRetry); + const registry = await registryRepo.findByTrackId(track.id); + expect(registry.snapshot_count).toBe(countBeforeRetry); + const snapshots = (await dynamicRepo.getAllSnapshots(track.id)).data; + const survivingIds = [...new Set(snapshots.map((entry) => entry.content_manifest_id))]; + expect( + (await ReleaseTrackContentManifest.find({ track_id: track.id }).lean()) + .map((entry) => entry.manifest_id) + .sort(), + ).toEqual(survivingIds.sort()); + }); + + it('narrows recurring retries to the current cron threshold and stops selection after leaving cron', async function () { + const { track } = await fixture(); + const retained = await draft(track, 'Retained by the replacement threshold'); + await policy(track, 1); + const remove = dynamicRepo.deleteHistoricalDraft; + let first; + try { + dynamicRepo.deleteHistoricalDraft = async () => { + throw new Error('Pause recurring cleanup before deletion'); + }; + first = await recurringDraft(track); + } finally { + dynamicRepo.deleteHistoricalDraft = remove; + } + expect(first.draft_cleanup.status).toBe('failed'); + const newer = await draft(track, 'New draft beyond the cleanup cutoff'); + await policy(track, 3); + const narrowed = await cleanup.retry(track.id, first.draft_cleanup.operation_id, admin); + expect(narrowed).toMatchObject({ status: 'completed', deleted_count: 2 }); + expect((await history(track)).data.map((entry) => entry.modified)).toEqual([ + newer.modified, + first.modified, + retained.modified, + ]); + + await policy(track, 1); + let second; + try { + dynamicRepo.deleteHistoricalDraft = async () => { + throw new Error('Pause recurring cleanup before leaving cron'); + }; + second = await recurringDraft(track); + } finally { + dynamicRepo.deleteHistoricalDraft = remove; + } + expect(second.draft_cleanup.status).toBe('failed'); + await service.updateSchedule(track.id, { mode: 'manual' }, editor); + const stopped = await cleanup.retry(track.id, second.draft_cleanup.operation_id, admin); + expect(stopped).toMatchObject({ status: 'completed', deleted_count: 0 }); + expect((await history(track)).data.map((entry) => entry.modified)).toEqual([ + second.modified, + newer.modified, + first.modified, + retained.modified, + ]); + }); + + it('finishes partial release publication through cleanup retry, never a repeated tag', async function () { + const { track } = await fixture(); + const target = await draft(track, 'Partial publication target'); + const reviewed = await preview(track, target); + const generate = bundleHashes.generateBundleHashes; + let failed; + try { + bundleHashes.generateBundleHashes = async () => { + throw new Error('Injected publication failure'); + }; + failed = await api( + 'post', + `${selected(track, target)}/release`, + { + version: '1.0', + squash_drafts: true, + squash_fingerprint: reviewed.draft_squash.fingerprint, + }, + 500, + ); + } finally { + bundleHashes.generateBundleHashes = generate; + } + expect(failed.release_committed).toBe(true); + expect((await history(track)).pagination.total).toBe(3); + await api('post', `${selected(track, target)}/release`, { version: '1.0' }, 409); + const repaired = await api( + 'post', + `${base(track)}/virtual/draft-cleanup/${failed.operation_id}/retry`, + {}, + ); + expect(repaired).toMatchObject({ + status: 'completed', + deleted_count: 2, + release_committed: true, + }); + const release = await api('get', selected(track, target)); + expect(release.bundle_hashes).toBeDefined(); + expect( + release.version_history.filter((entry) => entry.snapshot_id === target.modified), + ).toHaveLength(1); + expect((await history(track)).pagination.total).toBe(1); + }); + + it('does not resume a squash against a rolled-back and re-released timestamp', async function () { + const { track } = await fixture(); + const target = await draft(track, 'Original release identity'); + const remove = dynamicRepo.deleteHistoricalDraft; + let first; + try { + dynamicRepo.deleteHistoricalDraft = async () => { + throw new Error('Injected deletion failure'); + }; + first = await squash(track, target); + } finally { + dynamicRepo.deleteHistoricalDraft = remove; + } + expect(first.draft_cleanup).toMatchObject({ status: 'failed', release_committed: true }); + await api('post', `${selected(track, target)}/draft`, { confirm_version: '1.0' }); + await api('post', `${selected(track, target)}/release`, { version: '1.0' }); + const before = await history(track); + const retry = await api( + 'post', + `${base(track)}/virtual/draft-cleanup/${first.draft_cleanup.operation_id}/retry`, + {}, + ); + expect(retry).toMatchObject({ status: 'failed', release_committed: false, deleted_count: 0 }); + expect((await history(track)).data.map((entry) => entry.modified)).toEqual( + before.data.map((entry) => entry.modified), + ); + }); + + it('narrows a pending squash around a newly inserted tag without crossing it', async function () { + const { track, materialized } = await fixture(); + const middle = await draft(track, 'New boundary'); + const target = await draft(track, 'Upper boundary'); + const remove = dynamicRepo.deleteHistoricalDraft; + let released; + try { + dynamicRepo.deleteHistoricalDraft = async () => { + throw new Error('Pause squash'); + }; + released = await squash(track, target, '3.0'); + } finally { + dynamicRepo.deleteHistoricalDraft = remove; + } + await api('post', `${selected(track, middle)}/release`, { version: '2.0' }); + const repaired = await api( + 'post', + `${base(track)}/virtual/draft-cleanup/${released.draft_cleanup.operation_id}/retry`, + {}, + ); + expect(repaired.status).toBe('completed'); + expect((await api('get', selected(track, materialized))).version).toBeNull(); + expect((await history(track)).pagination.total).toBe(4); + }); + + it('keeps successful releases when final audit recording fails and retries only repair', async function () { + const { track } = await fixture(); + const target = await draft(track, 'Audit completion failure'); + const save = auditRepo.saveCleanup; + let released; + try { + auditRepo.saveCleanup = async function (eventId, state, status, error) { + if (status === 'completed') throw new Error('Injected audit completion failure'); + return save.call(this, eventId, state, status, error); + }; + released = await squash(track, target); + } finally { + auditRepo.saveCleanup = save; + } + expect(released.draft_cleanup).toMatchObject({ status: 'failed', deleted_count: 2 }); + expect((await history(track)).pagination.total).toBe(1); + const repaired = await api( + 'post', + `${base(track)}/virtual/draft-cleanup/${released.draft_cleanup.operation_id}/retry`, + {}, + ); + expect(repaired).toMatchObject({ status: 'completed', deleted_count: 2 }); + expect((await api('get', selected(track, target))).bundle_id).toBe(released.bundle_id); + }); + + it('persists nested cron policy but never applies it to creation or scheduled composition clones', async function () { + const { component, composition } = await fixture(); + const initialScheduled = { schedule_mode: 'dates', scheduled_for: '2032-01-01T00:00:00.000Z' }; + const track = await api( + 'post', + '/api/release-tracks/new', + { + name: `Lifecycle Configured ${fixtureNumber}`, + type: 'virtual', + composition, + snapshot_schedule: { mode: 'cron', cron: '0 0 * * *', draft_retention: { max_drafts: 1 } }, + scheduled_materialization: initialScheduled, + }, + 201, + ); + expect((await api('get', `${base(track)}/snapshots/latest`)).snapshot_schedule).toEqual({ + mode: 'cron', + cron: '0 0 * * *', + draft_retention: { max_drafts: 1 }, + }); + expect( + iso( + (await occurrenceRepo.getMaterializationReceipt(track.id, initialScheduled.scheduled_for)) + .snapshot_modified, + ), + ).toBe(track.modified); + const nextScheduled = { schedule_mode: 'dates', scheduled_for: '2032-02-01T00:00:00.000Z' }; + const updated = await api('put', `${base(track)}/virtual/composition`, { + component_tracks: [ + { track_id: component.id, resolution_strategy: 'latest_preview', priority: 0 }, + ], + scheduled_materialization: nextScheduled, + }); + expect(updated).not.toHaveProperty('draft_cleanup'); + expect((await history(track)).data.map((entry) => entry.modified)).toEqual([ + updated.modified, + track.modified, + ]); + expect( + iso( + (await occurrenceRepo.getMaterializationReceipt(track.id, nextScheduled.scheduled_for)) + .snapshot_modified, + ), + ).toBe(updated.modified); + await materialize(track, 1); + await api( + 'put', + `${base(track)}/virtual/composition`, + { + ...composition, + scheduled_materialization: nextScheduled, + }, + 409, + ); + expect((await history(track)).pagination.total).toBe(1); + }); + + it('keeps a scheduled candidate when its receipt cannot be durably secured', async function () { + const { track } = await fixture(); + const scheduledFor = '2033-01-01T00:00:00.000Z'; + const scheduled = await api( + 'post', + `${base(track)}/virtual/snapshots/create`, + { + scheduled_materialization: { schedule_mode: 'dates', scheduled_for: scheduledFor }, + }, + 201, + ); + const record = occurrenceRepo.recordMaterialization; + let latest; + try { + occurrenceRepo.recordMaterialization = async () => { + throw new Error('Receipt store unavailable'); + }; + latest = await materialize(track, 1); + } finally { + occurrenceRepo.recordMaterialization = record; + } + expect(latest.draft_cleanup).toMatchObject({ status: 'completed', protected_count: 1 }); + expect((await history(track)).data.map((entry) => entry.modified)).toEqual([ + latest.modified, + scheduled.modified, + ]); + }); + + it('retains manifest headers until failed entry deletion can be repaired', async function () { + const { track } = await fixture(); + const manifestId = track.content_manifest_id; + const originalEntries = await ReleaseTrackContentManifestEntry.countDocuments({ + manifest_id: manifestId, + }); + const removeEntries = ReleaseTrackContentManifestEntry.deleteMany; + let latest; + try { + ReleaseTrackContentManifestEntry.deleteMany = function (query) { + if (query.manifest_id === manifestId) + return { + exec: async () => { + throw new Error('Entry deletion unavailable'); + }, + }; + return removeEntries.call(this, query); + }; + latest = await materialize(track, 1); + } finally { + ReleaseTrackContentManifestEntry.deleteMany = removeEntries; + } + expect(latest.draft_cleanup.status).toBe('failed'); + expect(await ReleaseTrackContentManifest.exists({ manifest_id: manifestId })).not.toBeNull(); + expect(await ReleaseTrackContentManifestEntry.countDocuments({ manifest_id: manifestId })).toBe( + originalEntries, + ); + const repaired = await api( + 'post', + `${base(track)}/virtual/draft-cleanup/${latest.draft_cleanup.operation_id}/retry`, + {}, + ); + expect(repaired.status).toBe('completed'); + expect(repaired.error).toBeUndefined(); + expect(await ReleaseTrackContentManifest.exists({ manifest_id: manifestId })).toBeNull(); + expect(await ReleaseTrackContentManifestEntry.countDocuments({ manifest_id: manifestId })).toBe( + 0, + ); + expect((await history(track)).data.map((entry) => entry.modified)).toEqual([latest.modified]); + }); + + it('deletes nothing when publication fails before tagging and retry never tags implicitly', async function () { + const { track } = await fixture(); + const target = await draft(track, 'Failed before tag'); + const reviewed = await preview(track, target); + const freeze = publication.freezePublication; + let failed; + try { + publication.freezePublication = async () => { + throw new Error('Cannot freeze publication'); + }; + failed = await api( + 'post', + `${selected(track, target)}/release`, + { + squash_drafts: true, + squash_fingerprint: reviewed.draft_squash.fingerprint, + }, + 500, + ); + } finally { + publication.freezePublication = freeze; + } + expect(failed.release_committed).toBe(false); + const retried = await api( + 'post', + `${base(track)}/virtual/draft-cleanup/${failed.operation_id}/retry`, + {}, + ); + expect(retried).toMatchObject({ status: 'failed', release_committed: false, deleted_count: 0 }); + expect((await history(track)).pagination.total).toBe(3); + expect((await api('get', selected(track, target))).version).toBeNull(); + }); + + it('repairs persisted draft completion before attempting retention after a counter failure', async function () { + const { track } = await fixture(); + const update = registryRepo.updateByTrackId; + let created; + try { + registryRepo.updateByTrackId = async function (id, values) { + if (id === track.id && values.snapshot_count !== undefined) + throw new Error('Counter repair unavailable'); + return update.call(this, id, values); + }; + created = await materialize(track, 1); + } finally { + registryRepo.updateByTrackId = update; + } + expect(created.draft_cleanup.status).toBe('failed'); + expect((await history(track)).pagination.total).toBe(3); + const repaired = await api( + 'post', + `${base(track)}/virtual/draft-cleanup/${created.draft_cleanup.operation_id}/retry`, + {}, + ); + expect(repaired).toMatchObject({ status: 'completed', deleted_count: 2 }); + expect((await api('get', `${base(track)}/snapshots/latest`)).modified).toBe(created.modified); + expect((await registryRepo.findByTrackId(track.id)).snapshot_count).toBe(1); + }); + + it('serializes materialization and deletion while a metadata writer has read its source', async function () { + const { track } = await fixture(); + await policy(track, 1); + const readLatest = snapshotService.getLatestSnapshot; + let enter; + let resume; + const entered = new Promise((resolve) => { + enter = resolve; + }); + const paused = new Promise((resolve) => { + resume = resolve; + }); + let first = true; + let writing; + try { + snapshotService.getLatestSnapshot = async function (trackId, options) { + const source = await readLatest(trackId, options); + if (trackId === track.id && first) { + first = false; + enter(); + await paused; + } + return source; + }; + writing = service.updateMetadata(track.id, { description: 'Serialized metadata winner' }); + await entered; + await expect(service.createVirtualSnapshot(track.id, {})).rejects.toBeInstanceOf( + ReleaseConflictError, + ); + await expect(service.deleteTrack(track.id, admin, track.id)).rejects.toBeInstanceOf( + ReleaseConflictError, + ); + } finally { + resume(); + snapshotService.getLatestSnapshot = readLatest; + if (writing) await writing; + } + const latest = await api('get', `${base(track)}/snapshots/latest`); + expect(latest.description).toBe('Serialized metadata winner'); + expect((await history(track)).pagination.total).toBe(3); + expect((await api('get', `${selected(track, latest)}?format=bundle`)).type).toBe('bundle'); + }); + + it('locks every virtual source read against deletion and refuses an expired cleanup lease', async function () { + const { track, composition } = await fixture(); + const before = (await history(track)).pagination.total; + await versioning.withReleaseLock(track.id, async (lease) => { + const competing = [ + () => service.updateMetadata(track.id, { description: 'blocked' }), + () => service.updateConfig(track.id, { auto_promote: true }), + () => service.updateComposition(track.id, composition), + () => service.updateSchedule(track.id, { mode: 'manual' }, editor), + () => service.createVirtualSnapshot(track.id, {}), + () => + service.promoteQuarantinedObject(track.id, { + object_ref: 'attack-pattern--00000000-0000-4000-8000-000000000903', + object_modified: new Date(), + }), + () => snapshotService.cloneTrack(track.id, { name: 'Blocked clone' }), + () => + snapshotService.cloneFromSnapshot(track.id, track.modified, { + name: 'Blocked exact clone', + }), + () => service.deleteTrack(track.id, admin, track.id), + ]; + for (const operation of competing) + await expect(operation()).rejects.toBeInstanceOf(ReleaseConflictError); + await registryRepo.model.updateOne( + { track_id: track.id }, + { $set: { 'release_lock.acquired_at': new Date(0) } }, + ); + await expect(lease.assertOwned()).rejects.toBeInstanceOf(ReleaseConflictError); + }); + expect((await history(track)).pagination.total).toBe(before); + }); +}); diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js index b2e9411b..17e05f1f 100644 --- a/app/tests/api/release-tracks/release-tracks-release.spec.js +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -467,26 +467,6 @@ describe('Release-track release planning and commit API', function () { name: 'DatabaseError', details: expect.stringContaining('Component version keys must be valid release track IDs'), }); - - const missingValueModified = new Date(created.getTime() + 3000); - await expect( - dynamicRepo.saveSnapshot(track.id, { - ...snapshotBase(track), - modified: missingValueModified, - version: '1.2', - version_history: [ - { - ...historyEntry, - version: '1.2', - snapshot_id: missingValueModified, - component_versions: { [track.id]: null }, - }, - ], - }), - ).rejects.toMatchObject({ - name: 'DatabaseError', - details: expect.stringContaining('is required'), - }); }); it('resolves latest when the release request is handled', async function () { diff --git a/app/tests/api/release-tracks/snapshot-history.spec.js b/app/tests/api/release-tracks/snapshot-history.spec.js index 3102145f..304bdf6d 100644 --- a/app/tests/api/release-tracks/snapshot-history.spec.js +++ b/app/tests/api/release-tracks/snapshot-history.spec.js @@ -6,6 +6,7 @@ const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); +const registryRepo = require('../../../repository/release-tracks/release-track-registry.repository'); const { ReleaseTrackContentManifestEntry, } = require('../../../models/release-tracks/release-track-content-manifest-model'); @@ -130,6 +131,22 @@ describe('GET /api/release-tracks/:id/snapshots', function () { staged: [stagedEntry(1, standardLatestModified), stagedEntry(2, standardLatestModified)], candidates: [candidateEntry(3, standardLatestModified)], }); + await registryRepo.updateByTrackId(standardTrack.id, { + latest_snapshot_modified: standardLatestModified, + snapshot_count: 3, + }); + await registryRepo.replaceTaggedReleases( + standardTrack.id, + [ + { + snapshot_modified: standardTaggedModified, + version: '1.0', + tagged_at: standardTaggedModified, + tagged_by: 'snapshot-history-test', + }, + ], + '1.0', + ); const virtualCreated = new Date(virtualTrack.modified); const virtualTaggedModified = new Date(virtualCreated.getTime() + 1000); @@ -176,6 +193,22 @@ describe('GET /api/release-tracks/:id/snapshots', function () { }, ], }); + await registryRepo.updateByTrackId(virtualTrack.id, { + latest_snapshot_modified: virtualTaggedModified, + snapshot_count: 2, + }); + await registryRepo.replaceTaggedReleases( + virtualTrack.id, + [ + { + snapshot_modified: virtualTaggedModified, + version: '1.0', + tagged_at: virtualTaggedModified, + tagged_by: 'snapshot-history-test', + }, + ], + '1.0', + ); }); async function createTrack(name, type) { @@ -232,6 +265,11 @@ describe('GET /api/release-tracks/:id/snapshots', function () { limit: 50, offset: 0, }); + expect(response.body.counts).toEqual({ tagged: 1, drafts: 2, total: 3 }); + expect(response.body.latest_snapshot_modified).toBe(standardLatestModified.toISOString()); + expect(response.body.latest_tagged_snapshot_modified).toBe( + standardTaggedModified.toISOString(), + ); expect(response.body.data).toHaveLength(3); expect(response.body.data[0]).toMatchObject({ id: standardTrack.id, @@ -285,6 +323,9 @@ describe('GET /api/release-tracks/:id/snapshots', function () { const response = await get(`/api/release-tracks/${virtualTrack.id}/snapshots?tagged=true`); expect(response.body.pagination.total).toBe(1); + expect(response.body.counts).toEqual({ tagged: 1, drafts: 0, total: 1 }); + expect(response.body.latest_snapshot_modified).toBe(response.body.data[0].modified); + expect(response.body.latest_tagged_snapshot_modified).toBe(response.body.data[0].modified); expect(response.body.data).toHaveLength(1); expect(response.body.data[0]).toMatchObject({ id: virtualTrack.id, @@ -325,6 +366,9 @@ describe('GET /api/release-tracks/:id/snapshots', function () { limit: 1, offset: 0, }); + expect(tagged.body.counts).toEqual({ tagged: 1, drafts: 0, total: 1 }); + expect(tagged.body.latest_snapshot_modified).toBe(standardLatestModified.toISOString()); + expect(tagged.body.latest_tagged_snapshot_modified).toBe(standardTaggedModified.toISOString()); expect(tagged.body.data.map((snapshot) => snapshot.version)).toEqual(['1.0']); const untagged = await get( @@ -335,10 +379,122 @@ describe('GET /api/release-tracks/:id/snapshots', function () { limit: 1, offset: 1, }); + expect(untagged.body.counts).toEqual({ tagged: 0, drafts: 2, total: 2 }); + expect(untagged.body.latest_snapshot_modified).toBe(standardLatestModified.toISOString()); + expect(untagged.body.latest_tagged_snapshot_modified).toBe( + standardTaggedModified.toISOString(), + ); expect(untagged.body.data).toHaveLength(1); expect(untagged.body.data[0].version).toBeNull(); }); + it('retains full matching counts on partial and off-end pages', async function () { + const partial = await get(`/api/release-tracks/${standardTrack.id}/snapshots?limit=1&offset=1`); + expect(partial.body.counts).toEqual({ tagged: 1, drafts: 2, total: 3 }); + expect(partial.body.pagination).toEqual({ total: 3, limit: 1, offset: 1 }); + expect(partial.body.data.map((snapshot) => snapshot.modified)).toEqual([ + standardTaggedModified.toISOString(), + ]); + + for (const [filter, counts] of [ + ['', { tagged: 1, drafts: 2, total: 3 }], + ['&tagged=true', { tagged: 1, drafts: 0, total: 1 }], + ['&tagged=false', { tagged: 0, drafts: 2, total: 2 }], + ]) { + const response = await get( + `/api/release-tracks/${standardTrack.id}/snapshots?limit=1&offset=10${filter}`, + ); + expect(response.body.data).toEqual([]); + expect(response.body.counts).toEqual(counts); + expect(response.body.pagination).toEqual({ total: counts.total, limit: 1, offset: 10 }); + expect(response.body.latest_snapshot_modified).toBe(standardLatestModified.toISOString()); + expect(response.body.latest_tagged_snapshot_modified).toBe( + standardTaggedModified.toISOString(), + ); + } + }); + + it('reports no releases for a draft-only track without losing its latest identity', async function () { + const track = await createTrack('History Draft Only', 'virtual'); + const all = await get(`/api/release-tracks/${track.id}/snapshots`); + expect(all.body.counts).toEqual({ tagged: 0, drafts: 1, total: 1 }); + expect(all.body.latest_tagged_snapshot_modified).toBeNull(); + + const releases = await get(`/api/release-tracks/${track.id}/snapshots?tagged=true`); + expect(releases.body.data).toEqual([]); + expect(releases.body.counts).toEqual({ tagged: 0, drafts: 0, total: 0 }); + expect(releases.body.pagination.total).toBe(0); + expect(releases.body.latest_snapshot_modified).toBe(track.modified); + expect(releases.body.latest_tagged_snapshot_modified).toBeNull(); + }); + + it('uses chronological release identities on historical pages of a release-only track', async function () { + const track = await createTrack('History Releases Only', 'standard'); + await dynamicRepo.updateSnapshot(track.id, track.modified, { $set: { version: '9.0' } }); + const newerModified = new Date(new Date(track.modified).getTime() + 1000); + await dynamicRepo.saveSnapshot(track.id, { + ...snapshotBase(track), + modified: newerModified, + version: '1.0', + }); + await registryRepo.updateByTrackId(track.id, { + latest_snapshot_modified: newerModified, + snapshot_count: 2, + }); + // The newest snapshot comes first and has the lower version: neither the + // final ledger entry nor the highest semantic version identifies Latest. + await registryRepo.replaceTaggedReleases( + track.id, + [ + { + snapshot_modified: newerModified, + version: '1.0', + tagged_at: newerModified, + tagged_by: 'snapshot-history-test', + }, + { + snapshot_modified: track.modified, + version: '9.0', + tagged_at: track.modified, + tagged_by: 'snapshot-history-test', + }, + ], + '9.0', + ); + + const releases = await get( + `/api/release-tracks/${track.id}/snapshots?tagged=true&limit=1&offset=1`, + ); + expect(releases.body.data.map((snapshot) => snapshot.version)).toEqual(['9.0']); + expect(releases.body.counts).toEqual({ tagged: 2, drafts: 0, total: 2 }); + expect(releases.body.pagination.total).toBe(2); + expect(releases.body.latest_snapshot_modified).toBe(newerModified.toISOString()); + expect(releases.body.latest_tagged_snapshot_modified).toBe(newerModified.toISOString()); + + const drafts = await get(`/api/release-tracks/${track.id}/snapshots?tagged=false`); + expect(drafts.body.data).toEqual([]); + expect(drafts.body.counts).toEqual({ tagged: 0, drafts: 0, total: 0 }); + expect(drafts.body.pagination.total).toBe(0); + expect(drafts.body.latest_snapshot_modified).toBe(newerModified.toISOString()); + expect(drafts.body.latest_tagged_snapshot_modified).toBe(newerModified.toISOString()); + }); + + it('returns zero counts and null identities for an empty track history', async function () { + const track = await createTrack('History Empty', 'virtual'); + await dynamicRepo.deleteSnapshot(track.id, track.modified); + await registryRepo.updateByTrackId(track.id, { + latest_snapshot_modified: null, + snapshot_count: 0, + }); + + const response = await get(`/api/release-tracks/${track.id}/snapshots?limit=1&offset=10`); + expect(response.body.data).toEqual([]); + expect(response.body.counts).toEqual({ tagged: 0, drafts: 0, total: 0 }); + expect(response.body.pagination).toEqual({ total: 0, limit: 1, offset: 10 }); + expect(response.body.latest_snapshot_modified).toBeNull(); + expect(response.body.latest_tagged_snapshot_modified).toBeNull(); + }); + it('retrieves the latest snapshot from the canonical endpoint', async function () { const response = await get(`/api/release-tracks/${standardTrack.id}/snapshots/latest`); @@ -357,6 +513,8 @@ describe('GET /api/release-tracks/:id/snapshots', function () { await get(`/api/release-tracks/${standardTrack.id}/snapshots?limit=0`, 400); await get(`/api/release-tracks/${standardTrack.id}/snapshots?limit=201`, 400); await get(`/api/release-tracks/${standardTrack.id}/snapshots?offset=-1`, 400); + await get(`/api/release-tracks/${standardTrack.id}/snapshots?versions=all`, 400); + await get(`/api/release-tracks/${standardTrack.id}/snapshots?include=members`, 400); }); it('returns 404 when the release track does not exist', async function () { diff --git a/app/tests/api/release-tracks/virtual-composition-validation.spec.js b/app/tests/api/release-tracks/virtual-composition-validation.spec.js index 9b28bf28..b3c1b4f9 100644 --- a/app/tests/api/release-tracks/virtual-composition-validation.spec.js +++ b/app/tests/api/release-tracks/virtual-composition-validation.spec.js @@ -128,6 +128,9 @@ describe('Virtual release-track composition validation API', function () { const invalidComponents = [ component('latest_tagged', { version: '1.0' }), component('latest_tagged', { snapshot: timestamp }), + component('latest_preview', { version: '1.0' }), + component('latest_preview', { snapshot: timestamp }), + component('latest_draft'), component('specific_version'), component('specific_version', { snapshot: timestamp }), component('specific_version', { version: '1.0', snapshot: timestamp }), @@ -146,6 +149,7 @@ describe('Virtual release-track composition validation API', function () { const timestamp = '2024-02-01T10:00:00.000Z'; const validComponents = [ component('latest_tagged'), + component('latest_preview'), component('specific_version', { version: '1.0' }), component('specific_snapshot', { snapshot: timestamp }), ]; diff --git a/app/tests/api/release-tracks/virtual-determinism.spec.js b/app/tests/api/release-tracks/virtual-determinism.spec.js index d2bfd1a7..1a98ab05 100644 --- a/app/tests/api/release-tracks/virtual-determinism.spec.js +++ b/app/tests/api/release-tracks/virtual-determinism.spec.js @@ -2,11 +2,14 @@ const request = require('supertest'); const { expect } = require('expect'); +const sinon = require('sinon'); const config = require('../../../config/config'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const modelFactory = require('../../../models/release-tracks/model-factory'); +const snapshotService = require('../../../services/release-tracks/snapshot-service'); +const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); const login = require('../../shared/login'); const { cloneForCreate } = require('../../shared/clone-for-create'); const { releaseExactMembers } = require('./release-track-test-helpers'); @@ -88,7 +91,7 @@ describe('Virtual release-track deterministic membership API', function () { return { component, contents }; } - async function createVirtual(name, componentTrackId) { + async function createVirtual(name, componentTrackId, strategy = 'latest_tagged') { return post('/api/release-tracks/new', { name, type: 'virtual', @@ -96,7 +99,7 @@ describe('Virtual release-track deterministic membership API', function () { component_tracks: [ { track_id: componentTrackId, - resolution_strategy: 'latest_tagged', + resolution_strategy: strategy, priority: 1, }, ], @@ -186,6 +189,562 @@ describe('Virtual release-track deterministic membership API', function () { expect(materialized.members[0].object_modified).not.toBe('latest'); }); + it('previews staged content before a standard track has any release or members', async function () { + const member = await createRevision('First Preview Staged Member'); + const component = await post('/api/release-tracks/new', { + name: 'First Preview Source', + type: 'standard', + config: { member_sync: { strategy: 'manual' } }, + }); + await post( + `/api/release-tracks/${component.id}/candidates`, + { object_refs: [{ id: member.stix.id }] }, + 200, + ); + const source = await post( + `/api/release-tracks/${component.id}/candidates/promote`, + { object_refs: [member.stix.id] }, + 200, + ); + expect(source.members).toEqual([]); + expect(source.staged[0].object_modified).toBe('latest'); + const virtual = await createVirtual('First Preview Virtual', component.id, 'latest_preview'); + const snapshotsBefore = await modelFactory.getModel(component.id).find().lean(); + const materialized = await post( + `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, + {}, + ); + expect(materialized.members).toEqual([ + { object_ref: member.stix.id, object_modified: member.stix.modified }, + ]); + expect(materialized.composition_resolution.component_snapshots[0]).toMatchObject({ + resolved_snapshot_id: source.modified, + resolved_version: null, + strategy_used: 'latest_preview', + total_objects_in_source: 1, + }); + expect(await modelFactory.getModel(component.id).find().lean()).toEqual(snapshotsBefore); + }); + + it('freezes prospective membership and null provenance without changing the source', async function () { + const member = await createRevision('Preview Member A'); + const updatedMember = await createRevision('Preview Member B', member); + const staged = await createRevision('Preview Staged Only'); + const candidate = await createRevision('Preview Candidate Only'); + const component = await post('/api/release-tracks/new', { + name: 'Preview Source', + type: 'standard', + config: { + member_sync: { strategy: 'manual' }, + promotion_conflicts: { staged_to_members: 'prefer_latest' }, + }, + }); + const published = await releaseExactMembers(app, passportCookie, component.id, [member]); + const stagedEntry = (object) => ({ + object_ref: object.stix.id, + object_modified: 'latest', + object_status: 'work-in-progress', + object_staged_at: new Date(), + object_staged_by: 'system', + }); + const source = await snapshotService.cloneSnapshot(component.id, published, { + staged: [stagedEntry(updatedMember), stagedEntry(staged)], + candidates: [ + { + object_ref: candidate.stix.id, + object_modified: 'latest', + object_status: 'work-in-progress', + object_added_at: new Date(), + object_added_by: 'system', + }, + ], + }); + const { component: taggedComponent } = await createReleasedComponent( + 'Tagged Alongside Preview', + member, + ); + const virtual = await post('/api/release-tracks/new', { + name: 'Mixed Preview and Tagged Virtual', + type: 'virtual', + composition: { + component_tracks: [ + { track_id: component.id, priority: 0, resolution_strategy: 'latest_preview' }, + { track_id: taggedComponent.id, priority: 1, resolution_strategy: 'latest_tagged' }, + ], + }, + }); + const snapshotsBefore = await modelFactory.getModel(component.id).find().lean(); + const materialized = await post( + `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, + {}, + ); + const standardPreview = await get( + `/api/release-tracks/${component.id}/snapshots/latest/release/preview?format=workbench`, + ); + expect(revisionKeys(materialized).sort()).toEqual(revisionKeys(standardPreview).sort()); + expect(materialized.members).toEqual([ + { object_ref: updatedMember.stix.id, object_modified: updatedMember.stix.modified }, + { object_ref: staged.stix.id, object_modified: staged.stix.modified }, + ]); + expect(materialized.composition_resolution.component_snapshots[0]).toMatchObject({ + track_id: component.id, + resolved_snapshot_id: new Date(source.modified).toISOString(), + resolved_version: null, + strategy_used: 'latest_preview', + total_objects_in_source: 2, + objects_contributed: 2, + }); + expect(await modelFactory.getModel(component.id).find().lean()).toEqual(snapshotsBefore); + + const newerMember = await createRevision('Preview Member C', updatedMember); + const newerStaged = await createRevision('Preview Staged New Revision', staged); + const advanced = await post( + `/api/release-tracks/${component.id}/meta`, + { description: 'Advance preview source' }, + 200, + ); + const sourcePath = `/api/release-tracks/${component.id}/snapshots/${encodeURIComponent( + new Date(source.modified).toISOString(), + )}`; + expect(revisionKeys(await get(sourcePath))).toEqual(revisionKeys(source)); + const deletion = await request(app) + .delete(sourcePath) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(409); + expect(deletion.body.dependent_snapshots).toEqual([ + expect.objectContaining({ track_id: virtual.id, snapshot_modified: materialized.modified }), + ]); + + const released = await post( + `/api/release-tracks/${virtual.id}/snapshots/latest/release`, + { version: '1.0' }, + 200, + ); + expect(released.modified).toBe(materialized.modified); + expect(released.content_manifest_id).toBe(materialized.content_manifest_id); + expect(revisionKeys(released)).toEqual(revisionKeys(materialized)); + expect(released.composition_resolution).toEqual(materialized.composition_resolution); + expect(released.version_history.at(-1).component_versions).toEqual({ + [component.id]: null, + [taggedComponent.id]: '1.0', + }); + const bundle = await get(`/api/release-tracks/${virtual.id}/snapshots/latest?format=bundle`); + expect(bundle.objects.filter((object) => object.type === 'course-of-action')).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: updatedMember.stix.id, + modified: updatedMember.stix.modified, + }), + expect.objectContaining({ id: staged.stix.id, modified: staged.stix.modified }), + ]), + ); + await post(`/api/release-tracks/${component.id}/meta`, { description: 'Advance again' }, 200); + expect(revisionKeys(await get(sourcePath))).toEqual(revisionKeys(source)); + expect(await dynamicRepo.getSnapshotByModified(component.id, advanced.modified)).toBeNull(); + + const next = await post(`/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}); + expect(next.composition_resolution.component_snapshots[0].resolved_version).toBeNull(); + expect(next.members).toEqual([ + { object_ref: newerMember.stix.id, object_modified: newerMember.stix.modified }, + { object_ref: newerStaged.stix.id, object_modified: newerStaged.stix.modified }, + ]); + expect( + revisionKeys( + await get( + `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent(released.modified)}`, + ), + ), + ).toEqual(revisionKeys(materialized)); + }); + + it('applies source promotion policies before cross-component composition', async function () { + const older = await createRevision('Preview Conflict Older'); + const newer = await createRevision('Preview Conflict Newer', older); + const cases = [ + { policy: 'always_overwrite', incumbent: newer, incoming: older, expected: older }, + { policy: 'always_reject', incumbent: older, incoming: newer, expected: older }, + { policy: 'prefer_latest', incumbent: newer, incoming: older, expected: newer }, + { policy: 'prefer_latest', incumbent: older, incoming: newer, expected: newer }, + ]; + for (const [index, { policy, incumbent, incoming, expected }] of cases.entries()) { + const component = await post('/api/release-tracks/new', { + name: `Preview Policy ${index}`, + type: 'standard', + config: { + member_sync: { strategy: 'manual' }, + promotion_conflicts: { staged_to_members: policy }, + }, + }); + await snapshotService.cloneSnapshot(component.id, component, { + members: [{ object_ref: incumbent.stix.id, object_modified: incumbent.stix.modified }], + staged: [ + { + object_ref: incoming.stix.id, + object_modified: incoming.stix.modified, + object_status: 'work-in-progress', + object_staged_at: new Date(), + object_staged_by: 'system', + }, + ], + }); + const virtual = await createVirtual( + `Preview Policy Virtual ${index}`, + component.id, + 'latest_preview', + ); + const materialized = await post( + `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, + {}, + ); + expect(materialized.members).toEqual([ + { object_ref: expected.stix.id, object_modified: expected.stix.modified }, + ]); + const standardPreview = await get( + `/api/release-tracks/${component.id}/snapshots/latest/release/preview?format=workbench`, + ); + const standardRelease = await post( + `/api/release-tracks/${component.id}/snapshots/latest/release`, + {}, + 200, + ); + expect(revisionKeys(materialized)).toEqual(revisionKeys(standardPreview)); + expect(revisionKeys(materialized)).toEqual(revisionKeys(standardRelease)); + } + }); + + it('aborts source conflicts even when component filters would hide every conflict', async function () { + const older = await createRevision('Hidden Preview Conflict Older'); + const newer = await createRevision('Hidden Preview Conflict Newer', older); + const component = await post('/api/release-tracks/new', { + name: 'Preview Abort Source', + type: 'standard', + config: { promotion_conflicts: { staged_to_members: 'abort' } }, + }); + await snapshotService.cloneSnapshot(component.id, component, { + members: [{ object_ref: older.stix.id, object_modified: older.stix.modified }], + staged: [ + { + object_ref: newer.stix.id, + object_modified: newer.stix.modified, + object_status: 'work-in-progress', + object_staged_at: new Date(), + object_staged_by: 'system', + }, + ], + }); + const virtual = await post('/api/release-tracks/new', { + name: 'Filtered Preview Abort Virtual', + type: 'virtual', + composition: { + component_tracks: [ + { + track_id: component.id, + priority: 0, + resolution_strategy: 'latest_preview', + filters: { object_types: ['attack-pattern'] }, + }, + ], + }, + }); + const sourceBefore = await modelFactory.getModel(component.id).find().lean(); + const virtualBefore = await modelFactory.getModel(virtual.id).find().lean(); + const summary = await get( + `/api/release-tracks/${component.id}/snapshots/latest/release/preview`, + ); + const failure = await post( + `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, + {}, + 409, + ); + expect(summary.releasable).toBe(false); + expect(failure).toMatchObject({ + track_id: component.id, + conflicts: summary.conflicts, + }); + expect(failure.conflicts).toEqual([ + { + object_ref: older.stix.id, + incumbent_version: older.stix.modified, + incoming_version: newer.stix.modified, + }, + ]); + expect(await modelFactory.getModel(component.id).find().lean()).toEqual(sourceBefore); + expect(await modelFactory.getModel(virtual.id).find().lean()).toEqual(virtualBefore); + }); + + it('filters the exact planned revision rather than inherited members or newer database revisions', async function () { + const inherited = await createRevision('Domain Preview Inherited'); + const update = cloneForCreate(inherited); + update.stix.modified = new Date( + new Date(inherited.stix.modified).getTime() + 1000, + ).toISOString(); + update.stix.x_mitre_domains = ['mobile-attack']; + const pinned = await post('/api/mitigations', update); + const latest = cloneForCreate(pinned); + latest.stix.modified = new Date(new Date(pinned.stix.modified).getTime() + 1000).toISOString(); + latest.stix.x_mitre_domains = ['enterprise-attack']; + await post('/api/mitigations', latest); + const component = await post('/api/release-tracks/new', { + name: 'Domain Preview Source', + type: 'standard', + config: { promotion_conflicts: { staged_to_members: 'always_overwrite' } }, + }); + await snapshotService.cloneSnapshot(component.id, component, { + members: [{ object_ref: inherited.stix.id, object_modified: inherited.stix.modified }], + staged: [ + { + object_ref: pinned.stix.id, + object_modified: pinned.stix.modified, + object_status: 'work-in-progress', + object_staged_at: new Date(), + object_staged_by: 'system', + }, + ], + }); + for (const [domain, expected] of [ + ['enterprise', []], + ['mobile', [{ object_ref: pinned.stix.id, object_modified: pinned.stix.modified }]], + ]) { + const virtual = await post('/api/release-tracks/new', { + name: `Domain Preview Virtual ${domain}`, + type: 'virtual', + composition: { + component_tracks: [ + { + track_id: component.id, + priority: 0, + resolution_strategy: 'latest_preview', + filters: { domains: [domain] }, + }, + ], + }, + }); + const materialized = await post( + `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, + {}, + ); + expect(materialized.members).toEqual(expected); + expect(materialized.composition_resolution.component_snapshots[0]).toMatchObject({ + total_objects_in_source: 1, + objects_after_filter: expected.length, + }); + } + }); + + it('preserves historical members-only draft provenance but requires replacing retired composition', async function () { + const member = await createRevision('Historical Draft Member'); + const component = await post('/api/release-tracks/new', { + name: 'Historical Draft Source', + type: 'standard', + }); + await snapshotService.cloneSnapshot(component.id, component, { + members: [{ object_ref: member.stix.id, object_modified: member.stix.modified }], + }); + const virtual = await createVirtual('Historical Draft Virtual', component.id, 'latest_preview'); + const materialized = await post( + `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, + {}, + ); + // Simulate a snapshot persisted before latest_draft was retired. Its + // members-only output must remain labeled honestly and releasable. + await modelFactory.getModel(virtual.id).collection.updateOne( + { modified: new Date(materialized.modified) }, + { + $set: { + 'composition.component_tracks.0.resolution_strategy': 'latest_draft', + 'composition_resolution.component_snapshots.0.strategy_used': 'latest_draft', + }, + }, + ); + const historical = await get(`/api/release-tracks/${virtual.id}/snapshots/latest`); + expect(historical.composition.component_tracks[0].resolution_strategy).toBe('latest_draft'); + expect(historical.composition_resolution.component_snapshots[0]).toMatchObject({ + strategy_used: 'latest_draft', + resolved_version: null, + }); + const released = await post( + `/api/release-tracks/${virtual.id}/snapshots/latest/release`, + {}, + 200, + ); + expect(released.composition_resolution).toEqual(historical.composition_resolution); + expect(released.version_history.at(-1).component_versions).toEqual({ [component.id]: null }); + expect(revisionKeys(released)).toEqual(revisionKeys(materialized)); + const failure = await post( + `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, + {}, + 400, + ); + expect(failure.message).toContain(component.id); + expect((await get(`/api/release-tracks/${virtual.id}/snapshots/latest`)).modified).toBe( + released.modified, + ); + await request(app) + .put(`/api/release-tracks/${virtual.id}/virtual/composition`) + .send({ + component_tracks: [ + { track_id: component.id, priority: 0, resolution_strategy: 'latest_preview' }, + ], + }) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + const next = await post(`/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}); + expect(next.composition_resolution.component_snapshots[0].strategy_used).toBe('latest_preview'); + const retained = await get( + `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent(released.modified)}`, + ); + expect(retained.composition_resolution).toEqual(historical.composition_resolution); + }); + + it('uses the newest tagged snapshot for preview while explicit draft selection remains invalid', async function () { + const member = await createRevision('Newest Tagged Preview Member'); + const staged = await createRevision('Unpublished Legacy Tagged Content'); + const { component, contents: release } = await createReleasedComponent( + 'Newest Tagged Preview Source', + member, + ); + // Older tagged data may contain workflow entries. Published membership + // remains authoritative even when latest_preview selects that snapshot. + await modelFactory.getModel(component.id).collection.updateOne( + { modified: new Date(release.modified) }, + { + $set: { + staged: [ + { + object_ref: staged.stix.id, + object_modified: 'latest', + object_status: 'work-in-progress', + object_staged_at: new Date(), + object_staged_by: 'system', + }, + ], + }, + }, + ); + const virtual = await createVirtual( + 'Newest Tagged Preview Virtual', + component.id, + 'latest_preview', + ); + const snapshotsBefore = await modelFactory.getModel(component.id).find().lean(); + const materialized = await post( + `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, + {}, + ); + expect(revisionKeys(materialized)).toEqual(revisionKeys(release)); + expect(materialized.composition_resolution.component_snapshots[0]).toMatchObject({ + resolved_snapshot_id: release.modified, + resolved_version: release.version, + strategy_used: 'latest_preview', + }); + expect(await modelFactory.getModel(component.id).find().lean()).toEqual(snapshotsBefore); + const taggedVirtual = await post( + `/api/release-tracks/${virtual.id}/snapshots/latest/release`, + {}, + 200, + ); + expect(taggedVirtual.version_history.at(-1).component_versions).toEqual({ + [component.id]: release.version, + }); + await request(app) + .put(`/api/release-tracks/${virtual.id}/virtual/composition`) + .send({ + component_tracks: [ + { + track_id: component.id, + priority: 0, + resolution_strategy: 'specific_snapshot', + snapshot: release.release_source_modified, + }, + ], + }) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + await post(`/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}, 400); + }); + + it('serializes draft pruning with virtual materialization until provenance is persisted', async function () { + const component = await post('/api/release-tracks/new', { + name: 'Draft Pruning Race', + type: 'standard', + }); + const virtual = await createVirtual( + 'Draft Pruning Race Virtual', + component.id, + 'latest_preview', + ); + const save = dynamicRepo.saveSnapshot; + const stub = sinon.stub(dynamicRepo, 'saveSnapshot').callsFake(async (trackId, snapshot) => { + if (trackId === virtual.id) { + await post(`/api/release-tracks/${component.id}/meta`, { description: 'Racing edit' }, 409); + } + return save.call(dynamicRepo, trackId, snapshot); + }); + try { + await post(`/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}); + } finally { + stub.restore(); + } + await post(`/api/release-tracks/${component.id}/meta`, { description: 'After persist' }, 200); + expect( + await dynamicRepo.getSnapshotByModified(component.id, component.modified), + ).not.toBeNull(); + + const resolve = dynamicRepo.findResolvedComponentSnapshotIds; + const scan = sinon + .stub(dynamicRepo, 'findResolvedComponentSnapshotIds') + .callsFake(async (...args) => { + if (args[0] === virtual.id) { + await post(`/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}, 409); + } + return resolve.apply(dynamicRepo, args); + }); + try { + await post(`/api/release-tracks/${component.id}/meta`, { description: 'Pruning first' }, 200); + } finally { + scan.restore(); + } + expect( + await dynamicRepo.getSnapshotByModified(component.id, component.modified), + ).not.toBeNull(); + }); + + it('prunes a retained source after its last virtual dependent is deleted', async function () { + const component = await post('/api/release-tracks/new', { + name: 'Released Draft Retention', + type: 'standard', + }); + const virtual = await createVirtual( + 'Temporary Draft Dependent', + component.id, + 'latest_preview', + ); + const materialized = await post( + `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, + {}, + ); + await post( + `/api/release-tracks/${component.id}/meta`, + { description: 'Keep dependent source' }, + 200, + ); + expect( + await dynamicRepo.getSnapshotByModified(component.id, component.modified), + ).not.toBeNull(); + await request(app) + .delete( + `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent(materialized.modified)}`, + ) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(204); + await post( + `/api/release-tracks/${component.id}/meta`, + { description: 'Prune unused source' }, + 200, + ); + expect(await dynamicRepo.getSnapshotByModified(component.id, component.modified)).toBeNull(); + }); + after(async function () { await database.closeConnection(); }); diff --git a/app/tests/api/release-tracks/virtual-snapshot-schedule-validation.spec.js b/app/tests/api/release-tracks/virtual-snapshot-schedule-validation.spec.js index 9fdc8324..bae0d384 100644 --- a/app/tests/api/release-tracks/virtual-snapshot-schedule-validation.spec.js +++ b/app/tests/api/release-tracks/virtual-snapshot-schedule-validation.spec.js @@ -9,6 +9,8 @@ const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const ReleaseTrackRegistry = require('../../../models/release-tracks/release-track-registry-model'); const releaseTracksService = require('../../../services/release-tracks/release-tracks-service'); +const { InsufficientRoleError } = require('../../../exceptions'); +const editor = { role: 'editor' }; describe('Virtual release-track snapshot schedule validation API', function () { let app; @@ -65,6 +67,7 @@ describe('Virtual release-track snapshot schedule validation API', function () { const schedules = [ { mode: 'manual' }, { mode: 'cron', cron: '0 0 1 1,7 *' }, + { mode: 'cron', cron: '0 0 * * *', draft_retention: { max_drafts: 10 } }, { mode: 'dates', dates: ['2027-01-15T00:00:00.000Z', '2027-07-15T00:00:00.000Z'], @@ -137,11 +140,50 @@ describe('Virtual release-track snapshot schedule validation API', function () { expect(() => releaseTracksService.updateSchedule(virtual.body.id, { mode: 'cron' })).toThrow(); }); + it('allows editor timing edits but requires admin for every changed cron retention threshold', async function () { + const created = await createTrack({ + mode: 'cron', + cron: '0 0 * * *', + draft_retention: { max_drafts: 10 }, + }); + const trackId = created.body.id; + const changedTiming = { + mode: 'cron', + cron: '30 1 * * *', + draft_retention: { max_drafts: 10 }, + }; + await releaseTracksService.updateSchedule(trackId, changedTiming, editor); + expect((await getRegistryTrack(created.name)).snapshot_schedule).toEqual(changedTiming); + for (const draftRetention of [{ max_drafts: 1 }, { max_drafts: null }, undefined]) { + await expect( + releaseTracksService.updateSchedule( + trackId, + { mode: 'cron', cron: changedTiming.cron, draft_retention: draftRetention }, + editor, + ), + ).rejects.toBeInstanceOf(InsufficientRoleError); + } + expect((await getRegistryTrack(created.name)).snapshot_schedule).toEqual(changedTiming); + await releaseTracksService.updateSchedule( + trackId, + { mode: 'dates', dates: ['2030-01-01T00:00:00.000Z'] }, + editor, + ); + await expect( + releaseTracksService.updateSchedule(trackId, changedTiming, editor), + ).rejects.toBeInstanceOf(InsufficientRoleError); + await releaseTracksService.updateSchedule(trackId, { mode: 'manual' }, editor); + const registry = await getRegistryTrack(created.name); + expect(registry.snapshot_schedule).toEqual({ mode: 'manual' }); + expect(registry.snapshot_count).toBe(1); + }); + it('rejects fields that do not apply to manual schedules', async function () { const invalidSchedules = [ { mode: 'manual', cron: '0 0 1 1,7 *' }, { mode: 'manual', dates: ['2027-01-15T00:00:00.000Z'] }, { mode: 'manual', unexpected: true }, + { mode: 'manual', draft_retention: { max_drafts: 1 } }, ]; for (const schedule of invalidSchedules) { @@ -174,6 +216,11 @@ describe('Virtual release-track snapshot schedule validation API', function () { dates: ['2027-01-15T00:00:00.000Z'], cron: '0 0 1 1,7 *', }, + { + mode: 'dates', + dates: ['2027-01-15T00:00:00.000Z'], + draft_retention: { max_drafts: 1 }, + }, ]; for (const schedule of invalidSchedules) { diff --git a/app/tests/api/system-configuration/allowed-values.spec.js b/app/tests/api/system-configuration/allowed-values.spec.js new file mode 100644 index 00000000..9b82c720 --- /dev/null +++ b/app/tests/api/system-configuration/allowed-values.spec.js @@ -0,0 +1,745 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); +const fs = require('fs/promises'); +const os = require('os'); +const path = require('path'); +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const AllowedValuesConfiguration = require('../../../models/allowed-values-configuration-model'); +const UserAccount = require('../../../models/user-account-model'); +const service = require('../../../services/system/allowed-values-service'); +const systemConfigurationService = require('../../../services/system/system-configuration-service'); +const seed = require('../../../config/allowed-values.json'); +const ValidationBypassRule = require('../../../models/validation-bypass-rule-model'); + +const base = '/api/config/allowed-values'; +const rules = `${base}/rules`; +const platformRule = { propertyName: 'x_mitre_platforms', domainName: 'ics-attack' }; +const enterpriseRule = { propertyName: 'x_mitre_platforms', domainName: 'enterprise-attack' }; +const ruleKey = ({ propertyName, domainName }) => JSON.stringify([propertyName, domainName]); +const ruleUrl = ({ propertyName, domainName }) => `${rules}/${propertyName}/${domainName}`; +const sameRule = (left, right) => ruleKey(left) === ruleKey(right); +const expectedCatalog = []; +for (const { objectType, properties } of seed) { + for (const { propertyName, domains } of properties) { + for (const { domainName } of domains) { + let rule = expectedCatalog.find((item) => sameRule(item, { propertyName, domainName })); + if (!rule) { + rule = { propertyName, domainName, objectTypes: [] }; + expectedCatalog.push(rule); + } + rule.objectTypes.push(objectType); + } + } +} +// The original configured groups also admit asset platforms in the other +// domains under the installed ADM, without seeding any new choices. +for (const rule of expectedCatalog) { + if (rule.propertyName === 'x_mitre_platforms' && rule.domainName !== 'ics-attack') { + rule.objectTypes.push('asset'); + } +} +const expectedNested = structuredClone(seed); +expectedNested + .find(({ objectType }) => objectType === 'asset') + .properties.find(({ propertyName }) => propertyName === 'x_mitre_platforms') + .domains.push( + { domainName: 'enterprise-attack', allowedValues: [] }, + { domainName: 'mobile-attack', allowedValues: [] }, + ); + +function choices(nested, rule, objectType) { + return nested + .find((item) => item.objectType === objectType) + .properties.find((item) => item.propertyName === rule.propertyName) + .domains.find((item) => item.domainName === rule.domainName).allowedValues; +} + +function scopedValues(listing) { + return listing + .flatMap(({ propertyName, domainName, values }) => + values.flatMap(({ value, enabled, objectTypes }) => + objectTypes.map((objectType) => + JSON.stringify([objectType, propertyName, domainName, value, enabled]), + ), + ), + ) + .sort(); +} + +const seededValues = seed + .flatMap(({ objectType, properties }) => + properties.flatMap(({ propertyName, domains }) => + domains.flatMap(({ domainName, allowedValues }) => + allowedValues.map((value) => + JSON.stringify([objectType, propertyName, domainName, value, true]), + ), + ), + ), + ) + .sort(); + +describe('Runtime allowed values API', function () { + let app; + let cookie; + let anonymousUser; + let originalSeedPath; + let originalBasicAuthn; + + function api(method, url, body, status = 200) { + const call = request(app)[method](url).set('Cookie', cookie); + if (body !== undefined) call.send(body); + return call.expect(status); + } + + async function getRule(rule) { + return (await api('get', rules)).body.find((item) => sameRule(item, rule)); + } + + before(async function () { + originalSeedPath = config.configurationFiles.allowedValues; + originalBasicAuthn = { ...config.serviceAuthn.basicApikey }; + config.serviceAuthn.basicApikey.enable = true; + config.serviceAuthn.basicApikey.serviceAccounts = [ + { + name: 'allowed-values-reader', + apikey: 'allowed-values-test-key', + serviceRole: 'read-only', + }, + ]; + config.configurationFiles.allowedValues = path.resolve( + __dirname, + '../../../config/allowed-values.json', + ); + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + const passportCookie = await login.loginAnonymous(app); + cookie = `${passportCookie.name}=${passportCookie.value}`; + anonymousUser = await systemConfigurationService.retrieveAnonymousUserAccount(); + }); + + beforeEach(async function () { + await UserAccount.updateOne({ id: anonymousUser.id }, { $set: { role: 'admin' } }); + await AllowedValuesConfiguration.deleteMany({}); + await service.initialize(); + }); + + after(async function () { + config.configurationFiles.allowedValues = originalSeedPath; + Object.assign(config.serviceAuthn.basicApikey, originalBasicAuthn); + await database.closeConnection(); + }); + + it('groups the catalog uniquely without changing any seeded choice or object-type scope', async function () { + await AllowedValuesConfiguration.deleteMany({}); + await Promise.all(Array.from({ length: 8 }, () => service.initialize())); + expect((await api('get', base)).body).toEqual(expectedNested); + const listing = (await api('get', rules)).body; + expect( + listing.map(({ propertyName, domainName, objectTypes }) => ({ + propertyName, + domainName, + objectTypes, + })), + ).toEqual(expectedCatalog); + expect(new Set(listing.map(ruleKey)).size).toBe(expectedCatalog.length); + expect(scopedValues(listing)).toEqual(seededValues); + expect(listing.every(({ invalidValues }) => invalidValues.length === 0)).toBe(true); + for (const rule of listing) { + expect( + new Set(rule.values.map(({ value, enabled }) => JSON.stringify([value, enabled]))).size, + ).toBe(rule.values.length); + } + const ics = listing.find((item) => sameRule(item, platformRule)); + expect(ics.objectTypes).toContain('data-source'); + expect(ics.values.find((option) => option.value === 'None').objectTypes).toEqual([ + 'analytic', + 'technique', + ]); + expect(ics.values.some((option) => option.objectTypes.includes('data-source'))).toBe(false); + }); + + it('uses the configured seed once and keeps cleared rules and their complete catalog on restart', async function () { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'allowed-values-')); + const configuredPath = config.configurationFiles.allowedValues; + try { + const customPath = path.join(directory, 'seed.json'); + await fs.writeFile( + customPath, + JSON.stringify([ + { + objectType: 'data-source', + properties: [ + { + propertyName: platformRule.propertyName, + domains: [{ domainName: platformRule.domainName, allowedValues: ['Linux'] }], + }, + ], + }, + ]), + ); + config.configurationFiles.allowedValues = customPath; + await AllowedValuesConfiguration.deleteMany({}); + await service.initialize(); + const listing = (await api('get', rules)).body; + expect(scopedValues(listing)).toEqual([ + JSON.stringify([ + 'data-source', + platformRule.propertyName, + platformRule.domainName, + 'Linux', + true, + ]), + ]); + expect( + listing.map(({ propertyName, domainName, objectTypes }) => ({ + propertyName, + domainName, + objectTypes, + })), + ).toEqual(expectedCatalog); + const cleared = (await api('put', ruleUrl(platformRule), { values: [] })).body; + expect(cleared).toEqual({ + ...listing.find((item) => sameRule(item, platformRule)), + values: [], + }); + await Promise.all(Array.from({ length: 8 }, () => service.initialize())); + const after = (await api('get', rules)).body; + expect(after).toEqual( + expectedCatalog.map((rule) => ({ ...rule, values: [], invalidValues: [] })), + ); + for (const objectType of cleared.objectTypes) { + expect(choices((await api('get', base)).body, platformRule, objectType)).toEqual([]); + } + } finally { + config.configurationFiles.allowedValues = configuredPath; + await fs.rm(directory, { recursive: true, force: true }); + } + }); + + it('adds, toggles and removes options for multiple object types without broadening other scopes', async function () { + const initial = (await api('get', rules)).body; + const rule = initial.find((item) => sameRule(item, platformRule)); + const option = { + value: 'Network Devices', + enabled: true, + objectTypes: ['technique', 'data-source'], + }; + let values = [...rule.values, { ...option, value: ` ${option.value} ` }]; + let updated = (await api('put', ruleUrl(rule), { values })).body; + expect(updated).toEqual({ ...rule, values: [...rule.values, option] }); + for (const objectType of option.objectTypes) { + expect(choices((await api('get', base)).body, rule, objectType)).toContain(option.value); + } + expect(choices((await api('get', base)).body, rule, 'analytic')).not.toContain(option.value); + expect( + await systemConfigurationService.retrieveAllowedValuesForTypePropertyDomain( + 'data-source', + rule.propertyName, + rule.domainName, + ), + ).toEqual({ domainName: rule.domainName, allowedValues: [option.value] }); + + values = updated.values.map((item) => + item.value === option.value ? { ...item, enabled: false } : item, + ); + updated = (await api('put', ruleUrl(rule), { values })).body; + expect(updated.values).toContainEqual({ ...option, enabled: false }); + for (const objectType of option.objectTypes) { + expect(choices((await api('get', base)).body, rule, objectType)).not.toContain(option.value); + } + values = updated.values.map((item) => + item.value === option.value ? { ...item, enabled: true } : item, + ); + await api('put', ruleUrl(rule), { values }); + expect(choices((await api('get', base)).body, rule, 'data-source')).toEqual([option.value]); + values = values.filter((item) => item.value !== option.value); + await api('put', ruleUrl(rule), { values }); + expect(await getRule(rule)).toEqual(rule); + expect(choices((await api('get', base)).body, rule, 'data-source')).toEqual([]); + expect((await api('get', rules)).body.filter((item) => !sameRule(item, rule))).toEqual( + initial.filter((item) => !sameRule(item, rule)), + ); + }); + + it('preserves already persisted removals and mixed enabled variants through grouping, save and restart', async function () { + await AllowedValuesConfiguration.updateOne( + { _id: 'allowed-values' }, + { + $pull: { values: { ...enterpriseRule, objectType: 'technique', value: 'Linux' } }, + }, + ); + await AllowedValuesConfiguration.updateOne( + { + _id: 'allowed-values', + values: { $elemMatch: { ...enterpriseRule, objectType: 'technique', value: 'Windows' } }, + }, + { $set: { 'values.$.enabled': false } }, + ); + const grouped = await getRule(enterpriseRule); + expect(grouped.values.find((option) => option.value === 'Linux').objectTypes).not.toContain( + 'technique', + ); + expect(grouped.values).toContainEqual({ + value: 'Windows', + enabled: false, + objectTypes: ['technique'], + }); + expect( + grouped.values.find((option) => option.value === 'Windows' && option.enabled).objectTypes, + ).not.toContain('technique'); + expect((await api('put', ruleUrl(grouped), { values: grouped.values })).body).toEqual(grouped); + await Promise.all(Array.from({ length: 8 }, () => service.initialize())); + await databaseConfiguration.checkSystemConfiguration(); + expect(await getRule(grouped)).toEqual(grouped); + const nested = (await api('get', base)).body; + expect(choices(nested, grouped, 'technique')).not.toContain('Linux'); + expect(choices(nested, grouped, 'technique')).not.toContain('Windows'); + expect(choices(nested, grouped, 'analytic')).toContain('Linux'); + expect(choices(nested, grouped, 'analytic')).toContain('Windows'); + }); + + it('coalesces equal disjoint variants but keeps differing enabled states distinct', async function () { + const values = [ + { value: 'Windows', enabled: true, objectTypes: ['analytic'] }, + { value: 'Windows', enabled: true, objectTypes: ['technique'] }, + { value: 'Windows', enabled: false, objectTypes: ['software'] }, + ]; + const updated = (await api('put', ruleUrl(platformRule), { values })).body; + expect(updated.values).toEqual([ + { value: 'Windows', enabled: true, objectTypes: ['analytic', 'technique'] }, + { value: 'Windows', enabled: false, objectTypes: ['software'] }, + ]); + expect(await getRule(platformRule)).toEqual(updated); + const nested = (await api('get', base)).body; + expect(choices(nested, platformRule, 'analytic')).toEqual(['Windows']); + expect(choices(nested, platformRule, 'technique')).toEqual(['Windows']); + expect(choices(nested, platformRule, 'software')).toEqual([]); + expect(choices(nested, platformRule, 'data-source')).toEqual([]); + }); + + it('rejects unsupported rules, invalid bodies and invalid scopes without partially replacing any values', async function () { + const before = (await api('get', rules)).body; + const option = { value: 'Windows', enabled: true, objectTypes: ['technique'] }; + for (const body of [ + {}, + { values: null }, + { values: [option], extra: true }, + { values: [option, { ...option, value: ' ' }] }, + { values: [option, { ...option, value: 123 }] }, + { values: [option, { value: 'Other', objectTypes: ['technique'] }] }, + { values: [option, { ...option, enabled: 'false' }] }, + { values: [option, { ...option, enabled: null }] }, + { values: [option, { ...option, objectTypes: [] }] }, + { values: [option, { ...option, objectTypes: ['arbitrary-object'] }] }, + { values: [option, { ...option, objectTypes: ['identity'] }] }, + { values: [option, { ...option, objectTypes: ['technique', 'technique'] }] }, + { values: [option, { ...option, objectTypes: ['technique', null] }] }, + { values: [option, { ...option, extra: true }] }, + ]) { + await api('put', ruleUrl(platformRule), body, 400); + } + for (const rule of [ + { ...platformRule, propertyName: 'arbitrary-property' }, + { ...platformRule, domainName: 'stix' }, + ]) { + await api('put', ruleUrl(rule), { values: [] }, 400); + } + expect((await api('get', rules)).body).toEqual(before); + expect((await api('get', base)).body).toEqual(expectedNested); + }); + + it('rejects overlapping duplicate values after trimming regardless of enabled state', async function () { + const before = (await api('get', rules)).body; + for (const enabled of [true, false]) { + await api( + 'put', + ruleUrl(platformRule), + { + values: [ + { value: 'Windows', enabled: true, objectTypes: ['analytic', 'technique'] }, + { value: ' Windows ', enabled, objectTypes: ['technique', 'software'] }, + ], + }, + 409, + ); + } + expect((await api('get', rules)).body).toEqual(before); + }); + + it('atomically retains concurrent replacements of different rules and every untouched rule', async function () { + const before = (await api('get', rules)).body; + const targets = before.filter((rule) => rule.propertyName === 'x_mitre_platforms'); + const replacements = targets.map((rule) => ({ + ...rule, + values: [{ value: 'Windows', enabled: true, objectTypes: rule.objectTypes }], + })); + const responses = await Promise.all( + replacements.map((rule) => api('put', ruleUrl(rule), { values: rule.values })), + ); + expect(responses.map((response) => response.body)).toEqual(replacements); + expect((await api('get', rules)).body).toEqual( + before.map((rule) => replacements.find((replacement) => sameRule(replacement, rule)) || rule), + ); + const nested = (await api('get', base)).body; + for (const rule of replacements) { + for (const objectType of rule.objectTypes) { + expect(choices(nested, rule, objectType)).toEqual(['Windows']); + } + } + }); + + it('offers ADM-supported new scopes without expanding the original object/property whitelist', async function () { + const catalog = (await api('get', `${base}/catalog`)).body; + expect(catalog.admVersion).toBe( + require('@mitre-attack/attack-data-model/package.json').version, + ); + for (const definition of catalog.rules) { + for (const objectType of definition.objectTypes) { + expect( + seed + .find((item) => item.objectType === objectType) + .properties.map((item) => item.propertyName), + ).toContain(definition.propertyName); + } + } + expect(catalog.rules).toContainEqual( + expect.objectContaining({ + propertyName: 'x_mitre_sectors', + domainName: 'enterprise-attack', + objectTypes: ['asset'], + valueType: 'enum', + choices: expect.arrayContaining([{ value: 'Electric', objectTypes: ['asset'] }]), + }), + ); + expect( + catalog.rules.some( + (item) => + item.propertyName === 'x_mitre_effective_permissions' && + item.domainName === 'mobile-attack', + ), + ).toBe(false); + expect( + catalog.rules + .find((item) => item.propertyName === 'x_mitre_effective_permissions') + .choices.map(({ value }) => value), + ).not.toContain('Remote Desktop Users'); + expect( + catalog.rules.find((item) => item.propertyName === 'x_mitre_data_sources').valueType, + ).toBe('formatted'); + expect( + catalog.rules + .filter((item) => item.objectTypes.includes('identity')) + .every((item) => item.domainName === 'stix'), + ).toBe(true); + }); + + it('creates a new property/domain atomically and retains an empty configuration across restart', async function () { + const newRule = { propertyName: 'x_mitre_sectors', domainName: 'enterprise-attack' }; + await api('put', ruleUrl(newRule), { values: [] }, 404); + const responses = await Promise.all( + Array.from({ length: 8 }, () => + request(app) + .post(rules) + .set('Cookie', cookie) + .send({ ...newRule, values: [] }), + ), + ); + expect(responses.map(({ status }) => status).sort()).toEqual([ + 201, 409, 409, 409, 409, 409, 409, 409, + ]); + const created = responses.find(({ status }) => status === 201).body; + expect(created).toEqual({ ...newRule, objectTypes: ['asset'], values: [], invalidValues: [] }); + await Promise.all(Array.from({ length: 8 }, () => service.initialize())); + expect(await getRule(newRule)).toEqual(created); + expect(choices((await api('get', base)).body, newRule, 'asset')).toEqual([]); + const option = { value: 'Electric', enabled: true, objectTypes: ['asset'] }; + await api('put', ruleUrl(newRule), { values: [option] }); + expect(choices((await api('get', base)).body, newRule, 'asset')).toEqual(['Electric']); + }); + + it('validates formatted choices without mutation and applies the same checks to final writes', async function () { + const rule = { propertyName: 'x_mitre_data_sources', domainName: 'enterprise-attack' }; + const input = { ...rule, objectTypes: ['technique'], value: ' Custom Sensor: Custom Event ' }; + const before = (await api('get', rules)).body; + expect((await api('post', `${base}/validate`, input)).body).toEqual({ + value: 'Custom Sensor: Custom Event', + }); + expect((await api('get', rules)).body).toEqual(before); + for (const value of [ + 'Missing separator', + 'Source:', + ': Component', + 'Source: Component: Extra', + ]) { + await api('post', `${base}/validate`, { ...input, value }, 400); + await api( + 'post', + rules, + { ...rule, values: [{ value, enabled: false, objectTypes: ['technique'] }] }, + 400, + ); + } + const created = ( + await api( + 'post', + rules, + { + ...rule, + values: [{ value: input.value, enabled: true, objectTypes: input.objectTypes }], + }, + 201, + ) + ).body; + expect(created.values).toEqual([ + { value: 'Custom Sensor: Custom Event', enabled: true, objectTypes: ['technique'] }, + ]); + await api( + 'put', + ruleUrl(rule), + { values: [{ value: 'Bad', enabled: false, objectTypes: ['technique'] }] }, + 400, + ); + expect(await getRule(rule)).toEqual(created); + expect(choices((await api('get', base)).body, rule, 'technique')).toEqual([ + 'Custom Sensor: Custom Event', + ]); + }); + + it('rejects enum and domain incompatibilities even when disabled, bypassed, or general ADM validation is off', async function () { + const before = (await api('get', rules)).body; + const bypass = await ValidationBypassRule.create({ + fieldPath: ['x_mitre_platforms', '0'], + errorCode: 'invalid_value', + stixType: 'attack-pattern', + suppressError: true, + }); + const original = config.validateRequests.withAttackDataModel; + try { + config.validateRequests.withAttackDataModel = false; + for (const enabled of [true, false]) { + await api( + 'put', + ruleUrl(enterpriseRule), + { + values: [{ value: 'Custom Platform', enabled, objectTypes: ['technique'] }], + }, + 400, + ); + } + await api( + 'post', + `${base}/validate`, + { + ...enterpriseRule, + objectTypes: ['technique'], + value: 'Custom Platform', + }, + 400, + ); + for (const input of [ + { + propertyName: 'x_mitre_effective_permissions', + domainName: 'mobile-attack', + value: 'User', + }, + { + propertyName: 'x_mitre_tactic_type', + domainName: 'enterprise-attack', + value: 'Post-Adversary Device Access', + }, + { + propertyName: 'x_mitre_data_sources', + domainName: 'mobile-attack', + value: 'Source: Component', + }, + ]) { + await api('post', `${base}/validate`, { ...input, objectTypes: ['technique'] }, 400); + await api( + 'post', + rules, + { propertyName: input.propertyName, domainName: input.domainName, values: [] }, + 400, + ); + } + await api( + 'post', + rules, + { + propertyName: 'x_mitre_sectors', + domainName: 'enterprise-attack', + values: [{ value: 'Electric', enabled: true, objectTypes: ['technique'] }], + }, + 400, + ); + const relatedRule = { + propertyName: 'related_asset_sectors', + domainName: 'enterprise-attack', + }; + await api( + 'post', + rules, + { + ...relatedRule, + values: [{ value: 'Invented sector', enabled: false, objectTypes: ['asset'] }], + }, + 400, + ); + expect((await api('get', rules)).body).toEqual(before); + await api( + 'post', + rules, + { ...relatedRule, values: [{ value: 'Electric', enabled: true, objectTypes: ['asset'] }] }, + 201, + ); + expect(choices((await api('get', base)).body, relatedRule, 'asset')).toEqual(['Electric']); + } finally { + config.validateRequests.withAttackDataModel = original; + await ValidationBypassRule.deleteOne({ _id: bypass._id }); + } + }); + + it('migrates old configuration keys without restoring removals and quarantines invalid legacy values', async function () { + await AllowedValuesConfiguration.updateOne( + { _id: 'allowed-values' }, + { + $unset: { rules: '' }, + $pull: { values: { ...enterpriseRule, objectType: 'technique', value: 'Linux' } }, + }, + ); + await AllowedValuesConfiguration.updateOne( + { _id: 'allowed-values' }, + { + $push: { + values: { + ...enterpriseRule, + objectType: 'technique', + value: 'Legacy Platform', + enabled: true, + }, + }, + }, + ); + const configuredPath = config.configurationFiles.allowedValues; + const originalValidation = config.validateRequests.withAttackDataModel; + try { + config.configurationFiles.allowedValues = '/no-longer-present/seed.json'; + config.validateRequests.withAttackDataModel = false; + await Promise.all(Array.from({ length: 8 }, () => service.initialize())); + const rule = await getRule(enterpriseRule); + expect(rule.invalidValues).toEqual([ + expect.objectContaining({ + value: 'Legacy Platform', + enabled: true, + objectTypes: ['technique'], + reason: expect.any(String), + }), + ]); + expect(rule.values.some(({ value }) => value === 'Legacy Platform')).toBe(false); + const nested = (await api('get', base)).body; + expect(choices(nested, enterpriseRule, 'technique')).not.toContain('Legacy Platform'); + expect(choices(nested, enterpriseRule, 'technique')).not.toContain('Linux'); + expect(choices(nested, enterpriseRule, 'analytic')).toContain('Linux'); + const raw = await AllowedValuesConfiguration.findById('allowed-values').lean(); + expect(raw.values.some(({ value }) => value === 'Legacy Platform')).toBe(true); + expect(new Set(raw.rules.map(ruleKey)).size).toBe(expectedCatalog.length); + await api('put', ruleUrl(rule), { values: rule.values }); + expect((await getRule(rule)).invalidValues).toEqual([]); + expect( + (await AllowedValuesConfiguration.findById('allowed-values').lean()).values.some( + ({ value }) => value === 'Legacy Platform', + ), + ).toBe(false); + } finally { + config.configurationFiles.allowedValues = configuredPath; + config.validateRequests.withAttackDataModel = originalValidation; + } + }); + + it('rejects a fresh custom seed containing an ADM-invalid value without partially initializing', async function () { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'allowed-values-invalid-')); + const configuredPath = config.configurationFiles.allowedValues; + try { + config.configurationFiles.allowedValues = path.join(directory, 'seed.json'); + await fs.writeFile( + config.configurationFiles.allowedValues, + JSON.stringify([ + { + objectType: 'technique', + properties: [ + { + propertyName: 'x_mitre_platforms', + domains: [ + { + domainName: 'enterprise-attack', + allowedValues: ['Windows', 'Custom Platform'], + }, + ], + }, + ], + }, + ]), + ); + await AllowedValuesConfiguration.deleteMany({}); + await expect(service.initialize()).rejects.toMatchObject({ name: 'BadRequestError' }); + expect(await AllowedValuesConfiguration.countDocuments()).toBe(0); + } finally { + config.configurationFiles.allowedValues = configuredPath; + await fs.rm(directory, { recursive: true, force: true }); + } + }); + + it('keeps dropdown access for nonadmins but denies rule listing and replacement', async function () { + const before = (await api('get', rules)).body; + try { + for (const role of ['visitor', 'editor', 'team_lead']) { + await UserAccount.updateOne({ id: anonymousUser.id }, { $set: { role } }); + expect((await api('get', base)).body).toEqual(expectedNested); + await api('get', rules, undefined, 401); + await api('put', ruleUrl(platformRule), { values: [] }, 401); + await api('get', `${base}/catalog`, undefined, 401); + await api( + 'post', + rules, + { propertyName: 'x_mitre_sectors', domainName: 'enterprise-attack', values: [] }, + 401, + ); + await api( + 'post', + `${base}/validate`, + { ...platformRule, value: 'Windows', objectTypes: ['technique'] }, + 401, + ); + } + } finally { + await UserAccount.updateOne({ id: anonymousUser.id }, { $set: { role: 'admin' } }); + } + await request(app).get(rules).expect(401); + await request(app).put(ruleUrl(platformRule)).send({ values: [] }).expect(401); + expect((await api('get', rules)).body).toEqual(before); + }); + + it('retains read-only service dropdown access without granting administrator operations', async function () { + const before = (await api('get', rules)).body; + const serviceRequest = (method, url) => + request(app)[method](url).auth('allowed-values-reader', 'allowed-values-test-key'); + expect((await serviceRequest('get', base).expect(200)).body).toEqual(expectedNested); + await serviceRequest('get', rules).expect(401); + await serviceRequest('put', ruleUrl(platformRule)).send({ values: [] }).expect(401); + await serviceRequest('get', `${base}/catalog`).expect(401); + await serviceRequest('post', rules) + .send({ propertyName: 'x_mitre_sectors', domainName: 'enterprise-attack', values: [] }) + .expect(401); + await serviceRequest('post', `${base}/validate`) + .send({ ...platformRule, value: 'Windows', objectTypes: ['technique'] }) + .expect(401); + expect((await api('get', rules)).body).toEqual(before); + }); +}); diff --git a/app/tests/scheduler/virtual-track-snapshots-task.spec.js b/app/tests/scheduler/virtual-track-snapshots-task.spec.js index 885f57ff..807211db 100644 --- a/app/tests/scheduler/virtual-track-snapshots-task.spec.js +++ b/app/tests/scheduler/virtual-track-snapshots-task.spec.js @@ -10,6 +10,8 @@ const databaseConfiguration = require('../../lib/database-configuration'); const ReleaseTrackRegistry = require('../../models/release-tracks/release-track-registry-model'); const VirtualTrackScheduleOccurrence = require('../../models/release-tracks/virtual-track-schedule-occurrence-model'); const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); +const occurrenceRepo = require('../../repository/release-tracks/virtual-track-schedule-occurrence.repository'); +const { ReleaseConflictError } = require('../../exceptions'); const releaseTracksService = require('../../services/release-tracks/release-tracks-service'); describe('Scheduled virtual release-track materialization', function () { @@ -18,6 +20,7 @@ describe('Scheduled virtual release-track materialization', function () { before(async function () { config.scheduler.enableScheduler = false; + config.validateRequests.withAttackDataModel = true; await database.initializeConnection(); await databaseConfiguration.checkSystemConfiguration(); task = require('../../scheduler/virtual-track-snapshots-task'); @@ -58,6 +61,7 @@ describe('Scheduled virtual release-track materialization', function () { ], }, snapshot_schedule: snapshotSchedule, + actor: { role: 'admin' }, }); } @@ -178,6 +182,50 @@ describe('Scheduled virtual release-track materialization', function () { ).toBe(1); }); + it('applies only saved cron retention on trusted recurring execution, never on manual or date runs', async function () { + const component = await createComponent(); + const virtual = await createVirtual(component.id, { + mode: 'cron', + cron: '0 0 * * *', + draft_retention: { max_drafts: 2 }, + }); + await releaseTracksService.updateMetadata(virtual.id, { description: 'Count metadata too' }); + await releaseTracksService.createVirtualSnapshot(virtual.id); + const spoofed = await releaseTracksService.createVirtualSnapshot(virtual.id, { + scheduledMaterialization: { + schedule_mode: 'cron', + scheduled_for: new Date('2026-07-01T00:00:00.000Z'), + }, + }); + expect(spoofed).not.toHaveProperty('draft_cleanup'); + expect(await snapshotCount(virtual.id)).toBe(4); + const scheduled = await task.executeCronOccurrence( + virtual.id, + new Date('2026-07-02T00:00:00.000Z'), + ); + expect(scheduled.draft_cleanup).toMatchObject({ status: 'completed', deleted_count: 3 }); + expect(await snapshotCount(virtual.id)).toBe(2); + const remaining = (await dynamicRepo.getAllSnapshots(virtual.id)).data; + expect(remaining.map((entry) => entry.modified)).toEqual([ + scheduled.modified, + spoofed.modified, + ]); + + const scheduledFor = new Date('2026-07-03T00:00:00.000Z'); + await releaseTracksService.updateSchedule( + virtual.id, + { mode: 'dates', dates: [scheduledFor.toISOString()] }, + { role: 'editor' }, + ); + await task.reconcileSchedules(scheduledFor); + expect(await snapshotCount(virtual.id)).toBe(3); + const dateSnapshot = await dynamicRepo.getSnapshotByScheduledMaterialization( + virtual.id, + scheduledFor, + ); + expect(dateSnapshot.scheduled_materialization.schedule_mode).toBe('dates'); + }); + it('registers cron tracks in UTC and removes their jobs after track deletion', async function () { const component = await createComponent(); const virtual = await createVirtual(component.id, { @@ -218,6 +266,7 @@ describe('Scheduled virtual release-track materialization', function () { expect(occurrence).toMatchObject({ status: 'failed', attempt_count: 1, + snapshot_modified: null, }); expect(occurrence.last_error.message).toContain('has no tagged snapshots'); expect(await snapshotCount(virtual.id)).toBe(1); @@ -308,6 +357,11 @@ describe('Scheduled virtual release-track materialization', function () { scheduled_for: scheduledFor, }, }); + // Simulate a legacy save/crash before the receipt was durably written. + await VirtualTrackScheduleOccurrence.updateOne( + { track_id: virtual.id, scheduled_for: scheduledFor }, + { $set: { snapshot_modified: null } }, + ); // A persisted scheduled snapshot is the authoritative result. Recovery // must not depend on the component still being available. @@ -341,6 +395,189 @@ describe('Scheduled virtual release-track materialization', function () { }); }); + it('recovers a pruned result after save but before completion even after schedule removal', async function () { + const component = await createComponent(); + const scheduledFor = new Date('2026-08-01T00:00:00.000Z'); + const virtual = await createVirtual(component.id, { + mode: 'dates', + dates: [scheduledFor.toISOString()], + }); + await occurrenceRepo.register(virtual.id, 'dates', scheduledFor); + await occurrenceRepo.claim( + virtual.id, + scheduledFor, + scheduledFor, + new Date(scheduledFor.getTime() + 5 * 60 * 1000), + ); + const materialized = await releaseTracksService.createVirtualSnapshot(virtual.id, { + scheduledMaterialization: { schedule_mode: 'dates', scheduled_for: scheduledFor }, + }); + expect(await occurrenceRepo.getMaterializationReceipt(virtual.id, scheduledFor)).toMatchObject({ + status: 'running', + snapshot_modified: materialized.modified, + }); + + await dynamicRepo.deleteSnapshot(virtual.id, materialized.modified); + await releaseTracksService.deleteTrack(component.id); + await ReleaseTrackRegistry.updateOne( + { track_id: virtual.id }, + { $set: { snapshot_schedule: { mode: 'manual' } } }, + ); + await task.reconcileSchedules(new Date(scheduledFor.getTime() + 10 * 60 * 1000)); + + expect(await snapshotCount(virtual.id)).toBe(1); + expect(await occurrenceRepo.getMaterializationReceipt(virtual.id, scheduledFor)).toMatchObject({ + status: 'completed', + attempt_count: 2, + snapshot_modified: materialized.modified, + }); + const db = mongoose.connection.getClient().db(); + const run = await db.collection('automationRuns').findOne({ 'scope.track_id': virtual.id }); + expect(run).toMatchObject({ + status: 'completed', + counts: { materialized: 0, recovered: 1, failed: 0 }, + }); + const item = await db.collection('automationRunItems').findOne({ run_id: run.run_id }); + expect(item.details).toMatchObject({ + snapshot_modified: materialized.modified, + materialized_and_removed: true, + recovered: true, + }); + expect(item.details).not.toHaveProperty('members_count'); + }); + + it('fences stale completion, failure and skip and never replays a completed pruned result', async function () { + const component = await createComponent(); + const scheduledFor = new Date('2026-09-01T00:00:00.000Z'); + const virtual = await createVirtual(component.id, { + mode: 'dates', + dates: [scheduledFor.toISOString()], + }); + await occurrenceRepo.register(virtual.id, 'dates', scheduledFor); + const expires = new Date(scheduledFor.getTime() + 5 * 60 * 1000); + const stale = await occurrenceRepo.claim(virtual.id, scheduledFor, scheduledFor, expires); + const owner = await occurrenceRepo.claim( + virtual.id, + scheduledFor, + expires, + new Date(expires.getTime() + 5 * 60 * 1000), + ); + expect(await occurrenceRepo.fail(stale, { message: 'stale failure' }, expires)).toBeNull(); + expect(await occurrenceRepo.skip(stale, 'stale skip')).toBeNull(); + + const materialized = await releaseTracksService.createVirtualSnapshot(virtual.id, { + scheduledMaterialization: { schedule_mode: 'dates', scheduled_for: scheduledFor }, + }); + expect(await occurrenceRepo.complete(stale)).toBeNull(); + expect(await occurrenceRepo.skip(owner, 'cannot skip saved work')).toBeNull(); + await occurrenceRepo.complete(owner); + await dynamicRepo.deleteSnapshot(virtual.id, materialized.modified); + + expect(await occurrenceRepo.fail(owner, { message: 'late failure' }, expires)).toBeNull(); + expect(await occurrenceRepo.fail(stale, { message: 'stale failure' }, expires)).toBeNull(); + expect(await occurrenceRepo.skip(stale, 'late skip')).toBeNull(); + expect(await occurrenceRepo.complete(stale)).toBeNull(); + await task.reconcileSchedules(new Date(expires.getTime() + 10 * 60 * 1000)); + expect(await snapshotCount(virtual.id)).toBe(1); + expect(await occurrenceRepo.getMaterializationReceipt(virtual.id, scheduledFor)).toMatchObject({ + status: 'completed', + attempt_count: 2, + snapshot_modified: materialized.modified, + }); + }); + + it('repairs API-originated materialization without a ledger and refuses a different receipt', async function () { + const component = await createComponent(); + const virtual = await createVirtual(component.id, { mode: 'manual' }); + const scheduledMaterialization = { + schedule_mode: 'dates', + scheduled_for: new Date('2026-10-01T00:00:00.000Z'), + }; + const materialized = await releaseTracksService.createVirtualSnapshot(virtual.id, { + scheduledMaterialization, + }); + // Older API-originated metadata did not create an occurrence ledger. + await VirtualTrackScheduleOccurrence.deleteOne({ + track_id: virtual.id, + scheduled_for: scheduledMaterialization.scheduled_for, + }); + expect( + await occurrenceRepo.getMaterializationReceipt( + virtual.id, + scheduledMaterialization.scheduled_for, + ), + ).toBeNull(); + await Promise.all([ + occurrenceRepo.recordMaterialization( + virtual.id, + scheduledMaterialization, + materialized.modified, + ), + occurrenceRepo.recordMaterialization( + virtual.id, + scheduledMaterialization, + materialized.modified, + ), + ]); + await expect( + occurrenceRepo.recordMaterialization( + virtual.id, + scheduledMaterialization, + new Date(new Date(materialized.modified).getTime() + 1), + ), + ).rejects.toBeInstanceOf(ReleaseConflictError); + + const receipt = await occurrenceRepo.getMaterializationReceipt( + virtual.id, + scheduledMaterialization.scheduled_for, + ); + await dynamicRepo.deleteSnapshot(virtual.id, materialized.modified); + expect(await task.executeOccurrence(receipt)).toEqual({ + snapshot_modified: materialized.modified, + materialized_and_removed: true, + }); + expect(await snapshotCount(virtual.id)).toBe(1); + expect( + await occurrenceRepo.getMaterializationReceipt( + virtual.id, + scheduledMaterialization.scheduled_for, + ), + ).toMatchObject({ status: 'completed', snapshot_modified: materialized.modified }); + }); + + it('retains a receipt through audit failure and retries only recovery', async function () { + const component = await createComponent(); + const scheduledFor = new Date('2026-11-01T00:00:00.000Z'); + const virtual = await createVirtual(component.id, { + mode: 'dates', + dates: [scheduledFor.toISOString()], + }); + await occurrenceRepo.register(virtual.id, 'dates', scheduledFor); + const claimed = await occurrenceRepo.claim( + virtual.id, + scheduledFor, + scheduledFor, + new Date(scheduledFor.getTime() + 5 * 60 * 1000), + ); + const materialized = await releaseTracksService.createVirtualSnapshot(virtual.id, { + scheduledMaterialization: { schedule_mode: 'dates', scheduled_for: scheduledFor }, + }); + const retryAt = new Date(scheduledFor.getTime() + 60 * 1000); + await occurrenceRepo.fail(claimed, { message: 'audit unavailable after save' }, retryAt); + expect(await occurrenceRepo.getMaterializationReceipt(virtual.id, scheduledFor)).toMatchObject({ + status: 'failed', + snapshot_modified: materialized.modified, + }); + await dynamicRepo.deleteSnapshot(virtual.id, materialized.modified); + await task.reconcileSchedules(retryAt); + expect(await snapshotCount(virtual.id)).toBe(1); + expect(await occurrenceRepo.getMaterializationReceipt(virtual.id, scheduledFor)).toMatchObject({ + status: 'completed', + attempt_count: 2, + snapshot_modified: materialized.modified, + }); + }); + it('does not schedule or materialize manual tracks', async function () { const component = await createComponent(); const virtual = await createVirtual(component.id, { mode: 'manual' }); diff --git a/docs/README.md b/docs/README.md index 7953c3fb..84f3b033 100644 --- a/docs/README.md +++ b/docs/README.md @@ -33,6 +33,7 @@ Architecture, patterns, and implementation details for contributors. - [Build Information](developer/build-information.md): Build metadata provenance, runtime configuration, and frontend integration - [Data Quality Query Design](developer/data-quality-reports.md): Duplicate-report memory amplification, query optimization, and remaining response-size limits - [Data Model](developer/data-model.md): Database schema and STIX object structure +- [Allowed Values Model](developer/data-model.md#runtime-allowed-values-configuration): Schema catalog, initialization, persistence, and invalid settings - [Event Bus Architecture](developer/event-bus-architecture.md): Event-driven architecture for cross-document dependencies - [Lifecycle Hooks Guide](developer/lifecycle-hooks-guide.md): Service lifecycle hooks pattern - [Cross-Service Reads Pattern](developer/cross-service-reads-pattern.md): Cross-service communication patterns @@ -44,13 +45,11 @@ Architecture, patterns, and implementation details for contributors. ### Release Tracks (Internals) -- [Implementation Backlog](developer/TODO.md): Active release-track work and - completed implementation records -- [Frontend Handoff](developer/FRONTEND_TODO.md): Backend contract changes - requiring downstream Angular updates - [Entities](developer/release-tracks/entities.md): Database schemas and data models - [Snapshot Creation Causes](developer/release-tracks/snapshot-creation-causes.md): Persisted creation-cause enum, standard/virtual operation mapping, and historical fallback - [Sealed Content Manifests](developer/release-tracks/sealed-content-manifests.md): Why every snapshot seals its bill of materials, how the collection object is projected, and publication inheritance +- [Deletion Guardrails](developer/release-tracks/deletion-guardrails.md): Plain-language explanation of durable scheduler receipts, coordinated snapshot changes, and recoverable cleanup +- [Draft Lifecycle Plan](developer/release-tracks/draft-lifecycle-plan.md): Approved virtual draft-retention and release-time squash implementation plan - [Backref Reconciliation](developer/release-tracks/backref-reconciliation.md): How `workspace.release_tracks` backrefs stay in sync with snapshots - [Member Sync Strategies](developer/release-tracks/member-sync-strategies.md): Automatic tracking of member object revisions - [Error Handling](developer/release-tracks/error-handling.md): Error handling patterns @@ -63,6 +62,7 @@ Architecture, patterns, and implementation details for contributors. Configuration, deployment, and identity provider setup. - [Configuration](admin/configuration.md): Complete configuration guide (environment variables, JSON files) +- [Allowed Values Administration](admin/configuration.md#allowed-values): Initial values, ADM versions and upgrades, adding choices, and invalid settings - [Automation Run Audit Trail](admin/automation-runs.md): How to inspect migration and scheduler audit records - [Virtual Track Schedules](admin/virtual-track-schedules.md): UTC execution, restart recovery, retries, and observability - [Release-Track Membership Reconciliation](admin/release-track-reconciliation.md): Inspect and repair durable object-backref protection failures diff --git a/docs/admin/configuration.md b/docs/admin/configuration.md index 4fa6fede..f9dafe7c 100644 --- a/docs/admin/configuration.md +++ b/docs/admin/configuration.md @@ -29,6 +29,8 @@ JSON configuration files, or a combination of both. - [Validation](#validation) - [Collection Indexes](#collection-indexes) - [Configuration Files](#configuration-files) + - [Allowed Values](#allowed-values) + - [Static Marking Definitions](#static-marking-definitions) - [ATT\&CK Specific](#attck-specific) - [Additional Resources](#additional-resources) @@ -378,8 +380,8 @@ JSON_CONFIG_PATH=./config.json See sample configurations: -- [collection-manager-oidc-keycloak.json](../resources/sample-configurations/collection-manager-oidc-keycloak.json) -- [collection-manager-oidc-okta.json](../resources/sample-configurations/collection-manager-oidc-okta.json) +- [collection-manager-oidc-keycloak.json](../../resources/sample-configurations/collection-manager-oidc-keycloak.json) +- [collection-manager-oidc-okta.json](../../resources/sample-configurations/collection-manager-oidc-okta.json) #### Challenge API Key @@ -430,7 +432,7 @@ JSON_CONFIG_PATH=./config.json } ``` -See sample: [test-service-challenge-apikey.json](../resources/sample-configurations/test-service-challenge-apikey.json) +See sample: [test-service-challenge-apikey.json](../../resources/sample-configurations/test-service-challenge-apikey.json) #### Basic API Key @@ -477,7 +479,7 @@ JSON_CONFIG_PATH=./config.json } ``` -See sample: [navigator-basic-apikey.json](../resources/sample-configurations/navigator-basic-apikey.json) +See sample: [navigator-basic-apikey.json](../../resources/sample-configurations/navigator-basic-apikey.json) #### Multiple Service Authentication Methods @@ -519,7 +521,7 @@ You can enable multiple service authentication methods simultaneously: } ``` -See sample: [multiple-apikey-services.json](../resources/sample-configurations/multiple-apikey-services.json) +See sample: [multiple-apikey-services.json](../../resources/sample-configurations/multiple-apikey-services.json) ### Scheduler @@ -617,15 +619,154 @@ Paths to additional configuration and data files. | Option | Environment Variable | JSON Path | Type | Default | Description | |-----------------------------------|--------------------------------------|-----------------------------------------------------|--------|--------------------------------------------------|--------------------------------------------------| | JSON Config Path | `JSON_CONFIG_PATH` | `configurationFiles.jsonConfigFile` | string | *(empty)* | Path to JSON configuration file | -| Allowed Values Path | `ALLOWED_VALUES_PATH` | `configurationFiles.allowedValues` | string | `./app/config/allowed-values.json` | Path to allowed values configuration | +| Allowed Values Path | `ALLOWED_VALUES_PATH` | `configurationFiles.allowedValues` | string | `./app/config/allowed-values.json` | Seed values used when Allowed Values configuration does not yet exist | | Static Marking Definitions Path | `WB_REST_STATIC_MARKING_DEFS_PATH` | `configurationFiles.staticMarkingDefinitionsPath` | string | `./app/lib/default-static-marking-definitions/` | Directory containing static marking definitions | -**Allowed Values:** +#### Allowed Values -The allowed values file defines valid enum values for STIX object properties (platforms, permissions, etc.). -See `app/config/allowed-values.json` for the schema. +Allowed Values controls which choices appear in object editors. Administrators +manage one configuration per property/domain at **Dashboard → Admin → Allowed +Values**, below **Validation Bypasses**. -**Static Marking Definitions:** +The three sources have different responsibilities: + +| Source | Responsibility | +| --- | --- | +| Installed backend ADM package | Determines permissible values and object-type/domain constraints through its Zod schemas | +| Bundled `app/config/allowed-values.json` | Registers supported object/property pairs, supplies initial groups and values, and provides suggestions for formatted fields and the dropdown response structure | +| MongoDB | Stores the administrator's configured values and enabled states | + +##### Initialization and the JSON file + +When the backend starts and no Allowed Values configuration exists in MongoDB, +it reads the file selected by `ALLOWED_VALUES_PATH` (the bundled file by default). +It validates the complete seed against ADM before inserting it, with all seeded +values enabled. An invalid seed fails initialization without a partial insert. + +If configuration already exists, startup preserves it: additions, disabled +values, and removals are not reset from the seed. Older configuration documents +receive missing rule keys without restoring removed values. + +**The bundled JSON file is still required after initialization.** The backend +loads it for supported-property registration, initial rule keys, formatted-value +suggestions, and the nested dropdown response structure and ordering. +`ALLOWED_VALUES_PATH` replaces the source of initial values; it does not replace +that bundled registration. Editing seed values does not update an existing +database, but editing the bundled object/property structure can change what +Workbench exposes. Use the admin UI or API for runtime configuration. + +##### Managing a configuration + +**Add New Property** guides the administrator through property, domain and object +type, permitted values, and review. If the property/domain already exists, open +its editor instead of creating a duplicate. This configures a supported field; +it does not define a new STIX schema property. + +In **Edit values**, select one object type, then use the searchable checklist: + +- Check a value to enable it; uncheck it to retain it as disabled. +- Remove a value to clear its setting for that object type. +- Switch object types without discarding the other types' settings. +- Save the complete draft or cancel without changing configuration. + +For data-source/component names, enter the two names separately and choose +**Validate and add**. The value enters the draft only after ADM accepts its +format. Pending input must be validated or cleared before saving. + +An empty configuration persists and offers no choices. These operations do not +rewrite existing ATT&CK objects or change their validation requirements. + +##### Adding values + +A value does not generally need to be added to both ADM and the JSON file: + +| Situation | ADM change | JSON change | Operator action | +| --- | --- | --- | --- | +| ADM already permits the value, but it is not configured or enabled | None | None | Enable it in the relevant property/domain and object-type scope, then save | +| The value is a new enum member that ADM rejects | Required | Only if it should be a default for new configurations | Upgrade the backend ADM package, then enable the value through the UI | +| A new data-source/component string satisfies ADM's existing format validator | None | None | Use Validate and add, then save | + +For a new enum member: + +1. Add support to the appropriate ADM schema, including any scope constraints, + and publish a package release. +2. Update the backend's dependency and lockfile to resolve that release. +3. Rebuild and redeploy the backend, or restart it after installing the updated + dependency in a source deployment. +4. Reload the Allowed Values page and confirm its **Backend ADM** version. +5. Enable the newly available value for the intended scope and save. + +Changing ADM's GitHub source alone does not change a running Workbench. New enum +members without saved settings are not enabled automatically. The Allowed +Values UI reads the server catalog, so it does not need a matching frontend ADM +dependency upgrade for this workflow. + +Add a value to the JSON seed only when it should start enabled in a newly +initialized configuration. That edit does not update existing installations. +Adding an unsupported property/object-type pair is a separate Workbench code +and registration change, not an operator value addition. + +##### ADM version and enforcement + +`GET /api/config/allowed-values/catalog` returns `admVersion` from the installed +backend `@mitre-attack/attack-data-model` package's `package.json`. It is not the +dependency range, the frontend's ADM version, or the ATT&CK specification +version. The page and dialogs display it alongside choices derived from that +same installed package. + +Both enabled and disabled configuration values must pass ADM validation. +General validation switches and Validation Bypasses cannot exempt them. +Enum choices come from ADM; data-source strings use its format validator. +Software choices must be valid for both tool and malware objects. + +##### Legacy invalid settings + +A stored setting may be invalid under the current ADM because an earlier +implementation accepted it or an ADM upgrade changed the schema. +*Quarantine* means retaining that setting while excluding it from use: + +- The backend evaluates stored settings on reads; there is no quarantine + collection or persisted quarantine flag. +- Invalid settings never populate dropdowns, even if stored as enabled. +- The admin response lists them in `invalidValues`, with rejection reasons. +- The editor warns that saving replaces the configuration without those + invalid settings. Merely opening the editor does not delete them. + +Startup does not reject an existing configuration because it contains legacy +invalid settings. If a later ADM version accepts a retained setting again, its +stored enabled state applies; quarantine is not a permanent disable operation. +None of this changes existing STIX objects. + +##### API + +All management endpoints require an administrator. The dropdown endpoint retains +visitor-or-higher and read-only service access. + +| Method and path | Purpose | +| --- | --- | +| `GET /api/config/allowed-values` | Enabled, ADM-compliant choices in the existing nested object/property/domain format, including empty scopes | +| `GET /api/config/allowed-values/catalog` | Backend ADM version, supported scopes, choices, validation kind, and descriptions | +| `GET /api/config/allowed-values/rules` | Configured groups, compliant `values`, and legacy `invalidValues` warnings | +| `POST /api/config/allowed-values/rules` | Create `{ propertyName, domainName, values }`; returns `201` | +| `PUT /api/config/allowed-values/rules/{propertyName}/{domainName}` | Replace a configured group's complete `{ values }` set | +| `POST /api/config/allowed-values/validate` | Check `{ propertyName, domainName, objectTypes, value }` without saving; return the trimmed `{ value }` | + +Each option in `values` has `{ value, enabled, objectTypes }`: a nonempty trimmed +string, a boolean, and distinct supported object types. Values are case-sensitive. +The same value may have different enabled states in disjoint object-type scopes. + +PUT replaces the entire group, so retain every option and scope you intend to +keep. Creating or saving a rule validates its values independently of +`/validate`. Invalid values, scopes, or bodies return `400`; a supported but +unconfigured PUT target returns `404`; duplicate creation or overlapping +scope/value entries return `409`. Validation failure leaves configuration +unchanged. Saves to different groups do not overwrite each other. + +Object editors fetch current choices when opened. The legacy collection-bundle +ICS data-source filter also uses this configuration, so changing its choices +can affect that export projection. + +#### Static Marking Definitions Directory containing JSON files with STIX marking definitions that are automatically loaded into the system on startup. @@ -659,5 +800,5 @@ ATT&CK-specific configuration values. ## Additional Resources - [Authentication Documentation](./authentication/README.md) -- [Sample Configurations](../resources/sample-configurations/) -- [Template Environment File](../template.env) +- [Sample Configurations](../../resources/sample-configurations/) +- [Template Environment File](../../template.env) diff --git a/docs/admin/release-track-audit.md b/docs/admin/release-track-audit.md index 6e754a8f..a87289c2 100644 --- a/docs/admin/release-track-audit.md +++ b/docs/admin/release-track-audit.md @@ -1,10 +1,13 @@ # Release-Track Destructive Audit Events -Workbench stores administrator-initiated destructive attempts in -`releaseTrackAuditEvents`: full-track deletion (`delete_track`), rollback of a -track's most recent release (`convert_release_to_draft`), and release-version correction -(`retag_release`). The collection is empty until an administrator performs one -of those actions. +Workbench stores destructive attempts in `releaseTrackAuditEvents`: full-track +deletion (`delete_track`), release conversion (`convert_release_to_draft`), +version correction (`retag_release`), virtual draft retention (`draft_retention`), +and opt-in release-time draft squash (`draft_squash`). Recurring retention records +a system actor under the saved cron policy; one-shot retention and explicit +destructive actions record the authenticated administrator. Retention intents +record their source, applied threshold and original cutoff. Legacy unscoped +intents can repair storage but cannot select additional drafts. Older `delete_release` events retain their historical meaning. New snapshot DELETE requests reject tagged releases; conversion and draft deletion are @@ -50,8 +53,20 @@ db.releaseTrackAuditEvents ``` A `pending` event can mean the process stopped after the audit insert or the -track was deleted but the final audit update failed. Confirm whether the track -still exists before retrying. +operation committed before final audit/progress recording. Do not repeat a tag +to repair draft cleanup. Inspect +`GET /api/release-tracks/:id/virtual/draft-cleanup`; administrators can resume an +existing operation with +`POST /api/release-tracks/:id/virtual/draft-cleanup/:operationId/retry`. + +Cleanup events retain bounded selectors, the original release-event identity +where relevant, and a bounded write-ahead batch. Missing snapshots from a +partially processed batch are repaired idempotently. A retry never expands the +approved interval or attaches an old intent to a rollback/re-tagged release. +`release_committed: true` means the release exists even if cleanup failed; an +omitted outcome means a store failure prevented establishing it. + +The canonical safety explanation is [Deletion Guardrails](../developer/release-tracks/deletion-guardrails.md). These records have no automatic TTL. Establish retention and archive policy according to local audit requirements. diff --git a/docs/admin/virtual-track-schedules.md b/docs/admin/virtual-track-schedules.md index a184c35a..23cbd95d 100644 --- a/docs/admin/virtual-track-schedules.md +++ b/docs/admin/virtual-track-schedules.md @@ -26,20 +26,50 @@ Editors can replace the active schedule through immediately in track and Workbench-format snapshot responses; executable jobs are refreshed on the next `VIRTUAL_TRACK_SCHEDULES_CRON` reconciliation pass. +Recurring schedules may include administrator-managed retention: + +```json +{ + "mode": "cron", + "cron": "0 * * * *", + "draft_retention": { "max_drafts": 10 } +} +``` + +Only actual scheduler cron execution uses this persistent policy. Explicit +materialization uses only its optional request-scoped policy, and date schedules +never inherit recurring retention. Client-supplied occurrence metadata is not +authority to apply the saved policy. Manual/dates schedule payloads reject +retention fields. Missing/null limits disable cleanup. + +Editors can change timing while preserving the existing cron policy or switch +away from recurring mode; changing a cron retention limit requires an +administrator. Schedule-only updates create no snapshot and delete nothing +immediately. The former global policy is inert and must be configured explicitly +as recurring retention if desired. + ## Idempotency and multiple instances The `virtualTrackScheduleOccurrences` collection stores one durable occurrence per track and UTC timestamp. Workers atomically claim pending or retryable -occurrences. The resulting snapshot also records +occurrences with an ownership token. The resulting snapshot also records `scheduled_materialization.scheduled_for` under a unique track-local index. -Together, these controls prevent duplicate drafts across restarts, retry -delivery, and multiple scheduler-enabled API instances. - -If a worker persists the scheduled snapshot but exits before marking the -occurrence complete, the next worker treats that snapshot as the authoritative -result. It completes the occurrence from the persisted snapshot without -recomputing composition. The recovery attempt is audited as an unchanged -`recover_scheduled_virtual_snapshot` item with `counts.recovered: 1`. +The occurrence's monotonic `snapshot_modified` receipt survives snapshot cleanup; +it must not be TTL-expired while the track remains in use. + +If a worker persists a snapshot but exits before marking the occurrence complete, +the next worker recovers the existing snapshot or its receipt. A receipt whose +snapshot has been deleted means the occurrence already materialized and was +subsequently removed; it never authorizes recomputing composition. API-originated +scheduled metadata and legacy snapshots receive the same receipt protection +before deletion. Explicit attempts to recreate removed occurrences return `409`. + +Recovery is audited as `recover_scheduled_virtual_snapshot`, including +`materialized_and_removed` when no snapshot remains. Claim ownership prevents +stale worker failure/completion from reopening another worker's completed run. + +For why scheduled execution evidence must outlive retained snapshots, see +[Deletion Guardrails](../developer/release-tracks/deletion-guardrails.md). ## Failures and retries diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md deleted file mode 100644 index d78a481c..00000000 --- a/docs/developer/FRONTEND_TODO.md +++ /dev/null @@ -1,1049 +0,0 @@ -# Release Tracks Frontend TODO - -This developer handoff tracks backend release-track changes that require -corresponding work in the Angular frontend. It is intentionally task-oriented, -but each task also explains why the change matters so that it can be -implemented without reconstructing the backend design history. - -The current server contract is defined by: - -- [Release-track OpenAPI paths](../../app/api/definitions/paths/release-tracks-paths.yml) -- [Release-track API reference](../user/release-tracks/api-reference.md) -- [Virtual-track guide](../user/release-tracks/virtual-tracks.md) -- [Bruno release-track requests](https://gitlab.mitre.org/attack-strategy/bruno/-/tree/main/workbench/Release%20Tracks?ref_type=heads) -- [`internalattack` Python client](https://gitlab.mitre.org/attack-strategy/internalattack-python) - -## Endpoint conventions - -Keep these rules in mind while updating the connector: - -- Operations shared by standard and virtual tracks do not include a type - namespace. Snapshot retrieval and release operations are shared. -- New virtual-only operations include `/virtual/` in the path. -- The current OpenAPI document is authoritative. Some older standard-only - workflow routes, such as `/candidates` and `/staged`, predate the namespace - convention and do not currently include `/standard/`. -- A release preview is a read-only `GET`. A release commit is a `POST`. - -## P0 — Send canonical domains for domain-bearing content - -### [ ] Require `x_mitre_domains` in affected reviewed-object forms - -The backend no longer suppresses the ATT&CK Data Model error for a missing -`x_mitre_domains` property on campaigns, intrusion sets, detection strategies, -or matrices. Existing latest domainless content is repaired automatically at -server startup, including revoked and deprecated lineages, but new reviewed -revisions must carry their own canonical domain membership. - -Update the affected Angular create/edit payloads so the field contains the -object's complete domain union: - -```ts -x_mitre_domains: Array<'enterprise-attack' | 'ics-attack' | 'mobile-attack'>; -``` - -Do not reduce a cross-domain object to the currently selected screen or bundle -domain. For example, one object used by Enterprise and Mobile should persist -`['enterprise-attack', 'mobile-attack']`; both virtual domain filters will -include that same exact revision by set intersection. - -Workbench still permits incomplete `work-in-progress` objects under the -existing partial-ADM workflow contract. Before a form advances an affected -object to `awaiting-review` or `reviewed`, require at least one domain and -surface the backend's `x_mitre_domains` validation detail if it is missing. - -Done when: - -- Campaign, group, detection-strategy, and matrix form models expose canonical - domain selection. -- Reviewed create and new-revision payloads always include a nonempty domain - array. -- Multi-select state preserves every selected domain instead of choosing one - based on route context. -- Validation errors for `x_mitre_domains` are displayed next to the domain - control. -- Tests cover a cross-domain payload and rejection of a reviewed domainless - payload. - -## P0 — Model draft revision selectors separately from released member pins - -### [ ] Preserve `"latest"` in candidate and staged frontend state - -Candidate and staged entries no longer always contain an ISO timestamp. -Their `object_modified` field is a revision selector: - -```ts -type WorkflowRevisionSelector = string | 'latest'; - -interface CandidateOrStagedEntry { - object_ref: string; - object_modified: WorkflowRevisionSelector; -} -``` - -Here, `string` should be validated as an ISO timestamp when it is not the -literal `"latest"`. Member and quarantine models should remain stricter: -their `object_modified` value is always an exact ISO timestamp. - -When `POST /api/release-tracks/:id/candidates` omits `modified` or sends -`"latest"`, the response preserves `"latest"` instead of replacing it with -the current timestamp. Promotion to staged preserves that selector. The UI -should render it as a moving/latest reference and must not parse it as a date. -An explicitly supplied timestamp remains an exact pin. - -Release preview is the freezing boundary. Before a standard release preview is -rendered, the backend resolves every staged `"latest"` selector. Therefore, -`format=workbench` shows exact timestamps in the would-be `members`, and a -committed release always stores exact member revisions. Preview and commit are -separate resolutions; if an object changes between them, the committed member -may legitimately be newer than the previewed one. - -Done when: - -- Candidate and staged DTOs accept either an ISO timestamp or `"latest"`. -- Member and quarantine DTOs accept exact timestamps only. -- Candidate/staged views display a useful “latest” label without date parsing - errors. -- Add-candidate flows omit `modified` or send `"latest"` when the operator - chooses a moving reference, and send an ISO timestamp for an exact pin. -- Candidate-version updates and staged demotions can send `"latest"` as their - selector. -- Release-preview fixtures show dynamic staged input becoming exact - would-be members, and committed-release fixtures contain no dynamic members. - -## P0 — Surface fail-closed primary revision errors - -### [ ] Explain missing primary revisions instead of showing a generic failure - -The backend now verifies every exact `(object_ref, object_modified)` primary -reference at request ingress and again before it releases, clones, -materializes, or renders a snapshot. It no longer omits objects that could not -be hydrated. - -Two structured error cases are relevant to the UI: - -```ts -interface MissingPrimaryRevisions { - message: string; - missing_references: Array<{ - object_ref: string; - object_modified: string; - }>; -} -``` - -- HTTP `400` means the current request selected a revision that does not - exist. Candidate add/version-update flows should keep the dialog open, - identify the missing selections, and let the operator correct them. -- HTTP `409` means an existing draft or snapshot contains a dangling primary - reference. Snapshot retrieval, release preview/commit, cloning, virtual - materialization/quarantine promotion, and bundle export can return this - response. The UI should identify the affected revisions and explain that an - operator must repair the track/object data before continuing. - -Do not render a partial Workbench snapshot or treat a failed bundle request as -an empty export. - -Done when: - -- The release-track connector exposes `missing_references` on `400` and `409` - responses instead of flattening the response to a generic message. -- Candidate forms keep their input state after a `400` and highlight the - missing revisions. -- Snapshot, release, clone, virtual-materialization, and export views present - an actionable integrity error for `409`. -- Tests cover multiple missing references and prove no partial snapshot or - bundle is rendered. - -### [ ] Explain immutable-revision and snapshot-graph deletion conflicts - -Persisted STIX revisions are globally immutable. Any STIX-changing PUT returns -`409 Conflict` and should direct the operator to POST a new revision. Separately, -a tagged snapshot's opt-in graph manifest holds exact revision pointers. If an -object revision is a protected dependency of an active or linked-pending -manifest, exact-revision or full-lineage DELETE returns `409 Conflict`: - -```ts -{ - message: string; - details?: string; - snapshot_graph_pins: Array<{ - track_id: string; - snapshot_modified: string; - kind: 'root' | 'relationship' | 'secondary' | 'supporting' | 'link_target'; - tier?: 'members' | 'staged' | 'candidates' | 'quarantine'; - }>; -} -``` - -This can occur from ordinary object-management screens, not only from the -release-track UI. Present it as a versioning constraint: the operator should -create a new object revision, or remove the draft snapshots that no longer -need the old revision. Do not offer a force-delete path; administrator -authorization does not bypass graph integrity. - -Candidate and staged workspace metadata remains editable, but STIX content does -not. Relationship corrections are always new POST revisions; schema-v2 graph -manifests point to the exact older relationship revision and never rely on a -frozen clone. Marking definitions remain the narrow frozen-payload exception -because they are unversioned. - -Done when: - -- Shared object edit/delete error handling recognizes - `snapshot_graph_pins`. -- The message identifies the affected release track(s) and recommends a new - revision instead of a blind retry. -- The UI does not expose a force-delete action for graph-protected revisions. - -### [ ] Handle persisted mutations whose membership reconciliation failed - -A release-track mutation can persist its snapshot before a downstream object -backref write fails. The server now returns HTTP `500` instead of reporting -success and includes: - -```ts -{ - message: 'Release-track membership protection could not be reconciled'; - track_id: string; - reconciliation_id: string; - details?: string; -} -``` - -For a release request, the snapshot may already be tagged. Do not -automatically retry the POST: refresh snapshot history first, show the -reconciliation ID, and direct the operator to an administrator if protection -repair is still pending. - -Done when: - -- The connector preserves `track_id` and `reconciliation_id` from this `500`. -- Release and mutation dialogs explain that persistence may have succeeded - and do not offer a blind retry. -- The UI refreshes the relevant track before enabling another action. - -## P0 — Align the Angular connector with the current routes - -### [x] Remove direct snapshot mutation controls and client methods - -Persisted snapshot history is now immutable. The backend no longer exposes: - -```text -POST /api/release-tracks/:id/contents -POST /api/release-tracks/:id/snapshots/:modified/contents -POST /api/release-tracks/:id/snapshots/:modified/meta -``` - -Remove the corresponding connector methods, payload types, dialogs, buttons, -and tests. Standard-track content should move through candidates, staged, and -release. Virtual content should move through composition materialization and -quarantine resolution. Metadata can be changed only from the latest snapshot -via `POST /api/release-tracks/:id/meta`, which creates a new draft. - -Do not replace removed historical-edit actions with hidden calls or local -state edits. If an operator wants a different result, they should correct the -latest draft, delete it while deletion is still allowed, or create a newer -draft. - -Done when: - -- No Angular code calls or models any of the three removed routes. -- Snapshot history views are read-only except for supported release, clone, - and latest-draft deletion actions. -- Standard and virtual editors direct users to their respective supported - workflows. - -Completed 2026-07-30: the Angular connector methods, payload type, and -regression fixtures were removed. No component or menu called these methods, -so no UI control needed to be migrated. - -### [ ] Offer deletion only for the latest untagged draft - -`DELETE /api/release-tracks/:id/snapshots/:modified` is a narrow “undo latest -draft” operation. The server accepts it only when the selected snapshot is both -untagged and currently latest. Tagged releases and older drafts return `409` -because they are immutable history. - -In snapshot history, show Delete only on the latest item when `version == null`. -After a successful delete, refresh both the latest snapshot and the history; -the preceding snapshot becomes current. If a `409` occurs because another -operation created a newer draft, refresh instead of retrying the stale delete. - -Done when: - -- Tagged and historical rows never offer Delete. -- The confirmation explains that the track will revert to the preceding - snapshot. -- A stale `409` refreshes the view and preserves history. - -### [ ] Add administrator confirmation for full track deletion - -Full track deletion is administrator-only and requires the query parameter -`confirm_track_id` to exactly equal the `:id` path parameter: - -```text -DELETE /api/release-tracks/:id?confirm_track_id=:id -``` - -Do not expose this action to editors or team leads. Before sending the request, -show the track name and ID, explain that deletion removes all history, and -require an explicit confirmation interaction. A missing or stale ID returns -`400`; a non-administrator returns `401`. - -Done when: - -- Route guards and action visibility match the documented authorization - matrix. -- The connector sends the selected track's exact ID as `confirm_track_id`. -- Dialogs cannot reuse confirmation state after the selected track changes. -- Tests cover administrator success plus editor, missing-confirmation, and - mismatched-confirmation rejection. - -### [ ] Use only the explicit snapshot-retrieval endpoints - -The release-track resource path no longer doubles as an implicit request for -the latest snapshot. `GET /api/release-tracks/:id` was removed before the -feature was officially released, so there is no compatibility alias. - -Update the frontend to use: - -```text -GET /api/release-tracks/:id/snapshots -GET /api/release-tracks/:id/snapshots/latest -GET /api/release-tracks/:id/snapshots/:modified -``` - -Required work: - -- Keep `getLatestSnapshot()` on `/snapshots/latest`. -- Update the integration test that still calls `GET /release-tracks/:id`. -- Remove any fallback that interprets a full snapshot or `version_history` as - the snapshot-list response. -- Remove the unsupported `releases=only` option from track-list and - snapshot-retrieval types. Tagged-state filtering now belongs on the snapshot - history endpoint as `tagged=true`. - -Done when: - -- No Angular code calls `GET /api/release-tracks/:id`. -- Connector and integration tests assert the three explicit retrieval paths. - -### [ ] Move virtual-only operations under `/virtual/` - -Virtual composition and materialization are now visibly scoped in the URL: - -```text -PUT /api/release-tracks/:id/virtual/composition -POST /api/release-tracks/:id/virtual/snapshots/create -``` - -The Angular connector still calls the older `/composition` and -`/snapshots/create` paths. Update those paths and their tests. - -Also remove: - -```text -GET /api/release-tracks/:id/snapshots/preview -``` - -That endpoint no longer exists. It recomputed a hypothetical virtual -composition without persisting it, which overlapped confusingly with release -preview. Creating a virtual draft is now an explicit operation. The current -`onDraft()` flow should therefore open a confirmation dialog and call the -create endpoint directly instead of first calling `previewVirtualSnapshot()`. - -Done when: - -- `updateComposition()` calls `/virtual/composition`. -- `createVirtualSnapshot()` calls `/virtual/snapshots/create`. -- `previewVirtualSnapshot()` and its UI/test fixtures are removed. -- Creating a virtual draft still asks for confirmation, but does not depend on - a nonexistent preview payload. - -## P0 — Replace “bump” with the release contract - -### [ ] Rename bump-oriented frontend symbols and user-facing text - -“Release” is now the operation name throughout the API. The server has no -`/bump` routes, and retaining bump terminology in Angular makes logs, types, -tests, and UI copy disagree with the public contract. - -Suggested renames: - -```text -BumpPayload -> ReleasePayload -previewBump() -> previewRelease() -bumpByLatest() -> releaseLatest() -bumpByModified() -> releaseSnapshot() -bumpRelease() -> releaseSnapshot() -``` - -Update comments, test names, log messages, and errors such as “Failed to -preview release track bump” at the same time. - -Done when: - -- “bump” is absent from the release-track connector, models, components, and - tests unless it appears in a historical explanation. -- Angular method names distinguish previewing from committing a release. - -### [ ] Replace the old release request body - -The current frontend `BumpPayload` is obsolete: - -```ts -{ - type?: 'major' | 'minor'; - version?: string; - dry_run?: boolean; -} -``` - -The release body is now: - -```ts -{ - increment?: 'major' | 'minor'; - version?: string; // exact MAJOR.MINOR, for example "14.1" -} -``` - -The rules are: - -- `increment` and `version` are mutually exclusive. -- Supplying both returns `400 Bad Request`. -- Omitting both asks the server for the default minor increment. -- `dry_run` was removed. Use a release-preview representation instead. -- `expected_snapshot_modified` is not required. Choosing `latest` means the - caller accepts whichever snapshot is latest when the server handles the - request; choosing `:modified` explicitly pins the target. - -Use the same selector in the preview query and the release request body. For -example: - -```text -GET .../release/preview?format=summary&increment=major -POST .../release -Body: { "increment": "major" } -``` - -Done when: - -- Angular never sends `type`, `dry_run`, or - `expected_snapshot_modified` in a release request. -- The UI can select `major`, `minor`, or an explicitly entered `MAJOR.MINOR` - version and validates mutual exclusivity before calling the server. - -## P0 — Correct the release-preview flow - -### [ ] Treat release preview as a `GET` with representation-specific output - -Both track types use the same preview routes: - -```text -GET /api/release-tracks/:id/snapshots/latest/release/preview -GET /api/release-tracks/:id/snapshots/:modified/release/preview -``` - -The supported preview formats are: - -- `summary` — a before/after delta; this is the default -- `workbench` — the complete snapshot that would be persisted -- `bundle` — the publication-ready STIX bundle -- `filesystemstore` — reserved but currently returns `501 Not Implemented` - -The existing Angular flow requests `format=workbench` and then reads summary -fields such as `next_version_minor` and `staged_count`. Those are different -representations and must not be mixed. - -Recommended interaction: - -1. Ask the operator to choose `major`, `minor`, or an exact version. -2. Request `format=summary` using that selector. -3. Render the returned `version`, `before`, `after`, `changes`, and - `conflicts`. -4. Optionally let the operator inspect `format=workbench` or `format=bundle` - using the same selector. -5. Commit the release using the same selector only after confirmation. - -The summary contains the planned `version`; it does not return separate -`next_version_minor` and `next_version_major` fields. - -Done when: - -- The confirmation dialog is driven by a `summary` response. -- Workbench and bundle previews are treated as literal payloads, not deltas. -- The preview and commit always use the same target snapshot and version - selector. - -### [ ] Render standard and virtual summaries differently - -Standard and virtual snapshots share the preview endpoint, but their deltas -answer different questions. - -For a standard track: - -- `before` is the selected draft. -- `after` is the would-be release after staged objects become members. -- Counts are oriented around `members_count`, `staged_count`, and - `candidates_count`. - -For a virtual track: - -- `before` is the tagged release immediately preceding the selected draft. -- `after` is the selected, already-materialized virtual draft. -- Counts are oriented around `members_count` and `quarantine_count`. -- `previous_release` identifies the comparison baseline when one exists. -- `changes` can include `new_count`, `updated_count`, `removed_count`, and - `quarantined_count`. - -Virtual release does not promote staged objects because virtual tracks do not -have a staged tier. Avoid showing zero-valued staged/candidate statistics as -though they described the virtual workflow. - -Done when: - -- Summary rendering branches on the top-level `type`. -- Standard previews explain staged-to-member promotion. -- Virtual previews explain the delta from the preceding tagged release. -- A first virtual release handles `previous_release: null` cleanly. - -## P0 — Represent virtual materialization state - -### [ ] Show when a virtual draft is awaiting materialization - -Saving a new virtual composition now invalidates the preceding materialized -contents. The new latest draft intentionally has: - -```json -{ - "type": "virtual", - "members": [], - "quarantine": [], - "composition_resolution": null -} -``` - -This does not mean that the composition resolved to an empty release. It means -the composition has changed and the operator must explicitly materialize a new -draft with: - -```text -POST /api/release-tracks/:id/virtual/snapshots/create -``` - -Until that succeeds, shared release preview and release commit endpoints return -`409 Conflict`. - -Required work: - -- Treat `type === 'virtual' && composition_resolution == null` as “awaiting - materialization.” -- Show an explanatory state rather than an ordinary empty-members view. -- Disable release actions and make “Create Draft” the primary next action. -- After a composition update, refresh both the latest snapshot and history so - the newly invalidated draft is visible. -- Surface a helpful error when a component track has no tagged snapshots. A - virtual materialization can only consume tagged `members` from its component - standard tracks. - -Done when: - -- Operators cannot accidentally interpret an unmaterialized draft as a valid - empty virtual release. -- A `409` from preview/release explains that materialization is required - instead of being swallowed as a null preview. - -### [ ] Keep standard workflow controls out of virtual tracks - -Virtual membership has one authority: composition materialization followed by -optional quarantine resolution. Hide candidate and staged controls when -`type === 'virtual'`; keep them available for standard tracks on their current -routes. - -Done when: - -- A virtual-track screen does not offer actions the server will reject because - they belong to the standard workflow. - -## P0 — Consume snapshot history as summaries - -### [ ] Type and paginate the snapshot-history response - -`GET /api/release-tracks/:id/snapshots` returns a paginated envelope: - -```json -{ - "data": [], - "pagination": { - "total": 0, - "limit": 50, - "offset": 0 - } -} -``` - -Supported query parameters are: - -- `tagged=true` — tagged releases only -- `tagged=false` — untagged drafts only -- omit `tagged` — both; this is the no-filter default -- `limit` — 1 through 200, default 50 -- `offset` — zero or greater - -The current connector discards `pagination`, accepts no filters, and contains -fallback normalization for older response shapes. Replace that compatibility -logic with the explicit envelope. Otherwise tracks with more than 50 -snapshots are silently truncated. - -Done when: - -- The connector accepts `tagged`, `limit`, and `offset`. -- The component retains pagination metadata and supports paging or loading - more results. -- There is a clear UI control for all snapshots, releases only, and drafts - only. - -### [ ] Use the type-oriented snapshot summary fields - -Snapshot history entries are lightweight summaries, not full snapshots. All -entries include: - -```text -id, type, modified, version, name, description, members_count -``` - -Standard entries additionally include: - -```text -staged_count, candidates_count -``` - -Virtual entries additionally include: - -```text -quarantine_count -``` - -These counts are top-level fields. They are not nested beneath `summary`, and -the history endpoint does not return the tier arrays needed to derive object -deltas. The current `buildSnapshotHistory()` logic therefore reports zero for -several values and should not calculate “Added” or “Modified” by comparing -missing member arrays. - -Required work: - -- Define a discriminated union keyed by `type: 'standard' | 'virtual'`. -- Read the top-level count fields directly. -- Show standard counts as members/staged/candidates. -- Show virtual counts as members/quarantine. -- If the design still needs per-object added/updated/removed deltas, retrieve - an appropriate release summary or full snapshots explicitly rather than - inferring them from the lightweight list response. - -Done when: - -- Snapshot cards display accurate counts for both track types. -- No history calculation assumes the list response contains `members`, - `staged`, `candidates`, or `quarantine` arrays. - -## P1 — Add the virtual quarantine-resolution workflow - -### [ ] Let an operator select an exact quarantined revision - -The backend now provides: - -```text -POST /api/release-tracks/:id/virtual/quarantine/promote -``` - -Request body: - -```json -{ - "object_ref": "attack-pattern--11111111-1111-4111-8111-111111111111", - "object_modified": "2024-02-01T10:00:00Z" -} -``` - -The `(object_ref, object_modified)` pair must exactly match an entry in the -latest virtual snapshot's quarantine tier. On success, the server creates a -new draft, places the selected revision in `members`, removes every -quarantined alternative for that object, and preserves the original snapshot -for history and provenance. - -Suggested UI: - -- Group quarantine entries by `object_ref`. -- Show every conflicting revision, its modified timestamp, and any available - source/component context. -- Require the operator to choose one exact revision. -- Confirm that the other alternatives will be removed from the new draft. -- Refresh the latest snapshot and history after success. - -Error behavior: - -- `400` — malformed body or the target track is not virtual -- `404` — that exact revision is no longer quarantined in the latest snapshot - -A `404` is often a stale-screen condition. Refresh the latest snapshot and ask -the operator to review the current alternatives rather than retrying the old -selection automatically. - -Done when: - -- Each quarantined conflict has an explicit resolution action. -- The request always includes both the STIX ID and exact modified timestamp. -- Successful promotion updates members, quarantine, and snapshot history in - the UI. - -## P1 — Expose implemented virtual component filters - -### [ ] Add plural `filters.domains` to component-track editing - -Virtual composition can filter the exact revisions contributed by each -component track: - -```json -{ - "filters": { - "object_types": ["intrusion-set", "malware"], - "domains": ["enterprise", "mobile"] - } -} -``` - -Both public domain names (`enterprise`, `ics`, `mobile`) and their STIX names -(`enterprise-attack`, `ics-attack`, `mobile-attack`) are accepted. Prefer the -short public names in Angular controls for consistency. - -The key is plural: `domains`. Do not send `filters.domain`; that typo does not -enable filtering. The backend now rejects that typo with `400 Bad Request`. - -Objects without domain metadata are excluded when a domain filter is active. -This is expected behavior, not a partial match. - -Composition payloads are strict at every nested level. Do not preserve -frontend-only properties in the submitted composition, component, filter, or -deduplication objects. Component selector fields must also follow the selected -strategy: - -- `latest_tagged` sends neither `version` nor `snapshot`. -- `specific_version` sends `version` and omits `snapshot`. -- `specific_snapshot` sends `snapshot` and omits `version`. -- Every component sends a unique, non-negative integer `priority`; lower - numbers have higher priority. - -The server validates component identity during both creation and update. -Referenced tracks must already exist and must be standard tracks, and duplicate -component track IDs are rejected. Do not offer virtual tracks in a component -selector. - -Virtual tracks are purely compositional. Do not expose candidate, staged, -direct-member, or `native_members` controls for them. If operators need -aggregate-specific content, direct them to create or select a standard -component track that owns that content. - -Done when: - -- Each virtual component row can select zero or more domains. -- Saved and reloaded composition preserves `filters.domains`. -- Tests assert the plural key and multi-domain payload shape. -- Changing resolution strategy clears the selector from the previous strategy. -- Every component row requires a priority, and duplicate priorities or track - selections are blocked before submission. -- Component selectors list standard tracks only. -- Virtual-track forms never submit `native_members` or direct membership - fields. -- Submitted composition payloads contain only server-supported properties. - -## P1 — Treat virtual snapshot members as exact revision pins - -### [ ] Remove any lazy-resolution assumptions from virtual snapshot views - -Virtual composition is completed when -`POST /api/release-tracks/:id/virtual/snapshots/create` succeeds. The returned -draft directly contains `members`, `quarantine`, and -`composition_resolution`; every tier entry has an exact `object_ref` and -`object_modified` timestamp. - -Do not send a `resolve` query parameter and do not expect a -`resolved_content` response wrapper. Shared workbench retrieval returns the -persisted tier arrays directly. A component's `track_latest` policy may move -pins in newer standard-track drafts, but it cannot change a previously -materialized virtual snapshot. - -The `latest` path segment selects the newest release-track snapshot; it does -not mean “resolve every member to its latest object revision.” If the track has -not acquired another snapshot, repeated `/snapshots/latest` calls identify the -same primary revision set. - -Bundle downloads now replay the relationship/secondary graph captured when -the snapshot was created. The generated bundle-envelope ID may change, but -the object graph for a materialized virtual snapshot is stable. - -Done when: - -- Virtual views read `members` and `quarantine` directly from the snapshot. -- No connector or model exposes `resolve` or `resolved_content`. -- Member links and comparison keys use both `object_ref` and - `object_modified`. -- Tests prove that advancing a component after materialization does not change - the displayed virtual member revision. -- User-facing export guidance distinguishes a stable snapshot object graph - from the intentionally variable bundle-envelope UUID. - -## P1 — Submit mode-correct virtual snapshot schedules - -### [ ] Add conditional validation and complete the dates-mode UI - -`snapshot_schedule` is virtual-only and now has a strict discriminated -contract: - -```ts -type SnapshotSchedule = - | { mode: 'manual' } - | { mode: 'cron'; cron: string } - | { mode: 'dates'; dates: string[] }; -``` - -The modes are mutually exclusive. Do not retain hidden form values when the -mode changes: `manual` sends neither selector, `cron` sends only a valid -five-field cron expression, and `dates` sends only a nonempty array of ISO -timestamps. Standard-track payloads must omit `snapshot_schedule`. - -The current dialog already lists `dates`, but it has no date controls and -therefore submits only `{ mode: 'dates' }`, which the server rejects. The cron -control is also not conditionally required, allowing `{ mode: 'cron' }` to be -submitted. - -Automatic creation is now active when the backend scheduler is enabled. -Explain that cron and dates use UTC, cron occurrences are not backfilled after -downtime, and due dates are recovered after restart. A component-resolution -failure is retried by the backend; the UI does not need to resubmit the -schedule. - -Virtual drafts may include per-snapshot materialization metadata: - -```ts -scheduled_materialization?: { - schedule_mode: 'cron' | 'dates'; - scheduled_for: string; -}; -``` - -Clients may send this strict shape during `POST /api/release-tracks/new` for a -virtual track, `PUT /api/release-tracks/:id/virtual/composition`, and -`POST /api/release-tracks/:id/virtual/snapshots/create`. Include it only when -deliberately attaching the occurrence to the new snapshot; later snapshot -mutations do not inherit it. Standard tracks must omit it. Read it from track -listing, snapshot history, latest snapshot, or timestamp-selected snapshot -responses. - -Done when: - -- Selecting cron makes a valid cron expression required and clears dates. -- Selecting dates exposes date controls, requires at least one value, emits - ISO timestamps, and clears cron. -- Selecting manual clears both selector fields. -- Standard-track creation never sends schedule metadata. -- Tests cover all three modes and mode switching. -- User-facing copy explains UTC execution and the difference between cron and - restart-recoverable dates. -- Virtual create and composition update forms can deliberately submit - `scheduled_materialization`, and all supported GET representations tolerate - and preserve it. - -## P1 — Align virtual component object-type filters - -### [ ] Use the complete canonical Workbench STIX type vocabulary - -The backend now validates `composition.component_tracks[].filters.object_types` -against its canonical STIX type registry. When `object_types` is present, it -must be a nonempty array of unique, case-sensitive STIX type names. Omit the -property to include all types; do not send an empty array. - -The create dialog already removes the property when the user has no -selections, and `mat-select` naturally prevents duplicates. Its current -hard-coded options are only a subset of the server vocabulary, however. They -omit `identity`, `marking-definition`, `note`, `relationship`, -`x-mitre-collection`, and `x-mitre-data-source`. - -Done when: - -- Creation and composition editing use the same complete canonical option - list. -- Clearing all selections removes `object_types` from the submitted filter. -- Unknown values loaded from stale local state are rejected or removed before - submission. -- Tests cover the complete option list and clearing the filter. - -## P1 — Distinguish virtual duplicates from revision conflicts - -### [ ] Align resolution metrics and fixtures with deterministic deduplication - -Virtual materialization now reports duplicate contributions and revision -conflicts as related but different concepts: - -- `composition_resolution.deduplication.duplicates_found` counts STIX object - IDs contributed by more than one component, even when every component - supplies the same exact revision. -- `conflicts_resolved` contains only object IDs with genuinely different - `object_modified` revisions. -- An exact revision shared by multiple components produces one member and is - never quarantined. -- Each member is attributed to exactly one component. Consequently, the sum of - `component_snapshots[].objects_contributed` equals - `composition_resolution.summary.total_objects`. - -The current page already displays separate duplicate and conflict counters. -Preserve that distinction instead of assuming the two counts are equal. - -Done when: - -- Resolution fixtures include an identical revision shared across components - and a separate object with conflicting revisions. -- Duplicate and conflict counters render their respective backend fields. -- Component contribution counts add up to the resolved member total. -- Quarantine views never show repeated copies of the same exact revision. - -## P1 — Model and display immutable virtual release provenance - -### [ ] Type `component_versions` and resolve component display names - -Virtual release history entries now include: - -```ts -component_versions?: Record; -``` - -Each key is an immutable component release-track ID and each value is the -tagged component version frozen in the virtual draft's -`composition_resolution`. The map is present only for virtual releases; -standard release history entries omit it. Component display names are -deliberately not used as keys because names can change or collide. - -The existing `VersionHistoryEntry` interface currently types this property as -`any`. Replace that with `Record`. If the UI presents -provenance to operators, pair each track ID with the matching -`composition_resolution.component_snapshots[].track_name` from the same -released snapshot while retaining the ID as the authoritative identity. - -Do not fetch each component's latest release to construct this display. A -component may have advanced after virtual materialization; the embedded map is -the release's immutable provenance and must remain unchanged. - -Done when: - -- `component_versions` is strongly typed as an optional track-ID-to-version - map. -- Standard release-history fixtures omit the property. -- Virtual workbench preview and committed-release fixtures include the same - map. -- Any user-facing labels resolve names from the released snapshot's embedded - composition metadata and fall back to the track ID. -- Tests prove that a component's newer current release does not replace the - version shown for an older materialized virtual draft. - -## P1 — Separate snapshot and preview output-format types - -### [ ] Remove the invalid `snapshot` format and model `summary` correctly - -The frontend currently uses one `ExportFormat` enum for endpoints with -different contracts and includes `Snapshot = 'snapshot'`, which the server -rejects. - -Use separate types: - -```ts -type SnapshotOutputFormat = 'workbench' | 'bundle' | 'filesystemstore'; - -type ReleasePreviewFormat = 'summary' | 'workbench' | 'bundle' | 'filesystemstore'; -``` - -`filesystemstore` should remain disabled or clearly marked unavailable until -the server implementation exists. - -For workbench snapshot retrieval, `include` may select `members`, `staged`, -`candidates`, `quarantine`, or `all`. Ensure virtual inspection can request -quarantine and that the type does not restrict `include` to standard tiers. - -Done when: - -- Angular cannot send `format=snapshot`. -- `summary` is available only where the release-preview endpoint supports it. -- Virtual workbench retrieval can include quarantine. - -## P1 — Update frontend tests around the public contract - -### [ ] Replace stale fixtures and add standard/virtual lifecycle coverage - -Update connector, component, and integration tests together so old mock shapes -do not keep obsolete behavior alive. - -Minimum regression coverage: - -- Canonical latest retrieval uses `/snapshots/latest`. -- Snapshot history preserves `{ data, pagination }` and supports `tagged`. -- Release preview uses `GET`, `format=summary`, and query-based version - selection. -- Release commit uses `{ increment }` or `{ version }`. -- Latest and timestamp-selected releases use the same contract. -- Virtual composition and materialization use `/virtual/` paths. -- The removed virtual snapshot-preview method is absent. -- An unmaterialized virtual draft disables release and explains a `409`. -- Standard and virtual summary fixtures use their respective count fields. -- Quarantine promotion sends an exact object revision and handles stale `404` - responses. -- The frontend never offers standard-only contents/candidate/staged mutations - on a virtual track. - -## Deterministic bundle cache controls - -`GET /api/release-tracks/:id/snapshots` includes the opaque -`graph_manifest_id` on summaries whose tagged snapshot has a materialized -member graph. The History tab should translate that technical state into a -user-oriented bundle cache: - -- Show **Bundle cached** with a success indicator when `graph_manifest_id` is - present. -- Show **Not cached** with a warning icon otherwise. Explain that member-only - bundle exports are not guaranteed to be deterministic until cached. -- Offer **Cache bundle** only for uncached tagged snapshots and editors. It - calls `POST /api/release-tracks/:id/snapshots/:modified/graph`. -- Use the existing indeterminate Material spinner while a cache operation is - in progress. Cached snapshots offer editors a confirmed **Delete cache** - action backed by `DELETE /api/release-tracks/:id/snapshots/:modified/graph`. -- For cached snapshots, render the accompanying `graph_statistics` in a - compact **Graph cache** panel: Primary, Secondary, Relationships, and - Dependencies. Dependencies is the sum of supporting and LinkById entries; - show `total_count` as the overall cached-item count. -- Refresh snapshot history after cache creation or deletion so server-derived - state and statistics are visible immediately. -- Drafts remain uncached and must be tagged first. Candidate and staged - exports remain live even when the member graph is cached. -- Treat `graph_manifest_id` as an opaque read-only signal; never send or - persist a client-authored value. - -“Cache” is deliberately presentation language, not an implementation claim -about HTTP response caching. Tooltips should retain the member-only and -determinism qualifiers so users do not infer broader guarantees. - -## Backend changes that do not require Angular API changes - -The following changes are useful context but should not create extra connector -work: - -- Snapshot bundle exports include bounded secondary objects and their - relationships. Materialized schema-v2 graphs retain exact revision pointers - while graphless exports resolve live. A standard draft tier explicitly - stored as `"latest"` remains dynamic until release. -- Release-track object back-references are reconciled when snapshots change. - Frontend object refreshes will see the updated membership metadata without a - new endpoint. -- The server intentionally does not require - `expected_snapshot_modified`. Do not add a client-side precondition field. -- Virtual release preview and release use the same shared routes as standard - tracks after materialization; do not create separate virtual release - endpoints. diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md deleted file mode 100644 index 2be02d79..00000000 --- a/docs/developer/TODO.md +++ /dev/null @@ -1,2922 +0,0 @@ -# Release Track TODOs - -## Duplicate relationship report memory (2026-09-10) - -- [x] Trace dashboard request and identify historical endpoint fan-out. -- [x] Filter duplicates in MongoDB before hydrating relationships and latest endpoints. -- [x] Add regressions for history, lifecycle filters, missing endpoints, and bounded query output. -- [x] Run focused specs, full npm test, and lint; document findings and limits. - -Verification: focused report + relationship specs 38 passing; full `npm test` -passes (OpenAPI 2, config 22, API 1051, middleware 29, scheduler 10); lint -and `git diff --check` pass. Synthetic query output falls from 38,162,701 to -23,131 serialized bytes; production peak heap has not been measured. See -[data-quality report design](data-quality-reports.md) for the analysis. -No API contract change or frontend/Bruno update is needed. - -Proposed commit: `fix(reports): reduce duplicate relationship report memory use` - -Body: Filter duplicate groups in MongoDB before fetching full relationships, -limit endpoint lookups to their latest revisions, and consume results through -a batched cursor. Add history and lifecycle regressions and document remaining -response-size limits. - -Proposed AGENTS.md lesson: Analytical reports should filter findings before -joining full documents, and latest-endpoint lookups should limit revisions in -MongoDB rather than discarding history after materialization. - - -## Snapshot card header hierarchy - -- [x] Give snapshot identity and status labels their own full-width header area. -- [x] Move controls below metadata; remove redundant draft copy and style tagged timestamps as metadata pills. -- [x] Verify template regressions, responsive layouts, frontend suite, lint, and build. - -Accepted lifecycle work committed without a breaking marker: backend e9fd8562, -frontend 880c49e0, Bruno a9b8a97. Other provenance changes remain uncommitted. - -UI verification: component spec 87 passing; full frontend suite 168 files / -428 tests passing. Changed-file lint, formatting, and production build pass -(existing bundle/style budget warnings remain). Isolated Angular-rendered -headers with compiled styles were visually checked in headless Chrome at -1400, 900, 390, and 320px: Latest stays in the title row, controls remain below -metadata, and neither draft nor tagged header overflows. Timestamp pills use -existing MITRE theme tokens with readable light/dark foregrounds. - -Backend lifecycle assertion verification: 20 passing; full suite passes -(OpenAPI 2, config 22, API 1048, middleware 29, scheduler 10), and lint passes. -A transient content-manifest HTTP 404 passed in isolation (10 tests) and on -the complete rerun. No unrelated test-harness changes were made. - -Header refinements approved for commit on 2026-09-10: -`fix(release-tracks): clarify snapshot card headers` -Body: Separate snapshot identity and status from controls, remove redundant -draft subtitles, and present snapshot/tagging timestamps as metadata pills. - -## Separate tagging, conversion to draft, and draft deletion - -- [x] Preserve creation provenance while separating conversion and deletion in the API. -- [x] Keep admin confirmation, latest/sole-snapshot, dependency, and release-lock guards. -- [x] Add distinct frontend Convert to draft and Delete draft controls. -- [x] Update regressions, OpenAPI, user/developer docs, and Bruno requests. -- [x] Run focused tests, full backend/frontend suites, lint, and build. - -Verification (Node 24): backend focused group 76 passing; full `npm test` -passes (OpenAPI 2, config 22, API 1048, middleware 29, scheduler 10). -The first full run identified two obsolete DELETE-confirmation expectations, -which now assert draft-only rejection. Unrelated HTTP/authentication failures -passed in isolation and in the complete rerun. Backend lint passes. -Frontend focused tests 108 passing; all 168 files / 425 tests pass, changed-file -lint passes, and production build succeeds with existing size-budget warnings. -Logic-specialist review: ROBUST for the scoped state transitions, dependency -checks, preserved-source protection, publication cleanup, and release-lock use. -Both main beta checkouts retain the other agent's uncommitted provenance work. -No commits were created; Bruno conversion/deletion requests are updated. - -Accepted commit: `feat(release-tracks): separate draft conversion from snapshot deletion` - -Body: Add an explicitly confirmed release-to-draft endpoint and matching UI -control. Restore guarded draft deletion while preserving source drafts, -composition provenance, dependency checks, and release serialization. - -Snapshot DELETE is draft-only. Convert tagged releases with -POST /release-tracks/:id/snapshots/:modified/draft and a confirm_version body -before attempting a separate eligible-draft DELETE. - -## Snapshot creation provenance and user attribution (2026-09-09) - -- [x] Inspect beta and carry forward the prior uncommitted creation-cause work. -- [x] Persist immutable creation cause and invoking user, including standard release creation. -- [x] Return safe user display metadata with snapshot GETs and history; show initials avatars. -- [x] Cover user changes, automation, legacy records, tagging, and spoofing with regressions. -- [x] Update OpenAPI, user/developer documentation, and Bruno requests. -- [x] Run focused tests, full suites, lint, and frontend build. - -Verification: backend provenance/release/quarantine group 32 passing; final -provenance spec 7 passing. Complete backend suite under Node 24 passes: -OpenAPI 2, config 22, API 1044, middleware 29, scheduler 10. Backend lint -passes. First full-run transient HTTP failures passed in isolation (100) -and on the complete rerun. Frontend focused tests 97 passing; complete suite -168 files / 420 tests passing, changed-file lint and production build pass. -The existing save-dialog timing failure passed in isolation and on full rerun; -existing bundle/style budget warnings remain. Production build required -execution outside the sandbox. No test-harness changes were made. - -Work is in both main beta checkouts, preserving the newer rollback/composition -provenance work and unrelated local files. Commit preparation (2026-09-10) -excludes unrelated working-tree changes, including shared-document formatting. - -Commit messages: - -- Backend: `feat(release-tracks): record snapshot creation provenance` - Body: Persist creation causes and invoking users, distinguish standard - release creation, and expose safe creator metadata through snapshot GETs. -- Frontend: `feat(release-tracks): show snapshot causes and creator avatars` - Body: Display snapshot-local creator names and initials, with explicit - automation and historical-attribution fallbacks on Releases cards. -- Bruno: `docs(release-tracks): document snapshot creation attribution` - Body: Describe creation cause and actor fields on snapshot GET requests. - -## Merge scheduling and rollback branches into local beta (2026-09-09) - -- [x] Inspect all worktrees and confirm scheduling branches are already merged. -- [x] Merge rollback branches and preserve both features in documentation conflicts. -- [x] Run focused backend/frontend regressions, then full suites and lint/build checks. -- [x] Complete local merge commits and verify both feature tips are ancestors of beta. - -Verification: backend focused group 84 passing; full `npm test` under Node 24 -passes (OpenAPI 2, config 22, API 1037, middleware 29, scheduler 10), and -backend lint passes. One initial full-run HTTP 404 passed in isolation (23) -and in the complete rerun. Frontend focused group 100 passing; all 168 test -files / 416 tests, changed-file lint, and production build pass. The build -required execution outside the sandbox; existing bundle/style budget warnings -remain. Only documentation conflicted; both sides' behavior and records are -preserved. Scheduling branches were already ancestors of beta; uncommitted -work in the scheduling worktrees and existing untracked files are untouched. - -Merge messages: `chore(release-tracks): merge rollback support into beta` -(backend) and `chore(release-tracks): merge rollback controls into beta` -(frontend). - -## Virtual release-track schedule configuration - -- [x] Fix schedule saves submitting unsupported deduplication fields: remove - preferred tier/status controls and compare composition to its initial - editable state, including normalized priorities and defaults. -- [x] Verify server-shaped schedule-only save and strategy-change regressions, - full frontend tests, and production build. - Focused: 94 passing; full: 167 files / 410 tests passing; production build - and changed-file lint pass. Backend schema rejects precisely - `tier_resolution`/`status_resolution` and accepts the corrected payload. - Proposed commit: `fix(release-tracks): avoid invalid composition updates`. - Body: Remove unsupported deduplication controls and compare the edited - composition to its initial form state so schedule-only saves skip cloning. - -- [x] Add controlled natural-language schedule autocomplete with hourly and - 15/30-minute presets and guided customization. -- [x] Verify autocomplete regressions, full frontend suite, and production build. - Focused tests: 94 passing; full frontend: 167 files / 410 tests passing; - production build, changed-file lint, formatting, and diff checks pass. - Proposed commit: `feat(release-tracks): autocomplete schedule presets`. - Body: Map selected schedule phrases to deterministic UTC cron expressions - and support hourly and 15/30-minute guided customization. - -- [x] Review persisted schedule validation, storage, scheduler execution, and - existing regression coverage. -- [x] Add an authenticated virtual-track schedule update endpoint with strict - validation and persistence. -- [x] Add backend regression tests, OpenAPI documentation, user/developer - documentation, and Bruno coverage. -- [x] Add a controlled frontend schedule editor for manual, recurring cron, - and explicit-date schedules, without free-text cron entry. -- [x] Add frontend connector/component regressions and usage documentation. -- [x] Run focused checks, then the complete backend and frontend suites. -- [x] Propose conventional commit messages without committing. - -Verification (2026-09-09): - -- Backend focused schedule/API and scheduler specs: 19 passing; OpenAPI: 2 - passing; changed-file ESLint clean. -- Backend complete `npm test`: OpenAPI 2, config 22, API 1024, middleware 29, - and scheduler 10 passing. -- Frontend focused component/connector specs: 92 passing; complete suite: 167 - files and 408 tests passing; application TypeScript and production build - pass; changed-file ESLint has no errors. -- Proposed commits: `feat(release-tracks): add virtual schedule updates`, - `feat(release-tracks): add guided snapshot scheduling`, and - `docs(release-tracks): add virtual schedule request`. - -## Snapshot-scoped virtual composition provenance - -- [x] Return each virtual snapshot's immutable `composition_resolution` from - snapshot history and declare it in OpenAPI. -- [x] Add backend regression coverage and update user/API documentation plus - the Bruno snapshot-history request. -- [x] Move the frontend Composition Resolution view from HEAD into each - virtual snapshot's Releases card, including exact component version, - snapshot timestamp, strategy, filters, and contribution counts. -- [x] Add frontend regression coverage and update frontend documentation. -- [x] Replace the dense seven-column provenance table with a responsive - component list whose identifiers wrap within their own regions and whose - counts use independently wrapping metric labels. -- [x] Run focused backend/frontend specs, then each repository's complete - required verification suite; propose conventional commit messages. - -Verification (2026-09-09): - -- REST API: snapshot-history spec 7 passing; OpenAPI 2 passing; lint clean; - full `npm test` under Node 24 clean (2 OpenAPI, 22 config, 1020 API, - 29 middleware, 10 scheduler). Node 22 full-suite attempts reproduced the - documented roaming HTTP-response flake; every affected spec passed alone. -- Frontend: release-track page spec 71 passing; full `npm test` 404 passing; - application TypeScript compilation clean; changed-file ESLint clean; - production build clean with existing bundle/style budget warnings. The - repository-wide lint command still reports pre-existing errors outside the - changed files. -- Proposed commits: REST API `feat(release-tracks): expose virtual snapshot - provenance`; frontend `feat(release-tracks): scope composition provenance to - snapshots`; Bruno `docs(release-tracks): document snapshot composition - provenance`. - -Post-merge review (2026-09-09): no blocking findings; live registry scheduling -and historical composition provenance remain separate. Reverified with Node 24: -7 focused backend tests, 77 focused frontend tests, full backend suite -(2 OpenAPI, 22 config, 1024 API, 29 middleware, 10 scheduler), and all 411 -frontend tests passed. Backend lint, changed-file frontend lint, and the -frontend production build passed (bundle/style budget warnings remain). - -## Preserve pre-release drafts and protect virtual dependencies - -- [x] Change standard-track release commit from in-place tagging to creation - of a new tagged snapshot while retaining the exact source draft. -- [x] Prevent rolling-draft cleanup from pruning drafts retained as the source - of a tagged standard release. -- [x] Block release deletion when any persisted virtual snapshot resolved the - exact standard release snapshot, for implicit or explicit composition. -- [x] Add a post-hoc release-version update that preserves the snapshot and - validates the replacement against adjacent release versions. -- [x] Reconcile release catalogues, copied version ledgers, bundle hashes, - audit records, and current-snapshot backrefs for both operations. -- [x] Update OpenAPI, user/developer/operator docs, and Bruno requests. -- [x] Add ADM-valid API regressions and run focused specs, then full `npm test`. -- [x] Update the frontend release controls, wording, connector, and tests. -- [x] Propose conventional commit messages without committing. - -## Rollback / retag review follow-up - -- [x] Coordinate component release locks with virtual materialization. -- [x] Publish retag hashes atomically with the version and repair derived state on retry. -- [x] Expose preserved source pointers in snapshot history. -- [x] Validate deletion confirmation and capture audit identity under the release lock. -- [x] Add concurrency, failure-recovery, history, and exact-download hash regressions. -- [x] Update OpenAPI, user/developer docs, and Bruno smoke requests. -- [x] Run focused specs, full npm test, and lint; propose a commit without committing. - -Verification: focused backend group 32 passing, final destructive/retag spec -16 passing; full `npm test` passes (OpenAPI 2, config 22, API 1033, -middleware 29, scheduler 10). Backend lint and frontend page/connector tests -(90) pass. An initial unrelated technique-conversion 404 passed in isolation -(24) and on the final full run; no unrelated source changes were made. - -Proposed commit: `fix(release-tracks): make rollback and retag concurrency-safe` - -Coordinate materialization with component release locks, publish retag hashes -atomically, repair derived state on retry, expose preserved draft pointers, -and validate destructive confirmation under the audit lock. - -## Sealed snapshot content manifests (Problem 1) - -Design: [release-tracks/sealed-content-manifests.md](release-tracks/sealed-content-manifests.md). -Decisions confirmed by the developer on 2026-09-02: drop the relationship -advancement cascade, revert the frontend related-object reset to PUT, keep the -collection object in every STIX 2.1 bundle including drafts, track-scope -publication metadata with inherit-from-global as the default, collection -`modified` = snapshot `modified`, bundle `id` changes per snapshot while the -collection `id` is constant per track, and production-grade migration. - -### Backend (branch `beta`) - -- [x] Model: manifest gains `sealed_at`, `seal_reason`; snapshot gains - `content_manifest_id` (renamed from `graph_manifest_id`), `publication`, - `bundle_id`; `config.publication` with inherit/explicit identity and - markings plus optional `collection_id` and `created`; remove top-level - `object_marking_refs`. -- [x] Content manifest service (renamed from graph-manifest-service): seal - over exact members with ID-closed relationship selection, supporting and - link-target entries, no `secondary`, no predecessor carry-forward; - reference-counted discard; legacy schema-v1 reader retained. -- [x] Snapshot service: seal when `members` is written (create track, clone - with members override), inherit otherwise; prune drafts without - discarding shared manifests; notes editable on drafts only; remove - graph create/delete. -- [x] Versioning service: standard commit seals over planned members inside - the guarded tag update; virtual commit reuses the materialization seal; - freeze `publication`, generate `bundle_id`, store hashes; preview - reports relationship additions, removals, and stale authored endpoints. -- [x] Publication resolution: shared resolver for identity and markings - (track override or global), collection id and created overrides, - immutability after first release, config GET returns resolved values. -- [x] Export: single replay path; drafts may add live `include` tiers through - the same closure rule; tagged + `include` is 400; collection object - always present in 2.1, absent in 2.0; drafts omit `x_mitre_version`; - remove `includeToc`; bundle id stored for tagged, UUIDv5 for drafts. -- [x] Relationships: remove `handleEndpointRevisionCreated` and its - subscriptions; keep create-time endpoint pinning; drop the exact-endpoint - repository query if unused. -- [x] Routes/controller/OpenAPI: remove graph create/delete; reconstruct - accepts `replace_manifest_id`; rename response fields - (`content_manifest_id`, `content_statistics`); publication config - schema; remove `includeToc` and top-level marking refs. -- [x] Migration: rename field, seal unsealed tagged snapshots as baseline - reconstructions, drafts inherit or seal, migrate track marking refs into - `config.publication`, freeze `publication` and `bundle_id` on tagged - snapshots, recompute hashes, drop frozen `collection` entries; make the - 2026-07-30 migration's manifest backfill a no-op; dry-run preview - script; regression spec. -- [x] Tests: content-manifest lifecycle spec (replaces opt-in-graphs), - bundle spec updates, no-cascade relationship spec, publication config - spec, migration spec, virtual/history/description spec updates; run - focused specs then full `npm test`. -- [x] Docs: bundle-export.md, entities.md, implementation-notes.md, user - output-formats/versioning/api-reference/terminology, relationships doc. -- [x] Bruno: remove graph create/delete requests, update reconstruct, - config, snapshot export, and track creation requests. -- [x] Propose conventional commit messages. - -### Frontend (feature branch off `beta`) - -- [x] Revert relationship save to update related objects with PUT. -- [x] Remove bundle cache controls, cache status, and cache statistics; show - content statistics, hashes, and bundle id on tagged snapshots. -- [x] Publication section in track configuration: identity and markings with - inherit toggle and resolved-value display; collection id and created - overrides editable until first release. -- [x] Release preview: relationship inventory summary and stale-endpoint - warnings. -- [x] Snapshot notes read-only after release; field renames in classes, - connector, and specs; docs update; focused and full frontend - verification. - -Verification (2026-09-02): - -- Backend focused specs pass: content manifests 10, publication config 3, - manifest migrations 5, bundle export 19, snapshot descriptions 7, snapshot - history 7, virtual graph integrity 3, release planning 23, endpoint pins 3, - release tracks 3, attack objects, collection bundles, reconciliation. -- Three complete `npm test` runs: 1009/1012/1010 API cases passing with the - documented roaming shared-server failures only (attack-objects count, - collection-bundles 400, reconciliation 400, analytics ECONNRESET, groups - timeout); every affected spec passes in isolation. OpenAPI, config, - middleware, and scheduler suites pass. `npm run lint` is clean for the - changed files (pre-existing findings remain in untouched migrations and - `scripts/loadBundle.js`). -- Restored-production trial (2026-09-02): the first run crashed on - `release-track--4bf296be…`, one of eight unregistered collections left by - the first v19.1 bootstrap, whose members reference six technique revisions - replaced by the platform-ordering repair. The migration now migrates - registered tracks only, reports orphans, discards their manifests, and names - the failing snapshot, step, and missing references on any other error. -- The crashed API container kept restarting (`restart: unless-stopped`, 23 - restarts) on the old image, re-running the unfixed migration and re-creating - the four orphan manifests each time; the rebuilt image's idempotent run - discards them again. Stop or rebuild the container before judging the - database state. -- After the fix, a dry run against the restore reported 6 tracks, 21 - snapshots, 7 seals, 1 shared manifest, 4 renames, and 7 orphan collections; - applying it sealed and froze exactly that, recomputed 7 hash sets, and a - second apply changed nothing. Every released snapshot's stored STIX 2.1 - hash matches a fresh export and every draft exports with the version key - omitted. -- Design record: `docs/developer/release-tracks/sealed-content-manifests.md`. -- Follow-up for the developer's v19.1 bootstrap tooling (`.nocommit/`): read - `content_manifest_id` instead of `graph_manifest_id`, and replace the - delete-then-reconstruct recovery with a single reconstruct request naming - `replace_manifest_id`. -- Frontend (branch `feat/sealed-content-manifests`): focused specs pass - (page 53, connector, relationship, preview dialog); the complete vitest - suite passes (166 files, 379 tests); `tsc --noEmit`, ESLint on changed - files, and the production `ng build` succeed. -- Proposed REST commit: `feat(release-tracks): seal snapshot content manifests`. - Proposed frontend commit: `feat(release-tracks): surface sealed content and -publication settings`. - -### Review follow-ups (2026-09-02) - -- [x] Fix the virtual-track config editor crash: the connector's identity and - marking getters return functions that must be invoked as methods. -- [x] Rename the History tab to Releases; drop the per-snapshot "Sealed" chip - (every snapshot is sealed, so it carried no information) and label the - statistics section "Content". -- [x] Restore release deletion: administrators may delete the track's most - recent release with a typed version confirmation (`confirm_version`); - the ledger entry is retracted from every remaining snapshot, the manifest - is discarded when unreferenced, the registry is reconciled, and a - `delete_release` audit event is recorded. Frontend button on tagged - release cards for administrators. -- [x] Data-model review (KISS): rename manifest storage to - `releaseTrackContentManifest*` with `release-track-content-manifest--` - ids; drop `resolver_version` and `baseline_reconstruction` in favour of a - required `seal_reason`; remove the dead `config.include_secondary_objects` - block and its frontend section; keep `releaseTrackReconciliations` as an - outstanding-work queue (completed records are deleted, so it is normally - empty); keep `releaseTrackAuditEvents` (now used by both destructive - actions) and document every collection in `entities.md`. -- [x] Migration extended in place (unreleased): collection rename, id rewrite, - header normalization, dead-config removal, completed-reconciliation - cleanup; dry run stays accurate before the rename. - Verification (2026-09-02, review follow-ups): - -- Backend focused specs pass: manifest migrations 5, destructive authorization - 3, content manifests 10, snapshot history 7, virtual graph integrity 3, - reconciliation durability 2, releases by object 8, snapshot immutability 2, - backrefs 24; OpenAPI validation passes. -- Restored production database: dry run reports 131,534 legacy manifest - documents to move and 17 headers to normalize; apply completes, all 21 - snapshots export with matching hashes, storage now lists only - `releaseTrackContentManifests`, `releaseTrackContentManifestEntries`, - `releaseTrackRegistry`, `releaseTrackReconciliations` (empty), and - `releaseTrackAuditEvents`; a second dry run reports nothing left to do. -- Frontend: focused page and connector specs pass (71); complete suite - 381 tests with one unrelated save-dialog flake that passes alone; `tsc`, - ESLint, Prettier, and the production build are clean. -- [ ] Recommendation, not implemented: `version_history` is copied into every - snapshot document although only the tagged snapshot's own entry is read - (`tagMetadataForSnapshot`) and the registry's `tagged_releases` is the - catalogue. Storing the entry only on the tagged snapshot would remove the - duplication but touches release planning, cloning, and the frontend - history view; defer to a dedicated slice. - -## Remove nightly-only migration compatibility - -- [x] Remove regression code that imports the retired beta bundle-integrity - migration. -- [x] Remove canonical-domain correction logic used only to carry flawed beta - migration output forward. -- [x] Remove current documentation for the retired nightly migrations and - document the alpha/beta database reset policy. -- [x] Run the focused migration and current graph-invariant regression specs. -- [x] Run the complete `npm test` suite and propose a conventional commit - message without committing. - -Verification (2026-08-18): - -- Focused canonical-domain, deterministic-graph, opt-in-graph, and bundle - regressions pass (39); ESLint and whitespace checks also pass. -- Four complete `npm test` attempts reached 1001-1009 passing API tests but - each encountered the documented roaming shared-server failure (HTTP parse - error, transient 404, `ECONNRESET`, or socket hang-up). Every affected spec - passes independently, including techniques conversion (24), software - pagination (13), the grouped release-track cases (15), and change capture - (10). OpenAPI (2) and configuration (22) passed on every complete attempt. -- Proposed commit: `chore(migrations): remove nightly compatibility remnants`. - -## Frontend and REST API build information - -- [x] Source REST API build metadata from the Docker/runtime build variables, - with package/default fallbacks for non-Docker development. -- [x] Extend the public system-version endpoint and OpenAPI contract with the - REST API name, release version, Git commit, and build date. -- [x] Generate matching frontend metadata into production build artifacts and - display frontend plus REST API versions in the navigation footer. -- [x] Add REST API and frontend regressions for metadata loading, fallbacks, - endpoint access, and rendering. -- [x] Update REST API user/developer/admin docs, frontend docs, and the Bruno - collection for the expanded API response. -- [x] Run focused checks followed by the complete REST API and frontend test - suites, then propose conventional commit messages without committing. - -Verification (2026-08-07): - -- REST API focused system-version (19), configuration (22), and OpenAPI (2) - regressions pass; ESLint and whitespace checks also pass. -- The clean complete REST API suite passes: OpenAPI 2, configuration 22, API - 1012, middleware 29, and scheduler 10. An earlier run's documented roaming - `ECONNRESET` passed in isolation before the clean rerun. -- The complete frontend suite passes: 166 files and 385 tests. Focused service, - footer, and navigation tests (22), ESLint, Prettier, the metadata generator, - and a production build with release-like metadata also pass. -- Proposed REST API commit: `feat(config): expose REST API build information`. - Proposed frontend commit: `feat(shell): display component build versions`. - -## Targeted v19.1 source-graph recovery - -- [x] Add a read-only preflight that targets one exact tagged virtual snapshot, - reconstructs its canonical v19.1 pointer plan, and requires every prior - immutable baseline repair to already exist. -- [x] Add a separately confirmed apply mode that replaces only the target - snapshot's graph manifest and refuses track replacement or STIX writes. -- [x] Verify the snapshot members, source-pointer hydration, manifest - attestation, and final emitted bundle against the canonical source. -- [x] Tag future public v19.1 virtual baselines as `19.1` while retaining the - internal standard-track baseline tag. -- [x] Add operator documentation and regression coverage for safety, - idempotence, and the incorrect ordinary-manifest recovery scenario. -- [x] Run focused Python checks followed by the complete `npm test` suite. -- [x] Propose a conventional commit message without committing unless asked. - -Verification (2026-08-05): - -- Bootstrap regressions pass: 42 cases covering graph-only no-write preflight, - guarded replacement, missing-repair fail-closed behavior, manifest races, - and the public `19.1` virtual tag. Ruff, Python compilation, and diff - whitespace checks pass. -- The clean complete REST suite passes under repository-pinned Node 22.14.0: - OpenAPI 2, config 21, API 1012, middleware 29, and scheduler 10. -- Earlier complete runs encountered the documented roaming shared-server - failures; each affected bundle, pagination, backref, virtual-graph, and notes - spec passed independently before the clean run. -- Proposed commit: `fix(release-tracks): add targeted v19.1 graph recovery`. - -## Deterministic graph collection identity repair - -- [x] Reproduce the incorrect graph collection creator, STIX 2.0 TOC - inclusion, and persisted cross-manifest collection-ID drift. -- [x] Resolve graph collection `created_by_ref` from the configured - organization identity and enforce one collection ID per release track. -- [x] Exclude `x-mitre-collection` from STIX 2.0 snapshot bundles and hashes. -- [x] Add a rerunnable forward migration that repairs existing graph - collection entries and recomputes tagged-snapshot bundle hashes. -- [x] Update release-track user, developer, and administrator documentation. -- [x] Run focused regression specs. -- [x] Complete an all-green `npm test` run without the documented roaming - in-memory MongoDB/server flake. -- [x] Propose a conventional commit message without committing unless asked. - -Verification (2026-08-05): - -- Focused graph, migration, snapshot, ephemeral, virtual, and legacy bundle - specs pass, including exact SHA-256 comparisons against downloaded bundles - and a rerun proving the repair migration is idempotent. -- ESLint and `git diff --check` pass. -- Full-suite attempts reached 1009 passing/3 failures, 1000/5, and repeatedly - 1011/1. Each failure roamed to an unrelated spec as a transient 400/404, - `ECONNRESET`, or socket hangup; every affected spec passes in isolation, - including under the repository-pinned Node 22.14.0 runtime. -- The developer subsequently confirmed a complete all-green test run. -- Proposed commit: `fix(release-tracks): repair deterministic bundle integrity`. - -## Stateful snapshot collection objects and bundle hashes - -- [x] Persist one frozen `x-mitre-collection` entry in every graph manifest, - with a track-stable ID, first-manifest `created`, and current-manifest - `modified` timestamp. -- [x] Replay the frozen collection entry and a manifest-stable bundle envelope - ID for deterministic STIX 2.0 and STIX 2.1 downloads. -- [x] Generate SHA-256 hashes from the exact pretty-printed download bytes and - store both hashes on graph-backed snapshots with their manifest ID. -- [x] Reject snapshot-note edits while a graph exists so the frozen collection - and hashes remain immutable; require deletion and regeneration to edit. -- [x] Expose hashes through snapshot responses/OpenAPI and update REST docs, - Bruno coverage, and regression tests. -- [x] Replace frontend bundle prefetch/hashing with server-provided hashes, - preserving copy controls and exact download serialization. -- [x] Run focused and complete REST/frontend verification and propose - conventional commit messages without committing unless asked. - -Verification (2026-08-04): - -- Focused REST graph, description, history, and bundle specs pass, including - exact SHA-256 comparisons against both downloaded bundle serializations. -- The complete REST `npm test` suite passes after the repository's documented - roaming harness failures were confirmed in isolation and rerun. -- The complete frontend suite passes: 165 files and 381 tests. Targeted - TypeScript, ESLint, and Prettier checks also pass. -- Cached-note regression coverage proves the API returns 409 without changing - either bundle hash, then permits editing after graph deletion and freezes the - revised notes when the graph is recreated. -- Repeated complete REST runs encountered the documented roaming harness - failures in unrelated specs (transient 400/404/ECONNRESET responses); every - affected spec passes when rerun in isolation. -- Proposed REST commit: `feat(release-tracks): persist snapshot bundle hashes`. - Proposed frontend commit: `feat(release-tracks): display snapshot bundle hashes`. - -## Relationship review-state revision safety - -- [x] Reproduce the relationship-save source/target workflow reset and prove it - currently uses in-place PUT updates. -- [x] Reset related SDO workflow state through POST-created revisions so graph- - pinned revisions remain immutable. -- [x] Add frontend regression coverage for request method, WIP transition, and - sequential relationship/source/target saves. -- [x] Update frontend workflow documentation and run focused plus complete - frontend verification. -- [x] Propose a conventional commit message without committing unless asked. - -Verification (2026-08-04): - -- The focused relationship revision regression passes: 1 case proving ordered - relationship/source/target POSTs, WIP resets, and no related-object PUTs. -- The complete frontend suite passes: 164 files and 377 tests. The production - Angular build succeeds with existing bundle/style budget warnings. -- Prettier and the new regression's ESLint check pass. The legacy relationship - class retains its existing unrelated lint findings; the changed transport - line introduces none. -- Proposed frontend commit: `fix(relationships): revise related objects on save`. - -## Snapshot collection descriptions and bounded release versions - -- [x] Map each snapshot's user-authored description onto emitted - `x-mitre-collection.description` while preserving the track description - as the fallback for snapshots without notes. -- [x] Add backend regressions for bundle and graph-backed bundle exports using - snapshot descriptions. -- [x] Calculate relative and explicit release versions between the nearest - earlier and later tagged snapshots, with exclusive chronological bounds. -- [x] Add regression coverage for mixed explicit/relative tags, retroactive - releases, invalid boundary values, and exact-version uniqueness. -- [x] Wire exact `MAJOR.MINOR` release selection into the Angular release - preview dialog and connector flow with component/page tests. -- [x] Update OpenAPI, user/developer docs, and Bruno release requests. -- [x] Run focused backend and frontend tests, then the complete backend - `npm test` suite and the relevant frontend verification commands. -- [x] Propose conventional commit messages for both repositories. - -Verification (2026-08-04): - -- Focused backend release and bundle specs pass (23 and 18 cases), including - mixed explicit/relative tags, retroactive bounds, concurrent release locking, - and snapshot-description export. -- Backend lint, OpenAPI/config validation, middleware (29 cases), scheduler - (10 cases), and every isolated full-suite failure pass. Four complete - `npm test` attempts reached 1007-1008 passing API cases before the documented - shared-server flake roamed to a different unrelated spec on each run; the - isolated targets pass under both Node 22 and Node 24. -- The complete frontend suite passes (163 files, 376 tests), targeted ESLint and - Prettier checks pass, and the production build succeeds with existing budget - warnings. -- Proposed backend commit: `feat(release-tracks): bound snapshot publication -versions`. Proposed frontend commit: `feat(release-tracks): tag snapshots -with exact versions`. - -## Frontend graph cache lifecycle controls - -- [x] Replace the static cache-materialization hourglass with the existing - Angular Material indeterminate spinner. -- [x] Add an editor-only, confirmed delete action for cached snapshot graphs, - including progress and success/error feedback. -- [x] Add connector/component regressions, update frontend behavior notes, and - run focused plus complete frontend verification. - -Verification (2026-08-03): - -- Focused Angular connector/component regressions pass: 68 tests. The complete - frontend suite passes: 162 files and 366 tests. -- Targeted ESLint and Prettier checks pass. The production Angular build passes - with the local persistent cache temporarily disabled to avoid the documented - environment-specific native crash; `angular.json` was restored afterward. -- Proposed frontend commit: `feat(release-tracks): manage snapshot bundle -caches`. - -## Source-attested v19.1 graph reconstruction - -- [x] Add fail-closed regressions for canonical-domain migration when exact - collection TOC provenance is unavailable; never infer Enterprise from - missing evidence. -- [x] Add an administrator-only schema-v2 source reconstruction endpoint that - validates a closed pointer plan against tagged snapshot members and - persisted exact revisions. -- [x] Build v19.1 source plans without importing bundles, inferring legacy SRO - endpoint revisions from the unique objects emitted in each source bundle. -- [x] Validate the complete source plan against MongoDB before tagging, then - attach it atomically and require a final bundle comparison. -- [x] Update OpenAPI, Bruno, operator/developer documentation, and bootstrap - recovery guidance. -- [x] Run focused migration, reconstruction, and bootstrap regressions; then - lint and the complete `npm test` suite. - -Verification (2026-08-03): - -- Canonical-domain migration regressions pass: 9 cases. Unmapped domainless - objects remain unchanged, are reported, and keep the legacy bypasses active. -- Source reconstruction regressions pass: 5 cases. They prove exact historical - relationship and endpoint revisions replay after their live lineages advance, - reject incomplete pointer plans, and enforce source-attestation idempotence. -- Bootstrap regressions pass: 33 cases. Source bundles are never imported; - 30,649 emitted-object pointers and 54 non-emitted LinkById dependency - pointers are hydrated from MongoDB before tagging. The largest - reconstruction request remains below the 50 MiB API request limit and - outside the 16 MiB per-document BSON limit. -- The clean complete REST suite passes: OpenAPI 2, config 21, API 995, - middleware 29, and scheduler 10. Four roaming harness failures in the first - run passed independently (13, 24, 25, and 4 cases) before the clean rerun. -- Repository ESLint, Python Ruff, Python bytecode compilation, and diff - whitespace validation pass. -- Proposed commit: `fix(release-tracks): attest v19.1 snapshot graphs`. - -### Historical relationship hydration follow-up - -- [x] Reproduce the 21,025 missing Enterprise revisions against the restored - production-shaped database and classify payload differences. -- [x] Hydrate relationship pointers from the dedicated MongoDB collection and - render LinkById fields through the same deterministic export semantics. -- [x] Add regressions that fail on relationship content drift while accepting - exact persisted timestamps and export-only LinkById rendering. -- [x] Update bootstrap documentation and run focused plus complete verification. - -Verification (2026-08-03): - -- The 21,025 failures are exactly the Enterprise relationship count. Every - sampled exact timestamp exists in MongoDB's dedicated `relationships` - collection; the bootstrap had incorrectly queried `attackObjects` for all - pointer kinds. -- All 24,552 v19.1 relationships were audited read-only. Of those, 5,624 raw - payloads already match exactly and 18,928 differ only because exports render - persisted `(LinkById: ...)` tags as Markdown links. -- A production-shaped preflight reconstructs Enterprise (25,851 graph entries), - ICS (2,201), and Mobile (2,651) with zero missing, changed, or additional - emitted objects. The 54 entries beyond the 30,649 emitted objects are exact, - non-emitted cross-domain LinkById dependencies. -- Bootstrap regressions pass: 33 cases. Focused source-graph regressions pass: - 5 cases. The clean complete REST suite passes: OpenAPI 2, config 21, API 995, - middleware 29, and scheduler 10. -- A separate test-only correction serializes a Mongoose snapshot date before - placing it in a graph URL; its isolated virtual-graph-integrity spec passes: - 3 cases. -- Proposed implementation commit: `fix(release-tracks): hydrate historical -relationship graphs`. Proposed test-only commit: `test(release-tracks): -serialize snapshot timestamps in graph URLs`. - -### Production bootstrap replacement recovery - -- [x] Reproduce the production preflight failure against the restored database. -- [x] Keep dynamic `latest` workflow selectors out of exact historical-pin - compatibility comparisons. -- [x] Add an explicit, confirmed option to replace only the six exact-name - bootstrap tracks, deleting virtual tracks before their standard inputs. -- [x] Repair six persisted v19.1 technique revisions whose platform arrays - contain the correct values in a different order from the source bundles. -- [x] Run the corrected preflight against the restored production database and - complete focused script verification. - -Verification (2026-08-04): - -- The restored-production preflight completes without treating nine dynamic - `latest` candidates as timestamps and inventories all six existing tracks for - replacement. -- It identifies 297 immutable baseline repairs: 291 canonical-domain revisions - and six `x_mitre_platforms` ordering revisions. -- Two replacement applies complete successfully. The second reuses all 297 - semantic repair revisions, proving restart safety, and all three final bundle - comparisons report `identical_excluding_collection: true` with member counts - Enterprise 4,815, ICS 503, and Mobile 743. -- Bootstrap regressions pass: 36 cases. Python Ruff and the read-only production - preflight pass. - -### Closed-member deterministic relationship graphs - -- [x] Reproduce exact-revision leakage when a relationship endpoint pins a - different revision of an existing snapshot member. -- [x] Replace ID-frontier secondary expansion with indexed exact-endpoint - relationship selection requiring both endpoint revisions in `members`. -- [x] Select only the latest relationship revision for each exact endpoint - pair, with a later revoked or deprecated revision suppressing older - active history. -- [x] Seed the v19.1 transition from the preceding source-attested manifest so - historical relationships without truthful stored endpoint pins remain - available while their exact member endpoints survive. -- [x] Keep schema-v2 relationships pointer-only and reject duplicate emitted - STIX revisions or inconsistent relationship lineages. -- [x] Update release-track documentation and add regression coverage for - closed membership, predecessor carry-forward, relationship advancement, - and mutation protection. -- [x] Run focused release-track specs followed by the complete `npm test` - suite. - -Verification (2026-08-04): - -- Closed-member graph regressions pass: 9 opt-in graph cases and 17 bundle - cases. -- The complete release-track directory passes with 187 cases after two roaming - harness failures were rerun successfully in isolation (27 cases). -- Lint and the complete `npm test` suite pass, including OpenAPI, - configuration, API, middleware, and scheduler suites. - -## Snapshot-history graph cache statistics - -- [x] Add regression coverage for exact manifest-kind counts on cached - snapshot summaries and omission on uncached snapshots. -- [x] Aggregate graph cache statistics for every manifest on a history page in - one indexed query and expose the typed summary through OpenAPI. -- [x] Show CTI-oriented Primary, Secondary, Relationships, and Dependencies - statistics for cached snapshots in the frontend History tab. -- [x] Update release-track user/developer documentation and run focused plus - complete backend/frontend verification. - -Verification (2026-08-03): - -- Focused snapshot-history regressions pass: 7 REST cases and 65 Angular - connector/component cases. OpenAPI validation passes. -- The complete frontend suite passes: 162 files and 363 tests. The Angular - build passes with the local persistent cache temporarily disabled to avoid - the environment-specific native cache crash; `angular.json` was restored. -- The complete REST suite passes: OpenAPI 2, config 21, API 994, middleware 29, - and scheduler 10. A documented roaming backref setup flake passed all 24 - cases in isolation before the clean complete rerun. -- REST lint and targeted frontend ESLint/Prettier checks pass. The performance - audit is `PERFORMANT`: one indexed aggregate covers every manifest on the - bounded history page, with no per-snapshot query. -- Proposed REST commit: `feat(release-tracks): expose graph cache statistics`. -- Proposed frontend commit: `feat(release-tracks): show graph cache statistics`. - -## v19.1 bootstrap graph lifecycle and canonical-domain correction - -- [x] Add regressions proving canonical domains come from exact collection TOC - membership, not secondary bundle appearance or projected payload fields. -- [x] Correct the startup canonical-domain backfill and add a forward migration - for already-created domain-only successor revisions. -- [x] Replace the bootstrap's custom schema-v1 draft manifest writes with the - supported tagged-snapshot schema-v2 graph endpoint. -- [x] Make bootstrap resume and final verification require a persisted graph - and a post-graph v19.1 bundle comparison with no drift override. -- [x] Update the bootstrap runbook and canonical-domain documentation with the - corrected provenance contract and recovery behavior. -- [x] Run focused migration/bootstrap/release-track regressions, then lint and - the complete `npm test` suite. - -Verification (2026-08-03): - -- The local official v19.1 source audit finds exactly nine payload/TOC domain - mismatches, all campaigns; corrected virtual membership is Enterprise 4,815, - ICS 503, and Mobile 743. -- Canonical-domain migration regressions pass: 9 cases. Bootstrap regressions - pass: 29 cases, including pointer-only graph validation and semantic drift - rejection for relationships advanced by domain repairs. -- The complete `npm test` suite, repository ESLint, targeted migration ESLint, - Python Ruff, Python bytecode compilation, and diff whitespace checks pass. -- Proposed commit: `fix(release-tracks): correct v19.1 bootstrap provenance`. - -## Opt-in deterministic member graphs and rolling drafts - -- [x] Add regressions for immutable versioned STIX payloads, pointer-only - relationship manifests, and legacy frozen-manifest replay. -- [x] Add tagged-snapshot graph create/delete endpoints and make graphless - bundle exports resolve live while persisted graphs cover members only. -- [x] Stop automatic graph generation during snapshot cloning and release; - retain only the latest standard-track draft after a durable replacement. -- [x] Bound graph construction to relationship lineages that touch the - selected member frontier and batch exact-revision hydration. -- [x] Update OpenAPI, user/developer documentation, and Bruno requests for the - opt-in determinism and immutable-revision contracts. -- [x] Run focused regression specs, then the complete `npm test` suite and - review the final performance/architecture diff. - -Verification (2026-08-03): - -- Opt-in graph regressions pass: graphless release, tagged-only graph - creation/deletion, schema-v2 pointers, frozen marking definitions, - correction by POST, live graphless replay, and rolling-draft truncation. -- Release-track regressions pass: 176 cases. Immutable CRUD regressions pass: - 378 cases. The migration regression preserves schema-v1 frozen replay. -- The complete `npm test` suite passes: OpenAPI 2, config 21, API 993, - middleware 29, and scheduler 10. -- Repository lint and diff whitespace validation pass. -- Proposed commit: `feat(release-tracks): make deterministic graphs opt in`. - -### Frontend deterministic bundle cache controls - -- [x] Expose `graph_manifest_id` in lightweight snapshot-history summaries so - the UI can render cache state without per-snapshot requests. -- [x] Add the frontend connector and History-tab cache state, warning - tooltips, editor action, progress state, and success/error feedback. -- [x] Add focused REST and Angular regressions for summary propagation, - connector routing, cache-state mapping, and materialization. -- [x] Run formatting, lint, builds, and the complete frontend/backend suites; - record the final verification and proposed commits. - -Verification (2026-08-03): - -- Focused Angular connector/component regressions pass: 65 tests. The focused - REST snapshot-history regression passes: 7 tests. -- The complete frontend suite passes: 162 files and 363 tests. The Angular - build passes with the local persistent cache temporarily disabled to avoid - an environment-specific native `lmdb` crash; no cache setting was committed. -- The complete REST suite passes: OpenAPI 2, config 21, API 993, middleware 29, - and scheduler 10. REST lint passes. -- Changed frontend sources pass Prettier, targeted ESLint, and diff whitespace - checks. Repository-wide frontend lint remains red on 254 pre-existing - errors; the shared release-track API type retains one pre-existing - index-signature violation. -- Proposed frontend commit: `feat(release-tracks): add deterministic bundle -cache controls`. - -## Frontend canonical-domain preservation - -- [x] Inventory every frontend model and object view corresponding to the - canonical-domain migration's `TARGET_TYPES`. -- [x] Add a regression contract proving every target type preserves - `x_mitre_domains` through deserialize/serialize. -- [x] Add domain model support and editable domain fields to the missing - campaign, intrusion-set, detection-strategy, and matrix views. -- [x] Document the frontend domain-editing contract and run focused tests, - lint/format checks, and the complete frontend test suite. - -Verification (2026-08-03): - -- Canonical-domain model and view contracts pass: 25 cases covering every - migration target type. -- Complete frontend suite passes: 158 files and 323 tests. -- Angular build and targeted ESLint/Prettier checks for every changed source - file pass. -- Repository-wide lint remains red on 256 pre-existing errors outside this - change; no new lint errors remain in the hotfix files. -- Proposed frontend commit: `fix(stix): preserve canonical domains in -editors`. - -## C0028 campaign revision / released virtual-snapshot investigation - -- [x] Trace the submitted campaign payload through REST create handling and ADM - citation validation against the authoritative ADM source. -- [x] Reproduce the reported 400 response and isolate whether the defect is in - the payload, frontend transformation, REST API, or ADM. -- [x] Document the supported repair path and, as a fallback, enumerate every - database invariant/provenance record a manual repair would have to keep - consistent. -- [x] Record evidence, recommended regressions/fix scope, and a proposed - conventional commit message without mutating production data. - -Investigation (2026-08-03): - -- The reported request cites `Booz Allen Hamilton` in both campaign temporal - citation fields but sends only the `mitre-attack` external reference. ADM - 4.11.7 correctly reports both missing-reference refinements. Adding the - released Booz Allen reference makes the composed campaign pass the WIP ADM - schema. -- The Angular `Campaign` model does not deserialize or serialize - `x_mitre_domains`, and the campaign view exposes no domain editor. The - reported request consequently also omits the intended canonical-domain - correction. This is a frontend payload defect, not an ADM defect. -- The supplied database record is a `releaseTrackGraphManifestEntries` root - with an operationally frozen payload, not the authoritative campaign entity - in `attackObjects`. Editing it would rewrite an immutable released artifact - while retaining the old revision key and object timestamp. -- Supported hotfix: dry-run and then POST a new C0028 revision containing the - Booz Allen external reference and - `x_mitre_domains: ["enterprise-attack", "ics-attack"]`; let created-event - relationship advancement and standard-track member sync create the next - candidate/draft, then release the component track and materialize/tag a new - virtual snapshot. Do not alter the already-tagged virtual snapshot. -- Recommended regressions: frontend campaign/group round-trip coverage for - canonical domains; campaign save coverage that retains temporal citation - references; backend campaign regression proving a missing cited source is - rejected and the corrected revision succeeds with ADM validation enabled. -- Proposed implementation commit: `fix(campaigns): preserve domains and cited -references in revisions`. - -Verification (2026-08-03): - -- Direct ADM schema reproduction returns the two production error paths for - the reported composed STIX object and succeeds after adding the cited - reference and canonical domains in work-in-progress, awaiting-review, and - reviewed states. -- Existing campaign API regression passes: 21 cases. Its placeholder - organization identity causes an earlier suppressible ADM issue, so it does - not currently exercise the citation refinement and needs the targeted - regression above. - -Follow-up frontend citation-loss investigation (2026-08-03): - -- REST `BaseService.create()` removes only ATT&CK-owned external references, - preserves every submitted user reference, regenerates the canonical ATT&CK - reference, and validates that composed object. It deliberately does not - merge omitted user references from the previous revision. -- Angular initially retains the C0028 `Booz Allen Hamilton` reference when it - deserializes the GET response. `StixObject.base_validate()` first sends a - valid dry-run payload, then calls the mutating `ExternalReferences.validate()` - with only `description` and `aliases` as campaign citation fields. -- That incomplete field list treats the temporal citation reference as unused - and removes it before the real save POST. Commit `4f04ac70` added server - dry-run validation and explicitly removed `first_seen_citation` and - `last_seen_citation` from this field list, creating a time-of-check/time-of-use - mismatch. `ExternalReferences.parseObjectCitations()` still has the correct - campaign field list. -- Minimum repair: restore both temporal citation fields to campaign reference - validation. Durable repair: centralize the field list and complete all - reference synchronization before the server dry run so validation and save - serialize the same object state. - -## STIX 2.0 virtual snapshot bundles - -- [x] Add a virtual-track regression proving materialized snapshots emit STIX - 2.0 bundles when `stixVersion=2.0` and remain STIX 2.1 by default. -- [x] Align the virtual snapshot OpenAPI, user documentation, and Bruno request - with the explicit STIX-version contract. -- [x] Run the focused regression followed by the complete `npm test` suite, - review the final diff, and propose a conventional commit message. - -Verification (2026-07-30): - -- Virtual STIX-version bundle regression passes: 2 cases. -- Existing snapshot-bundle regression passes: 17 cases. -- Backend lint, Prettier, and diff whitespace validation pass. -- One aggregate attempt exposed the documented roaming References search 404; - the affected spec passed all 17 cases in isolation. -- The clean complete suite passes: OpenAPI 2, config 21, API 989, middleware - 29, and scheduler 10. - -## Client-managed virtual scheduled materialization - -- [x] Add API regressions proving virtual-track POST and composition PUT - requests persist `scheduled_materialization`. -- [x] Validate the client-supplied shape at controller, service, and Mongoose - boundaries and reject it for standard tracks. -- [x] Expose the value through track listing, snapshot history, latest - snapshot, and timestamp-selected snapshot GET responses. -- [x] Align OpenAPI, user/developer documentation, frontend guidance, and - Bruno requests with the client-managed contract. -- [x] Run the focused regression spec followed by the complete `npm test` - suite, review the final diff, and propose a conventional commit message. - -Verification (2026-07-30): - -- Focused scheduler and scheduled-materialization API regressions pass: 13 - cases. -- Previously roaming group-query and virtual-deduplication failures pass in - isolation: 14 cases. -- The complete `npm test` suite passes, including OpenAPI, configuration, API, - middleware, and scheduler stages. -- Backend lint and diff whitespace validation pass. - -## Caller-supplied configuration on track creation - -- [x] Add a regression proving `POST /api/release-tracks/new` accepts and - persists supported `config` options on the initial snapshot. -- [x] Reuse the release-track config validation contract in the create request - and pass the validated config through the snapshot creation service. -- [x] Update OpenAPI guidance, user documentation, and the Bruno request. -- [ ] Run the focused regression and the complete `npm test` suite. - -Verification (2026-07-30): - -- Focused release-track API regression passes: 3 cases. -- Backend lint and OpenAPI validation pass. -- The complete suite was run and reached API 975 passing with four failures - in unrelated, pre-existing work: three canonical-domain migration failures - and one roaming virtual-composition failure. -- The virtual-composition spec passes in isolation. The in-progress - canonical-domain migration spec still has three isolated failures, so a - clean aggregate run remains outstanding. - -## Embedded canonical-domain migration and enforcement - -- [x] Batch the canonical-domain migration so active revisions use bounded - service-layer concurrency and verification/audit records avoid - unnecessary per-object database round trips. -- [x] Replace the v19.1 object manifest with persisted canonical collection - provenance and scan the latest revision of all 13 domain-bearing ATT&CK - types, irrespective of active, deprecated, or revoked state. -- [x] Let the native migration driver generate inactive-clone `_id` values so - Mongoose BSON 6 values are never passed to MongoDB driver/BSON 7 writes. -- [x] Serialize concurrent release-track member-sync mutations per track so - batched reposts cannot overwrite candidates created by sibling workers. -- [x] Add batch-size, audit-sequence, shared-track concurrency, and - idempotency regressions; update operator documentation and rerun the - focused and complete test suites. -- [x] Add an idempotent startup migration that reposts every active latest - domainless object through its normal service create lifecycle. -- [x] Include deprecated and revoked latest revisions as immutable direct - clones, preserving lifecycle state and creating a new `modified` - revision without relying on inactive-content POST guardrails. -- [x] Initialize release-track member synchronization during the migration so - newly created revisions follow ordinary track-driven candidacy behavior. -- [x] Remove static `x_mitre_domains` validation bypasses and delete their - already-persisted database copies during migration. -- [x] Default a latest domainless object that cannot be mapped to canonical - collection provenance to `["enterprise-attack"]`, and audit the fallback. -- [x] Add migration, idempotency, inactive-state, member-sync, and ADM - enforcement regressions. -- [x] Update migration and domain-contract documentation, then run focused - tests, lint, and the complete `npm test` suite. - -Verification (2026-07-30): - -- Release-agnostic canonical-domain migration regression: 8 passing, covering - all 13 domain-bearing types and the MongoDB 7/Mongoose MongoDB 6 driver - boundary. -- Focused virtual-domain and bundle regressions: 23 passing. -- Backend and migration lint plus diff checks pass. -- Complete suite passes: OpenAPI 2, config 21, API 982, middleware 29, and - scheduler 10. -- Existing release-track change-capture regression: 13 passing. -- Batch-related lint and formatting checks pass. -- Two complete-suite runs reached 977 and 975 API passes respectively. The - remaining failures were the documented roaming HTTP/Mongo test-harness - failures in unrelated specs; every affected spec, including virtual - determinism, passes in isolation. -- Focused canonical-domain migration, virtual-filter, and bundle regressions: - 110 passing. -- Application and migration lint plus diff checks pass. -- The pre-batching clean full-suite baseline was OpenAPI 2, config 21, API 978, - middleware 29, and scheduler 10. - -## ATT&CK v19.1 canonical domain repair - -- [x] Add a dry-run/apply operational migration that derives canonical - `x_mitre_domains` values from object presence across the Enterprise, ICS, - and Mobile v19.1 collection TOCs. -- [x] Repost each affected latest active object through its normal create - endpoint so the repair creates a new revision and triggers ordinary - release-track member synchronization. -- [x] Preserve canonical multi-domain arrays during legacy and ephemeral bundle - export instead of narrowing them to the requested bundle domain. -- [x] Keep virtual `filters.domains` matching inclusive: any matching canonical - domain includes an object, while no matching domain excludes it. -- [x] Document the canonical-domain contract and the required follow-up - standard-track release after the repair creates new candidate revisions. -- [x] Run focused migration and API regressions, then lint, OpenAPI validation, - and the complete `npm test` suite. - -Verification (2026-07-30): - -- Migration/bootstrap Python regressions: 20 passing. -- Focused bundle and virtual-domain API regressions: 23 passing. -- Lint and diff checks pass. -- Complete server suite passes: OpenAPI 2, config 21, API 973, middleware 29, - and scheduler 10. - -## Bootstrap hotfix — ATT&CK-branded track names - -- [x] Permit ampersands in release-track names at the request and persistence - validation boundaries. -- [x] Add an API regression using the production bootstrap name - `Enterprise ATT&CK`. -- [x] Align OpenAPI and developer naming documentation with the accepted - contract. - -## Production-readiness branch — `fix/release-tracks-production-readiness` - -This branch implements the prioritized findings in -`.nocommit/project-review-release-tracks/15-recommendations.md`. Each numbered -recommendation is kept as a separate conventional commit so the merge request -can be reviewed or reverted item by item. - -### Current implementation slice — Immutable snapshot history - -- [x] Remove direct latest and historical snapshot member-replacement - endpoints (`POST /:id/contents` and - `POST /:id/snapshots/:modified/contents`) from routes, controllers, - services, validation, and OpenAPI. -- [x] Remove historical metadata rewriting - (`POST /:id/snapshots/:modified/meta`); retain latest metadata updates, - which create a new draft snapshot. -- [x] Permit snapshot deletion only for the latest untagged draft and return a - typed `409 Conflict` for tagged or historical snapshots. -- [x] Replace test setup that depended on direct member replacement with - supported bootstrap/candidate/promotion/release workflows, and add - regressions for the removed endpoints and deletion boundary. -- [x] Update user/developer/admin documentation and the Angular, Python, and - Bruno clients so no downstream surface suggests that persisted snapshot - history can be rewritten. -- [x] Run focused server/client checks, then the complete server `npm test` - suite, and record the results. - -Verification (2026-07-30): - -- Server: focused release-track regression files, OpenAPI validation, lint, - and the complete `npm test` suite passed. -- Angular: connector regression tests, changed-file formatting/lint checks, - and the complete frontend test suite passed. -- `internalattack`: release-track regression tests, changed-file Ruff checks, - and the complete Python test suite passed. -- Bruno: the release-track collection no longer contains the removed mutation - requests, and its modified request files pass scoped whitespace validation. - -### P0.1 — Enforce release version uniqueness - -- [x] Add a unique partial index for tagged `version` strings in every dynamic - release-track snapshot collection. -- [x] Convert duplicate-version races into a typed `409 Conflict` that - identifies the track and requested version. -- [x] Add a regression that releases two distinct drafts concurrently with the - same version and proves exactly one succeeds. -- [x] Adopt the pre-release reset policy for collections created with the - former non-unique index. No shared deployment retains beta release-track - data, so this change deliberately does not establish a permanent - migration contract for local development state. -- [x] Update release-version documentation and run focused, middleware, and - lint verification. -- [ ] Obtain one clean aggregate `npm test` run for the migration cleanup. Three - attempts exposed the repository's roaming cross-spec isolation failure; - every affected spec passed immediately in isolation. - -Original implementation verification (2026-07-30): - -- The deterministic concurrent-release, middleware, and isolated - roaming-failure group passes. -- The required clean full suite passes: OpenAPI 2, config 21, API 947, - middleware 25, and scheduler 10. -- New dynamic track collections create the unique partial index before their - initial snapshot is persisted. Existing personal development tracks created - under the former beta schema are reset or recreated. - -Pre-release migration cleanup verification (2026-07-30): - -- Release planning and concurrent-version coverage passes all 20 cases. -- Error middleware passes all 9 cases; lint and diff checks pass. -- Aggregate attempts failed in different, unrelated modules: user accounts, - groups, releases-by-object, and tagged-content immutability. Those modules - pass in isolation (9, 23, 8, and 1 cases respectively), confirming no - reproducible migration-cleanup regression. - -### P0.2 — Make primary release membership fail closed - -- [x] Add one shared batch hydrator that resolves dynamic selectors, validates - every exact `(object_ref, object_modified)` pair, and reports all missing - primary revisions without swallowing repository failures. -- [x] Reject nonexistent exact candidate pins, candidate pin updates, the - then-supported direct member replacement requests, track cloning, and - virtual materialization before snapshot persistence. Direct replacement - was subsequently removed by the immutable-history slice. -- [x] Revalidate existing and promoted members at release-preview and - release-commit boundaries; return a typed `409 Conflict` for corrupt stored - drafts. -- [x] Abort bundle import before creating a track when any authoritative - primary object failed to import or cannot be hydrated. -- [x] Abort bundle/workbench export when selected primary revisions cannot be - hydrated; return every missing reference instead of a partial result. -- [x] Add ingress, partial-import, deleted-staged-revision, virtual - materialization, and incomplete-export regressions. -- [x] Update user/developer documentation and run focused, lint, OpenAPI, and - complete-suite verification. - -Verification result (2026-07-30): - -- The shared integrity, release, export, virtual determinism/quarantine, and - middleware regression group passes (54); the complete release-track API - regression group passes (143). -- OpenAPI validation (2) and backend lint pass. -- The required full suite passes: OpenAPI 2, config 21, API 955, middleware - 27, and scheduler 10. -- Bruno documents the structured `400`/`409` integrity response on companion - branch `fix/release-tracks-production-readiness`. - -### Remaining prioritized recommendations - -### P0.3 — Make tagged-content immutability authoritative and durable - -- [x] Guard object revision update/delete and delete-all by querying tagged - snapshot membership, even when `workspace.release_tracks` is missing or - stale. -- [x] Make release-track backref reconciliation failures propagate to the - triggering request so a release is never reported as fully successful when - protection writes failed. -- [x] Persist every reconciliation attempt and its terminal outcome so - process crashes and partial listener failures remain operator-visible. -- [x] Provide an idempotent repair command for failed/pending reconciliation - records and a full-scan mode for legacy drift. -- [x] Add failure-injection, missing-backref, repair, and historical-release - regressions; update user/developer/admin documentation. -- [x] Run focused, lint, OpenAPI, and complete-suite verification. - -Verification result (2026-07-30): - -- Lint and OpenAPI validation pass. -- The complete release-track API group passes (146), including - failure-injection, repair, and authoritative historical-membership - regressions. Scheduler/date/cron integration (15) and middleware (11) - focused groups pass. -- Repeated complete-suite runs execute all 958 API cases and consistently - pass the release-track cases. The repository's documented roaming - Supertest transport flake still moves among unrelated isolated-pass cases - (socket resets, transient status mismatches, or timeouts). Dynamic - release-track models are now evicted between dropped test databases and the - Mongoose connection is reused, reducing the API run from roughly six - minutes to roughly one minute; the remaining unrelated transport flake is - tracked separately from this completed integrity change. - -### P0.4 — Correct destructive authorization and add durable audit records - -This records the earlier beta contract. The immutable-history slice later -removed both member-replacement routes and their audit action types; durable -auditing now applies only to full-track deletion. - -- [x] Require administrator authorization for full track deletion and both - direct member-replacement routes. -- [x] Require an exact `confirm_track_id` precondition on each destructive - request so stale or accidental UI actions fail before persistence. -- [x] Persist a durable, actor-attributed audit event before each operation - and record completion or failure without hiding partial persistence. -- [x] Add an authorization matrix and operator-facing audit documentation. -- [x] Update OpenAPI, frontend tasks, and Bruno requests for the confirmation - contract. -- [x] Add admin/editor, missing/mismatched confirmation, success/failure - audit, lint, OpenAPI, focused, and complete-suite verification. - -Verification result (2026-07-30): - -- Lint and OpenAPI validation pass. -- The focused authorization/audit and middleware group passes (11). The - complete release-track and virtual-scheduler group passes all 148 relevant - cases; one roaming setup 404 passed immediately in isolation (8). -- The required clean full suite passes: OpenAPI 2, config 21, API 960, - middleware 29, and scheduler 10. -- The `internalattack` focused release-track suite passes (33), its complete - suite passes (246), and changed-file Ruff checks pass. - -- [ ] P0.5 — Complete the Angular contract migration and end-to-end smoke gate. -- [ ] P0.6 — Finish scheduled-materialization fencing, retry bounds, and - operator intervention. -- [ ] P0.7 — Establish and enforce a safe storage operating envelope. -- [ ] P0.8 — Harden deployment, database readiness, backup/restore, rollback, - and post-deploy verification. -- [ ] Address P1 recommendations in documented criticality order. - -## Current implementation slice — Deterministic snapshot bundle graphs - -This slice replaces export-time relationship and secondary-object discovery -with a frozen graph manifest for every persisted release-track snapshot. The -manifest is authoritative for bundle replay and for protecting every exact -revision on which that bundle depends. - -### Shared graph resolution - -- [x] Extract the bounded ATT&CK graph-selection rules from the mutable - `stix-bundles-service` singleton into a request-local resolver. -- [x] Preserve the existing one-hop and named special-case behavior for - `detects`, `attributed-to`, `revoked-by`, detection strategies, analytics, - and required supporting objects without introducing unrestricted graph - traversal. -- [x] Make the legacy/ephemeral exporter and release-track snapshot capture - use thin adapters around the same resolver. -- [x] Add parity and concurrent-request regression coverage before changing - release-track persistence. - -### Revision-pinned relationships - -- [x] Store server-controlled exact source and target revision pins under - `workspace.relationship_endpoints`; do not add non-ADM fields to emitted - STIX payloads. -- [x] Resolve endpoint revisions when relationships are created, including - bundle-import and automated relationship-creation paths, and fail closed - when an exact endpoint cannot be established. -- [x] Create new SRO revisions when a referenced SDO advances instead of - mutating an existing `(stix.id, stix.modified)` revision. -- [x] Reject in-place source, target, and relationship-type changes. - Description-only corrections remain allowed because manifests freeze the - relationship STIX payload used by existing snapshots. - -### Snapshot graph manifests - -- [x] Persist an exact, tier-aware manifest for every newly created standard - and virtual snapshot. Include primary roots, relationships, secondary - objects, identities, marking definitions, and LinkById render dependencies. -- [x] Store manifest entries in one indexed collection so exact revision - hydration and mutation-protection checks do not scan dynamic snapshot - collections. -- [x] Make snapshot capture fail closed and concurrency-safe. Complete - manifests are linked before activation; linked pending manifests remain - replayable and self-activate after an interrupted write. -- [x] Replace a standard draft's manifest from the resolved release plan in - the same conditional tag update, so staged `"latest"` selectors become - exact released members without creating a second snapshot timestamp. -- [x] Replay `format=bundle` entirely from the frozen manifest, with no live - relationship, secondary, supporting-object, or LinkById discovery. The - deliberate exception is an explicitly included standard draft tier whose - stored selector is `"latest"`; that tier remains dynamic until release. - -### Mutation and deletion protection - -- [x] Centralize exact graph-pin checks in the shared object service layer. -- [x] Return `409 Conflict` when a PUT or exact-revision delete would mutate - an emitted revision referenced by any active or pending manifest. -- [x] Reject lineage deletion when any revision in the lineage is manifest - referenced; administrator authorization must not bypass graph integrity. -- [x] Remove protection entries when their owning snapshot or track is - deleted, with idempotent reconciliation for interrupted cleanup. - -### Migration, documentation, and verification - -- [x] Add a dry-run-capable, idempotent migration that resolves endpoint pins - for the latest revision of each relationship directly from the underlying - collections. Do not write through `view.relationships.latest`. -- [x] Backfill existing snapshots with manifests reconstructed from the graph - visible at migration time and label them as baseline reconstructions rather - than historically exact captures. -- [x] Add regression coverage for newer SDO/SRO revisions, relationship - deprecation, missing dependencies, PUT/delete guards, standard and virtual - snapshots, query-tier filtering, migration reruns, and concurrent capture. -- [x] Update OpenAPI error contracts, user/developer/admin documentation, - frontend guidance, `internalattack`, and Bruno where the observable contract - changes. No request route or parameter changed, so generated clients and - Bruno request definitions require no transport change. -- [x] Run focused specs while iterating, then lint, OpenAPI validation, and the - complete `npm test` suite. - -### Production-shaped migration repair - -- [x] Scope legacy endpoint pinning to active latest relationships, matching - the relationship set eligible for deterministic snapshot graphs. -- [x] Preserve fail-closed behavior for active dangling relationships while - allowing deprecated or revoked dangling history to remain untouched. -- [x] Surface actionable missing-endpoint diagnostics in the migration preview - and startup failure. -- [x] Establish manifest indexes independently of whether relationship pin - updates happen to be required. -- [x] Add production-shaped migration regressions and update the operator - documentation. - -Verification result (2026-07-30): - -- The focused migration spec passes all 4 cases. -- A read-only preview against the restored production database scans 24,818 - active relationship revisions without encountering the 118 dangling - endpoints confined to deprecated relationship history. -- Lint, formatting, diff checks, and the complete `npm test` suite pass; the - API suite passes all 972 cases. - -- [x] Propose conventional commits split by independently reviewable - architectural slice; do not commit until requested. - -Verification result (2026-07-30): - -- The focused deterministic-graph group passes all 101 cases; the final - compatibility group for relationship pagination, reports, and collection - imports passes all 27 cases. -- Lint, formatting, OpenAPI validation, and diff checks pass. -- The required clean full suite passes: OpenAPI 2, config 21, API 971, - middleware 29, and scheduler 10. - -## Current implementation slice — Scheduler regression and virtual schedule coverage - -- [x] Repair the legacy collection-index scheduler spec so it imports the - refactored `sync-collection-indexes-task` module without auto-registering - background jobs during the test. -- [x] Add virtual-track coverage proving reconciliation registers scheduled - cron jobs in UTC and removes jobs for tracks that no longer exist. -- [x] Add date-schedule boundary coverage for multiple due dates and future - dates. -- [x] Add crash-window recovery coverage for a scheduled virtual snapshot that - was persisted before its occurrence ledger reached `completed`. -- [x] Add stale-claim recovery coverage and document any remaining - multi-process lease/fencing limitation. -- [x] Run the legacy scheduler spec, the virtual scheduler spec, the aggregate - scheduler suite, lint, and the complete `npm test` suite. -- [x] Record the coverage conclusion and propose a conventional commit message. - -Coverage conclusion (2026-07-30): - -- Functional coverage is sufficient for the current `manual`, `cron`, and - `dates` contracts. It exercises UTC cron registration and cleanup, duplicate - delivery, multiple due and future dates, missed-date recovery, retryable - component failures, expired claims, and recovery after snapshot persistence. -- Scheduler regressions now run under the default `npm test` and Cobertura - coverage gates instead of requiring a separate developer-only command. -- Remaining production hardening is explicitly tracked below; it does not - change the single-worker schedule contract covered by this slice. - -Verification result (2026-07-30): - -- The deterministic legacy collection-index scheduler spec passes (2), the - expanded virtual scheduler spec passes (8), and the aggregate scheduler - suite passes (10). -- Backend lint passes. The required clean full suite passes: OpenAPI 2, - config 21, API 945, middleware 24, and scheduler 10. -- Three roaming API-suite failures seen during earlier runs passed together in - isolation (23) before the clean full-suite run. - -### Remaining scheduled-materialization hardening - -- [ ] Add an owner token (fencing token) to occurrence claims, make terminal - updates conditional on the active token, and renew leases for work that may - exceed the claim duration. Add a true multi-worker regression proving that - an expired worker cannot overwrite the succeeding worker's result. -- [ ] Decide and document an operator policy for permanent failures. If - indefinite one-minute retries are not acceptable, add bounded exponential - backoff plus a terminal/dead-letter state and operator-visible recovery - controls. - -## Current implementation slice — Deterministic standard releases - -- [x] Preserve `modified: "latest"` and omitted candidate selectors as dynamic - references through the candidate and staged tiers; preserve explicit - timestamps as exact revision pins. -- [x] Resolve every dynamic staged reference to the actual latest - `stix.modified` timestamp during standard release planning, before conflict - detection, preview rendering, or commit. -- [x] Ensure tagged members contain exact revisions only and that preview and - commit use the same release-planning rules. -- [x] Make dynamic candidate/staged references safe in tier comparison, - Workbench enrichment, bundle rendering, back-reference reconciliation, and - member-sync paths. -- [x] Add regression coverage for dynamic and explicit candidate promotion, - release-time resolution after a newer revision is created, historical - release targeting, conflict handling, and member immutability. -- [x] Update OpenAPI, user/developer documentation, frontend guidance, - `internalattack`, and Bruno as required by the corrected contract. -- [x] Run focused tests, lint, OpenAPI validation, and the complete `npm test` - suite. -- [x] Apply logic review, inspect the final diff, and propose conventional - commit messages. - -Verification result (2026-07-30): - -- The combined release, back-reference, change-capture, bundle, - tier-invariant, and virtual-determinism regression group passes (81); the - strengthened release-planning spec passes (19). -- OpenAPI validation passes (2), backend lint passes, and the required clean - full suite passes with routine logs suppressed (OpenAPI 2, config 21, API - 945, middleware 24). -- The focused `internalattack` release-track suite passes (30), its complete - suite passes (247), and changed-file Ruff checks pass. -- Relevant REST API, `internalattack`, and Bruno diffs pass whitespace checks. - -## Virtual release tracks - -This section records the 2026-07-29 documentation-to-implementation audit of -virtual release tracks. Items are ordered by integrity risk and implementation -dependency. A checked item must include regression coverage and any necessary -OpenAPI, user/developer documentation, client, and Bruno updates. - -The completed P0 implementation and verification records remain in the dated -sections below. The following items constitute the active virtual-track -completion backlog. - -### P1 — Composition validation and deterministic resolution - -- [x] Make request validation strict so misspelled keys such as - `filters.domain` return 400 instead of silently disabling filtering. -- [x] Validate component selectors according to `resolution_strategy`: - - `specific_version` requires `version` and rejects `snapshot`; - - `specific_snapshot` requires `snapshot` and rejects `version`; - - `latest_tagged` rejects both selector fields. -- [x] Make `priority` consistently required in Zod, Mongoose, OpenAPI, docs, - and examples; reject duplicate priorities at the request boundary. -- [x] Validate component existence, standard-track type, duplicate track IDs, - and duplicate priorities when a virtual track is initially created, not only - when composition is later updated or materialized. -- [x] Validate `snapshot_schedule` by mode: - - `manual` rejects `cron` and `dates`; - - `cron` requires `cron` and rejects `dates`; - - `dates` requires at least one date and rejects `cron`. -- [x] Constrain or document accepted `filters.object_types` values and add - direct regression coverage for exact-revision filtering. - -### P1 — Deduplication correctness - -- [x] Treat the same exact object revision contributed by multiple components - as one duplicate, not a conflicting revision. -- [x] Ensure the `quarantine` strategy only quarantines genuinely different - revisions of the same object. -- [x] Attribute each surviving revision to one deterministic component so - `objects_contributed` totals cannot exceed `summary.total_objects`. -- [x] Add dedicated tests for all four strategies: - `prioritize_latest_object`, `prioritize_latest_snapshot`, - `prioritize_higher_priority`, and `quarantine`. - -### P1 — Release provenance - -- [x] Populate virtual release `version_history[].component_versions` from the - materialized snapshot's immutable `composition_resolution`. -- [x] Define and test the provenance shape in Mongoose, OpenAPI, and user and - developer documentation. - -### P2 — Scheduled materialization - -- [x] Connect virtual `snapshot_schedule` metadata to the existing task - scheduler. This is required for virtual-track completion, not an optional - future enhancement. -- [x] Implement `cron` execution so each matching schedule occurrence - materializes a new virtual draft through the same lifecycle and validation - used by `POST /api/release-tracks/:id/virtual/snapshots/create`. -- [x] Implement `dates` execution so every configured timestamp materializes - exactly one virtual draft, including deterministic handling for restart - recovery, missed timestamps, and duplicate-delivery prevention. -- [x] Preserve `manual` semantics: store no executable schedule and create - drafts only through the explicit virtual snapshot-creation endpoint. -- [x] Define failure behavior when a component has no matching tagged - snapshot, including automation-run audit records and retry policy. -- [x] Add scheduler integration tests for both `cron` and `dates`, including - successful execution, restart recovery, idempotency, component-resolution - failure, and retry behavior. -- [x] Add operational documentation covering scheduler activation, UTC - interpretation, observability, failures, and retries. - -### Current implementation slice — Scheduled virtual materialization - -- [x] Add a scheduler reconciliation task for persisted virtual-track - `cron` and `dates` schedules while preserving explicit-only `manual` mode. -- [x] Persist schedule occurrences and claim them atomically so multiple - scheduler instances cannot concurrently process the same occurrence. -- [x] Make snapshot persistence idempotent by recording the scheduled - occurrence on the resulting virtual draft. -- [x] Recover missed `dates` occurrences and failed `cron` or `dates` - occurrences during reconciliation. -- [x] Record every materialization attempt in the automation-run audit trail. -- [x] Add scheduler integration coverage for success, restart recovery, - duplicate delivery, component failure, and retry. -- [x] Update OpenAPI, user/developer/operations documentation, frontend - guidance, and Bruno. -- [x] Run focused scheduler tests, lint, and the complete `npm test` suite. -- [x] Review the final diff and propose conventional commit messages. - -Verification result (2026-07-29): - -- The focused scheduler integration spec passes (4), OpenAPI validation - passes (2), and backend lint passes. -- The first complete run encountered five unrelated roaming failures after - 933 API tests passed. Each affected spec passed in isolation. -- The required clean `npm test` rerun passes (OpenAPI 2, config 21, API 938, - middleware 24). -- Proposed REST API commit: - - ```text - feat(release-tracks): schedule virtual snapshot materialization - - Execute persisted cron and date schedules through the existing virtual - snapshot lifecycle. Add durable occurrence claims, restart-safe - idempotency, automation-run auditing, retry behavior, scheduled snapshot - provenance, and aligned API and operations documentation. - ``` - -- Proposed companion Bruno commit: - - ```text - docs(release-tracks): document scheduled materialization - - Describe UTC cron and date execution, restart recovery, idempotency, - and retry behavior for virtual snapshot schedules. - ``` - -### P2 — Contract decisions - -- [x] Virtual tracks cannot compose virtual tracks. Components must be - standard tracks; revisit nesting only if a concrete future use case requires - it. -- [x] Do not implement the documented native-members/hybrid model. Virtual - tracks are purely compositional; content that is not already represented - belongs in a dedicated standard component track. -- [x] Do not implement `resolve=true` or `resolved_content`. Virtual - composition is resolved eagerly into exact object revisions when a draft is - materialized; retrieval must never re-resolve a persisted snapshot. -- [x] Do not implement caching or component-release notifications without - measured scale or an approved operator workflow. Persisted snapshots already - avoid composition recomputation, and no notification recipient, channel, or - expected action has been defined. - -### Current implementation slice — Deterministic virtual membership - -- [x] Resolve the `latest` request shorthand to the actual latest - `stix.modified` value before standard-track contents are persisted. -- [x] Defensively lock any unresolved component member to an exact revision - during virtual materialization, while preserving exact revisions already - frozen into tagged component snapshots. -- [x] Add regression coverage proving that component `track_latest` behavior - cannot move a materialized virtual member and repeated snapshot retrieval - returns the same exact revision set. -- [x] Remove `resolve=true` and `resolved_content` from the documented - retrieval contract. -- [x] Initially document the distinction between deterministic primary - membership and the then-dynamic bundle graph; the later deterministic graph - manifest slice below supersedes that accepted limitation. -- [x] Update OpenAPI, frontend guidance, and Bruno where the clarified - contract affects consumers. -- [x] Run focused tests, lint, OpenAPI validation, and the complete `npm test` - suite. -- [x] Apply logic review, inspect the final diff, and propose conventional - commit messages. - -Verification result (2026-07-29): - -- The dedicated virtual-determinism spec passes (2), and the combined - determinism, release-track lifecycle, and virtual-domain regression group - passes (4). -- OpenAPI validation passes (2), backend lint passes, and the complete - `npm test` suite passes (OpenAPI 2, config 21, API 941, middleware 24). -- Logic review result: `ROBUST`. Request-time `latest` resolution, immutable - tagged component pins, legacy unresolved-member locking, invalid-date - rejection, and repeated-reference resolution were covered without finding a - remaining correctness defect. -- Proposed commits: - - ```text - fix(release-tracks): enforce pure virtual composition - - Require virtual components to be standard tracks and reject unsupported - native-member input across the API contract and documentation. - ``` - - ```text - fix(release-tracks): freeze virtual member revisions - - Resolve latest member shorthand before persistence, lock virtual composition - to exact revisions, and document the bundle graph consistency boundary. - ``` - - ```text - docs(release-tracks): clarify snapshot determinism - - Document exact virtual member pins and the snapshot graph consistency - boundary in the Bruno collection. - ``` - -### Future architecture — Deterministic bundle graphs - -- [x] Design version-controlled STIX Relationship Objects whose source and - target references identify exact `(object_id, object_modified)` revisions - rather than an entire STIX object provenance chain. -- [x] Evaluate cloning every affected SRO when a new SDO revision is created, - including atomicity, fan-out, concurrency, migration, and rollback behavior. -- [x] Avoid cloning SROs per snapshot by recording exact endpoint metadata on - each SRO revision and storing compact manifest references plus a frozen SRO - payload where description-only PUT compatibility requires it. -- [x] Define and persist an export manifest that pins every secondary object, - supporting object, and relationship revision required to reproduce a bundle. -- [x] Document the resulting guarantee: the emitted object graph is - deterministic, while the generated bundle-envelope UUID is not byte-stable. - -### Current implementation slice — Pure standard-track composition - -- [x] Make standard component tracks a positive service-layer requirement, - preserving rejection during both virtual-track creation and composition - replacement. -- [x] Reject unsupported top-level creation properties such as - `native_members` instead of silently stripping them. -- [x] Add regression coverage for virtual-track nesting on both creation and - composition update, and for attempted native-member creation. -- [x] Remove nesting and hybrid/native-member claims from OpenAPI, user and - developer documentation, frontend guidance, and Bruno. -- [x] Run the focused virtual-composition spec, lint, and complete `npm test` - suite. -- [x] Review the final diff and propose conventional commit messages. - -Verification result (2026-07-29): - -- The focused virtual-composition validation spec passes (6), OpenAPI - validation passes (2), and backend lint passes. -- The first complete run encountered six unrelated roaming failures after - 910 API tests passed. All affected specs passed in isolation. -- The required clean `npm test` rerun passes in full, including OpenAPI, - configuration, API, and middleware suites. -- Architecture review result: the positive standard-track allowlist and strict - creation schema keep the contract explicit without adding a parallel - composition path or new abstraction. -- Proposed REST API commit: - - ```text - fix(release-tracks): enforce pure virtual composition - - Require every virtual component to be a standard track during creation and - composition updates. Reject unsupported native-member input and align - OpenAPI, documentation, frontend guidance, and regression coverage. - ``` - -- Proposed companion Bruno commit: - - ```text - docs(release-tracks): clarify pure virtual composition - - Document standard-only components, rejected virtual nesting, and the absence - of native virtual members. - ``` - -### Documentation corrections - -- [ ] Replace `stix.type = "virtual"` with the top-level snapshot - `type: "virtual"`. -- [ ] Remove the nonexistent snapshot-level `snapshot_id`; retain - `version_history[].snapshot_id`. -- [ ] Correct response envelopes and the virtual-create response example. -- [ ] Align `composition_resolution` examples with fields actually generated, - or implement the documented `by_type`, `by_tier`, and native statistics. -- [ ] Align documented error envelopes with centralized error-handler output. -- [x] Include required `priority` values in every composition example. -- [x] Clearly distinguish configured composition from a materialized draft and - document scheduler activation, timing, recovery, and retry behavior. - -### Verified complete - -- [x] Composition changes invalidate inherited materialized contents and - require explicit rematerialization before release. -- [x] Generic contents replacement rejects virtual tracks. -- [x] Exact-revision quarantine resolution is available at - `POST /api/release-tracks/:id/virtual/quarantine/promote`. -- [x] `filters.domains` hydrates and evaluates exact pinned revisions. -- [x] Public domain names and STIX `*-attack` names are normalized. -- [x] Multiple domain values are supported. -- [x] Objects without domain metadata are excluded when a domain filter is set. -- [x] Primary Enterprise, ICS, and Mobile matrices use their ATT&CK external ID - as the established domain fallback. -- [x] Virtual tracks resolve only tagged snapshots and consume only component - `members`. -- [x] Virtual tracks maintain independent draft/release history and use the - shared snapshot retrieval and release endpoints after materialization. - -### Current implementation slice — Strict composition contracts - -- [x] Add API regression coverage for unknown composition/filter keys on both - virtual-track creation and composition update. -- [x] Require the selector appropriate to each `resolution_strategy` and - reject selectors that do not apply to that strategy. -- [x] Make the composition, component, filter, and deduplication request - objects strict without changing persisted response shapes. -- [x] Update OpenAPI, user/developer documentation, and Bruno examples. -- [x] Run the focused regression spec, then lint and the complete `npm test` - suite. -- [x] Review the final diff and propose a conventional commit message. - -Verification result (2026-07-29): - -- The focused virtual-composition contract spec passes (3), OpenAPI validation - passes (2), and backend lint passes. -- The first complete run encountered one roaming 404 in the new spec after 917 - API tests passed. The spec passed both in isolation (3) and alongside its - preceding snapshot-history spec (10). -- The required clean `npm test` rerun passes (OpenAPI 2, config 21, API 918, - middleware 24). -- Proposed commit: - - ```text - fix(release-tracks): validate virtual composition contracts - - Reject unknown composition properties and enforce strategy-specific - component selectors across virtual-track creation and updates. Align - OpenAPI, documentation, frontend guidance, and Bruno examples. - ``` - -### Current implementation slice — Component identity and priority validation - -- [x] Add creation and composition-update regression coverage for required - priorities, duplicate priorities, and duplicate component track IDs. -- [x] Reject missing component tracks and virtual component tracks before an - initial virtual track is persisted. -- [x] Make component priority required and non-negative across Zod, Mongoose, - OpenAPI, user/developer documentation, and Bruno examples. -- [x] Keep service-layer component validation as a defense for non-HTTP - callers while moving deterministic duplicates to request validation. -- [x] Run the focused regression specs, then lint and the complete `npm test` - suite. -- [x] Review the final diff and propose a conventional commit message. - -Verification result (2026-07-29): - -- The focused release-track regression group passes (22), the isolated - backrefs spec passes (23), OpenAPI validation passes (2), and backend lint - passes. -- The first complete run encountered one unrelated shared-suite failure in the - backrefs manual-sync case after 919 API tests passed. The affected spec - passed in isolation (23). -- The required clean `npm test` rerun passes (OpenAPI 2, config 21, API 920, - middleware 24). -- Proposed commit: - - ```text - fix(release-tracks): validate virtual component identities - - Require unique component priorities and track IDs, validate referenced - standard tracks before initial virtual-track persistence, and align request, - persistence, OpenAPI, documentation, and frontend contracts. - ``` - -- Proposed companion Bruno commit: - - ```text - docs(release-tracks): document component priority constraints - - Document required unique priorities and standard component references for - virtual-track creation and composition updates. - ``` - -### Current implementation slice — Snapshot schedule contracts - -- [x] Add creation regressions for valid and invalid `manual`, `cron`, and - `dates` schedule payloads. -- [x] Enforce a strict mode-discriminated request contract: - - `manual` accepts only `mode`; - - `cron` requires `cron` and rejects `dates`; - - `dates` requires at least one date and rejects `cron`. -- [x] Reject `snapshot_schedule` on standard-track creation instead of silently - dropping it. -- [x] Repeat schedule invariants at the service and Mongoose boundaries for - non-HTTP callers. -- [x] Align OpenAPI, user/developer documentation, frontend guidance, the - `internalattack` test fixture, and Bruno. -- [x] Run focused regression specs, lint, and the complete `npm test` suite. -- [x] Review the final diff and propose conventional commit messages. - -Verification result (2026-07-29): - -- The focused schedule-contract spec passes (7), the focused virtual-track - regression group passes (29), OpenAPI validation passes (2), and backend - lint passes. -- The required complete `npm test` suite passes (OpenAPI 2, config 21, API - 927, middleware 24). -- The `internalattack` focused release-track suite passes (30), and its - complete suite passes (247). -- Proposed REST API commit: - - ```text - fix(release-tracks): validate virtual snapshot schedules - - Enforce strict mode-specific virtual snapshot schedules across request, - service, persistence, OpenAPI, documentation, and frontend contracts. - Reject schedule metadata for standard tracks. - ``` - -- Proposed companion Bruno commit: - - ```text - docs(release-tracks): document snapshot schedule modes - - Document the strict manual, cron, and dates schedule payloads and clarify - that automated execution is not yet implemented. - ``` - -- Proposed companion `internalattack` commit: - - ```text - test(release-tracks): align virtual composition fixture - - Include the required component priority in virtual-track creation coverage. - ``` - -### Current implementation slice — Object-type filter contracts - -- [x] Define `filters.object_types` against the canonical Workbench STIX type - vocabulary instead of accepting arbitrary strings. -- [x] Reject empty arrays, duplicate values, malformed values, and unsupported - object types on both virtual-track creation and composition update. -- [x] Repeat the accepted-value constraint at the Mongoose persistence - boundary. -- [x] Add direct materialization coverage proving that object-type filtering - preserves the exact revision pinned by the tagged component snapshot rather - than resolving the latest database revision. -- [x] Align OpenAPI, user/developer documentation, frontend guidance, and - Bruno; verify whether `internalattack` needs a typed client change. -- [x] Run focused regression specs, lint, and the complete `npm test` suite. -- [x] Review the final diff and propose conventional commit messages. - -Verification result (2026-07-29): - -- The dedicated object-type contract and exact-revision materialization spec - passes (5); the combined virtual composition, domain, schedule, and - object-type filter group passes (18). -- OpenAPI validation passes (2), backend lint passes, and the required clean - `npm test` run passes (OpenAPI 2, config 21, API 923, middleware 24). -- Earlier complete runs encountered unrelated shared-suite flakes in user - account startup, analytics socket handling, and campaign/group HTTP - handling. The affected specs pass in isolation (14, 12, and 44 - respectively). -- `internalattack` already accepts composition filters as a mapping, so this - contract clarification does not require a typed client change. -- Proposed REST API commit: - - ```text - fix(release-tracks): validate virtual object type filters - - Constrain virtual component object-type filters to the canonical Workbench - STIX vocabulary across request, service, persistence, OpenAPI, and - documentation boundaries. Preserve exact component snapshot revisions. - ``` - -- Proposed Bruno commit: - - ```text - docs(release-tracks): document object type filters - - Document canonical virtual component object-type values, omission semantics, - and exact-revision behavior. - ``` - -### Current implementation slice — Deterministic virtual deduplication - -- [x] Add materialization regressions for all four deduplication strategies - using both an exact revision shared by multiple components and genuinely - different revisions of the same STIX object. -- [x] Collapse repeated contributions of the same `(object_ref, -object_modified)` revision before applying conflict resolution. -- [x] Count an object contributed by multiple components once in - `duplicates_found`, but include it in `conflicts_resolved` only when multiple - distinct revisions remain after exact-revision collapse. -- [x] Choose one deterministic source component for every surviving revision: - use the active strategy's ordering and use component priority as the stable - tie-breaker. -- [x] Quarantine one entry per distinct conflicting revision and leave an - identical revision shared by multiple components in `members`. -- [x] Derive `objects_contributed` from explicit survivor attribution so its - component total equals `summary.total_objects`. -- [x] Align OpenAPI, user/developer documentation, frontend guidance, Bruno, - and `internalattack` if the clarified response semantics require downstream - changes. -- [x] Run focused regression specs, lint, and the complete `npm test` suite. -- [x] Review the final diff and propose conventional commit messages. - -Verification result (2026-07-29): - -- The dedicated four-strategy deduplication spec passes (4); the combined - deduplication, quarantine, and back-reference release-track group passes - (29). -- OpenAPI validation passes (2), backend lint passes, and the required clean - `npm test` run passes (OpenAPI 2, config 21, API 936, middleware 24). -- An earlier complete run encountered unrelated shared-suite 404, 400, and - connection-reset failures in Recent Activity, References, and Ephemeral - Bundle tests. Those three specs pass together in isolation (30). -- `internalattack` exposes the resolution response as an untyped mapping, so - the clarified metric semantics do not require a Python client change. -- Performance review result: `PERFORMANT`. The implementation replaces the - prior input-to-output nested survivor scan with linear source attribution; - no database, blocking, or resource-management regression was found. -- Proposed REST API commit: - - ```text - fix(release-tracks): deduplicate virtual revisions deterministically - - Collapse exact component revision duplicates before resolving conflicts, - attribute every surviving member to one deterministic source, and quarantine - only genuinely different revisions. - ``` - -- Proposed Bruno commit: - - ```text - docs(release-tracks): clarify virtual deduplication - - Document exact-revision collapse, genuine conflict handling, and deterministic - component contribution accounting. - ``` - -### Current implementation slice — Virtual release provenance - -- [x] Add release preview and commit regressions proving that virtual - `version_history[].component_versions` comes from the selected draft's - immutable `composition_resolution`, even if a component is released again - before the virtual draft is tagged. -- [x] Define `component_versions` as an optional object keyed by immutable - component track ID with tagged `MAJOR.MINOR` version values. -- [x] Populate provenance only for virtual release history entries and leave - standard release history unchanged. -- [x] Enforce the provenance value shape at the Mongoose persistence boundary - and describe it in OpenAPI. -- [x] Align user/developer documentation, frontend guidance, Bruno, and - `internalattack` if the response contract requires downstream changes. -- [x] Run focused regression specs, lint, and the complete `npm test` suite. -- [x] Apply logic and performance review checklists, inspect the final diff, - and propose conventional commit messages. - -Verification result (2026-07-29): - -- The focused release-planning and commit spec passes (16), including - workbench preview, in-place release persistence, standard-track omission, - immutable component advancement, and invalid Mongoose key/value cases. -- OpenAPI validation passes (2), backend lint passes, and the required clean - `npm test` run passes (OpenAPI 2, config 21, API 938, middleware 24). -- An earlier complete run encountered unrelated roaming 404s in Attack Objects - pagination and References after 936 API tests passed. The affected specs pass - together in isolation (30). -- `internalattack` returns release preview and commit responses as raw mappings, - so the additive history field requires no Python client change. -- Logic review result: `ROBUST`. Preview and commit both derive provenance from - the selected persisted draft, malformed map keys/values are rejected, and - standard release history remains unchanged. -- Performance review result: `PERFORMANT`. Provenance construction is a linear - in-memory pass over already-loaded component resolution metadata and adds no - database reads, blocking work, or resource lifecycle. -- Proposed REST API commit: - - ```text - fix(release-tracks): record virtual release provenance - - Persist immutable component track versions from the materialized virtual - draft in release history, validate the provenance map, and align API, - documentation, frontend, and regression contracts. - ``` - -- Proposed Bruno commit: - - ```text - docs(release-tracks): document virtual release provenance - - Describe the track-ID-keyed component version map returned by virtual release - previews and commits. - ``` - -### Tracker consolidation - -- [x] Consolidate the virtual-track completion backlog into this section. -- [x] Preserve completed implementation evidence in the dated records below. -- [x] Move the downstream Angular handoff to - `docs/developer/FRONTEND_TODO.md`. -- [x] Remove the superseded root-level tracker files. - -## Document downstream frontend work - -- [x] Inventory the current release-track API contract and recent endpoint, - terminology, lifecycle, validation, and response-shape changes. -- [x] Inspect the Angular release-track consumers so the handoff identifies - concrete downstream work instead of restating backend implementation notes. -- [x] Create `docs/developer/FRONTEND_TODO.md` with task-oriented guidance, - contextual explanations, and acceptance criteria. -- [x] Cross-check the handoff against OpenAPI, user/developer documentation, - Bruno, and the `internalattack` client. -- [x] Review formatting and the final diff. - -Verification result (2026-07-29): - -- The handoff was cross-checked against the current OpenAPI paths, release-track - documentation, Bruno requests, `internalattack` methods, and Angular - release-track consumers. -- `git diff --check` passes. -- Proposed commit: - - ```text - docs(release-tracks): track required frontend updates - - Document the route, request, response, lifecycle, and terminology changes - that the Angular release-track client must adopt. - ``` - -## Implement virtual quarantine resolution - -- [x] Add end-to-end regression coverage for exact-revision quarantine - promotion, snapshot immutability, back-reference reconciliation, validation, - and virtual-track type enforcement. -- [x] Add `POST /api/release-tracks/:id/virtual/quarantine/promote`. -- [x] Promote the selected revision to members in a new draft and remove all - quarantined alternatives for the same object. -- [x] Preserve the immutable composition-resolution record and historical - materialized snapshot. -- [x] Update OpenAPI, user/developer documentation, and Bruno. -- [x] Run focused regression specs, then lint and the complete `npm test` suite. -- [x] Review the final diff and propose a conventional commit message. - -Verification result (2026-07-29): - -- Focused quarantine, release, back-reference, and virtual-domain specs pass - (40); backend lint passes. -- The first complete run encountered six unrelated shared-suite failures in - collection bundles, data-component pagination, and user accounts. All three - specs passed in isolation (30, 13, and 14 tests respectively). -- The required clean `npm test` rerun passes (OpenAPI 2, config 21, API 915, - middleware 24). -- The `internalattack` focused release-track suite passes (30), its complete - suite passes (247), and changed-file Ruff and pre-commit checks pass. -- Proposed commit: - - ```text - feat(release-tracks): resolve virtual quarantine conflicts - - Add an explicitly virtual-scoped endpoint for selecting an exact - quarantined revision into a new draft. Preserve materialization provenance, - reconcile back-references, and update supported clients and documentation. - ``` - -## Harden virtual materialization lifecycle - -- [x] Record the complete virtual-track audit in the dedicated virtual release - tracks section of this file. -- [x] Add regression coverage for stale composition state, unmaterialized - release attempts, and virtual use of standard contents endpoints. -- [x] Clear inherited materialized state when virtual composition changes. -- [x] Require a materialized virtual draft for release preview and commit. -- [x] Restrict generic contents replacement to standard tracks. -- [x] Update OpenAPI, user/developer documentation, and Bruno. -- [x] Run focused regression specs, then the complete `npm test` suite. -- [x] Review the final diff and propose a conventional commit message. - -Verification result (2026-07-29): - -- Focused release, back-reference, release-by-object, and virtual-domain specs - pass (46); backend lint passes. -- The complete suite passes (OpenAPI 2, config 21, API 913, middleware 24). -- Proposed commit: - - ```text - fix(release-tracks): enforce virtual materialization lifecycle - - Invalidate materialized contents when composition changes and reject release - planning until the virtual draft is rematerialized. Restrict direct contents - replacement to standard tracks and document the remaining virtual-track work. - ``` - -## Consolidate virtual draft creation and shared release previews - -- [x] Move virtual-only composition and draft-creation operations under an - explicit `/virtual` capability namespace. -- [x] Remove the standalone virtual snapshot-preview endpoint without an - alias. -- [x] Enhance shared virtual release summaries to compare the persisted draft - with its preceding tagged release without recomputing composition. -- [x] Add regression coverage for route removal, type enforcement, latest and - historical virtual previews, and release-preview non-persistence. -- [x] Update OpenAPI, user/developer documentation, Bruno, and the - `internalattack` Python client. -- [x] Run focused regression specs, then the complete `npm test` suite. -- [x] Review the final diff and propose a conventional commit message. - -Verification result (2026-07-29): - -- Focused release, back-reference, release-by-object, and virtual-domain specs - pass; backend lint passes. -- The first complete run encountered two unrelated full-suite flakes in Assets - and Campaigns; both passed in isolation. The required second complete - `npm test` run passed. -- The `internalattack` focused suite passes (29), its complete suite passes - (246), and changed-file Ruff and pre-commit checks pass. -- Proposed commit: - `feat(release-tracks): clarify virtual draft and release lifecycle` - -## Bootstrap faster-release core, defense, and virtual tracks - -- [x] Reconcile the clarified ownership partition with the current release-track - and virtual-composition API. -- [x] Add regression coverage for functional virtual domain filters and - relationship-complete snapshot bundle exports. -- [x] Implement virtual `filters.domains` using the established ATT&CK domain - inference rules. -- [x] Reuse/extract existing bundle relationship logic so snapshot - `format=bundle` exports dynamically include valid secondary relationships. -- [x] Inventory and report any additional release-track no-op placeholders. -- [x] Update user/developer docs and OpenAPI for the effective contract change; - Bruno has no new or changed request parameter to mirror. -- [x] Run focused release-track regression specs, then the complete `npm test` - suite. -- [x] Scan all three ATT&CK v19.1 bundles and construct a disjoint exact-revision - partition for Enterprise Core, ICS Core, Mobile Core, and Defense. -- [x] Assign the shared identity and marking definitions to Enterprise Core - using the representations supported by release-track snapshots. -- [x] Preflight exact track names and refuse conflicting duplicate tracks. -- [x] Create and verify the four v19.1-pinned standard tracks. -- [x] Create and verify the three domain-filtered virtual track definitions. -- [x] Verify that every in-scope v19.1 object is owned by exactly one standard - track and record intentional relationship/collection exclusions. CTI owns - `course-of-action`; ICS Core owns `x-mitre-asset`. -- [x] Defer materializing virtual snapshots until the component standard tracks - have tagged releases; no release/tag action was authorized in this bootstrap. -- [x] Review the final repository diff and propose a conventional commit - message. - -Operational result (2026-07-28): - -- Standard tracks: Enterprise Core - (`release-track--48be5319-2f98-435a-ba36-5533236a991a`, 875 members), - ICS Core (`release-track--73147f31-2598-42a3-9cb4-125d458c4490`, 149), - Mobile Core (`release-track--ae6df6f6-3856-4d54-af40-22db856baa2d`, 206), - Defense (`release-track--84cb1147-9dba-445f-948e-6eecc51fa7e8`, 3,151), - and CTI (`release-track--469b126a-6081-462e-8b4c-709cdbb4eac4`, 1,575). -- CTI now includes 60 campaigns, 358 courses of action, 194 intrusion sets, - 866 malware objects, and 97 tools, pinned to the latest database revisions. -- Virtual definitions: Enterprise - (`release-track--a42a6f32-80c6-43a7-b1e7-26ef0814d0cb`), ICS - (`release-track--83ede842-58c8-42ce-a3fb-c38c5dd0e74c`), and Mobile - (`release-track--05615c60-bca8-4074-b8d3-b537eed52d30`). Each composes all - five standard tracks with `latest_tagged`, `prioritize_latest_object`, and - its domain filter. -- Verified 5,928 unique v19.1 owned object IDs form a disjoint partition; - relationships remain indirect, collections are generated at export, and - marking definitions are supporting metadata. -- Focused domain-filter and bundle-export specs pass (1 and 15 tests); - lint passes; the complete suite passes (OpenAPI 2, config 21, API 909, - middleware 24). -- Proposed commit: - `feat(release-tracks): filter virtual tracks and export relationships` - -## Bootstrap CTI faster-release tracks - -- [x] Read the local environment mapping and release-track documentation. -- [x] Inspect the internalattack release-track client and reference script. -- [x] Scan the ATT&CK v19.1 ICS and Mobile bundles and report every object type. -- [x] Preflight the production-mirroring Workbench API and existing tracks. -- [x] Create the CTI standard track with the latest intrusion-set, malware, - tool, and campaign revisions as members. -- [x] Verify the persisted CTI snapshot, object-type coverage, exact latest - revision pins, and counts. -- [x] Record operational results and propose a conventional commit message for - the committable scratchpad update. - -Operational result (2026-07-28): - -- Created standard track `CTI` - (`release-track--469b126a-6081-462e-8b4c-709cdbb4eac4`). -- Initially pinned 1,217 exact latest revisions as members: 60 campaigns, 194 - intrusion sets, 866 malware objects, and 97 tools. The clarified ownership - bootstrap subsequently added 358 courses of action for 1,575 total members. -- Verified the persisted snapshot, registry count, and all 1,575 member - backrefs; candidates and staged are empty. -- Proposed commit: `docs(release-tracks): record CTI bootstrap run` - -## Harden release version selection - -- [x] Reject simultaneous `increment` and `version` selectors inside the - release planner, even when controller validation is bypassed. -- [x] Add regression coverage for planner-level mutual exclusivity. -- [x] Make exact, incremental, default, and ambiguous selection behavior - explicit in OpenAPI, user/developer docs, and Bruno. -- [x] Run the focused release-track spec, lint, and complete `npm test` suite. - The focused release spec passes (10 tests), lint passes, and the complete - backend suite passes (OpenAPI: 2, config: 21, API: 907, middleware: 24). - Targeted frontend Prettier and ESLint pass; TypeScript remains blocked by - the checkout's existing Angular dependency-resolution and unrelated type - errors. -- [x] Review the final diff and propose a conventional commit message. - -## Release command and unified previews - -- [x] Replace bump routes and symbols with explicit release operations for - latest and historical snapshots. -- [x] Implement one pure release planner shared by summary, workbench, bundle, - and commit paths. -- [x] Remove `dry_run`, rename version `type` to `increment`, and reject - conflicting version-selection inputs. -- [x] Keep release targeting semantics explicit: `latest` resolves at request - time, while `:modified` pins a specific snapshot; no client precondition is - required. -- [x] Add regression coverage for preview parity, non-persistence, conflicts, - formats, validation, historical releases, and removed bump routes. -- [x] Update OpenAPI, user/developer documentation, Bruno, and frontend - consumers. -- [x] Run focused tests and frontend checks, then the complete `npm test` - backend suite. - Focused release-track suites pass (49 tests), and the affected backref suite - passes again in isolation (23 tests). The complete backend suite passes on - retry. Targeted frontend formatting and ESLint pass; frontend Vitest and - TypeScript startup remain blocked by the checkout's existing - ESM/dependency-resolution errors. -- [x] Review the final diff and propose a conventional commit message. - -## Remove implicit latest-snapshot route - -- [x] Remove `GET /api/release-tracks/:id` while preserving track deletion. -- [x] Make `/snapshots/latest` canonical across OpenAPI, tests, docs, Bruno, - and the frontend consumer. -- [x] Add regression coverage proving the removed method returns 405. -- [x] Run focused regression specs followed by the complete `npm test` suite. - The focused suites pass. The aggregate run reached 894 passing with three - unrelated documented roaming failures; all three affected specs pass - together in isolation (51 passing). -- [x] Review the final diff and propose a conventional commit message. - -## Snapshot history collection endpoint - -- [x] Add `GET /api/release-tracks/:id/snapshots` with strict tagged filtering - and pagination, plus an explicit `/snapshots/latest` alias. -- [x] Return lightweight, type-oriented summaries: standard snapshots include - member/staged/candidate counts; virtual snapshots include member/quarantine - counts. -- [x] Add regression coverage for defaults, filters, pagination, validation, - track types, and not-found behavior. -- [x] Update OpenAPI, user/developer documentation, and Bruno requests. -- [x] Run the focused regression spec followed by the complete `npm test` - suite. -- [x] Review the final diff and propose a conventional commit message. - -## Regression Tests - -- [ ] Implement regression tests - -- [x] **Investigate the recurring full-suite flake.** Two root causes found and fixed (2026-07-10) in `app/lib/database-in-memory.js`: - 1. _Port collision_: every spec file stopped and restarted the `mongodb-memory-server` instance, and a fresh mongod would intermittently fail with `Port already in use` — breaking that file's `before` hook (surfacing as `loginAnonymous` 404s) and cascading failures through the file. Fixed by reusing one mongod for all spec files in the process (`closeConnection` drops the database and disconnects but keeps the server running) plus `--exit` on the mocha scripts. - 2. _Vanishing unique indexes_: dropping the database between spec files also drops its indexes, and mongoose's per-model `init()` is memoized per process — so the `stix.id + stix.modified` unique index was intermittently missing for later files, letting duplicate-POST tests (and dependent count tests) fail in roaming pairs. Fixed by explicitly awaiting `createIndexes()` for all registered models after each reconnect. - - Residual: rare (≈1 per run under heavy machine load) single-test failures of a different character (a count assertion, a 20s timeout in a pagination GET) still appear occasionally and pass in isolation — likely load-related; keep observing before chasing further. - -## Release-track cross-tier revision uniqueness - -- [x] Read the release-track user and developer documentation and identify the - intended exact-revision invariant. -- [x] Trace every standard/virtual tier ingress and transition path. -- [x] Add regression coverage proving one `(stix.id, stix.modified)` revision - cannot occupy multiple tiers while different revisions of one ID can. -- [x] Enforce the invariant for candidate adds, promotions, demotions, bulk - status transitions, release bumps, member sync, and quarantine workflows. -- [x] Update user/developer documentation (and OpenAPI/Bruno only if the API - contract changes). -- [x] Run focused specs and the complete `npm test` suite. The task-specific - and constituent suites pass; repeated aggregate runs each encountered one - unrelated roaming API failure that passed immediately in isolation. -- [x] Review the final diff and propose a conventional commit message. - -## Snapshot Output Format - -**TASK Summary**: Implement support for the `bundle` output format for snapshots - -`bundle` refers to a STIX 2.1 bundle that contains all of the objects in the snapshot. The bundle should be emitted as a JSON object with the following structure: - -```json -{ - "type": "bundle", - "id": "bundle--", - "spec_version": "2.0", // omit if STIX 2.1, include for STIX 2.0 - "objects": [ - // All objects in the snapshot - ] -} -``` - -The following release-track snapshot retrieval endpoints support `include` and -`format` query parameters: - -- `GET /api/release-tracks/:id/snapshots/latest` (get latest snapshot) -- `GET /api/release-tracks/:id/snapshots/:modified` (get specific snapshot) - -> [!Note] -> The ephemeral bundle endpoint (`GET /api/release-tracks/ephemeral/{domain}`) supports `format`, but not tier `include`, because it does not read from a persisted release-track snapshot. Rather, it "blindly" includes all objects in the domain. - -**Include Parameter** (controls which tiers are returned): - -``` -GET /api/release-tracks/:id/snapshots/latest # Default: all tiers -GET /api/release-tracks/:id/snapshots/latest?include=members # Members tier only -GET /api/release-tracks/:id/snapshots/latest?include=staged # Members and staged tiers -GET /api/release-tracks/:id/snapshots/latest?include=candidates # Members and candidates tiers -GET /api/release-tracks/:id/snapshots/latest?include=quarantine # Members and quarantine tiers -GET /api/release-tracks/:id/snapshots/latest?include=all # All tiers -``` - -**Format Parameter** (controls output format): - -``` -GET /api/release-tracks/:id/snapshots/latest?format=workbench # Workbench snapshot with metadata (default) -GET /api/release-tracks/:id/snapshots/latest?format=bundle # Standard STIX 2.1 bundle -GET /api/release-tracks/:id/snapshots/latest?format=filesystemstore # Not implemented; returns 501 -``` - -**Combined Example:** - -``` -GET /api/release-tracks/:id/snapshots/latest?include=all&format=workbench -``` - -> [!Note] -> The `workbench` format is the default output format and is already implemented. The `bundle` format is a new output format that needs to be implemented. The `filesystemstore` format is not implemented and will return a 501 error if requested. - -### Replacing the legacy `GET /api/stix-bundles` endpoint - -Importantly, the release track retrieval method with `format=bundle` as well as the ephemeral bundle endpoint will supplant the `GET /api/stix-bundles/` endpoint defined in `stix-bundles-routes.js`. The `stix-bundles` endpoint will be deprecated and removed in a future release. We thus need to inspect the `stix-bundles-controller.js` module and identify any logic that needs to be preserved with respect to preserving existing functionality in the new endpoints. - -The `stix-bundles` endpoint currently supports generating a `x-mitre-collection` object that is emitted in the bundle. We need to ensure that this functionality is preserved in the new endpoints. Users specify how the `x-mitre-collection` object is generated via the `includeCollectionObject`, `collectionObjectVersion`, `collectionObjectModified`, and `collectionAttackSpecVersion` query parameters. We can simplify this functionality in the new endpoints: - -- `collectionObjectVersion` can just default to `v0.1` to signify that the collection was generated ephemerally and is not connected to a particular release track. -- `collectionObjectModified` can default to the current timestamp. -- `collectionAttackSpecVersion` can default to the global default attack spec version (tracked in `config.js` and exposed via `app.attackSpecVersion`). -- The `includeCollectionObject` parameter can be renamed to `includeToc` to signify that the user wants to include a table of contents object in the bundle (which is what the `x-mitre-collection` object effectively is; moreover, the term, "collection", is oversaturated in the context of STIX and Workbench, so this renaming will help reduce confusion). The `includeToc` parameter can default to `true`. - -Here is how each of the other query parameters should be handled/mapped to the newer ephemeral bundle retrieval endpoint (`/api/release-tracks/ephemeral/{domain}`): - -- `includeNotes` can be **removed**. We originally implemented notes in Workbench such that they could be included in emitted STIX bundles because we treat notes as STIX objects. However, this concept never really took off, and we have decided to treat notes as second-class Workbench-native objects that are not STIX objects, and thus cannot be included in emitted STIX bundles. -- `includeMissingAttackId` should be **preserved** as `includeObjectsWithMissingAttackId`. This parameter allows users to control whether or not objects without ATT&CK IDs are included in the emitted bundle. It defaults to `false`. -- `stixVersion` should be **preserved**. This parameter allows users to control which STIX version is used in the emitted bundle (`2.0` or `2.1`). It defaults to `2.1`. -- `useLegacyMethod` should be **removed**. The `stix-bundles-service.js` module has a legacy method for generating STIX bundles that we no longer use. The new endpoints should not support this legacy method, and thus this parameter can be removed. -- `includeDataSources` should be **removed**. For context, Data Sources are officially considered a deprecated concept in ATT&CK as of ATT&CK Spec v3.3.0. They were marked as either deprecated or revoked in the corresponding ATT&CK content release (v18.0). Because we already have `includeDeprecated` and `includeRevoked` query parameters, we can remove `includeDataSources` and instead rely on the `includeDeprecated` and `includeRevoked` query parameters to control whether or not deprecated/revoked Data Sources are included in the emitted bundle. This will simplify the API and reduce confusion. -- `state` can be **removed**. The `state` parameter was originally implemented to allow users to control which objects are included based on their workflow status (`work-in-progress`, `awaiting-review`, `reviewed`). Before the introduction of release tracks, workflow status was globally scoped. Now, with release tracks, workflow status is scoped to a release track. The ephemeral bundle endpoint is domain scoped, not release-track scoped, and thus it does not have a concept of workflow status. The `state` parameter can be removed from the new endpoints. - -### Updates to the release-track retrieval endpoints - -For release track retrieval requests that include the `format=bundle` query parameter, the following query parameters must be supported: - -- `include: ['candidate', 'staged']`: If specified, the value must be equal to an array of at least one value. The parameter acts as a filter, allowing users to specify whether release-track candidates and/or staged objects should be included in the bundle. If the `include` parameter is omitted, only members should be included. -- `state: ['work-in-progress', 'awaiting-review']`: If specified, the value must be equal to an array of at least one value. Notably, objects marked as `"reviewed"` are always included (by nature of all members being included —— all members are inherently "reviewed"), irrespective of this query parameter. The parameter acts as a union filter that logically combines with `include`. In other words, when `include` and `state` are both specified, `include` is applied first, then `state` is applied to the remaining `include`-filtered subset. (i.e., Of the candidates and/or staged objects that are ready to be included in the emitted bundle, only include the ones that are marked as "work-in-progress", "awaiting-review", or either). -- `stixVersion` should be **preserved**. This parameter allows users to control which STIX version is used in the emitted bundle (`2.0` or `2.1`). It defaults to `2.1`. - -### Fixing the /bump/preview endpoint - -Currently there exists support for the `format` query parameter on the `GET /api/release-tracks/:id/bump/preview` endpoint. It's not actually functional (has no impact on the response body) and should be removed. - -### In Summary: - -- [x] Read the existing release track user + developer documentation in `docs/user/release-tracks/` and `docs/developer/release-tracks/`, respectively. -- [x] Review the new `GET /api/release-tracks/ephemeral/:domain` endpoint implementation as well as the legacy `GET /api/stix-bundles` endpoint. -- [x] Implement support for the `format=bundle` query parameter in the following two endpoints: - - `GET /api/release-tracks/:id/snapshots/latest` (get latest snapshot) - - `GET /api/release-tracks/:id/snapshots/:modified` (get specific snapshot) -- [x] Ensure that all required logic (query parameters) is/are implemented in the new endpoints as outlined above. -- [x] Implement regression tests for the new functionality (`release-tracks-bundle.spec.js`, `ephemeral-bundle.spec.js`) -- [x] Update the aforementioned user + developer documentation. The user documentation should simply describe how the behavior _is_ while the developer documentation should described _why_ and _how_, and additionally cover what has been described here: explaining what _was_ and how the functionality has evolved from before the introduction of release tracks to after. (See `docs/developer/release-tracks/bundle-export.md`.) -- [] Remove support for the `query` parameter on the `GET /api/release-tracks/:id/bump/preview` endpoint - -## Bidirectional References - -- [x] Implement bidirectional refs between objects and snapshots. Users should be able to get individual objects via standard getters (e.g., `GET /api/techniques/:id`) and see which snapshots they are part of in the object's metadata. - -> **Implemented** as `workspace.release_tracks` (`[{ id, tier, status }]`, tiers `members`/`staged`/`candidates`/`quarantine` — matching the snapshot tier array names; the sketch below predates the rename of `phase` → `tier`), maintained via snapshot-driven reconciliation over the `release-track::contents-changed` EventBus event. See `docs/developer/release-tracks/backref-reconciliation.md` (why/how) and `docs/user/release-tracks/object-backrefs.md` (behavior). Regression tests: `app/tests/api/release-tracks/release-tracks-backrefs.spec.js`. - -Currently, it is impossible to delineate which release tracks (if any) an object belongs to _from the object's perspective_. By "the object's perspective", I mean from a given STIX object document in the `attackObjects` Mongo collection —— you cannot look at a document in the `attackObjects` collection and see which release track(s) the object is a part of. Instead, you must scan all existing release tracks for the object's `stix.id` value in either the `candidates`, `staged`, `members`, or `quarantine` list. - -This is easily correctable. When an object is either added or removed from a release track, the object document should be updated. We just need to include a small piece of metadata in the STIX object's document. Fortunately, we already have a pattern in place for tracking metadata: `workspace`. Moreover, we actually have an equivalent bidirectional ref tracker in place for the release tracks' predecessor: Workbench collections. They are/were tracked in each object's `workspace.collection` field. So, we may be able to copy/mimic this existing workflow. - -I am imagining STIX object documents containing backwards pointers to their containing release track(s) looking something like this: - -```yaml -# A Technique document -workspace: - release_tracks: - - id: String - phase: String; Options: ['candidate', 'staged', 'member'] - status: String; Options: ['work-in-progress', 'awaiting-review', or 'reviewed'] -stix: # ... -``` - -For example: - -```yaml -workspace: - release_tracks: - - id: 'release-track--3a0e2537-1153-4b16-8ff5-1993f2d9cd7d' - phase: 'candidate' - status: 'work-in-progress' -stix: # ... -``` - -The `phase` and `status` fields will need to change for the appropriate `release_tracks` list element when user moves the object between the candidate, staged, and member phases; and when the object's status changes. We can make use of the event bus architecture here, following the same pattern that some services (like `detection-strategies-service.js` and `analytics-service.js`) use to track embedded relationships between two objects. Similarly, the release tracks service would just need to fire off an event that each of the STIX services listen; and when heard, they set the `workspace.release_tracks` field for the relevant STIX object document(s) accordingly. - -## Release-Track Change Capture (in-place mutation hardening) - -Object CRUD paths can mutate or destroy revisions that release tracks pin, without the track ever hearing about it. Design decisions locked in 2026-07-10. The `workspace.release_tracks` backrefs make every guard below a cheap document-local check (no track scanning). - -- [x] **Reject revision re-keying on PUT.** `updateFull` merged body `stix.id`/`stix.modified` over the stored document, so a PUT could silently re-key a revision and strand any track pins. Now returns 400 when the body identity fields differ from the path parameters. Re-keying must go through POST (a new revision), which member sync captures. Tests: `app/tests/api/base-services/update-identity-guard.spec.js`. - -- [x] **Capture in-place PUTs of pinned revisions.** Implemented 2026-07-13: `BaseService.updateFull` rejects (409, `MemberPinnedRevisionError`) when the revision is pinned in any track's `members` tier — released content is immutable in place; POST a new revision instead. `staged`/`candidates`-pinned revisions ride the `::updated` → revision-sync path and are marked with the server-assigned **`modified-in-place`** status (content changed with no revision history to diff — reviewers are told _that_ something changed, not _what_; the marker is cleared via the review endpoint). Placement is centralized in the **workflow gate** (`app/lib/release-tracks/workflow-gate.js`): tier is decided against `candidacy_threshold`/`auto_promote` (`modified-in-place` ranks with `work-in-progress`), so permissive tracks keep in-place-edited staged entries staged while strict tracks demote them for re-review — and threshold-qualifying placements land directly in `staged` in a single snapshot (no more candidates bounce). Covers in-place deprecation (`x_mitre_deprecated` via PUT). The member-sync misfire (same-key duplicate cross-tier enrollment) is fixed by skipping enrollment of already-pinned revisions and skipping no-op snapshot clones. Future: an in-document changelog of in-place modifications would let the marker say _what_ changed. Tests: `app/tests/api/release-tracks/release-tracks-change-capture.spec.js`. - -- [x] **DELETE of tracked objects.** Implemented 2026-07-13 with a simplified decision: DELETE (single version or all versions) is _rejected_ (409) when a revision is `members`-pinned, with guidance to retire the object via a new `x_mitre_deprecated` revision instead — members-pinned revisions are immutable and must never be deleted. (The earlier auto-convert-to-deprecation idea was dropped in favor of explicit rejection.) `candidates`/`staged`-pinned deletes remain allowed unless the same revision is also a frozen graph dependency. The deterministic-graph slice extended the guard to secondary/supporting revisions and to `CollectionsService`'s custom exact, lineage, and `deleteAllContents` paths. Legacy delete controllers use the service-exception middleware (`next(err)`) so the 409 maps correctly. - -- [x] **Revoke must reach member sync.** Implemented 2026-07-13: member sync subscribes to the 11 per-type `::revoked` events via a payload adapter (`handleStixObjectRevokedEvent`), so the revoked revision (`revoked: true`) is enrolled as a candidate in member tracks and candidate/staged pins move to it — treated exactly like any new revision. The revoke response's primary document carries the resulting backrefs. Member sync is not extended to relationships; the bounded `revoked-by` edge is captured in each new snapshot's graph manifest and replayed from there. - -- [x] **Technique conversion should reach revision sync.** Implemented 2026-07-13 with the adapter approach (same pattern as `handleStixObjectRevokedEvent`): the `TECHNIQUE_CONVERTED_TO_SUBTECHNIQUE` / `SUBTECHNIQUE_CONVERTED_TO_TECHNIQUE` event payloads now carry the converted revision (`document`) and acting user, and member sync subscribes via `handleStixObjectConvertedEvent`, treating the conversion as a `new-revision` trigger through the workflow gate — candidate/staged pins move to the converted revision, member tracks enroll it as a candidate. The conversion responses refresh `workspace.release_tracks` after event processing (read-your-own-writes). Tests: conversion cases in `release-tracks-change-capture.spec.js` and the updated clone-strip test in `release-tracks-backrefs.spec.js`. - -## Get Releases By Object - -- [x] Implement `GET /api/release-tracks/objects/:objectRef/releases` so a - caller can retrieve every tagged snapshot whose `members` tier directly - contains the supplied STIX ID, across all object revisions and release - tracks. - -### Design - -The existing `workspace.release_tracks` backrefs cannot answer this query: -they intentionally describe only each track's latest snapshot. A release that -historically contained an object must still be returned after a later snapshot -removes it. Conversely, copying all tagged snapshots into a new global MongoDB -collection would duplicate the existing per-track source data and undermine -the collection-per-track storage boundary. - -Use `releaseTrackRegistry` as a compact global forward catalogue instead. Its -single document per track gains a server-maintained `tagged_releases` array: - -```javascript -tagged_releases: [ - { - snapshot_modified: Date, // (track_id, snapshot_modified) identifies the snapshot - version: String, - tagged_at: Date, - tagged_by: String, - }, -]; -``` - -`tagged_release_count` is derived from `tagged_releases.length`. The actual -snapshot — including the authoritative `members` pins — remains in the track's -dynamic collection. Tagging reconciles this registry projection from the -source snapshots rather than incrementally appending, so retries and -retroactive tagging are idempotent and self-healing. A migration backfills -existing tracks. - -The endpoint is stateless but necessarily fan-outs: read registry documents -with tagged releases, then issue one bounded-concurrency query per eligible -track using all of that track's tagged `snapshot_modified` values. Flatten, -sort deterministically, and paginate the matches. Registry references reduce -the search to tagged snapshots, but they are a forward index (track → release), -not an inverted object → release index; eliminating the per-track fan-out would -require a separate denormalized membership index and is deliberately out of -scope. - -Add a partial multikey index to every dynamic track collection for -`members.object_ref`, limited to snapshots whose `version` is a string. Drafts -therefore incur no index cost, and draft squashing does not affect the lookup. - -### Semantics - -- Match the STIX ID across all revisions; return the pinned `object_modified` - for each release. -- Include standard and virtual tracks by default; optional `type` filtering. -- Include only direct `members` entries from tagged snapshots. Do not include - candidates, staged/quarantined entries, or secondary objects added during - bundle export. -- Support `order=asc|desc` by `snapshot_modified`, plus `limit` and `offset`. -- Return 200 with an empty result for a valid STIX ID with no tagged releases; - malformed IDs return 400. -- Ascending order describes first _published/tagged_ appearance, not the time - the object first entered an untagged draft. - -### Checklist - -- [x] Registry schema/repository: add `tagged_releases`, reconciliation, and - derived count/latest-version maintenance. -- [x] Dynamic snapshot schema/repository: add the tagged-member partial index - and a projected `findTaggedSnapshotsContainingObject` query. -- [x] Versioning: reconcile registry metadata after tagging and validate - version progression against track-wide tagged releases rather than a - potentially stale historical snapshot's embedded `version_history`. -- [x] API: route, controller Zod validation, facade/service orchestration, - deterministic pagination, and OpenAPI contract. -- [x] Migration: backfill registry tagged-release refs and ensure the new index - on all existing dynamic track collections. -- [x] Regression tests: multiple tracks/releases/revisions, removal after an - earlier release, retroactive tag, virtual track, draft/non-member exclusion, - filtering/order/pagination, empty/malformed input, and backfill behavior. -- [x] User/developer docs and Bruno request. -- [ ] Verification: targeted spec first, then the complete `npm test` suite. - - Targeted endpoint spec: 8 passing; release-track directory: 69 passing; - lint, OpenAPI validation, and middleware suite pass. - - `npm test` was attempted three times on 2026-07-16. Each API run reached - 861-880 passing but hit different roaming failures in unrelated legacy - specs (collection-bundle timeout, missing anonymous-session cookie, and - transient version lookups). Every failed file passed when rerun in - isolation. A clean full-suite run is still required before this task meets - the repository definition of done. - -## Snapshot Retention (Squash on Tag) - -- [ ] Implement draft-snapshot squashing so release cycles don't accumulate - unbounded snapshot storage. Design captured 2026-07-15; assessed as sound — - see analysis below. - -### Why - -Every mutation clones the full snapshot document (`cloneSnapshot` in -`snapshot-service.js`): metadata edits, config edits, tier operations, and — -critically — every member-sync enrollment. Each snapshot embeds the complete -`members`/`staged`/`candidates` arrays (~100–150 bytes BSON per pin entry). - -At ATT&CK scale (~10k–20k tracked objects), each snapshot document is -~1–3 MB. A release cycle where 10% of a 10k-object track is edited produces -~1,000 member-sync snapshots ≈ 1–3 GB of drafts per track per cycle — nearly -all of it intermediate states nobody will ever read again. Storage per cycle -is O(edits × track_size); the per-write clone is the root cause, but squashing -at the tag checkpoint caps the steady state without touching the write path. - -Mitigating facts (verified in code): - -- Bulk endpoints already exist: `addCandidates`, `promoteCandidates`, - `reviewCandidates`, `demoteStaged` all take arrays and produce **one** - snapshot per call. Initial population of a track is 3 snapshots (create → - bulk-add → bulk-promote), plus an in-place tag (tagging via - `tagSnapshotInPlace` creates **zero** snapshots). The N-snapshot trap is - calling the bulk endpoints once per object — document this loudly in user - docs, but no code change needed there. -- `::created` events for brand-new objects are no-ops for member sync - (`findTracksReferencingObject` only matches already-tracked `stix.id`s). - The O(N²) trap is bulk _re-imports/updates_ of already-tracked objects - (e.g. re-importing a modified 20k-object bundle → 20k snapshots × MBs each). -- `version_history` is embedded in and carried forward by every clone, so the - release ledger survives squashing — tagged snapshots and the latest draft - always hold the full history. -- Backref reconciliation (`emitContentsChanged`) only ever reads the **latest** - snapshot; deleting non-latest drafts requires no backref work. - -### Semantics - -"Squash" = bulk-delete draft snapshots (`version == null`) older than a -boundary, preserving: all tagged snapshots, the boundary snapshot, and always -the latest snapshot. Like `git rebase --squash`ing the commits behind a tag. - -1. **Squash-on-tag (opt-in):** `POST /api/release-tracks/:id/bump` (and - `.../snapshots/:modified/bump`) accept `squash: boolean` (default `false`). - After a successful tag of snapshot S, delete all snapshots matching - `{ id, version: null, modified: { $lt: S.modified } }`. Drafts newer than S - (work already underway toward the next release) survive. Response gains - `squashed_count`. -2. **Standalone maintenance endpoint** (recovery from bulk-operation - accidents, no tag required): `POST /api/release-tracks/:id/snapshots/squash` - with optional `before` (ISO timestamp; defaults to the latest tagged - snapshot's `modified`; if no tagged release exists and `before` is omitted, - 400). Same delete filter; never deletes the latest snapshot even if it is - an untagged draft and `before` post-dates it. -3. **Concurrency safety:** the filter can't race member sync — concurrent - clones get `modified = now`, which is always ≥ the boundary, so they are - never matched. Tag-then-squash need not be atomic: a crash between the two - just leaves drafts behind (retryable via the maintenance endpoint). -4. After deletion: one `syncRegistryCounters(trackId)` call; **no** - `emitContentsChanged` (latest snapshot unchanged by construction). Add a - repo-level `deleteDraftSnapshotsBefore(trackId, boundary)` (`deleteMany`) - rather than looping `deleteSnapshot` (which emits per-delete events). - -### Drawbacks accepted (documented trade-offs, not blockers) - -- **Provenance loss.** Intermediate drafts are the only record of the journey: - who added/staged what when (`object_added_by`, `object_staged_at`), status - transitions, `modified-in-place` markers that were later cleared. Promotion - strips staged metadata from member entries, so after squash only the final - state remains. This is exactly git-squash semantics and is why the flag is - opt-in, but teams that need review audit trails must not squash (or we later - add a roll-up audit record — see Future). -- **Retro-tagging is foreclosed.** `bumpByModified` can no longer tag a - squashed draft. Consistent by construction: squashing is the declaration - that intermediates don't matter. Note the "undo/move the tag" worry is - already moot — versions are immutable once set, re-tagging throws - `AlreadyReleasedError`, and no untag endpoint exists. The genuine loss is - forensic/DR, mitigated only by Mongo backups. -- **Virtual tracks: excluded from v1.** Their scheduled snapshots - (`snapshot_schedule`) exist precisely to build a periodic history; - squash-on-tag would destroy the thing the schedule creates. Reject - (or no-op with a warning) squash on virtual tracks until there's a - considered retention policy for them. - -### Alternatives considered - -- _Amend-in-place_ (member sync mutates the latest draft instead of cloning): - attacks the root cause but breaks the "every modification is a new - snapshot" invariant, complicates concurrent reads, and silently degrades - the audit trail for everyone. Rejected for now. -- _Delta/structural-sharing storage_: large refactor of the snapshot store; - revisit only if squash proves insufficient. -- _TTL/retention config_ (e.g. `config.retention.auto_squash_on_tag`, - max-draft-age): natural follow-on once manual squash exists. - -### Checklist - -- [ ] Repo: `deleteDraftSnapshotsBefore(trackId, boundary)` in - `release-track-dynamic.repository.js` (deleteMany on - `{ id, version: null, modified: { $lt: boundary } }`, excluding the latest - snapshot's `modified`). -- [ ] Service: squash logic in `versioning-service.js` (`squash` option on - `_doBump`) + standalone squash operation (probably `snapshot-service.js`); - reject for virtual tracks; return `squashed_count`. -- [ ] Controller/routes: `squash` in the Zod bump body schema; new - `POST /api/release-tracks/:id/snapshots/squash` route with Zod-validated - optional `before`. -- [ ] OpenAPI: bump request body + new squash path. -- [ ] Regression tests (`release-tracks-squash.spec.js`): squash-on-tag - deletes only pre-tag drafts; tagged snapshots survive; drafts newer than - the tagged snapshot survive; latest-draft never deleted by maintenance - squash; registry counters resync; backrefs untouched; virtual track - rejected; no-tagged-release + no `before` → 400; idempotent re-squash. -- [ ] Docs: `docs/user/release-tracks/versioning.md` (squash behavior + - the bulk-endpoints-vs-per-object-loop warning for initial population), - `docs/developer/release-tracks/` (why, trade-offs, provenance loss). -- [ ] Bruno: bump `.bru` gains `~squash` toggle; new squash request file. - -### Future (not in scope) - -- Roll-up audit record written at squash time (compact per-object journey - summary appended to the version_history entry or a side collection) to - soften the provenance loss. -- Retention config for auto-squash and for virtual-track snapshot history. -- Coalescing/debouncing member-sync snapshots during bulk update storms - (the re-import O(N²) trap) — e.g. a bulk-import context that suspends - per-object snapshotting and emits one consolidated snapshot at the end. - -## Small Fixes - -- [x] **Composition schema mismatch: `priority`.** Resolved 2026-07-29 by - requiring a unique, non-negative integer priority in request validation, - persistence, OpenAPI, documentation, and Bruno examples. - -- [x] **Restrict `deleteSnapshot` to the latest untagged draft.** - `DELETE /api/release-tracks/:id/snapshots/:modified` now returns `409` - for tagged releases and historical drafts, preserving immutable history. - Completed by the immutable-history slice. - -- [ ] **`syncRegistryCounters` scales with snapshot count.** It fetches _all_ snapshots (`getAllSnapshots` with projection) on every clone to recount — O(snapshot_count) reads per write, on the hottest path (member sync). Fine post-squash; consider a count query or incremental counters if draft accumulation between tags is large. - -## Diffing Endpoint - -- [ ] Implement object diffing endpoints for snapshots. Users should be able to effectively preview changes to objects before tier transitions (candidates, staged, members). - -### Idea 1 - Diffing endpoint specifically for release tracks - -In this approach, we would implement a workflow-driven diffing endpoint that is specific to release tracks. The endpoint would allow users to diff objects in the candidate snapshot against their previous revisions in the staged or member snapshots. - -``` -GET /api/release-tracks/:id/candidates/:objectRef/diff -GET /api/release-tracks/:id/staged/:objectRef/diff -``` - -If `:objectRef` is a reference to an object that is not part of the candidate snapshot, the endpoint should return a 404 error. If it is part of the candidate snapshot, the endpoint should return a diff between the object in the candidate snapshot and the object in the next lifecycle stage. - -To clarify, snapshot objects transition linearly and unidirectionally through the following tier transitions: Candidate -> Staged -> Member - -An object exists as a set of one or more revisions. An object is identified by its `stix.id` field, whereas an object revision is identified by its `stix.id` and `stix.modified` fields. - -A revision can exist in exactly one tier at a time. - -- If a revision exists in the candidate snapshot, it will not exist in the staged or member snapshots. -- If it exists in the staged snapshot, it will not exist in the candidate or member snapshots. -- If it exists in the member snapshot, it will not exist in the candidate or staged snapshots. - -If a revision exists in the candidate snapshot, it will not exist in the staged or member snapshots. However, a _previous_ revision may exist in the staged or member tiers (though it is not guaranteed). Because the tier transitions are unidirectional, revisions must be temporally ordered as it relates to how they are distributed across the tiers. It should not be possible for a newer revision to exist in a previous tier. For example, if a revision exists in the candidate snapshot, it is not possible for a newer revision to exist in the staged or member snapshots. - -This rigidity allows us to implement a diffing endpoint that is specific to release tracks. The diffing endpoint should return a diff between the candidate revision and the next lifecycle stage revision (staged or member). - -So, if an object exists in the candidate snapshot, and another/previous revision of it exists in the members state, the diff endpoint should return a diff between the candidate revision and the member revision. If no previous revision exists in the members state, the diff endpoint should return a diff between the candidate revision and an empty object. - -As another example, if an object exists in the staged tier, the `GET /api/release-tracks/:id/staged/:objectRef/diff` endpoint should return a diff between it and the previous revision that exists in the member tier. If no previous revision exists in the member state, the diff endpoint should return a diff between the candidate revision and an empty object. - -Member revisions are considered immutable and thus cannot be diffed from. Hence, there is no `GET /api/release-tracks/:id/members/:objectRef/diff` endpoint. - -There is one edge case that needs special consideration. If a revision exists as a candidate, a previous revision exists as a member, but no previous revision exists in the staged tier, the diff endpoint now becomes unclear: If the candidate transitions to the next tier, one could argue that the diff should be between the candidate revision and an empty object (since no previous revision exists in the staged tier). However, one could also argue that the diff should be between the candidate revision and the previous member revision. I think the most intuitive approach is to return a diff between the candidate revision and the previous member revision. This is because the candidate revision will eventually transition to the staged tier, and it is more intuitive to compare it against the most recent revision that exists in the next lifecycle stage (member) rather than an empty object. - -To stick with the example, if a revision exists as a candidate, a previous revision exists as staged, and a previous revision exists as a member, the `GET /api/release-tracks/:id/candidates/:objectRef/diff` diff endpoint should return a diff between the candidate revision and the previous staged revision. This is because the candidate revision will eventually transition to the staged tier, and it is more intuitive to compare it against the most recent revision that exists in the next lifecycle stage (staged) rather than an empty object. Similarly, the `GET /api/release-tracks/:id/staged/:objectRef/diff` diff endpoint should return a diff between the staged revision and the previous member revision. This is because the staged revision will eventually transition to the member tier. - -### Idea 2 - Diffing endpoint for all objects (not just release tracks) - -Type-centric: - -``` -GET /api/:type/:id/diff -GET /api/:type/:id/modified/:modified/diff -``` - -Type-agnostic: - -Embed the - -``` -GET /api/attack-objects/:id/diff -GET /api/attack-objects/:id/modified/:modified/diff -{ - "compareTo": { - "type": "attack-pattern", - "id": "attack-pattern--1234", - "modified": "2024-02-01T00:00:00.000Z", - } -} -``` - -Set up a diffing endpoint that is type-agnostic and allows users to compare any two revisions of an object. The endpoint should accept a request body that specifies the `compareTo` revision, and the endpoint should return a diff between the current revision and the specified `compareTo` revision. - -``` -GET /api/compare -{ - "compareFrom": { - "type": "attack-pattern", - "id": "attack-pattern--1234", - "modified": "2024-01-01T00:00:00.000Z", - }, - "compareTo": { - "type": "attack-pattern", - "id": "attack-pattern--1234", - "modified": "2024-02-01T00:00:00.000Z", - } -} -``` - -## Repurposing the `note` object - -- [ ] Implement support for tracking notes on snapshot objects (can be candidates, staged, or members). Notes should be stored in a separate Mongo collection and linked to the snapshot object via a reference field. Users should be able to add, edit, and delete notes via the API. Notably, we already have a notes service that can be leveraged for this purpose. However, it needs some modifications. The service was originally implemented with STIX in mind. The idea was to treat/represent notes as STIX objects and enable users to include them in emitted STIX bundles. However, the concept never really took off. We should modify the service to treat notes as second-class objects that are entirely separate from STIX, but rather as Workbench-native objects. Notes should be capable of being linked/attached to snapshot objects (candidates, staged, or members) as well as to objects independent of snapshots (documents in the `attackObjects` collection). - -Make a new Mongo collection called `notes` to store notes. Each note should have the following fields: - -```json -{ - "_id": "ObjectId", - "content": "string", - "created_by": "string", - "last_modified_by": "string", - "created_at": "Date", - "modified_at": "Date", - "snapshot_object_id": "ObjectId", // Reference to the snapshot object (if applicable) - "object_id": "ObjectId" // Reference to the attack object (if applicable) -} -``` - -Notes will NOT be version controlled. If they are edited or deleted, the changes will be reflected immediately in the database, and recovery and undo functionality will not be supported. - -Links/references between notes and snapshot objects will be one-to-many. A single snapshot object can have multiple notes attached to it, but a note can only be linked to one snapshot object at a time. Similarly, links/references between notes and attack objects will also be one-to-many. These should be bidirectionally tracked, meaning that if a note is linked to an attack object, the attack object should have a reference to the note in its metadata, and vice versa. - -```json -// attackObjects collection -{ - "_id": "ObjectId", - "workspace": { - "notes": ["ObjectId"] // Array of references to notes linked to this attack object - }, - "stix": "StixObject" -} -``` - -## Deterministic v19.1 virtual-track bootstrap graph - -- [x] Preserve the materialized virtual snapshot graph when previewing and committing a release. -- [x] Enforce virtual component-domain filters throughout graph traversal, including secondary objects. -- [x] Prevent `LinkById` rendering from selecting revoked or deprecated ATT&CK-ID collisions. -- [x] Seed virtual snapshot graph manifests from the canonical v19.1 bundles in the bootstrap script. -- [x] Make bootstrap bundle comparisons detect duplicate revisions and explain unexpected drift. -- [x] Add regression coverage and document the deterministic-primary/non-deterministic-graph boundary. -- [x] Run focused tests and the complete `npm test` suite. -- [x] Delete and recreate only the Enterprise ATT&CK virtual track, then assess its emitted bundle against v19.1. - -Verification (2026-07-30): - -- The clean complete server suite passes: OpenAPI 2, config 21, API 992, - middleware 29, and scheduler 10. -- Focused graph-integrity regressions pass (3), the affected release-track - group passes (40), and the bootstrap regression suite passes (23). -- The guarded bootstrap completed without accepting unexplained bundle drift; - all three standard and virtual baselines are tagged `1.0`. -- Enterprise contains 4,815 exact members, zero quarantine entries, and a - 25,842-object publication graph excluding its generated collection. It has - no missing, additional, or duplicate STIX IDs; type counts and all 25,841 - adjusted TOC pins match canonical v19.1. -- The 312 raw payload differences consist only of the expected canonical-domain - repairs and domain-array ordering. After those agreed normalizations, zero - payloads differ. - -## Snapshot descriptions - -- [x] Persist a bounded, snapshot-local description separately from release-track metadata. -- [x] Allow editors to set the description while materializing or tagging a snapshot and edit it later without changing snapshot identity or contents. -- [x] Return descriptions in snapshot history and Workbench snapshot responses. -- [x] Document the API and update the Bruno collection. -- [x] Add frontend creation, display, edit, clear, and feedback flows. -- [x] Add backend and frontend regression coverage. -- [x] Run focused tests and the complete backend and frontend verification suites. - -## Release-track UX and API follow-ups (2026-09-03) - -Raised after testing the sealed-manifest work on a restored production -database. - -- [x] Remove bundle `include`/`state`: bundles always replay the sealed - manifest; `format=bundle` rejects `include` (400). Frontend Export - Latest no longer sends `include=all` for bundles. -- [x] Workbench tier entries carry `type` and `x_mitre_version` so the - release preview no longer downloads the whole object catalogue. -- [x] Release preview computes relationship changes without loading supporting - objects and LinkById targets. -- [x] Track URL aliases (unique slug per track resolving to the track ID on - every `:id` route; set at creation or via `/meta`). -- [x] Inter-domain relationship report: `GET /api/reports/domain-consistency` - (active SROs whose endpoints share no domain; domain-bearing SDOs - lacking domains) and a Data Quality page section. -- [x] Frontend: Delete release only on the most recent release; Preview & - Release in-progress state. -- [x] Frontend: draft-then-tag flow (header keeps only Create Draft; tagging - from draft cards), DETAILS tab renamed Board, track deletion in a CONFIG - danger zone. diff --git a/docs/developer/data-model.md b/docs/developer/data-model.md index d674a9d7..da8d20b2 100644 --- a/docs/developer/data-model.md +++ b/docs/developer/data-model.md @@ -77,6 +77,88 @@ The Mongoose discriminator capability supports storing multiple object types in The `__t` property is created and managed by Mongoose to distinguish between the different types of objects stored in the `attackObjects` collection. +### Runtime Allowed Values Configuration + +`AllowedValuesConfiguration` is a singleton MongoDB document with +`_id: "allowed-values"`. It stores property/domain `rules` keys separately from +value rows, so a configured group can remain empty. Each row has an ObjectId, +object type, property, domain, value, and enabled state. This is application +configuration, not STIX content or object history. + +#### Catalog and file dependencies + +`app/lib/allowed-values-catalog.js` builds the catalog when its module loads: + +- The bundled `app/config/allowed-values.json` registers the supported + object/property pairs and initial rule keys. Its values supply suggestions + for non-enum fields. +- `getSchema` in `app/lib/validation-schemas.js` supplies the installed ADM + partial object schemas. The catalog reads enum choices from Zod and checks + each candidate in its object-type/domain context. Software must satisfy both + tool and malware schemas; related-asset sectors use their nested context. +- Data-source/component strings use ADM's custom validator. Seed suggestions + are examples, not an exhaustive list of permissible strings. +- `admVersion` comes from the installed ADM package's `package.json`, alongside + the schemas used for validation. `/catalog` returns both the version and + definitions; the frontend displays that version and uses those definitions. + +The catalog is immutable for the process lifetime. Upgrading the installed +package requires restarting the backend to rebuild it; reload the UI to fetch +the new catalog. Runtime settings are read from MongoDB, not cached with it. + +The bundled JSON remains a dependency on an initialized database: +`retrieveAllowedValues` also uses its object/property/domain layout and ordering +to construct the nested dropdown response. `ALLOWED_VALUES_PATH` overrides only +the file read for initial values, not these bundled metadata uses. Do not remove +the bundled file on the assumption that seeding is its only role. + +#### Initialization and writes + +`checkSystemConfiguration` calls the Allowed Values service during startup. +If the singleton is absent, the service reads the configured seed file, +validates all scopes and values, and inserts the configuration with atomic +`$setOnInsert`. Existing configuration is not reseeded. For legacy documents +missing rule keys, an aggregation update combines initial keys with persisted +row scopes while retaining the rows. + +Creation requires an absent rule key and adds its key and values in one update. +Replacement requires an existing key and filters/replaces that group's rows +inside MongoDB. Concurrent saves to different groups retain both changes; +duplicate creation has one winner. No multi-document transactions are required. + +Every option, including disabled options, must pass ADM field and contextual +validation before a write. Duplicate object-type/value combinations are rejected. +General STIX validation flags and Validation Bypasses do not participate in +these checks. `/validate` offers the same candidate check without persistence; +create and update still validate their own requests. + +#### Reads and invalid legacy settings + +`projectRules` groups saved rows by property/domain, combining equal value/state +pairs across object types. On every read it separates compliant `values` from +`invalidValues`, which carry rejection reasons. The dropdown response excludes +invalid or disabled rows. + +This is the entire quarantine mechanism: no rows are moved, no quarantine flag +is stored, and no stored enabled flag is changed. An ADM upgrade can therefore +change how a retained row is classified. The admin editor omits invalid rows +from its replacement payload and warns before saving; that save removes them. +Existing STIX objects are not modified. + +#### Extending supported choices + +For an existing enum property, a new ADM enum member is discovered from the +upgraded backend package; it does not also need a JSON entry. Add it to the seed +only if new configurations should enable it by default. Existing databases still +need an administrator to enable new, unconfigured choices. Formatted values can +be added without a library change when the current ADM validator accepts them. + +A new property/object-type pair is different: register it in Workbench, confirm +the schema mapping and editor support, and retain ADM validation. The admin +workflow does not create arbitrary schema properties. See the +[operator procedures](../admin/configuration.md#allowed-values) for initialization, +adding values, ADM upgrades, and API usage. + ## Sample Objects ### Technique Object Example diff --git a/docs/developer/release-tracks/authorization.md b/docs/developer/release-tracks/authorization.md index abef107b..e360cfa9 100644 --- a/docs/developer/release-tracks/authorization.md +++ b/docs/developer/release-tracks/authorization.md @@ -17,6 +17,9 @@ history requires an administrator. | Convert the track's most recent release to draft | No | No | Yes | | Change a tagged release's semantic version | No | No | Yes | | Delete an entire track and all snapshot history | No | No | Yes | +| Inspect pending/failed virtual draft cleanup | Yes | Yes | Yes | +| Set an ad-hoc limit or change recurring draft retention | No | No | Yes | +| Opt into draft squash when tagging or retry draft cleanup | No | No | Yes | Full-track deletion also requires `confirm_track_id` to equal the `:id` path parameter. `POST /snapshots/:modified/draft` requires a JSON `confirm_version` @@ -35,10 +38,20 @@ release lock. Both conversion and retag capture audit identity under that same lock, so a competing version correction cannot invalidate confirmation or change the version between audit capture and mutation. +Supplying an ad-hoc retention policy, changing a recurring retention limit, and +explicit squash/retry use the existing global administrator role; there is no +track-specific administrator role. Editors may change cron timing while keeping +its retention policy unchanged, or switch away from recurring mode. Ordinary +tagging retains editor-or-higher access. Squash authorization and the reviewed +fingerprint are checked before tagging. Recurring retention records its system +actor; ad-hoc cleanup records the invoking administrator. + ## Audited destructive actions -The `delete_track`, `convert_release_to_draft`, and `retag_release` actions create a -`releaseTrackAuditEvents` record before the business operation begins. +The `delete_track`, `convert_release_to_draft`, `retag_release`, `draft_retention`, +and `draft_squash` actions create `releaseTrackAuditEvents` records. +Cleanup actions use bounded durable intent/progress; see +[Deletion Guardrails](deletion-guardrails.md) for the recovery rationale. The legacy `delete_release` value remains readable for historical audit events; new requests never use it. @@ -49,4 +62,9 @@ persists but final audit-state recording fails, the API returns a structured `500` containing the audit event ID instead of reporting unconditional success. +Cleanup-only failures after successful creation/release are reported in +`draft_cleanup` without undoing the committed snapshot. Interrupted publication +uses a structured `500` with the cleanup operation ID and the known release +outcome. Administrators resume via the cleanup retry endpoint, not another tag. + See the [operator audit guide](../../admin/release-track-audit.md). diff --git a/docs/developer/release-tracks/deletion-guardrails.md b/docs/developer/release-tracks/deletion-guardrails.md new file mode 100644 index 00000000..9ca682f7 --- /dev/null +++ b/docs/developer/release-tracks/deletion-guardrails.md @@ -0,0 +1,124 @@ +# Deletion Guardrails + +These safeguards prevent automatic cleanup from **recreating snapshots, deleting a newly tagged release, or leaving the database inconsistent if an operation fails halfway through**. + +They are implementation requirements for the proposed features—not additional settings you would need to manage. + +## 1. “Scheduler receipts must survive snapshot deletion” + +**The system needs to remember that a scheduled run already happened, even after its snapshot has been deleted.** + +Suppose a virtual track creates a snapshot every hour: + +1. The 10:00 scheduled run creates snapshot A. +2. The server crashes before recording that the scheduled run finished. +3. After restarting, the scheduler sees the unfinished 10:00 run and considers retrying it. + +Today, the scheduler can find snapshot A and conclude: “The snapshot was already created; I just need to finish recording success.” + +Automatic truncation introduces a problem: + +4. Newer snapshots are created, and retention deletes A. +5. The scheduler retries the unfinished 10:00 run. +6. It cannot find A, so it could create another snapshot for the same scheduled occurrence. + +That replacement might contain different content because the component tracks have changed since 10:00. + +### The safeguard + +Keep a small, separate record—a **receipt**—saying: + +> The 10:00 occurrence already created snapshot A. + +Deleting A must not delete that receipt. A retry then knows that the work already happened and must not repeat it. + +The backend already has records for scheduled occurrences, so we can build on those rather than introduce a separate scheduling system. These small records would remain even while the much larger snapshot history and associated metadata are trimmed. + +--- + +## 2. “Serialize the virtual target’s lifecycle” + +Here, **serialize** means “make conflicting operations take turns.” The **target** is the virtual track being changed. + +Two operations can happen at almost the same time: + +- The scheduler creates a draft and starts pruning old drafts. +- A user tags one of those old drafts as a release. + +Without coordination, this sequence is possible: + +1. Cleanup checks snapshot A and sees that it is an old draft. +2. The user tags A as a release. +3. Cleanup deletes A based on its earlier check. + +That would violate our most important rule: **never delete tagged releases.** + +There is another possible race: one operation reads a snapshot as the basis for a new draft while another deletes that snapshot and cleans up metadata the new draft still needs. + +### The safeguard + +Use a database-backed lock for the affected virtual track. Conceptually: + +> “I’m changing this track’s snapshots. Another conflicting operation cannot change them until I finish.” + +The backend already uses this locking mechanism for releases and some other operations. We need to extend its coverage so virtual-track creation and cleanup participate too. + +We should also make the final deletion conditional: + +> Delete A only if it is **still an eligible draft**. + +That provides an additional check rather than trusting an earlier decision. + +This does **not** mean stopping all Workbench activity. It coordinates conflicting changes to the affected track. Under the existing locking convention, a competing operation may receive a conflict response and need to retry. + +--- + +## 3. “Make cleanup recoverable” + +**Cleanup has several steps. A failure halfway through must not lose the new release or leave cleanup permanently unfinished.** + +For example, a user tags a draft and requests squashing: + +1. The release is successfully created. +2. Cleanup deletes three earlier drafts. +3. The database connection fails before the remaining seven drafts are deleted. + +At that point, two different things are true: + +- **The release succeeded.** +- **Cleanup is incomplete.** + +It would be misleading to show only “Release failed—try again.” Retrying the tagging operation is not the right way to finish deleting old drafts. + +### The safeguard + +Record the cleanup request and its progress separately. The system can then report: + +> Release created successfully. Draft cleanup is incomplete and can be retried. + +A retry should finish only the remaining cleanup. It must not create another release, change the released contents, or broaden the set of drafts originally approved for deletion. + +### Shared metadata also needs care + +Snapshots can share a **content manifest**: a stored list of the exact object versions and relationships needed to reproduce their exports. + +If drafts A and B share the same manifest: + +- Deleting A must leave the manifest intact because B still needs it. +- Deleting the last snapshot that references that manifest can make the manifest eligible for removal. + +Cleanup must also update snapshot counts, so the UI does not report snapshots that no longer exist. + +None of this should delete the underlying ATT&CK/STIX objects. We are removing obsolete snapshot history and metadata that no surviving snapshot needs. + +--- + +In short: + +| Safeguard | What it prevents | +|---|---| +| Keep scheduler receipts | A deleted scheduled snapshot being recreated by a retry | +| Coordinate conflicting operations | Cleanup deleting something that another operation just released or still needs | +| Record and resume cleanup | Partial failures leaving misleading results, stale counts, or unfinished database cleanup | + +The intended user experience remains simple: configure a retention count, or check a squash option when releasing. These safeguards make those actions safe behind the scenes. diff --git a/docs/developer/release-tracks/draft-lifecycle-plan.md b/docs/developer/release-tracks/draft-lifecycle-plan.md new file mode 100644 index 00000000..abc4ff45 --- /dev/null +++ b/docs/developer/release-tracks/draft-lifecycle-plan.md @@ -0,0 +1,291 @@ +# Virtual draft retention and release-time squash + +Status: implemented and verified. Work is on +`feat/virtual-track-draft-retention` in both repositories, created from local +`next` with the prior draft-composition commits preserved: + +- REST API: `b1acfaad` — `feat(release-tracks): support draft components in virtual tracks` +- Frontend: `60b3ac93` — `feat(release-tracks): expose draft component selection` + +The frontend's subsequent save-dialog timing regression fix is also preserved. + +## Recommendation + +Both features are viable. Implement them as two selection policies using one +internal, guarded historical-draft cleanup operation. Keep normal latest-draft +DELETE semantics unchanged. Add a compact history view independently of storage +retention: hiding drafts is reversible; deleting them is not. + +The defaults below include the subsequent frontend-feedback refinements: + +| Decision | Recommended behavior | +| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| Scope | Virtual tracks only; leave standard rolling drafts and rollback sources unchanged | +| Retention triggers | One-shot manual materialization policy or persistent cron-schedule policy, both disabled by default | +| Suggested starting threshold | 10; positive integer, minimum 1; no zero-as-delete-all behavior | +| Counted drafts | All virtual drafts, including manual, scheduled, composition/configuration/metadata and quarantine-resolution drafts | +| Tagged snapshots | Never count against N and never delete | +| Policy changes | Saved cron schedule updates create no content draft/deletion; manual policies last one request | +| New track clones | Retention disabled rather than inheriting a destructive policy silently | +| Release-time squash | Explicit, unchecked opt-in on each virtual release | +| First-release squash | Delete all eligible drafts strictly before the selected snapshot | +| Protected snapshots | Skip and report; safety overrides the configured count | +| Authorization | Existing application administrators configure retention and opt into bulk squash; ordinary tagging permissions remain unchanged | + +There is no track-specific administrator/ACL model today. A new per-track role +system is outside this proposal. Extending these controls to existing team leads +or editors is a product decision, not a reason to introduce ACL infrastructure. + +## Baseline implementation findings + +- `snapshot-service.cloneSnapshot` locks and prunes standard tracks, but retains + virtual history (`app/services/release-tracks/snapshot-service.js:427-521`). +- Virtual materialization locks its standard components, not its virtual target + (`app/services/release-tracks/virtual-track-service.js:523-600`). Other virtual + creation paths include composition updates, quarantine promotion, and shared + metadata/configuration updates. Schedule updates and snapshot notes do not + create drafts. +- Public snapshot DELETE refuses historical drafts and reverts latest membership + to the predecessor (`snapshot-service.js:999-1045`). It is not a suitable + implementation of retention or squash. +- Virtual tagging keeps the selected snapshot's timestamp, members, manifest and + composition provenance. Historical tagging is supported. The existing release + planner finds the preceding tag by snapshot `modified`, not `tagged_at` or + highest version (`versioning-service.js:121-347`; existing retroactive release + cases in `app/tests/api/release-tracks/release-tracks-release.spec.js`). +- Scheduled recovery looks for the persisted scheduled snapshot before retrying. + The occurrence is marked complete only after materialization and audit finish + (`app/scheduler/virtual-track-snapshots-task.js:104-158`). Deleting the recovery + snapshot without a durable receipt can cause a completed occurrence to run again. +- Manifests can be shared by snapshots. `content-manifest-service` provides + reference-aware deletion and orphan repair. Its current concurrent deletion of + entries and header can strand entries if only header deletion succeeds + (`app/services/release-tracks/content-manifest-service.js:341-398`). +- The frontend connector requests 200 history summaries; the page ignores + pagination metadata and renders every returned item. Older history beyond that + page is not navigable through these controls + (`release-tracks.service.ts:223-245`, `release-track-page.component.ts:1058-1081`, + `release-track-page.component.html:545-570` in the frontend repository). + +## Feature A: trigger-scoped count-based retention + +Frontend feedback replaced the original global policy with two distinct choices: + +- **Ad-hoc:** optional `draft_retention: { max_drafts: N }` on + `POST /api/release-tracks/:id/virtual/snapshots/create`. It applies only to this + materialization, is off by default, and never changes/inherits recurring policy. +- **Recurring:** optional `snapshot_schedule.draft_retention` within the cron + schedule, saved through `PUT /api/release-tracks/:id/virtual/schedule` or initial + virtual-track creation. Only trusted scheduler cron execution applies it. + +Positive safe integers are accepted; missing/null limits disable cleanup. +Manual/dates schedule shapes reject retention fields. Metadata, configuration, +composition and quarantine writes do not invoke retention. Both policies still +count all untagged snapshots across the track, not separate cause pools. + +Supplying an ad-hoc policy or changing a cron limit requires an administrator. +Editors can preserve the policy while editing cron timing or switch away from +recurring mode. Schedule-only saves create no draft or immediate deletion. + +Materialization locks its target before reading its source, chooses the policy +from the trusted trigger, and creates durable bounded intent before saving. +After successful persistence it removes eligible older drafts, preserving the +newest N, tagged snapshots, and source/receipt protections. Shared manifests and +registry counters are repaired through the existing cleanup path. + +Intent records preserve `source`, `max_drafts`, and the original cutoff. Manual +retry retains its immutable request limit independently of later schedule edits; +recurring retry also respects the current applicable cron policy. Disabling or +leaving recurring mode stops additional recurring selection. Legacy intents with +no trigger/limit may repair already-deleted storage but cannot select more rows. + +The earlier global field/endpoint are removed, with no compatibility alias. +Stored legacy global values are inert rather than silently adopted as new +destructive policies. The existing deletion guardrails and squash behavior remain. + +## Feature B: opt-in release-time squash + +This is history deletion, not a merge of contents. Do not combine members, +quarantine choices, notes, manifests or provenance into a new synthetic snapshot. +The reviewed release remains the exact artifact being tagged. + +For selected virtual draft R and preceding tagged snapshot P, eligible drafts D +satisfy: + +```text +D.version == null +P.modified < D.modified < R.modified +``` + +If P does not exist, omit the lower bound. Always retain R, all tags, and every +snapshot at or after R. Historical tagging therefore cannot remove newer drafts +or a later release. An active draft means the latest snapshot when untagged, not +an arbitrary older untagged snapshot remaining after the target is tagged. + +Example: + +```text +v1.0 -> draft A -> draft B -> selected R -> newer draft -> v3.0 +``` + +Tagging R with squash removes A/B only. It preserves v1.0, R, the newer draft and +v3.0. Use snapshot chronology, never wall-clock tagging order or semantic-version +magnitude, to determine these boundaries. + +### API and UI + +- Extend the existing strict release payload with `squash_drafts: boolean`, false + by default. Reject true for standard tracks and unauthorized actors before + tagging. Keep existing increment/version and notes behavior. +- Extend the virtual release summary preview with a `draft_squash` section: + boundaries, eligible count, protected count/reasons, and a preview fingerprint. + Use the same selector as commit; preview performs no deletion. +- Recommend requiring that fingerprint for destructive opt-in. Under the target + lock, recompute eligibility; return 409 and request a fresh preview if the + reviewed boundary/candidate set changed. This avoids silently expanding what + the user approved. Ordinary tagging remains unaffected. +- In the preview dialog, show an unchecked **Delete earlier drafts after tagging** + checkbox, the count and preceding release boundary (or “since track creation”), + and an irreversible-deletion warning. Do not imply content merging. +- Submit to the previewed exact `:modified` endpoint, not a moving `latest` target. +- Commit and finish release publication/artifacts before deleting any old drafts. + A failure before successful release completion deletes no squash candidates. +- Return the normal snapshot response plus operation-only `draft_cleanup` status, + counts and audit operation ID. Do not put mutable cleanup progress in the + released snapshot's content or history fields. + +Converting the newest virtual release back to a draft still preserves that +snapshot and its content. It does not resurrect squashed earlier drafts. + +## Shared prerequisites and safeguards + +The rationale and failure examples are documented in +[Deletion Guardrails](deletion-guardrails.md). That document is the canonical +plain-language explanation of scheduler receipts, target-track serialization, +and recoverable cleanup; do not duplicate that explanation here. + +Implementation requirements specific to this plan: + +- Acquire the virtual target lock before source reads; hold standard component + locks in sorted order during materialization. Cover every creation path and + whole-track deletion; avoid nested non-reentrant acquisition. Use bounded + cleanup batches and verify lock ownership before destructive writes. +- Preserve monotonic scheduled-occurrence receipts, including legacy and + API-supplied occurrence metadata. Receipt-only recovery must never rematerialize + deleted results, and stale workers must not reopen completed work. +- Persist bounded audit-backed cleanup intent before deletion. Bind squash to + its original release event and non-expanding interval. Expose cleanup-only + status/retry, independently of normal release POST. +- Reference-check shared manifests, make entry/header deletion retryable, repair + registry counts without rolling back latest membership, and never delete STIX + objects or component snapshots as a side effect. + +## Frontend history changes + +Virtual-track history now offers **Drafts only**, **Releases only**, and **All +releases** (the default). All releases includes both states. Use exact server +tagged filtering with no pinned-draft injection, and retain the 25-item paginator. +The small summary reports filtered tagged/draft/total counts across matching +pages, including zero results, rather than registry-wide totals or page length. + +Refresh lightweight history and cleanup status every 30 seconds while Releases +is visible, plus immediately on entry/focus/visibility return. Pause during +editing, dialogs, mutations and active requests; preserve filter/page/scroll and +clamp an emptied page. Tear down timers/listeners and cancel stale requests. +Unfiltered server latest identities control Latest markers and latest-only +actions. Keep standard-track source-draft hiding behavior unchanged. + +The **Create Draft** dialog offers a fresh, off-by-default ad-hoc policy. +Saved recurring policy controls appear only for **Recurring** mode and follow +**Edit Config**, **Cancel**, and **Save Config**. Center the history controls, +refresh authoritative counts with stale-response protection, and omit the +redundant pinned label. Completed cleanup uses a floating dismissible notification; +pending/failed cleanup remains discoverable with cleanup-only retry. + +## Implementation sequence on the same feature branches + +1. **Shared lifecycle safety:** target-lock coverage, guarded historical cleanup, + recoverable shared-manifest cleanup, durable intent/status/repair, scheduler + receipt preservation and non-replay checks. Keep both policies disabled until + these prerequisites are verified. +2. **Automatic retention:** registry field, strict API/authorization, creation + integration, config read/write projection and documentation/Bruno updates. +3. **Release-time squash:** pure interval selection, preview/fingerprint, opt-in + release commit, audit-bound cleanup-only recovery and OpenAPI documentation. +4. **Frontend:** retention configuration, virtual history filters/pagination, + opt-in preview control and partial-outcome handling. These consume the agreed + backend schemas; avoid a parallel alternative frontend contract. +5. **Verification:** focused API and scheduler regressions, real-browser flows + against isolated data, then the full backend suite and affected frontend + tests. Commit backend/frontend behavior as separate `feat(release-tracks)` + changes; use appropriately scoped commits for prerequisite fixes. + +## Acceptance checks for implementation + +- Disabled retention and omitted/false squash preserve existing history. +- N=1 and N=10 behave correctly with interleaved releases and every draft cause; + invalid settings fail, policy-only save creates no draft, new clones are safe. +- All tagged snapshots, current/target snapshots and protected sources survive. +- First-release and historical-release squash respect strict boundaries, including + tagging order different from snapshot chronology; stale preview refreshes. +- Surviving release exports/hashes, manifests, provenance and latest backrefs are + unchanged; unreferenced manifest entries really disappear; STIX objects remain. +- A scheduled result pruned before/after occurrence completion is never recreated + after restart, duplicate delivery, stale-worker failure or explicit API retry. +- Fault injection after draft/tag persistence, during deletes, manifest cleanup, + counter repair and audit completion converges via cleanup-only recovery without + a duplicate release or a widened deletion interval. +- Concurrent materialization/configuration/tagging/rollback/deletion either + serializes or reports 409; no tagged row or live shared manifest is deleted. +- Authorization is enforced in the service, not only hidden frontend controls. +- Real UI checks cover many drafts, more than one page of tags, default compact + view, policy save, unchecked squash default, clear deletion counts, disappearing + drafts and a committed release with deferred cleanup. + +## Initial evaluation + +Read the implementation, request contracts and existing regression scenarios; +no retention/squash implementation was changed and no database cleanup ran. +Executed an in-memory model of the proposed selectors covering count thresholds, +interleaved tags, historical and first releases, newer drafts, and protected +snapshots. All six scenario groups passed. This validates the proposed boundary +rules only, not MongoDB locking, scheduler durability, or production behavior. + +## Execution evidence + +The approved defaults are implemented. Browser verification against an isolated +API demonstrated compact history, 25-item draft pagination, policy-only saves, +31 drafts reduced to 10 while preserving a tag, first-release squash, and +cleanup-only repair after an injected manifest-deletion failure. The release's +bundle hashes remained unchanged through repair. A regression also verifies +that a successful repair clears the previous failure from its response. + +Focused lifecycle/scheduler and frontend regressions exercise the API contracts, +fault recovery, authorization and UI transitions. See the +[implementation backlog](../TODO.md#virtual-draft-retention-and-release-time-squash-2026-09-23) +for verification completion. Public behavior is documented in the +[API reference](../../user/release-tracks/api-reference.md#virtual-draft-retention); +the explanation requested during planning is preserved in +[Deletion Guardrails](deletion-guardrails.md). + +Initial lifecycle verification completed: 1,143 backend tests across all `npm test` stages and +143 focused frontend tests passed. Backend lint and changed frontend source lint +passed. Real-browser checks also covered 26 tagged releases across two pages, +with the current draft pinned and global counts unchanged between pages. + +The local test environment exposed a Supertest transport issue: it hardcodes an +IPv4 client URL even when the ephemeral listener uses IPv6, occasionally reaching +an unrelated local listener. Full-suite verification used a temporary adapter +matching the listener's address family, without changing test selection, +assertions, or production code. That diagnostic tooling was removed afterward. +Repository-wide frontend lint still reports existing errors in untouched files; +the changed lifecycle files pass their scoped lint check. + +Trigger-policy feedback verification: all 1,149 backend tests and 132 focused +frontend tests passed, together with backend and changed-source frontend lint. +The real browser/API flow verified centered controls, no redundant pinned text, +manual counter growth (4 to 5), one-shot cleanup and fresh-dialog reset, floating +notification dismissal without refresh resurrection, Edit Config gating/Cancel, +and schedule-only saving without a new snapshot. Actual cron execution reduced +five drafts to its saved limit of three; a separate ad-hoc limit of one left that +saved recurring limit unchanged. diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index 44ad7218..20be9f92 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -6,13 +6,31 @@ This document tracks new database schemas, interfaces, etc.; as well as changes | Collection | Purpose | Written by | Growth and retention | | ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | -| `releaseTrackRegistry` | One document per track: name, type, denormalized counters, the tagged-release catalogue (`tagged_releases`), the release lock, and virtual schedules. The index that maps a track to its own snapshot collection. | Track create/delete, every snapshot write (counters), release commit and conversion to draft (catalogue). | One document per track. | -| `release-track--` | The track's snapshots: one active rolling draft, a preserved source draft per tagged standard release, and every tagged release; every materialized draft plus releases for a virtual track. | Snapshot service and release commit. | Standard tracks grow by two snapshots per release plus one active draft; virtual tracks by materializations. | +| `releaseTrackRegistry` | Track identity, counters, tagged catalogue, lifecycle lock, and live virtual schedule with cron-only nested retention. | Track creation/deletion, snapshot lifecycle and schedule updates. | One document per track. | +| `release-track--` | Standard rolling drafts, preserved release sources, virtual-pinned source drafts and tags; virtual drafts and releases. | Snapshot service and release commit. | Virtual drafts can be bounded by count retention or explicit release-time squash; tagged snapshots are never pruned. | | `releaseTrackContentManifests` | The sealed bill of materials each snapshot references (`content_manifest_id`). Several snapshots share one manifest when their member sets are identical. | Sealed whenever members are written; discarded when no snapshot references it. | Bounded by member-changing writes, not by snapshot count. | | `releaseTrackContentManifestEntries` | One exact-revision pointer per object a manifest emits or depends on. The `(object_ref, object_modified)` index is what protects referenced revisions from deletion. | With its manifest. | Roughly members + relationships + a few supporting objects per manifest. | | `releaseTrackReconciliations` | Outstanding backref reconciliation work only: a record is created before the `workspace.release_tracks` listeners run and deleted when they succeed, so anything present is pending or failed and needs repair. | Every snapshot write. | Normally empty. | -| `releaseTrackAuditEvents` | Audit trail for administrator-only track deletion, release conversion to draft, and release retagging (`delete_track`, `convert_release_to_draft` (legacy: `delete_release`), `retag_release`). | Those operations. | Empty until an administrator performs one of those operations. | -| `virtualTrackScheduleOccurrences` | Durable claims for scheduled virtual materialization (cron or dated schedules) so restarts and duplicate delivery execute each occurrence once. | The scheduler. | One record per scheduled occurrence; empty when no virtual track has a schedule. | +| `releaseTrackAuditEvents` | Destructive audit and bounded cleanup intent/progress for deletion, conversion, retag, `draft_retention`, and `draft_squash`. | Destructive operations and cleanup recovery. | Durable audit records; cleanup results are discoverable independently of snapshot survival. | +| `virtualTrackScheduleOccurrences` | Ownership-fenced claims and monotonic `snapshot_modified` materialization receipts. | Scheduler and scheduled snapshot lifecycle. | One record per track/time, retained after snapshot cleanup; also covers API-originated occurrence metadata. | + +Retention is `{ max_drafts: null | positive safe integer }`, supplied either on +the manual materialization request or inside the live cron `snapshot_schedule`. +There is no global registry retention policy. Workbench virtual responses +project the schedule plus registry `snapshot_count` and `tagged_release_count` +for paginated clients; these are not historical snapshot content. + +Retention audit intents persist their `source` (`manual` or `recurring`), applied +`max_drafts`, and original cutoff. Manual retries use that fixed request policy; +recurring retries additionally honor the currently enabled cron policy. Legacy +intents lacking these fields cannot select further drafts, but storage repair +remains available. + +A server-controlled `release_event_id` binds release-time cleanup to the original +tagging event, independently of mutable version labels and reusable virtual +snapshot timestamps. Cleanup progress belongs to its audit record, not the +released manifest or version-history content. See [Deletion Guardrails](deletion-guardrails.md) +and the [API lifecycle contracts](../../user/release-tracks/api-reference.md#virtual-draft-retention). Removed by the sealed-manifest work: the former `releaseTrackGraphManifests` and `releaseTrackGraphManifestEntries` collections (renamed in place by the @@ -393,7 +411,7 @@ Virtual release tracks compute their contents by aggregating objects from compon component_tracks: [ { track_id: "release-track--groups-monthly", - resolution_strategy: "latest_tagged", // "latest_tagged" | "specific_version" | "specific_snapshot" + resolution_strategy: "latest_tagged", // "latest_tagged" | "latest_preview" | "specific_version" | "specific_snapshot" priority: 1, // Always required and unique (lower number = higher priority) // Optional: filters to limit which objects are included @@ -617,9 +635,11 @@ on the resulting snapshot and projected into track-list and snapshot-history responses. Snapshot clones clear inherited occurrence metadata unless the mutation explicitly supplies a replacement. -The track-local unique index on `scheduled_for`, together with the durable -`virtualTrackScheduleOccurrences` claim record, makes duplicate delivery and -restart recovery idempotent. Failed occurrences remain retryable. +The track-local unique index prevents duplicate surviving scheduled snapshots. +The occurrence ledger also retains a monotonic materialization receipt after +cleanup, so missing snapshots do not make completed work executable again. +Unmaterialized failures remain retryable; stale claim owners cannot overwrite a +new worker's completion. See [Deletion Guardrails](deletion-guardrails.md). **Key Differences from Standard Tracks:** @@ -627,30 +647,30 @@ restart recovery idempotent. Failed occurrences remain retryable. 2. **Two-Tier System**: Only `members` and `quarantine` (no `candidates` or `staged` tiers) 3. **Composition Rules**: Defines which component tracks to aggregate and how 4. **Composition Resolution**: Immutable metadata about how snapshot was computed -5. **Sync from Members Only**: Always pulls from component tracks' `members` tier (never staged or candidates) +5. **Effective Membership**: Published sources contribute members; `latest_preview` drafts contribute prospective membership (members plus staged changes under source conflict rules, never candidates) 6. **No Workflow States**: No work-in-progress, awaiting-review, or reviewed states 7. **Scheduled Snapshots**: Can auto-generate snapshots on schedule 8. **Component Version Tracking**: Version history records which component versions were included **Virtual Track Constraints:** -- Can only reference **tagged snapshots** from component tracks (not drafts) -- Can only sync from component tracks' **`members` tier** (released objects only) +- Select published snapshots or the newest standard snapshot with `latest_preview` +- Preview drafts through standard release membership planning; never mutate the source or include candidates - Can only compose from **standard release tracks** (not other virtual tracks - no nesting allowed) - Is purely compositional and has no `native_members` or second membership authority; aggregate-specific content belongs in another standard component track - Snapshots are created **manually or on schedule** (never event-driven) - All snapshots start as **drafts** and must be explicitly tagged -- Component tracks must exist and have at least one tagged release +- Component tracks must exist; selected source snapshot eligibility is checked at materialization - Each component track must have a unique **priority** value (no duplicates) - Priority is a required non-negative integer for every component, regardless of deduplication strategy - Component IDs and priorities are validated before initial virtual-track persistence as well as during composition updates and materialization - Snapshot schedules are strict and mode-discriminated: `manual` accepts only - `mode`, `cron` requires only a five-field `cron` expression, and `dates` - requires only a nonempty `dates` array + `mode`, `cron` requires a five-field `cron` expression and accepts optional + nested `draft_retention`, and `dates` requires only a nonempty `dates` array - Standard tracks reject `snapshot_schedule`; virtual `cron` and `dates` schedules execute through the global scheduler - `filters.object_types` uses the canonical Workbench STIX type names from @@ -668,12 +688,15 @@ restart recovery idempotent. Failed occurrences remain retryable. `version_history[].component_versions`. This is an object keyed by immutable component `track_id`, not display name. It records the frozen materialization inputs even when a component has newer releases by the time the virtual draft - is tagged. Standard release history entries omit the field + is tagged. Draft components record `null`; tagged components record their + version string. Standard release history entries omit the field - Composition request objects are strict; unknown composition, component, filter, and deduplication keys return `400 Bad Request` - Selector fields form a discriminated request contract: - - `latest_tagged` rejects `version` and `snapshot` + - `latest_tagged` and `latest_preview` reject `version` and `snapshot` - `specific_version` requires `version` and rejects `snapshot` - `specific_snapshot` requires `snapshot` and rejects `version` - Quarantine promotion selects an exact revision in a new draft and preserves the source snapshot's immutable `composition_resolution` +- Retired `latest_draft` is read-only historical composition/provenance, not an + active request strategy or an alias for preview behavior diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 2d153600..28cbffe8 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -204,7 +204,7 @@ Virtual-only operations are deliberately scoped beneath rules, empty members/quarantine tiers, and `composition_resolution: null`. Clearing all three prevents a materialized result from surviving a change to the rules that produced it. -- `POST /virtual/snapshots/create` resolves tagged component snapshots and +- `POST /virtual/snapshots/create` resolves published or preview component snapshots and persists the concrete members, quarantine, and immutable `composition_resolution`. - Every persisted member and quarantine entry uses an exact @@ -215,20 +215,48 @@ Virtual-only operations are deliberately scoped beneath moving reference. - `member_sync.strategy = track_latest` applies only to standard tracks. New object revisions may update a component's newer candidate/staged draft, but - they cannot rewrite the members of the tagged component snapshot selected - during virtual materialization or an already-persisted virtual snapshot. + they cannot rewrite the members of the component snapshot selected during + virtual materialization or an already-persisted virtual snapshot. - `POST /virtual/quarantine/promote` clones the latest virtual snapshot, selects one exact quarantined revision for members, and removes all quarantined alternatives for that object. Composition input uses strict Zod objects at the composition, component, filter, and deduplication levels. Components form a discriminated union on -`resolution_strategy`: `latest_tagged` accepts no selector, +`resolution_strategy`: `latest_tagged` and `latest_preview` accept no selector, `specific_version` requires only `version`, and `specific_snapshot` requires only `snapshot`. This prevents misspelled filters or irrelevant selectors from being silently stripped before persistence. The same schema is used for initial virtual-track creation and composition updates. +`latest_preview` resolves the newest standard snapshot regardless of tag state. +For a draft, `versioning-service.planPreviewMembers` shares `resolveReleaseInput` +and `planMembership` with standard release planning: resolve exact revisions, +normalize tiers and apply the source's staged-to-members conflict policy. It +does not allocate versions, freeze publication, reject already-released source +drafts, or write source snapshots. Tagged inputs contribute published members +without incorporating staged entries. + +This planning occurs before component filters/domain hydration and virtual +deduplication. A blocking source conflict returns `409` with the component +`track_id` and standard conflict details. Source counts describe planned +membership; stored source members can still be empty. Candidates are excluded. +Existing `latest_tagged`, `specific_version` and `specific_snapshot` continue to +select published members only. + +The active request enum rejects retired `latest_draft`; it is not an alias. +Read-only stored composition and provenance schemas retain the historical token +so existing members-only results remain truthful and releasable. Saved rules +must be explicitly replaced before rematerialization. No nightly-data migration +silently opts existing operators into staged content. + +Standard clone save/prune and virtual materialization share the existing release +lock; contention returns `409 Conflict`. See [Deletion Guardrails](deletion-guardrails.md) +for the concurrency rationale. Pruning retains every source snapshot named by +persisted virtual provenance, including historical virtual drafts. Once the last +dependent disappears, the next standard clone can prune the source if no other +retention rule protects it. + Component `priority` is always required, even when the selected deduplication strategy does not inspect it. Zod rejects duplicate component IDs and priorities before service delegation. The facade also asks the virtual-track @@ -339,13 +367,14 @@ planned snapshot, and the commit path tags that snapshot in place. Virtual release planning also derives `version_history[].component_versions` directly from the selected draft's immutable `composition_resolution.component_snapshots`. The property is a -component track ID to tagged `MAJOR.MINOR` version map. It deliberately does -not query the component tracks at preview or commit time: a component can -advance after virtual materialization without changing the provenance of the -already-frozen draft. Standard release history entries omit the virtual-only -property. Mongoose validates every map value with the shared release-version -validator and requires every persisted component resolution to identify its -tagged `resolved_version`. +component track ID to tagged `MAJOR.MINOR` version or `null` map. It deliberately +does not query the component tracks at preview or commit time: a component can +advance after materialization without changing the already-frozen provenance. +Standard release history entries omit the property. Mongoose validates string +values with the shared release-version validator. Component resolutions require +a tagged `resolved_version` for tagged strategies. `latest_preview` records +`null` for a draft or the actual version for a tagged source. Historical +`latest_draft` provenance continues to permit its original null version. Standard release commit assigns a fresh timestamp, stores `release_source_modified`, and inserts the tagged clone while retaining the @@ -371,9 +400,10 @@ and a list response. `release-track-dynamic.repository.getSnapshotSummaries` performs tagged-state filtering, descending timestamp ordering, pagination, and tier counts in -MongoDB. It projects counts with `$size` rather than hydrating the potentially -large tier arrays. The filter is applied to both the data query and -`countDocuments`, making `pagination.total` the filtered total. +MongoDB. A metadata-only grouping computes filtered `counts.tagged`, +`counts.drafts`, and `counts.total` before pagination. The page query projects +tier counts with `$size` only for its bounded result, rather than hydrating every +snapshot's potentially large arrays. `pagination.total` equals `counts.total`. The service shapes projected counts according to `snapshot.type`: @@ -393,6 +423,15 @@ match, and shared manifests are counted once. The service fills zero-valued categories for empty manifests. This keeps history latency to one additional bounded query rather than one query per snapshot. +The history service also projects unfiltered `latest_snapshot_modified` and +`latest_tagged_snapshot_modified` from the registry metadata it already reads. +The latter uses the newest catalogue snapshot timestamp, not highest semantic +version or tagging time. Filtered pages therefore retain correct latest-only +action identities. The frontend polls this lightweight history plus outstanding +cleanup while Releases is visible; it does not poll full member/configuration +payloads. Visibility/focus and tab entry refresh immediately, while edits, +dialogs, mutations and active requests pause background refresh. + ## Integrating with the Event-Driven Architecture ### Events Published diff --git a/docs/developer/release-tracks/member-sync-strategies.md b/docs/developer/release-tracks/member-sync-strategies.md index 720d4cf8..86998eeb 100644 --- a/docs/developer/release-tracks/member-sync-strategies.md +++ b/docs/developer/release-tracks/member-sync-strategies.md @@ -131,7 +131,7 @@ Member sync strategies integrate with several existing release track features: - **Candidacy Threshold:** When a new revision is auto-enrolled as a candidate, it may be immediately promoted to `staged` if its status meets the candidacy threshold. - **Conflict Resolution Policies:** Member sync resolves overlaps with existing `candidates`/`staged` entries through its own `supplant` config (below). *Manual* candidate adds and demotions instead go through `config.promotion_conflicts.into_candidates` (default `prefer_latest`) — see `release-workflow.md`. The two are deliberately separate: supplant expresses sync intent (replace/queue/ignore), while `into_candidates` uses the same policy vocabulary as the other tier transitions. -- **Snapshot Creation:** Any change to a release track's object lists (`candidates`, `staged`, `members`) creates a replacement draft snapshot. Standard tracks retain only the newest untagged draft after it is durably saved; tagged snapshots remain historical. Member sync follows this convention. +- **Snapshot Creation:** Any change to a release track's object lists (`candidates`, `staged`, `members`) creates a replacement draft snapshot. After it is durably saved, older untagged drafts are pruned unless they are preserved release sources or referenced by virtual provenance. Tagged snapshots remain historical. Member sync follows this convention. --- diff --git a/docs/developer/release-tracks/sealed-content-manifests.md b/docs/developer/release-tracks/sealed-content-manifests.md index eec15044..931b54ea 100644 --- a/docs/developer/release-tracks/sealed-content-manifests.md +++ b/docs/developer/release-tracks/sealed-content-manifests.md @@ -128,12 +128,14 @@ never rewritten by conversion. Legacy `delete_release` audit entries remain valid; new conversions use `convert_release_to_draft`. Virtual materialization acquires the existing database-backed release locks -for all component tracks in sorted order, before resolving any release, and +for all component tracks in sorted order, before resolving any source snapshot, and holds them through snapshot persistence. Partial acquisition and failed -materialization unwind the locks. Contention fails fast with 409. The rollback -dependency scan therefore cannot miss an in-flight materialization: either -rollback owns the lock first, or it sees the persisted virtual dependency -after materialization releases the lock. +materialization unwind the locks. Contention fails fast with 409. Standard +clone save/prune acquires the same component lock. The concurrency and shared +manifest deletion rationale is documented in [Deletion Guardrails](deletion-guardrails.md), +rather than repeated here. Pruning retains drafts referenced +by virtual provenance; after the final dependent is removed, a later standard +clone can prune an otherwise-unprotected draft. Retag prepares both bundle serializations before writing, then atomically publishes the version, publication metadata, bundle ID, and hashes on the @@ -151,7 +153,8 @@ History's repository projection and service summary both expose and never queries relationships, and a draft replays its inherited members graph. - Manifest storage is bounded by the number of member-changing writes, not - the number of snapshots. Standard tracks already keep only one draft. + the number of snapshots. Standard tracks retain one active draft plus release + sources and drafts referenced by virtual provenance. - Editing an object creates no relationship revisions; editing a relationship creates exactly one relationship revision. - Existing databases are migrated in place: the `graph_manifest_id` field is diff --git a/docs/developer/release-tracks/snapshot-creation-causes.md b/docs/developer/release-tracks/snapshot-creation-causes.md index 32eba523..6f0fd1a6 100644 --- a/docs/developer/release-tracks/snapshot-creation-causes.md +++ b/docs/developer/release-tracks/snapshot-creation-causes.md @@ -72,8 +72,9 @@ publication configuration. That config write creates a draft labelled "Configuration updated", even if the supplied config equals its prior value. An actual scheduled materialization is labelled "Scheduled snapshot". -Standard tracks keep only their latest rolling draft, so this is provenance -for each surviving snapshot, not a complete event log. If auto-promotion +Standard tracks retain their latest rolling draft plus release sources and +drafts referenced by virtual provenance. This is provenance for each surviving +snapshot, not a complete event log. If auto-promotion immediately replaces a candidate-add/review draft, the surviving snapshot is labelled "Candidates automatically promoted". Tagged snapshots retain their creation cause throughout their lifetime. Standard release creation persists a diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 424c93cf..e1e108bb 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -455,6 +455,18 @@ GET /api/release-tracks/:id/snapshots Filtering occurs before pagination, so `pagination.total` is the total number of snapshots matching `tagged`, not the total number in the track. +The response also includes `counts: { tagged, drafts, total }`. All three counts +use the same `tagged` filter before pagination: `counts.total` equals +`pagination.total` and `counts.tagged + counts.drafts`. They remain query-wide +when the requested page is partial or empty. Thus `tagged=true` always reports +zero drafts, and `tagged=false` always reports zero tagged releases. + +`latest_snapshot_modified` and `latest_tagged_snapshot_modified` are unfiltered +track identities, not the first row of the selected page. They are nullable ISO +timestamps; the tagged identity is chronological rather than the highest semantic +version. Clients use them to mark the actual latest snapshot and gate latest-only +actions while refreshing filtered/paginated history. + Every summary contains `id`, `type`, `modified`, `version`, `name`, the track-level `description` (when set), `snapshot_description` (when the snapshot has user-authored notes), `members_count`, the opaque `content_manifest_id` @@ -513,10 +525,17 @@ Inapplicable count keys are omitted rather than returned as zero. } ], "pagination": { - "total": 47, + "total": 1, "limit": 50, "offset": 0 - } + }, + "counts": { + "tagged": 1, + "drafts": 0, + "total": 1 + }, + "latest_snapshot_modified": "2024-01-15T16:20:00.000Z", + "latest_tagged_snapshot_modified": "2024-01-15T16:20:00.000Z" } ``` @@ -659,7 +678,7 @@ The virtual release response records the materialized component provenance in ``` Keys are immutable component track IDs and values are the tagged versions -stored in the selected draft's `composition_resolution`. The server does not +or `null` for draft components stored in the selected draft's `composition_resolution`. The server does not look up the components' current releases, so advancing a component after materialization does not rewrite the virtual release's provenance. Standard release history entries omit `component_versions`. @@ -1372,7 +1391,7 @@ multiple standard component tracks based on configurable rules. - Compute contents only from standard component tracks; virtual-track nesting is rejected - Are purely compositional and cannot own native members -- Only reference **tagged snapshots** from component tracks (never drafts) +- Reference **tagged snapshots** or explicitly selected **active drafts** - Create snapshots **manually or on schedule** (never event-driven) - All snapshots start as **drafts** and must be explicitly tagged - Support **resolution strategies** to control which component versions are included @@ -1381,10 +1400,166 @@ multiple standard component tracks based on configurable rules. 1. `latest_tagged` - Always use the most recent tagged snapshot from component 2. `specific_version` - Pin to a specific semantic version (e.g., "5.0") -3. `specific_snapshot` - Pin to a specific snapshot by timestamp +3. `specific_snapshot` - Pin to a specific tagged snapshot by timestamp +4. `latest_preview` - Use the newest standard snapshot's prospective release membership + +`latest_preview` accepts drafts and tagged snapshots. Draft input contributes +members plus staged changes through the standard release membership planner, +including dynamic revision resolution and source conflict policies. Tagged input +contributes published members. Candidates are excluded. The source is not tagged +or mutated, and standard release version allocation/publication is not performed. +Filters and cross-component deduplication run after source membership planning; +blocking source conflicts return `409` with the source track ID and conflicts. + +Published and pinned strategies remain members-only. Preview provenance records +the exact source snapshot timestamp and `resolved_version: null` for a draft, or +the actual version for a tagged source. `total_objects_in_source` counts planned +preview members before filters, not just the source draft's stored members. +Later changes cannot rewrite the frozen virtual materialization. + +New composition requests reject retired `latest_draft`. Existing saved rules +must be explicitly changed to a supported strategy before materialization. +Historical composition/provenance retains its original label and contents. See [virtual-tracks.md](./virtual-tracks.md) for complete documentation. +### Virtual Draft Retention + +Retention has two independent, administrator-controlled triggers. Both count +untagged snapshots across the whole virtual track, regardless of creation cause; +tags and protected snapshots remain excluded from deletion. + +**Ad-hoc policy:** pass a one-shot limit when explicitly creating a draft: + +```http +POST /api/release-tracks/:id/virtual/snapshots/create +Content-Type: application/json + +{ "description": "Reviewed composition", "draft_retention": { "max_drafts": 10 } } +``` + +Omitting `draft_retention`, or setting `max_drafts` to `null`, means no retention +for this operation. The policy is not saved and never inherits or changes the +recurring schedule's policy. A supplied policy requires an administrator. + +**Recurring policy:** save a limit inside the cron schedule: + +```http +PUT /api/release-tracks/:id/virtual/schedule +Content-Type: application/json + +{ "mode": "cron", "cron": "0 * * * *", "draft_retention": { "max_drafts": 10 } } +``` + +Only trusted scheduler cron execution applies this saved policy. Manual +materialization never inherits it, and dated schedules do not use it. Clients +cannot activate it by supplying `scheduled_materialization` provenance. The +policy can also be nested in `snapshot_schedule` when creating a virtual track. + +Counts must be positive safe integers; missing/null limits disable cleanup. +The `manual` and `dates` schedule shapes reject retention fields. Changing a +cron retention policy requires an administrator; editors can change timing while +preserving that policy, or switch away from recurring mode. Saving only the +schedule/policy creates no draft and deletes nothing immediately. + +Configuration, metadata, composition and quarantine updates never trigger +retention. Cleanup still runs only after successful materialization. Manual +cleanup retries preserve the approved request limit and original cutoff; +recurring retries also honor the current saved policy, stopping further deletion +if it is disabled or the schedule changes away from cron. + +The former global policy, top-level track-creation field, and +`PUT /virtual/draft-retention` endpoint are retired. Existing root/global settings +are inert; configure the recurring policy explicitly. Legacy cleanup intents +without trigger/limit provenance may repair already-deleted storage but cannot +select additional drafts. + +### Release-Time Draft Squash + +The existing summary release preview includes virtual-only `draft_squash`: + +```json +{ + "draft_squash": { + "lower_bound": null, + "upper_bound": "2026-09-23T10:00:00.000Z", + "eligible_count": 3, + "protected_count": 0, + "fingerprint": "opaque-preview-fingerprint" + } +} +``` + +An administrator can explicitly opt into cleanup when tagging the exact +previewed draft: + +```http +POST /api/release-tracks/:id/snapshots/:modified/release +Content-Type: application/json + +{ "increment": "minor", "squash_drafts": true, "squash_fingerprint": "opaque-preview-fingerprint" } +``` + +Use the actual preview fingerprint. A changed selection returns `409` before +tagging; fetch a fresh preview rather than retrying blindly. Omitted/false squash +retains ordinary tagging behavior, including editor/team-lead access. True is +virtual-only and administrator-only. Both latest and exact release endpoints +accept the option, but the UI uses the exact previewed timestamp. + +Cleanup removes only eligible untagged snapshots strictly after the preceding +tagged snapshot and strictly before the selected snapshot, ordered by `modified`, +not `tagged_at` or version magnitude. With no preceding tag, all strictly earlier +eligible drafts are considered. Every tagged snapshot and every newer snapshot +is preserved. Release content, its manifest and composition provenance are not +merged or rewritten. Deleted history is not restored by release-to-draft conversion. + +### Inspect or Retry Draft Cleanup + +Creation, opted-in release, and retry responses can include an operation-only +`draft_cleanup` result: + +```json +{ + "operation_id": "97508a36-62cf-4a72-9ed6-49f2690d3609", + "status": "failed", + "kind": "squash", + "eligible_count": 3, + "deleted_count": 1, + "protected_count": 0, + "target_modified": "2026-09-23T10:00:00.000Z", + "release_committed": true, + "error": "Cleanup could not finish" +} +``` + +`status` is `pending`, `completed`, or `failed`; `kind` is `retention` or +`squash`. `release_committed` applies to squash; it is omitted if a store failure +prevents determining the release outcome. A successful release with failed +cleanup remains a successful release response with a failed cleanup result. +Interrupted publication may return structured `500` with top-level +`operation_id`, `release_committed` when known, and `draft_cleanup`. + +```http +GET /api/release-tracks/:id/virtual/draft-cleanup +POST /api/release-tracks/:id/virtual/draft-cleanup/:operationId/retry +``` + +GET returns `{ "data": [...] }`, up to 25 recent pending/failed operations, and +requires ordinary read access. POST takes `{}`, requires an administrator, and +returns the cleanup result. It resumes an existing intent; it cannot authorize +arbitrary new deletion or implicitly tag a draft. Normal repeated release POST +still rejects an already-tagged snapshot. + +Each cleanup invocation processes at most ten 100-snapshot pages. Large histories +can return `pending`; retry the same operation until complete. Repair is bounded +by the original intent and current protections. Disabling retention prevents +further candidate selection but does not prevent repairing already-deleted +storage. Shared manifests survive while referenced, and underlying STIX objects +are not cleanup targets. + +For the safety rationale and failure examples, see +[Deletion Guardrails](../../developer/release-tracks/deletion-guardrails.md). + ### Create Virtual Track ``` @@ -1455,7 +1630,7 @@ by the resolved component snapshot. is enabled. Its shape depends on `mode`: - `manual` accepts only `{ "mode": "manual" }`; -- `cron` requires a five-field `cron` expression and rejects `dates`; +- `cron` requires a five-field `cron` expression, accepts optional `draft_retention`, and rejects `dates`; - `dates` requires at least one ISO timestamp and rejects `cron`. Unknown schedule properties return `400 Bad Request`. Standard tracks also @@ -1474,8 +1649,8 @@ snapshot, and timestamp-selected snapshot GET requests. Composition, component, filter, and deduplication objects are strict. Unknown keys, including the incorrect singular `filters.domain`, return `400 Bad Request`. Component selectors are also strategy-specific: -`latest_tagged` rejects `version` and `snapshot`; `specific_version` requires -only `version`; and `specific_snapshot` requires only `snapshot`. +`latest_tagged` and `latest_preview` reject `version` and `snapshot`; +`specific_version` requires only `version`; `specific_snapshot` requires only `snapshot`. Every component requires a unique, non-negative integer `priority`; lower numbers have higher priority. When composition is supplied during creation, each referenced track must already exist and must be a standard track. Virtual @@ -1613,7 +1788,7 @@ readiness marker for those shared release operations. Each resulting `members` and `quarantine` entry contains an exact `(object_ref, object_modified)` pair. Virtual materialization preserves exact -revisions already frozen in the selected tagged component snapshots. It also +revisions already frozen in the selected tagged or active-draft component snapshots. It also resolves any unresolved legacy component entry before persistence. The virtual snapshot never stores `"latest"` and does not inherit a standard component's `track_latest` member-sync behavior. diff --git a/docs/user/release-tracks/release-workflow.md b/docs/user/release-tracks/release-workflow.md index 2b0967cc..51cd890d 100644 --- a/docs/user/release-tracks/release-workflow.md +++ b/docs/user/release-tracks/release-workflow.md @@ -1137,8 +1137,8 @@ July 16 (manual): ### Virtual Track Constraints -- Only references **tagged snapshots** from components (never drafts) +- Published strategies compose members; `latest_preview` also includes staged changes through standard release planning without tagging the source - Snapshots created **manually or on schedule** (never event-driven) - All snapshots start as **drafts** (must explicitly tag) -- Component tracks must have at least one tagged release +- Component tracks must have a source eligible for their selected strategy at materialization - Circular dependencies not allowed diff --git a/docs/user/release-tracks/summary.md b/docs/user/release-tracks/summary.md index 4efc2b91..94e222bd 100644 --- a/docs/user/release-tracks/summary.md +++ b/docs/user/release-tracks/summary.md @@ -36,7 +36,7 @@ The Release Tracks API supports two types of release tracks: - No duplicate object tracking - objects managed in source tracks only - Purely compositional - virtual tracks cannot add native members of their own - Create snapshots manually or on schedule (never event-driven) -- Always compose from tagged snapshots only (never drafts) +- Compose published members or prospective release membership selected with `latest_preview` - Examples: "EnterpriseTwiceAnnual" (aggregates Groups + Techniques + Software) **Use Case for Virtual Tracks:** @@ -102,7 +102,8 @@ We borrow heavily concepts from git. Snapshots are sort of like commits and tagg - Identified by `stix.modified` timestamp - Immutable once created - Standard tracks retain one active rolling draft plus the hidden source draft - for each tagged release; tagged releases remain historical + for each tagged release and any drafts referenced by virtual provenance; + tagged releases remain historical - May be a **draft release** (untagged) or **tagged release** (has version number) **Tagged Releases** (like Git tags) diff --git a/docs/user/release-tracks/terminology.md b/docs/user/release-tracks/terminology.md index 7d5d81e2..8023e2cc 100644 --- a/docs/user/release-tracks/terminology.md +++ b/docs/user/release-tracks/terminology.md @@ -250,7 +250,7 @@ When the release is exported as a `bundle`, all `members` will be included in th #### Virtual Release Tracks: Two-Tier System -Virtual release tracks use a simplified two-tier system since they aggregate already-released content: +Virtual release tracks use a simplified two-tier system: they freeze published members or prospective release membership, rather than maintaining a separate authoring workflow. ##### 1. Member Objects @@ -259,7 +259,7 @@ Virtual release tracks use a simplified two-tier system since they aggregate alr **Definition:** Successfully synced objects from component tracks. **Characteristics:** -- Contains objects synced from component tracks' `members` tiers +- Contains published source members or planned members plus staged changes selected with `latest_preview` - Objects that were automatically resolved using the deduplication strategy - OR objects manually promoted from quarantine - These objects are included in published STIX bundles @@ -301,7 +301,7 @@ A **virtual release track** is a special type of release track that computes its **Characteristics:** - Does NOT manage objects through candidate/staged/released workflow - Aggregates content only from **standard component tracks** -- Only references **tagged snapshots** from component tracks (never drafts) +- References published snapshots or the newest standard snapshot's prospective release contents with `latest_preview` - Creates snapshots **manually** or **on schedule** (*never* event-driven; see [Types of Release Tracks](#types-of-release-tracks) for explanation) - All snapshots start as drafts and must be explicitly tagged - Is purely compositional and cannot own native objects; place additional @@ -412,12 +412,14 @@ A **resolution strategy** determines which snapshot from a component track to us **Options:** 1. **latest_tagged** - Use the most recent tagged snapshot from the component track 2. **specific_version** - Use a specific semantic version (e.g., "5.0") -3. **specific_snapshot** - Use a specific snapshot by timestamp +3. **specific_snapshot** - Use a specific tagged snapshot by timestamp +4. **latest_preview** - Use the newest standard snapshot: planned members plus staged changes for a draft, published members for a release; never tag the source **Examples:** - `{ resolution_strategy: "latest_tagged", priority: 0 }` → Always gets latest - `{ resolution_strategy: "specific_version", version: "5.0", priority: 0 }` → Always uses v5.0 - `{ resolution_strategy: "specific_snapshot", snapshot: "2024-02-01T10:00:00Z", priority: 0 }` → Always uses that exact snapshot +- `{ resolution_strategy: "latest_preview", priority: 0 }` → Previews staged changes under source release conflict rules; candidates stay excluded --- diff --git a/docs/user/release-tracks/versioning.md b/docs/user/release-tracks/versioning.md index e1dc9d04..5f83d6bc 100644 --- a/docs/user/release-tracks/versioning.md +++ b/docs/user/release-tracks/versioning.md @@ -22,7 +22,8 @@ A **snapshot** is an immutable state of a release track at a specific point in t Every content-changing operation creates a replacement snapshot with a new `modified` timestamp. For a standard track, the replacement is saved first and -then the older untagged draft is removed. Tagged snapshots are never pruned. +then older untagged drafts are pruned, except preserved release sources and +drafts referenced by persisted virtual provenance. Tagged snapshots are never pruned. A snapshot may be either a **draft release** (untagged) or a **tagged release** (has version number). @@ -67,8 +68,9 @@ id: "release-track--123", modified: "2024-01-15T16:20:00.000Z" ] ``` -The timeline lists the first draft only to illustrate its replacement. Once the -second draft is durably stored, the first draft is no longer retrievable. +The timeline lists the first draft only to illustrate its replacement. In this +example it has no virtual dependents or tagged release preserving it, so once +the second draft is durably stored, the first draft is no longer retrievable. ## The Release Operation diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index 7c8d8f1b..e7a437ed 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -7,7 +7,7 @@ Virtual release tracks are computed aggregations of standard release tracks. The **Key Characteristics:** - Virtual tracks **compute** their contents from component standard tracks -- Only reference **tagged snapshots** from standard tracks (never drafts) +- Reference published snapshots or the newest standard snapshot's prospective release contents - Maintain their own **independent snapshot history and versioning** - Create snapshots **manually or on schedule** (never event-driven) - All snapshots start as **drafts** and must be explicitly tagged @@ -17,6 +17,33 @@ content. Replace it with `PUT /api/release-tracks/:id/virtual/schedule`; this does not create a draft. Workbench-format snapshot responses project the current schedule for configuration interfaces. +## Draft Retention and Release-Time Squash + +Administrators choose an optional **ad-hoc** retention limit in **Create Draft** +or configure a **persistent recurring** limit inside a cron snapshot schedule. +Both default to disabled and accept a positive safe integer such as 10. +The one-shot policy applies only to its manual materialization; the recurring +policy applies only to actual scheduler cron executions. Neither inherits from +the other. Dated schedules, configuration/metadata/composition changes, and +quarantine promotion do not trigger retention. + +Both policies retain the newest N untagged snapshots across the track's history, +not separate manual/scheduled pools. Tagged releases are never removed. +Saving only the schedule/policy creates no draft and deletes nothing immediately. + +When tagging a reviewed draft, administrators can separately opt into deleting +earlier drafts since the preceding tagged snapshot. The preview reports the +strict timestamp bounds and eligible/protected counts, and its fingerprint must +still match at commit. First-release squash considers all earlier eligible +drafts. Historical tagging preserves all newer snapshots. This removes history, +not content: the selected snapshot and its manifest remain unchanged. + +Both controls preserve protected snapshots and expose incomplete cleanup as a +retryable operation distinct from release success. Deleted drafts and their +notes cannot be restored by rollback. See the +[API retention, squash and recovery contracts](api-reference.md#virtual-draft-retention) +and [Deletion Guardrails](../../developer/release-tracks/deletion-guardrails.md). + ## Use Cases ### Scenario 1: Different Cadences for Different Object Types @@ -190,9 +217,54 @@ Resolves to a specific snapshot by its `modified` timestamp. **Use case:** "Lock to exact snapshot for reproducibility" +#### 4. `latest_preview` + +Resolves the newest standard-track snapshot, whether draft or tagged: + +```javascript +{ + track_id: "release-track--uuid-1", + resolution_strategy: "latest_preview", + priority: 0 +} +``` + +For an untagged source, composition uses the same membership calculation as a +standard release preview: existing members plus staged objects under the source +track's `promotion_conflicts.staged_to_members` policy. Exact duplicates are +normalized, dynamic `"latest"` selectors resolve to exact revisions, and +candidates are excluded. This is not a simple concatenation of the two tiers. + +For a tagged newest snapshot, composition uses its published members. A new +source draft is not required after tagging, and no older draft is selected. +`latest_tagged` remains distinct: it selects the newest published release even +when a newer draft exists. + +Component planning happens before object/domain filters and cross-component +deduplication. A blocking source promotion conflict aborts materialization with +`409 Conflict`, identifying the component and conflict. Virtual filters cannot +bypass a conflict that would prevent releasing that source. + +Neither materialization nor tagging the virtual snapshot tags, promotes, +allocates a version for, or otherwise mutates the standard source. Operators can +stage changes, create a virtual snapshot to review the downstream composition, +and leave standard-track publication to its own release cadence. Revisions +freeze at virtual materialization; later source edits do not rewrite that result. + +Provenance records `strategy_used: "latest_preview"` and the original source +`resolved_snapshot_id`. `resolved_version` is `null` for a source draft and its +actual version for a tagged source. Source/contribution counts use the planned +member set, which can be nonempty even when the draft's stored members are empty. + +**Cutover:** `latest_draft` is retired and rejected in new composition requests. +Existing rules must explicitly select `latest_preview` or `latest_tagged` before +rematerialization; they are not silently upgraded to include staged content. +Historical members-only provenance retains its original `latest_draft` label and +contents and remains readable/releasable. + Component selectors are strict and strategy-specific: -- `latest_tagged` rejects both `version` and `snapshot`. +- `latest_tagged` and `latest_preview` reject both `version` and `snapshot`. - `specific_version` requires `version` and rejects `snapshot`. - `specific_snapshot` requires `snapshot` and rejects `version`. @@ -201,15 +273,18 @@ not silently discarded. ### Component Track Sync Rules -Virtual tracks **only sync from component tracks' `members` tier** (`x_mitre_contents`). This ensures that virtual tracks only aggregate objects that have been officially released in their source tracks. +Published and pinned strategies use the selected source's members. `latest_preview` uses prospective release membership for a draft, or published members for a tagged newest snapshot. **Important:** -- Virtual tracks reference **tagged snapshots only** (never drafts) -- Virtual tracks pull objects from **`members` tier only** (never staged or candidates) -- This guarantees that virtual track releases are composed of stable, released content +- `latest_preview` includes staged changes through the standard release membership planner +- Candidates are never composed; the other strategies remain tagged-members-only +- Each materialization freezes exact member revisions and source provenance -**Rationale:** Since virtual tracks can only reference tagged snapshots from component tracks, it makes sense to only pull from the `members` tier, which contains the released objects from those snapshots. +Source drafts referenced by virtual snapshots are retained when the standard +track advances. Deletion remains blocked while a virtual snapshot depends on +the source. After the last dependent is removed, a later standard draft write +can prune that otherwise-unprotected source. ### Filters @@ -226,7 +301,7 @@ filters: { } ``` -Domain filters hydrate the exact revisions pinned by the component's tagged +Domain filters hydrate the exact revisions pinned by the selected component snapshot; they do not inspect the latest database revision. Matching uses inclusive **any-match** semantics, not exact-array equality: an object is included when at least one value in its canonical `x_mitre_domains` array @@ -462,7 +537,7 @@ Unlike standard release tracks (which use a three-tier system: candidates → st **Why only two tiers?** -Virtual tracks aggregate content from component tracks that have already gone through the full workflow (candidates → staged → members). Virtual tracks don't need the intermediate `staged` tier because they're composing already-released content. The only workflow step is resolving conflicts via the `quarantine` tier. +Virtual tracks compose an effective member set rather than managing a separate authoring workflow. Published sources contribute members; draft previews calculate members plus eligible staged changes without promoting them in the source. The virtual snapshot freezes that result directly into members or quarantine, so it does not need its own staged tier. ## Virtual Track Snapshot Lifecycle @@ -543,8 +618,8 @@ description. 1. For each component track in `composition.component_tracks`: - Resolve snapshot based on `resolution_strategy` - - **Validate that resolved snapshot is tagged** (version !== null) - - Pull objects from component track's **`members` tier only** (`x_mitre_contents`) + - Require a tagged source for published/pinned strategies; `latest_preview` accepts the newest draft or release + - For a previewed draft, calculate prospective members with standard release conflict rules; otherwise use published members - Apply `filters` to get subset of objects - Collect all object references with source metadata 2. Apply deduplication rules across all components: @@ -689,11 +764,11 @@ POST /api/release-tracks/:id/snapshots/:modified/release } ``` -`component_versions` is keyed by immutable component track ID. Its values come -from the selected draft's `composition_resolution`, not from the component -tracks' current releases. If a component advances after this virtual draft was -materialized, the virtual release still records the version that actually -produced its frozen contents. Standard release history entries omit this +`component_versions` is keyed by immutable component track ID. Values are +tagged version strings or `null` for draft components, copied from the selected +virtual draft's `composition_resolution`. If a component advances or is tagged +after materialization, the virtual release still records its original source +state and exact snapshot ID. Standard release history entries omit this virtual-only property. The snapshot-history endpoint (`GET /api/release-tracks/:id/snapshots`) also @@ -703,7 +778,7 @@ clients present provenance beside the draft or release it describes rather than presenting only the virtual track's current HEAD resolution. In each component entry, `resolved_snapshot_id` is the exact component snapshot's creation timestamp and stable retrieval key; `resolved_version` names its -tagged version. The stored source, filtered, and contributed counts belong to +tagged version or is `null` for a draft. The stored source, filtered, and contributed counts belong to that materialization and are not recomputed from the component track's current state. Full snapshot retrieval additionally returns the deduplication report and resolution summary. @@ -832,37 +907,13 @@ Each virtual track snapshot stores metadata about how it was composed: ### Validation Rules -#### 1. Component tracks must have tagged snapshots - -```javascript -// When creating virtual snapshot -for (const component of composition.component_tracks) { - const snapshot = await resolveSnapshot(component); - - if (snapshot.version === null) { - throw new ValidationError( - `Component track ${component.track_id} resolved to draft snapshot. ` + - `Virtual tracks can only reference tagged snapshots.`, - ); - } -} -``` - -**User experience:** +#### 1. Component snapshots must satisfy their resolution strategy -```bash -POST /api/release-tracks/release-track--uuid-virtual/virtual/snapshots/create - -# Error response: -{ - "error": "ValidationError", - "message": "Cannot create virtual snapshot: component track 'GroupsMonthly' has no tagged releases", - "details": { - "component": "release-track--uuid-1", - "issue": "No tagged snapshots found (all snapshots are drafts)" - } -} -``` +`latest_tagged`, `specific_version`, and `specific_snapshot` require a tagged +source. `latest_preview` accepts the newest standard snapshot whether tagged or +untagged. Draft preview conflicts block materialization before filters. Component +identity and type are validated when composition is configured; snapshot +eligibility and prospective membership are checked at materialization. #### 2. Component tracks must be standard tracks @@ -1294,15 +1345,28 @@ Virtual tracks cannot transition workflow status of composed objects. **Alternative:** If you need to change object status, do it in the source standard track. -### 3. Only Reference Tagged Snapshots +### 3. Preview Excludes Candidates -Virtual tracks cannot compose from draft snapshots. +`latest_preview` composes members plus staged changes under the source's release +conflict policy, without actually releasing or promoting source content. +Candidates remain excluded. Stage an intended change before previewing it in a +virtual composition; source publication is not required. -**Rationale:** Ensures stability and prevents virtual snapshots from inadvertently including WIP content. +## Error Handling -**Alternative:** Tag the standard track snapshot first, then create virtual snapshot. +### Error: Component Preview Has a Blocking Conflict -## Error Handling +A draft whose staged-to-members promotion would fail also fails virtual +materialization with `409 Conflict`. The response identifies the source track +and conflicts. Resolve the source conflict or deliberately change its promotion +policy; component filters do not bypass this requirement. + +### Error: Retired `latest_draft` Rule + +New requests reject the retired rule with `400 Bad Request`. Existing saved +rules also block materialization until an operator explicitly replaces them in +Config. Choose `latest_preview` for staged changes or `latest_tagged` for +published content. Historical snapshots are not relabeled or recomputed. ### Error: Component Has No Tagged Snapshots diff --git a/template.env b/template.env index 8dfcc8c7..e49e3452 100644 --- a/template.env +++ b/template.env @@ -54,7 +54,7 @@ DATABASE_URL= # Default: empty (disabled) #JSON_CONFIG_PATH= -# ALLOWED_VALUES_PATH (string) - Path to allowed values configuration file +# ALLOWED_VALUES_PATH (string) - Seed file read only when Allowed Values configuration is first initialized # Default: ./app/config/allowed-values.json #ALLOWED_VALUES_PATH=./app/config/allowed-values.json