From 7fa26eda33058d1196ba81ff974d5624129489fe Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:25:35 -0400 Subject: [PATCH 1/7] fix(release-tracks): enforce unique release versions Enforce tagged-version uniqueness with a database partial index, return a typed conflict for concurrent release races, and add a fail-closed migration for existing and orphan track collections. --- app/exceptions/index.js | 11 ++ app/lib/error-handler.js | 2 + .../release-track-snapshot-schema.js | 12 +- .../release-track-dynamic.repository.js | 9 +- .../release-tracks-release.spec.js | 50 +++++ ...lease-version-uniqueness-migration.spec.js | 74 ++++++++ app/tests/middleware/error-handler.spec.js | 29 ++- docs/developer/TODO.md | 42 +++++ .../release-tracks/implementation-notes.md | 15 ++ docs/user/release-tracks/versioning.md | 10 + ...nforce-release-track-version-uniqueness.js | 173 ++++++++++++++++++ 11 files changed, 421 insertions(+), 6 deletions(-) create mode 100644 app/tests/api/release-tracks/release-version-uniqueness-migration.spec.js create mode 100644 migrations/20260730040000-enforce-release-track-version-uniqueness.js diff --git a/app/exceptions/index.js b/app/exceptions/index.js index ad9a2977..a4db28ef 100644 --- a/app/exceptions/index.js +++ b/app/exceptions/index.js @@ -297,6 +297,16 @@ class AlreadyReleasedError extends CustomError { } } +class DuplicateReleaseVersionError extends CustomError { + constructor(trackId, version, options = {}) { + super(`Release track ${trackId} already has tagged version ${version}`, { + ...options, + track_id: trackId, + version, + }); + } +} + class TaggedSnapshotDeletionError extends CustomError { constructor(version, options) { super(`Tagged snapshot version ${version} cannot be deleted`, options); @@ -386,6 +396,7 @@ module.exports = { //** Version control errors */ AlreadyReleasedError, + DuplicateReleaseVersionError, TaggedSnapshotDeletionError, InvalidVersionError, diff --git a/app/lib/error-handler.js b/app/lib/error-handler.js index b0701431..6e4e292d 100644 --- a/app/lib/error-handler.js +++ b/app/lib/error-handler.js @@ -36,6 +36,7 @@ const { AlreadyRevokedError, SelfRevocationError, AlreadyReleasedError, + DuplicateReleaseVersionError, TaggedSnapshotDeletionError, InvalidVersionError, ReleaseConflictError, @@ -132,6 +133,7 @@ exports.serviceExceptions = function (err, req, res, next) { err instanceof DuplicateNameError || err instanceof AlreadyRevokedError || err instanceof AlreadyReleasedError || + err instanceof DuplicateReleaseVersionError || err instanceof TaggedSnapshotDeletionError || err instanceof ReleaseConflictError || err instanceof VirtualSnapshotNotMaterializedError || diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index fee1a485..814a7190 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -413,8 +413,16 @@ const releaseTrackSnapshotSchema = new mongoose.Schema(releaseTrackSnapshotDefin // Primary lookup: find snapshot by track id + modified timestamp releaseTrackSnapshotSchema.index({ id: 1, modified: -1 }, { unique: true }); -// Find the latest tagged version -releaseTrackSnapshotSchema.index({ id: 1, version: 1 }); +// A tagged version identifies exactly one snapshot within a release track. +// Drafts are excluded so any number of snapshots may retain version: null. +releaseTrackSnapshotSchema.index( + { id: 1, version: 1 }, + { + name: 'unique_tagged_version', + unique: true, + partialFilterExpression: { version: { $type: 'string' } }, + }, +); // A scheduled occurrence may materialize at most one snapshot, including // after restart recovery or duplicate delivery by multiple scheduler nodes. diff --git a/app/repository/release-tracks/release-track-dynamic.repository.js b/app/repository/release-tracks/release-track-dynamic.repository.js index b761ee15..f1675d5b 100644 --- a/app/repository/release-tracks/release-track-dynamic.repository.js +++ b/app/repository/release-tracks/release-track-dynamic.repository.js @@ -4,6 +4,7 @@ const modelFactory = require('../../models/release-tracks/model-factory'); const { DatabaseError, DuplicateIdError, + DuplicateReleaseVersionError, BadlyFormattedParameterError, } = require('../../exceptions'); const logger = require('../../lib/logger'); @@ -266,8 +267,12 @@ class ReleaseTrackDynamicRepository { return saved.toObject(); } catch (err) { if (err.name === 'MongoServerError' && err.code === 11000) { + if (err.keyPattern?.version && typeof snapshotData.version === 'string') { + throw new DuplicateReleaseVersionError(trackId, snapshotData.version, { cause: err }); + } throw new DuplicateIdError({ details: `Snapshot with modified '${snapshotData.modified}' already exists for track '${trackId}'.`, + cause: err, }); } throw new DatabaseError(err); @@ -305,9 +310,7 @@ class ReleaseTrackDynamicRepository { return result; } catch (err) { if (err.name === 'MongoServerError' && err.code === 11000) { - throw new DuplicateIdError({ - details: `Version conflict while tagging snapshot for track '${trackId}'.`, - }); + throw new DuplicateReleaseVersionError(trackId, versionData.version, { cause: err }); } throw new DatabaseError(err); } diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js index 67b5df71..6362eff5 100644 --- a/app/tests/api/release-tracks/release-tracks-release.spec.js +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -2,11 +2,13 @@ const request = require('supertest'); const { expect } = require('expect'); +const sinon = require('sinon'); const config = require('../../../config/config'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); +const releaseHistoryService = require('../../../services/release-tracks/release-history-service'); const versioningService = require('../../../services/release-tracks/versioning-service'); const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); @@ -171,6 +173,54 @@ describe('Release-track release planning and commit API', function () { expect(released.body.version_history.at(-1)).not.toHaveProperty('component_versions'); }); + it('allows only one concurrent release to claim a version', async function () { + const track = await createTrack('Concurrent Release Version'); + const newerDraft = await post(`/api/release-tracks/${track.id}/meta`, { + description: 'A distinct draft racing for the same release version', + }); + const originalHistoryLookup = releaseHistoryService.getTrackWideVersionHistory; + let waiting = 0; + let releaseBarrier; + const bothPlanned = new Promise((resolve) => { + releaseBarrier = resolve; + }); + const historyStub = sinon + .stub(releaseHistoryService, 'getTrackWideVersionHistory') + .callsFake(async (...args) => { + const history = await originalHistoryLookup(...args); + waiting += 1; + if (waiting === 2) releaseBarrier(); + await bothPlanned; + return history; + }); + + const release = (modified) => + request(app) + .post(`/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(modified)}/release`) + .send({ version: '2.0' }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + + let responses; + try { + responses = await Promise.all([release(track.modified), release(newerDraft.body.modified)]); + } finally { + historyStub.restore(); + } + expect(responses.map((response) => response.status).sort()).toEqual([200, 409]); + + const conflict = responses.find((response) => response.status === 409); + expect(conflict.body).toEqual({ + message: `Release track ${track.id} already has tagged version 2.0`, + track_id: track.id, + version: '2.0', + }); + + const tagged = await dynamicRepo.getAllSnapshots(track.id, { taggedOnly: true }); + expect(tagged.pagination.total).toBe(1); + expect(tagged.data[0].version).toBe('2.0'); + }); + it('freezes a dynamic staged reference to the latest revision during release', async function () { const revisionA = (await post('/api/techniques', buildTechnique('Dynamic Release A'), 201)) .body; diff --git a/app/tests/api/release-tracks/release-version-uniqueness-migration.spec.js b/app/tests/api/release-tracks/release-version-uniqueness-migration.spec.js new file mode 100644 index 00000000..895080d1 --- /dev/null +++ b/app/tests/api/release-tracks/release-version-uniqueness-migration.spec.js @@ -0,0 +1,74 @@ +'use strict'; + +const { expect } = require('expect'); +const mongoose = require('mongoose'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const releaseTracksService = require('../../../services/release-tracks/release-tracks-service'); +const migration = require('../../../../migrations/20260730040000-enforce-release-track-version-uniqueness'); + +const UNIQUE_INDEX = 'unique_tagged_version'; +const LEGACY_INDEX = 'id_1_version_1'; + +describe('Release-track tagged-version uniqueness migration', function () { + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + }); + + after(async function () { + await database.closeConnection(); + }); + + it('fails closed on legacy duplicates before replacing indexes and is rerunnable after repair', async function () { + const track = await releaseTracksService.createTrack({ + name: 'Legacy Duplicate Release Versions', + type: 'standard', + }); + const released = await releaseTracksService.releaseLatest(track.id, { + version: '1.0', + userAccountId: 'migration-test', + }); + const collection = mongoose.connection.db.collection(track.id); + + await collection.dropIndex(UNIQUE_INDEX); + await collection.createIndex({ id: 1, version: 1 }, { name: LEGACY_INDEX }); + + const duplicate = { ...released }; + delete duplicate._id; + duplicate.modified = new Date(new Date(released.modified).getTime() + 1000); + await collection.insertOne(duplicate); + await mongoose.connection.db + .collection('releaseTrackRegistry') + .deleteOne({ track_id: track.id }); + + await expect(migration.up(mongoose.connection.db)).rejects.toMatchObject({ + message: expect.stringContaining(`${track.id} version 1.0 (2 snapshots)`), + duplicates: [ + expect.objectContaining({ + track_id: track.id, + version: '1.0', + }), + ], + }); + + let indexes = await collection.indexes(); + expect(indexes.some((index) => index.name === LEGACY_INDEX)).toBe(true); + expect(indexes.some((index) => index.name === UNIQUE_INDEX)).toBe(false); + + await collection.deleteOne({ modified: duplicate.modified }); + await migration.up(mongoose.connection.db); + await migration.up(mongoose.connection.db); + + indexes = await collection.indexes(); + expect(indexes.some((index) => index.name === LEGACY_INDEX)).toBe(false); + expect(indexes.find((index) => index.name === UNIQUE_INDEX)).toMatchObject({ + key: { id: 1, version: 1 }, + unique: true, + partialFilterExpression: { version: { $type: 'string' } }, + }); + }); +}); diff --git a/app/tests/middleware/error-handler.spec.js b/app/tests/middleware/error-handler.spec.js index f84f85ae..0ab1829f 100644 --- a/app/tests/middleware/error-handler.spec.js +++ b/app/tests/middleware/error-handler.spec.js @@ -5,7 +5,12 @@ const sinon = require('sinon'); const logger = require('../../lib/logger'); const errorHandler = require('../../lib/error-handler'); -const { DatabaseError, DuplicateIdError, InvalidPostOperationError } = require('../../exceptions'); +const { + DatabaseError, + DuplicateIdError, + DuplicateReleaseVersionError, + InvalidPostOperationError, +} = require('../../exceptions'); describe('error-handler middleware', function () { beforeEach(function () { @@ -72,6 +77,28 @@ describe('error-handler middleware', function () { expect(next.called).toBe(false); }); + it('should return a structured conflict for DuplicateReleaseVersionError', function () { + const trackId = 'release-track--00000000-0000-4000-8000-000000000001'; + const err = new DuplicateReleaseVersionError(trackId, '2.0'); + const res = { + status: sinon.stub().returnsThis(), + send: sinon.stub().returnsThis(), + }; + const next = sinon.stub(); + + errorHandler.serviceExceptions(err, {}, res, next); + + expect(res.status.calledOnceWithExactly(409)).toBe(true); + expect( + res.send.calledOnceWithExactly({ + message: `Release track ${trackId} already has tagged version 2.0`, + track_id: trackId, + version: '2.0', + }), + ).toBe(true); + expect(next.called).toBe(false); + }); + it('should preserve wrapped error details for DatabaseError', function () { const err = new DatabaseError(new Error('Mongo connection failed')); const res = { diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index fdc2c78c..5ed481dd 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,47 @@ # Release Track TODOs +## Production-readiness branch — `fix/release-tracks-production-readiness` + +This branch implements the prioritized findings in +`.nocommit/project-review-release-tracks/15-recommendations.md`. Each numbered +recommendation is kept as a separate conventional commit so the merge request +can be reviewed or reverted item by item. + +### P0.1 — Enforce release version uniqueness + +- [x] Add a unique partial index for tagged `version` strings in every dynamic + release-track snapshot collection. +- [x] Convert duplicate-version races into a typed `409 Conflict` that + identifies the track and requested version. +- [x] Add a regression that releases two distinct drafts concurrently with the + same version and proves exactly one succeeds. +- [x] Add a rerunnable migration that fails closed on pre-existing duplicates + before replacing the legacy non-unique index. +- [x] Update release-version documentation and run focused, migration, + middleware, lint, and complete-suite verification. + +Verification result (2026-07-30): + +- The deterministic concurrent-release, migration, middleware, and isolated + roaming-failure group passes (39). +- The required clean full suite passes: OpenAPI 2, config 21, API 947, + middleware 25, and scheduler 10. +- The migration preflights the union of registry IDs and canonical orphan + release-track collection names before making any index changes. + +### Remaining prioritized recommendations + +- [ ] P0.2 — Make primary release membership fail closed. +- [ ] P0.3 — Make tagged-content immutability authoritative and durable. +- [ ] P0.4 — Correct destructive authorization and add durable audit records. +- [ ] P0.5 — Complete the Angular contract migration and end-to-end smoke gate. +- [ ] P0.6 — Finish scheduled-materialization fencing, retry bounds, and + operator intervention. +- [ ] P0.7 — Establish and enforce a safe storage operating envelope. +- [ ] P0.8 — Harden deployment, database readiness, backup/restore, rollback, + and post-deploy verification. +- [ ] Address P1 recommendations in documented criticality order. + ## Current implementation slice — Scheduler regression and virtual schedule coverage - [x] Repair the legacy collection-index scheduler spec so it imports the diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 95d0da3c..43bfceea 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -13,6 +13,21 @@ db.objects.createIndex({ 'workspace.collections.staged': 1 }); db.objects.createIndex({ 'workspace.workflow.status': 1 }); ``` +Each release track also owns a dynamic snapshot collection. Tagged versions +use a unique partial index on `{ id: 1, version: 1 }`, restricted to documents +whose `version` is a string. Drafts therefore remain unlimited at +`version: null`, while the database—not an application-level preflight—decides +which concurrent release may claim a version. + +Migration `20260730040000-enforce-release-track-version-uniqueness` scans the +union of registered tracks and canonical `release-track--` collection +names before changing any indexes. Including orphan collections matters +because track creation predates transaction-backed registry coordination. If any +`(track_id, version)` has multiple tagged snapshots, it reports all offending +snapshot timestamps and performs no index changes. After operators repair the +data, rerunning the migration replaces the legacy non-unique index +idempotently. + ## Validation Rules - **Same revision selector** can only be in one tier per release-track snapshot diff --git a/docs/user/release-tracks/versioning.md b/docs/user/release-tracks/versioning.md index 925e314f..3290e359 100644 --- a/docs/user/release-tracks/versioning.md +++ b/docs/user/release-tracks/versioning.md @@ -191,6 +191,16 @@ Collections use a **two-part versioning scheme** (MAJOR.MINOR), inspired by sema 2. **Immutable once set** - Once a snapshot has `version` assigned, it cannot be changed 3. **Cannot re-tag** - A snapshot can only be tagged once (throws `AlreadyReleasedError` if attempted) 4. **Valid version format** - Must match `/^\d+\.\d+$/` (MAJOR.MINOR only, no patch component) +5. **Unique within the track** - Exactly one snapshot may hold a given tagged + version. If concurrent release requests race for the same version, one + succeeds and the other receives `409 Conflict` with the conflicting + `track_id` and `version`. + +Deployments upgrading from an earlier release run a database migration before +serving traffic. The migration checks every release-track collection for +pre-existing duplicate tagged versions and stops without changing indexes if +it finds any. Operators must resolve every reported track/version pair and +rerun the migration; the server does not guess which tagged snapshot to keep. ### First Tagged Release diff --git a/migrations/20260730040000-enforce-release-track-version-uniqueness.js b/migrations/20260730040000-enforce-release-track-version-uniqueness.js new file mode 100644 index 00000000..5aed38e6 --- /dev/null +++ b/migrations/20260730040000-enforce-release-track-version-uniqueness.js @@ -0,0 +1,173 @@ +'use strict'; + +/** + * Replace the legacy non-unique (id, version) index in every dynamic release + * track collection with a unique partial index over tagged snapshots. + * + * The migration preflights every collection before changing any indexes. If a + * deployment already contains duplicate tagged versions, migration stops and + * reports every offending track/version so an operator can repair the data + * deliberately. + */ + +const INDEX_NAME = 'unique_tagged_version'; +const LEGACY_INDEX_NAME = 'id_1_version_1'; +const CONCURRENCY = 8; +const TRACK_COLLECTION_PATTERN = + /^release-track--[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +async function mapWithConcurrency(items, mapper) { + let nextIndex = 0; + + async function worker() { + while (nextIndex < items.length) { + const index = nextIndex++; + await mapper(items[index]); + } + } + + await Promise.all(Array.from({ length: Math.min(CONCURRENCY, items.length) }, () => worker())); +} + +async function collectionExists(db, trackId) { + return db.listCollections({ name: trackId }, { nameOnly: true }).hasNext(); +} + +async function findTrackCollectionIds(db) { + const [registeredTracks, collections] = await Promise.all([ + db.collection('releaseTrackRegistry').find({}).project({ track_id: 1, _id: 0 }).toArray(), + db.listCollections({}, { nameOnly: true }).toArray(), + ]); + + return Array.from( + new Set([ + ...registeredTracks.map((track) => track.track_id), + ...collections + .map((collection) => collection.name) + .filter((name) => TRACK_COLLECTION_PATTERN.test(name)), + ]), + ).sort(); +} + +async function findDuplicateVersions(db, trackIds) { + const duplicates = []; + + await mapWithConcurrency(trackIds, async (trackId) => { + if (!(await collectionExists(db, trackId))) return; + + const matches = await db + .collection(trackId) + .aggregate([ + { $match: { version: { $type: 'string' } } }, + { + $group: { + _id: { id: '$id', version: '$version' }, + count: { $sum: 1 }, + snapshots: { $push: '$modified' }, + }, + }, + { $match: { count: { $gt: 1 } } }, + { $sort: { '_id.version': 1 } }, + ]) + .toArray(); + + for (const match of matches) { + duplicates.push({ + track_id: trackId, + version: match._id.version, + snapshots: match.snapshots, + }); + } + }); + + return duplicates.sort( + (left, right) => + left.track_id.localeCompare(right.track_id) || left.version.localeCompare(right.version), + ); +} + +function isDesiredIndex(index) { + return ( + index?.name === INDEX_NAME && + index.unique === true && + index.key?.id === 1 && + index.key?.version === 1 && + index.partialFilterExpression?.version?.$type === 'string' + ); +} + +async function installUniqueIndex(db, trackId) { + if (!(await collectionExists(db, trackId))) return; + + const collection = db.collection(trackId); + const indexes = await collection.indexes(); + const desired = indexes.find((index) => index.name === INDEX_NAME); + if (isDesiredIndex(desired)) { + if (indexes.some((index) => index.name === LEGACY_INDEX_NAME)) { + await collection.dropIndex(LEGACY_INDEX_NAME); + } + return; + } + + if (desired) await collection.dropIndex(INDEX_NAME); + if (indexes.some((index) => index.name === LEGACY_INDEX_NAME)) { + await collection.dropIndex(LEGACY_INDEX_NAME); + } + + await collection.createIndex( + { id: 1, version: 1 }, + { + name: INDEX_NAME, + unique: true, + partialFilterExpression: { version: { $type: 'string' } }, + }, + ); +} + +module.exports = { + async up(db) { + const trackIds = await findTrackCollectionIds(db); + const duplicates = await findDuplicateVersions(db, trackIds); + + if (duplicates.length > 0) { + const summary = duplicates + .map( + (duplicate) => + `${duplicate.track_id} version ${duplicate.version} ` + + `(${duplicate.snapshots.length} snapshots)`, + ) + .join('; '); + const error = new Error( + `Duplicate tagged release versions detected; repair them before retrying migration: ${summary}`, + ); + error.duplicates = duplicates; + throw error; + } + + await mapWithConcurrency(trackIds, (trackId) => installUniqueIndex(db, trackId)); + }, + + async down(db) { + const trackIds = await findTrackCollectionIds(db); + + await mapWithConcurrency(trackIds, async (trackId) => { + if (!(await collectionExists(db, trackId))) return; + + const collection = db.collection(trackId); + const indexes = await collection.indexes(); + if (indexes.some((index) => index.name === INDEX_NAME)) { + await collection.dropIndex(INDEX_NAME); + } + if (!indexes.some((index) => index.name === LEGACY_INDEX_NAME)) { + await collection.createIndex({ id: 1, version: 1 }, { name: LEGACY_INDEX_NAME }); + } + }); + }, + + _private: { + findTrackCollectionIds, + findDuplicateVersions, + installUniqueIndex, + isDesiredIndex, + }, +}; From aacfeffc60658612de979a732eb16f34c307789a Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:51:29 -0400 Subject: [PATCH 2/7] fix(release-tracks): reject unresolved primary revisions Validate exact primary content at request, release, clone, materialization, import, and export boundaries. Return structured missing references instead of persisting or rendering partial release data. --- .../definitions/components/release-tracks.yml | 29 ++ .../paths/release-tracks-paths.yml | 56 ++- app/exceptions/index.js | 20 ++ app/lib/error-handler.js | 4 + .../release-tracks/bundle-import-service.js | 42 ++- app/services/release-tracks/export-service.js | 79 +---- .../primary-revision-service.js | 151 +++++++++ .../release-tracks/release-tracks-service.js | 7 + .../release-tracks/snapshot-service.js | 28 +- .../release-tracks/standard-track-service.js | 27 +- .../release-tracks/versioning-service.js | 6 + .../release-tracks/virtual-track-service.js | 3 + .../primary-revision-integrity.spec.js | 320 ++++++++++++++++++ .../release-tracks-release.spec.js | 46 ++- .../release-tracks/snapshot-history.spec.js | 61 +++- app/tests/middleware/error-handler.spec.js | 54 +++ docs/developer/FRONTEND_TODO.md | 45 +++ docs/developer/TODO.md | 32 +- .../developer/release-tracks/bundle-export.md | 9 +- .../release-tracks/implementation-notes.md | 27 ++ docs/user/release-tracks/api-reference.md | 6 + docs/user/release-tracks/output-formats.md | 7 + docs/user/release-tracks/release-workflow.md | 13 +- 23 files changed, 915 insertions(+), 157 deletions(-) create mode 100644 app/services/release-tracks/primary-revision-service.js create mode 100644 app/tests/api/release-tracks/primary-revision-integrity.spec.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 243ef13d..5ba8efe7 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -676,3 +676,32 @@ components: type: string format: date-time description: 'UTC occurrence timestamp; also serves as the idempotency key' + + object-revision-reference: + type: object + required: + - object_ref + - object_modified + properties: + object_ref: + type: string + description: 'STIX object ID' + object_modified: + type: string + format: date-time + description: 'Exact STIX revision timestamp' + + object-revision-error: + type: object + required: + - message + - missing_references + properties: + message: + type: string + description: 'Whether request input or persisted primary content failed validation' + missing_references: + type: array + minItems: 1 + items: + $ref: '#/components/schemas/object-revision-reference' diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 9b1c4c64..daf6ff1f 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -256,8 +256,8 @@ paths: responses: '201': description: 'Release track created from bundle' - '501': - description: 'Not yet implemented' + '400': + description: 'A bundle primary object is unsupported, invalid, or cannot be persisted' /api/release-tracks/import: post: @@ -354,7 +354,7 @@ paths: '200': description: 'Contents updated successfully' '400': - description: 'Track is virtual or the contents request is invalid' + description: 'Track is virtual, the contents request is invalid, or a requested revision does not exist' /api/release-tracks/{id}/clone: post: @@ -374,6 +374,12 @@ paths: responses: '201': description: 'Release track cloned successfully' + '409': + description: 'The source snapshot references missing primary revisions' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/object-revision-error' /api/release-tracks/{id}/snapshots/latest/release: post: @@ -412,7 +418,7 @@ paths: '400': description: 'Invalid release request' '409': - description: 'Already released or conflicting snapshot' + description: 'Already released, conflicting snapshot, or missing persisted primary revisions' /api/release-tracks/{id}/snapshots/latest/release/preview: get: @@ -488,7 +494,7 @@ paths: '200': description: 'Release preview generated' '409': - description: 'Virtual draft is unmaterialized or release is blocked by a conflict' + description: 'Virtual draft is unmaterialized, release is blocked by a conflict, or persisted primary revisions are missing' '501': description: 'Requested format is not yet implemented' @@ -558,6 +564,8 @@ paths: responses: '200': description: 'Candidates added successfully' + '400': + description: 'A requested exact or latest object revision does not exist' /api/release-tracks/{id}/candidates/review: post: @@ -654,6 +662,8 @@ paths: responses: '200': description: 'Candidate version pin updated successfully' + '400': + description: 'The requested replacement revision does not exist' # ============================================================================= # Staged objects @@ -873,6 +883,12 @@ paths: description: 'Virtual snapshot created successfully' '400': description: 'Track is not virtual or cannot resolve its composition' + '409': + description: 'A resolved component snapshot references missing primary revisions' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/object-revision-error' /api/release-tracks/{id}/virtual/quarantine/promote: post: @@ -906,6 +922,12 @@ paths: description: 'Track is not virtual or the request body is invalid' '404': description: 'The selected exact revision is not quarantined' + '409': + description: 'The resulting virtual snapshot would reference missing primary revisions' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/object-revision-error' # ============================================================================= # Snapshot-specific operations @@ -1061,6 +1083,12 @@ paths: $ref: '../components/release-tracks.yml#/components/schemas/release-track-snapshot' '404': description: 'Release track not found' + '409': + description: 'The selected snapshot representation references missing primary revisions' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/object-revision-error' '501': description: 'Requested format is not yet implemented' @@ -1148,6 +1176,12 @@ paths: description: 'Snapshot retrieved successfully' '404': description: 'Snapshot not found' + '409': + description: 'The selected snapshot representation references missing primary revisions' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/object-revision-error' '501': description: 'Requested format is not yet implemented' @@ -1234,7 +1268,7 @@ paths: '200': description: 'Contents updated successfully' '400': - description: 'Track is virtual or the contents request is invalid' + description: 'Track is virtual, the contents request is invalid, or a requested revision does not exist' /api/release-tracks/{id}/snapshots/{modified}/clone: post: @@ -1259,6 +1293,12 @@ paths: responses: '201': description: 'Release track cloned successfully' + '409': + description: 'The source snapshot references missing primary revisions' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/object-revision-error' /api/release-tracks/{id}/snapshots/{modified}/release: post: @@ -1302,7 +1342,7 @@ paths: '400': description: 'Invalid release request' '409': - description: 'Already released or conflicting snapshot' + description: 'Already released, conflicting snapshot, or missing persisted primary revisions' /api/release-tracks/{id}/snapshots/{modified}/release/preview: get: @@ -1374,6 +1414,6 @@ paths: '200': description: 'Release preview generated' '409': - description: 'Virtual draft is unmaterialized or release is blocked by a conflict' + description: 'Release is blocked because the draft is unmaterialized, conflicting, or references missing primary revisions' '501': description: 'Requested format is not yet implemented' diff --git a/app/exceptions/index.js b/app/exceptions/index.js index a4db28ef..c387fd83 100644 --- a/app/exceptions/index.js +++ b/app/exceptions/index.js @@ -307,6 +307,24 @@ class DuplicateReleaseVersionError extends CustomError { } } +class InvalidObjectRevisionError extends CustomError { + constructor(missingReferences, options = {}) { + super('One or more object revisions do not exist', { + ...options, + missing_references: missingReferences, + }); + } +} + +class ReleaseContentIntegrityError extends CustomError { + constructor(missingReferences, options = {}) { + super('Release-track primary content is incomplete', { + ...options, + missing_references: missingReferences, + }); + } +} + class TaggedSnapshotDeletionError extends CustomError { constructor(version, options) { super(`Tagged snapshot version ${version} cannot be deleted`, options); @@ -397,11 +415,13 @@ module.exports = { //** Version control errors */ AlreadyReleasedError, DuplicateReleaseVersionError, + InvalidObjectRevisionError, TaggedSnapshotDeletionError, InvalidVersionError, //** Release track errors */ ReleaseConflictError, + ReleaseContentIntegrityError, NoTaggedSnapshotsError, InvalidComponentTypeError, VirtualSnapshotNotMaterializedError, diff --git a/app/lib/error-handler.js b/app/lib/error-handler.js index 6e4e292d..0911c02e 100644 --- a/app/lib/error-handler.js +++ b/app/lib/error-handler.js @@ -37,9 +37,11 @@ const { SelfRevocationError, AlreadyReleasedError, DuplicateReleaseVersionError, + InvalidObjectRevisionError, TaggedSnapshotDeletionError, InvalidVersionError, ReleaseConflictError, + ReleaseContentIntegrityError, NoTaggedSnapshotsError, InvalidComponentTypeError, VirtualSnapshotNotMaterializedError, @@ -106,6 +108,7 @@ exports.serviceExceptions = function (err, req, res, next) { err instanceof ValidationError || err instanceof MitreIdentityWriteError || err instanceof InvalidVersionError || + err instanceof InvalidObjectRevisionError || err instanceof NoTaggedSnapshotsError || err instanceof InvalidComponentTypeError ) { @@ -136,6 +139,7 @@ exports.serviceExceptions = function (err, req, res, next) { err instanceof DuplicateReleaseVersionError || err instanceof TaggedSnapshotDeletionError || err instanceof ReleaseConflictError || + err instanceof ReleaseContentIntegrityError || err instanceof VirtualSnapshotNotMaterializedError || err instanceof MemberPinnedRevisionError || err instanceof ObjectHasValidationIssuesError || diff --git a/app/services/release-tracks/bundle-import-service.js b/app/services/release-tracks/bundle-import-service.js index 412fcf51..6740c064 100644 --- a/app/services/release-tracks/bundle-import-service.js +++ b/app/services/release-tracks/bundle-import-service.js @@ -20,6 +20,7 @@ const types = require('../../lib/types'); const logger = require('../../lib/logger'); const snapshotService = require('./snapshot-service'); +const primaryRevisionService = require('./primary-revision-service'); const { BadRequestError, DuplicateIdError } = require('../../exceptions'); // --------------------------------------------------------------------------- @@ -134,18 +135,24 @@ function sortByDependencyOrder(objects) { async function importObject(stixObj, serviceMap) { const service = serviceMap[stixObj.type]; if (!service) { - logger.warn( - `BundleImportService: No service for type "${stixObj.type}", skipping "${stixObj.id}"`, - ); - return { imported: false, ref: null }; + throw new BadRequestError({ + message: 'Bundle contains an unsupported primary object type', + details: { + object_ref: stixObj.id, + type: stixObj.type, + }, + }); } // Validate required fields if (!stixObj.id || !stixObj.modified) { - logger.warn( - `BundleImportService: Object missing id or modified, skipping: ${JSON.stringify({ id: stixObj.id, type: stixObj.type })}`, - ); - return { imported: false, ref: null }; + throw new BadRequestError({ + message: 'Bundle primary object is missing an exact revision identifier', + details: { + object_ref: stixObj.id, + type: stixObj.type, + }, + }); } const ref = { @@ -189,9 +196,18 @@ async function importObject(stixObj, serviceMap) { return { imported: false, ref }; } - // Non-duplicate errors are logged but don't abort the entire import + // A track must never retain a primary reference whose object failed to + // import. Previously this path logged the failure and returned the ref. logger.error(`BundleImportService: Failed to import "${stixObj.id}":`, err); - return { imported: false, ref }; + throw new BadRequestError({ + message: 'Failed to import a bundle primary object', + details: { + object_ref: stixObj.id, + object_modified: stixObj.modified, + type: stixObj.type, + }, + cause: err, + }); } } @@ -280,6 +296,12 @@ exports.createTrackFromBundle = async function createTrackFromBundle(bundleData) ); } + // Validate the authoritative member list before creating the dynamic track + // collection or registry entry. Successfully imported standalone objects + // remain available if a later primary is invalid, but no partial track is + // persisted. + memberEntries = (await primaryRevisionService.assertRequestEntries(memberEntries)).entries; + // ------------------------------------------------------------------ // Step 4: Create the release track // ------------------------------------------------------------------ diff --git a/app/services/release-tracks/export-service.js b/app/services/release-tracks/export-service.js index b02e8447..fb469a4b 100644 --- a/app/services/release-tracks/export-service.js +++ b/app/services/release-tracks/export-service.js @@ -25,48 +25,15 @@ const EventBus = require('../../lib/event-bus'); const Events = require('../../lib/event-constants'); const { selectRelationshipsForBundle } = require('../../lib/stix-bundle-relationships'); const revisionReference = require('../../lib/release-tracks/revision-reference'); +const primaryRevisionService = require('./primary-revision-service'); const { bundleTransformSchema, workbenchTransformSchema, filesystemStoreTransformSchema, } = require('../../lib/release-tracks/export-schemas'); -// --------------------------------------------------------------------------- -// Repository map — lazy-loaded to avoid circular dependency issues at startup. -// -// Maps STIX type prefixes to their corresponding repositories so we can -// batch-query each repository's `findManyByIdAndModified` in parallel. -// --------------------------------------------------------------------------- - -let _repoMap = null; - function getRepositoryMap() { - if (_repoMap) return _repoMap; - - _repoMap = { - [types.Technique]: require('../../repository/techniques-repository'), - [types.Tactic]: require('../../repository/tactics-repository'), - [types.Group]: require('../../repository/groups-repository'), - [types.Campaign]: require('../../repository/campaigns-repository'), - [types.Mitigation]: require('../../repository/mitigations-repository'), - [types.Matrix]: require('../../repository/matrix-repository'), - [types.Relationship]: require('../../repository/relationships-repository'), - [types.MarkingDefinition]: require('../../repository/marking-definitions-repository'), - [types.Identity]: require('../../repository/identities-repository'), - [types.Note]: require('../../repository/notes-repository'), - [types.DataSource]: require('../../repository/data-sources-repository'), - [types.DataComponent]: require('../../repository/data-components-repository'), - [types.Asset]: require('../../repository/assets-repository'), - [types.Analytic]: require('../../repository/analytics-repository'), - [types.DetectionStrategy]: require('../../repository/detection-strategies-repository'), - }; - - // Software types share a single repository - const softwareRepo = require('../../repository/software-repository'); - _repoMap[types.Malware] = softwareRepo; - _repoMap[types.Tool] = softwareRepo; - - return _repoMap; + return primaryRevisionService.getRepositoryMap(); } // ============================================================================= @@ -83,47 +50,7 @@ function getRepositoryMap() { * @returns {Promise>} Full Mongoose lean documents ({ stix, workspace, ... }) */ exports.hydrateMembers = async function hydrateMembers(entries) { - if (!entries || entries.length === 0) return []; - const resolvedEntries = await revisionReference.resolveEntries(entries); - const uniqueResolvedEntries = []; - const seenResolvedEntries = new Set(); - for (const entry of resolvedEntries) { - const key = `${entry.object_ref}::` + revisionReference.modifiedKey(entry.object_modified); - if (seenResolvedEntries.has(key)) continue; - seenResolvedEntries.add(key); - uniqueResolvedEntries.push(entry); - } - - // Group entries by STIX type prefix - const byType = {}; - for (const entry of uniqueResolvedEntries) { - const type = entry.object_ref.split('--')[0]; - if (!byType[type]) byType[type] = []; - byType[type].push(entry); - } - - const repoMap = getRepositoryMap(); - const hydrated = []; - - await Promise.all( - Object.entries(byType).map(async ([type, refs]) => { - const repo = repoMap[type]; - if (!repo) { - logger.warn( - `ExportService: No repository for type "${type}", skipping ${refs.length} object(s)`, - ); - return; - } - try { - const docs = await repo.findManyByIdAndModified(refs); - hydrated.push(...docs); - } catch (err) { - logger.error(`ExportService: Failed to hydrate ${refs.length} "${type}" object(s):`, err); - } - }), - ); - - return hydrated; + return (await primaryRevisionService.assertStoredEntries(entries)).documents; }; // ============================================================================= diff --git a/app/services/release-tracks/primary-revision-service.js b/app/services/release-tracks/primary-revision-service.js new file mode 100644 index 00000000..9b12b7ea --- /dev/null +++ b/app/services/release-tracks/primary-revision-service.js @@ -0,0 +1,151 @@ +'use strict'; + +// Authoritative hydration and existence validation for release-track primary +// content. Cross-service reads are intentionally centralized here so ingress, +// release planning, virtual materialization, import, and export share one +// exact-revision invariant. + +const types = require('../../lib/types'); +const revisionReference = require('../../lib/release-tracks/revision-reference'); +const { InvalidObjectRevisionError, ReleaseContentIntegrityError } = require('../../exceptions'); + +let repositoryMap; + +function getRepositoryMap() { + if (repositoryMap) return repositoryMap; + + repositoryMap = { + [types.Technique]: require('../../repository/techniques-repository'), + [types.Tactic]: require('../../repository/tactics-repository'), + [types.Group]: require('../../repository/groups-repository'), + [types.Campaign]: require('../../repository/campaigns-repository'), + [types.Mitigation]: require('../../repository/mitigations-repository'), + [types.Matrix]: require('../../repository/matrix-repository'), + [types.Relationship]: require('../../repository/relationships-repository'), + [types.MarkingDefinition]: require('../../repository/marking-definitions-repository'), + [types.Identity]: require('../../repository/identities-repository'), + [types.Note]: require('../../repository/notes-repository'), + [types.DataSource]: require('../../repository/data-sources-repository'), + [types.DataComponent]: require('../../repository/data-components-repository'), + [types.Asset]: require('../../repository/assets-repository'), + [types.Analytic]: require('../../repository/analytics-repository'), + [types.DetectionStrategy]: require('../../repository/detection-strategies-repository'), + }; + + const softwareRepo = require('../../repository/software-repository'); + repositoryMap[types.Malware] = softwareRepo; + repositoryMap[types.Tool] = softwareRepo; + + return repositoryMap; +} + +function revisionKey(entry) { + return `${entry.object_ref}::${revisionReference.modifiedKey(entry.object_modified)}`; +} + +function serializeReference(entry) { + const modified = new Date(entry.object_modified); + return { + object_ref: entry.object_ref, + object_modified: Number.isNaN(modified.getTime()) + ? String(entry.object_modified) + : modified.toISOString(), + }; +} + +function uniqueEntries(entries) { + const seen = new Set(); + return entries.filter((entry) => { + const key = revisionKey(entry); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +/** + * Resolve dynamic selectors and hydrate every unique exact revision. + * Repository failures deliberately propagate; only a successful query with a + * missing result is classified as unresolved primary content. + */ +async function hydrateEntries(entries) { + if (!entries || entries.length === 0) { + return { entries: [], documents: [], missing: [] }; + } + + const resolvedEntries = uniqueEntries(await revisionReference.resolveEntries(entries)); + const byType = new Map(); + for (const entry of resolvedEntries) { + const type = entry.object_ref.split('--')[0]; + if (!byType.has(type)) byType.set(type, []); + byType.get(type).push(entry); + } + + const documentsByRevision = new Map(); + const unsupported = []; + const repositories = getRepositoryMap(); + + await Promise.all( + Array.from(byType.entries()).map(async ([type, refs]) => { + const repository = repositories[type]; + if (!repository) { + unsupported.push(...refs); + return; + } + + const documents = await repository.findManyByIdAndModified(refs); + for (const document of documents) { + documentsByRevision.set( + revisionKey({ + object_ref: document.stix.id, + object_modified: document.stix.modified, + }), + document, + ); + } + }), + ); + + const missing = [ + ...unsupported, + ...resolvedEntries.filter((entry) => !documentsByRevision.has(revisionKey(entry))), + ] + .filter( + (entry, index, all) => + all.findIndex((item) => revisionKey(item) === revisionKey(entry)) === index, + ) + .map(serializeReference); + const documents = resolvedEntries + .map((entry) => documentsByRevision.get(revisionKey(entry))) + .filter(Boolean); + + return { entries: resolvedEntries, documents, missing }; +} + +async function assertRequestEntries(entries) { + const result = await hydrateEntries(entries); + if (result.missing.length > 0) { + throw new InvalidObjectRevisionError(result.missing); + } + return result; +} + +async function assertStoredEntries(entries) { + const result = await hydrateEntries(entries); + if (result.missing.length > 0) { + throw new ReleaseContentIntegrityError(result.missing); + } + return result; +} + +module.exports = { + getRepositoryMap, + hydrateEntries, + assertRequestEntries, + assertStoredEntries, + _private: { + revisionKey, + serializeReference, + uniqueEntries, + }, +}; diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index 6aa14bb6..aa2bc8de 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -24,6 +24,7 @@ const standardTrackService = require('./standard-track-service'); const versioningService = require('./versioning-service'); const virtualTrackService = require('./virtual-track-service'); const exportService = require('./export-service'); +const primaryRevisionService = require('./primary-revision-service'); const ephemeralService = require('./ephemeral-service'); const bundleImportService = require('./bundle-import-service'); const memberSyncService = require('./member-sync-service'); @@ -173,6 +174,12 @@ function filterSnapshotTiers(snapshot, include) { } async function formatWorkbenchSnapshot(snapshot, options) { + const include = options?.include; + const selectedTiers = + !include || include === 'all' ? TIER_NAMES : [...new Set(['members', include])]; + await primaryRevisionService.assertStoredEntries( + selectedTiers.flatMap((tierName) => snapshot[tierName] || []), + ); const enriched = await addObjectInfoToSnapshot(snapshot); return filterSnapshotTiers(enriched, options?.include); } diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index b148a061..57bbffa5 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -20,8 +20,8 @@ const logger = require('../../lib/logger'); const EventBus = require('../../lib/event-bus'); const EventConstants = require('../../lib/event-constants'); const versionUtils = require('../../lib/release-tracks/version-utils'); -const objectResolver = require('../../lib/release-tracks/object-resolver'); const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); +const primaryRevisionService = require('./primary-revision-service'); const { TrackNotFoundError, NotFoundError, @@ -75,23 +75,12 @@ function assertStandardTrack(snapshot) { * @returns {Promise>} */ async function resolveContentsMembers(contents) { - const latestByObjectRef = new Map(); - const resolveLatest = (objectRef) => { - if (!latestByObjectRef.has(objectRef)) { - latestByObjectRef.set(objectRef, objectResolver.resolveLatestModified(objectRef)); - } - return latestByObjectRef.get(objectRef); - }; - - return Promise.all( - contents.map(async (entry) => ({ - object_ref: entry.obj_ref, - object_modified: - entry.obj_modified === 'latest' - ? await resolveLatest(entry.obj_ref) - : new Date(entry.obj_modified), - })), - ); + const requested = contents.map((entry) => ({ + object_ref: entry.obj_ref, + object_modified: + entry.obj_modified === 'latest' ? entry.obj_modified : new Date(entry.obj_modified), + })); + return (await primaryRevisionService.assertRequestEntries(requested)).entries; } /** @@ -406,6 +395,9 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { delete clone.scheduled_materialization; const normalized = tierRevisionInvariant.normalizeSnapshot(clone); + await primaryRevisionService.assertStoredEntries( + tierRevisionInvariant.TIER_PRECEDENCE.flatMap((tier) => normalized.snapshot[tier] || []), + ); await modelFactory.ensureIndexes(newTrackId); const saved = await dynamicRepo.saveSnapshot(newTrackId, normalized.snapshot); diff --git a/app/services/release-tracks/standard-track-service.js b/app/services/release-tracks/standard-track-service.js index 47655146..60a94f0d 100644 --- a/app/services/release-tracks/standard-track-service.js +++ b/app/services/release-tracks/standard-track-service.js @@ -13,7 +13,7 @@ // ============================================================================= const snapshotService = require('./snapshot-service'); -const objectResolver = require('../../lib/release-tracks/object-resolver'); +const primaryRevisionService = require('./primary-revision-service'); const revisionReference = require('../../lib/release-tracks/revision-reference'); const conflictResolution = require('../../lib/release-tracks/conflict-resolution'); const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); @@ -103,16 +103,13 @@ exports.addCandidates = async function addCandidates(trackId, objectRefs, userId for (const raw of objectRefs) { const entry = normalizeObjectRef(raw); - // `latest` is a dynamic workflow-tier selector. Resolve it once here to - // validate that the object exists, but preserve the selector until the - // staged entry is frozen by a release operation. - let modified; - if (!entry.modified || entry.modified === 'latest') { - await objectResolver.resolveLatestModified(entry.id); - modified = revisionReference.LATEST; - } else { - modified = new Date(entry.modified); - } + // `latest` remains dynamic through the candidate/staged workflow. The + // shared primary-revision boundary resolves it only for existence + // validation and does not mutate the persisted selector. + const modified = + !entry.modified || entry.modified === 'latest' + ? revisionReference.LATEST + : new Date(entry.modified); const revision = { object_ref: entry.id, object_modified: modified }; const revisionKey = tierRevisionInvariant.revisionKey(revision); @@ -140,6 +137,8 @@ exports.addCandidates = async function addCandidates(trackId, objectRefs, userId : source; } + await primaryRevisionService.assertRequestEntries(newEntries); + // Same-object conflicts (the object_ref is already pinned in candidates at // a different revision) are resolved by the into_candidates policy. const conflictPolicy = source.config?.promotion_conflicts?.into_candidates || 'prefer_latest'; @@ -382,16 +381,18 @@ exports.updateCandidateVersion = async function updateCandidateVersion(trackId, const existingCandidates = source.candidates || []; let found = false; + let updatedEntry; const updatedCandidates = existingCandidates.map((candidate) => { if ( candidate.object_ref === objectRef && revisionReference.sameModified(candidate.object_modified, data.old_modified) ) { found = true; - return { + updatedEntry = { ...candidate, object_modified: revisionReference.normalize(data.new_modified), }; + return updatedEntry; } return candidate; }); @@ -404,6 +405,8 @@ exports.updateCandidateVersion = async function updateCandidateVersion(trackId, }); } + await primaryRevisionService.assertRequestEntries([updatedEntry]); + const snapshot = await snapshotService.cloneSnapshot(trackId, source, { candidates: updatedCandidates, }); diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index 857cb238..6f902d2b 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -11,6 +11,7 @@ const conflictResolution = require('../../lib/release-tracks/conflict-resolution const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); const revisionReference = require('../../lib/release-tracks/revision-reference'); const releaseHistoryService = require('./release-history-service'); +const primaryRevisionService = require('./primary-revision-service'); const logger = require('../../lib/logger'); const { AlreadyReleasedError, @@ -261,6 +262,11 @@ async function planLoadedSnapshot(trackId, snapshot, options) { } : snapshot; + await primaryRevisionService.assertStoredEntries([ + ...(releaseInput.members || []), + ...(releaseInput.staged || []), + ]); + return planRelease( trackId, releaseInput, diff --git a/app/services/release-tracks/virtual-track-service.js b/app/services/release-tracks/virtual-track-service.js index 2697a63b..58a2bf30 100644 --- a/app/services/release-tracks/virtual-track-service.js +++ b/app/services/release-tracks/virtual-track-service.js @@ -17,6 +17,7 @@ // ============================================================================= const snapshotService = require('./snapshot-service'); +const primaryRevisionService = require('./primary-revision-service'); const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); const deduplicationStrategies = require('../../lib/release-tracks/deduplication-strategies'); @@ -488,6 +489,7 @@ exports.createVirtualSnapshot = async function createVirtualSnapshot(trackId, op source, registryMap, ); + await primaryRevisionService.assertStoredEntries([...members, ...quarantined]); // Build overrides for the new snapshot const overrides = { @@ -559,6 +561,7 @@ exports.promoteQuarantinedObject = async function promoteQuarantinedObject(track const quarantine = (source.quarantine || []).filter( (entry) => entry.object_ref !== selected.object_ref, ); + await primaryRevisionService.assertStoredEntries([...members, ...quarantine]); const snapshot = await snapshotService.cloneSnapshot(trackId, source, { members, diff --git a/app/tests/api/release-tracks/primary-revision-integrity.spec.js b/app/tests/api/release-tracks/primary-revision-integrity.spec.js new file mode 100644 index 00000000..2b32c0a4 --- /dev/null +++ b/app/tests/api/release-tracks/primary-revision-integrity.spec.js @@ -0,0 +1,320 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); +const sinon = require('sinon'); +const { v4: uuidv4 } = require('uuid'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const Technique = require('../../../models/technique-model'); +const ReleaseTrackRegistry = require('../../../models/release-tracks/release-track-registry-model'); +const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); +const techniquesRepo = require('../../../repository/techniques-repository'); +const { DatabaseError } = require('../../../exceptions'); + +const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; + +function buildTechnique(name, previous) { + const timestamp = previous + ? new Date(new Date(previous.stix.modified).getTime() + 1000).toISOString() + : new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + id: previous?.stix.id, + created: previous?.stix.created || timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [markingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }; +} + +describe('Release-track primary revision integrity API', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + after(async function () { + await database.closeConnection(); + }); + + async function api(method, path, body, status) { + const call = request(app) + [method](path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + if (body !== undefined) call.send(body); + return call.expect(status); + } + + async function post(path, body, status = 200) { + return (await api('post', path, body, status)).body; + } + + async function get(path, status = 200) { + return (await api('get', path, undefined, status)).body; + } + + async function createTechnique(name, previous) { + return post('/api/techniques', buildTechnique(name, previous), 201); + } + + async function createTrack(name, type = 'standard', extra = {}) { + return post('/api/release-tracks/new', { name, type, ...extra }, 201); + } + + function missingRevision(objectRef = `attack-pattern--${uuidv4()}`) { + return { + object_ref: objectRef, + object_modified: '2026-01-01T00:00:00.000Z', + }; + } + + async function deleteTechniqueRevision(technique) { + await Technique.deleteOne({ + 'stix.id': technique.stix.id, + 'stix.modified': new Date(technique.stix.modified), + }); + } + + it('rejects nonexistent exact candidate pins without creating a snapshot', async function () { + const track = await createTrack('Reject Missing Candidate'); + const missing = missingRevision(); + + const response = await api( + 'post', + `/api/release-tracks/${track.id}/candidates`, + { + object_refs: [{ id: missing.object_ref, modified: missing.object_modified }], + }, + 400, + ); + expect(response.body).toEqual({ + message: 'One or more object revisions do not exist', + missing_references: [missing], + }); + + expect((await dynamicRepo.getAllSnapshots(track.id)).pagination.total).toBe(1); + }); + + it('rejects a candidate pin update to a nonexistent revision', async function () { + const technique = await createTechnique('Reject Missing Candidate Update'); + const track = await createTrack('Reject Missing Candidate Update Track'); + await post(`/api/release-tracks/${track.id}/candidates`, { + object_refs: [{ id: technique.stix.id, modified: technique.stix.modified }], + }); + const missing = missingRevision(technique.stix.id); + + const response = await api( + 'post', + `/api/release-tracks/${track.id}/candidates/${technique.stix.id}/update-version`, + { + old_modified: technique.stix.modified, + new_modified: missing.object_modified, + }, + 400, + ); + expect(response.body.missing_references).toEqual([missing]); + + const latest = await get(`/api/release-tracks/${track.id}/snapshots/latest`); + expect(latest.candidates[0].object_modified).toBe(technique.stix.modified); + }); + + it('rejects direct member replacement atomically when one exact revision is missing', async function () { + const technique = await createTechnique('Reject Missing Direct Member'); + const track = await createTrack('Reject Missing Direct Member Track'); + const missing = missingRevision(); + + const response = await api( + 'post', + `/api/release-tracks/${track.id}/contents`, + { + x_mitre_contents: [ + { obj_ref: technique.stix.id, obj_modified: technique.stix.modified }, + { obj_ref: missing.object_ref, obj_modified: missing.object_modified }, + ], + }, + 400, + ); + expect(response.body.missing_references).toEqual([missing]); + expect((await dynamicRepo.getAllSnapshots(track.id)).pagination.total).toBe(1); + }); + + it('fails preview and release when a staged revision was deleted', async function () { + const technique = await createTechnique('Deleted Staged Revision'); + const track = await createTrack('Deleted Staged Revision Track'); + await post(`/api/release-tracks/${track.id}/candidates`, { + object_refs: [{ id: technique.stix.id, modified: technique.stix.modified }], + }); + await post(`/api/release-tracks/${track.id}/candidates/promote`, { + object_refs: [technique.stix.id], + }); + await deleteTechniqueRevision(technique); + + const expectedMissing = { + object_ref: technique.stix.id, + object_modified: technique.stix.modified, + }; + const preview = await api( + 'get', + `/api/release-tracks/${track.id}/snapshots/latest/release/preview`, + undefined, + 409, + ); + expect(preview.body).toEqual({ + message: 'Release-track primary content is incomplete', + missing_references: [expectedMissing], + }); + + const release = await api( + 'post', + `/api/release-tracks/${track.id}/snapshots/latest/release`, + {}, + 409, + ); + expect(release.body.missing_references).toEqual([expectedMissing]); + expect( + (await dynamicRepo.getAllSnapshots(track.id, { taggedOnly: true })).pagination.total, + ).toBe(0); + }); + + it('rejects cloning and export when a stored primary member is missing', async function () { + const technique = await createTechnique('Missing Stored Member'); + const track = await createTrack('Missing Stored Member Track'); + await post(`/api/release-tracks/${track.id}/contents`, { + x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], + }); + await deleteTechniqueRevision(technique); + const registryCount = await ReleaseTrackRegistry.countDocuments(); + + const bundle = await api( + 'get', + `/api/release-tracks/${track.id}/snapshots/latest?format=bundle`, + undefined, + 409, + ); + expect(bundle.body.missing_references).toEqual([ + { + object_ref: technique.stix.id, + object_modified: technique.stix.modified, + }, + ]); + + const workbench = await api( + 'get', + `/api/release-tracks/${track.id}/snapshots/latest`, + undefined, + 409, + ); + expect(workbench.body.missing_references).toEqual(bundle.body.missing_references); + + await api('post', `/api/release-tracks/${track.id}/clone`, {}, 409); + expect(await ReleaseTrackRegistry.countDocuments()).toBe(registryCount); + }); + + it('propagates repository hydration failures instead of returning a partial export', async function () { + const technique = await createTechnique('Failed Primary Hydration'); + const track = await createTrack('Failed Primary Hydration Track'); + await post(`/api/release-tracks/${track.id}/contents`, { + x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], + }); + const hydrationStub = sinon + .stub(techniquesRepo, 'findManyByIdAndModified') + .rejects(new DatabaseError(new Error('injected hydration failure'))); + + try { + await api( + 'get', + `/api/release-tracks/${track.id}/snapshots/latest?format=bundle`, + undefined, + 500, + ); + } finally { + hydrationStub.restore(); + } + }); + + it('aborts virtual materialization when a component member is missing', async function () { + const technique = await createTechnique('Missing Virtual Component Member'); + const component = await createTrack('Missing Virtual Component'); + await post(`/api/release-tracks/${component.id}/contents`, { + x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], + }); + await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, { + version: '1.0', + }); + const virtual = await createTrack('Missing Virtual Primary', 'virtual', { + composition: { + component_tracks: [ + { + track_id: component.id, + resolution_strategy: 'latest_tagged', + priority: 1, + }, + ], + }, + }); + await deleteTechniqueRevision(technique); + + const response = await api( + 'post', + `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, + {}, + 409, + ); + expect(response.body.missing_references).toEqual([ + { + object_ref: technique.stix.id, + object_modified: technique.stix.modified, + }, + ]); + expect((await dynamicRepo.getAllSnapshots(virtual.id)).pagination.total).toBe(1); + }); + + it('does not create a track when any bundle primary object cannot be imported', async function () { + const technique = await createTechnique('Existing Bundle Primary'); + const registryCount = await ReleaseTrackRegistry.countDocuments(); + const unsupported = { + type: 'x-unsupported-primary', + id: `x-unsupported-primary--${uuidv4()}`, + modified: '2026-01-01T00:00:00.000Z', + }; + + const response = await api( + 'post', + '/api/release-tracks/new-from-bundle', + { + type: 'bundle', + id: `bundle--${uuidv4()}`, + objects: [technique.stix, unsupported], + }, + 400, + ); + expect(response.body).toMatchObject({ + message: 'Bundle contains an unsupported primary object type', + details: { + object_ref: unsupported.id, + type: unsupported.type, + }, + }); + expect(await ReleaseTrackRegistry.countDocuments()).toBe(registryCount); + }); +}); diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js index 6362eff5..72f862dc 100644 --- a/app/tests/api/release-tracks/release-tracks-release.spec.js +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -529,30 +529,41 @@ describe('Release-track release planning and commit API', function () { }); it('compares the latest virtual draft with its preceding tagged release', async function () { + const updatedOld = ( + await post('/api/techniques', buildTechnique('Virtual Preview Updated Old'), 201) + ).body; + const updatedNew = ( + await post('/api/techniques', buildTechnique('Virtual Preview Updated New', updatedOld), 201) + ).body; + const removed = (await post('/api/techniques', buildTechnique('Virtual Preview Removed'), 201)) + .body; + const added = (await post('/api/techniques', buildTechnique('Virtual Preview Added'), 201)) + .body; + const quarantined = ( + await post('/api/techniques', buildTechnique('Virtual Preview Quarantined'), 201) + ).body; const track = await createTrack('Virtual Release Preview', 'virtual'); const created = new Date(track.modified); const taggedModified = new Date(created.getTime() + 1000); const draftModified = new Date(created.getTime() + 2000); - const oldRevision = new Date(created.getTime() - 2000); - const newRevision = new Date(created.getTime() - 1000); await dynamicRepo.saveSnapshot(track.id, { ...snapshotBase(track), modified: taggedModified, version: '1.0', members: [ - memberEntry(virtualObjectRefs[0], oldRevision), - memberEntry(virtualObjectRefs[1], oldRevision), + memberEntry(updatedOld.stix.id, updatedOld.stix.modified), + memberEntry(removed.stix.id, removed.stix.modified), ], - quarantine: [quarantineEntry(virtualObjectRefs[3], oldRevision, track.id)], + quarantine: [quarantineEntry(quarantined.stix.id, quarantined.stix.modified, track.id)], }); await dynamicRepo.saveSnapshot(track.id, { ...snapshotBase(track), modified: draftModified, version: null, members: [ - memberEntry(virtualObjectRefs[0], newRevision), - memberEntry(virtualObjectRefs[2], newRevision), + memberEntry(updatedNew.stix.id, updatedNew.stix.modified), + memberEntry(added.stix.id, added.stix.modified), ], quarantine: [], composition_resolution: compositionResolution(draftModified), @@ -586,32 +597,43 @@ describe('Release-track release planning and commit API', function () { }); it('compares a historical virtual draft with the tagged release that preceded it', async function () { + const updatedOld = ( + await post('/api/techniques', buildTechnique('Historical Virtual Updated Old'), 201) + ).body; + const updatedNew = ( + await post( + '/api/techniques', + buildTechnique('Historical Virtual Updated New', updatedOld), + 201, + ) + ).body; + const laterMember = ( + await post('/api/techniques', buildTechnique('Historical Virtual Later Member'), 201) + ).body; const track = await createTrack('Historical Virtual Release Preview', 'virtual'); const created = new Date(track.modified); const firstTaggedModified = new Date(created.getTime() + 1000); const historicalDraftModified = new Date(created.getTime() + 2000); const laterTaggedModified = new Date(created.getTime() + 3000); - const oldRevision = new Date(created.getTime() - 2000); - const newRevision = new Date(created.getTime() - 1000); await dynamicRepo.saveSnapshot(track.id, { ...snapshotBase(track), modified: firstTaggedModified, version: '1.0', - members: [memberEntry(virtualObjectRefs[0], oldRevision)], + members: [memberEntry(updatedOld.stix.id, updatedOld.stix.modified)], }); await dynamicRepo.saveSnapshot(track.id, { ...snapshotBase(track), modified: historicalDraftModified, version: null, - members: [memberEntry(virtualObjectRefs[0], newRevision)], + members: [memberEntry(updatedNew.stix.id, updatedNew.stix.modified)], composition_resolution: compositionResolution(historicalDraftModified), }); await dynamicRepo.saveSnapshot(track.id, { ...snapshotBase(track), modified: laterTaggedModified, version: '2.0', - members: [memberEntry(virtualObjectRefs[3], newRevision)], + members: [memberEntry(laterMember.stix.id, laterMember.stix.modified)], }); const preview = await get( diff --git a/app/tests/api/release-tracks/snapshot-history.spec.js b/app/tests/api/release-tracks/snapshot-history.spec.js index 7e1f2fe5..adc7b717 100644 --- a/app/tests/api/release-tracks/snapshot-history.spec.js +++ b/app/tests/api/release-tracks/snapshot-history.spec.js @@ -7,25 +7,19 @@ const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); -const objectRefs = [ - 'attack-pattern--00000000-0000-4000-8000-000000000001', - 'attack-pattern--00000000-0000-4000-8000-000000000002', - 'attack-pattern--00000000-0000-4000-8000-000000000003', - 'attack-pattern--00000000-0000-4000-8000-000000000004', - 'attack-pattern--00000000-0000-4000-8000-000000000005', - 'attack-pattern--00000000-0000-4000-8000-000000000006', -]; - -function memberEntry(index, modified) { +const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; +const objectRevisions = []; + +function memberEntry(index) { return { - object_ref: objectRefs[index], - object_modified: modified, + object_ref: objectRevisions[index].id, + object_modified: objectRevisions[index].modified, }; } function stagedEntry(index, modified) { return { - ...memberEntry(index, modified), + ...memberEntry(index), object_status: 'reviewed', object_staged_at: modified, object_staged_by: 'snapshot-history-test', @@ -34,7 +28,7 @@ function stagedEntry(index, modified) { function candidateEntry(index, modified) { return { - ...memberEntry(index, modified), + ...memberEntry(index), object_status: 'work-in-progress', object_added_at: modified, object_added_by: 'snapshot-history-test', @@ -66,6 +60,9 @@ describe('GET /api/release-tracks/:id/snapshots', function () { app = await require('../../../index').initializeApp(); passportCookie = await login.loginAnonymous(app); + for (let index = 0; index < 6; index++) { + objectRevisions.push(await createTechnique(`Snapshot History Technique ${index + 1}`)); + } standardTrack = await createTrack('Snapshot History Standard', 'standard'); virtualTrack = await createTrack('Snapshot History Virtual', 'virtual'); @@ -77,7 +74,7 @@ describe('GET /api/release-tracks/:id/snapshots', function () { ...snapshotBase(standardTrack), modified: standardTaggedModified, version: '1.0', - members: [memberEntry(0, standardTaggedModified), memberEntry(1, standardTaggedModified)], + members: [memberEntry(0), memberEntry(1)], staged: [stagedEntry(2, standardTaggedModified)], candidates: [ candidateEntry(3, standardTaggedModified), @@ -89,7 +86,7 @@ describe('GET /api/release-tracks/:id/snapshots', function () { ...snapshotBase(standardTrack), modified: standardLatestModified, version: null, - members: [memberEntry(0, standardLatestModified)], + members: [memberEntry(0)], staged: [stagedEntry(1, standardLatestModified), stagedEntry(2, standardLatestModified)], candidates: [candidateEntry(3, standardLatestModified)], }); @@ -100,10 +97,10 @@ describe('GET /api/release-tracks/:id/snapshots', function () { ...snapshotBase(virtualTrack), modified: virtualTaggedModified, version: '1.0', - members: [memberEntry(0, virtualTaggedModified), memberEntry(1, virtualTaggedModified)], + members: [memberEntry(0), memberEntry(1)], quarantine: [ { - ...memberEntry(2, virtualTaggedModified), + ...memberEntry(2), source_track_id: standardTrack.id, source_track_name: standardTrack.name, source_snapshot_version: '1.0', @@ -123,6 +120,34 @@ describe('GET /api/release-tracks/:id/snapshots', function () { return response.body; } + async function createTechnique(name) { + const timestamp = new Date().toISOString(); + const response = await request(app) + .post('/api/techniques') + .send({ + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'attack-pattern', + spec_version: '2.1', + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + object_marking_refs: [markingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(201); + return { + id: response.body.stix.id, + modified: response.body.stix.modified, + }; + } + function get(path, status = 200) { return request(app) .get(path) diff --git a/app/tests/middleware/error-handler.spec.js b/app/tests/middleware/error-handler.spec.js index 0ab1829f..a553a44f 100644 --- a/app/tests/middleware/error-handler.spec.js +++ b/app/tests/middleware/error-handler.spec.js @@ -9,7 +9,9 @@ const { DatabaseError, DuplicateIdError, DuplicateReleaseVersionError, + InvalidObjectRevisionError, InvalidPostOperationError, + ReleaseContentIntegrityError, } = require('../../exceptions'); describe('error-handler middleware', function () { @@ -99,6 +101,58 @@ describe('error-handler middleware', function () { expect(next.called).toBe(false); }); + it('should return missing request revisions as a structured bad request', function () { + const missing = [ + { + object_ref: 'attack-pattern--00000000-0000-4000-8000-000000000001', + object_modified: '2026-01-01T00:00:00.000Z', + }, + ]; + const err = new InvalidObjectRevisionError(missing); + const res = { + status: sinon.stub().returnsThis(), + send: sinon.stub().returnsThis(), + }; + const next = sinon.stub(); + + errorHandler.serviceExceptions(err, {}, res, next); + + expect(res.status.calledOnceWithExactly(400)).toBe(true); + expect( + res.send.calledOnceWithExactly({ + message: 'One or more object revisions do not exist', + missing_references: missing, + }), + ).toBe(true); + expect(next.called).toBe(false); + }); + + it('should return missing stored revisions as a structured conflict', function () { + const missing = [ + { + object_ref: 'attack-pattern--00000000-0000-4000-8000-000000000001', + object_modified: '2026-01-01T00:00:00.000Z', + }, + ]; + const err = new ReleaseContentIntegrityError(missing); + const res = { + status: sinon.stub().returnsThis(), + send: sinon.stub().returnsThis(), + }; + const next = sinon.stub(); + + errorHandler.serviceExceptions(err, {}, res, next); + + expect(res.status.calledOnceWithExactly(409)).toBe(true); + expect( + res.send.calledOnceWithExactly({ + message: 'Release-track primary content is incomplete', + missing_references: missing, + }), + ).toBe(true); + expect(next.called).toBe(false); + }); + it('should preserve wrapped error details for DatabaseError', function () { const err = new DatabaseError(new Error('Mongo connection failed')); const res = { diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index 1a1fe4b8..b0301b13 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -71,6 +71,51 @@ Done when: - Release-preview fixtures show dynamic staged input becoming exact would-be members, and committed-release fixtures contain no dynamic members. +## P0 — Surface fail-closed primary revision errors + +### [ ] Explain missing primary revisions instead of showing a generic failure + +The backend now verifies every exact `(object_ref, object_modified)` primary +reference at request ingress and again before it releases, clones, +materializes, or renders a snapshot. It no longer omits objects that could not +be hydrated. + +Two structured error cases are relevant to the UI: + +```ts +interface MissingPrimaryRevisions { + message: string; + missing_references: Array<{ + object_ref: string; + object_modified: string; + }>; +} +``` + +- HTTP `400` means the current request selected a revision that does not + exist. Candidate add/version-update and direct standard member replacement + flows should keep the dialog open, identify the missing selections, and let + the operator correct them. +- HTTP `409` means an existing draft or snapshot contains a dangling primary + reference. Snapshot retrieval, release preview/commit, cloning, virtual + materialization/quarantine promotion, and bundle export can return this + response. The UI should identify the affected revisions and explain that an + operator must repair the track/object data before continuing. + +Do not render a partial Workbench snapshot or treat a failed bundle request as +an empty export. + +Done when: + +- The release-track connector exposes `missing_references` on `400` and `409` + responses instead of flattening the response to a generic message. +- Candidate and direct-content forms keep their input state after a `400` and + highlight the missing revisions. +- Snapshot, release, clone, virtual-materialization, and export views present + an actionable integrity error for `409`. +- Tests cover multiple missing references and prove no partial snapshot or + bundle is rendered. + ## P0 — Align the Angular connector with the current routes ### [ ] Use only the explicit snapshot-retrieval endpoints diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 5ed481dd..26bb6c4e 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -29,9 +29,39 @@ Verification result (2026-07-30): - The migration preflights the union of registry IDs and canonical orphan release-track collection names before making any index changes. +### P0.2 — Make primary release membership fail closed + +- [x] Add one shared batch hydrator that resolves dynamic selectors, validates + every exact `(object_ref, object_modified)` pair, and reports all missing + primary revisions without swallowing repository failures. +- [x] Reject nonexistent exact candidate pins, candidate pin updates, direct + member replacement, track cloning, and virtual materialization before + snapshot persistence. +- [x] Revalidate existing and promoted members at release-preview and + release-commit boundaries; return a typed `409 Conflict` for corrupt stored + drafts. +- [x] Abort bundle import before creating a track when any authoritative + primary object failed to import or cannot be hydrated. +- [x] Abort bundle/workbench export when selected primary revisions cannot be + hydrated; return every missing reference instead of a partial result. +- [x] Add ingress, partial-import, deleted-staged-revision, virtual + materialization, and incomplete-export regressions. +- [x] Update user/developer documentation and run focused, lint, OpenAPI, and + complete-suite verification. + +Verification result (2026-07-30): + +- The shared integrity, release, export, virtual determinism/quarantine, and + middleware regression group passes (54); the complete release-track API + regression group passes (143). +- OpenAPI validation (2) and backend lint pass. +- The required full suite passes: OpenAPI 2, config 21, API 955, middleware + 27, and scheduler 10. +- Bruno documents the structured `400`/`409` integrity response on companion + branch `fix/release-tracks-production-readiness`. + ### Remaining prioritized recommendations -- [ ] P0.2 — Make primary release membership fail closed. - [ ] P0.3 — Make tagged-content immutability authoritative and durable. - [ ] P0.4 — Correct destructive authorization and add durable audit records. - [ ] P0.5 — Complete the Angular contract migration and end-to-end smoke gate. diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index 074eebe9..5f9abb0c 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -104,7 +104,9 @@ Implemented in resolved for this export request, then the concrete `{object_ref, object_modified}` pairs are batch-fetched per STIX type via each repository's `findManyByIdAndModified`. The stored draft selectors are - not mutated. + not mutated. Hydration is fail-closed: if any selected primary revision is + missing, the request returns `409` with `missing_references` and emits no + partial bundle. Database failures propagate as server errors. 3. **Relationships** — the relationship service fetches the latest active relationship revisions whose `source_ref` and `target_ref` are both among the selected objects. Deprecated data-component `detects` relationships @@ -196,6 +198,11 @@ comma-separated and repeated-parameter forms reach the Zod layer, which normalizes and enforces the enums. Invalid values produce a 400 `InvalidQueryStringParameterError`. +Primary revision existence is validated separately in +`primary-revision-service.js`. This is intentionally a service-layer +invariant, because snapshot cloning, scheduled virtual materialization, and +release planning also enter through non-controller paths. + ### Regression tests - [release-tracks-bundle.spec.js](../../../app/tests/api/release-tracks/release-tracks-bundle.spec.js) diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 43bfceea..739b71f3 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -43,6 +43,33 @@ idempotently. and `version-utils.calculateNextVersion` repeats the invariant so internal release-planning callers cannot silently choose one selector. +### Primary revision integrity boundary + +`app/services/release-tracks/primary-revision-service.js` is the shared +existence and hydration boundary for primary snapshot content. It resolves +dynamic selectors, batches exact `(object_ref, object_modified)` reads by STIX +type, preserves request order, and reports every missing revision instead of +silently dropping it. + +The error contract distinguishes who can correct the problem: + +- Request ingress returns `400` with `missing_references` when candidate + selection or direct member replacement names a revision that does not exist. +- Operations over already-persisted content return `409` with + `missing_references` when a release preview/commit, track clone, virtual + materialization, quarantine promotion, or bundle export encounters a + dangling primary reference. +- Repository failures propagate as server errors. They are never interpreted + as an empty query result, because doing so could emit a partial release. + +Bundle bootstrap is also fail-closed. Every primary bundle object must have a +supported Workbench repository and must either be persisted successfully or +already exist as the exact revision being imported. The track registry and +initial snapshot are not created if any primary object fails. Import is not a +database transaction across the heterogeneous object collections, so objects +successfully created before a later failure may remain as ordinary Workbench +objects; no partial release track points at them. + ### Cross-tier revision enforcement `app/lib/release-tracks/tier-revision-invariant.js` owns selector identity diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 6a3d0584..b7dfb798 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -285,6 +285,12 @@ POST /api/release-tracks/new-from-bundle **Note:** All objects are added directly to the `members` tier. To add objects as candidates instead, use the standard [Create New Release Track](#create-new-release-track) endpoint followed by [Add Candidates](#add-candidates). +Bundle bootstrap is fail-closed. Unsupported primary object types, invalid +objects, and primary revisions that cannot be persisted cause HTTP `400`, and +the release track is not created. Objects successfully persisted before a +later object fails may remain available in Workbench, but no partial track or +snapshot is registered. + ### Import Release Track (Not Implemented) Comprehensively importing a release track would necessitate including the full snapshot history of the source release track. We don't presently have a solution for serializing an entire release track, including its snapshot history, into an atomic structure that can be exchanged between different Workbench deployments. diff --git a/docs/user/release-tracks/output-formats.md b/docs/user/release-tracks/output-formats.md index baed84c6..4a27bb59 100644 --- a/docs/user/release-tracks/output-formats.md +++ b/docs/user/release-tracks/output-formats.md @@ -47,6 +47,8 @@ shape for snapshot retrieval endpoints and is intended for the Workbench fronten response enriches them from the currently latest object revision without replacing the stored selector. - Adds UI-friendly object details to tier entries +- Fails with HTTP `409` and `missing_references` rather than returning + partially enriched tier content when a selected primary revision is missing - Suitable for Workbench UI rendering and release-track management workflows Use `include` to narrow tier arrays in `workbench` responses: @@ -100,6 +102,11 @@ Standard STIX bundle format: - If a draft export explicitly includes candidate or staged tiers, dynamic `"latest"` selectors are resolved for that export request. Tagged member contents remain exact. +- Bundle export is fail-closed for primary content. If any selected exact + revision no longer exists, the server returns HTTP `409` with every missing + `(object_ref, object_modified)` pair in `missing_references`; it never emits + a partial bundle. A repository/database failure is returned as a server + error rather than being mistaken for missing content. - Notes are never included (notes are Workbench-native objects, not STIX objects) - Suitable for external publication diff --git a/docs/user/release-tracks/release-workflow.md b/docs/user/release-tracks/release-workflow.md index b62b26b3..acdefd8e 100644 --- a/docs/user/release-tracks/release-workflow.md +++ b/docs/user/release-tracks/release-workflow.md @@ -58,6 +58,12 @@ Snapshot tagged → dynamic staged selectors resolved and exact revisions moved Snapshot exported → members reflected in stix.x_mitre_contents of the output bundle ``` +Immediately before previewing or committing a release, the server hydrates +every resulting exact member revision. If persisted track content points to a +revision that no longer exists, the operation returns HTTP `409` with +`missing_references` and does not tag the snapshot. This check protects both +standard and virtual releases from publishing incomplete primary membership. + ### STIX Freeze Solution Version pinning solves the "STIX freeze" problem: @@ -161,7 +167,9 @@ POST /api/release-tracks/:id/candidates ``` **Business Logic:** -1. Validate all object_refs exist +1. Validate that every selected exact revision exists. A missing exact pin, or + a `"latest"` selector for an object with no current revision, returns HTTP + `400` with `missing_references`; no snapshot is created. 2. Establish the `object_modified` selector: - If an ISO timestamp is provided: retain that exact revision pin - If `"latest"` is provided or `modified` is omitted: persist the dynamic @@ -171,6 +179,9 @@ POST /api/release-tracks/:id/candidates 5. If status meets `candidacy_threshold`, auto-promote to `workspace.staged` 6. Update object's `workspace.referenced_by` array +The same existence check applies when changing a candidate version pin and +when replacing a standard snapshot's member contents directly. + Importantly, candidate removal/deletion must occur separately using the `DELETE` operation: ```bash DELETE /api/release-tracks/:id/candidates From 173fff25cbeb1977df53d0138b4c1b1bb5d8db1c Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:25:07 -0400 Subject: [PATCH 3/7] test(release-tracks): evict dropped dynamic models Clear per-track Mongoose models when the in-memory database is dropped so later specs do not recreate indexes for collections that no longer exist. --- app/lib/database-in-memory.js | 17 ++++++++++++----- app/models/release-tracks/model-factory.js | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/app/lib/database-in-memory.js b/app/lib/database-in-memory.js index 66564f09..6c0efb16 100644 --- a/app/lib/database-in-memory.js +++ b/app/lib/database-in-memory.js @@ -25,7 +25,9 @@ exports.initializeConnection = async function () { // Bootstrap db connection logger.info('Mongoose attempting to connect to in memory database at ' + uri); try { - await mongoose.connect(uri); + if (mongoose.connection.readyState === 0) { + await mongoose.connect(uri); + } } catch (error) { handleError(error); } @@ -41,12 +43,17 @@ exports.initializeConnection = async function () { }; exports.closeConnection = async function () { - // Drop data and disconnect, but leave the mongod instance running for the - // next spec file. The mocha scripts run with --exit, so the process does - // not linger after the last spec. + // Drop data, but keep both mongod and the Mongoose connection alive for the + // next spec file. Disconnecting while an event listener is finishing can + // reset an otherwise unrelated Supertest request in a later suite. The + // mocha scripts run with --exit, so the process does not linger after the + // last spec. if (mongod && mongoose.connection.readyState !== 0) { await mongoose.connection.dropDatabase(); - await mongoose.connection.close(); + // Dynamic release-track collections no longer exist after the drop. + // Evict their models so the next spec does not rebuild indexes for every + // track created by all preceding specs in this process. + require('../models/release-tracks/model-factory').clearModels(); } }; diff --git a/app/models/release-tracks/model-factory.js b/app/models/release-tracks/model-factory.js index e4ba1771..c9383d7f 100644 --- a/app/models/release-tracks/model-factory.js +++ b/app/models/release-tracks/model-factory.js @@ -53,6 +53,20 @@ class ModelFactory { } } + /** + * Remove every cached dynamic release-track model. + * + * Test databases drop every dynamic collection between spec files. Keeping + * those models registered makes the next connection recreate indexes for + * every track used by every preceding spec, even though none of those + * collections still exists. + */ + clearModels() { + for (const trackId of [...this._cache.keys()]) { + this.removeModel(trackId); + } + } + /** * Ensure indexes are created on a release track's collection. * Call this after creating a new track to build the indexes defined in the schema. From a3734c1017568056ee65a87d5b9bb888c0297ad2 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:34:17 -0400 Subject: [PATCH 4/7] fix(release-tracks): make release protection durable Guard tagged revisions from authoritative snapshot history and make backref reconciliation required, durable, observable, and repairable after partial failures. --- .../definitions/components/release-tracks.yml | 22 ++ .../paths/release-tracks-paths.yml | 12 + app/exceptions/index.js | 11 + app/lib/error-handler.js | 4 +- app/lib/event-bus.js | 38 +++- .../release-track-reconciliation-model.js | 69 ++++++ app/repository/_base.repository.js | 13 ++ .../release-track-dynamic.repository.js | 37 ++++ ...release-track-reconciliation.repository.js | 105 +++++++++ app/services/meta-classes/base.service.js | 54 ++++- .../release-tracks/reconciliation-service.js | 139 ++++++++++++ .../release-tracks/snapshot-service.js | 5 +- .../tagged-membership-service.js | 68 ++++++ app/services/stix/attack-objects-service.js | 19 +- app/services/stix/relationships-service.js | 19 +- .../reconciliation-durability.spec.js | 208 ++++++++++++++++++ .../tagged-content-immutability.spec.js | 141 ++++++++++++ app/tests/middleware/error-handler.spec.js | 25 +++ docs/README.md | 1 + docs/admin/release-track-reconciliation.md | 79 +++++++ docs/developer/FRONTEND_TODO.md | 27 +++ docs/developer/TODO.md | 33 ++- docs/developer/event-bus-architecture.md | 26 ++- .../release-tracks/backref-reconciliation.md | 54 ++++- docs/user/release-tracks/object-backrefs.md | 15 +- docs/user/release-tracks/release-workflow.md | 7 + package.json | 1 + scripts/README.md | 21 ++ scripts/reconcileReleaseTrackBackrefs.js | 63 ++++++ 29 files changed, 1268 insertions(+), 48 deletions(-) create mode 100644 app/models/release-tracks/release-track-reconciliation-model.js create mode 100644 app/repository/release-tracks/release-track-reconciliation.repository.js create mode 100644 app/services/release-tracks/reconciliation-service.js create mode 100644 app/services/release-tracks/tagged-membership-service.js create mode 100644 app/tests/api/release-tracks/reconciliation-durability.spec.js create mode 100644 app/tests/api/release-tracks/tagged-content-immutability.spec.js create mode 100644 docs/admin/release-track-reconciliation.md create mode 100644 scripts/reconcileReleaseTrackBackrefs.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 5ba8efe7..1e5683a4 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -705,3 +705,25 @@ components: minItems: 1 items: $ref: '#/components/schemas/object-revision-reference' + + release-track-reconciliation-error: + type: object + required: + - message + - track_id + - reconciliation_id + properties: + message: + type: string + enum: + - 'Release-track membership protection could not be reconciled' + details: + type: string + description: 'Operator guidance; the track mutation may already be persisted' + track_id: + type: string + description: 'Release track whose object backrefs require repair' + reconciliation_id: + type: string + format: uuid + description: 'Durable reconciliation record to inspect or repair' diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index daf6ff1f..4ed5c2d3 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -419,6 +419,12 @@ paths: description: 'Invalid release request' '409': description: 'Already released, conflicting snapshot, or missing persisted primary revisions' + '500': + description: 'The release may be tagged, but durable membership-protection reconciliation failed' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' /api/release-tracks/{id}/snapshots/latest/release/preview: get: @@ -1343,6 +1349,12 @@ paths: description: 'Invalid release request' '409': description: 'Already released, conflicting snapshot, or missing persisted primary revisions' + '500': + description: 'The release may be tagged, but durable membership-protection reconciliation failed' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' /api/release-tracks/{id}/snapshots/{modified}/release/preview: get: diff --git a/app/exceptions/index.js b/app/exceptions/index.js index c387fd83..8186a9ce 100644 --- a/app/exceptions/index.js +++ b/app/exceptions/index.js @@ -325,6 +325,16 @@ class ReleaseContentIntegrityError extends CustomError { } } +class ReleaseTrackReconciliationError extends CustomError { + constructor(trackId, reconciliationId, options = {}) { + super('Release-track membership protection could not be reconciled', { + ...options, + track_id: trackId, + reconciliation_id: reconciliationId, + }); + } +} + class TaggedSnapshotDeletionError extends CustomError { constructor(version, options) { super(`Tagged snapshot version ${version} cannot be deleted`, options); @@ -422,6 +432,7 @@ module.exports = { //** Release track errors */ ReleaseConflictError, ReleaseContentIntegrityError, + ReleaseTrackReconciliationError, NoTaggedSnapshotsError, InvalidComponentTypeError, VirtualSnapshotNotMaterializedError, diff --git a/app/lib/error-handler.js b/app/lib/error-handler.js index 0911c02e..4e9d1b14 100644 --- a/app/lib/error-handler.js +++ b/app/lib/error-handler.js @@ -42,6 +42,7 @@ const { InvalidVersionError, ReleaseConflictError, ReleaseContentIntegrityError, + ReleaseTrackReconciliationError, NoTaggedSnapshotsError, InvalidComponentTypeError, VirtualSnapshotNotMaterializedError, @@ -155,7 +156,8 @@ exports.serviceExceptions = function (err, req, res, next) { err instanceof TechniquesServiceError || err instanceof TacticsServiceError || err instanceof GenericServiceError || - err instanceof DatabaseError + err instanceof DatabaseError || + err instanceof ReleaseTrackReconciliationError ) { logger.error('Service error: %s', JSON.stringify(buildErrorResponse(err))); return res.status(500).send(buildErrorResponse(err)); diff --git a/app/lib/event-bus.js b/app/lib/event-bus.js index fa00dd3c..1766c604 100644 --- a/app/lib/event-bus.js +++ b/app/lib/event-bus.js @@ -42,7 +42,7 @@ class EventBus extends EventEmitter { * @param {object} payload - Data to pass to event handlers * @returns {Promise} */ - async emit(eventName, payload) { + async _dispatch(eventName, payload, options = {}) { const timestamp = new Date().toISOString(); // Log the event @@ -51,6 +51,12 @@ class EventBus extends EventEmitter { logger.debug(`EventBus: Emitting '${eventName}'`); const listeners = this.listeners(eventName); + if (listeners.length < (options.minimumListeners || 0)) { + throw new Error( + `Event '${eventName}' requires at least ${options.minimumListeners} listener(s); ` + + `found ${listeners.length}`, + ); + } if (listeners.length === 0) { logger.debug(`EventBus: No listeners for '${eventName}'`); return; @@ -79,12 +85,42 @@ class EventBus extends EventEmitter { logger.warn( `EventBus: ${failures.length}/${listeners.length} listeners failed for '${eventName}'`, ); + if (options.required) { + const error = new AggregateError( + failures.map((failure) => failure.reason), + `${failures.length}/${listeners.length} required listener(s) failed for '${eventName}'`, + ); + error.eventName = eventName; + error.failures = failures.map((failure) => failure.reason); + throw error; + } } // Return fulfilled handler results for callers that need them (e.g., WorkflowResult) return results.filter((r) => r.status === 'fulfilled' && r.value != null).map((r) => r.value); } + async emit(eventName, payload) { + return this._dispatch(eventName, payload); + } + + /** + * Emit an event whose listener side effects are part of the caller's + * success contract. Any listener failure rejects the emission. + * + * @param {string} eventName + * @param {object} payload + * @param {object} [options] + * @param {number} [options.minimumListeners] + * @returns {Promise} + */ + async emitRequired(eventName, payload, options = {}) { + return this._dispatch(eventName, payload, { + ...options, + required: true, + }); + } + /** * Log an event for debugging and auditing * @param {object} event - Event details diff --git a/app/models/release-tracks/release-track-reconciliation-model.js b/app/models/release-tracks/release-track-reconciliation-model.js new file mode 100644 index 00000000..397847fe --- /dev/null +++ b/app/models/release-tracks/release-track-reconciliation-model.js @@ -0,0 +1,69 @@ +'use strict'; + +const mongoose = require('mongoose'); +const { validateTrackId } = require('../../lib/release-tracks/release-track-validators'); + +const releaseTrackReconciliationSchema = new mongoose.Schema( + { + reconciliation_id: { + type: String, + required: true, + unique: true, + }, + track_id: { + type: String, + required: true, + validate: validateTrackId, + }, + requested_snapshot_modified: { + type: Date, + default: null, + }, + reconciled_snapshot_modified: { + type: Date, + default: null, + }, + source: { + type: String, + required: true, + enum: ['contents_changed', 'repair', 'full_scan'], + }, + status: { + type: String, + required: true, + enum: ['pending', 'completed', 'failed'], + default: 'pending', + }, + attempts: { + type: Number, + required: true, + default: 0, + min: 0, + }, + created_at: { + type: Date, + required: true, + }, + updated_at: { + type: Date, + required: true, + }, + completed_at: { + type: Date, + default: null, + }, + last_error: { + name: String, + message: String, + }, + }, + { + collection: 'releaseTrackReconciliations', + bufferCommands: false, + }, +); + +releaseTrackReconciliationSchema.index({ status: 1, updated_at: 1 }); +releaseTrackReconciliationSchema.index({ track_id: 1, created_at: -1 }); + +module.exports = mongoose.model('ReleaseTrackReconciliation', releaseTrackReconciliationSchema); diff --git a/app/repository/_base.repository.js b/app/repository/_base.repository.js index fd024a38..1cec5a72 100644 --- a/app/repository/_base.repository.js +++ b/app/repository/_base.repository.js @@ -591,6 +591,19 @@ class BaseRepository extends AbstractRepository { } } + /** + * Return every release-track ID present in denormalized object backrefs. + * Used only by administrative full-scan repair so deleted tracks with stale + * backrefs are included alongside registry-backed tracks. + */ + async distinctReleaseTrackIds() { + try { + return await this.model.distinct('workspace.release_tracks.id').exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + /** * Resolve specific object revisions to their document _ids. Lean, minimal * projection — used by release-track backref reconciliation. diff --git a/app/repository/release-tracks/release-track-dynamic.repository.js b/app/repository/release-tracks/release-track-dynamic.repository.js index f1675d5b..80926151 100644 --- a/app/repository/release-tracks/release-track-dynamic.repository.js +++ b/app/repository/release-tracks/release-track-dynamic.repository.js @@ -167,6 +167,43 @@ class ReleaseTrackDynamicRepository { } } + /** + * Find tagged snapshots whose members tier contains an object revision. + * Omitting objectModified matches every released revision for the STIX ID. + * This query reads the tagged snapshots themselves rather than relying on + * denormalized object backrefs or registry release metadata. + */ + async findTaggedSnapshotsContainingRevision(trackId, objectRef, objectModified) { + try { + const Model = this._getModel(trackId); + const memberMatch = { object_ref: objectRef }; + if (objectModified !== undefined) { + memberMatch.object_modified = new Date(objectModified); + } + + return await Model.find( + { + id: trackId, + version: { $type: 'string' }, + members: { $elemMatch: memberMatch }, + }, + { + id: 1, + type: 1, + name: 1, + modified: 1, + version: 1, + members: { $elemMatch: memberMatch }, + }, + ) + .sort({ modified: 1 }) + .lean() + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + async getAllSnapshots(trackId, options = {}) { try { const Model = this._getModel(trackId); diff --git a/app/repository/release-tracks/release-track-reconciliation.repository.js b/app/repository/release-tracks/release-track-reconciliation.repository.js new file mode 100644 index 00000000..154dec54 --- /dev/null +++ b/app/repository/release-tracks/release-track-reconciliation.repository.js @@ -0,0 +1,105 @@ +'use strict'; + +const { v4: uuidv4 } = require('uuid'); +const ReleaseTrackReconciliation = require('../../models/release-tracks/release-track-reconciliation-model'); +const { DatabaseError } = require('../../exceptions'); + +class ReleaseTrackReconciliationRepository { + async create({ trackId, snapshotModified, source }) { + const now = new Date(); + try { + const record = await ReleaseTrackReconciliation.create({ + reconciliation_id: uuidv4(), + track_id: trackId, + requested_snapshot_modified: snapshotModified || null, + source, + status: 'pending', + attempts: 0, + created_at: now, + updated_at: now, + }); + return record.toObject(); + } catch (error) { + throw new DatabaseError(error); + } + } + + async startAttempt(reconciliationId) { + try { + return await ReleaseTrackReconciliation.findOneAndUpdate( + { reconciliation_id: reconciliationId }, + { + $inc: { attempts: 1 }, + $set: { + status: 'pending', + updated_at: new Date(), + completed_at: null, + last_error: null, + }, + }, + { new: true, lean: true }, + ).exec(); + } catch (error) { + throw new DatabaseError(error); + } + } + + async complete(reconciliationId, snapshotModified) { + const now = new Date(); + try { + return await ReleaseTrackReconciliation.findOneAndUpdate( + { reconciliation_id: reconciliationId }, + { + $set: { + status: 'completed', + reconciled_snapshot_modified: snapshotModified || null, + updated_at: now, + completed_at: now, + last_error: null, + }, + }, + { new: true, lean: true }, + ).exec(); + } catch (error) { + throw new DatabaseError(error); + } + } + + async fail(reconciliationId, error) { + try { + return await ReleaseTrackReconciliation.findOneAndUpdate( + { reconciliation_id: reconciliationId }, + { + $set: { + status: 'failed', + updated_at: new Date(), + completed_at: null, + last_error: { + name: error?.name || 'Error', + message: error?.message || String(error), + }, + }, + }, + { new: true, lean: true }, + ).exec(); + } catch (repositoryError) { + throw new DatabaseError(repositoryError); + } + } + + async findRepairable(limit = 100) { + try { + return await ReleaseTrackReconciliation.find({ + status: { $in: ['pending', 'failed'] }, + }) + .sort({ updated_at: 1, created_at: 1 }) + .limit(limit) + .lean() + .exec(); + } catch (error) { + throw new DatabaseError(error); + } + } +} + +module.exports = new ReleaseTrackReconciliationRepository(); diff --git a/app/services/meta-classes/base.service.js b/app/services/meta-classes/base.service.js index 864ae03b..30a9b0c6 100644 --- a/app/services/meta-classes/base.service.js +++ b/app/services/meta-classes/base.service.js @@ -713,18 +713,56 @@ class BaseService extends ServiceWithHooks { * @param {Object} document - The stored document ({ workspace, stix }) * @param {string} operation - Verb for the error message ('updated'|'deleted') */ - static assertNotMemberPinned(document, operation) { - const memberPins = (document.workspace?.release_tracks || []).filter( + static async assertNotMemberPinned(document, operation) { + const currentMemberPins = (document.workspace?.release_tracks || []).filter( (entry) => entry.tier === 'members', ); - if (memberPins.length > 0) { + const taggedMembershipService = require('../release-tracks/tagged-membership-service'); + const taggedPins = await taggedMembershipService.findPinsForRevision( + document.stix.id, + document.stix.modified, + ); + + if (currentMemberPins.length > 0 || taggedPins.length > 0) { + const trackIds = [ + ...new Set([ + ...currentMemberPins.map((entry) => entry.id), + ...taggedPins.map((entry) => entry.track_id), + ]), + ]; throw new MemberPinnedRevisionError({ details: `Revision ${document.stix.id} (modified ` + `${new Date(document.stix.modified).toISOString()}) is pinned in the members tier of ` + - `release track(s) ${memberPins.map((entry) => entry.id).join(', ')} and cannot be ` + + `release track(s) ${trackIds.join(', ')} and cannot be ` + `${operation} in place. Create a new revision instead (set x_mitre_deprecated on a ` + `new revision to retire the object).`, + release_tracks: trackIds, + tagged_releases: taggedPins, + }); + } + } + + static async assertNoMemberPinnedVersions(stixId, currentMemberPinned, operation) { + const taggedMembershipService = require('../release-tracks/tagged-membership-service'); + const taggedPins = await taggedMembershipService.findPinsForObject(stixId); + const currentTrackIds = currentMemberPinned.flatMap((document) => + (document.workspace?.release_tracks || []) + .filter((entry) => entry.tier === 'members') + .map((entry) => entry.id), + ); + const trackIds = [ + ...new Set([...currentTrackIds, ...taggedPins.map((entry) => entry.track_id)]), + ]; + + if (trackIds.length > 0) { + throw new MemberPinnedRevisionError({ + details: + `Object ${stixId} has revision(s) pinned in the members tier of release track(s) ` + + `${trackIds.join(', ')} and cannot be ${operation}. Create a new revision instead ` + + `(set x_mitre_deprecated on a new revision to retire the object).`, + release_tracks: trackIds, + tagged_releases: taggedPins, }); } } @@ -946,7 +984,7 @@ class BaseService extends ServiceWithHooks { } // Members-pinned revisions are released content — immutable in place. - BaseService.assertNotMemberPinned(document, 'updated'); + await BaseService.assertNotMemberPinned(document, 'updated'); // TODO: diff analysis — detect field-level changes vs document // TODO: if no changes detected, short-circuit (no-op) @@ -1069,7 +1107,7 @@ class BaseService extends ServiceWithHooks { if (!existing) { return null; } - BaseService.assertNotMemberPinned(existing, 'deleted'); + await BaseService.assertNotMemberPinned(existing, 'deleted'); const document = await this.repository.findOneAndDelete(stixId, stixModified); @@ -1376,9 +1414,7 @@ class BaseService extends ServiceWithHooks { // Deleting all versions must not destroy a members-pinned revision const memberPinned = await this.repository.retrieveMemberPinnedVersionsLean(stixId); - for (const pinnedDocument of memberPinned) { - BaseService.assertNotMemberPinned(pinnedDocument, 'deleted'); - } + await BaseService.assertNoMemberPinnedVersions(stixId, memberPinned, 'deleted'); const result = await this.repository.deleteMany(stixId); if (result.deletedCount > 0) { diff --git a/app/services/release-tracks/reconciliation-service.js b/app/services/release-tracks/reconciliation-service.js new file mode 100644 index 00000000..394ea3f2 --- /dev/null +++ b/app/services/release-tracks/reconciliation-service.js @@ -0,0 +1,139 @@ +'use strict'; + +// Durable orchestration for workspace.release_tracks reconciliation. Each +// attempt is persisted before required EventBus listeners run. Repair always +// reconciles against the track's current latest snapshot, so replay is +// idempotent and cannot restore obsolete membership from an old event. + +const EventBus = require('../../lib/event-bus'); +const Events = require('../../lib/event-constants'); +const logger = require('../../lib/logger'); +const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); +const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); +const reconciliationRepo = require('../../repository/release-tracks/release-track-reconciliation.repository'); +const attackObjectsRepo = require('../../repository/attack-objects-repository'); +const relationshipsRepo = require('../../repository/relationships-repository'); +const { ReleaseTrackReconciliationError } = require('../../exceptions'); + +// Reconciliation is also invoked by scheduler/migration tests and operator +// scripts that call the service facade without initializing Express. Loading +// both owning services guarantees the two required listeners are registered. +require('../stix/attack-objects-service'); +require('../stix/relationships-service'); + +async function dispatch(record, snapshot) { + await reconciliationRepo.startAttempt(record.reconciliation_id); + + try { + await EventBus.emitRequired( + Events.RELEASE_TRACK_CONTENTS_CHANGED, + { + trackId: record.track_id, + snapshot, + reconciliationId: record.reconciliation_id, + }, + { minimumListeners: 2 }, + ); + return await reconciliationRepo.complete(record.reconciliation_id, snapshot?.modified); + } catch (error) { + try { + await reconciliationRepo.fail(record.reconciliation_id, error); + } catch (recordError) { + logger.error( + `ReconciliationService: Failed to record reconciliation ${record.reconciliation_id} ` + + `failure: ${recordError.message}`, + ); + } + + throw new ReleaseTrackReconciliationError(record.track_id, record.reconciliation_id, { + details: + 'The release-track change was persisted, but one or more object backref protections ' + + 'failed. Run the release-track reconciliation repair command before retrying.', + cause: error, + }); + } +} + +async function currentSnapshot(trackId) { + const registry = await registryRepo.findByTrackId(trackId); + return registry ? dynamicRepo.getLatestSnapshot(trackId) : null; +} + +async function createAndDispatch(trackId, snapshot, source) { + const record = await reconciliationRepo.create({ + trackId, + snapshotModified: snapshot?.modified, + source, + }); + return dispatch(record, snapshot); +} + +exports.reconcileContentsChanged = function reconcileContentsChanged(trackId, snapshot) { + return createAndDispatch(trackId, snapshot, 'contents_changed'); +}; + +exports.repairOutstanding = async function repairOutstanding(options = {}) { + const records = await reconciliationRepo.findRepairable(options.limit || 100); + const results = []; + + for (const record of records) { + try { + const snapshot = await currentSnapshot(record.track_id); + const completed = await dispatch(record, snapshot); + results.push({ + reconciliation_id: record.reconciliation_id, + track_id: record.track_id, + status: completed.status, + }); + } catch (error) { + results.push({ + reconciliation_id: record.reconciliation_id, + track_id: record.track_id, + status: 'failed', + error: error.message, + }); + if (!options.continueOnError) throw error; + } + } + + return results; +}; + +exports.reconcileAll = async function reconcileAll(options = {}) { + const [registered, attackObjectTrackIds, relationshipTrackIds] = await Promise.all([ + registryRepo.findAll(), + attackObjectsRepo.distinctReleaseTrackIds(), + relationshipsRepo.distinctReleaseTrackIds(), + ]); + const trackIds = [ + ...new Set([ + ...registered.data.map((track) => track.track_id), + ...attackObjectTrackIds, + ...relationshipTrackIds, + ]), + ].sort(); + const results = []; + + for (const trackId of trackIds) { + try { + const snapshot = await currentSnapshot(trackId); + const completed = await createAndDispatch(trackId, snapshot, 'full_scan'); + results.push({ + reconciliation_id: completed.reconciliation_id, + track_id: trackId, + status: completed.status, + }); + } catch (error) { + results.push({ track_id: trackId, status: 'failed', error: error.message }); + if (!options.continueOnError) throw error; + } + } + + return results; +}; + +exports._private = { + createAndDispatch, + currentSnapshot, + dispatch, +}; diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 57bbffa5..45ee635a 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -17,11 +17,10 @@ const registryRepo = require('../../repository/release-tracks/release-track-regi const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); const modelFactory = require('../../models/release-tracks/model-factory'); const logger = require('../../lib/logger'); -const EventBus = require('../../lib/event-bus'); -const EventConstants = require('../../lib/event-constants'); const versionUtils = require('../../lib/release-tracks/version-utils'); const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); const primaryRevisionService = require('./primary-revision-service'); +const reconciliationService = require('./reconciliation-service'); const { TrackNotFoundError, NotFoundError, @@ -129,7 +128,7 @@ async function syncRegistryCounters(trackId) { * track (or its only snapshot) was deleted */ async function emitContentsChanged(trackId, snapshot) { - await EventBus.emit(EventConstants.RELEASE_TRACK_CONTENTS_CHANGED, { trackId, snapshot }); + await reconciliationService.reconcileContentsChanged(trackId, snapshot); } exports.emitContentsChanged = emitContentsChanged; diff --git a/app/services/release-tracks/tagged-membership-service.js b/app/services/release-tracks/tagged-membership-service.js new file mode 100644 index 00000000..ff4fcff5 --- /dev/null +++ b/app/services/release-tracks/tagged-membership-service.js @@ -0,0 +1,68 @@ +'use strict'; + +// Authoritative tagged-membership reads used by object mutation guards. +// workspace.release_tracks remains a useful denormalized current-snapshot +// pointer, but it is not authoritative for historical tagged releases and may +// be temporarily stale when reconciliation needs repair. + +const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); +const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); + +const QUERY_CONCURRENCY = 12; + +async function mapWithConcurrency(items, mapper) { + const results = new Array(items.length); + let nextIndex = 0; + + async function worker() { + while (nextIndex < items.length) { + const index = nextIndex++; + results[index] = await mapper(items[index], index); + } + } + + await Promise.all( + Array.from({ length: Math.min(QUERY_CONCURRENCY, items.length) }, () => worker()), + ); + return results; +} + +function pinFromSnapshot(snapshot) { + const member = snapshot.members[0]; + return { + track_id: snapshot.id, + track_type: snapshot.type, + track_name: snapshot.name, + version: snapshot.version, + snapshot_modified: snapshot.modified, + object_ref: member.object_ref, + object_modified: member.object_modified, + }; +} + +async function findPins(objectRef, objectModified) { + const tracks = (await registryRepo.findAll()).data; + const matchesByTrack = await mapWithConcurrency(tracks, async (track) => { + const snapshots = await dynamicRepo.findTaggedSnapshotsContainingRevision( + track.track_id, + objectRef, + objectModified, + ); + return snapshots.map(pinFromSnapshot); + }); + + return matchesByTrack.flat(); +} + +exports.findPinsForRevision = function findPinsForRevision(objectRef, objectModified) { + return findPins(objectRef, objectModified); +}; + +exports.findPinsForObject = function findPinsForObject(objectRef) { + return findPins(objectRef); +}; + +exports._private = { + mapWithConcurrency, + pinFromSnapshot, +}; diff --git a/app/services/stix/attack-objects-service.js b/app/services/stix/attack-objects-service.js index 51917ffc..eb2baf5c 100644 --- a/app/services/stix/attack-objects-service.js +++ b/app/services/stix/attack-objects-service.js @@ -234,19 +234,12 @@ class AttackObjectsService extends BaseService { */ static async handleReleaseTrackContentsChanged(payload) { const backrefReconciler = require('../../lib/release-tracks/backref-reconciler'); - - try { - await backrefReconciler.reconcile( - attackObjectsRepository, - payload.trackId, - payload.snapshot, - (objectRef) => !objectRef.startsWith('relationship--'), - ); - } catch (error) { - logger.error( - `AttackObjectsService: Error reconciling release track backrefs for ${payload.trackId}: ${error.message}`, - ); - } + return backrefReconciler.reconcile( + attackObjectsRepository, + payload.trackId, + payload.snapshot, + (objectRef) => !objectRef.startsWith('relationship--'), + ); } /** diff --git a/app/services/stix/relationships-service.js b/app/services/stix/relationships-service.js index 2e377219..8f32da1f 100644 --- a/app/services/stix/relationships-service.js +++ b/app/services/stix/relationships-service.js @@ -96,19 +96,12 @@ class RelationshipsService extends BaseService { */ static async handleReleaseTrackContentsChanged(payload) { const backrefReconciler = require('../../lib/release-tracks/backref-reconciler'); - - try { - await backrefReconciler.reconcile( - relationshipsRepository, - payload.trackId, - payload.snapshot, - (objectRef) => objectRef.startsWith('relationship--'), - ); - } catch (error) { - logger.error( - `RelationshipsService: Error reconciling release track backrefs for ${payload.trackId}: ${error.message}`, - ); - } + return backrefReconciler.reconcile( + relationshipsRepository, + payload.trackId, + payload.snapshot, + (objectRef) => objectRef.startsWith('relationship--'), + ); } /** diff --git a/app/tests/api/release-tracks/reconciliation-durability.spec.js b/app/tests/api/release-tracks/reconciliation-durability.spec.js new file mode 100644 index 00000000..2468b472 --- /dev/null +++ b/app/tests/api/release-tracks/reconciliation-durability.spec.js @@ -0,0 +1,208 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); +const sinon = require('sinon'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const Technique = require('../../../models/technique-model'); +const ReleaseTrackReconciliation = require('../../../models/release-tracks/release-track-reconciliation-model'); +const attackObjectsRepo = require('../../../repository/attack-objects-repository'); +const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); +const reconciliationService = require('../../../services/release-tracks/reconciliation-service'); +const { DatabaseError } = require('../../../exceptions'); + +const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; + +function buildTechnique(name) { + const timestamp = new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [markingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }; +} + +describe('Release-track durable backref reconciliation', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + afterEach(function () { + sinon.restore(); + }); + + after(async function () { + await database.closeConnection(); + }); + + async function api(method, path, body, status) { + const call = request(app) + [method](path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + if (body !== undefined) call.send(body); + return call.expect(status); + } + + async function post(path, body, status = 200) { + return (await api('post', path, body, status)).body; + } + + async function getTechnique(technique) { + return ( + await api( + 'get', + `/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`, + undefined, + 200, + ) + ).body; + } + + it('returns failure, persists the failed attempt, and repairs a committed release', async function () { + const technique = await post('/api/techniques', buildTechnique('Reconciliation Failure'), 201); + const track = await post( + '/api/release-tracks/new', + { name: 'Reconciliation Failure Track', type: 'standard' }, + 201, + ); + await post(`/api/release-tracks/${track.id}/candidates`, { + object_refs: [{ id: technique.stix.id, modified: technique.stix.modified }], + }); + await post(`/api/release-tracks/${track.id}/candidates/promote`, { + object_refs: [technique.stix.id], + }); + + sinon + .stub(attackObjectsRepo, 'bulkWrite') + .rejects(new DatabaseError(new Error('injected backref write failure'))); + + const release = await api( + 'post', + `/api/release-tracks/${track.id}/snapshots/latest/release`, + { version: '1.0' }, + 500, + ); + expect(release.body).toMatchObject({ + message: 'Release-track membership protection could not be reconciled', + track_id: track.id, + reconciliation_id: expect.any(String), + }); + + const tagged = await dynamicRepo.getLatestTaggedSnapshot(track.id); + expect(tagged.version).toBe('1.0'); + + let record = await ReleaseTrackReconciliation.findOne({ + reconciliation_id: release.body.reconciliation_id, + }) + .lean() + .exec(); + expect(record).toMatchObject({ + track_id: track.id, + status: 'failed', + attempts: 1, + last_error: { + name: 'AggregateError', + message: expect.stringContaining('required listener'), + }, + }); + + let stored = await getTechnique(technique); + expect(stored.workspace.release_tracks).toEqual([ + expect.objectContaining({ id: track.id, tier: 'staged' }), + ]); + + sinon.restore(); + const results = await reconciliationService.repairOutstanding({ + limit: 100, + continueOnError: false, + }); + expect(results).toContainEqual({ + reconciliation_id: release.body.reconciliation_id, + track_id: track.id, + status: 'completed', + }); + + record = await ReleaseTrackReconciliation.findOne({ + reconciliation_id: release.body.reconciliation_id, + }) + .lean() + .exec(); + expect(record.status).toBe('completed'); + expect(record.attempts).toBe(2); + expect(record.completed_at).toBeInstanceOf(Date); + + stored = await getTechnique(technique); + expect(stored.workspace.release_tracks).toEqual([ + { + id: track.id, + type: 'standard', + tier: 'members', + status: 'reviewed', + }, + ]); + }); + + it('repairs legacy drift with an idempotent full scan', async function () { + const technique = await post('/api/techniques', buildTechnique('Full Scan Repair'), 201); + const track = await post( + '/api/release-tracks/new', + { name: 'Full Scan Repair Track', type: 'standard' }, + 201, + ); + await post(`/api/release-tracks/${track.id}/contents`, { + x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], + }); + + await Technique.updateOne( + { + 'stix.id': technique.stix.id, + 'stix.modified': new Date(technique.stix.modified), + }, + { $pull: { 'workspace.release_tracks': { id: track.id } } }, + ); + expect((await getTechnique(technique)).workspace.release_tracks || []).toHaveLength(0); + + const first = await reconciliationService.reconcileAll({ continueOnError: false }); + expect(first).toContainEqual( + expect.objectContaining({ + track_id: track.id, + status: 'completed', + }), + ); + expect((await getTechnique(technique)).workspace.release_tracks).toEqual([ + expect.objectContaining({ id: track.id, tier: 'members' }), + ]); + + const second = await reconciliationService.reconcileAll({ continueOnError: false }); + expect(second).toContainEqual( + expect.objectContaining({ + track_id: track.id, + status: 'completed', + }), + ); + expect((await getTechnique(technique)).workspace.release_tracks).toHaveLength(1); + }); +}); diff --git a/app/tests/api/release-tracks/tagged-content-immutability.spec.js b/app/tests/api/release-tracks/tagged-content-immutability.spec.js new file mode 100644 index 00000000..e0c03567 --- /dev/null +++ b/app/tests/api/release-tracks/tagged-content-immutability.spec.js @@ -0,0 +1,141 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const ReleaseTrackRegistry = require('../../../models/release-tracks/release-track-registry-model'); + +const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; + +function buildTechnique(name, previous) { + const timestamp = previous + ? new Date(new Date(previous.stix.modified).getTime() + 1000).toISOString() + : new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + id: previous?.stix.id, + created: previous?.stix.created || timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [markingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }; +} + +describe('Release-track authoritative tagged-content immutability', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + after(async function () { + await database.closeConnection(); + }); + + async function api(method, path, body, status) { + const call = request(app) + [method](path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + if (body !== undefined) call.send(body); + return call.expect(status); + } + + async function post(path, body, status = 200) { + return (await api('post', path, body, status)).body; + } + + it('blocks mutation from historical tagged membership when current backrefs are absent', async function () { + const technique = await post('/api/techniques', buildTechnique('Historical Member'), 201); + const replacement = await post('/api/techniques', buildTechnique('Current Draft Member'), 201); + const track = await post( + '/api/release-tracks/new', + { name: 'Historical Immutability', type: 'standard' }, + 201, + ); + await post(`/api/release-tracks/${track.id}/contents`, { + x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], + }); + await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, { version: '1.0' }); + + // A newer draft removes the member, so latest-snapshot reconciliation + // deliberately removes the object's denormalized backref. The historical + // tagged snapshot remains the immutable authority. + await post(`/api/release-tracks/${track.id}/contents`, { + x_mitre_contents: [{ obj_ref: replacement.stix.id, obj_modified: replacement.stix.modified }], + }); + const current = ( + await api( + 'get', + `/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`, + undefined, + 200, + ) + ).body; + expect(current.workspace.release_tracks || []).toHaveLength(0); + + // Clear the registry's denormalized tagged-release catalogue as well. + // The guard must query tagged snapshots, not either derived index. + await ReleaseTrackRegistry.updateOne( + { track_id: track.id }, + { + $set: { + tagged_releases: [], + tagged_release_count: 0, + latest_tagged_version: null, + }, + }, + ); + + const updated = buildTechnique('Historical Member (edited)', technique); + updated.stix.modified = technique.stix.modified; + const putResponse = await api( + 'put', + `/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`, + updated, + 409, + ); + expect(putResponse.body.release_tracks).toEqual([track.id]); + expect(putResponse.body.tagged_releases).toEqual([ + expect.objectContaining({ + track_id: track.id, + version: '1.0', + object_ref: technique.stix.id, + object_modified: technique.stix.modified, + }), + ]); + + await api( + 'delete', + `/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`, + undefined, + 409, + ); + await api('delete', `/api/techniques/${technique.stix.id}`, undefined, 409); + + await api( + 'get', + `/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`, + undefined, + 200, + ); + }); +}); diff --git a/app/tests/middleware/error-handler.spec.js b/app/tests/middleware/error-handler.spec.js index a553a44f..05549787 100644 --- a/app/tests/middleware/error-handler.spec.js +++ b/app/tests/middleware/error-handler.spec.js @@ -12,6 +12,7 @@ const { InvalidObjectRevisionError, InvalidPostOperationError, ReleaseContentIntegrityError, + ReleaseTrackReconciliationError, } = require('../../exceptions'); describe('error-handler middleware', function () { @@ -175,4 +176,28 @@ describe('error-handler middleware', function () { expect(Object.keys(err)).not.toContain('cause'); expect(next.called).toBe(false); }); + + it('should return durable reconciliation identifiers on protection failures', function () { + const err = new ReleaseTrackReconciliationError('release-track--track', 'repair-id', { + details: 'Run repair.', + }); + const res = { + status: sinon.stub().returnsThis(), + send: sinon.stub().returnsThis(), + }; + const next = sinon.stub(); + + errorHandler.serviceExceptions(err, {}, res, next); + + expect(res.status.calledOnceWithExactly(500)).toBe(true); + expect( + res.send.calledOnceWithExactly({ + message: 'Release-track membership protection could not be reconciled', + details: 'Run repair.', + track_id: 'release-track--track', + reconciliation_id: 'repair-id', + }), + ).toBe(true); + expect(next.called).toBe(false); + }); }); diff --git a/docs/README.md b/docs/README.md index bf0ca6bf..65a8e53f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -58,6 +58,7 @@ Configuration, deployment, and identity provider setup. - [Configuration](admin/configuration.md): Complete configuration guide (environment variables, JSON files) - [Automation Run Audit Trail](admin/automation-runs.md): How to inspect migration and scheduler audit records - [Virtual Track Schedules](admin/virtual-track-schedules.md): UTC execution, restart recovery, retries, and observability +- [Release-Track Membership Reconciliation](admin/release-track-reconciliation.md): Inspect and repair durable object-backref protection failures ### Authentication diff --git a/docs/admin/release-track-reconciliation.md b/docs/admin/release-track-reconciliation.md new file mode 100644 index 00000000..53ca08c3 --- /dev/null +++ b/docs/admin/release-track-reconciliation.md @@ -0,0 +1,79 @@ +# Release-Track Membership Reconciliation + +Release-track snapshots are authoritative. Object documents carry +`workspace.release_tracks` as a derived current-snapshot index used for +navigation and mutation protection. + +## Durable records + +Every snapshot membership change creates a document in +`releaseTrackReconciliations` before the server updates object backrefs. + +Important fields: + +| Field | Meaning | +|---|---| +| `reconciliation_id` | Stable UUID returned to API callers when reconciliation fails | +| `track_id` | Track whose latest snapshot is being reconciled | +| `requested_snapshot_modified` | Snapshot current when the record was created; null means track deletion | +| `reconciled_snapshot_modified` | Snapshot actually used by the successful attempt | +| `source` | `contents_changed`, `repair`, or `full_scan` | +| `status` | `pending`, `completed`, or `failed` | +| `attempts` | Number of listener dispatch attempts | +| `last_error` | Most recent failure name and message | + +API HTTP `500` responses containing a `reconciliation_id` mean the +release-track mutation may already be persisted. In particular, a release may +already be tagged. Inspect the track before repeating any mutation. + +## Inspect failures + +```javascript +db.releaseTrackReconciliations.find({ + status: { $in: ["pending", "failed"] } +}).sort({ updated_at: 1 }).pretty() +``` + +Inspect one response identifier: + +```javascript +db.releaseTrackReconciliations.findOne({ + reconciliation_id: "" +}) +``` + +## Repair outstanding attempts + +```bash +DATABASE_URL=mongodb://... npm run repair:release-track-backrefs +``` + +The default repairs up to 100 oldest pending/failed records. Set a bound: + +```bash +DATABASE_URL=mongodb://... npm run repair:release-track-backrefs -- --limit=500 +``` + +Each retry reads the track's current latest snapshot. It does not replay an +obsolete snapshot payload, so repeated repair is idempotent. + +## Full scan + +Run a full scan after an unclean shutdown or when legacy drift is suspected: + +```bash +DATABASE_URL=mongodb://... npm run repair:release-track-backrefs -- --all +``` + +This unions registered track IDs with IDs found in object and relationship +backrefs. Existing tracks are reconciled to their current latest snapshots; +backrefs for tracks that no longer exist are removed. + +The command prints JSON and exits nonzero if any track still fails. Preserve +failed records and command output for incident review. + +## Known crash window + +Snapshot persistence and reconciliation-record creation do not share a MongoDB +transaction. A hard crash between those writes can leave no pending record. +The full scan is the recovery mechanism for that narrow interval. diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index b0301b13..0bfed042 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -116,6 +116,33 @@ Done when: - Tests cover multiple missing references and prove no partial snapshot or bundle is rendered. +### [ ] Handle persisted mutations whose membership reconciliation failed + +A release-track mutation can persist its snapshot before a downstream object +backref write fails. The server now returns HTTP `500` instead of reporting +success and includes: + +```ts +{ + message: 'Release-track membership protection could not be reconciled'; + track_id: string; + reconciliation_id: string; + details?: string; +} +``` + +For a release request, the snapshot may already be tagged. Do not +automatically retry the POST: refresh snapshot history first, show the +reconciliation ID, and direct the operator to an administrator if protection +repair is still pending. + +Done when: + +- The connector preserves `track_id` and `reconciliation_id` from this `500`. +- Release and mutation dialogs explain that persistence may have succeeded + and do not offer a blind retry. +- The UI refreshes the relevant track before enabling another action. + ## P0 — Align the Angular connector with the current routes ### [ ] Use only the explicit snapshot-retrieval endpoints diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 26bb6c4e..0c0995e4 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -62,7 +62,38 @@ Verification result (2026-07-30): ### Remaining prioritized recommendations -- [ ] P0.3 — Make tagged-content immutability authoritative and durable. +### P0.3 — Make tagged-content immutability authoritative and durable + +- [x] Guard object revision update/delete and delete-all by querying tagged + snapshot membership, even when `workspace.release_tracks` is missing or + stale. +- [x] Make release-track backref reconciliation failures propagate to the + triggering request so a release is never reported as fully successful when + protection writes failed. +- [x] Persist every reconciliation attempt and its terminal outcome so + process crashes and partial listener failures remain operator-visible. +- [x] Provide an idempotent repair command for failed/pending reconciliation + records and a full-scan mode for legacy drift. +- [x] Add failure-injection, missing-backref, repair, and historical-release + regressions; update user/developer/admin documentation. +- [x] Run focused, lint, OpenAPI, and complete-suite verification. + +Verification result (2026-07-30): + +- Lint and OpenAPI validation pass. +- The complete release-track API group passes (146), including + failure-injection, repair, and authoritative historical-membership + regressions. Scheduler/date/cron integration (15) and middleware (11) + focused groups pass. +- Repeated complete-suite runs execute all 958 API cases and consistently + pass the release-track cases. The repository's documented roaming + Supertest transport flake still moves among unrelated isolated-pass cases + (socket resets, transient status mismatches, or timeouts). Dynamic + release-track models are now evicted between dropped test databases and the + Mongoose connection is reused, reducing the API run from roughly six + minutes to roughly one minute; the remaining unrelated transport flake is + tracked separately from this completed integrity change. + - [ ] P0.4 — Correct destructive authorization and add durable audit records. - [ ] P0.5 — Complete the Angular contract migration and end-to-end smoke gate. - [ ] P0.6 — Finish scheduled-materialization fencing, retry bounds, and diff --git a/docs/developer/event-bus-architecture.md b/docs/developer/event-bus-architecture.md index 66ad2fc0..8112ce2f 100644 --- a/docs/developer/event-bus-architecture.md +++ b/docs/developer/event-bus-architecture.md @@ -105,6 +105,30 @@ Each STIX document has two top-level keys: ### 4. Event Bus Messaging +The default `EventBus.emit()` method waits for every listener with +`Promise.allSettled()`, logs individual failures, and returns successful +listener values. It is appropriate when a listener is advisory or when the +caller has a separate recovery contract. + +Use `EventBus.emitRequired()` when listener-owned writes are part of the +caller's success contract. It still lets every listener finish, but rejects +when a listener fails or when fewer than the declared `minimumListeners` are +registered. The caller must make the failure durable when the originating +write has already been persisted. + +Release-track membership reconciliation is the first required-event workflow: + +1. Persist the snapshot mutation. +2. Create a pending `releaseTrackReconciliations` record. +3. Call `emitRequired()` for the attack-object and relationship backref + owners. +4. Mark the record completed, or mark it failed and return a structured + service error containing its reconciliation ID. + +See +[backref-reconciliation.md](release-tracks/backref-reconciliation.md) and the +[operator repair procedure](../admin/release-track-reconciliation.md). + **Event Naming Convention:** ``` @@ -160,7 +184,7 @@ Where `{type}` is the STIX type (e.g., `attack-pattern`, `x-mitre-analytic`, `x- | `x-mitre-detection-strategy::analytics-referenced` | DetectionStrategiesService | When detection strategy references analytics (create/update) | `{ detectionStrategyId, detectionStrategy, analyticIds }` | AnalyticsService | | `x-mitre-detection-strategy::analytics-removed` | DetectionStrategiesService | When analytics removed from detection strategy | `{ detectionStrategyId, analyticIds }` | AnalyticsService | | `x-mitre-analytic::parent-changed` | AnalyticsService | When analytic's parent detection strategy changes | `{ analyticId, oldParentId, newParentId, analytic }` | (Future: for cascading updates) | -| `release-track::contents-changed` | snapshot-service / versioning-service | After any persisted change to a track's latest snapshot (or track/snapshot deletion) | `{ trackId, snapshot }` (`snapshot` null when the track or its only snapshot was deleted) | AttackObjectsService, RelationshipsService (reconcile `workspace.release_tracks` backrefs; see [backref-reconciliation.md](release-tracks/backref-reconciliation.md)) | +| `release-track::contents-changed` | snapshot-service / versioning-service | After a durable reconciliation record is created for any persisted change to a track's latest snapshot (or track/snapshot deletion) | `{ trackId, snapshot, reconciliationId }` (`snapshot` null when the track or its only snapshot was deleted) | AttackObjectsService, RelationshipsService (required listeners that reconcile `workspace.release_tracks` backrefs; see [backref-reconciliation.md](release-tracks/backref-reconciliation.md)) | ## Workflow Examples diff --git a/docs/developer/release-tracks/backref-reconciliation.md b/docs/developer/release-tracks/backref-reconciliation.md index e3b427e3..6e2d3a9d 100644 --- a/docs/developer/release-tracks/backref-reconciliation.md +++ b/docs/developer/release-tracks/backref-reconciliation.md @@ -38,7 +38,10 @@ snapshot-service.deleteSnapshot │ bundle import, updateContents, ...) snapshot-service.deleteTrack │ versioning-service.releaseLatest/releaseByModified ┘ (staged → members via tagSnapshotInPlace) │ - ▼ awaited EventBus.emit release-track::contents-changed { trackId, snapshot } + ▼ persist releaseTrackReconciliations record (pending) + │ + ▼ awaited EventBus.emitRequired release-track::contents-changed + │ { trackId, snapshot, reconciliationId } │ snapshot = track's latest snapshot, │ or null when the track (or its only │ snapshot) was deleted @@ -63,8 +66,17 @@ can reference its ID yet. `releaseByModified` may tag an older snapshot; the rel path therefore re-reads the *latest* snapshot before emitting rather than using the tagged one. -Emissions are awaited (the request/response-blocking convention), so backrefs -are consistent by the time the triggering API call returns. +Emissions are awaited and required. The EventBus rejects when either owning +listener fails or is not registered. A successful response therefore means +both object collections were reconciled. A failure returns HTTP `500` with +the durable `reconciliation_id`; the release-track mutation may already be +persisted and must not be retried blindly. + +Every attempt is written to `releaseTrackReconciliations` before listeners +run. Records move through `pending`, `completed`, or `failed` and retain the +requested snapshot, attempt count, timestamps, and last error. If recording +completion fails after the listeners succeeded, the record remains pending; +replaying it is safe because reconciliation is idempotent. ## Reconciliation algorithm @@ -133,5 +145,37 @@ backref to the newly latest revision without rewriting the stored selector revision is later re-created, its backref is restored on the next contents-changed event for that track, not immediately. - **Historical snapshots.** Backrefs describe only the *latest* snapshot per - track. Membership in older snapshots remains discoverable only from the - track side. + track. Object mutation guards do not trust that derived view: they query + every registered track's tagged snapshots for the exact revision before an + in-place update or delete. Historical tagged membership therefore remains + immutable even after the latest draft removes the object or the registry's + tagged-release catalogue is stale. +- **Crash window before record creation.** Snapshot persistence and the + central reconciliation record are not in one MongoDB transaction. A hard + process failure in that narrow interval can leave no pending record. + Operators should run the full-scan repair after an unclean shutdown; it + compares every registered track and every track ID found in object + backrefs against current latest snapshots. + +## Repair + +Repair outstanding `pending`/`failed` attempts: + +```bash +npm run repair:release-track-backrefs +``` + +Limit one invocation with `--limit`: + +```bash +npm run repair:release-track-backrefs -- --limit=500 +``` + +Perform a full idempotent scan, including stale backrefs for deleted tracks: + +```bash +npm run repair:release-track-backrefs -- --all +``` + +The command exits nonzero if any track still fails and prints a JSON summary +with track and reconciliation identifiers. diff --git a/docs/user/release-tracks/object-backrefs.md b/docs/user/release-tracks/object-backrefs.md index ad7e0049..7d960a68 100644 --- a/docs/user/release-tracks/object-backrefs.md +++ b/docs/user/release-tracks/object-backrefs.md @@ -81,7 +81,11 @@ Release tracks are never blind to changes in the objects they pin: `409 Conflict` — released content cannot be changed or destroyed under the track. Make changes by creating a new revision (`POST`); retire an object by creating a new revision with `x_mitre_deprecated: true`. Revision sync - captures either one. + captures either one. This guard checks tagged snapshots authoritatively, not + only the current `workspace.release_tracks` value. A revision remains + protected when it belongs only to a historical tagged release, when a newer + draft has removed it, or when a reconciliation failure temporarily omitted + its backref. - **Candidate/staged-pinned revisions can be edited in place, but the track sees it.** An in-place `PUT` (including one that only sets `x_mitre_deprecated`) marks the pinned entry `modified-in-place`: the @@ -112,3 +116,12 @@ POST /api/release-tracks/:id/candidates/promote → { tier: "staged", sta POST /api/release-tracks/:id/snapshots/latest/release → { tier: "members", status: "reviewed" } DELETE /api/release-tracks/:id → entry removed ``` + +## Reconciliation failures + +Track mutations reconcile object backrefs before reporting success. If either +object collection cannot be updated, the API returns HTTP `500` with +`track_id` and `reconciliation_id`. The track mutation may already have been +persisted—including a release tag—so do not repeat it blindly. Give the +reconciliation ID to an administrator, who can inspect the durable failure +record and run the idempotent repair command. diff --git a/docs/user/release-tracks/release-workflow.md b/docs/user/release-tracks/release-workflow.md index acdefd8e..bbae7b7d 100644 --- a/docs/user/release-tracks/release-workflow.md +++ b/docs/user/release-tracks/release-workflow.md @@ -64,6 +64,13 @@ revision that no longer exists, the operation returns HTTP `409` with `missing_references` and does not tag the snapshot. This check protects both standard and virtual releases from publishing incomplete primary membership. +After tagging, the server reconciles the member protections stored on object +documents. A successful response means both object collections were updated. +HTTP `500` with a `reconciliation_id` means the release may already be tagged, +but one or more protection writes failed. Do not repeat the release request +without checking the selected snapshot first; an administrator can safely +replay the idempotent reconciliation using that durable record. + ### STIX Freeze Solution Version pinning solves the "STIX freeze" problem: diff --git a/package.json b/package.json index 1c1ae037..deeb337a 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "test:fuzz": "mocha --timeout 10000 --recursive ./app/tests/fuzz --exit", "test:scheduler": "mocha --timeout 60000 --recursive ./app/tests/scheduler --exit", "test:file": "mocha --timeout 10000 --exit", + "repair:release-track-backrefs": "node scripts/reconcileReleaseTrackBackrefs.js", "check:lockfile": "bash scripts/check-package-lock.sh" }, "dependencies": { diff --git a/scripts/README.md b/scripts/README.md index e4928f56..e874529b 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1 +1,22 @@ This directory holds utility scripts that are used for system configuration during software development. + +## Release-track backref repair + +`reconcileReleaseTrackBackrefs.js` repairs durable failed or pending +`workspace.release_tracks` reconciliation attempts: + +```bash +npm run repair:release-track-backrefs +npm run repair:release-track-backrefs -- --limit=500 +``` + +Use `--all` after an unclean shutdown or when legacy drift is suspected. It +reconciles all registry tracks and removes stale backrefs whose track no +longer exists: + +```bash +npm run repair:release-track-backrefs -- --all +``` + +The script requires the normal `DATABASE_URL`, prints a JSON result, and exits +nonzero when any repair still fails. diff --git a/scripts/reconcileReleaseTrackBackrefs.js b/scripts/reconcileReleaseTrackBackrefs.js new file mode 100644 index 00000000..e7e9d49d --- /dev/null +++ b/scripts/reconcileReleaseTrackBackrefs.js @@ -0,0 +1,63 @@ +#!/usr/bin/env node + +'use strict'; + +const mongoose = require('mongoose'); + +function parseOptions(argv) { + const all = argv.includes('--all'); + const limitArgument = argv.find((argument) => argument.startsWith('--limit=')); + const limit = limitArgument ? Number(limitArgument.split('=')[1]) : 100; + + if (!Number.isInteger(limit) || limit < 1 || limit > 10000) { + throw new Error('--limit must be an integer between 1 and 10000'); + } + + return { all, limit }; +} + +async function run() { + const options = parseOptions(process.argv.slice(2)); + await require('../app/lib/database-connection').initializeConnection(); + + // Loading the owning services registers both required reconciliation + // listeners before the repair dispatches any events. + require('../app/services/stix/attack-objects-service'); + require('../app/services/stix/relationships-service'); + const reconciliationService = require('../app/services/release-tracks/reconciliation-service'); + + const results = options.all + ? await reconciliationService.reconcileAll({ continueOnError: true }) + : await reconciliationService.repairOutstanding({ + limit: options.limit, + continueOnError: true, + }); + const failed = results.filter((result) => result.status === 'failed'); + + console.log( + JSON.stringify( + { + mode: options.all ? 'full_scan' : 'outstanding', + processed: results.length, + completed: results.length - failed.length, + failed: failed.length, + results, + }, + null, + 2, + ), + ); + + if (failed.length > 0) { + process.exitCode = 1; + } +} + +run() + .catch((error) => { + console.error(error); + process.exitCode = 1; + }) + .finally(async () => { + await mongoose.disconnect(); + }); From 256f86e18db5603fd32cbb258cbf0a0669ea6f2f Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:52:36 -0400 Subject: [PATCH 5/7] fix(release-tracks): protect destructive operations Require administrator authorization and exact track confirmation for member replacement and full-track deletion. Persist actor-attributed audit events before execution and expose durable identifiers when finalization fails. --- .../definitions/components/release-tracks.yml | 20 ++ .../paths/release-tracks-paths.yml | 58 ++++- app/controllers/release-tracks-controller.js | 34 ++- app/exceptions/index.js | 11 + app/lib/error-handler.js | 4 +- .../release-track-audit-event-model.js | 42 ++++ .../release-track-audit-event.repository.js | 75 ++++++ app/routes/release-tracks-routes.js | 4 +- .../destructive-audit-service.js | 45 ++++ .../release-tracks/release-tracks-service.js | 53 +++- .../destructive-authorization.spec.js | 226 ++++++++++++++++++ .../primary-revision-integrity.spec.js | 8 +- .../reconciliation-durability.spec.js | 2 +- .../release-tracks-backrefs.spec.js | 7 +- .../release-tracks-bundle.spec.js | 2 +- .../release-tracks-change-capture.spec.js | 2 +- .../release-tracks-release.spec.js | 16 +- .../release-tracks-tier-invariant.spec.js | 2 +- .../api/release-tracks/release-tracks.spec.js | 1 + .../release-tracks/releases-by-object.spec.js | 2 +- .../tagged-content-immutability.spec.js | 4 +- .../virtual-deduplication.spec.js | 2 +- .../virtual-determinism.spec.js | 2 +- .../virtual-domain-filters.spec.js | 2 +- .../virtual-object-type-filters.spec.js | 2 +- .../release-tracks/virtual-quarantine.spec.js | 2 +- app/tests/middleware/error-handler.spec.js | 25 ++ docs/README.md | 2 + docs/admin/release-track-audit.md | 50 ++++ docs/developer/FRONTEND_TODO.md | 27 +++ docs/developer/TODO.md | 26 +- .../developer/release-tracks/authorization.md | 41 ++++ docs/user/release-tracks/api-reference.md | 28 ++- 33 files changed, 778 insertions(+), 49 deletions(-) create mode 100644 app/models/release-tracks/release-track-audit-event-model.js create mode 100644 app/repository/release-tracks/release-track-audit-event.repository.js create mode 100644 app/services/release-tracks/destructive-audit-service.js create mode 100644 app/tests/api/release-tracks/destructive-authorization.spec.js create mode 100644 docs/admin/release-track-audit.md create mode 100644 docs/developer/release-tracks/authorization.md diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 1e5683a4..d1549685 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -727,3 +727,23 @@ components: type: string format: uuid description: 'Durable reconciliation record to inspect or repair' + + release-track-audit-error: + type: object + required: + - message + - track_id + - audit_event_id + properties: + message: + type: string + enum: + - 'Release-track audit recording could not be finalized' + details: + type: string + description: 'Operator guidance; the destructive operation may already be persisted' + track_id: + type: string + audit_event_id: + type: string + format: uuid diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 4ed5c2d3..94917925 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -283,7 +283,9 @@ paths: operationId: 'release-tracks-delete' description: | Delete an entire release track including all snapshots and version history. - This operation cannot be undone. + This administrator-only operation cannot be undone. The caller must + confirm the exact target with confirm_track_id. The server writes a + durable audit event before deletion begins. tags: - 'Release Tracks' parameters: @@ -293,11 +295,29 @@ paths: description: 'Release track ID' schema: type: string + - name: confirm_track_id + in: query + required: true + description: 'Must exactly equal the id path parameter' + schema: + type: string responses: '204': description: 'Release track deleted successfully' '404': description: 'Release track not found' + '400': + description: 'Missing or mismatched destructive confirmation' + '401': + description: 'Administrator role required' + '500': + description: 'The deletion may be persisted, but its durable audit or membership reconciliation failed' + content: + application/json: + schema: + oneOf: + - $ref: '../components/release-tracks.yml#/components/schemas/release-track-audit-error' + - $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' # ============================================================================= # Latest snapshot operations @@ -341,6 +361,8 @@ paths: Exact revisions already present in another tier are retained only in members; different revisions of the same object remain valid across tiers. Creates a new snapshot clone. + This administrator-only operation requires confirm_track_id to exactly + match the target track and writes a durable audit event. Request body validated via Zod in controller. tags: - 'Release Tracks' @@ -350,11 +372,27 @@ paths: required: true schema: type: string + - name: confirm_track_id + in: query + required: true + description: 'Must exactly equal the id path parameter' + schema: + type: string responses: '200': description: 'Contents updated successfully' '400': description: 'Track is virtual, the contents request is invalid, or a requested revision does not exist' + '401': + description: 'Administrator role required' + '500': + description: 'The replacement may be persisted, but its durable audit or membership reconciliation failed' + content: + application/json: + schema: + oneOf: + - $ref: '../components/release-tracks.yml#/components/schemas/release-track-audit-error' + - $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' /api/release-tracks/{id}/clone: post: @@ -1256,6 +1294,8 @@ paths: store exact revision pins. Exact revisions already present in another tier are retained only in members; different revisions of the same object remain valid across tiers. + This administrator-only operation requires confirm_track_id to exactly + match the target track and writes a durable audit event. Request body validated via Zod in controller. tags: - 'Release Tracks' @@ -1270,11 +1310,27 @@ paths: required: true schema: type: string + - name: confirm_track_id + in: query + required: true + description: 'Must exactly equal the id path parameter' + schema: + type: string responses: '200': description: 'Contents updated successfully' '400': description: 'Track is virtual, the contents request is invalid, or a requested revision does not exist' + '401': + description: 'Administrator role required' + '500': + description: 'The replacement may be persisted, but its durable audit or membership reconciliation failed' + content: + application/json: + schema: + oneOf: + - $ref: '../components/release-tracks.yml#/components/schemas/release-track-audit-error' + - $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' /api/release-tracks/{id}/snapshots/{modified}/clone: post: diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index 3bc462c5..fb61574b 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -79,6 +79,25 @@ function parseOptionalQueryStrict(value, schema, defaultValue, parameterName) { }); } +function requireDestructiveConfirmation(req) { + if (req.query.confirm_track_id !== req.params.id) { + throw new BadRequestError({ + message: 'Destructive release-track confirmation is required', + details: `Set confirm_track_id to the exact target track ID '${req.params.id}'.`, + parameter_name: 'confirm_track_id', + expected_track_id: req.params.id, + }); + } +} + +function destructiveActor(req) { + return { + user_account_id: req.user?.userAccountId, + role: req.user?.role, + authentication_strategy: req.user?.strategy, + }; +} + function rejectFilesystemStoreFormat(format, methodName) { if (format !== 'filesystemstore') return null; @@ -445,6 +464,7 @@ exports.updateMetadataByLatest = async function updateMetadataByLatest(req, res, /** POST /api/release-tracks/:id/contents */ exports.updateContentsByLatest = async function updateContentsByLatest(req, res, next) { try { + requireDestructiveConfirmation(req); const bodyResult = updateContentsBodySchema.safeParse(req.body); if (!bodyResult.success) { return next( @@ -458,7 +478,8 @@ exports.updateContentsByLatest = async function updateContentsByLatest(req, res, const result = await releaseTracksService.updateContents( req.params.id, bodyResult.data, - req.user?.userAccountId, + destructiveActor(req), + req.query.confirm_track_id, ); logger.debug(`Success: Updated contents for track ${req.params.id}`); return res.status(200).send(result); @@ -521,7 +542,12 @@ exports.cloneByLatest = async function cloneByLatest(req, res, next) { /** DELETE /api/release-tracks/:id */ exports.deleteReleaseTrack = async function deleteReleaseTrack(req, res, next) { try { - await releaseTracksService.deleteTrack(req.params.id); + requireDestructiveConfirmation(req); + await releaseTracksService.deleteTrack( + req.params.id, + destructiveActor(req), + req.query.confirm_track_id, + ); logger.debug(`Success: Deleted track ${req.params.id}`); return res.status(204).end(); } catch (err) { @@ -589,6 +615,7 @@ exports.updateMetadataByModified = async function updateMetadataByModified(req, /** POST /api/release-tracks/:id/snapshots/:modified/contents */ exports.updateContentsByModified = async function updateContentsByModified(req, res, next) { try { + requireDestructiveConfirmation(req); const bodyResult = updateContentsBodySchema.safeParse(req.body); if (!bodyResult.success) { return next( @@ -603,7 +630,8 @@ exports.updateContentsByModified = async function updateContentsByModified(req, req.params.id, req.params.modified, bodyResult.data, - req.user?.userAccountId, + destructiveActor(req), + req.query.confirm_track_id, ); logger.debug(`Success: Updated contents for snapshot ${req.params.modified}`); return res.status(200).send(result); diff --git a/app/exceptions/index.js b/app/exceptions/index.js index 8186a9ce..8be2df56 100644 --- a/app/exceptions/index.js +++ b/app/exceptions/index.js @@ -335,6 +335,16 @@ class ReleaseTrackReconciliationError extends CustomError { } } +class ReleaseTrackAuditError extends CustomError { + constructor(trackId, auditEventId, options = {}) { + super('Release-track audit recording could not be finalized', { + ...options, + track_id: trackId, + audit_event_id: auditEventId, + }); + } +} + class TaggedSnapshotDeletionError extends CustomError { constructor(version, options) { super(`Tagged snapshot version ${version} cannot be deleted`, options); @@ -433,6 +443,7 @@ module.exports = { ReleaseConflictError, ReleaseContentIntegrityError, ReleaseTrackReconciliationError, + ReleaseTrackAuditError, NoTaggedSnapshotsError, InvalidComponentTypeError, VirtualSnapshotNotMaterializedError, diff --git a/app/lib/error-handler.js b/app/lib/error-handler.js index 4e9d1b14..c187dc47 100644 --- a/app/lib/error-handler.js +++ b/app/lib/error-handler.js @@ -43,6 +43,7 @@ const { ReleaseConflictError, ReleaseContentIntegrityError, ReleaseTrackReconciliationError, + ReleaseTrackAuditError, NoTaggedSnapshotsError, InvalidComponentTypeError, VirtualSnapshotNotMaterializedError, @@ -157,7 +158,8 @@ exports.serviceExceptions = function (err, req, res, next) { err instanceof TacticsServiceError || err instanceof GenericServiceError || err instanceof DatabaseError || - err instanceof ReleaseTrackReconciliationError + err instanceof ReleaseTrackReconciliationError || + err instanceof ReleaseTrackAuditError ) { logger.error('Service error: %s', JSON.stringify(buildErrorResponse(err))); return res.status(500).send(buildErrorResponse(err)); diff --git a/app/models/release-tracks/release-track-audit-event-model.js b/app/models/release-tracks/release-track-audit-event-model.js new file mode 100644 index 00000000..ded1fa76 --- /dev/null +++ b/app/models/release-tracks/release-track-audit-event-model.js @@ -0,0 +1,42 @@ +'use strict'; + +const mongoose = require('mongoose'); +const { validateTrackId } = require('../../lib/release-tracks/release-track-validators'); + +const releaseTrackAuditEventSchema = new mongoose.Schema( + { + event_id: { type: String, required: true, unique: true }, + action: { + type: String, + required: true, + enum: ['replace_members_latest', 'replace_members_historical', 'delete_track'], + }, + track_id: { type: String, required: true, validate: validateTrackId }, + status: { + type: String, + required: true, + enum: ['pending', 'completed', 'failed'], + default: 'pending', + }, + actor: { type: mongoose.Schema.Types.Mixed, required: true }, + confirmation: { type: String, required: true }, + request: { type: mongoose.Schema.Types.Mixed, default: {} }, + result: { type: mongoose.Schema.Types.Mixed, default: null }, + error: { + name: String, + message: String, + }, + started_at: { type: Date, required: true }, + finished_at: { type: Date, default: null }, + }, + { + collection: 'releaseTrackAuditEvents', + bufferCommands: false, + }, +); + +releaseTrackAuditEventSchema.index({ track_id: 1, started_at: -1 }); +releaseTrackAuditEventSchema.index({ action: 1, started_at: -1 }); +releaseTrackAuditEventSchema.index({ 'actor.user_account_id': 1, started_at: -1 }); + +module.exports = mongoose.model('ReleaseTrackAuditEvent', releaseTrackAuditEventSchema); diff --git a/app/repository/release-tracks/release-track-audit-event.repository.js b/app/repository/release-tracks/release-track-audit-event.repository.js new file mode 100644 index 00000000..884e1679 --- /dev/null +++ b/app/repository/release-tracks/release-track-audit-event.repository.js @@ -0,0 +1,75 @@ +'use strict'; + +const { v4: uuidv4 } = require('uuid'); +const ReleaseTrackAuditEvent = require('../../models/release-tracks/release-track-audit-event-model'); +const { DatabaseError } = require('../../exceptions'); + +class ReleaseTrackAuditEventRepository { + async create({ action, trackId, actor, confirmation, request }) { + try { + const event = await ReleaseTrackAuditEvent.create({ + event_id: uuidv4(), + action, + track_id: trackId, + status: 'pending', + actor, + confirmation, + request, + started_at: new Date(), + }); + return event.toObject(); + } catch (error) { + throw new DatabaseError(error); + } + } + + async complete(eventId, result) { + try { + const event = await ReleaseTrackAuditEvent.findOneAndUpdate( + { event_id: eventId }, + { + $set: { + status: 'completed', + result: result || null, + error: null, + finished_at: new Date(), + }, + }, + { new: true, lean: true }, + ).exec(); + if (!event) { + throw new Error(`Release-track audit event ${eventId} no longer exists`); + } + return event; + } catch (error) { + throw new DatabaseError(error); + } + } + + async fail(eventId, error) { + try { + const event = await ReleaseTrackAuditEvent.findOneAndUpdate( + { event_id: eventId }, + { + $set: { + status: 'failed', + error: { + name: error?.name || 'Error', + message: error?.message || String(error), + }, + finished_at: new Date(), + }, + }, + { new: true, lean: true }, + ).exec(); + if (!event) { + throw new Error(`Release-track audit event ${eventId} no longer exists`); + } + return event; + } catch (repositoryError) { + throw new DatabaseError(repositoryError); + } + } +} + +module.exports = new ReleaseTrackAuditEventRepository(); diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index 263caf51..40838dc7 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -263,7 +263,7 @@ router .route('/release-tracks/:id/snapshots/:modified/contents') .post( authn.authenticate, - authz.requireRole(authz.editorOrHigher), + authz.requireRole(authz.admin), releaseTracksController.updateContentsByModified, ); @@ -324,7 +324,7 @@ router .route('/release-tracks/:id') .delete( authn.authenticate, - authz.requireRole(authz.editorOrHigher), + authz.requireRole(authz.admin), releaseTracksController.deleteReleaseTrack, ); diff --git a/app/services/release-tracks/destructive-audit-service.js b/app/services/release-tracks/destructive-audit-service.js new file mode 100644 index 00000000..1b399951 --- /dev/null +++ b/app/services/release-tracks/destructive-audit-service.js @@ -0,0 +1,45 @@ +'use strict'; + +const logger = require('../../lib/logger'); +const auditRepo = require('../../repository/release-tracks/release-track-audit-event.repository'); +const { ReleaseTrackAuditError } = require('../../exceptions'); + +function snapshotResult(snapshot) { + if (!snapshot) return null; + return { + snapshot_modified: snapshot.modified, + version: snapshot.version ?? null, + members_count: snapshot.members?.length || 0, + }; +} + +exports.execute = async function execute(options, operation) { + const event = await auditRepo.create(options); + let operationCompleted = false; + + try { + const result = await operation(); + operationCompleted = true; + await auditRepo.complete(event.event_id, options.result?.(result) ?? snapshotResult(result)); + return result; + } catch (error) { + if (!operationCompleted) { + try { + await auditRepo.fail(event.event_id, error); + } catch (auditError) { + logger.error( + `DestructiveAuditService: Failed to record ${event.event_id} failure: ` + + auditError.message, + ); + } + throw error; + } + + throw new ReleaseTrackAuditError(options.trackId, event.event_id, { + details: + 'The destructive release-track operation completed, but its audit record could not be ' + + 'finalized. Inspect the track and audit event before retrying.', + cause: error, + }); + } +}; diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index aa2bc8de..7a2ebab0 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -29,6 +29,7 @@ const ephemeralService = require('./ephemeral-service'); const bundleImportService = require('./bundle-import-service'); const memberSyncService = require('./member-sync-service'); const releaseHistoryService = require('./release-history-service'); +const destructiveAuditService = require('./destructive-audit-service'); const attackObjectsService = require('../stix/attack-objects-service'); const userAccountsService = require('../system/user-accounts-service'); const revisionReference = require('../../lib/release-tracks/revision-reference'); @@ -40,6 +41,16 @@ function notImplemented(methodName) { throw new NotImplementedError(MODULE, methodName); } +function destructiveIdentity(trackId, actor, confirmation) { + return { + actor: actor || { + kind: 'system', + name: 'internal-service', + }, + confirmation: confirmation || trackId, + }; +} + function rejectFilesystemStoreFormat(format, methodName) { if (format !== 'filesystemstore') return; @@ -287,17 +298,38 @@ exports.updateMetadataByModified = function updateMetadataByModified( return snapshotService.updateMetadataByModified(trackId, modified, updates, userId); }; -exports.updateContents = function updateContents(trackId, contents, userId) { - return snapshotService.updateContents(trackId, contents, userId); +exports.updateContents = function updateContents(trackId, contents, actor, confirmation) { + return destructiveAuditService.execute( + { + action: 'replace_members_latest', + trackId, + ...destructiveIdentity(trackId, actor, confirmation), + request: { members_count: contents.x_mitre_contents.length }, + }, + () => snapshotService.updateContents(trackId, contents, actor?.user_account_id), + ); }; exports.updateContentsByModified = function updateContentsByModified( trackId, modified, contents, - userId, + actor, + confirmation, ) { - return snapshotService.updateContentsByModified(trackId, modified, contents, userId); + return destructiveAuditService.execute( + { + action: 'replace_members_historical', + trackId, + ...destructiveIdentity(trackId, actor, confirmation), + request: { + source_snapshot_modified: modified, + members_count: contents.x_mitre_contents.length, + }, + }, + () => + snapshotService.updateContentsByModified(trackId, modified, contents, actor?.user_account_id), + ); }; exports.cloneTrack = function cloneTrack(trackId, options) { @@ -308,8 +340,17 @@ exports.cloneFromSnapshot = function cloneFromSnapshot(trackId, modified, option return snapshotService.cloneFromSnapshot(trackId, modified, options); }; -exports.deleteTrack = function deleteTrack(trackId) { - return snapshotService.deleteTrack(trackId); +exports.deleteTrack = function deleteTrack(trackId, actor, confirmation) { + return destructiveAuditService.execute( + { + action: 'delete_track', + trackId, + ...destructiveIdentity(trackId, actor, confirmation), + request: {}, + result: () => ({ deleted: true }), + }, + () => snapshotService.deleteTrack(trackId), + ); }; exports.deleteSnapshot = function deleteSnapshot(trackId, modified) { diff --git a/app/tests/api/release-tracks/destructive-authorization.spec.js b/app/tests/api/release-tracks/destructive-authorization.spec.js new file mode 100644 index 00000000..8dd05d0e --- /dev/null +++ b/app/tests/api/release-tracks/destructive-authorization.spec.js @@ -0,0 +1,226 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); +const sinon = require('sinon'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const UserAccount = require('../../../models/user-account-model'); +const ReleaseTrackAuditEvent = require('../../../models/release-tracks/release-track-audit-event-model'); +const auditRepository = require('../../../repository/release-tracks/release-track-audit-event.repository'); +const systemConfigurationService = require('../../../services/system/system-configuration-service'); + +const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; + +function techniquePayload() { + const timestamp = new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + name: 'Destructive authorization member', + description: 'Member used by destructive authorization tests.', + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [markingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }; +} + +describe('Release-track destructive authorization and audit', function () { + let app; + let passportCookie; + let anonymousUser; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + anonymousUser = await systemConfigurationService.retrieveAnonymousUserAccount(); + }); + + after(async function () { + sinon.restore(); + await UserAccount.updateOne({ id: anonymousUser.id }, { $set: { role: 'admin' } }); + await database.closeConnection(); + }); + + afterEach(function () { + sinon.restore(); + }); + + async function setRole(role) { + await UserAccount.updateOne({ id: anonymousUser.id }, { $set: { role } }); + } + + function api(method, path, body, status, query) { + const call = request(app) + [method](path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + if (query) call.query(query); + if (body !== undefined) call.send(body); + return call.expect(status); + } + + async function post(path, body, status = 200, query) { + return (await api('post', path, body, status, query)).body; + } + + it('requires admin role, exact confirmation, and durable outcome records', async function () { + await setRole('admin'); + const technique = await post('/api/techniques', techniquePayload(), 201); + const track = await post( + '/api/release-tracks/new', + { name: 'Destructive authorization standard', type: 'standard' }, + 201, + ); + const contents = { + x_mitre_contents: [ + { + obj_ref: technique.stix.id, + obj_modified: technique.stix.modified, + }, + ], + }; + + await setRole('editor'); + await api('post', `/api/release-tracks/${track.id}/contents`, contents, 401, { + confirm_track_id: track.id, + }); + await api( + 'post', + `/api/release-tracks/${track.id}/snapshots/${track.modified}/contents`, + contents, + 401, + { confirm_track_id: track.id }, + ); + await api('delete', `/api/release-tracks/${track.id}`, undefined, 401, { + confirm_track_id: track.id, + }); + expect(await ReleaseTrackAuditEvent.countDocuments()).toBe(0); + + await setRole('admin'); + await api('post', `/api/release-tracks/${track.id}/contents`, contents, 400); + await api('post', `/api/release-tracks/${track.id}/contents`, contents, 400, { + confirm_track_id: 'release-track--00000000-0000-4000-8000-000000000099', + }); + await api('delete', `/api/release-tracks/${track.id}`, undefined, 400); + expect(await ReleaseTrackAuditEvent.countDocuments()).toBe(0); + + const latest = await post(`/api/release-tracks/${track.id}/contents`, contents, 200, { + confirm_track_id: track.id, + }); + await post( + `/api/release-tracks/${track.id}/snapshots/${track.modified}/contents`, + contents, + 200, + { confirm_track_id: track.id }, + ); + + const virtual = await post( + '/api/release-tracks/new', + { name: 'Destructive authorization virtual', type: 'virtual' }, + 201, + ); + await api('post', `/api/release-tracks/${virtual.id}/contents`, contents, 400, { + confirm_track_id: virtual.id, + }); + + await api('delete', `/api/release-tracks/${track.id}`, undefined, 204, { + confirm_track_id: track.id, + }); + + const events = await ReleaseTrackAuditEvent.find().sort({ started_at: 1 }).lean().exec(); + expect(events).toHaveLength(4); + expect(events.map((event) => [event.action, event.status])).toEqual([ + ['replace_members_latest', 'completed'], + ['replace_members_historical', 'completed'], + ['replace_members_latest', 'failed'], + ['delete_track', 'completed'], + ]); + expect(events[0]).toMatchObject({ + track_id: track.id, + confirmation: track.id, + actor: { + user_account_id: anonymousUser.id, + role: 'admin', + authentication_strategy: 'anonymId', + }, + request: { members_count: 1 }, + result: { + snapshot_modified: new Date(latest.modified), + members_count: 1, + }, + }); + expect(events[2].track_id).toBe(virtual.id); + expect(events[2].error.message).toContain( + 'Direct contents updates are only available for standard release tracks', + ); + expect(events[3].result).toEqual({ deleted: true }); + }); + + it('reports an audit-finalization failure without hiding the persisted mutation', async function () { + await setRole('admin'); + const technique = await post('/api/techniques', techniquePayload(), 201); + const track = await post( + '/api/release-tracks/new', + { name: 'Audit finalization failure standard', type: 'standard' }, + 201, + ); + const contents = { + x_mitre_contents: [ + { + obj_ref: technique.stix.id, + obj_modified: technique.stix.modified, + }, + ], + }; + + sinon.stub(auditRepository, 'complete').rejects(new Error('injected audit update failure')); + const response = await api('post', `/api/release-tracks/${track.id}/contents`, contents, 500, { + confirm_track_id: track.id, + }); + auditRepository.complete.restore(); + + expect(response.body).toMatchObject({ + message: 'Release-track audit recording could not be finalized', + track_id: track.id, + }); + expect(response.body.audit_event_id).toEqual(expect.any(String)); + + const latest = await api( + 'get', + `/api/release-tracks/${track.id}/snapshots/latest`, + undefined, + 200, + ); + expect(latest.body.members).toHaveLength(1); + expect(latest.body.members[0]).toMatchObject({ + object_ref: technique.stix.id, + object_modified: technique.stix.modified, + }); + + const pendingEvent = await ReleaseTrackAuditEvent.findOne({ + event_id: response.body.audit_event_id, + }) + .lean() + .exec(); + expect(pendingEvent).toMatchObject({ + action: 'replace_members_latest', + track_id: track.id, + status: 'pending', + }); + expect(pendingEvent.finished_at).toBeNull(); + }); +}); diff --git a/app/tests/api/release-tracks/primary-revision-integrity.spec.js b/app/tests/api/release-tracks/primary-revision-integrity.spec.js index 2b32c0a4..81c53614 100644 --- a/app/tests/api/release-tracks/primary-revision-integrity.spec.js +++ b/app/tests/api/release-tracks/primary-revision-integrity.spec.js @@ -145,7 +145,7 @@ describe('Release-track primary revision integrity API', function () { const response = await api( 'post', - `/api/release-tracks/${track.id}/contents`, + `/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { x_mitre_contents: [ { obj_ref: technique.stix.id, obj_modified: technique.stix.modified }, @@ -199,7 +199,7 @@ describe('Release-track primary revision integrity API', function () { it('rejects cloning and export when a stored primary member is missing', async function () { const technique = await createTechnique('Missing Stored Member'); const track = await createTrack('Missing Stored Member Track'); - await post(`/api/release-tracks/${track.id}/contents`, { + await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], }); await deleteTechniqueRevision(technique); @@ -233,7 +233,7 @@ describe('Release-track primary revision integrity API', function () { it('propagates repository hydration failures instead of returning a partial export', async function () { const technique = await createTechnique('Failed Primary Hydration'); const track = await createTrack('Failed Primary Hydration Track'); - await post(`/api/release-tracks/${track.id}/contents`, { + await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], }); const hydrationStub = sinon @@ -255,7 +255,7 @@ describe('Release-track primary revision integrity API', function () { it('aborts virtual materialization when a component member is missing', async function () { const technique = await createTechnique('Missing Virtual Component Member'); const component = await createTrack('Missing Virtual Component'); - await post(`/api/release-tracks/${component.id}/contents`, { + await post(`/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], }); await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, { diff --git a/app/tests/api/release-tracks/reconciliation-durability.spec.js b/app/tests/api/release-tracks/reconciliation-durability.spec.js index 2468b472..52e0792e 100644 --- a/app/tests/api/release-tracks/reconciliation-durability.spec.js +++ b/app/tests/api/release-tracks/reconciliation-durability.spec.js @@ -172,7 +172,7 @@ describe('Release-track durable backref reconciliation', function () { { name: 'Full Scan Repair Track', type: 'standard' }, 201, ); - await post(`/api/release-tracks/${track.id}/contents`, { + await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], }); diff --git a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js index e862a3a3..20361bf8 100644 --- a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js +++ b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js @@ -196,6 +196,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { it('deleting the track removes its backrefs', async function () { await request(app) .delete(`/api/release-tracks/${trackId}`) + .query({ confirm_track_id: trackId }) .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(204); @@ -301,7 +302,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { const trackId = await createTrack('Backref Contents Track'); const contentsSnapshot = await postObject( - `/api/release-tracks/${trackId}/contents`, + `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, { x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], }, @@ -364,7 +365,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { const trackId = await createTrack('Backref Member Sync Track'); await postObject( - `/api/release-tracks/${trackId}/contents`, + `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, { x_mitre_contents: [{ obj_ref: revisionA.stix.id, obj_modified: revisionA.stix.modified }], }, @@ -469,7 +470,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { ); const trackId = await createTrack('Backref Dynamic Ignore Track'); await postObject( - `/api/release-tracks/${trackId}/contents`, + `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, { x_mitre_contents: [{ obj_ref: revisionA.stix.id, obj_modified: revisionA.stix.modified }], }, diff --git a/app/tests/api/release-tracks/release-tracks-bundle.spec.js b/app/tests/api/release-tracks/release-tracks-bundle.spec.js index 8aca6aae..f95977a1 100644 --- a/app/tests/api/release-tracks/release-tracks-bundle.spec.js +++ b/app/tests/api/release-tracks/release-tracks-bundle.spec.js @@ -217,7 +217,7 @@ describe('Release Tracks Bundle Export API', function () { .expect(200); // Members - await postAction(`/api/release-tracks/${trackId}/contents`, { + await postAction(`/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, { x_mitre_contents: [ { obj_ref: memberObject.stix.id, obj_modified: memberObject.stix.modified }, { obj_ref: linkedMemberObject.stix.id, obj_modified: linkedMemberObject.stix.modified }, diff --git a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js index f1c42d55..1f313696 100644 --- a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js +++ b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js @@ -101,7 +101,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { async function setMembers(trackId, technique) { return postObject( - `/api/release-tracks/${trackId}/contents`, + `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, { x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], }, diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js index 72f862dc..5a1c7843 100644 --- a/app/tests/api/release-tracks/release-tracks-release.spec.js +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -363,7 +363,7 @@ describe('Release-track release planning and commit API', function () { it('records immutable component versions when previewing and releasing a virtual draft', async function () { const member = (await post('/api/techniques', buildTechnique('Provenance Member'), 201)).body; const component = await createTrack('Provenance Component'); - await post(`/api/release-tracks/${component.id}/contents`, { + await post(`/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { x_mitre_contents: [{ obj_ref: member.stix.id, obj_modified: member.stix.modified }], }); const firstComponentRelease = await post( @@ -401,7 +401,7 @@ describe('Release-track release planning and commit API', function () { // Advance the component after materialization. Virtual release provenance // must remain tied to the frozen component resolution, not current state. - await post(`/api/release-tracks/${component.id}/contents`, { + await post(`/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { x_mitre_contents: [{ obj_ref: member.stix.id, obj_modified: member.stix.modified }], }); const secondComponentRelease = await post( @@ -696,7 +696,7 @@ describe('Release-track release planning and commit API', function () { await post('/api/techniques', buildTechnique('Virtual Materialization Member'), 201) ).body; const component = await createTrack('Virtual Materialization Component'); - await post(`/api/release-tracks/${component.id}/contents`, { + await post(`/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { x_mitre_contents: [{ obj_ref: member.stix.id, obj_modified: member.stix.modified }], }); await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, {}); @@ -769,9 +769,13 @@ describe('Release-track release planning and commit API', function () { ], }; - await post(`/api/release-tracks/${virtual.id}/contents`, contents, 400); await post( - `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent(virtual.modified)}/contents`, + `/api/release-tracks/${virtual.id}/contents?confirm_track_id=${virtual.id}`, + contents, + 400, + ); + await post( + `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent(virtual.modified)}/contents?confirm_track_id=${virtual.id}`, contents, 400, ); @@ -788,7 +792,7 @@ describe('Release-track release planning and commit API', function () { await post('/api/techniques', buildTechnique('Release Conflict B', revisionA), 201) ).body; const track = await createTrack('Release Conflict'); - await post(`/api/release-tracks/${track.id}/contents`, { + await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { x_mitre_contents: [{ obj_ref: revisionA.stix.id, obj_modified: revisionA.stix.modified }], }); await post(`/api/release-tracks/${track.id}/candidates`, { diff --git a/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js b/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js index 8cbbeb03..5096b577 100644 --- a/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js +++ b/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js @@ -125,7 +125,7 @@ describe('Release-track cross-tier revision uniqueness', function () { } async function setMembers(trackId, objects) { - return post(`/api/release-tracks/${trackId}/contents`, { + return post(`/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, { x_mitre_contents: objects.map((object) => ({ obj_ref: object.stix.id, obj_modified: object.stix.modified, diff --git a/app/tests/api/release-tracks/release-tracks.spec.js b/app/tests/api/release-tracks/release-tracks.spec.js index 45691556..cf46f412 100644 --- a/app/tests/api/release-tracks/release-tracks.spec.js +++ b/app/tests/api/release-tracks/release-tracks.spec.js @@ -109,6 +109,7 @@ describe('Release Tracks API', function () { await request(app) .post(`/api/release-tracks/${trackId}/contents`) + .query({ confirm_track_id: trackId }) .send({ x_mitre_contents: [ { diff --git a/app/tests/api/release-tracks/releases-by-object.spec.js b/app/tests/api/release-tracks/releases-by-object.spec.js index f152f89b..a89e59ca 100644 --- a/app/tests/api/release-tracks/releases-by-object.spec.js +++ b/app/tests/api/release-tracks/releases-by-object.spec.js @@ -152,7 +152,7 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { async function setMembers(trackId, objects) { return post( - `/api/release-tracks/${trackId}/contents`, + `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, { x_mitre_contents: objects.map((object) => ({ obj_ref: object.stix.id, diff --git a/app/tests/api/release-tracks/tagged-content-immutability.spec.js b/app/tests/api/release-tracks/tagged-content-immutability.spec.js index e0c03567..d72c675a 100644 --- a/app/tests/api/release-tracks/tagged-content-immutability.spec.js +++ b/app/tests/api/release-tracks/tagged-content-immutability.spec.js @@ -71,7 +71,7 @@ describe('Release-track authoritative tagged-content immutability', function () { name: 'Historical Immutability', type: 'standard' }, 201, ); - await post(`/api/release-tracks/${track.id}/contents`, { + await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], }); await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, { version: '1.0' }); @@ -79,7 +79,7 @@ describe('Release-track authoritative tagged-content immutability', function () // A newer draft removes the member, so latest-snapshot reconciliation // deliberately removes the object's denormalized backref. The historical // tagged snapshot remains the immutable authority. - await post(`/api/release-tracks/${track.id}/contents`, { + await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { x_mitre_contents: [{ obj_ref: replacement.stix.id, obj_modified: replacement.stix.modified }], }); const current = ( diff --git a/app/tests/api/release-tracks/virtual-deduplication.spec.js b/app/tests/api/release-tracks/virtual-deduplication.spec.js index 8950a190..b35b9d27 100644 --- a/app/tests/api/release-tracks/virtual-deduplication.spec.js +++ b/app/tests/api/release-tracks/virtual-deduplication.spec.js @@ -89,7 +89,7 @@ describe('Virtual release-track deduplication API', function () { async function createReleasedComponent(name, members) { const track = await post('/api/release-tracks/new', { name, type: 'standard' }); await post( - `/api/release-tracks/${track.id}/contents`, + `/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { x_mitre_contents: members.map((member) => ({ obj_ref: member.stix.id, diff --git a/app/tests/api/release-tracks/virtual-determinism.spec.js b/app/tests/api/release-tracks/virtual-determinism.spec.js index f4f70380..a58d028b 100644 --- a/app/tests/api/release-tracks/virtual-determinism.spec.js +++ b/app/tests/api/release-tracks/virtual-determinism.spec.js @@ -82,7 +82,7 @@ describe('Virtual release-track deterministic membership API', function () { type: 'standard', }); const contents = await post( - `/api/release-tracks/${component.id}/contents`, + `/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { x_mitre_contents: [ { diff --git a/app/tests/api/release-tracks/virtual-domain-filters.spec.js b/app/tests/api/release-tracks/virtual-domain-filters.spec.js index f158ee29..ae38757d 100644 --- a/app/tests/api/release-tracks/virtual-domain-filters.spec.js +++ b/app/tests/api/release-tracks/virtual-domain-filters.spec.js @@ -114,7 +114,7 @@ describe('Virtual Release Track Domain Filters API', function () { type: 'standard', }); await post( - `/api/release-tracks/${component.id}/contents`, + `/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { x_mitre_contents: [enterprise, ics, shared, noDomain, enterpriseMatrix].map((object) => ({ obj_ref: object.stix.id, diff --git a/app/tests/api/release-tracks/virtual-object-type-filters.spec.js b/app/tests/api/release-tracks/virtual-object-type-filters.spec.js index fb18ac67..c580c436 100644 --- a/app/tests/api/release-tracks/virtual-object-type-filters.spec.js +++ b/app/tests/api/release-tracks/virtual-object-type-filters.spec.js @@ -203,7 +203,7 @@ describe('Virtual release-track object-type filters API', function () { const matrix = await post('/api/matrices', buildMatrix('Excluded Type Member')); await post( - `/api/release-tracks/${componentTrack.id}/contents`, + `/api/release-tracks/${componentTrack.id}/contents?confirm_track_id=${componentTrack.id}`, { x_mitre_contents: [mitigation, matrix].map((object) => ({ obj_ref: object.stix.id, diff --git a/app/tests/api/release-tracks/virtual-quarantine.spec.js b/app/tests/api/release-tracks/virtual-quarantine.spec.js index 34f91d15..d2e0a24d 100644 --- a/app/tests/api/release-tracks/virtual-quarantine.spec.js +++ b/app/tests/api/release-tracks/virtual-quarantine.spec.js @@ -73,7 +73,7 @@ describe('Virtual release-track quarantine API', function () { async function createReleasedComponent(name, member) { const track = await createTrack(name); - await post(`/api/release-tracks/${track.id}/contents`, { + await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { x_mitre_contents: [ { obj_ref: member.stix.id, diff --git a/app/tests/middleware/error-handler.spec.js b/app/tests/middleware/error-handler.spec.js index 05549787..cd139b5b 100644 --- a/app/tests/middleware/error-handler.spec.js +++ b/app/tests/middleware/error-handler.spec.js @@ -13,6 +13,7 @@ const { InvalidPostOperationError, ReleaseContentIntegrityError, ReleaseTrackReconciliationError, + ReleaseTrackAuditError, } = require('../../exceptions'); describe('error-handler middleware', function () { @@ -200,4 +201,28 @@ describe('error-handler middleware', function () { ).toBe(true); expect(next.called).toBe(false); }); + + it('should return durable audit identifiers when finalization fails', function () { + const err = new ReleaseTrackAuditError('release-track--track', 'audit-id', { + details: 'Inspect the operation.', + }); + const res = { + status: sinon.stub().returnsThis(), + send: sinon.stub().returnsThis(), + }; + const next = sinon.stub(); + + errorHandler.serviceExceptions(err, {}, res, next); + + expect(res.status.calledOnceWithExactly(500)).toBe(true); + expect( + res.send.calledOnceWithExactly({ + message: 'Release-track audit recording could not be finalized', + details: 'Inspect the operation.', + track_id: 'release-track--track', + audit_event_id: 'audit-id', + }), + ).toBe(true); + expect(next.called).toBe(false); + }); }); diff --git a/docs/README.md b/docs/README.md index 65a8e53f..b4e1ee03 100644 --- a/docs/README.md +++ b/docs/README.md @@ -50,6 +50,7 @@ Architecture, patterns, and implementation details for contributors. - [Error Handling](developer/release-tracks/error-handling.md): Error handling patterns - [Implementation Notes](developer/release-tracks/implementation-notes.md): Implementation notes and decisions - [Releases By Object](developer/release-tracks/releases-by-object.md): Registry catalogue, fan-out query, and indexing design +- [Authorization](developer/release-tracks/authorization.md): Role matrix, destructive confirmation, and audit contract ## Admin Documentation @@ -59,6 +60,7 @@ Configuration, deployment, and identity provider setup. - [Automation Run Audit Trail](admin/automation-runs.md): How to inspect migration and scheduler audit records - [Virtual Track Schedules](admin/virtual-track-schedules.md): UTC execution, restart recovery, retries, and observability - [Release-Track Membership Reconciliation](admin/release-track-reconciliation.md): Inspect and repair durable object-backref protection failures +- [Release-Track Destructive Audit Events](admin/release-track-audit.md): Inspect administrator member replacements and track deletions ### Authentication diff --git a/docs/admin/release-track-audit.md b/docs/admin/release-track-audit.md new file mode 100644 index 00000000..548e7a68 --- /dev/null +++ b/docs/admin/release-track-audit.md @@ -0,0 +1,50 @@ +# Release-Track Destructive Audit Events + +Workbench stores administrator-initiated member replacement and full-track +deletion attempts in `releaseTrackAuditEvents`. + +Each record contains: + +- `event_id`, `action`, and `track_id` +- the authenticated `actor` +- the exact `confirmation` supplied by the caller +- a bounded request/result summary +- `pending`, `completed`, or `failed` status +- start/finish timestamps and failure detail + +Inspect recent events: + +```javascript +db.releaseTrackAuditEvents.find().sort({ started_at: -1 }).limit(50).pretty(); +``` + +Inspect destructive actions for one track: + +```javascript +db.releaseTrackAuditEvents + .find({ + track_id: 'release-track--...', + }) + .sort({ started_at: -1 }) + .pretty(); +``` + +Inspect incomplete or failed attempts: + +```javascript +db.releaseTrackAuditEvents + .find({ + status: { $in: ['pending', 'failed'] }, + }) + .sort({ started_at: 1 }) + .pretty(); +``` + +A `pending` event can mean the process stopped after the audit insert or the +operation completed but the final audit update failed. Inspect the target +track before retrying. A failed member replacement may also have persisted a +new snapshot if backref reconciliation subsequently failed; correlate its +timestamp with `releaseTrackReconciliations`. + +These records have no automatic TTL. Establish retention and archive policy +according to local audit requirements. diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index 0bfed042..c46b2791 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -145,6 +145,33 @@ Done when: ## P0 — Align the Angular connector with the current routes +### [ ] Add administrator confirmation for destructive release-track actions + +Full track deletion and both direct standard-track member replacement routes +are administrator-only. They now require the query parameter +`confirm_track_id` to exactly equal the `:id` path parameter: + +```text +DELETE /api/release-tracks/:id?confirm_track_id=:id +POST /api/release-tracks/:id/contents?confirm_track_id=:id +POST /api/release-tracks/:id/snapshots/:modified/contents?confirm_track_id=:id +``` + +Do not expose these actions to editors or team leads. Before sending a request, +show the track name and ID, explain that direct replacement bypasses the normal +candidate/staged workflow or that deletion removes all history, and require an +explicit confirmation interaction. A missing or stale ID returns `400`; a +non-administrator returns `401`. + +Done when: + +- Route guards and action visibility match the documented authorization + matrix. +- The connector sends the selected track's exact ID as `confirm_track_id`. +- Dialogs cannot reuse confirmation state after the selected track changes. +- Tests cover administrator success plus editor, missing-confirmation, and + mismatched-confirmation rejection. + ### [ ] Use only the explicit snapshot-retrieval endpoints The release-track resource path no longer doubles as an implicit request for diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 0c0995e4..efb91503 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -94,7 +94,31 @@ Verification result (2026-07-30): minutes to roughly one minute; the remaining unrelated transport flake is tracked separately from this completed integrity change. -- [ ] P0.4 — Correct destructive authorization and add durable audit records. +### P0.4 — Correct destructive authorization and add durable audit records + +- [x] Require administrator authorization for full track deletion and both + direct member-replacement routes. +- [x] Require an exact `confirm_track_id` precondition on each destructive + request so stale or accidental UI actions fail before persistence. +- [x] Persist a durable, actor-attributed audit event before each operation + and record completion or failure without hiding partial persistence. +- [x] Add an authorization matrix and operator-facing audit documentation. +- [x] Update OpenAPI, frontend tasks, and Bruno requests for the confirmation + contract. +- [x] Add admin/editor, missing/mismatched confirmation, success/failure + audit, lint, OpenAPI, focused, and complete-suite verification. + +Verification result (2026-07-30): + +- Lint and OpenAPI validation pass. +- The focused authorization/audit and middleware group passes (11). The + complete release-track and virtual-scheduler group passes all 148 relevant + cases; one roaming setup 404 passed immediately in isolation (8). +- The required clean full suite passes: OpenAPI 2, config 21, API 960, + middleware 29, and scheduler 10. +- The `internalattack` focused release-track suite passes (33), its complete + suite passes (246), and changed-file Ruff checks pass. + - [ ] P0.5 — Complete the Angular contract migration and end-to-end smoke gate. - [ ] P0.6 — Finish scheduled-materialization fencing, retry bounds, and operator intervention. diff --git a/docs/developer/release-tracks/authorization.md b/docs/developer/release-tracks/authorization.md new file mode 100644 index 00000000..11fec021 --- /dev/null +++ b/docs/developer/release-tracks/authorization.md @@ -0,0 +1,41 @@ +# Release-Track Authorization + +Release-track access follows the existing Workbench roles. Read operations are +available to visitors and higher. Normal draft workflow operations require an +editor, team lead, or administrator. Operations that can replace authoritative +membership or destroy history require an administrator. + +## Authorization matrix + +| Capability | Visitor | Editor / team lead | Administrator | +| --------------------------------------------------------------------- | ------: | -----------------: | ------------: | +| List tracks, snapshots, candidates, and staged objects | Yes | Yes | Yes | +| Preview releases and export snapshots | Yes | Yes | Yes | +| Create tracks and drafts; manage candidates/staged/config/composition | No | Yes | Yes | +| Tag a standard or virtual snapshot | No | Yes | Yes | +| Delete an untagged individual snapshot | No | Yes | Yes | +| Replace standard-track members directly | No | No | Yes | +| Delete an entire track and all snapshot history | No | No | Yes | + +The two direct replacement routes and full-track deletion also require +`confirm_track_id` to equal the `:id` path parameter. Authorization runs before +the controller, and confirmation runs before request-body validation or +persistence. + +## Audited destructive actions + +The following actions create a `releaseTrackAuditEvents` record before their +business operation begins: + +- `replace_members_latest` +- `replace_members_historical` +- `delete_track` + +Each event records the authenticated actor, confirmation value, target track, +request summary, timestamps, and a `pending`, `completed`, or `failed` status. +An audit insert failure prevents the destructive operation. If the operation +persists but final audit-state recording fails, the API returns a structured +`500` containing the audit event ID instead of reporting unconditional +success. + +See the [operator audit guide](../../admin/release-track-audit.md). diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index b7dfb798..f722075a 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -47,10 +47,10 @@ POST /api/release-tracks/new POST /api/release-tracks/new-from-bundle POST /api/release-tracks/import POST /api/release-tracks/:id/meta -POST /api/release-tracks/:id/contents +POST /api/release-tracks/:id/contents?confirm_track_id=:id POST /api/release-tracks/:id/snapshots/latest/release POST /api/release-tracks/:id/clone -DELETE /api/release-tracks/:id +DELETE /api/release-tracks/:id?confirm_track_id=:id ``` ### Snapshot Operations @@ -470,6 +470,11 @@ POST /api/release-tracks/:id/contents Creates new snapshot with updated member objects. **This is intended for retroactive hotfixes only.** The main workflow for enrolling new member objects into `x_mitre_contents` is through the candidate-staging promotion cycle described in [versioning.md](./versioning.md). +This operation requires the administrator role. The +`confirm_track_id` query parameter must exactly equal the `:id` path +parameter. Every accepted attempt is recorded in the durable release-track +destructive audit trail. + This operation is available only for standard tracks. Virtual membership is computed from component releases and can only be updated by materializing a virtual draft with `POST /api/release-tracks/:id/virtual/snapshots/create`. @@ -568,12 +573,13 @@ POST /api/release-tracks/:id/clone ### Delete Release Track ``` -DELETE /api/release-tracks/:id +DELETE /api/release-tracks/:id?confirm_track_id=:id ``` -**Query Parameters:** - -- `versions` - `latest` (delete only latest, default: all) +This irreversible operation requires the administrator role and removes the +track's complete snapshot history. `confirm_track_id` must exactly equal the +`:id` path parameter. Every accepted attempt is recorded in the durable +release-track destructive audit trail. --- @@ -628,14 +634,16 @@ Creates new snapshot with updated metadata. ### Update Contents (Specific Snapshot) ``` -POST /api/release-tracks/:id/snapshots/:modified/contents +POST /api/release-tracks/:id/snapshots/:modified/contents?confirm_track_id=:id ``` Creates new snapshot with updated member objects. **This is intended for retroactive hotfixes only.** **Request Body:** Same as [Update Contents](#update-contents) for latest snapshot. -Like the latest form, this operation is restricted to standard tracks. +Like the latest form, this operation is restricted to standard tracks, +requires the administrator role and exact track-ID confirmation, and creates +a durable audit event. ### Release/Tag Specific Snapshot @@ -764,7 +772,7 @@ contains the same exact revision in `members` and `candidates`, the transition repairs the duplicate and retains the `members` occurrence. A dynamic candidate remains `"latest"` if it is promoted to staged. -``` +```` POST /api/release-tracks/:id/candidates/review ```/ @@ -776,7 +784,7 @@ POST /api/release-tracks/:id/candidates/review "to": "awaiting-review", "object_refs": [{ "id": "attack-pattern--uuid", "modified": "2024-01-15T10:00:00Z" }] } -``` +```` --- From 8730e8820dd25c095a7759d7bd669cf9b1da2d9e Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:13:12 -0400 Subject: [PATCH 6/7] chore(release-tracks): remove pre-release version migration Retain database-enforced release version uniqueness while treating existing beta track collections as disposable development state. --- ...lease-version-uniqueness-migration.spec.js | 74 -------- docs/developer/TODO.md | 37 ++-- .../release-tracks/implementation-notes.md | 14 +- docs/user/release-tracks/versioning.md | 6 - ...nforce-release-track-version-uniqueness.js | 173 ------------------ 5 files changed, 32 insertions(+), 272 deletions(-) delete mode 100644 app/tests/api/release-tracks/release-version-uniqueness-migration.spec.js delete mode 100644 migrations/20260730040000-enforce-release-track-version-uniqueness.js diff --git a/app/tests/api/release-tracks/release-version-uniqueness-migration.spec.js b/app/tests/api/release-tracks/release-version-uniqueness-migration.spec.js deleted file mode 100644 index 895080d1..00000000 --- a/app/tests/api/release-tracks/release-version-uniqueness-migration.spec.js +++ /dev/null @@ -1,74 +0,0 @@ -'use strict'; - -const { expect } = require('expect'); -const mongoose = require('mongoose'); - -const config = require('../../../config/config'); -const database = require('../../../lib/database-in-memory'); -const databaseConfiguration = require('../../../lib/database-configuration'); -const releaseTracksService = require('../../../services/release-tracks/release-tracks-service'); -const migration = require('../../../../migrations/20260730040000-enforce-release-track-version-uniqueness'); - -const UNIQUE_INDEX = 'unique_tagged_version'; -const LEGACY_INDEX = 'id_1_version_1'; - -describe('Release-track tagged-version uniqueness migration', function () { - before(async function () { - await database.initializeConnection(); - await databaseConfiguration.checkSystemConfiguration(); - config.validateRequests.withAttackDataModel = true; - }); - - after(async function () { - await database.closeConnection(); - }); - - it('fails closed on legacy duplicates before replacing indexes and is rerunnable after repair', async function () { - const track = await releaseTracksService.createTrack({ - name: 'Legacy Duplicate Release Versions', - type: 'standard', - }); - const released = await releaseTracksService.releaseLatest(track.id, { - version: '1.0', - userAccountId: 'migration-test', - }); - const collection = mongoose.connection.db.collection(track.id); - - await collection.dropIndex(UNIQUE_INDEX); - await collection.createIndex({ id: 1, version: 1 }, { name: LEGACY_INDEX }); - - const duplicate = { ...released }; - delete duplicate._id; - duplicate.modified = new Date(new Date(released.modified).getTime() + 1000); - await collection.insertOne(duplicate); - await mongoose.connection.db - .collection('releaseTrackRegistry') - .deleteOne({ track_id: track.id }); - - await expect(migration.up(mongoose.connection.db)).rejects.toMatchObject({ - message: expect.stringContaining(`${track.id} version 1.0 (2 snapshots)`), - duplicates: [ - expect.objectContaining({ - track_id: track.id, - version: '1.0', - }), - ], - }); - - let indexes = await collection.indexes(); - expect(indexes.some((index) => index.name === LEGACY_INDEX)).toBe(true); - expect(indexes.some((index) => index.name === UNIQUE_INDEX)).toBe(false); - - await collection.deleteOne({ modified: duplicate.modified }); - await migration.up(mongoose.connection.db); - await migration.up(mongoose.connection.db); - - indexes = await collection.indexes(); - expect(indexes.some((index) => index.name === LEGACY_INDEX)).toBe(false); - expect(indexes.find((index) => index.name === UNIQUE_INDEX)).toMatchObject({ - key: { id: 1, version: 1 }, - unique: true, - partialFilterExpression: { version: { $type: 'string' } }, - }); - }); -}); diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index efb91503..04d21c2a 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -15,19 +15,34 @@ can be reviewed or reverted item by item. identifies the track and requested version. - [x] Add a regression that releases two distinct drafts concurrently with the same version and proves exactly one succeeds. -- [x] Add a rerunnable migration that fails closed on pre-existing duplicates - before replacing the legacy non-unique index. -- [x] Update release-version documentation and run focused, migration, - middleware, lint, and complete-suite verification. - -Verification result (2026-07-30): - -- The deterministic concurrent-release, migration, middleware, and isolated - roaming-failure group passes (39). +- [x] Adopt the pre-release reset policy for collections created with the + former non-unique index. No shared deployment retains beta release-track + data, so this change deliberately does not establish a permanent + migration contract for local development state. +- [x] Update release-version documentation and run focused, middleware, and + lint verification. +- [ ] Obtain one clean aggregate `npm test` run for the migration cleanup. Three + attempts exposed the repository's roaming cross-spec isolation failure; + every affected spec passed immediately in isolation. + +Original implementation verification (2026-07-30): + +- The deterministic concurrent-release, middleware, and isolated + roaming-failure group passes. - The required clean full suite passes: OpenAPI 2, config 21, API 947, middleware 25, and scheduler 10. -- The migration preflights the union of registry IDs and canonical orphan - release-track collection names before making any index changes. +- New dynamic track collections create the unique partial index before their + initial snapshot is persisted. Existing personal development tracks created + under the former beta schema are reset or recreated. + +Pre-release migration cleanup verification (2026-07-30): + +- Release planning and concurrent-version coverage passes all 20 cases. +- Error middleware passes all 9 cases; lint and diff checks pass. +- Aggregate attempts failed in different, unrelated modules: user accounts, + groups, releases-by-object, and tagged-content immutability. Those modules + pass in isolation (9, 23, 8, and 1 cases respectively), confirming no + reproducible migration-cleanup regression. ### P0.2 — Make primary release membership fail closed diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 739b71f3..ec2db9dd 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -19,14 +19,12 @@ whose `version` is a string. Drafts therefore remain unlimited at `version: null`, while the database—not an application-level preflight—decides which concurrent release may claim a version. -Migration `20260730040000-enforce-release-track-version-uniqueness` scans the -union of registered tracks and canonical `release-track--` collection -names before changing any indexes. Including orphan collections matters -because track creation predates transaction-backed registry coordination. If any -`(track_id, version)` has multiple tagged snapshots, it reports all offending -snapshot timestamps and performs no index changes. After operators repair the -data, rerunning the migration replaces the legacy non-unique index -idempotently. +Release tracks are still pre-release, and no shared deployment retains track +data written under the former non-unique index. Existing personal development +tracks are therefore reset or recreated instead of establishing a permanent +upgrade contract for beta data. Once release tracks are formally released, +future index or persistence changes must include an appropriate migration for +supported deployments. ## Validation Rules diff --git a/docs/user/release-tracks/versioning.md b/docs/user/release-tracks/versioning.md index 3290e359..44e6962f 100644 --- a/docs/user/release-tracks/versioning.md +++ b/docs/user/release-tracks/versioning.md @@ -196,12 +196,6 @@ Collections use a **two-part versioning scheme** (MAJOR.MINOR), inspired by sema succeeds and the other receives `409 Conflict` with the conflicting `track_id` and `version`. -Deployments upgrading from an earlier release run a database migration before -serving traffic. The migration checks every release-track collection for -pre-existing duplicate tagged versions and stops without changing indexes if -it finds any. Operators must resolve every reported track/version pair and -rerun the migration; the server does not guess which tagged snapshot to keep. - ### First Tagged Release For release tracks with no prior tagged releases: diff --git a/migrations/20260730040000-enforce-release-track-version-uniqueness.js b/migrations/20260730040000-enforce-release-track-version-uniqueness.js deleted file mode 100644 index 5aed38e6..00000000 --- a/migrations/20260730040000-enforce-release-track-version-uniqueness.js +++ /dev/null @@ -1,173 +0,0 @@ -'use strict'; - -/** - * Replace the legacy non-unique (id, version) index in every dynamic release - * track collection with a unique partial index over tagged snapshots. - * - * The migration preflights every collection before changing any indexes. If a - * deployment already contains duplicate tagged versions, migration stops and - * reports every offending track/version so an operator can repair the data - * deliberately. - */ - -const INDEX_NAME = 'unique_tagged_version'; -const LEGACY_INDEX_NAME = 'id_1_version_1'; -const CONCURRENCY = 8; -const TRACK_COLLECTION_PATTERN = - /^release-track--[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; - -async function mapWithConcurrency(items, mapper) { - let nextIndex = 0; - - async function worker() { - while (nextIndex < items.length) { - const index = nextIndex++; - await mapper(items[index]); - } - } - - await Promise.all(Array.from({ length: Math.min(CONCURRENCY, items.length) }, () => worker())); -} - -async function collectionExists(db, trackId) { - return db.listCollections({ name: trackId }, { nameOnly: true }).hasNext(); -} - -async function findTrackCollectionIds(db) { - const [registeredTracks, collections] = await Promise.all([ - db.collection('releaseTrackRegistry').find({}).project({ track_id: 1, _id: 0 }).toArray(), - db.listCollections({}, { nameOnly: true }).toArray(), - ]); - - return Array.from( - new Set([ - ...registeredTracks.map((track) => track.track_id), - ...collections - .map((collection) => collection.name) - .filter((name) => TRACK_COLLECTION_PATTERN.test(name)), - ]), - ).sort(); -} - -async function findDuplicateVersions(db, trackIds) { - const duplicates = []; - - await mapWithConcurrency(trackIds, async (trackId) => { - if (!(await collectionExists(db, trackId))) return; - - const matches = await db - .collection(trackId) - .aggregate([ - { $match: { version: { $type: 'string' } } }, - { - $group: { - _id: { id: '$id', version: '$version' }, - count: { $sum: 1 }, - snapshots: { $push: '$modified' }, - }, - }, - { $match: { count: { $gt: 1 } } }, - { $sort: { '_id.version': 1 } }, - ]) - .toArray(); - - for (const match of matches) { - duplicates.push({ - track_id: trackId, - version: match._id.version, - snapshots: match.snapshots, - }); - } - }); - - return duplicates.sort( - (left, right) => - left.track_id.localeCompare(right.track_id) || left.version.localeCompare(right.version), - ); -} - -function isDesiredIndex(index) { - return ( - index?.name === INDEX_NAME && - index.unique === true && - index.key?.id === 1 && - index.key?.version === 1 && - index.partialFilterExpression?.version?.$type === 'string' - ); -} - -async function installUniqueIndex(db, trackId) { - if (!(await collectionExists(db, trackId))) return; - - const collection = db.collection(trackId); - const indexes = await collection.indexes(); - const desired = indexes.find((index) => index.name === INDEX_NAME); - if (isDesiredIndex(desired)) { - if (indexes.some((index) => index.name === LEGACY_INDEX_NAME)) { - await collection.dropIndex(LEGACY_INDEX_NAME); - } - return; - } - - if (desired) await collection.dropIndex(INDEX_NAME); - if (indexes.some((index) => index.name === LEGACY_INDEX_NAME)) { - await collection.dropIndex(LEGACY_INDEX_NAME); - } - - await collection.createIndex( - { id: 1, version: 1 }, - { - name: INDEX_NAME, - unique: true, - partialFilterExpression: { version: { $type: 'string' } }, - }, - ); -} - -module.exports = { - async up(db) { - const trackIds = await findTrackCollectionIds(db); - const duplicates = await findDuplicateVersions(db, trackIds); - - if (duplicates.length > 0) { - const summary = duplicates - .map( - (duplicate) => - `${duplicate.track_id} version ${duplicate.version} ` + - `(${duplicate.snapshots.length} snapshots)`, - ) - .join('; '); - const error = new Error( - `Duplicate tagged release versions detected; repair them before retrying migration: ${summary}`, - ); - error.duplicates = duplicates; - throw error; - } - - await mapWithConcurrency(trackIds, (trackId) => installUniqueIndex(db, trackId)); - }, - - async down(db) { - const trackIds = await findTrackCollectionIds(db); - - await mapWithConcurrency(trackIds, async (trackId) => { - if (!(await collectionExists(db, trackId))) return; - - const collection = db.collection(trackId); - const indexes = await collection.indexes(); - if (indexes.some((index) => index.name === INDEX_NAME)) { - await collection.dropIndex(INDEX_NAME); - } - if (!indexes.some((index) => index.name === LEGACY_INDEX_NAME)) { - await collection.createIndex({ id: 1, version: 1 }, { name: LEGACY_INDEX_NAME }); - } - }); - }, - - _private: { - findTrackCollectionIds, - findDuplicateVersions, - installUniqueIndex, - isDesiredIndex, - }, -}; From 86d11ed661c590e9edc1d465af826328732d3406 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:56:52 -0400 Subject: [PATCH 7/7] feat(release-tracks): make snapshot bundles deterministic Persist exact primary, relationship, and secondary revisions in snapshot graph manifests. Pin relationship endpoints, protect referenced revisions, and add migration tooling for existing data. Make persisted snapshot history immutable by removing direct contents and historical metadata mutation routes. Limit snapshot deletion to the latest untagged draft and align the API contract, tests, and documentation. --- .../definitions/components/release-tracks.yml | 6 + app/api/definitions/openapi.yml | 9 - app/api/definitions/paths/analytics-paths.yml | 6 +- app/api/definitions/paths/assets-paths.yml | 4 +- app/api/definitions/paths/campaigns-paths.yml | 6 +- .../definitions/paths/collections-paths.yml | 4 + .../paths/data-components-paths.yml | 6 +- .../definitions/paths/data-sources-paths.yml | 4 +- .../paths/detection-strategies-paths.yml | 6 +- app/api/definitions/paths/groups-paths.yml | 6 +- .../definitions/paths/identities-paths.yml | 4 +- app/api/definitions/paths/matrices-paths.yml | 6 +- .../definitions/paths/mitigations-paths.yml | 6 +- app/api/definitions/paths/notes-paths.yml | 6 +- .../definitions/paths/relationships-paths.yml | 6 +- .../paths/release-tracks-paths.yml | 132 +--- app/api/definitions/paths/software-paths.yml | 6 +- app/api/definitions/paths/tactics-paths.yml | 6 +- .../definitions/paths/techniques-paths.yml | 6 +- app/controllers/collections-controller.js | 4 +- app/controllers/release-tracks-controller.js | 85 -- app/exceptions/index.js | 24 + app/lib/error-handler.js | 4 + app/lib/linkById.js | 17 + app/lib/release-tracks/backref-reconciler.js | 2 +- .../release-tracks/release-track-schemas.js | 13 - app/models/relationship-model.js | 20 + .../release-track-audit-event-model.js | 2 +- .../release-track-graph-manifest-model.js | 85 ++ .../release-track-snapshot-schema.js | 1 + app/routes/release-tracks-routes.js | 31 +- app/services/meta-classes/base.service.js | 50 ++ app/services/release-tracks/export-service.js | 157 +--- .../release-tracks/graph-manifest-service.js | 542 +++++++++++++ .../primary-revision-service.js | 1 + .../release-tracks/release-tracks-service.js | 50 +- .../release-tracks/snapshot-service.js | 167 ++-- .../release-tracks/versioning-service.js | 41 +- app/services/stix/bundle-graph-resolver.js | 469 +++++++++++ app/services/stix/collections-service.js | 42 + app/services/stix/relationships-service.js | 161 ++++ app/services/stix/stix-bundles-service.js | 394 +--------- .../api/attack-objects/attack-objects.spec.js | 6 +- .../relationship-endpoint-pins.spec.js | 170 ++++ .../relationships-pagination.spec.js | 33 + .../api/relationships/relationships.spec.js | 104 +++ .../destructive-authorization.spec.js | 109 +-- .../deterministic-graph-migration.spec.js | 198 +++++ .../release-tracks/ephemeral-bundle.spec.js | 68 +- .../primary-revision-integrity.spec.js | 34 +- .../reconciliation-durability.spec.js | 5 +- .../release-track-test-helpers.js | 49 ++ .../release-tracks-backrefs.spec.js | 35 +- .../release-tracks-bundle.spec.js | 152 +++- .../release-tracks-change-capture.spec.js | 9 +- .../release-tracks-release.spec.js | 32 +- .../release-tracks-tier-invariant.spec.js | 8 +- .../api/release-tracks/release-tracks.spec.js | 17 +- .../release-tracks/releases-by-object.spec.js | 55 +- .../snapshot-immutability.spec.js | 104 +++ .../tagged-content-immutability.spec.js | 23 +- .../virtual-deduplication.spec.js | 13 +- .../virtual-determinism.spec.js | 17 +- .../virtual-domain-filters.spec.js | 19 +- .../virtual-object-type-filters.spec.js | 13 +- .../release-tracks/virtual-quarantine.spec.js | 11 +- app/tests/api/reports/reports.spec.js | 15 + docs/README.md | 3 +- docs/admin/release-track-audit.md | 10 +- docs/admin/release-track-graph-migration.md | 53 ++ docs/developer/FRONTEND_TODO.md | 161 +++- docs/developer/TODO.md | 743 +++++++++++------- .../developer/release-tracks/authorization.md | 22 +- .../release-tracks/backref-reconciliation.md | 8 +- .../developer/release-tracks/bundle-export.md | 121 +-- docs/developer/release-tracks/entities.md | 10 +- .../release-tracks/error-handling.md | 30 +- .../release-tracks/implementation-notes.md | 14 +- .../release-tracks/member-sync-strategies.md | 4 + docs/user/release-tracks/api-reference.md | 95 +-- docs/user/release-tracks/object-backrefs.md | 15 +- docs/user/release-tracks/summary.md | 20 +- docs/user/release-tracks/virtual-tracks.md | 13 +- docs/user/release-tracks/workflow-examples.md | 97 ++- ...-backfill-deterministic-snapshot-graphs.js | 276 +++++++ package.json | 1 + ...viewDeterministicSnapshotGraphMigration.js | 22 + 87 files changed, 3745 insertions(+), 1869 deletions(-) create mode 100644 app/models/release-tracks/release-track-graph-manifest-model.js create mode 100644 app/services/release-tracks/graph-manifest-service.js create mode 100644 app/services/stix/bundle-graph-resolver.js create mode 100644 app/tests/api/relationships/relationship-endpoint-pins.spec.js create mode 100644 app/tests/api/release-tracks/deterministic-graph-migration.spec.js create mode 100644 app/tests/api/release-tracks/release-track-test-helpers.js create mode 100644 app/tests/api/release-tracks/snapshot-immutability.spec.js create mode 100644 docs/admin/release-track-graph-migration.md create mode 100644 migrations/20260730180000-backfill-deterministic-snapshot-graphs.js create mode 100644 scripts/previewDeterministicSnapshotGraphMigration.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index d1549685..d973f9d4 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -23,6 +23,12 @@ components: nullable: true description: 'Semantic version (e.g., "1.0", "2.1") if tagged, null for draft snapshots' example: '1.0' + graph_manifest_id: + type: string + readOnly: true + description: | + Server-controlled identifier for the frozen bundle graph associated + with this snapshot. Clients should treat this value as opaque. name: type: string description: 'Human-readable track name' diff --git a/app/api/definitions/openapi.yml b/app/api/definitions/openapi.yml index dec2b7a3..07a7c20f 100644 --- a/app/api/definitions/openapi.yml +++ b/app/api/definitions/openapi.yml @@ -352,9 +352,6 @@ paths: /api/release-tracks/{id}/meta: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1meta' - /api/release-tracks/{id}/contents: - $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1contents' - /api/release-tracks/{id}/clone: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1clone' @@ -409,12 +406,6 @@ paths: /api/release-tracks/{id}/snapshots/{modified}: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}' - /api/release-tracks/{id}/snapshots/{modified}/meta: - $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1meta' - - /api/release-tracks/{id}/snapshots/{modified}/contents: - $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1contents' - /api/release-tracks/{id}/snapshots/{modified}/clone: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1clone' diff --git a/app/api/definitions/paths/analytics-paths.yml b/app/api/definitions/paths/analytics-paths.yml index 16d84795..dbab5651 100644 --- a/app/api/definitions/paths/analytics-paths.yml +++ b/app/api/definitions/paths/analytics-paths.yml @@ -206,7 +206,7 @@ paths: '404': description: 'A analytic with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/analytics/{stixId}/modified/{modified}: get: @@ -278,7 +278,7 @@ paths: '404': description: 'A analytic with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a analytic' operationId: 'analytic-delete' @@ -306,4 +306,4 @@ paths: '404': description: 'A analytic with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' diff --git a/app/api/definitions/paths/assets-paths.yml b/app/api/definitions/paths/assets-paths.yml index 594eac6c..4f76dde8 100644 --- a/app/api/definitions/paths/assets-paths.yml +++ b/app/api/definitions/paths/assets-paths.yml @@ -270,7 +270,7 @@ paths: '404': description: 'An asset with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete an asset' operationId: 'asset-delete' @@ -298,7 +298,7 @@ paths: '404': description: 'An asset with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/assets/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/campaigns-paths.yml b/app/api/definitions/paths/campaigns-paths.yml index 37ec83bb..e02dff77 100644 --- a/app/api/definitions/paths/campaigns-paths.yml +++ b/app/api/definitions/paths/campaigns-paths.yml @@ -178,7 +178,7 @@ paths: '404': description: 'A campaign with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/campaigns/{stixId}/modified/{modified}: get: @@ -250,7 +250,7 @@ paths: '404': description: 'A campaign with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a campaign' operationId: 'campaign-delete' @@ -278,7 +278,7 @@ paths: '404': description: 'A campaign with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/campaigns/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/collections-paths.yml b/app/api/definitions/paths/collections-paths.yml index f2bcb30f..b7e8b832 100644 --- a/app/api/definitions/paths/collections-paths.yml +++ b/app/api/definitions/paths/collections-paths.yml @@ -199,6 +199,8 @@ paths: description: 'The collections were successfully deleted.' '404': description: 'A collection with the requested STIX id was not found.' + '409': + description: 'The collection revision or a requested cascade-delete target is pinned by release-track membership or a snapshot graph manifest and cannot be deleted.' /api/collections/{stixId}/modified/{modified}: get: @@ -277,3 +279,5 @@ paths: description: 'The collection was successfully deleted.' '404': description: 'A collection with the requested STIX id was not found.' + '409': + description: 'The collection revision or a requested cascade-delete target is pinned by release-track membership or a snapshot graph manifest and cannot be deleted.' diff --git a/app/api/definitions/paths/data-components-paths.yml b/app/api/definitions/paths/data-components-paths.yml index 1e260c73..b110bd3f 100644 --- a/app/api/definitions/paths/data-components-paths.yml +++ b/app/api/definitions/paths/data-components-paths.yml @@ -190,7 +190,7 @@ paths: '404': description: 'A data component with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/data-components/{stixId}/channels: get: @@ -316,7 +316,7 @@ paths: '404': description: 'A data component with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a data component' operationId: 'data-component-delete' @@ -344,7 +344,7 @@ paths: '404': description: 'A data component with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/data-components/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/data-sources-paths.yml b/app/api/definitions/paths/data-sources-paths.yml index c301a6af..bc33482c 100644 --- a/app/api/definitions/paths/data-sources-paths.yml +++ b/app/api/definitions/paths/data-sources-paths.yml @@ -286,7 +286,7 @@ paths: '404': description: 'A data source with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a data source' operationId: 'data-source-delete' @@ -314,7 +314,7 @@ paths: '404': description: 'A data source with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/data-sources/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/detection-strategies-paths.yml b/app/api/definitions/paths/detection-strategies-paths.yml index a038c768..d83dc126 100644 --- a/app/api/definitions/paths/detection-strategies-paths.yml +++ b/app/api/definitions/paths/detection-strategies-paths.yml @@ -190,7 +190,7 @@ paths: '404': description: 'A detection strategy with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/detection-strategies/{stixId}/modified/{modified}: get: @@ -262,7 +262,7 @@ paths: '404': description: 'A detection strategy with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a detection strategy' operationId: 'detection-strategy-delete' @@ -290,4 +290,4 @@ paths: '404': description: 'A detection strategy with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' diff --git a/app/api/definitions/paths/groups-paths.yml b/app/api/definitions/paths/groups-paths.yml index cbe8dd3b..c6e39178 100644 --- a/app/api/definitions/paths/groups-paths.yml +++ b/app/api/definitions/paths/groups-paths.yml @@ -178,7 +178,7 @@ paths: '404': description: 'A group with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/groups/{stixId}/modified/{modified}: get: @@ -250,7 +250,7 @@ paths: '404': description: 'A group with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a group' operationId: 'group-delete' @@ -278,7 +278,7 @@ paths: '404': description: 'A group with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/groups/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/identities-paths.yml b/app/api/definitions/paths/identities-paths.yml index 1c104789..10e83651 100644 --- a/app/api/definitions/paths/identities-paths.yml +++ b/app/api/definitions/paths/identities-paths.yml @@ -227,7 +227,7 @@ paths: '404': description: 'An identity with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a identity' operationId: 'identity-delete' @@ -255,4 +255,4 @@ paths: '404': description: 'An identity with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' diff --git a/app/api/definitions/paths/matrices-paths.yml b/app/api/definitions/paths/matrices-paths.yml index a49d0123..45cc2118 100644 --- a/app/api/definitions/paths/matrices-paths.yml +++ b/app/api/definitions/paths/matrices-paths.yml @@ -178,7 +178,7 @@ paths: '404': description: 'A matrix with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/matrices/{stixId}/modified/{modified}: get: @@ -250,7 +250,7 @@ paths: '404': description: 'A matrix with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a matrix' operationId: 'matrix-delete' @@ -278,7 +278,7 @@ paths: '404': description: 'A matrix with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/matrices/{stixId}/modified/{modified}/techniques: get: diff --git a/app/api/definitions/paths/mitigations-paths.yml b/app/api/definitions/paths/mitigations-paths.yml index f1253584..2e3bde43 100644 --- a/app/api/definitions/paths/mitigations-paths.yml +++ b/app/api/definitions/paths/mitigations-paths.yml @@ -190,7 +190,7 @@ paths: '404': description: 'A mitigation with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/mitigations/{stixId}/modified/{modified}: get: @@ -262,7 +262,7 @@ paths: '404': description: 'A mitigation with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a mitigation' operationId: 'mitigation-delete' @@ -290,7 +290,7 @@ paths: '404': description: 'A mitigation with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/mitigations/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/notes-paths.yml b/app/api/definitions/paths/notes-paths.yml index f83752ef..eebad19a 100644 --- a/app/api/definitions/paths/notes-paths.yml +++ b/app/api/definitions/paths/notes-paths.yml @@ -176,7 +176,7 @@ paths: '404': description: 'A note with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/notes/{stixId}/modified/{modified}: get: @@ -247,7 +247,7 @@ paths: '404': description: 'A note with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a note' operationId: 'note-delete-version' @@ -275,4 +275,4 @@ paths: '404': description: 'A note with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' diff --git a/app/api/definitions/paths/relationships-paths.yml b/app/api/definitions/paths/relationships-paths.yml index 0de8060f..69fd5c03 100644 --- a/app/api/definitions/paths/relationships-paths.yml +++ b/app/api/definitions/paths/relationships-paths.yml @@ -248,7 +248,7 @@ paths: '404': description: 'A relationship with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/relationships/{stixId}/modified/{modified}: get: @@ -320,7 +320,7 @@ paths: '404': description: 'A relationship with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a relationship' operationId: 'relationship-delete' @@ -348,4 +348,4 @@ paths: '404': description: 'A relationship with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 94917925..4a001ef4 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -346,54 +346,6 @@ paths: schema: $ref: '../components/release-tracks.yml#/components/schemas/release-track-snapshot' - /api/release-tracks/{id}/contents: - post: - summary: 'Update member contents on the latest snapshot' - operationId: 'release-tracks-update-contents-latest' - description: | - Replace the members tier of a standard track with new contents - (x_mitre_contents format). Virtual membership can only be produced by - POST /api/release-tracks/{id}/virtual/snapshots/create. - obj_modified may be an exact ISO timestamp or the request-time - shorthand "latest"; the server resolves "latest" to the actual latest - stix.modified timestamp before persistence. Snapshot members always - store exact revision pins. - Exact revisions already present in another tier are retained only in members; - different revisions of the same object remain valid across tiers. - Creates a new snapshot clone. - This administrator-only operation requires confirm_track_id to exactly - match the target track and writes a durable audit event. - Request body validated via Zod in controller. - tags: - - 'Release Tracks' - parameters: - - name: id - in: path - required: true - schema: - type: string - - name: confirm_track_id - in: query - required: true - description: 'Must exactly equal the id path parameter' - schema: - type: string - responses: - '200': - description: 'Contents updated successfully' - '400': - description: 'Track is virtual, the contents request is invalid, or a requested revision does not exist' - '401': - description: 'Administrator role required' - '500': - description: 'The replacement may be persisted, but its durable audit or membership reconciliation failed' - content: - application/json: - schema: - oneOf: - - $ref: '../components/release-tracks.yml#/components/schemas/release-track-audit-error' - - $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' - /api/release-tracks/{id}/clone: post: summary: 'Clone the latest snapshot into a new release track' @@ -1233,8 +1185,10 @@ paths: summary: 'Delete a specific snapshot' operationId: 'release-tracks-snapshot-delete' description: | - Delete a snapshot by its modified timestamp. - Cannot delete tagged snapshots. + Delete the latest untagged draft snapshot by its modified timestamp. + Tagged snapshots and historical drafts are immutable and cannot be + deleted. Deleting the latest draft reverts the track to its immediately + preceding snapshot. tags: - 'Release Tracks' parameters: @@ -1252,86 +1206,10 @@ paths: '204': description: 'Snapshot deleted successfully' '409': - description: 'Cannot delete tagged snapshot' + description: 'Cannot delete a tagged snapshot or a historical draft' '404': description: 'Snapshot not found' - /api/release-tracks/{id}/snapshots/{modified}/meta: - post: - summary: 'Update metadata on a specific snapshot' - operationId: 'release-tracks-update-meta-by-modified' - description: | - Update metadata on a historical snapshot. - Request body validated via Zod in controller. - tags: - - 'Release Tracks' - parameters: - - name: id - in: path - required: true - schema: - type: string - - name: modified - in: path - required: true - schema: - type: string - responses: - '200': - description: 'Metadata updated successfully' - - /api/release-tracks/{id}/snapshots/{modified}/contents: - post: - summary: 'Update contents on a specific snapshot' - operationId: 'release-tracks-update-contents-by-modified' - description: | - Update member contents on a historical standard-track snapshot. - Virtual membership can only be produced by - POST /api/release-tracks/{id}/virtual/snapshots/create. - obj_modified may be an exact ISO timestamp or the request-time - shorthand "latest"; the server resolves "latest" to the actual latest - stix.modified timestamp before persistence. Snapshot members always - store exact revision pins. - Exact revisions already present in another tier are retained only in members; - different revisions of the same object remain valid across tiers. - This administrator-only operation requires confirm_track_id to exactly - match the target track and writes a durable audit event. - Request body validated via Zod in controller. - tags: - - 'Release Tracks' - parameters: - - name: id - in: path - required: true - schema: - type: string - - name: modified - in: path - required: true - schema: - type: string - - name: confirm_track_id - in: query - required: true - description: 'Must exactly equal the id path parameter' - schema: - type: string - responses: - '200': - description: 'Contents updated successfully' - '400': - description: 'Track is virtual, the contents request is invalid, or a requested revision does not exist' - '401': - description: 'Administrator role required' - '500': - description: 'The replacement may be persisted, but its durable audit or membership reconciliation failed' - content: - application/json: - schema: - oneOf: - - $ref: '../components/release-tracks.yml#/components/schemas/release-track-audit-error' - - $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' - /api/release-tracks/{id}/snapshots/{modified}/clone: post: summary: 'Clone a specific snapshot into a new release track' diff --git a/app/api/definitions/paths/software-paths.yml b/app/api/definitions/paths/software-paths.yml index 33831f85..2d90bc18 100644 --- a/app/api/definitions/paths/software-paths.yml +++ b/app/api/definitions/paths/software-paths.yml @@ -201,7 +201,7 @@ paths: '404': description: 'A tactic with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/software/{stixId}/modified/{modified}: get: @@ -272,7 +272,7 @@ paths: '404': description: 'A software object with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a software object' operationId: 'software-delete' @@ -300,7 +300,7 @@ paths: '404': description: 'A software object with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/software/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/tactics-paths.yml b/app/api/definitions/paths/tactics-paths.yml index 6caadde9..e007ca7d 100644 --- a/app/api/definitions/paths/tactics-paths.yml +++ b/app/api/definitions/paths/tactics-paths.yml @@ -190,7 +190,7 @@ paths: '404': description: 'A tactic with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/tactics/{stixId}/modified/{modified}: get: @@ -262,7 +262,7 @@ paths: '404': description: 'A tactic with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a tactic' operationId: 'tactic-delete' @@ -290,7 +290,7 @@ paths: '404': description: 'A tactic with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/tactics/{stixId}/modified/{modified}/techniques: get: diff --git a/app/api/definitions/paths/techniques-paths.yml b/app/api/definitions/paths/techniques-paths.yml index 050535e6..773af422 100644 --- a/app/api/definitions/paths/techniques-paths.yml +++ b/app/api/definitions/paths/techniques-paths.yml @@ -214,7 +214,7 @@ paths: '404': description: 'A technique with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/techniques/{stixId}/modified/{modified}: get: @@ -286,7 +286,7 @@ paths: '404': description: 'A technique with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a technique' operationId: 'technique-delete' @@ -314,7 +314,7 @@ paths: '404': description: 'A technique with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/techniques/{stixId}/modified/{modified}/tactics: get: diff --git a/app/controllers/collections-controller.js b/app/controllers/collections-controller.js index 255a2a49..701ca9d9 100644 --- a/app/controllers/collections-controller.js +++ b/app/controllers/collections-controller.js @@ -191,7 +191,7 @@ exports.create = async function (req, res) { } }; -exports.delete = async function (req, res) { +exports.delete = async function (req, res, next) { try { const removedCollections = await collectionsService.delete( req.params.stixId, @@ -205,7 +205,7 @@ exports.delete = async function (req, res) { } } catch (error) { logger.error('Delete collections failed. ' + error); - return res.status(500).send('Unable to delete collections. Server error.'); + return next(error); } }; diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index fb61574b..4b1f9fec 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -38,7 +38,6 @@ const { createTrackBodySchema, createFromBundleBodySchema, updateMetadataBodySchema, - updateContentsBodySchema, releaseBodySchema, releaseVersionSelectionSchema, cloneBodySchema, @@ -461,34 +460,6 @@ exports.updateMetadataByLatest = async function updateMetadataByLatest(req, res, } }; -/** POST /api/release-tracks/:id/contents */ -exports.updateContentsByLatest = async function updateContentsByLatest(req, res, next) { - try { - requireDestructiveConfirmation(req); - const bodyResult = updateContentsBodySchema.safeParse(req.body); - if (!bodyResult.success) { - return next( - new BadRequestError({ - message: 'Invalid contents update', - details: bodyResult.error.errors, - }), - ); - } - - const result = await releaseTracksService.updateContents( - req.params.id, - bodyResult.data, - destructiveActor(req), - req.query.confirm_track_id, - ); - logger.debug(`Success: Updated contents for track ${req.params.id}`); - return res.status(200).send(result); - } catch (err) { - logger.error('Failed to update track contents: ' + err); - return next(err); - } -}; - /** POST /api/release-tracks/:id/snapshots/latest/release */ exports.releaseLatest = async function releaseLatest(req, res, next) { try { @@ -585,62 +556,6 @@ exports.retrieveSnapshotByModified = async function retrieveSnapshotByModified(r } }; -/** POST /api/release-tracks/:id/snapshots/:modified/meta */ -exports.updateMetadataByModified = async function updateMetadataByModified(req, res, next) { - try { - const bodyResult = updateMetadataBodySchema.safeParse(req.body); - if (!bodyResult.success) { - return next( - new BadRequestError({ - message: 'Invalid metadata update', - details: bodyResult.error.errors, - }), - ); - } - - const result = await releaseTracksService.updateMetadataByModified( - req.params.id, - req.params.modified, - bodyResult.data, - req.user?.userAccountId, - ); - logger.debug(`Success: Updated metadata for snapshot ${req.params.modified}`); - return res.status(200).send(result); - } catch (err) { - logger.error('Failed to update snapshot metadata: ' + err); - return next(err); - } -}; - -/** POST /api/release-tracks/:id/snapshots/:modified/contents */ -exports.updateContentsByModified = async function updateContentsByModified(req, res, next) { - try { - requireDestructiveConfirmation(req); - const bodyResult = updateContentsBodySchema.safeParse(req.body); - if (!bodyResult.success) { - return next( - new BadRequestError({ - message: 'Invalid contents update', - details: bodyResult.error.errors, - }), - ); - } - - const result = await releaseTracksService.updateContentsByModified( - req.params.id, - req.params.modified, - bodyResult.data, - destructiveActor(req), - req.query.confirm_track_id, - ); - logger.debug(`Success: Updated contents for snapshot ${req.params.modified}`); - return res.status(200).send(result); - } catch (err) { - logger.error('Failed to update snapshot contents: ' + err); - return next(err); - } -}; - /** POST /api/release-tracks/:id/snapshots/:modified/release */ exports.releaseByModified = async function releaseByModified(req, res, next) { try { diff --git a/app/exceptions/index.js b/app/exceptions/index.js index 8be2df56..88747877 100644 --- a/app/exceptions/index.js +++ b/app/exceptions/index.js @@ -351,6 +351,18 @@ class TaggedSnapshotDeletionError extends CustomError { } } +class HistoricalSnapshotDeletionError extends CustomError { + constructor(snapshotModified, latestSnapshotModified, options = {}) { + super('Only the latest untagged snapshot can be deleted', { + ...options, + snapshot_modified: new Date(snapshotModified).toISOString(), + latest_snapshot_modified: latestSnapshotModified + ? new Date(latestSnapshotModified).toISOString() + : null, + }); + } +} + class MemberPinnedRevisionError extends CustomError { constructor(options) { super( @@ -362,6 +374,16 @@ class MemberPinnedRevisionError extends CustomError { } } +class SnapshotGraphPinnedRevisionError extends CustomError { + constructor(options) { + super( + 'This revision is frozen in a release-track snapshot graph and cannot be modified or ' + + 'deleted in place. Create a new revision instead.', + options, + ); + } +} + class InvalidVersionError extends CustomError { constructor(message, options) { super(message || 'Invalid version', options); @@ -437,6 +459,7 @@ module.exports = { DuplicateReleaseVersionError, InvalidObjectRevisionError, TaggedSnapshotDeletionError, + HistoricalSnapshotDeletionError, InvalidVersionError, //** Release track errors */ @@ -449,6 +472,7 @@ module.exports = { VirtualSnapshotNotMaterializedError, TrackNotFoundError, MemberPinnedRevisionError, + SnapshotGraphPinnedRevisionError, //** Database-related errors */ DuplicateIdError, diff --git a/app/lib/error-handler.js b/app/lib/error-handler.js index c187dc47..7ad83715 100644 --- a/app/lib/error-handler.js +++ b/app/lib/error-handler.js @@ -39,6 +39,7 @@ const { DuplicateReleaseVersionError, InvalidObjectRevisionError, TaggedSnapshotDeletionError, + HistoricalSnapshotDeletionError, InvalidVersionError, ReleaseConflictError, ReleaseContentIntegrityError, @@ -49,6 +50,7 @@ const { VirtualSnapshotNotMaterializedError, TrackNotFoundError, MemberPinnedRevisionError, + SnapshotGraphPinnedRevisionError, ObjectHasValidationIssuesError, } = require('../exceptions'); @@ -140,10 +142,12 @@ exports.serviceExceptions = function (err, req, res, next) { err instanceof AlreadyReleasedError || err instanceof DuplicateReleaseVersionError || err instanceof TaggedSnapshotDeletionError || + err instanceof HistoricalSnapshotDeletionError || err instanceof ReleaseConflictError || err instanceof ReleaseContentIntegrityError || err instanceof VirtualSnapshotNotMaterializedError || err instanceof MemberPinnedRevisionError || + err instanceof SnapshotGraphPinnedRevisionError || err instanceof ObjectHasValidationIssuesError || err instanceof ActiveOrganizationIdentityDeleteError ) { diff --git a/app/lib/linkById.js b/app/lib/linkById.js index c37707ec..c46188f6 100644 --- a/app/lib/linkById.js +++ b/app/lib/linkById.js @@ -23,6 +23,23 @@ function attackReference(externalReferences) { } const linkByIdRegex = /\(LinkById: ([A-Z]+[0-9]+(\.[0-9]+)?)\)/g; + +function extractLinkByIds(stixObject) { + const values = [ + stixObject?.description, + stixObject?.type === 'attack-pattern' ? stixObject.x_mitre_detection : undefined, + ...(stixObject?.external_references || []).map((reference) => reference.description), + ]; + const attackIds = new Set(); + for (const value of values) { + for (const match of value?.matchAll(linkByIdRegex) || []) { + attackIds.add(match[1]); + } + } + return [...attackIds]; +} +exports.extractLinkByIds = extractLinkByIds; + async function convertLinkById(text, getAttackObject) { if (text) { let convertedText = ''; diff --git a/app/lib/release-tracks/backref-reconciler.js b/app/lib/release-tracks/backref-reconciler.js index 0a539cc2..b83ddf08 100644 --- a/app/lib/release-tracks/backref-reconciler.js +++ b/app/lib/release-tracks/backref-reconciler.js @@ -21,7 +21,7 @@ // (latest) snapshot, compute the desired set of backrefs and diff it against // the documents that currently carry an entry for that track. This single // code path covers every membership mutation (add/remove/review/promote/ -// demote/release/member-sync/clone/bundle-import/updateContents) as well as +// demote/release/member-sync/clone/bundle-import) as well as // snapshot deletion (membership reverts to the new latest snapshot) and // track deletion (snapshot = null removes all entries). // diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index 8f0c6d85..aaf9d8df 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -398,18 +398,6 @@ const updateMetadataBodySchema = z.object({ object_marking_refs: z.array(stixIdentifierSchema).optional(), }); -/** POST /release-tracks/:id/contents */ -const updateContentsBodySchema = z.object({ - x_mitre_contents: z - .array( - z.object({ - obj_ref: stixIdentifierSchema, - obj_modified: z.iso.datetime().or(z.literal('latest')), - }), - ) - .min(1), -}); - const releaseVersionSelectionSchema = z .object({ increment: releaseIncrementSchema.optional(), @@ -567,7 +555,6 @@ module.exports = { createTrackBodySchema, createFromBundleBodySchema, updateMetadataBodySchema, - updateContentsBodySchema, releaseBodySchema, cloneBodySchema, addCandidatesBodySchema, diff --git a/app/models/relationship-model.js b/app/models/relationship-model.js index 223aa4d6..2a794940 100644 --- a/app/models/relationship-model.js +++ b/app/models/relationship-model.js @@ -23,10 +23,22 @@ const relationshipProperties = { x_mitre_log_source_channel: String, }; +const exactObjectRevision = { + object_ref: { type: String, required: true }, + object_modified: { type: Date, required: true }, +}; +const exactObjectRevisionSchema = new mongoose.Schema(exactObjectRevision, { + _id: false, +}); + // Create the definition const relationshipDefinition = { workspace: { ...workspaceDefinitions.common, + relationship_endpoints: { + source: { type: exactObjectRevisionSchema, required: true }, + target: { type: exactObjectRevisionSchema, required: true }, + }, }, stix: { ...stixCoreDefinitions.commonRequiredSDO, @@ -43,6 +55,14 @@ relationshipSchema.index({ 'stix.id': 1, 'stix.modified': -1 }, { unique: true } // Multikey index supporting reverse lookups from release tracks // (release-track backref reconciliation queries by workspace.release_tracks.id) relationshipSchema.index({ 'workspace.release_tracks.id': 1 }, { sparse: true }); +relationshipSchema.index({ + 'workspace.relationship_endpoints.source.object_ref': 1, + 'workspace.relationship_endpoints.source.object_modified': 1, +}); +relationshipSchema.index({ + 'workspace.relationship_endpoints.target.object_ref': 1, + 'workspace.relationship_endpoints.target.object_modified': 1, +}); // Create the model const RelationshipModel = mongoose.model(ModelName.Relationship, relationshipSchema); diff --git a/app/models/release-tracks/release-track-audit-event-model.js b/app/models/release-tracks/release-track-audit-event-model.js index ded1fa76..2218f47f 100644 --- a/app/models/release-tracks/release-track-audit-event-model.js +++ b/app/models/release-tracks/release-track-audit-event-model.js @@ -9,7 +9,7 @@ const releaseTrackAuditEventSchema = new mongoose.Schema( action: { type: String, required: true, - enum: ['replace_members_latest', 'replace_members_historical', 'delete_track'], + enum: ['delete_track'], }, track_id: { type: String, required: true, validate: validateTrackId }, status: { diff --git a/app/models/release-tracks/release-track-graph-manifest-model.js b/app/models/release-tracks/release-track-graph-manifest-model.js new file mode 100644 index 00000000..07f049d1 --- /dev/null +++ b/app/models/release-tracks/release-track-graph-manifest-model.js @@ -0,0 +1,85 @@ +'use strict'; + +const mongoose = require('mongoose'); + +const exactRevisionSchema = new mongoose.Schema( + { + object_ref: { type: String, required: true }, + object_modified: { type: Date, required: true }, + }, + { _id: false }, +); + +const manifestSchema = new mongoose.Schema( + { + manifest_id: { type: String, required: true, unique: true }, + track_id: { type: String, required: true }, + snapshot_modified: { type: Date, required: true }, + state: { + type: String, + enum: ['pending', 'active'], + required: true, + default: 'pending', + }, + schema_version: { type: Number, required: true, default: 1 }, + resolver_version: { type: String, required: true }, + baseline_reconstruction: { type: Boolean, required: true, default: false }, + created_at: { type: Date, required: true, default: Date.now }, + }, + { collection: 'releaseTrackGraphManifests' }, +); + +manifestSchema.index( + { track_id: 1, snapshot_modified: 1, state: 1 }, + { name: 'manifest_by_snapshot' }, +); + +const entrySchema = new mongoose.Schema( + { + manifest_id: { type: String, required: true }, + track_id: { type: String, required: true }, + snapshot_modified: { type: Date, required: true }, + revision_key: { type: String, required: true }, + kind: { + type: String, + enum: ['root', 'relationship', 'secondary', 'supporting', 'link_target'], + required: true, + }, + tier: { + type: String, + enum: ['members', 'staged', 'candidates', 'quarantine'], + }, + object_status: { type: String }, + object_ref: { type: String, required: true }, + object_modified: { type: Date }, + source: { type: exactRevisionSchema }, + target: { type: exactRevisionSchema }, + discovered_from: { type: [exactRevisionSchema], default: undefined }, + // Relationship payloads are frozen so description-only corrections do + // not change older bundles. Marking definitions are not STIX-versioned, + // so their complete payload is frozen for the same replay guarantee. + frozen_stix: { type: mongoose.Schema.Types.Mixed }, + }, + { collection: 'releaseTrackGraphManifestEntries' }, +); + +entrySchema.index( + { manifest_id: 1, revision_key: 1, kind: 1, tier: 1 }, + { unique: true, name: 'unique_manifest_entry' }, +); +entrySchema.index( + { object_ref: 1, object_modified: 1, manifest_id: 1 }, + { name: 'manifest_revision_protection' }, +); +entrySchema.index({ manifest_id: 1, kind: 1, tier: 1 }); + +const ReleaseTrackGraphManifest = mongoose.model('ReleaseTrackGraphManifest', manifestSchema); +const ReleaseTrackGraphManifestEntry = mongoose.model( + 'ReleaseTrackGraphManifestEntry', + entrySchema, +); + +module.exports = { + ReleaseTrackGraphManifest, + ReleaseTrackGraphManifestEntry, +}; diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index 814a7190..ee25cfcc 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -364,6 +364,7 @@ const releaseTrackSnapshotDefinition = { default: null, validate: validateVersion, }, + graph_manifest_id: { type: String }, // Release track metadata name: { diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index 40838dc7..8204fca0 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -76,19 +76,6 @@ router releaseTracksController.updateMetadataByLatest, ); -/** - * !!IMPORTANT - * The following endpoint is considered dangerous. It is intended for retroactive hotfixes only. Thus, only admins may use it. - * The main workflow for enrolling new member objects into members is through the candidate-staging promotion cycle. - */ -router - .route('/release-tracks/:id/contents') - .post( - authn.authenticate, - authz.requireRole(authz.admin), - releaseTracksController.updateContentsByLatest, - ); - router .route('/release-tracks/:id/clone') .post( @@ -248,25 +235,9 @@ router ); // ============================================================================= -// Snapshot-specific operations (parameterised by :modified) +// Snapshot-specific read, release, clone, and deletion operations // ============================================================================= -router - .route('/release-tracks/:id/snapshots/:modified/meta') - .post( - authn.authenticate, - authz.requireRole(authz.editorOrHigher), - releaseTracksController.updateMetadataByModified, - ); - -router - .route('/release-tracks/:id/snapshots/:modified/contents') - .post( - authn.authenticate, - authz.requireRole(authz.admin), - releaseTracksController.updateContentsByModified, - ); - router .route('/release-tracks/:id/snapshots/:modified/release/preview') .get( diff --git a/app/services/meta-classes/base.service.js b/app/services/meta-classes/base.service.js index 30a9b0c6..4bf2512d 100644 --- a/app/services/meta-classes/base.service.js +++ b/app/services/meta-classes/base.service.js @@ -23,6 +23,7 @@ const { AlreadyRevokedError, SelfRevocationError, MemberPinnedRevisionError, + SnapshotGraphPinnedRevisionError, } = require('../../exceptions'); const { getSchema } = require('../../lib/validation-schemas'); const { deepFreezeStix } = require('../../lib/import-safety'); @@ -374,9 +375,12 @@ class BaseService extends ServiceWithHooks { // Strip workspace.release_tracks — server-controlled; maintained by // release-track backref reconciliation, and pinned to specific revisions, // so a copy from a prior GET must not ride along onto a new version. + // Strip workspace.relationship_endpoints — relationship services resolve + // these exact endpoint pins from authoritative object revisions. if (data.workspace) { delete data.workspace.validation; delete data.workspace.release_tracks; + delete data.workspace.relationship_endpoints; } if (!options.preserveAttackId) { @@ -767,6 +771,48 @@ class BaseService extends ServiceWithHooks { } } + /** + * Protect every exact revision captured by an active or in-progress graph + * manifest. Relationship payloads are frozen in the manifest, so a + * non-topology PUT remains safe; relationship endpoint changes are rejected + * separately by RelationshipsService. Hard deletion is always rejected. + */ + static async assertNotGraphPinned(document, operation) { + const graphManifestService = require('../release-tracks/graph-manifest-service'); + const pins = await graphManifestService.findPinsForRevision( + document.stix.id, + document.stix.modified, + ); + if ( + pins.length === 0 || + (operation === 'updated' && pins.every((pin) => pin.kind === 'relationship')) + ) { + return; + } + + throw new SnapshotGraphPinnedRevisionError({ + details: + `Revision ${document.stix.id} (modified ` + + `${new Date(document.stix.modified).toISOString()}) is frozen in ` + + `${pins.length} release-track snapshot graph manifest(s) and cannot be ${operation} ` + + 'in place.', + snapshot_graph_pins: pins, + }); + } + + static async assertNoGraphPinnedVersions(stixId, operation) { + const graphManifestService = require('../release-tracks/graph-manifest-service'); + const pins = await graphManifestService.findPinsForObject(stixId); + if (pins.length === 0) return; + + throw new SnapshotGraphPinnedRevisionError({ + details: + `Object ${stixId} has revision(s) frozen in ${pins.length} release-track snapshot ` + + `graph manifest(s) and cannot be ${operation}.`, + snapshot_graph_pins: pins, + }); + } + /** * Refresh workspace.release_tracks on a response object after domain * events have run. The created/updated event is awaited, and its listeners @@ -876,6 +922,7 @@ class BaseService extends ServiceWithHooks { if (data.workspace) { delete data.workspace.validation; delete data.workspace.release_tracks; + delete data.workspace.relationship_endpoints; } // Extract ATT&CK ID from external_references and propagate to workspace.attack_id @@ -985,6 +1032,7 @@ class BaseService extends ServiceWithHooks { // Members-pinned revisions are released content — immutable in place. await BaseService.assertNotMemberPinned(document, 'updated'); + await BaseService.assertNotGraphPinned(document, 'updated'); // TODO: diff analysis — detect field-level changes vs document // TODO: if no changes detected, short-circuit (no-op) @@ -1108,6 +1156,7 @@ class BaseService extends ServiceWithHooks { return null; } await BaseService.assertNotMemberPinned(existing, 'deleted'); + await BaseService.assertNotGraphPinned(existing, 'deleted'); const document = await this.repository.findOneAndDelete(stixId, stixModified); @@ -1415,6 +1464,7 @@ class BaseService extends ServiceWithHooks { // Deleting all versions must not destroy a members-pinned revision const memberPinned = await this.repository.retrieveMemberPinnedVersionsLean(stixId); await BaseService.assertNoMemberPinnedVersions(stixId, memberPinned, 'deleted'); + await BaseService.assertNoGraphPinnedVersions(stixId, 'deleted'); const result = await this.repository.deleteMany(stixId); if (result.deletedCount > 0) { diff --git a/app/services/release-tracks/export-service.js b/app/services/release-tracks/export-service.js index fb469a4b..06911881 100644 --- a/app/services/release-tracks/export-service.js +++ b/app/services/release-tracks/export-service.js @@ -18,24 +18,16 @@ // ============================================================================= const config = require('../../config/config'); -const types = require('../../lib/types'); const logger = require('../../lib/logger'); const linkById = require('../../lib/linkById'); -const EventBus = require('../../lib/event-bus'); -const Events = require('../../lib/event-constants'); -const { selectRelationshipsForBundle } = require('../../lib/stix-bundle-relationships'); -const revisionReference = require('../../lib/release-tracks/revision-reference'); const primaryRevisionService = require('./primary-revision-service'); +const graphManifestService = require('./graph-manifest-service'); const { bundleTransformSchema, workbenchTransformSchema, filesystemStoreTransformSchema, } = require('../../lib/release-tracks/export-schemas'); -function getRepositoryMap() { - return primaryRevisionService.getRepositoryMap(); -} - // ============================================================================= // Hydration // ============================================================================= @@ -57,145 +49,34 @@ exports.hydrateMembers = async function hydrateMembers(entries) { // Bundle assembly helpers // ============================================================================= -/** - * Select the tier entries that belong in a bundle export. - * - * Members are always included. Staged and candidate entries are included only - * when named in `include`. When `state` is provided it further narrows the - * staged/candidate entries to those whose workflow status matches — except - * entries marked 'reviewed', which are always included irrespective of - * `state` (reviewed content is release-ready by definition, mirroring how all - * members are inherently reviewed). - * - * @param {Object} snapshot - The raw snapshot document - * @param {Object} options - { include?: Array<'staged'|'candidates'>, state?: Array } - * @returns {Array<{object_ref: string, object_modified: string|Date}>} Deduplicated entries - */ -function collectBundleEntries(snapshot, options) { - const include = options.include || []; - const state = options.state; - - const filterByState = (entries) => { - if (!state) return entries; - return entries.filter( - (entry) => entry.object_status === 'reviewed' || state.includes(entry.object_status), - ); - }; - - const entries = [...(snapshot.members || [])]; - if (include.includes('staged')) { - entries.push(...filterByState(snapshot.staged || [])); - } - if (include.includes('candidates')) { - entries.push(...filterByState(snapshot.candidates || [])); - } - - // Deduplicate by object_ref + object_modified - const seen = new Set(); - const deduped = []; - for (const entry of entries) { - const key = `${entry.object_ref}::` + revisionReference.modifiedKey(entry.object_modified); - if (seen.has(key)) continue; - seen.add(key); - deduped.push(entry); - } - - return deduped; -} - -/** - * Fetch identities and marking definitions referenced by the hydrated - * documents (via created_by_ref / object_marking_refs) that are not already - * part of the export. Emitted bundles must be self-contained, so referenced - * supporting objects are appended even though they are not tier entries. - * - * @param {Array} documents - Hydrated lean documents ({ stix, ... }) - * @returns {Promise>} Supporting lean documents - */ -async function fetchSupportingObjects(documents) { - const repoMap = getRepositoryMap(); - const existingIds = new Set(documents.map((doc) => doc.stix.id)); - - const identityIds = new Set(); - const markingIds = new Set(); - for (const doc of documents) { - if (doc.stix.created_by_ref && !existingIds.has(doc.stix.created_by_ref)) { - identityIds.add(doc.stix.created_by_ref); - } - for (const ref of doc.stix.object_marking_refs || []) { - if (!existingIds.has(ref)) markingIds.add(ref); - } - } - - const supportingObjects = []; - const fetchLatest = async (repo, stixId, description) => { - try { - const doc = await repo.retrieveLatestByStixIdLean(stixId); - if (doc) supportingObjects.push(doc); - else logger.warn(`ExportService: Referenced ${description} not found: ${stixId}`); - } catch (err) { - logger.warn(`ExportService: Could not fetch ${description} "${stixId}": ${err.message}`); - } - }; - - await Promise.all([ - ...[...identityIds].map((id) => fetchLatest(repoMap[types.Identity], id, 'identity')), - ...[...markingIds].map((id) => - fetchLatest(repoMap[types.MarkingDefinition], id, 'marking definition'), - ), - ]); - - return supportingObjects; -} - -/** - * Fetch the latest publishable relationships connecting selected bundle - * objects. Relationship revisions remain indirect export-time content rather - * than snapshot members. - * - * @param {Array} documents - Hydrated selected object documents - * @returns {Promise>} - */ -async function fetchRelationships(documents) { - const selectedIds = new Set(documents.map((document) => document.stix.id)); - if (selectedIds.size === 0) return []; - - const results = await EventBus.emit(Events.BUNDLE_RELATIONSHIPS_REQUESTED, { - objectRefs: [...selectedIds], - }); - const relationships = results?.[0]; - if (!relationships) { - throw new Error('Unable to retrieve relationships for release-track bundle export'); - } - - return selectRelationshipsForBundle(relationships, selectedIds).filter( - (relationship) => !selectedIds.has(relationship.stix.id), - ); -} - /** * Convert LinkById tags (e.g. "(LinkById: T1234)") in descriptions to - * markdown citations, preferring objects already in the export before - * falling back to a database lookup. Mirrors the legacy stix-bundles-service - * behavior so bundles emitted from release tracks match published output. + * markdown citations using only object revisions frozen in the manifest. * * @param {Array} documents - Hydrated lean documents ({ stix, ... }) */ -async function convertLinkByIdTags(documents) { +async function convertLinkByIdTags(documents, linkTargetDocuments) { const byAttackId = new Map(); - for (const doc of documents) { + for (const doc of [...documents, ...linkTargetDocuments]) { const attackId = linkById.getAttackId(doc.stix); if (attackId) byAttackId.set(attackId, doc); } - const getAttackObject = async (attackId) => - byAttackId.get(attackId) || (await linkById.getAttackObjectFromDatabase(attackId)); + const getAttackObject = async (attackId) => byAttackId.get(attackId); for (const doc of documents) { await linkById.convertLinkByIdTags(doc.stix, getAttackObject); } } +function selectedDraftTierIsDynamic(snapshot, options) { + return (options.include || []).some( + (tier) => + ['staged', 'candidates'].includes(tier) && + (snapshot[tier] || []).some((entry) => entry.object_modified === 'latest'), + ); +} + // ============================================================================= // Format helpers (delegating to Zod transform schemas) // ============================================================================= @@ -266,12 +147,12 @@ exports.formatAsFilesystemStore = function formatAsFilesystemStore(snapshot, hyd */ exports.exportSnapshot = async function exportSnapshot(snapshot, format, options = {}) { if (format === 'bundle') { - const entries = collectBundleEntries(snapshot, options); - const hydratedObjects = await exports.hydrateMembers(entries); - const relationships = await fetchRelationships(hydratedObjects); - const supportingObjects = await fetchSupportingObjects([...hydratedObjects, ...relationships]); - const allObjects = [...hydratedObjects, ...relationships, ...supportingObjects]; - await convertLinkByIdTags(allObjects); + const graph = + options.captureGraph || selectedDraftTierIsDynamic(snapshot, options) + ? await graphManifestService.replayPlannedSnapshot(snapshot, options) + : await graphManifestService.replay(snapshot, options); + const allObjects = graph.documents; + await convertLinkByIdTags(allObjects, graph.linkTargetDocuments); return exports.formatAsBundle(snapshot, allObjects, { stixVersion: options.stixVersion, diff --git a/app/services/release-tracks/graph-manifest-service.js b/app/services/release-tracks/graph-manifest-service.js new file mode 100644 index 00000000..1ed69ade --- /dev/null +++ b/app/services/release-tracks/graph-manifest-service.js @@ -0,0 +1,542 @@ +'use strict'; + +const { v4: uuidv4 } = require('uuid'); +const linkById = require('../../lib/linkById'); +const bundleRelationships = require('../../lib/stix-bundle-relationships'); +const attackObjectsRepository = require('../../repository/attack-objects-repository'); +const relationshipsRepository = require('../../repository/relationships-repository'); +const detectionStrategiesRepository = require('../../repository/detection-strategies-repository'); +const BundleGraphResolver = require('../stix/bundle-graph-resolver'); +const { + ReleaseTrackGraphManifest, + ReleaseTrackGraphManifestEntry, +} = require('../../models/release-tracks/release-track-graph-manifest-model'); +const { ReleaseContentIntegrityError } = require('../../exceptions'); +const primaryRevisionService = require('./primary-revision-service'); + +const RESOLVER_VERSION = 'bounded-attack-graph-v1'; +const TIERS = ['members', 'staged', 'candidates', 'quarantine']; +const MUTATION_PROTECTED_ENTRY_FILTER = { + $or: [ + { kind: { $ne: 'root' } }, + { kind: 'root', tier: { $in: ['members', 'quarantine'] } }, + { kind: 'root', 'discovered_from.0': { $exists: true } }, + ], +}; + +function revisionKey(objectRef, objectModified) { + return `${objectRef}::${new Date(objectModified).getTime()}`; +} + +function endpointFor(relationship, side) { + const endpoint = relationship.workspace?.relationship_endpoints?.[side]; + const objectRef = relationship.stix[`${side}_ref`]; + if (!endpoint || endpoint.object_ref !== objectRef || !endpoint.object_modified) { + return null; + } + return { + object_ref: endpoint.object_ref, + object_modified: endpoint.object_modified, + }; +} + +async function buildManifestEntries(snapshot) { + const rootRequests = []; + for (const tier of TIERS) { + for (const entry of snapshot[tier] || []) { + rootRequests.push({ ...entry, tier }); + } + } + + const hydratedRoots = await primaryRevisionService.assertStoredEntries(rootRequests); + const rootMetadata = new Map( + hydratedRoots.entries.map((entry) => [ + revisionKey(entry.object_ref, entry.object_modified), + entry, + ]), + ); + + const rootObjectRefs = new Set(hydratedRoots.entries.map((entry) => entry.object_ref)); + + const relationships = await relationshipsRepository.retrieveAllForBundle({ + includeRevoked: false, + includeDeprecated: false, + }); + const missing = []; + const pinnedRelationships = []; + for (const relationship of relationships) { + const source = endpointFor(relationship, 'source'); + const target = endpointFor(relationship, 'target'); + if (!source || !target) { + // Legacy relationships outside this snapshot's bounded graph cannot + // affect its replay. Fail closed only when an unpinned relationship + // touches a primary member by STIX ID. + if ( + rootObjectRefs.has(relationship.stix.source_ref) || + rootObjectRefs.has(relationship.stix.target_ref) + ) { + missing.push({ + object_ref: relationship.stix.id, + object_modified: new Date(relationship.stix.modified).toISOString(), + dependency: 'relationship_endpoints', + }); + } + continue; + } + pinnedRelationships.push({ relationship, source, target }); + } + if (missing.length > 0) { + throw new ReleaseContentIntegrityError(missing, { + details: 'Snapshot graph capture found relationships without exact endpoint pins.', + }); + } + + const graphResolver = new BundleGraphResolver({ + attackObjectsRepository, + detectionStrategiesRepository, + repositoryMap: primaryRevisionService.getRepositoryMap(), + policy: { + isDeprecatedPattern: bundleRelationships.isDeprecatedPattern, + relationshipIsActive: bundleRelationships.relationshipIsActive, + secondaryObjectIsValid: (document) => Boolean(document), + }, + options: { + inferDomains: false, + includeRevoked: true, + includeDeprecated: true, + includeMissingAttackId: true, + }, + relationships: pinnedRelationships.map((candidate) => candidate.relationship), + onMissingDependency(reference) { + missing.push({ + ...reference, + object_modified: new Date(reference.object_modified).toISOString(), + }); + }, + }); + const resolvedGraph = await graphResolver.resolve(hydratedRoots.documents); + if (missing.length > 0) { + const uniqueMissing = [ + ...new Map( + missing.map((reference) => [ + `${reference.object_ref}::${reference.object_modified}`, + reference, + ]), + ).values(), + ]; + throw new ReleaseContentIntegrityError(uniqueMissing, { + details: 'Snapshot graph capture could not hydrate every exact dependency.', + }); + } + const selectedDocuments = new Map( + resolvedGraph.documents.map((document) => [ + revisionKey(document.stix.id, document.stix.modified), + document, + ]), + ); + const selectedRelationships = resolvedGraph.relationships.map((relationship) => ({ + relationship, + source: endpointFor(relationship, 'source'), + target: endpointFor(relationship, 'target'), + })); + const relationshipDocuments = selectedRelationships.map((candidate) => candidate.relationship); + const discoverySources = resolvedGraph.dependencies; + const supportingDocuments = await graphResolver.loadSupportingDocuments(resolvedGraph.objects); + + const linkTargets = new Map(); + for (const document of [...selectedDocuments.values(), ...relationshipDocuments]) { + for (const attackId of linkById.extractLinkByIds(document.stix)) { + if (!linkTargets.has(attackId)) { + const target = await linkById.getAttackObjectFromDatabase(attackId); + if (target) { + linkTargets.set(attackId, target); + } + } + } + } + + const entries = []; + for (const [key, document] of selectedDocuments) { + const root = rootMetadata.get(key); + entries.push({ + revision_key: key, + kind: root ? 'root' : 'secondary', + tier: root?.tier, + object_status: root?.object_status, + object_ref: document.stix.id, + object_modified: document.stix.modified, + discovered_from: discoverySources.get(key) || [], + }); + } + for (const candidate of selectedRelationships) { + entries.push({ + revision_key: revisionKey( + candidate.relationship.stix.id, + candidate.relationship.stix.modified, + ), + kind: 'relationship', + object_ref: candidate.relationship.stix.id, + object_modified: candidate.relationship.stix.modified, + source: candidate.source, + target: candidate.target, + frozen_stix: candidate.relationship.stix, + }); + } + for (const document of supportingDocuments) { + const isVersioned = Boolean(document.stix.modified); + entries.push({ + revision_key: isVersioned + ? revisionKey(document.stix.id, document.stix.modified) + : `${document.stix.id}::unversioned`, + kind: 'supporting', + object_ref: document.stix.id, + object_modified: document.stix.modified, + frozen_stix: isVersioned ? undefined : document.stix, + }); + } + for (const document of linkTargets.values()) { + entries.push({ + revision_key: revisionKey(document.stix.id, document.stix.modified), + kind: 'link_target', + object_ref: document.stix.id, + object_modified: document.stix.modified, + }); + } + + return entries; +} + +async function prepare(snapshot, options = {}) { + const manifestId = `release-track-graph-manifest--${uuidv4()}`; + const entries = await buildManifestEntries(snapshot); + const common = { + manifest_id: manifestId, + track_id: snapshot.id, + snapshot_modified: snapshot.modified, + }; + + await ReleaseTrackGraphManifest.create({ + ...common, + state: 'pending', + resolver_version: RESOLVER_VERSION, + baseline_reconstruction: options.baselineReconstruction === true, + }); + try { + if (entries.length > 0) { + await ReleaseTrackGraphManifestEntry.insertMany( + entries.map((entry) => ({ ...common, ...entry })), + ); + } + } catch (err) { + await discard(manifestId); + throw err; + } + return manifestId; +} + +async function activate(manifestId) { + await ReleaseTrackGraphManifest.updateOne( + { manifest_id: manifestId, state: 'pending' }, + { $set: { state: 'active' } }, + ).exec(); +} + +async function discard(manifestId) { + await Promise.all([ + ReleaseTrackGraphManifestEntry.deleteMany({ manifest_id: manifestId }).exec(), + ReleaseTrackGraphManifest.deleteOne({ manifest_id: manifestId }).exec(), + ]); +} + +async function discardSnapshot(trackId, snapshotModified) { + const manifests = await ReleaseTrackGraphManifest.find({ + track_id: trackId, + snapshot_modified: snapshotModified, + }) + .select({ manifest_id: 1, _id: 0 }) + .lean() + .exec(); + const manifestIds = manifests.map((manifest) => manifest.manifest_id); + if (manifestIds.length === 0) return; + + await Promise.all([ + ReleaseTrackGraphManifestEntry.deleteMany({ + manifest_id: { $in: manifestIds }, + }).exec(), + ReleaseTrackGraphManifest.deleteMany({ + manifest_id: { $in: manifestIds }, + }).exec(), + ]); +} + +async function discardTrack(trackId) { + const manifests = await ReleaseTrackGraphManifest.find({ track_id: trackId }) + .select({ manifest_id: 1, _id: 0 }) + .lean() + .exec(); + const manifestIds = manifests.map((manifest) => manifest.manifest_id); + + await Promise.all([ + manifestIds.length > 0 + ? ReleaseTrackGraphManifestEntry.deleteMany({ + manifest_id: { $in: manifestIds }, + }).exec() + : Promise.resolve(), + ReleaseTrackGraphManifest.deleteMany({ track_id: trackId }).exec(), + ]); +} + +function rootIsSelected(entry, options) { + if (entry.tier === 'members') return true; + if (!['staged', 'candidates'].includes(entry.tier)) return false; + if (!(options.include || []).includes(entry.tier)) return false; + if (!options.state) return true; + return entry.object_status === 'reviewed' || options.state.includes(entry.object_status); +} + +async function replayEntries(entries, manifest, options) { + const versionedEntries = entries.filter( + (entry) => entry.object_modified && entry.kind !== 'relationship', + ); + const hydrated = await primaryRevisionService.assertStoredEntries( + versionedEntries.map((entry) => ({ + object_ref: entry.object_ref, + object_modified: entry.object_modified, + })), + ); + const documentsByRevision = new Map( + hydrated.documents.map((document) => [ + revisionKey(document.stix.id, document.stix.modified), + document, + ]), + ); + for (const entry of entries) { + if (entry.kind === 'relationship' && entry.frozen_stix) { + documentsByRevision.set(entry.revision_key, { + stix: entry.frozen_stix, + }); + } + } + + const selectedRevisionKeys = new Set( + entries + .filter((entry) => entry.kind === 'root' && rootIsSelected(entry, options)) + .map((entry) => entry.revision_key), + ); + + // Special embedded-reference dependencies can be chained (for example, a + // detection strategy discovered through an analytic that was itself a + // relationship secondary). Replay only follows edges frozen in the + // manifest; it never asks the live database to expand the graph. + let added; + do { + added = false; + for (const entry of entries) { + if ( + !['root', 'secondary'].includes(entry.kind) || + selectedRevisionKeys.has(entry.revision_key) + ) { + continue; + } + if ( + (entry.discovered_from || []).some((source) => + selectedRevisionKeys.has(revisionKey(source.object_ref, source.object_modified)), + ) + ) { + selectedRevisionKeys.add(entry.revision_key); + added = true; + } + } + } while (added); + + const selectedRelationships = entries.filter( + (entry) => + entry.kind === 'relationship' && + selectedRevisionKeys.has( + revisionKey(entry.source.object_ref, entry.source.object_modified), + ) && + selectedRevisionKeys.has(revisionKey(entry.target.object_ref, entry.target.object_modified)), + ); + for (const entry of selectedRelationships) { + selectedRevisionKeys.add(entry.revision_key); + } + + const selectedDocuments = [...selectedRevisionKeys] + .map((key) => documentsByRevision.get(key)) + .filter(Boolean); + const supportingRefs = new Set(); + for (const document of selectedDocuments) { + if (document.stix.created_by_ref) { + supportingRefs.add(document.stix.created_by_ref); + } + for (const objectRef of document.stix.object_marking_refs || []) { + supportingRefs.add(objectRef); + } + } + + const supportingDocuments = entries + .filter((entry) => entry.kind === 'supporting' && supportingRefs.has(entry.object_ref)) + .map((entry) => + entry.object_modified + ? documentsByRevision.get(entry.revision_key) + : { stix: entry.frozen_stix }, + ) + .filter(Boolean); + const linkTargetDocuments = entries + .filter((entry) => entry.kind === 'link_target') + .map((entry) => documentsByRevision.get(entry.revision_key)) + .filter(Boolean); + + const emittedByRevision = new Map(); + for (const document of [...selectedDocuments, ...supportingDocuments]) { + const key = document.stix.modified + ? revisionKey(document.stix.id, document.stix.modified) + : `${document.stix.id}::unversioned`; + emittedByRevision.set(key, document); + } + + return { + documents: [...emittedByRevision.values()], + linkTargetDocuments, + manifest, + }; +} + +async function replay(snapshot, options = {}) { + if (!snapshot.graph_manifest_id) { + throw new ReleaseContentIntegrityError( + [ + { + track_id: snapshot.id, + snapshot_modified: new Date(snapshot.modified).toISOString(), + dependency: 'graph_manifest', + }, + ], + { details: 'Snapshot does not reference a deterministic graph manifest.' }, + ); + } + + const manifest = await ReleaseTrackGraphManifest.findOne({ + manifest_id: snapshot.graph_manifest_id, + track_id: snapshot.id, + snapshot_modified: snapshot.modified, + state: { $in: ['pending', 'active'] }, + }) + .lean() + .exec(); + if (!manifest) { + throw new ReleaseContentIntegrityError( + [{ manifest_id: snapshot.graph_manifest_id, dependency: 'graph_manifest' }], + { details: 'Snapshot graph manifest is missing.' }, + ); + } + + // A snapshot link is the durable commit record. If the process stopped + // after linking a complete pending manifest but before activation, replay + // remains deterministic and repairs the visibility marker opportunistically. + if (manifest.state === 'pending') { + await activate(manifest.manifest_id); + manifest.state = 'active'; + } + + const entries = await ReleaseTrackGraphManifestEntry.find({ + manifest_id: manifest.manifest_id, + }) + .lean() + .exec(); + return replayEntries(entries, manifest, options); +} + +async function replayPlannedSnapshot(snapshot, options = {}) { + const entries = await buildManifestEntries(snapshot); + return replayEntries( + entries, + { + manifest_id: null, + track_id: snapshot.id, + snapshot_modified: snapshot.modified, + state: 'preview', + resolver_version: RESOLVER_VERSION, + }, + options, + ); +} + +async function findPinsForRevision(objectRef, objectModified) { + const entries = await ReleaseTrackGraphManifestEntry.find({ + object_ref: objectRef, + object_modified: objectModified, + ...MUTATION_PROTECTED_ENTRY_FILTER, + }) + .select({ + manifest_id: 1, + track_id: 1, + snapshot_modified: 1, + kind: 1, + tier: 1, + _id: 0, + }) + .lean() + .exec(); + if (entries.length === 0) return []; + + const protectedManifestIds = new Set( + ( + await ReleaseTrackGraphManifest.find({ + manifest_id: { $in: entries.map((entry) => entry.manifest_id) }, + state: { $in: ['pending', 'active'] }, + }) + .select({ manifest_id: 1, _id: 0 }) + .lean() + .exec() + ).map((manifest) => manifest.manifest_id), + ); + return entries.filter((entry) => protectedManifestIds.has(entry.manifest_id)); +} + +async function findPinsForObject(objectRef) { + const entries = await ReleaseTrackGraphManifestEntry.find({ + object_ref: objectRef, + object_modified: { $ne: null }, + ...MUTATION_PROTECTED_ENTRY_FILTER, + }) + .select({ + manifest_id: 1, + track_id: 1, + snapshot_modified: 1, + object_modified: 1, + kind: 1, + tier: 1, + _id: 0, + }) + .lean() + .exec(); + if (entries.length === 0) return []; + + const protectedManifestIds = new Set( + ( + await ReleaseTrackGraphManifest.find({ + manifest_id: { $in: entries.map((entry) => entry.manifest_id) }, + state: { $in: ['pending', 'active'] }, + }) + .select({ manifest_id: 1, _id: 0 }) + .lean() + .exec() + ).map((manifest) => manifest.manifest_id), + ); + return entries.filter((entry) => protectedManifestIds.has(entry.manifest_id)); +} + +module.exports = { + prepare, + activate, + discard, + discardSnapshot, + discardTrack, + replay, + replayPlannedSnapshot, + findPinsForRevision, + findPinsForObject, + buildManifestEntries, + RESOLVER_VERSION, +}; diff --git a/app/services/release-tracks/primary-revision-service.js b/app/services/release-tracks/primary-revision-service.js index 9b12b7ea..e8863355 100644 --- a/app/services/release-tracks/primary-revision-service.js +++ b/app/services/release-tracks/primary-revision-service.js @@ -19,6 +19,7 @@ function getRepositoryMap() { [types.Tactic]: require('../../repository/tactics-repository'), [types.Group]: require('../../repository/groups-repository'), [types.Campaign]: require('../../repository/campaigns-repository'), + [types.Collection]: require('../../repository/collections-repository'), [types.Mitigation]: require('../../repository/mitigations-repository'), [types.Matrix]: require('../../repository/matrix-repository'), [types.Relationship]: require('../../repository/relationships-repository'), diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index 7a2ebab0..234f52ca 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -6,7 +6,7 @@ // Orchestrator that delegates to domain-specific sub-services. This is the // single entry point consumed by the controller layer. // -// Phase 1: Track management, snapshot CRUD, config → snapshot-service +// Phase 1: Track management, snapshot lifecycle, config → snapshot-service // Phase 2: Candidates, staged, object versions → standard-track-service // Phase 3: Auto-promotion, workflow → workflow-service // Phase 4: Release planning and versioning → versioning-service @@ -289,49 +289,6 @@ exports.updateMetadata = function updateMetadata(trackId, updates, userId) { return snapshotService.updateMetadata(trackId, updates, userId); }; -exports.updateMetadataByModified = function updateMetadataByModified( - trackId, - modified, - updates, - userId, -) { - return snapshotService.updateMetadataByModified(trackId, modified, updates, userId); -}; - -exports.updateContents = function updateContents(trackId, contents, actor, confirmation) { - return destructiveAuditService.execute( - { - action: 'replace_members_latest', - trackId, - ...destructiveIdentity(trackId, actor, confirmation), - request: { members_count: contents.x_mitre_contents.length }, - }, - () => snapshotService.updateContents(trackId, contents, actor?.user_account_id), - ); -}; - -exports.updateContentsByModified = function updateContentsByModified( - trackId, - modified, - contents, - actor, - confirmation, -) { - return destructiveAuditService.execute( - { - action: 'replace_members_historical', - trackId, - ...destructiveIdentity(trackId, actor, confirmation), - request: { - source_snapshot_modified: modified, - members_count: contents.x_mitre_contents.length, - }, - }, - () => - snapshotService.updateContentsByModified(trackId, modified, contents, actor?.user_account_id), - ); -}; - exports.cloneTrack = function cloneTrack(trackId, options) { return snapshotService.cloneTrack(trackId, options); }; @@ -425,7 +382,10 @@ async function renderReleasePlan(plan, options) { if (format === 'summary') return plan.summary; if (plan.blockingError) throw plan.blockingError; if (format === 'bundle') { - return exportService.exportSnapshot(plan.plannedSnapshot, format, options); + return exportService.exportSnapshot(plan.plannedSnapshot, format, { + ...options, + captureGraph: true, + }); } return formatWorkbenchSnapshot(plan.plannedSnapshot, options); } diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 45ee635a..ddf8e334 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -4,7 +4,7 @@ // Snapshot Service // // Core snapshot lifecycle operations: track creation, retrieval, cloning, -// metadata/contents updates, configuration, and deletion. +// metadata updates, configuration, and deletion. // // This is the foundational sub-service consumed by the facade and by other // sub-services (standard-track, versioning, virtual-track) that need to @@ -21,11 +21,12 @@ const versionUtils = require('../../lib/release-tracks/version-utils'); const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); const primaryRevisionService = require('./primary-revision-service'); const reconciliationService = require('./reconciliation-service'); +const graphManifestService = require('./graph-manifest-service'); const { TrackNotFoundError, NotFoundError, TaggedSnapshotDeletionError, - BadRequestError, + HistoricalSnapshotDeletionError, } = require('../../exceptions'); // ============================================================================= @@ -53,35 +54,6 @@ function normalizeTierSummary(summary) { }; } -function assertStandardTrack(snapshot) { - if (snapshot.type !== 'standard') { - throw new BadRequestError({ - message: 'Direct contents updates are only available for standard release tracks', - details: - 'Virtual members are computed from component tracks; create a virtual snapshot to update them', - }); - } -} - -/** - * Convert contents request entries into exact revision pins. - * - * `latest` is request-time shorthand only. It must never be persisted because - * snapshot membership is defined by an immutable `(object_ref, - * object_modified)` pair. - * - * @param {Array<{obj_ref: string, obj_modified: string}>} contents - * @returns {Promise>} - */ -async function resolveContentsMembers(contents) { - const requested = contents.map((entry) => ({ - object_ref: entry.obj_ref, - object_modified: - entry.obj_modified === 'latest' ? entry.obj_modified : new Date(entry.obj_modified), - })); - return (await primaryRevisionService.assertRequestEntries(requested)).entries; -} - /** * Recompute and persist denormalized registry counters from actual snapshot data. * @@ -132,6 +104,38 @@ async function emitContentsChanged(trackId, snapshot) { } exports.emitContentsChanged = emitContentsChanged; +/** + * Persist a snapshot and its graph manifest as one logical operation. + * + * Mongo transactions are not available across the dynamically named snapshot + * collections in every supported deployment. A pending manifest plus + * compensation keeps partially completed writes invisible to replay and + * protection queries. + */ +async function saveSnapshotWithManifest(trackId, snapshotData) { + const manifestId = await graphManifestService.prepare(snapshotData); + snapshotData.graph_manifest_id = manifestId; + + try { + const saved = await dynamicRepo.saveSnapshot(trackId, snapshotData); + try { + await graphManifestService.activate(manifestId); + } catch (err) { + // The saved snapshot is already linked to a complete pending manifest. + // Replay can safely activate it later, so do not turn a committed + // snapshot into an ambiguous client-visible failure. + logger.warn( + `SnapshotService: Deferred activation for graph manifest "${manifestId}": ${err.message}`, + ); + } + return saved; + } catch (err) { + await graphManifestService.discard(manifestId); + throw err; + } +} +exports.saveSnapshotWithManifest = saveSnapshotWithManifest; + // ============================================================================= // Track management // ============================================================================= @@ -192,7 +196,7 @@ exports.createTrack = async function createTrack(data) { // Create collection + indexes, then persist the initial snapshot await modelFactory.ensureIndexes(trackId); - const snapshot = await dynamicRepo.saveSnapshot(trackId, initialSnapshot); + const snapshot = await saveSnapshotWithManifest(trackId, initialSnapshot); // Register in the central registry await registryRepo.create({ @@ -317,6 +321,7 @@ exports.getSnapshotByModified = async function getSnapshotByModified(trackId, mo */ exports.cloneSnapshot = async function cloneSnapshot(trackId, sourceSnapshot, overrides) { const clone = deepClone(sourceSnapshot); + delete clone.graph_manifest_id; clone.modified = new Date(); clone.version = null; // clones are always drafts delete clone.scheduled_materialization; @@ -331,7 +336,7 @@ exports.cloneSnapshot = async function cloneSnapshot(trackId, sourceSnapshot, ov } const normalized = tierRevisionInvariant.normalizeSnapshot(clone); - const saved = await dynamicRepo.saveSnapshot(trackId, normalized.snapshot); + const saved = await saveSnapshotWithManifest(trackId, normalized.snapshot); await syncRegistryCounters(trackId); // The clone (modified = now) is the track's new latest snapshot @@ -384,6 +389,7 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { const now = new Date(); const clone = deepClone(sourceSnapshot); + delete clone.graph_manifest_id; clone.id = newTrackId; clone.modified = now; clone.version = null; @@ -399,7 +405,7 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { ); await modelFactory.ensureIndexes(newTrackId); - const saved = await dynamicRepo.saveSnapshot(newTrackId, normalized.snapshot); + const saved = await saveSnapshotWithManifest(newTrackId, normalized.snapshot); await registryRepo.create({ track_id: newTrackId, @@ -459,82 +465,6 @@ exports.updateMetadata = async function updateMetadata(trackId, updates, _userId return exports.cloneSnapshot(trackId, source, overrides); }; -/** - * Update metadata on a specific snapshot (creates a new snapshot clone). - * - * @param {string} trackId - * @param {string|Date} modified - * @param {Object} updates - { name?, description?, object_marking_refs? } - * @param {string} [_userId] - * @returns {Promise} The new snapshot - */ -exports.updateMetadataByModified = async function updateMetadataByModified( - trackId, - modified, - updates, - // eslint-disable-next-line no-unused-vars - _userId, -) { - const source = await exports.getSnapshotByModified(trackId, modified); - const overrides = {}; - if (updates.name !== undefined) overrides.name = updates.name; - if (updates.description !== undefined) overrides.description = updates.description; - if (updates.object_marking_refs !== undefined) - overrides.object_marking_refs = updates.object_marking_refs; - - const registryUpdates = {}; - if (updates.name !== undefined) registryUpdates.name = updates.name; - if (updates.description !== undefined) registryUpdates.description = updates.description; - if (Object.keys(registryUpdates).length > 0) { - registryUpdates.updated_at = new Date(); - await registryRepo.updateByTrackId(trackId, registryUpdates); - } - - return exports.cloneSnapshot(trackId, source, overrides); -}; - -// ============================================================================= -// Contents updates -// ============================================================================= - -/** - * Replace member contents on the latest snapshot (creates a new snapshot clone). - * - * @param {string} trackId - * @param {Object} contents - { x_mitre_contents: [{ obj_ref, obj_modified }] } - * @param {string} [_userId] - * @returns {Promise} The new snapshot - */ -// eslint-disable-next-line no-unused-vars -exports.updateContents = async function updateContents(trackId, contents, _userId) { - const source = await exports.getLatestSnapshot(trackId); - assertStandardTrack(source); - const members = await resolveContentsMembers(contents.x_mitre_contents); - return exports.cloneSnapshot(trackId, source, { members }); -}; - -/** - * Replace member contents on a specific snapshot (creates a new snapshot clone). - * - * @param {string} trackId - * @param {string|Date} modified - * @param {Object} contents - { x_mitre_contents: [{ obj_ref, obj_modified }] } - * @param {string} [_userId] - * @returns {Promise} The new snapshot - */ -exports.updateContentsByModified = async function updateContentsByModified( - trackId, - modified, - contents, - // eslint-disable-next-line no-unused-vars - _userId, -) { - const source = await exports.getSnapshotByModified(trackId, modified); - assertStandardTrack(source); - const members = await resolveContentsMembers(contents.x_mitre_contents); - return exports.cloneSnapshot(trackId, source, { members }); -}; - // ============================================================================= // Configuration // ============================================================================= @@ -608,10 +538,14 @@ exports.updateConfig = async function updateConfig(trackId, config, _userId) { exports.deleteTrack = async function deleteTrack(trackId) { const registry = await registryRepo.findByTrackId(trackId); if (!registry) { + // A previous delete may have removed the registry only after dropping the + // dynamic snapshot collection but stopped before manifest cleanup. + await graphManifestService.discardTrack(trackId); throw new TrackNotFoundError(trackId); } await dynamicRepo.dropCollection(trackId); + await graphManifestService.discardTrack(trackId); await registryRepo.deleteByTrackId(trackId); // Remove all backrefs to the deleted track @@ -630,6 +564,9 @@ exports.deleteTrack = async function deleteTrack(trackId) { exports.deleteSnapshot = async function deleteSnapshot(trackId, modified) { const snapshot = await dynamicRepo.getSnapshotByModified(trackId, modified); if (!snapshot) { + // Make a retry after an interrupted delete clean any orphaned manifests + // even though the snapshot document is already gone. + await graphManifestService.discardSnapshot(trackId, modified); throw new NotFoundError({ details: `Snapshot with modified '${modified}' not found for track '${trackId}'`, }); @@ -639,13 +576,19 @@ exports.deleteSnapshot = async function deleteSnapshot(trackId, modified) { throw new TaggedSnapshotDeletionError(snapshot.version); } + const latest = await dynamicRepo.getLatestSnapshot(trackId); + if (!latest || new Date(latest.modified).getTime() !== new Date(snapshot.modified).getTime()) { + throw new HistoricalSnapshotDeletionError(snapshot.modified, latest?.modified); + } + await dynamicRepo.deleteSnapshot(trackId, modified); + await graphManifestService.discardSnapshot(trackId, snapshot.modified); await syncRegistryCounters(trackId); // Deleting the latest snapshot reverts membership to the previous snapshot // (or clears it if no snapshots remain) - const latest = await dynamicRepo.getLatestSnapshot(trackId); - await emitContentsChanged(trackId, latest); + const revertedLatest = await dynamicRepo.getLatestSnapshot(trackId); + await emitContentsChanged(trackId, revertedLatest); logger.verbose(`SnapshotService: Deleted snapshot '${modified}' from track "${trackId}"`); }; diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index 6f902d2b..702511fc 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -12,6 +12,7 @@ const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-in const revisionReference = require('../../lib/release-tracks/revision-reference'); const releaseHistoryService = require('./release-history-service'); const primaryRevisionService = require('./primary-revision-service'); +const graphManifestService = require('./graph-manifest-service'); const logger = require('../../lib/logger'); const { AlreadyReleasedError, @@ -280,17 +281,47 @@ async function planLoadedSnapshot(trackId, snapshot, options) { async function commitPlan(plan) { if (plan.blockingError) throw plan.blockingError; - const tagged = await dynamicRepo.tagSnapshotInPlace(plan.trackId, plan.sourceSnapshot.modified, { - version: plan.version, - versionHistoryEntry: plan.versionHistoryEntry, - additionalOps: Object.keys(plan.additionalOps).length > 0 ? plan.additionalOps : undefined, - }); + const manifestId = await graphManifestService.prepare(plan.plannedSnapshot); + + let tagged; + try { + tagged = await dynamicRepo.tagSnapshotInPlace(plan.trackId, plan.sourceSnapshot.modified, { + version: plan.version, + versionHistoryEntry: plan.versionHistoryEntry, + additionalOps: { + ...plan.additionalOps, + graph_manifest_id: manifestId, + }, + }); + } catch (err) { + await graphManifestService.discard(manifestId); + throw err; + } if (!tagged) { + await graphManifestService.discard(manifestId); await releaseHistoryService.reconcileTaggedReleases(plan.trackId); throw new AlreadyReleasedError('(concurrent release)'); } + // Link the complete pending manifest before activation. The snapshot link + // is the durable commit record, and replay can recover a linked pending + // manifest if the process stops in this narrow window. + try { + await graphManifestService.activate(manifestId); + } catch (err) { + logger.warn( + `VersioningService: Deferred activation for graph manifest "${manifestId}": ${err.message}`, + ); + } + + if ( + plan.sourceSnapshot.graph_manifest_id && + plan.sourceSnapshot.graph_manifest_id !== manifestId + ) { + await graphManifestService.discard(plan.sourceSnapshot.graph_manifest_id); + } + await releaseHistoryService.reconcileTaggedReleases(plan.trackId); const latest = await dynamicRepo.getLatestSnapshot(plan.trackId); await snapshotService.emitContentsChanged(plan.trackId, latest); diff --git a/app/services/stix/bundle-graph-resolver.js b/app/services/stix/bundle-graph-resolver.js new file mode 100644 index 00000000..9374bafb --- /dev/null +++ b/app/services/stix/bundle-graph-resolver.js @@ -0,0 +1,469 @@ +'use strict'; + +const _ = require('lodash'); +const linkById = require('../../lib/linkById'); +const logger = require('../../lib/logger'); + +/** + * Resolves the bounded ATT&CK object graph used by bundle exports. + * + * A resolver instance belongs to exactly one export request. Keeping its + * caches, relationship set, and inferred-domain state request-local prevents + * overlapping exports from observing or overwriting each other's state. + * + * This resolver deliberately preserves the legacy one-hop relationship + * expansion and named ATT&CK special cases. It is not a general transitive + * graph walker. + */ +class BundleGraphResolver { + /** + * @param {Object} dependencies + * @param {Object} dependencies.attackObjectsRepository + * @param {Object} dependencies.detectionStrategiesRepository + * @param {Object} dependencies.policy + * @param {Function} dependencies.policy.isDeprecatedPattern + * @param {Function} dependencies.policy.relationshipIsActive + * @param {Function} dependencies.policy.secondaryObjectIsValid + * @param {Object} dependencies.options + * @param {Array} dependencies.relationships + */ + constructor({ + attackObjectsRepository, + detectionStrategiesRepository, + repositoryMap, + policy, + options, + relationships, + onMissingDependency, + }) { + this.attackObjectsRepository = attackObjectsRepository; + this.detectionStrategiesRepository = detectionStrategiesRepository; + this.repositoryMap = repositoryMap; + this.exactEndpoints = Boolean(repositoryMap); + this.policy = policy; + this.options = options; + this.relationships = _.cloneDeep(relationships); + this.onMissingDependency = onMissingDependency; + + this.attackObjectCache = new Map(); + this.attackObjectByAttackIdCache = new Map(); + this.domainCache = new Map(); + this.dependencies = new Map(); + } + + revisionKey(objectRef, objectModified) { + return `${objectRef}::${new Date(objectModified).getTime()}`; + } + + documentKey(document) { + return this.exactEndpoints + ? this.revisionKey(document.stix.id, document.stix.modified) + : document.stix.id; + } + + endpointKey(relationship, side) { + const objectRef = relationship.stix[`${side}_ref`]; + if (!this.exactEndpoints) return objectRef; + const endpoint = relationship.workspace?.relationship_endpoints?.[side]; + if (endpoint?.object_ref !== objectRef || !endpoint.object_modified) { + return `${objectRef}::unpinned`; + } + return this.revisionKey(endpoint.object_ref, endpoint.object_modified); + } + + hasEndpoint(objectsMap, relationship, side) { + return objectsMap.has(this.endpointKey(relationship, side)); + } + + endpointDocument(objectsMap, relationship, side) { + return objectsMap.get(this.endpointKey(relationship, side)); + } + + rememberDependency(document, sourceDocument) { + if (!document || !sourceDocument) return; + const key = this.documentKey(document); + const sources = this.dependencies.get(key) || new Map(); + sources.set(this.documentKey(sourceDocument), { + object_ref: sourceDocument.stix.id, + object_modified: sourceDocument.stix.modified, + }); + this.dependencies.set(key, sources); + } + + /** + * Resolve the object and relationship graph for the supplied primary roots. + * + * @param {Array} primaryObjects Workbench-shaped primary documents + * @returns {Promise<{ + * objects: Array, + * documents: Array, + * relationships: Array, + * attackObjectByAttackIdCache: Map + * }>} + */ + async resolve(primaryObjects) { + const objects = []; + const objectsMap = new Map(); + + for (const primaryObject of _.cloneDeep(primaryObjects)) { + this.addAttackObject(primaryObject, objects, objectsMap); + } + + const primaryObjectRelationships = this.relationships.filter( + (relationship) => + this.hasEndpoint(objectsMap, relationship, 'source') || + this.hasEndpoint(objectsMap, relationship, 'target'), + ); + + await this.addSecondaryObjects(primaryObjectRelationships, objectsMap, objects); + await this.processSecondaryRelationships(objects, objectsMap); + + const selectedRelationships = []; + for (const relationship of this.relationships) { + if (this.relationshipCanBeEmitted(relationship, objectsMap)) { + objects.push(relationship.stix); + selectedRelationships.push(relationship); + } + } + + return { + objects, + documents: [...objectsMap.values()], + relationships: selectedRelationships, + attackObjectByAttackIdCache: this.attackObjectByAttackIdCache, + dependencies: new Map( + [...this.dependencies].map(([key, sources]) => [key, [...sources.values()]]), + ), + }; + } + + /** + * Load identities and marking definitions referenced by the resolved graph. + * + * @param {Array} stixObjects + * @returns {Promise>} STIX-shaped supporting objects + */ + async loadSupportingObjects(stixObjects) { + return (await this.loadSupportingDocuments(stixObjects)).map((document) => document.stix); + } + + /** + * Load Workbench-shaped supporting documents for manifest capture. + * + * @param {Array} stixObjects + * @returns {Promise>} + */ + async loadSupportingDocuments(stixObjects) { + const identityRefs = new Set(); + const markingRefs = new Set(); + + for (const stixObject of stixObjects) { + if (stixObject.created_by_ref) { + identityRefs.add(stixObject.created_by_ref); + } + for (const markingRef of stixObject.object_marking_refs || []) { + markingRefs.add(markingRef); + } + } + + const supportingDocuments = []; + for (const stixId of identityRefs) { + const identity = await this.getAttackObject(stixId); + if (identity) { + supportingDocuments.push(identity); + } else { + logger.warn(`Referenced identity not found: ${stixId}`); + } + } + + for (const stixId of markingRefs) { + const markingDefinition = await this.getAttackObject(stixId); + if (markingDefinition) { + supportingDocuments.push(markingDefinition); + } + } + + return supportingDocuments; + } + + /** + * Resolve one attack object by STIX ID within this request. + * + * The legacy exporter is intentionally best-effort. Deterministic snapshot + * capture will use a strict adapter that treats missing dependencies as an + * integrity failure. + * + * @param {string} stixId + * @returns {Promise} + */ + async getAttackObject(stixId) { + try { + if (this.attackObjectCache.has(stixId)) { + return this.attackObjectCache.get(stixId); + } + + const attackObject = await this.attackObjectsRepository.retrieveLatestByStixIdLean(stixId); + const requestLocalObject = attackObject ? _.cloneDeep(attackObject) : null; + + if (requestLocalObject) { + this.attackObjectCache.set(stixId, requestLocalObject); + } + return requestLocalObject; + } catch (err) { + logger.error(`Error retrieving attack object ${stixId}:`, err); + return null; + } + } + + async getAttackObjectRevision(objectRef, objectModified) { + if (!objectModified || !this.repositoryMap) { + return this.getAttackObject(objectRef); + } + + const cacheKey = `${objectRef}::${new Date(objectModified).getTime()}`; + if (this.attackObjectCache.has(cacheKey)) { + return this.attackObjectCache.get(cacheKey); + } + + const repository = this.repositoryMap[objectRef.split('--')[0]]; + if (!repository) { + return null; + } + + const attackObject = ( + await repository.findManyByIdAndModified([ + { + object_ref: objectRef, + object_modified: objectModified, + }, + ]) + )[0]; + const requestLocalObject = attackObject ? _.cloneDeep(attackObject) : null; + this.attackObjectCache.set(cacheKey, requestLocalObject); + return requestLocalObject; + } + + async getRelationshipEndpoint(relationship, side) { + const endpoint = relationship.workspace?.relationship_endpoints?.[side]; + const objectRef = relationship.stix[`${side}_ref`]; + const object = await this.getAttackObjectRevision( + objectRef, + endpoint?.object_ref === objectRef ? endpoint.object_modified : undefined, + ); + if (!object && endpoint?.object_ref === objectRef && endpoint.object_modified) { + this.onMissingDependency?.({ + object_ref: objectRef, + object_modified: endpoint.object_modified, + dependency: 'relationship_endpoint', + }); + } + return object; + } + + addAttackObject(attackObject, objects, objectsMap) { + if (!attackObject || objectsMap.has(this.documentKey(attackObject))) { + return; + } + + objects.push(attackObject.stix); + objectsMap.set(this.documentKey(attackObject), attackObject); + const attackId = linkById.getAttackId(attackObject.stix); + if (attackId) { + this.attackObjectByAttackIdCache.set(attackId, attackObject); + } + } + + relationshipCanBeEmitted(relationship, objectsMap) { + return ( + !this.policy.isDeprecatedPattern(relationship.stix) && + this.policy.relationshipIsActive(relationship) && + this.hasEndpoint(objectsMap, relationship, 'source') && + this.hasEndpoint(objectsMap, relationship, 'target') + ); + } + + async processSecondaryObject(secondaryObject) { + if (!this.policy.secondaryObjectIsValid(secondaryObject, this.options)) { + return false; + } + + if ( + this.options.inferDomains !== false && + (secondaryObject.stix.type === 'intrusion-set' || secondaryObject.stix.type === 'campaign') + ) { + if (secondaryObject.stix.x_mitre_domains) { + this.domainCache.set(secondaryObject.stix.id, secondaryObject.stix.x_mitre_domains); + } + secondaryObject.stix.x_mitre_domains = + await this.getDomainsForSecondaryObject(secondaryObject); + } + return true; + } + + async getDomainsForSecondaryObject(attackObject) { + const relationships = this.relationships.filter( + (relationship) => relationship.stix.source_ref === attackObject.stix.id, + ); + + const domains = new Set(); + for (const relationship of relationships) { + const targetObject = await this.getRelationshipEndpoint(relationship, 'target'); + const targetDomains = + this.domainCache.get(targetObject?.stix.id) || targetObject?.stix.x_mitre_domains || []; + for (const domain of targetDomains) { + domains.add(domain); + } + } + return [...domains]; + } + + async addSecondaryObjects(primaryObjectRelationships, objectsMap, objects) { + for (const relationship of primaryObjectRelationships) { + if (relationship.stix.relationship_type === 'detects') { + continue; + } + + let secondarySide; + if (!this.hasEndpoint(objectsMap, relationship, 'source')) { + secondarySide = 'source'; + } else if (!this.hasEndpoint(objectsMap, relationship, 'target')) { + secondarySide = 'target'; + } + + if (!secondarySide) { + continue; + } + + const secondaryObject = await this.getRelationshipEndpoint(relationship, secondarySide); + if (await this.processSecondaryObject(secondaryObject)) { + const primarySide = secondarySide === 'source' ? 'target' : 'source'; + this.rememberDependency( + secondaryObject, + this.endpointDocument(objectsMap, relationship, primarySide), + ); + this.addAttackObject(secondaryObject, objects, objectsMap); + } + } + } + + async processSecondaryRelationships(objects, objectsMap) { + for (const relationship of this.relationships) { + await this.addAttributedGroup(relationship, objects, objectsMap); + await this.addDetectionStrategy(relationship, objects, objectsMap); + await this.addRevokedSecondaryObject(relationship, objects, objectsMap); + } + + const analyticIds = objects + .filter((object) => object.type === 'x-mitre-analytic') + .map((analytic) => analytic.id); + + if (analyticIds.length === 0) { + return; + } + + const detectionStrategyDocs = await this.detectionStrategiesRepository.findByAnalyticRefs( + analyticIds, + this.options, + ); + + for (const sourceDoc of detectionStrategyDocs) { + const detectionStrategyDoc = _.cloneDeep(sourceDoc); + if ( + !objectsMap.has(this.documentKey(detectionStrategyDoc)) && + this.policy.secondaryObjectIsValid(detectionStrategyDoc, this.options) + ) { + for (const analyticId of detectionStrategyDoc.stix.x_mitre_analytic_refs || []) { + for (const candidate of objectsMap.values()) { + if (candidate.stix.id === analyticId) { + this.rememberDependency(detectionStrategyDoc, candidate); + } + } + } + this.rememberAndSetDomains(detectionStrategyDoc, [this.options.domain]); + this.addAttackObject(detectionStrategyDoc, objects, objectsMap); + } + } + } + + async addAttributedGroup(relationship, objects, objectsMap) { + if ( + relationship.stix.relationship_type !== 'attributed-to' || + !this.hasEndpoint(objectsMap, relationship, 'source') || + this.hasEndpoint(objectsMap, relationship, 'target') + ) { + return; + } + + const groupObject = await this.getRelationshipEndpoint(relationship, 'target'); + if ( + groupObject?.stix.type === 'intrusion-set' && + this.policy.secondaryObjectIsValid(groupObject, this.options) + ) { + this.rememberDependency( + groupObject, + this.endpointDocument(objectsMap, relationship, 'source'), + ); + this.rememberAndSetDomains(groupObject, [this.options.domain]); + this.addAttackObject(groupObject, objects, objectsMap); + } + } + + async addDetectionStrategy(relationship, objects, objectsMap) { + if ( + relationship.stix.relationship_type !== 'detects' || + !this.hasEndpoint(objectsMap, relationship, 'target') || + this.hasEndpoint(objectsMap, relationship, 'source') + ) { + return; + } + + const detectionStrategy = await this.getRelationshipEndpoint(relationship, 'source'); + if ( + detectionStrategy?.stix.type === 'x-mitre-detection-strategy' && + this.policy.secondaryObjectIsValid(detectionStrategy, this.options) + ) { + this.rememberDependency( + detectionStrategy, + this.endpointDocument(objectsMap, relationship, 'target'), + ); + this.rememberAndSetDomains(detectionStrategy, [this.options.domain]); + this.addAttackObject(detectionStrategy, objects, objectsMap); + } + } + + async addRevokedSecondaryObject(relationship, objects, objectsMap) { + if ( + relationship.stix.relationship_type !== 'revoked-by' || + this.hasEndpoint(objectsMap, relationship, 'source') || + !this.hasEndpoint(objectsMap, relationship, 'target') + ) { + return; + } + + const revokedObject = await this.getRelationshipEndpoint(relationship, 'source'); + if (!this.policy.secondaryObjectIsValid(revokedObject, this.options)) { + return; + } + + this.rememberDependency( + revokedObject, + this.endpointDocument(objectsMap, relationship, 'target'), + ); + if (revokedObject.stix.type === 'intrusion-set' || revokedObject.stix.type === 'campaign') { + this.rememberAndSetDomains(revokedObject, [this.options.domain]); + } + this.addAttackObject(revokedObject, objects, objectsMap); + } + + rememberAndSetDomains(attackObject, domains) { + if (this.options.inferDomains === false) { + return; + } + if (attackObject.stix.x_mitre_domains) { + this.domainCache.set(attackObject.stix.id, attackObject.stix.x_mitre_domains); + } + attackObject.stix.x_mitre_domains = domains; + } +} + +module.exports = BundleGraphResolver; diff --git a/app/services/stix/collections-service.js b/app/services/stix/collections-service.js index 4551d14f..4b2b63d5 100644 --- a/app/services/stix/collections-service.js +++ b/app/services/stix/collections-service.js @@ -256,6 +256,36 @@ class CollectionsService extends BaseService { } } + async assertCollectionContentsCanBeDeleted(collection, stixId, modified) { + for (const reference of collection.stix.x_mitre_contents || []) { + const referenceObj = await attackObjectsService.retrieveOneByVersionLean( + reference.object_ref, + reference.object_modified, + ); + if (!referenceObj) continue; + + const matchQuery = { + 'stix.id': { $ne: stixId }, + 'stix.x_mitre_contents': { + $elemMatch: { + object_ref: reference.object_ref, + object_modified: reference.object_modified, + }, + }, + }; + if (modified) { + delete matchQuery['stix.id']; + matchQuery.$or = [{ 'stix.id': { $ne: stixId } }, { 'stix.modified': { $ne: modified } }]; + } + + const matches = await this.repository.findWithContents(matchQuery, { lean: true }); + if (matches.length === 0) { + await BaseService.assertNotMemberPinned(referenceObj, 'deleted'); + await BaseService.assertNotGraphPinned(referenceObj, 'deleted'); + } + } + } + async delete(stixId, deleteAllContents = false) { if (!stixId) { throw new MissingParameterError('stixId'); @@ -266,7 +296,15 @@ class CollectionsService extends BaseService { throw new BadlyFormattedParameterError({ parameterName: 'stixId' }); } + for (const collection of collections) { + await BaseService.assertNotMemberPinned(collection, 'deleted'); + await BaseService.assertNotGraphPinned(collection, 'deleted'); + } + if (deleteAllContents) { + for (const collection of collections) { + await this.assertCollectionContentsCanBeDeleted(collection, stixId); + } for (const collection of collections) { await this.deleteAllContentsOfCollection(collection, stixId); } @@ -290,7 +328,11 @@ class CollectionsService extends BaseService { throw new BadlyFormattedParameterError({ parameterName: 'stixId' }); } + await BaseService.assertNotMemberPinned(collection, 'deleted'); + await BaseService.assertNotGraphPinned(collection, 'deleted'); + if (deleteAllContents) { + await this.assertCollectionContentsCanBeDeleted(collection, stixId, modified); await this.deleteAllContentsOfCollection(collection, stixId, modified); } diff --git a/app/services/stix/relationships-service.js b/app/services/stix/relationships-service.js index 8f32da1f..79f9c9e4 100644 --- a/app/services/stix/relationships-service.js +++ b/app/services/stix/relationships-service.js @@ -1,11 +1,14 @@ 'use strict'; +const _ = require('lodash'); const { BaseService } = require('../meta-classes'); const relationshipsRepository = require('../../repository/relationships-repository'); +const attackObjectsRepository = require('../../repository/attack-objects-repository'); const { Relationship: RelationshipType } = require('../../lib/types'); const EventBus = require('../../lib/event-bus'); const EventConstants = require('../../lib/event-constants'); const logger = require('../../lib/logger'); +const { BadRequestError, InvalidObjectRevisionError } = require('../../exceptions'); // Map STIX types to ATT&CK types const objectTypeMap = new Map([ @@ -23,11 +26,95 @@ const objectTypeMap = new Map([ ]); class RelationshipsService extends BaseService { + /** + * Resolve STIX ID-only relationship endpoints to the exact object revisions + * they mean when this relationship revision is created. + * + * The pins are Workbench metadata rather than custom STIX properties. They + * are therefore validated by Mongoose, remain server-controlled, and are + * naturally omitted from emitted STIX bundles. + * + * @param {Object} data Workbench-shaped relationship document + * @returns {Promise} + */ + static async pinEndpointRevisions(data) { + const [source, target] = await Promise.all([ + attackObjectsRepository.retrieveLatestByStixIdLean(data.stix.source_ref), + attackObjectsRepository.retrieveLatestByStixIdLean(data.stix.target_ref), + ]); + + const missing = []; + if (!source) { + missing.push({ + endpoint: 'source', + object_ref: data.stix.source_ref, + object_modified: 'latest', + }); + } + if (!target) { + missing.push({ + endpoint: 'target', + object_ref: data.stix.target_ref, + object_modified: 'latest', + }); + } + if (missing.length > 0) { + throw new InvalidObjectRevisionError(missing, { + details: + 'Relationship endpoints must resolve to exact object revisions before the ' + + 'relationship can be persisted.', + }); + } + + data.workspace = data.workspace || {}; + data.workspace.relationship_endpoints = { + source: { + object_ref: source.stix.id, + object_modified: source.stix.modified, + }, + target: { + object_ref: target.stix.id, + object_modified: target.stix.modified, + }, + }; + } + + async beforeCreate(data) { + await RelationshipsService.pinEndpointRevisions(data); + } + + async beforeUpdate(_stixId, _stixModified, data, existingDocument) { + for (const field of ['source_ref', 'target_ref', 'relationship_type']) { + if (data.stix[field] !== existingDocument.stix[field]) { + throw new BadRequestError({ + details: + `Relationship ${field} cannot be changed in place because that would alter the ` + + 'meaning of an existing graph revision. Create a new relationship revision instead.', + immutable_property: field, + }); + } + } + + data.workspace = data.workspace || {}; + data.workspace.relationship_endpoints = existingDocument.workspace.relationship_endpoints; + } + /** * Initialize event listeners. * Called once on module load. */ static initializeEventListeners() { + const endpointRevisionEvents = [ + ...new Set([ + ...Object.values(EventConstants).filter((eventName) => eventName.endsWith('::created')), + 'identity::created', + 'note::created', + ]), + ]; + for (const event of endpointRevisionEvents) { + EventBus.on(event, this.handleEndpointRevisionCreated.bind(this)); + } + const revokedEvents = [ EventConstants.ATTACK_PATTERN_REVOKED, EventConstants.TACTIC_REVOKED, @@ -69,6 +156,80 @@ class RelationshipsService extends BaseService { logger.info('RelationshipsService: Event listeners initialized'); } + /** + * Carry active relationship edges forward when one of their exact endpoint + * revisions advances. + * + * The prior SRO revision remains pinned to the prior endpoint revisions. + * A new SRO revision is created for the new endpoint state, preserving STIX + * revision immutability while retaining the current graph. + * + * @param {Object} payload Standard BaseService created-event payload + * @returns {Promise<{created: Array}>} + */ + static async handleEndpointRevisionCreated(payload) { + const document = payload?.document; + if (!document?.stix?.id || !document?.stix?.modified) { + return { created: [] }; + } + + const versions = await attackObjectsRepository.retrieveAllById(document.stix.id); + const createdRevisionIndex = versions.findIndex( + (version) => + new Date(version.stix.modified).getTime() === new Date(document.stix.modified).getTime(), + ); + + // Only the latest revision advances the current graph. Older revisions + // arriving in a bulk import retain their historical position. + if (createdRevisionIndex !== 0 || versions.length < 2) { + return { created: [] }; + } + + const previousRevision = versions[1]; + const relationships = await relationshipsRepository.retrieveAllBySourceOrTarget( + document.stix.id, + ); + const relationshipsToAdvance = relationships.filter((relationship) => { + if (relationship.stix.revoked || relationship.stix.x_mitre_deprecated) { + return false; + } + + const endpoints = relationship.workspace?.relationship_endpoints; + return ['source', 'target'].some( + (side) => + endpoints?.[side]?.object_ref === previousRevision.stix.id && + new Date(endpoints[side].object_modified).getTime() === + new Date(previousRevision.stix.modified).getTime(), + ); + }); + + const created = []; + for (const relationship of relationshipsToAdvance) { + const relationshipData = _.cloneDeep(relationship); + delete relationshipData._id; + delete relationshipData.__v; + delete relationshipData.__t; + if (relationshipData.workspace) { + delete relationshipData.workspace.release_tracks; + delete relationshipData.workspace.relationship_endpoints; + } + + const previousRelationshipModified = new Date(relationship.stix.modified).getTime(); + relationshipData.stix.modified = new Date( + Math.max(Date.now(), previousRelationshipModified + 1), + ).toISOString(); + + created.push( + await module.exports.create(relationshipData, { + userAccountId: payload.options?.userAccountId, + automationContext: payload.options?.automationContext, + }), + ); + } + + return { created }; + } + /** * Return the latest active relationship revisions whose endpoints are both * in the requested bundle object set. diff --git a/app/services/stix/stix-bundles-service.js b/app/services/stix/stix-bundles-service.js index abd0750b..199b7f60 100644 --- a/app/services/stix/stix-bundles-service.js +++ b/app/services/stix/stix-bundles-service.js @@ -5,9 +5,9 @@ const config = require('../../config/config'); const { BaseService } = require('../meta-classes'); const linkById = require('../../lib/linkById'); const bundleRelationships = require('../../lib/stix-bundle-relationships'); -const logger = require('../../lib/logger'); const { requiresAttackId } = require('../../lib/attack-id-generator'); const stixConformance = require('../../lib/stix-conformance'); +const BundleGraphResolver = require('./bundle-graph-resolver'); // Import repositories const analyticsRepository = require('../../repository/analytics-repository'); @@ -243,47 +243,6 @@ class StixBundlesService extends BaseService { } } - /** - * Adds an ATT&CK object to the STIX bundle - * @param {Object} attackObject - The ATT&CK object to add - * @param {Object} bundle - The STIX bundle being built - * @param {Map} objectsMap - Map tracking objects in the bundle - - */ - addAttackObjectToBundle(attackObject, bundle, objectsMap) { - if (!objectsMap.has(attackObject.stix.id)) { - bundle.objects.push(attackObject.stix); - objectsMap.set(attackObject.stix.id, true); - const attackId = linkById.getAttackId(attackObject.stix); - if (attackId) { - this.attackObjectByAttackIdCache.set(attackId, attackObject); - } - } - } - - /** - * Processes a secondary object for inclusion in the bundle. - * Validates the object and updates necessary data structures. - * @param {Object} secondaryObject - The secondary object to process - * @param {Object} options - Bundle generation options - * @returns {Promise} True if object was successfully processed - */ - async processSecondaryObject(secondaryObject, options) { - if (!StixBundlesService.secondaryObjectIsValid(secondaryObject, options)) { - return false; - } - - // Handle domains for groups and campaigns - if (secondaryObject.stix.type === 'intrusion-set' || secondaryObject.stix.type === 'campaign') { - if (secondaryObject.stix.x_mitre_domains) { - this.domainCache.set(secondaryObject.stix.id, secondaryObject.stix.x_mitre_domains); - } - secondaryObject.stix.x_mitre_domains = - await this.getDomainsForSecondaryObject(secondaryObject); - } - return true; - } - /** * Validates if a secondary object meets all inclusion criteria for the bundle. * @param {Object} secondaryObject - The object to validate @@ -309,38 +268,6 @@ class StixBundlesService extends BaseService { ); } - /** - * Determines the domains associated with a secondary object based on its relationships. - * @param {Object} attackObject - The secondary object to process - * @returns {Promise>} Array of domain names - */ - async getDomainsForSecondaryObject(attackObject) { - const relationships = this.allRelationships.filter( - (relationship) => relationship.stix.source_ref == attackObject.stix.id, - ); - - const domainMap = new Map(); - for (const relationship of relationships) { - const targetObject = await this.getAttackObject(relationship.stix.target_ref); - // domainCache is used to accurately reflect the STIX bundle post-refactoring in project Orion. - // The additional domains that would otherwise be added are likely correct, but that will - // be handled in a separate data cleanup effort not coinciding with the imminent v17 ATT&CK release. - if (this.domainCache.has(targetObject?.stix.id)) { - for (const domain of this.domainCache.get(targetObject.stix.id)) { - domainMap.set(domain, true); - } - } else { - if (targetObject?.stix.x_mitre_domains) { - for (const domain of targetObject.stix.x_mitre_domains) { - domainMap.set(domain, true); - } - } - } - } - - return [...domainMap.keys()]; - } - // ============================ // Collection Object Management // ============================ @@ -446,13 +373,6 @@ class StixBundlesService extends BaseService { * @returns {Promise} The generated STIX bundle */ async exportBundle(options) { - // Initialize caches for efficient object lookup - this.attackObjectCache = new Map(); // Maps STIX IDs to attack objects - this.identityCache = new Map(); // Maps identity STIX IDs to identity objects - this.markingDefinitionsCache = new Map(); // Maps marking definition STIX IDs to marking objects - this.attackObjectByAttackIdCache = new Map(); // Maps attack IDs to attack objects - this.domainCache = new Map(); // Stores original x-mitre-domains if we change them at runtime - // Initialize bundle const bundle = { type: 'bundle', @@ -517,32 +437,20 @@ class StixBundlesService extends BaseService { primaryObjects = primaryObjects.filter((o) => StixBundlesService.hasAttackId(o)); } - // Put the primary objects in the bundle - // Also create a map of the objects added to the bundle (use the id as the key, since relationships only reference the id) - const objectsMap = new Map(); - for (const primaryObject of primaryObjects) { - this.addAttackObjectToBundle(primaryObject, bundle, objectsMap); - } - - // Since we're querying all relationships, save them for later to prevent future database queries. - this.allRelationships = await this.repositories.relationship.retrieveAllForBundle(options); - - // Filter relationships that have a source_ref or target_ref that points at a primary object - const primaryObjectRelationships = this.allRelationships.filter( - (relationship) => - objectsMap.has(relationship.stix.source_ref) || - objectsMap.has(relationship.stix.target_ref), - ); - - // Get the secondary objects (additional objects pointed to by a relationship) - await this.addSecondaryObjects(primaryObjectRelationships, objectsMap, bundle, options); - - await this.processSecondaryRelationships(bundle, objectsMap, options); - - // Add all valid relationships to the bundle - for (const relationship of this.allRelationships) { - StixBundlesService.addRelationshipToBundle(relationship, bundle, objectsMap); - } + const relationships = await this.repositories.relationship.retrieveAllForBundle(options); + const graphResolver = new BundleGraphResolver({ + attackObjectsRepository: this.repositories.attackObject, + detectionStrategiesRepository: this.repositories.detectionStrategy, + policy: { + isDeprecatedPattern: StixBundlesService.isDeprecatedPattern, + relationshipIsActive: StixBundlesService.relationshipIsActive, + secondaryObjectIsValid: StixBundlesService.secondaryObjectIsValid, + }, + options, + relationships, + }); + const resolvedGraph = await graphResolver.resolve(primaryObjects); + bundle.objects.push(...resolvedGraph.objects); // Add notes if requested if (options.includeNotes) { @@ -550,10 +458,10 @@ class StixBundlesService extends BaseService { } // Convert LinkById tags to markdown citations - await this.convertLinkByIdTags(bundle.objects, this.attackObjectByAttackIdCache); + await this.convertLinkByIdTags(bundle.objects, resolvedGraph.attackObjectByAttackIdCache); // Process identities and marking definitions - await this.processIdentitiesAndMarkings(bundle); + bundle.objects.push(...(await graphResolver.loadSupportingObjects(bundle.objects))); // Conform to STIX version for (const stixObject of bundle.objects) { @@ -566,274 +474,6 @@ class StixBundlesService extends BaseService { return bundle; } - /** - * Add secondary objects to the bundle - those objects which have a relationship - * to a primary object but did not have the proper domain in the database. - * - * Note: 'detects' relationships are skipped here and handled separately in - * processSecondaryRelationships() to support the new ATT&CK spec where only - * detection strategies (not data components) can detect techniques. - * - * @param {Array} primaryObjectRelationships - The relationships to process - * @param {Map} objectsMap - Map of objects currently in the bundle - * @param {Object} bundle - The STIX bundle being built - * @param {Object} options - Bundle generation options - * @returns {Promise} - */ - async addSecondaryObjects(primaryObjectRelationships, objectsMap, bundle, options) { - for (const relationship of primaryObjectRelationships) { - // Skip 'detects' relationships - they require special handling - // - // CONTEXT: The ATT&CK specification changed how detection works: - // - OLD (pre-v17): Data components could detect techniques via 'detects' relationships - // - NEW (v17+): Only detection strategies can detect techniques via 'detects' relationships - // - // WHY WE SKIP HERE: - // 1. Data components are now PRIMARY objects (retrieved by domain), not secondary - // 2. If we processed 'detects' relationships here, we would incorrectly add data - // components as secondary objects based on deprecated relationships - // 3. Detection strategies ARE secondary objects, but they need special domain - // inference logic (they get the domain of the technique they detect) - // - // WHERE THEY'RE HANDLED: - // 'detects' relationships are processed in processSecondaryRelationships() where: - // - We verify the source is a detection strategy (not a data component) - // - We set the detection strategy's x_mitre_domains to match the target technique - // - Deprecated 'detects' from data components are silently ignored - if (relationship.stix.relationship_type === 'detects') { - continue; - } - - if (!objectsMap.has(relationship.stix.source_ref)) { - const secondaryObject = await this.getAttackObject(relationship.stix.source_ref); - - // Only process if the secondary object meets our inclusion criteria - if (await this.processSecondaryObject(secondaryObject, options)) { - this.addAttackObjectToBundle(secondaryObject, bundle, objectsMap); - } - } else if (!objectsMap.has(relationship.stix.target_ref)) { - const secondaryObject = await this.getAttackObject(relationship.stix.target_ref); - - // Only process if the secondary object meets our inclusion criteria - if (await this.processSecondaryObject(secondaryObject, options)) { - this.addAttackObjectToBundle(secondaryObject, bundle, objectsMap); - } - } - } - } - - /** - * Processes all identities and marking definitions referenced in the bundle. - * This ensures that all necessary context objects are included. - * - * Steps: - * 1. Collect all identity references (created_by_ref) - * 2. Collect all marking definition references (object_marking_refs) - * 3. Retrieve objects from cache or database - * 4. Add valid objects to bundle - * 5. Log warnings for missing references - * - * @param {Object} bundle - The STIX bundle being built - * @returns {Promise} - */ - async processIdentitiesAndMarkings(bundle) { - // Map referenced identities and marking definitions - const identitiesMap = new Map(); - const markingDefinitionsMap = new Map(); - - for (const bundleObject of bundle.objects) { - if (bundleObject.created_by_ref) { - identitiesMap.set(bundleObject.created_by_ref, true); - } - - if (bundleObject.object_marking_refs) { - for (const markingRef of bundleObject.object_marking_refs) { - markingDefinitionsMap.set(markingRef, true); - } - } - } - - // Process identities - for (const stixId of identitiesMap.keys()) { - if (this.identityCache.has(stixId)) { - bundle.objects.push(this.identityCache.get(stixId)); - continue; - } - - const identity = await this.getAttackObject(stixId); - if (identity) { - bundle.objects.push(identity.stix); - this.identityCache.set(stixId, identity.stix); - } else { - logger.warn(`Referenced identity not found: ${stixId}`); - } - } - - // Process marking definitions - for (const stixId of markingDefinitionsMap.keys()) { - if (this.markingDefinitionsCache.has(stixId)) { - bundle.objects.push(this.markingDefinitionsCache.get(stixId)); - continue; - } - - const markingDefinition = await this.getAttackObject(stixId); - if (markingDefinition) { - bundle.objects.push(markingDefinition.stix); - this.markingDefinitionsCache.set(stixId, markingDefinition.stix); - } - } - } - - /** - * Processes relationships between secondary objects and handles special cases that need separate processing: - * - Groups referenced by campaigns through 'attributed-to' relationships - * - Detection strategies that detect techniques in the bundle - * - Detection strategies referenced by analytics in the bundle - * - Secondary objects that were revoked by other secondary objects - * - * @param {Object} bundle - The STIX bundle being built - * @param {Map} objectsMap - Map tracking objects currently in bundle - * @param {Object} options - Bundle generation options - * @param {string} options.domain - The domain being processed - * @returns {Promise} - */ - async processSecondaryRelationships(bundle, objectsMap, options) { - for (const relationship of this.allRelationships) { - // Add groups referenced by campaigns through 'attributed-to' relationships - if ( - relationship.stix.relationship_type === 'attributed-to' && - objectsMap.has(relationship.stix.source_ref) && - !objectsMap.has(relationship.stix.target_ref) - ) { - const groupObject = await this.getAttackObject(relationship.stix.target_ref); - if ( - groupObject.stix.type === 'intrusion-set' && - StixBundlesService.secondaryObjectIsValid(groupObject, options) - ) { - if (groupObject.stix.x_mitre_domains) { - this.domainCache.set(groupObject.stix.id, groupObject.stix.x_mitre_domains); - } - groupObject.stix.x_mitre_domains = [options.domain]; - this.addAttackObjectToBundle(groupObject, bundle, objectsMap); - } - } - - // Add detection strategies that detect techniques in the bundle - if ( - relationship.stix.relationship_type === 'detects' && - objectsMap.has(relationship.stix.target_ref) && - !objectsMap.has(relationship.stix.source_ref) - ) { - const detectionStrategy = await this.getAttackObject(relationship.stix.source_ref); - if ( - detectionStrategy.stix.type === 'x-mitre-detection-strategy' && - StixBundlesService.secondaryObjectIsValid(detectionStrategy, options) - ) { - if (detectionStrategy.stix.x_mitre_domains) { - this.domainCache.set(detectionStrategy.stix.id, detectionStrategy.stix.x_mitre_domains); - } - // Set x_mitre_domains on each exported detection strategy - detectionStrategy.stix.x_mitre_domains = [options.domain]; - this.addAttackObjectToBundle(detectionStrategy, bundle, objectsMap); - } - } - - // Add secondary objects that were revoked by other secondary objects - if ( - relationship.stix.relationship_type === 'revoked-by' && - !objectsMap.has(relationship.stix.source_ref) && - objectsMap.has(relationship.stix.target_ref) - ) { - const revokedObject = await this.getAttackObject(relationship.stix.source_ref); - if (StixBundlesService.secondaryObjectIsValid(revokedObject, options)) { - if ( - revokedObject.stix.type === 'intrusion-set' || - revokedObject.stix.type === 'campaign' - ) { - if (revokedObject.stix.x_mitre_domains) { - this.domainCache.set(revokedObject.stix.id, revokedObject.stix.x_mitre_domains); - } - revokedObject.stix.x_mitre_domains = [options.domain]; - } - this.addAttackObjectToBundle(revokedObject, bundle, objectsMap); - } - } - } - - // Add detection strategies referenced by analytics in the bundle - // This is a key requirement of the new ATT&CK spec: detection strategies should be - // included if they reference an analytic that is in the domain - const analyticsInBundle = bundle.objects.filter((obj) => obj.type === 'x-mitre-analytic'); - - if (analyticsInBundle.length > 0) { - // Collect all analytic IDs in the bundle - const analyticIds = analyticsInBundle.map((analytic) => analytic.id); - - // Single batch query to find all detection strategies that reference any of these analytics - // This replaces the N+1 query pattern that was causing timeouts - const detectionStrategyDocs = await this.repositories.detectionStrategy.findByAnalyticRefs( - analyticIds, - options, - ); - - for (const detectionStrategyDoc of detectionStrategyDocs) { - if ( - !objectsMap.has(detectionStrategyDoc.stix.id) && - StixBundlesService.secondaryObjectIsValid(detectionStrategyDoc, options) - ) { - if (detectionStrategyDoc.stix.x_mitre_domains) { - this.domainCache.set( - detectionStrategyDoc.stix.id, - detectionStrategyDoc.stix.x_mitre_domains, - ); - } - // Set x_mitre_domains on each exported detection strategy - detectionStrategyDoc.stix.x_mitre_domains = [options.domain]; - this.addAttackObjectToBundle(detectionStrategyDoc, bundle, objectsMap); - } - } - } - } - - // ============================ - // Repository Access Methods (+Cache Management) - // ============================ - - /** - * Retrieves an attack object by its STIX ID, using cache when possible. - * Implements a caching strategy to minimize database queries. - * - * Process: - * 1. Check cache using STIX ID - * 2. If not found, query database - * 3. If found in database, cache for future use - * 4. Handle errors gracefully - * - * @param {string} stixId - The STIX ID of the object to retrieve - * @returns {Promise} The attack object or null if not found/error - */ - async getAttackObject(stixId) { - try { - // First check cache - const cacheKey = stixId; - if (this.attackObjectCache.has(cacheKey)) { - return this.attackObjectCache.get(cacheKey); - } - - // Use the existing repository method that exactly matches the original logic - const attackObject = await this.repositories.attackObject.retrieveLatestByStixIdLean(stixId); - - if (attackObject) { - this.attackObjectCache.set(cacheKey, attackObject); - } - - return attackObject; - } catch (err) { - logger.error(`Error retrieving attack object ${stixId}:`, err); - return null; - } - } - /** * Converts LinkById tags to markdown citations * @param {Array} bundleObjects - Objects in the bundle diff --git a/app/tests/api/attack-objects/attack-objects.spec.js b/app/tests/api/attack-objects/attack-objects.spec.js index 287d8bc4..bd4a9184 100644 --- a/app/tests/api/attack-objects/attack-objects.spec.js +++ b/app/tests/api/attack-objects/attack-objects.spec.js @@ -141,8 +141,10 @@ describe('ATT&CK Objects API', function () { const markingDefinitions = attackObjects.filter((x) => x.stix.type === 'marking-definition'); expect(markingDefinitions.length).toBe(5); - // Placeholder identity, 4 TLP marking definitions, 18 collection contents, 2 collection objects - expect(attackObjects.length).toBe(1 + 4 + 18 + 2); + // Placeholder identity, 4 TLP marking definitions, 18 imported collection contents, + // 2 collection objects, and the propagated relationship revision pinned to the + // second bundle's newer target revision. + expect(attackObjects.length).toBe(1 + 4 + 18 + 2 + 1); }); it('GET /api/attack-objects returns zero objects with an ATT&CK ID that does not exist', async function () { diff --git a/app/tests/api/relationships/relationship-endpoint-pins.spec.js b/app/tests/api/relationships/relationship-endpoint-pins.spec.js new file mode 100644 index 00000000..672e98a4 --- /dev/null +++ b/app/tests/api/relationships/relationship-endpoint-pins.spec.js @@ -0,0 +1,170 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const { cloneForCreate } = require('../../shared/clone-for-create'); + +const markingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; + +describe('Relationship endpoint revision pins', function () { + let app; + let passportCookie; + let source; + let target; + let relationship; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + async function post(path, body, expectedStatus = 201) { + const response = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(expectedStatus); + return response.body; + } + + before('create endpoint objects and their relationship', async function () { + const sourceTimestamp = new Date().toISOString(); + source = await post('/api/software', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'malware', + spec_version: '2.1', + created: sourceTimestamp, + modified: sourceTimestamp, + name: 'Revision-pinned source', + description: 'Source object for relationship revision pin tests.', + is_family: false, + x_mitre_domains: ['enterprise-attack'], + x_mitre_platforms: ['Windows'], + object_marking_refs: [markingDefinitionId], + }, + }); + + const targetTimestamp = new Date().toISOString(); + target = await post('/api/techniques', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'attack-pattern', + spec_version: '2.1', + created: targetTimestamp, + modified: targetTimestamp, + name: 'Revision-pinned target', + description: 'Target object for relationship revision pin tests.', + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'execution' }], + x_mitre_is_subtechnique: false, + x_mitre_domains: ['enterprise-attack'], + x_mitre_platforms: ['Windows'], + object_marking_refs: [markingDefinitionId], + }, + }); + + const relationshipTimestamp = new Date().toISOString(); + relationship = await post('/api/relationships', { + workspace: { + workflow: { state: 'work-in-progress' }, + relationship_endpoints: { + source: { + object_ref: target.stix.id, + object_modified: target.stix.modified, + }, + target: { + object_ref: source.stix.id, + object_modified: source.stix.modified, + }, + }, + }, + stix: { + type: 'relationship', + spec_version: '2.1', + created: relationshipTimestamp, + modified: relationshipTimestamp, + relationship_type: 'uses', + source_ref: source.stix.id, + target_ref: target.stix.id, + object_marking_refs: [markingDefinitionId], + }, + }); + }); + + it('stores server-resolved exact endpoint revisions outside the STIX payload', function () { + expect(relationship.workspace.relationship_endpoints).toEqual({ + source: { + object_ref: source.stix.id, + object_modified: source.stix.modified, + }, + target: { + object_ref: target.stix.id, + object_modified: target.stix.modified, + }, + }); + expect(relationship.stix.x_mitre_source_ref_modified).toBeUndefined(); + expect(relationship.stix.x_mitre_target_ref_modified).toBeUndefined(); + }); + + it('creates a new SRO revision when an endpoint advances', async function () { + const sourceRevision = cloneForCreate(source); + sourceRevision.stix.modified = new Date( + new Date(source.stix.modified).getTime() + 1000, + ).toISOString(); + sourceRevision.stix.description = 'A newer source revision.'; + + const newSource = await post('/api/software', sourceRevision); + const response = await request(app) + .get(`/api/relationships/${relationship.stix.id}?versions=all`) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + + expect(response.body).toHaveLength(2); + const [latestRelationship, originalRelationship] = response.body; + expect(latestRelationship.stix.id).toBe(relationship.stix.id); + expect(latestRelationship.stix.modified).not.toBe(originalRelationship.stix.modified); + expect(latestRelationship.workspace.relationship_endpoints.source).toEqual({ + object_ref: source.stix.id, + object_modified: newSource.stix.modified, + }); + expect(latestRelationship.workspace.relationship_endpoints.target).toEqual({ + object_ref: target.stix.id, + object_modified: target.stix.modified, + }); + expect(originalRelationship.workspace.relationship_endpoints.source).toEqual({ + object_ref: source.stix.id, + object_modified: source.stix.modified, + }); + }); + + it('does not emit internal endpoint pins in STIX bundles', async function () { + const response = await request(app) + .get('/api/release-tracks/ephemeral/enterprise?includeToc=false') + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + + const emittedRelationship = response.body.objects.find( + (object) => object.id === relationship.stix.id, + ); + expect(emittedRelationship).toBeDefined(); + expect(emittedRelationship.workspace).toBeUndefined(); + expect(emittedRelationship.x_mitre_source_ref_modified).toBeUndefined(); + expect(emittedRelationship.x_mitre_target_ref_modified).toBeUndefined(); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/app/tests/api/relationships/relationships-pagination.spec.js b/app/tests/api/relationships/relationships-pagination.spec.js index de2f5fd9..63f1fd5e 100644 --- a/app/tests/api/relationships/relationships-pagination.spec.js +++ b/app/tests/api/relationships/relationships-pagination.spec.js @@ -1,6 +1,8 @@ const relationshipsService = require('../../../services/stix/relationships-service'); const PaginationTests = require('../../shared/pagination'); const config = require('../../../config/config'); +const Software = require('../../../models/software-model'); +const Technique = require('../../../models/technique-model'); config.validateRequests.withOpenApi = true; @@ -33,8 +35,39 @@ const options = { label: 'Relationships', validateWithAdm: true, }; +let endpointsCreated = false; const relationshipsPaginationService = { async create(data, options) { + if (!endpointsCreated) { + const endpointModified = new Date(); + await Promise.all([ + Software.create({ + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'malware', + spec_version: '2.1', + id: sourceRef1, + created: endpointModified, + modified: endpointModified, + name: 'Pagination relationship source', + is_family: false, + }, + }), + Technique.create({ + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'attack-pattern', + spec_version: '2.1', + id: targetRef1, + created: endpointModified, + modified: endpointModified, + name: 'Pagination relationship target', + x_mitre_is_subtechnique: false, + }, + }), + ]); + endpointsCreated = true; + } delete data.stix.name; return relationshipsService.create(data, options); }, diff --git a/app/tests/api/relationships/relationships.spec.js b/app/tests/api/relationships/relationships.spec.js index 9e71a4b8..982968e7 100644 --- a/app/tests/api/relationships/relationships.spec.js +++ b/app/tests/api/relationships/relationships.spec.js @@ -7,6 +7,8 @@ const databaseConfiguration = require('../../../lib/database-configuration'); const config = require('../../../config/config'); const login = require('../../shared/login'); const { cloneForCreate } = require('../../shared/clone-for-create'); +const Software = require('../../../models/software-model'); +const Technique = require('../../../models/technique-model'); const logger = require('../../../lib/logger'); logger.level = 'debug'; @@ -27,6 +29,16 @@ const initialObjectData = { workflow: { state: 'work-in-progress', }, + relationship_endpoints: { + source: { + object_ref: 'malware--00000000-0000-4000-8000-000000000000', + object_modified: '2000-01-01T00:00:00.000Z', + }, + target: { + object_ref: 'attack-pattern--00000000-0000-4000-8000-000000000000', + object_modified: '2000-01-01T00:00:00.000Z', + }, + }, }, stix: { spec_version: '2.1', @@ -44,6 +56,7 @@ const initialObjectData = { describe('Relationships API', function () { let app; let passportCookie; + let endpointModified; before(async function () { // Establish the database connection @@ -62,6 +75,37 @@ describe('Relationships API', function () { // Log into the app passportCookie = await login.loginAnonymous(app); + + endpointModified = new Date(); + const endpointCreated = new Date(endpointModified); + await Software.create( + [sourceRef1, sourceRef2].map((id, index) => ({ + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'malware', + spec_version: '2.1', + id, + created: endpointCreated, + modified: endpointModified, + name: `Relationship source ${index + 1}`, + is_family: false, + }, + })), + ); + await Technique.create( + [targetRef1, targetRef2].map((id, index) => ({ + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'attack-pattern', + spec_version: '2.1', + id, + created: endpointCreated, + modified: endpointModified, + name: `Relationship target ${index + 1}`, + x_mitre_is_subtechnique: false, + }, + })), + ); }); it('GET /api/relationships returns an empty array of relationships', async function () { @@ -110,6 +154,47 @@ describe('Relationships API', function () { expect(relationship1a.stix.created).toBeDefined(); expect(relationship1a.stix.modified).toBeDefined(); expect(relationship1a.stix.x_mitre_attack_spec_version).toBe(config.app.attackSpecVersion); + expect(relationship1a.workspace.relationship_endpoints).toEqual({ + source: { + object_ref: sourceRef1, + object_modified: endpointModified.toISOString(), + }, + target: { + object_ref: targetRef1, + object_modified: endpointModified.toISOString(), + }, + }); + }); + + it('POST /api/relationships rejects endpoints that cannot be revision-pinned', async function () { + const timestamp = new Date().toISOString(); + const body = { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'relationship', + spec_version: '2.1', + created: timestamp, + modified: timestamp, + relationship_type: 'uses', + source_ref: sourceRef1, + target_ref: targetRef3, + }, + }; + + const res = await request(app) + .post('/api/relationships') + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(400); + + expect(res.body.missing_references).toEqual([ + { + endpoint: 'target', + object_ref: targetRef3, + object_modified: 'latest', + }, + ]); }); it('GET /api/relationships returns the added relationship', async function () { @@ -189,6 +274,25 @@ describe('Relationships API', function () { expect(relationship.stix.modified).toBe(relationship1a.stix.modified); }); + it('PUT /api/relationships rejects in-place endpoint changes', async function () { + const body = structuredClone(relationship1a); + body.stix.source_ref = sourceRef2; + + const res = await request(app) + .put( + '/api/relationships/' + + relationship1a.stix.id + + '/modified/' + + relationship1a.stix.modified, + ) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(400); + + expect(res.body.immutable_property).toBe('source_ref'); + }); + it('POST /api/relationships does not create a relationship with the same id and modified date', async function () { const body = relationship1a; await request(app) diff --git a/app/tests/api/release-tracks/destructive-authorization.spec.js b/app/tests/api/release-tracks/destructive-authorization.spec.js index 8dd05d0e..682fee1c 100644 --- a/app/tests/api/release-tracks/destructive-authorization.spec.js +++ b/app/tests/api/release-tracks/destructive-authorization.spec.js @@ -13,27 +13,6 @@ const ReleaseTrackAuditEvent = require('../../../models/release-tracks/release-t const auditRepository = require('../../../repository/release-tracks/release-track-audit-event.repository'); const systemConfigurationService = require('../../../services/system/system-configuration-service'); -const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; - -function techniquePayload() { - const timestamp = new Date().toISOString(); - return { - workspace: { workflow: { state: 'work-in-progress' } }, - stix: { - created: timestamp, - modified: timestamp, - name: 'Destructive authorization member', - description: 'Member used by destructive authorization tests.', - spec_version: '2.1', - type: 'attack-pattern', - object_marking_refs: [markingDefinitionId], - kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], - x_mitre_is_subtechnique: false, - x_mitre_platforms: ['Windows'], - }, - }; -} - describe('Release-track destructive authorization and audit', function () { let app; let passportCookie; @@ -77,76 +56,34 @@ describe('Release-track destructive authorization and audit', function () { return (await api('post', path, body, status, query)).body; } - it('requires admin role, exact confirmation, and durable outcome records', async function () { + it('requires admin role, exact confirmation, and a durable outcome record', async function () { await setRole('admin'); - const technique = await post('/api/techniques', techniquePayload(), 201); const track = await post( '/api/release-tracks/new', { name: 'Destructive authorization standard', type: 'standard' }, 201, ); - const contents = { - x_mitre_contents: [ - { - obj_ref: technique.stix.id, - obj_modified: technique.stix.modified, - }, - ], - }; await setRole('editor'); - await api('post', `/api/release-tracks/${track.id}/contents`, contents, 401, { - confirm_track_id: track.id, - }); - await api( - 'post', - `/api/release-tracks/${track.id}/snapshots/${track.modified}/contents`, - contents, - 401, - { confirm_track_id: track.id }, - ); await api('delete', `/api/release-tracks/${track.id}`, undefined, 401, { confirm_track_id: track.id, }); expect(await ReleaseTrackAuditEvent.countDocuments()).toBe(0); await setRole('admin'); - await api('post', `/api/release-tracks/${track.id}/contents`, contents, 400); - await api('post', `/api/release-tracks/${track.id}/contents`, contents, 400, { + await api('delete', `/api/release-tracks/${track.id}`, undefined, 400); + await api('delete', `/api/release-tracks/${track.id}`, undefined, 400, { confirm_track_id: 'release-track--00000000-0000-4000-8000-000000000099', }); - await api('delete', `/api/release-tracks/${track.id}`, undefined, 400); expect(await ReleaseTrackAuditEvent.countDocuments()).toBe(0); - const latest = await post(`/api/release-tracks/${track.id}/contents`, contents, 200, { - confirm_track_id: track.id, - }); - await post( - `/api/release-tracks/${track.id}/snapshots/${track.modified}/contents`, - contents, - 200, - { confirm_track_id: track.id }, - ); - - const virtual = await post( - '/api/release-tracks/new', - { name: 'Destructive authorization virtual', type: 'virtual' }, - 201, - ); - await api('post', `/api/release-tracks/${virtual.id}/contents`, contents, 400, { - confirm_track_id: virtual.id, - }); - await api('delete', `/api/release-tracks/${track.id}`, undefined, 204, { confirm_track_id: track.id, }); const events = await ReleaseTrackAuditEvent.find().sort({ started_at: 1 }).lean().exec(); - expect(events).toHaveLength(4); + expect(events).toHaveLength(1); expect(events.map((event) => [event.action, event.status])).toEqual([ - ['replace_members_latest', 'completed'], - ['replace_members_historical', 'completed'], - ['replace_members_latest', 'failed'], ['delete_track', 'completed'], ]); expect(events[0]).toMatchObject({ @@ -157,38 +94,20 @@ describe('Release-track destructive authorization and audit', function () { role: 'admin', authentication_strategy: 'anonymId', }, - request: { members_count: 1 }, - result: { - snapshot_modified: new Date(latest.modified), - members_count: 1, - }, + result: { deleted: true }, }); - expect(events[2].track_id).toBe(virtual.id); - expect(events[2].error.message).toContain( - 'Direct contents updates are only available for standard release tracks', - ); - expect(events[3].result).toEqual({ deleted: true }); }); it('reports an audit-finalization failure without hiding the persisted mutation', async function () { await setRole('admin'); - const technique = await post('/api/techniques', techniquePayload(), 201); const track = await post( '/api/release-tracks/new', - { name: 'Audit finalization failure standard', type: 'standard' }, + { name: 'Track deletion audit finalization failure', type: 'standard' }, 201, ); - const contents = { - x_mitre_contents: [ - { - obj_ref: technique.stix.id, - obj_modified: technique.stix.modified, - }, - ], - }; sinon.stub(auditRepository, 'complete').rejects(new Error('injected audit update failure')); - const response = await api('post', `/api/release-tracks/${track.id}/contents`, contents, 500, { + const response = await api('delete', `/api/release-tracks/${track.id}`, undefined, 500, { confirm_track_id: track.id, }); auditRepository.complete.restore(); @@ -199,17 +118,7 @@ describe('Release-track destructive authorization and audit', function () { }); expect(response.body.audit_event_id).toEqual(expect.any(String)); - const latest = await api( - 'get', - `/api/release-tracks/${track.id}/snapshots/latest`, - undefined, - 200, - ); - expect(latest.body.members).toHaveLength(1); - expect(latest.body.members[0]).toMatchObject({ - object_ref: technique.stix.id, - object_modified: technique.stix.modified, - }); + await api('get', `/api/release-tracks/${track.id}/snapshots/latest`, undefined, 404); const pendingEvent = await ReleaseTrackAuditEvent.findOne({ event_id: response.body.audit_event_id, @@ -217,7 +126,7 @@ describe('Release-track destructive authorization and audit', function () { .lean() .exec(); expect(pendingEvent).toMatchObject({ - action: 'replace_members_latest', + action: 'delete_track', track_id: track.id, status: 'pending', }); diff --git a/app/tests/api/release-tracks/deterministic-graph-migration.spec.js b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js new file mode 100644 index 00000000..7a149c2e --- /dev/null +++ b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js @@ -0,0 +1,198 @@ +'use strict'; + +const mongoose = require('mongoose'); +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const migration = require('../../../../migrations/20260730180000-backfill-deterministic-snapshot-graphs'); +const Relationship = require('../../../models/relationship-model'); +const { + ReleaseTrackGraphManifest, + ReleaseTrackGraphManifestEntry, +} = require('../../../models/release-tracks/release-track-graph-manifest-model'); +const { releaseExactMembers } = require('./release-track-test-helpers'); + +const markingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; + +describe('Deterministic snapshot graph migration', function () { + let app; + let passportCookie; + let technique; + let group; + let relationship; + let trackId; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + async function post(path, body, status = 201) { + const response = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + return response.body; + } + + before('create and then downgrade representative legacy data', async function () { + const timestamp = new Date().toISOString(); + technique = await post('/api/techniques', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'attack-pattern', + spec_version: '2.1', + created: timestamp, + modified: timestamp, + name: 'Migration graph technique', + description: 'A primary migration fixture.', + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'execution' }], + x_mitre_is_subtechnique: false, + x_mitre_domains: ['enterprise-attack'], + x_mitre_platforms: ['Windows'], + object_marking_refs: [markingDefinitionId], + }, + }); + group = await post('/api/groups', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'intrusion-set', + spec_version: '2.1', + created: timestamp, + modified: timestamp, + name: 'Migration graph secondary', + description: 'A secondary migration fixture.', + object_marking_refs: [markingDefinitionId], + }, + }); + relationship = await post('/api/relationships', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'relationship', + spec_version: '2.1', + created: timestamp, + modified: timestamp, + relationship_type: 'uses', + source_ref: group.stix.id, + target_ref: technique.stix.id, + object_marking_refs: [markingDefinitionId], + }, + }); + const track = await post( + '/api/release-tracks/new', + { name: 'Legacy migration track', type: 'standard' }, + 201, + ); + trackId = track.id; + await releaseExactMembers(app, passportCookie, trackId, [technique]); + + await Relationship.updateOne( + { + 'stix.id': relationship.stix.id, + 'stix.modified': relationship.stix.modified, + }, + { $unset: { 'workspace.relationship_endpoints': '' } }, + ); + await mongoose.connection.db + .collection(trackId) + .updateMany({}, { $unset: { graph_manifest_id: '' } }); + await Promise.all([ + ReleaseTrackGraphManifest.deleteMany({ track_id: trackId }), + ReleaseTrackGraphManifestEntry.deleteMany({ track_id: trackId }), + ]); + }); + + it('supports a non-mutating dry run', async function () { + const report = await migration._private.run(mongoose.connection.db, { + dryRun: true, + }); + + expect(report.dry_run).toBe(true); + expect(report.relationship_pins_written).toBeGreaterThan(0); + expect(report.manifests_created).toBeGreaterThan(0); + const storedRelationship = await Relationship.findOne({ + 'stix.id': relationship.stix.id, + }) + .lean() + .exec(); + expect(storedRelationship.workspace.relationship_endpoints).toBeUndefined(); + expect(await ReleaseTrackGraphManifest.countDocuments({ track_id: trackId })).toBe(0); + }); + + it('pins latest relationships and rerunnably backfills baseline manifests', async function () { + await migration.up(mongoose.connection.db); + + const storedRelationship = await Relationship.findOne({ + 'stix.id': relationship.stix.id, + }) + .lean() + .exec(); + expect(storedRelationship.workspace.relationship_endpoints.source).toEqual({ + object_ref: group.stix.id, + object_modified: new Date(group.stix.modified), + }); + expect(storedRelationship.workspace.relationship_endpoints.target).toEqual({ + object_ref: technique.stix.id, + object_modified: new Date(technique.stix.modified), + }); + + const manifests = await ReleaseTrackGraphManifest.find({ + track_id: trackId, + }) + .lean() + .exec(); + expect(manifests.length).toBeGreaterThan(0); + expect(manifests.every((manifest) => manifest.baseline_reconstruction === true)).toBe(true); + const countAfterFirstRun = manifests.length; + + await migration.up(mongoose.connection.db); + expect(await ReleaseTrackGraphManifest.countDocuments({ track_id: trackId })).toBe( + countAfterFirstRun, + ); + + const response = await request(app) + .get(`/api/release-tracks/${trackId}/snapshots/latest?format=bundle&includeToc=false`) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + const objectIds = response.body.objects.map((object) => object.id); + expect(objectIds).toContain(technique.stix.id); + expect(objectIds).toContain(group.stix.id); + expect(objectIds).toContain(relationship.stix.id); + }); + + it('replays and activates a complete linked pending manifest after interruption', async function () { + const snapshot = await mongoose.connection.db + .collection(trackId) + .findOne({}, { sort: { modified: -1 } }); + await ReleaseTrackGraphManifest.updateOne( + { manifest_id: snapshot.graph_manifest_id }, + { $set: { state: 'pending' } }, + ).exec(); + + await request(app) + .get(`/api/release-tracks/${trackId}/snapshots/latest?format=bundle&includeToc=false`) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + + const manifest = await ReleaseTrackGraphManifest.findOne({ + manifest_id: snapshot.graph_manifest_id, + }) + .lean() + .exec(); + expect(manifest.state).toBe('active'); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/app/tests/api/release-tracks/ephemeral-bundle.spec.js b/app/tests/api/release-tracks/ephemeral-bundle.spec.js index d411a18e..3a930581 100644 --- a/app/tests/api/release-tracks/ephemeral-bundle.spec.js +++ b/app/tests/api/release-tracks/ephemeral-bundle.spec.js @@ -49,6 +49,8 @@ describe('Ephemeral Bundle API', function () { let icsTechnique; let group; let relationship; + let icsGroup; + let icsRelationship; before(async function () { await database.initializeConnection(); @@ -72,15 +74,19 @@ describe('Ephemeral Bundle API', function () { return res.body; } - async function getEphemeral(query = '', expectedStatus = 200) { + async function getEphemeralForDomain(domain, query = '', expectedStatus = 200) { const res = await request(app) - .get(`/api/release-tracks/ephemeral/enterprise${query}`) + .get(`/api/release-tracks/ephemeral/${domain}${query}`) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(expectedStatus); return res.body; } + async function getEphemeral(query = '', expectedStatus = 200) { + return getEphemeralForDomain('enterprise', query, expectedStatus); + } + function buildTechnique(name, domains, overrides = {}) { const timestamp = new Date().toISOString(); return { @@ -179,6 +185,33 @@ describe('Ephemeral Bundle API', function () { object_marking_refs: [staticMarkingDefinitionId], }, }); + + icsGroup = await postObject('/api/groups', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + name: 'Ephemeral ICS Test Group', + spec_version: '2.1', + type: 'intrusion-set', + description: 'Group used to verify request-local graph resolution.', + object_marking_refs: [staticMarkingDefinitionId], + }, + }); + + icsRelationship = await postObject('/api/relationships', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + spec_version: '2.1', + type: 'relationship', + relationship_type: 'uses', + source_ref: icsGroup.stix.id, + target_ref: icsTechnique.stix.id, + object_marking_refs: [staticMarkingDefinitionId], + }, + }); }); it('GET /api/release-tracks/ephemeral/:domain returns a STIX 2.1 bundle with legacy-parity contents', async function () { @@ -231,6 +264,37 @@ describe('Ephemeral Bundle API', function () { expect(toc.object_marking_refs).toContain(staticMarkingDefinitionId); }); + it('isolates graph state across concurrent domain exports', async function () { + const requests = Array.from({ length: 6 }, () => + Promise.all([ + getEphemeralForDomain('enterprise', '?includeToc=false'), + getEphemeralForDomain('ics', '?includeToc=false'), + ]), + ); + + for (const [enterpriseBundle, icsBundle] of await Promise.all(requests)) { + const enterpriseIds = bundleObjectIds(enterpriseBundle); + const icsIds = bundleObjectIds(icsBundle); + + expect(enterpriseIds).toContain(group.stix.id); + expect(enterpriseIds).toContain(relationship.stix.id); + expect(enterpriseIds).not.toContain(icsGroup.stix.id); + expect(enterpriseIds).not.toContain(icsRelationship.stix.id); + + expect(icsIds).toContain(icsGroup.stix.id); + expect(icsIds).toContain(icsRelationship.stix.id); + expect(icsIds).not.toContain(group.stix.id); + expect(icsIds).not.toContain(relationship.stix.id); + + expect( + enterpriseBundle.objects.find((object) => object.id === group.stix.id).x_mitre_domains, + ).toEqual([enterpriseDomain]); + expect( + icsBundle.objects.find((object) => object.id === icsGroup.stix.id).x_mitre_domains, + ).toEqual([icsDomain]); + } + }); + it('includeToc=false omits the TOC object', async function () { const bundle = await getEphemeral('?includeToc=false'); const tocObjects = bundle.objects.filter((o) => o.type === 'x-mitre-collection'); diff --git a/app/tests/api/release-tracks/primary-revision-integrity.spec.js b/app/tests/api/release-tracks/primary-revision-integrity.spec.js index 81c53614..4a043001 100644 --- a/app/tests/api/release-tracks/primary-revision-integrity.spec.js +++ b/app/tests/api/release-tracks/primary-revision-integrity.spec.js @@ -14,6 +14,7 @@ const ReleaseTrackRegistry = require('../../../models/release-tracks/release-tra const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); const techniquesRepo = require('../../../repository/techniques-repository'); const { DatabaseError } = require('../../../exceptions'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; @@ -138,26 +139,6 @@ describe('Release-track primary revision integrity API', function () { expect(latest.candidates[0].object_modified).toBe(technique.stix.modified); }); - it('rejects direct member replacement atomically when one exact revision is missing', async function () { - const technique = await createTechnique('Reject Missing Direct Member'); - const track = await createTrack('Reject Missing Direct Member Track'); - const missing = missingRevision(); - - const response = await api( - 'post', - `/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, - { - x_mitre_contents: [ - { obj_ref: technique.stix.id, obj_modified: technique.stix.modified }, - { obj_ref: missing.object_ref, obj_modified: missing.object_modified }, - ], - }, - 400, - ); - expect(response.body.missing_references).toEqual([missing]); - expect((await dynamicRepo.getAllSnapshots(track.id)).pagination.total).toBe(1); - }); - it('fails preview and release when a staged revision was deleted', async function () { const technique = await createTechnique('Deleted Staged Revision'); const track = await createTrack('Deleted Staged Revision Track'); @@ -199,9 +180,7 @@ describe('Release-track primary revision integrity API', function () { it('rejects cloning and export when a stored primary member is missing', async function () { const technique = await createTechnique('Missing Stored Member'); const track = await createTrack('Missing Stored Member Track'); - await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { - x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], - }); + await releaseExactMembers(app, passportCookie, track.id, [technique]); await deleteTechniqueRevision(technique); const registryCount = await ReleaseTrackRegistry.countDocuments(); @@ -233,9 +212,7 @@ describe('Release-track primary revision integrity API', function () { it('propagates repository hydration failures instead of returning a partial export', async function () { const technique = await createTechnique('Failed Primary Hydration'); const track = await createTrack('Failed Primary Hydration Track'); - await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { - x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], - }); + await releaseExactMembers(app, passportCookie, track.id, [technique]); const hydrationStub = sinon .stub(techniquesRepo, 'findManyByIdAndModified') .rejects(new DatabaseError(new Error('injected hydration failure'))); @@ -255,10 +232,7 @@ describe('Release-track primary revision integrity API', function () { it('aborts virtual materialization when a component member is missing', async function () { const technique = await createTechnique('Missing Virtual Component Member'); const component = await createTrack('Missing Virtual Component'); - await post(`/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { - x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], - }); - await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, { + await releaseExactMembers(app, passportCookie, component.id, [technique], { version: '1.0', }); const virtual = await createTrack('Missing Virtual Primary', 'virtual', { diff --git a/app/tests/api/release-tracks/reconciliation-durability.spec.js b/app/tests/api/release-tracks/reconciliation-durability.spec.js index 52e0792e..ca9ad18d 100644 --- a/app/tests/api/release-tracks/reconciliation-durability.spec.js +++ b/app/tests/api/release-tracks/reconciliation-durability.spec.js @@ -14,6 +14,7 @@ const attackObjectsRepo = require('../../../repository/attack-objects-repository const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); const reconciliationService = require('../../../services/release-tracks/reconciliation-service'); const { DatabaseError } = require('../../../exceptions'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; @@ -172,9 +173,7 @@ describe('Release-track durable backref reconciliation', function () { { name: 'Full Scan Repair Track', type: 'standard' }, 201, ); - await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { - x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], - }); + await releaseExactMembers(app, passportCookie, track.id, [technique]); await Technique.updateOne( { diff --git a/app/tests/api/release-tracks/release-track-test-helpers.js b/app/tests/api/release-tracks/release-track-test-helpers.js new file mode 100644 index 00000000..5f28b73d --- /dev/null +++ b/app/tests/api/release-tracks/release-track-test-helpers.js @@ -0,0 +1,49 @@ +'use strict'; + +const request = require('supertest'); + +function exactObjectRef(object) { + if (object.stix) { + return { id: object.stix.id, modified: object.stix.modified }; + } + if (object.object_ref) { + return { id: object.object_ref, modified: object.object_modified }; + } + return { id: object.id, modified: object.modified }; +} + +function authenticated(requestBuilder, passportCookie) { + return requestBuilder + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); +} + +async function stageExactMembers(app, passportCookie, trackId, objects) { + const refs = objects.map(exactObjectRef); + await authenticated( + request(app).post(`/api/release-tracks/${trackId}/candidates`).send({ object_refs: refs }), + passportCookie, + ).expect(200); + + const response = await authenticated( + request(app) + .post(`/api/release-tracks/${trackId}/candidates/promote`) + .send({ object_refs: refs.map((ref) => ref.id) }), + passportCookie, + ).expect(200); + return response.body; +} + +async function releaseExactMembers(app, passportCookie, trackId, objects, releaseBody = {}) { + await stageExactMembers(app, passportCookie, trackId, objects); + const response = await authenticated( + request(app).post(`/api/release-tracks/${trackId}/snapshots/latest/release`).send(releaseBody), + passportCookie, + ).expect(200); + return response.body; +} + +module.exports = { + releaseExactMembers, + stageExactMembers, +}; diff --git a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js index 20361bf8..0b3b54bd 100644 --- a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js +++ b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js @@ -5,6 +5,7 @@ const config = require('../../../config/config'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const logger = require('../../../lib/logger'); logger.level = 'debug'; @@ -297,30 +298,24 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { }); describe('members and snapshots', function () { - it('setting track contents adds member backrefs and reverts on snapshot delete', async function () { + it('deleting the latest draft reverts its candidate backrefs', async function () { const technique = await postObject('/api/techniques', buildTechnique('Backref Contents')); const trackId = await createTrack('Backref Contents Track'); - const contentsSnapshot = await postObject( - `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, - { - x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], - }, - 200, - ); + const candidateSnapshot = await addCandidates(trackId, [technique]); let retrieved = await getTechniqueVersion(technique); expect(entryForTrack(retrieved, trackId)).toEqual({ id: trackId, type: 'standard', - tier: 'members', - status: 'reviewed', + tier: 'candidates', + status: 'work-in-progress', }); - // Deleting the latest snapshot reverts membership to the previous + // Deleting the latest snapshot reverts contents to the previous // (empty) snapshot — the backref disappears await request(app) - .delete(`/api/release-tracks/${trackId}/snapshots/${contentsSnapshot.modified}`) + .delete(`/api/release-tracks/${trackId}/snapshots/${candidateSnapshot.modified}`) .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(204); @@ -364,13 +359,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { const revisionA = await postObject('/api/techniques', buildTechnique('Backref Member Sync')); const trackId = await createTrack('Backref Member Sync Track'); - await postObject( - `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, - { - x_mitre_contents: [{ obj_ref: revisionA.stix.id, obj_modified: revisionA.stix.modified }], - }, - 200, - ); + await releaseExactMembers(app, passportCookie, trackId, [revisionA]); // Creating a new revision triggers member sync (default strategy: // track_latest) which auto-enrolls the new revision as a candidate @@ -469,13 +458,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { buildTechnique('Backref Dynamic Ignore'), ); const trackId = await createTrack('Backref Dynamic Ignore Track'); - await postObject( - `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, - { - x_mitre_contents: [{ obj_ref: revisionA.stix.id, obj_modified: revisionA.stix.modified }], - }, - 200, - ); + await releaseExactMembers(app, passportCookie, trackId, [revisionA]); await request(app) .put(`/api/release-tracks/${trackId}/config`) .send({ diff --git a/app/tests/api/release-tracks/release-tracks-bundle.spec.js b/app/tests/api/release-tracks/release-tracks-bundle.spec.js index f95977a1..02017e9c 100644 --- a/app/tests/api/release-tracks/release-tracks-bundle.spec.js +++ b/app/tests/api/release-tracks/release-tracks-bundle.spec.js @@ -11,8 +11,8 @@ * Covered behavior: * - Default bundle contains members only, plus referenced identities and * marking definitions (self-contained bundle) - * - Active relationships whose endpoints are both selected are added - * dynamically; relationships with an endpoint outside the export are not + * - Active relationships and their bounded secondary objects are frozen in + * a snapshot graph manifest * - `include` adds staged and/or candidate tiers (comma-separated or * repeated, singular or plural tier names) * - `state` narrows the included staged/candidate entries by workflow @@ -31,6 +31,7 @@ const config = require('../../../config/config'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const logger = require('../../../lib/logger'); logger.level = 'debug'; @@ -53,6 +54,8 @@ describe('Release Tracks Bundle Export API', function () { let relationshipSource; let includedRelationship; let excludedRelationship; + let secondaryGroup; + let secondaryRelationship; let linkedAttackId; let linkedAttackUrl; let candidateWip; @@ -195,6 +198,32 @@ describe('Release Tracks Bundle Export API', function () { object_marking_refs: [staticMarkingDefinitionId], }, }); + secondaryGroup = await postObject('/api/groups', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + name: 'Bundle Secondary Group', + description: 'A relationship-discovered secondary object.', + spec_version: '2.1', + type: 'intrusion-set', + object_marking_refs: [staticMarkingDefinitionId], + }, + }); + secondaryRelationship = await postObject('/api/relationships', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + description: 'Frozen relationship description.', + spec_version: '2.1', + type: 'relationship', + relationship_type: 'uses', + source_ref: secondaryGroup.stix.id, + target_ref: memberObject.stix.id, + object_marking_refs: [staticMarkingDefinitionId], + }, + }); const track = await postAction( '/api/release-tracks/new', @@ -216,14 +245,13 @@ describe('Release Tracks Bundle Export API', function () { .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(200); - // Members - await postAction(`/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, { - x_mitre_contents: [ - { obj_ref: memberObject.stix.id, obj_modified: memberObject.stix.modified }, - { obj_ref: linkedMemberObject.stix.id, obj_modified: linkedMemberObject.stix.modified }, - { obj_ref: relationshipSource.stix.id, obj_modified: relationshipSource.stix.modified }, - ], - }); + // Members enter through the supported candidate → staged → release + // lifecycle. + await releaseExactMembers(app, passportCookie, trackId, [ + memberObject, + linkedMemberObject, + relationshipSource, + ]); // Candidates (all start as work-in-progress) await postAction(`/api/release-tracks/${trackId}/candidates`, { @@ -265,6 +293,8 @@ describe('Release Tracks Bundle Export API', function () { const ids = bundleObjectIds(bundle); expect(ids).toContain(memberObject.stix.id); expect(ids).toContain(linkedMemberObject.stix.id); + expect(ids).toContain(secondaryGroup.stix.id); + expect(ids).toContain(secondaryRelationship.stix.id); // Tier entries not selected via include are excluded expect(ids).not.toContain(candidateWip.stix.id); @@ -314,6 +344,7 @@ describe('Release Tracks Bundle Export API', function () { const ids = bundleObjectIds(bundle); expect(ids).toContain(includedRelationship.stix.id); + expect(ids).toContain(secondaryRelationship.stix.id); expect(ids).not.toContain(excludedRelationship.stix.id); const snapshot = await getBundle(`/api/release-tracks/${trackId}/snapshots/latest`); @@ -322,6 +353,107 @@ describe('Release Tracks Bundle Export API', function () { ); }); + it('replays frozen relationship payloads and protects graph dependencies', async function () { + const relationshipUpdate = JSON.parse(JSON.stringify(secondaryRelationship)); + relationshipUpdate.stix.description = 'A later in-place typo correction.'; + relationshipUpdate.stix.external_references = [ + { + source_name: 'deterministic-bundle-test', + description: 'Regression-test relationship source.', + }, + ]; + + await request(app) + .put( + `/api/relationships/${secondaryRelationship.stix.id}/modified/` + + encodeURIComponent(secondaryRelationship.stix.modified), + ) + .send(relationshipUpdate) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + + const bundle = await getBundle( + `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&includeToc=false`, + ); + const frozenRelationship = bundle.objects.find( + (object) => object.id === secondaryRelationship.stix.id, + ); + expect(frozenRelationship.description).toBe('Frozen relationship description.'); + + await request(app) + .delete( + `/api/relationships/${secondaryRelationship.stix.id}/modified/` + + encodeURIComponent(secondaryRelationship.stix.modified), + ) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(409); + + const secondaryUpdate = JSON.parse(JSON.stringify(secondaryGroup)); + secondaryUpdate.stix.description = 'Attempted in-place graph drift.'; + await request(app) + .put( + `/api/groups/${secondaryGroup.stix.id}/modified/` + + encodeURIComponent(secondaryGroup.stix.modified), + ) + .send(secondaryUpdate) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(409); + + await request(app) + .delete( + `/api/groups/${secondaryGroup.stix.id}/modified/` + + encodeURIComponent(secondaryGroup.stix.modified), + ) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(409); + }); + + it('protects graph dependencies from collection cascade deletion', async function () { + const timestamp = new Date().toISOString(); + const collection = await postObject('/api/collections', { + workspace: { + imported: timestamp, + import_categories: {}, + workflow: {}, + }, + stix: { + id: `x-mitre-collection--${trackUuid}`, + type: 'x-mitre-collection', + spec_version: '2.1', + created: timestamp, + modified: timestamp, + name: 'Graph protection cascade fixture', + description: 'Attempts to cascade-delete a protected secondary object.', + x_mitre_version: '1.0', + x_mitre_contents: [ + { + object_ref: secondaryGroup.stix.id, + object_modified: secondaryGroup.stix.modified, + }, + ], + object_marking_refs: [staticMarkingDefinitionId], + }, + }); + + await request(app) + .delete( + `/api/collections/${collection.stix.id}/modified/` + + `${encodeURIComponent(collection.stix.modified)}?deleteAllContents=true`, + ) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(409); + + await request(app) + .get( + `/api/groups/${secondaryGroup.stix.id}/modified/` + + encodeURIComponent(secondaryGroup.stix.modified), + ) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + }); + it('GET /api/release-tracks/:id/snapshots/latest?format=bundle&includeToc=false omits the TOC', async function () { const bundle = await getBundle( `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&includeToc=false`, diff --git a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js index 1f313696..246645e6 100644 --- a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js +++ b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js @@ -5,6 +5,7 @@ const config = require('../../../config/config'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const logger = require('../../../lib/logger'); logger.level = 'debug'; @@ -100,13 +101,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { } async function setMembers(trackId, technique) { - return postObject( - `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, - { - x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], - }, - 200, - ); + return releaseExactMembers(app, passportCookie, trackId, [technique]); } async function latestSnapshotModified(trackId) { diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js index 5a1c7843..a2d6686f 100644 --- a/app/tests/api/release-tracks/release-tracks-release.spec.js +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -11,6 +11,7 @@ const login = require('../../shared/login'); const releaseHistoryService = require('../../../services/release-tracks/release-history-service'); const versioningService = require('../../../services/release-tracks/versioning-service'); const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const staticMarkingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; const virtualObjectRefs = [ @@ -363,14 +364,10 @@ describe('Release-track release planning and commit API', function () { it('records immutable component versions when previewing and releasing a virtual draft', async function () { const member = (await post('/api/techniques', buildTechnique('Provenance Member'), 201)).body; const component = await createTrack('Provenance Component'); - await post(`/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { - x_mitre_contents: [{ obj_ref: member.stix.id, obj_modified: member.stix.modified }], - }); - const firstComponentRelease = await post( - `/api/release-tracks/${component.id}/snapshots/latest/release`, - {}, - ); - expect(firstComponentRelease.body.version).toBe('1.0'); + const firstComponentRelease = await releaseExactMembers(app, passportCookie, component.id, [ + member, + ]); + expect(firstComponentRelease.version).toBe('1.0'); const virtual = ( await post( @@ -401,8 +398,8 @@ describe('Release-track release planning and commit API', function () { // Advance the component after materialization. Virtual release provenance // must remain tied to the frozen component resolution, not current state. - await post(`/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { - x_mitre_contents: [{ obj_ref: member.stix.id, obj_modified: member.stix.modified }], + await post(`/api/release-tracks/${component.id}/meta`, { + description: 'Component draft created after virtual materialization', }); const secondComponentRelease = await post( `/api/release-tracks/${component.id}/snapshots/latest/release`, @@ -696,10 +693,7 @@ describe('Release-track release planning and commit API', function () { await post('/api/techniques', buildTechnique('Virtual Materialization Member'), 201) ).body; const component = await createTrack('Virtual Materialization Component'); - await post(`/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { - x_mitre_contents: [{ obj_ref: member.stix.id, obj_modified: member.stix.modified }], - }); - await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, {}); + await releaseExactMembers(app, passportCookie, component.id, [member]); const virtual = ( await post( @@ -758,7 +752,7 @@ describe('Release-track release planning and commit API', function () { expect(preview.body.releasable).toBe(true); }); - it('rejects generic contents replacement for virtual tracks', async function () { + it('does not expose generic contents replacement for virtual tracks', async function () { const virtual = await createTrack('Virtual Contents Guard', 'virtual'); const contents = { x_mitre_contents: [ @@ -772,12 +766,12 @@ describe('Release-track release planning and commit API', function () { await post( `/api/release-tracks/${virtual.id}/contents?confirm_track_id=${virtual.id}`, contents, - 400, + 404, ); await post( `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent(virtual.modified)}/contents?confirm_track_id=${virtual.id}`, contents, - 400, + 404, ); const latest = await get(`/api/release-tracks/${virtual.id}/snapshots/latest`); @@ -792,9 +786,7 @@ describe('Release-track release planning and commit API', function () { await post('/api/techniques', buildTechnique('Release Conflict B', revisionA), 201) ).body; const track = await createTrack('Release Conflict'); - await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { - x_mitre_contents: [{ obj_ref: revisionA.stix.id, obj_modified: revisionA.stix.modified }], - }); + await releaseExactMembers(app, passportCookie, track.id, [revisionA]); await post(`/api/release-tracks/${track.id}/candidates`, { object_refs: [{ id: revisionB.stix.id, modified: 'latest' }], }); diff --git a/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js b/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js index 5096b577..272ff916 100644 --- a/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js +++ b/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js @@ -7,6 +7,7 @@ const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); const snapshotService = require('../../../services/release-tracks/snapshot-service'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const staticMarkingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; const tiers = ['members', 'staged', 'candidates', 'quarantine']; @@ -125,12 +126,7 @@ describe('Release-track cross-tier revision uniqueness', function () { } async function setMembers(trackId, objects) { - return post(`/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, { - x_mitre_contents: objects.map((object) => ({ - obj_ref: object.stix.id, - obj_modified: object.stix.modified, - })), - }); + return releaseExactMembers(app, passportCookie, trackId, objects); } async function useManualMemberSync(trackId) { diff --git a/app/tests/api/release-tracks/release-tracks.spec.js b/app/tests/api/release-tracks/release-tracks.spec.js index cf46f412..33079058 100644 --- a/app/tests/api/release-tracks/release-tracks.spec.js +++ b/app/tests/api/release-tracks/release-tracks.spec.js @@ -7,6 +7,7 @@ const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const AttackObject = require('../../../models/attack-object-model'); const snapshotService = require('../../../services/release-tracks/snapshot-service'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const logger = require('../../../lib/logger'); logger.level = 'debug'; @@ -107,21 +108,7 @@ describe('Release Tracks API', function () { const trackId = createRes.body.id; - await request(app) - .post(`/api/release-tracks/${trackId}/contents`) - .query({ confirm_track_id: trackId }) - .send({ - x_mitre_contents: [ - { - obj_ref: memberObject.stix.id, - obj_modified: memberObject.stix.modified, - }, - ], - }) - .set('Accept', 'application/json') - .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) - .expect('Content-Type', /json/); + await releaseExactMembers(app, passportCookie, trackId, [memberObject]); await request(app) .post(`/api/release-tracks/${trackId}/candidates`) diff --git a/app/tests/api/release-tracks/releases-by-object.spec.js b/app/tests/api/release-tracks/releases-by-object.spec.js index a89e59ca..1c24567a 100644 --- a/app/tests/api/release-tracks/releases-by-object.spec.js +++ b/app/tests/api/release-tracks/releases-by-object.spec.js @@ -8,6 +8,7 @@ const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const ReleaseTrackRegistry = require('../../../models/release-tracks/release-track-registry-model'); const backfillMigration = require('../../../../migrations/20260716000000-backfill-release-track-tagged-releases'); +const { stageExactMembers } = require('./release-track-test-helpers'); const staticMarkingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; @@ -67,12 +68,12 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { const createdA = await createTrack('Releases By Object A'); trackA = createdA.id; const initialSnapshotModified = createdA.modified; - trackATaggedSnapshot = await setMembers(trackA, [objectRevisionA]); - await releaseLatest(trackA); + await setMembers(trackA, [objectRevisionA]); + trackATaggedSnapshot = await releaseLatest(trackA); - // Remove the requested object from the latest state and tag that state. - // The earlier tagged release must remain discoverable despite its current - // backref disappearing. + // Append another object in a later release. Existing members remain part + // of the immutable lineage because direct member replacement is not + // supported. await setMembers(trackA, [otherObject]); await releaseLatest(trackA); @@ -151,16 +152,7 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { } async function setMembers(trackId, objects) { - return post( - `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, - { - x_mitre_contents: objects.map((object) => ({ - obj_ref: object.stix.id, - obj_modified: object.stix.modified, - })), - }, - 200, - ); + return stageExactMembers(app, passportCookie, trackId, objects); } async function releaseLatest(trackId, increment = 'minor') { @@ -177,19 +169,30 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { const response = await get(`/api/release-tracks/objects/${objectRevisionA.stix.id}/releases`); expect(response.body.object_ref).toBe(objectRevisionA.stix.id); - expect(response.body.pagination).toEqual({ total: 3, limit: 50, offset: 0 }); - expect(response.body.data).toHaveLength(3); + expect(response.body.pagination).toEqual({ total: 4, limit: 50, offset: 0 }); + expect(response.body.data).toHaveLength(4); - const standardA = response.body.data.find((entry) => entry.track_id === trackA); + const standardA = response.body.data.filter((entry) => entry.track_id === trackA); const standardB = response.body.data.find((entry) => entry.track_id === trackB); const virtual = response.body.data.find((entry) => entry.track_id === virtualTrack); - expect(standardA).toMatchObject({ - track_type: 'standard', - track_name: 'Releases By Object A', - version: '1.0', - object_modified: objectRevisionA.stix.modified, - }); + expect(standardA).toHaveLength(2); + expect(standardA).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + track_type: 'standard', + track_name: 'Releases By Object A', + version: '1.0', + object_modified: objectRevisionA.stix.modified, + }), + expect.objectContaining({ + track_type: 'standard', + track_name: 'Releases By Object A', + version: '1.1', + object_modified: objectRevisionA.stix.modified, + }), + ]), + ); expect(standardB).toMatchObject({ track_type: 'standard', version: '1.0', @@ -238,7 +241,7 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { const standard = await get( `/api/release-tracks/objects/${objectRevisionA.stix.id}/releases?type=standard&order=desc&limit=1&offset=1`, ); - expect(standard.body.pagination).toEqual({ total: 2, limit: 1, offset: 1 }); + expect(standard.body.pagination).toEqual({ total: 3, limit: 1, offset: 1 }); expect(standard.body.data).toHaveLength(1); expect(standard.body.data[0].track_type).toBe('standard'); @@ -293,6 +296,6 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { const response = await get( `/api/release-tracks/objects/${objectRevisionA.stix.id}/releases?type=standard`, ); - expect(response.body.pagination.total).toBe(2); + expect(response.body.pagination.total).toBe(3); }); }); diff --git a/app/tests/api/release-tracks/snapshot-immutability.spec.js b/app/tests/api/release-tracks/snapshot-immutability.spec.js new file mode 100644 index 00000000..3bd00ee5 --- /dev/null +++ b/app/tests/api/release-tracks/snapshot-immutability.spec.js @@ -0,0 +1,104 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); + +describe('Release-track snapshot immutability contract', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + after(async function () { + await database.closeConnection(); + }); + + function api(method, path, body, status) { + const call = request(app) + [method](path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + if (body !== undefined) call.send(body); + return call.expect(status); + } + + async function post(path, body, status = 200) { + return (await api('post', path, body, status)).body; + } + + it('does not expose direct snapshot metadata or member-replacement routes', async function () { + const track = await post( + '/api/release-tracks/new', + { name: 'Removed snapshot mutation routes', type: 'standard' }, + 201, + ); + const modified = encodeURIComponent(track.modified); + + await api('post', `/api/release-tracks/${track.id}/contents`, {}, 404); + await api('post', `/api/release-tracks/${track.id}/snapshots/${modified}/meta`, {}, 404); + await api('post', `/api/release-tracks/${track.id}/snapshots/${modified}/contents`, {}, 404); + }); + + it('deletes only the latest untagged draft', async function () { + const initial = await post( + '/api/release-tracks/new', + { name: 'Latest draft deletion boundary', type: 'standard' }, + 201, + ); + const middle = await post(`/api/release-tracks/${initial.id}/meta`, { + description: 'Middle draft', + }); + const latest = await post(`/api/release-tracks/${initial.id}/meta`, { + description: 'Latest draft', + }); + + const historicalDelete = await api( + 'delete', + `/api/release-tracks/${initial.id}/snapshots/${encodeURIComponent(initial.modified)}`, + undefined, + 409, + ); + expect(historicalDelete.body).toEqual({ + message: 'Only the latest untagged snapshot can be deleted', + snapshot_modified: initial.modified, + latest_snapshot_modified: latest.modified, + }); + + await api( + 'delete', + `/api/release-tracks/${initial.id}/snapshots/${encodeURIComponent(latest.modified)}`, + undefined, + 204, + ); + const reverted = await api( + 'get', + `/api/release-tracks/${initial.id}/snapshots/latest`, + undefined, + 200, + ); + expect(reverted.body.modified).toBe(middle.modified); + + await post(`/api/release-tracks/${initial.id}/snapshots/latest/release`, { + version: '1.0', + }); + const taggedDelete = await api( + 'delete', + `/api/release-tracks/${initial.id}/snapshots/${encodeURIComponent(middle.modified)}`, + undefined, + 409, + ); + expect(taggedDelete.text).toContain('Tagged snapshot version 1.0 cannot be deleted'); + }); +}); diff --git a/app/tests/api/release-tracks/tagged-content-immutability.spec.js b/app/tests/api/release-tracks/tagged-content-immutability.spec.js index d72c675a..3118c9ab 100644 --- a/app/tests/api/release-tracks/tagged-content-immutability.spec.js +++ b/app/tests/api/release-tracks/tagged-content-immutability.spec.js @@ -8,6 +8,8 @@ const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const ReleaseTrackRegistry = require('../../../models/release-tracks/release-track-registry-model'); +const Technique = require('../../../models/technique-model'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; @@ -65,23 +67,24 @@ describe('Release-track authoritative tagged-content immutability', function () it('blocks mutation from historical tagged membership when current backrefs are absent', async function () { const technique = await post('/api/techniques', buildTechnique('Historical Member'), 201); - const replacement = await post('/api/techniques', buildTechnique('Current Draft Member'), 201); const track = await post( '/api/release-tracks/new', { name: 'Historical Immutability', type: 'standard' }, 201, ); - await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { - x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], + await releaseExactMembers(app, passportCookie, track.id, [technique], { + version: '1.0', }); - await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, { version: '1.0' }); - // A newer draft removes the member, so latest-snapshot reconciliation - // deliberately removes the object's denormalized backref. The historical - // tagged snapshot remains the immutable authority. - await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { - x_mitre_contents: [{ obj_ref: replacement.stix.id, obj_modified: replacement.stix.modified }], - }); + // Simulate a stale derived backref. The tagged snapshot remains the + // immutable authority even when both denormalized indexes are missing. + await Technique.updateOne( + { + 'stix.id': technique.stix.id, + 'stix.modified': new Date(technique.stix.modified), + }, + { $pull: { 'workspace.release_tracks': { id: track.id } } }, + ); const current = ( await api( 'get', diff --git a/app/tests/api/release-tracks/virtual-deduplication.spec.js b/app/tests/api/release-tracks/virtual-deduplication.spec.js index b35b9d27..64b70c76 100644 --- a/app/tests/api/release-tracks/virtual-deduplication.spec.js +++ b/app/tests/api/release-tracks/virtual-deduplication.spec.js @@ -7,6 +7,7 @@ const config = require('../../../config/config'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const staticMarkingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; @@ -88,17 +89,7 @@ describe('Virtual release-track deduplication API', function () { async function createReleasedComponent(name, members) { const track = await post('/api/release-tracks/new', { name, type: 'standard' }); - await post( - `/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, - { - x_mitre_contents: members.map((member) => ({ - obj_ref: member.stix.id, - obj_modified: member.stix.modified, - })), - }, - 200, - ); - const release = await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, {}, 200); + const release = await releaseExactMembers(app, passportCookie, track.id, members); return { ...track, release }; } diff --git a/app/tests/api/release-tracks/virtual-determinism.spec.js b/app/tests/api/release-tracks/virtual-determinism.spec.js index a58d028b..d2bfd1a7 100644 --- a/app/tests/api/release-tracks/virtual-determinism.spec.js +++ b/app/tests/api/release-tracks/virtual-determinism.spec.js @@ -9,6 +9,7 @@ const databaseConfiguration = require('../../../lib/database-configuration'); const modelFactory = require('../../../models/release-tracks/model-factory'); const login = require('../../shared/login'); const { cloneForCreate } = require('../../shared/clone-for-create'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const staticMarkingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; @@ -81,19 +82,9 @@ describe('Virtual release-track deterministic membership API', function () { name, type: 'standard', }); - const contents = await post( - `/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, - { - x_mitre_contents: [ - { - obj_ref: member.stix.id, - obj_modified: modified, - }, - ], - }, - 200, - ); - await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, {}, 200); + const contents = await releaseExactMembers(app, passportCookie, component.id, [ + { id: member.stix.id, modified }, + ]); return { component, contents }; } diff --git a/app/tests/api/release-tracks/virtual-domain-filters.spec.js b/app/tests/api/release-tracks/virtual-domain-filters.spec.js index ae38757d..98e3c984 100644 --- a/app/tests/api/release-tracks/virtual-domain-filters.spec.js +++ b/app/tests/api/release-tracks/virtual-domain-filters.spec.js @@ -6,6 +6,7 @@ const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const { cloneForCreate } = require('../../shared/clone-for-create'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const staticMarkingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; @@ -113,17 +114,13 @@ describe('Virtual Release Track Domain Filters API', function () { name: 'Domain Filter Component', type: 'standard', }); - await post( - `/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, - { - x_mitre_contents: [enterprise, ics, shared, noDomain, enterpriseMatrix].map((object) => ({ - obj_ref: object.stix.id, - obj_modified: object.stix.modified, - })), - }, - 200, - ); - await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, {}, 200); + await releaseExactMembers(app, passportCookie, component.id, [ + enterprise, + ics, + shared, + noDomain, + enterpriseMatrix, + ]); // A newer revision has a different domain, but virtual composition must // evaluate the exact revision pinned in the tagged component snapshot. diff --git a/app/tests/api/release-tracks/virtual-object-type-filters.spec.js b/app/tests/api/release-tracks/virtual-object-type-filters.spec.js index c580c436..9353d49e 100644 --- a/app/tests/api/release-tracks/virtual-object-type-filters.spec.js +++ b/app/tests/api/release-tracks/virtual-object-type-filters.spec.js @@ -14,6 +14,7 @@ const { compositionSchema, } = require('../../../models/release-tracks/release-track-snapshot-schema'); const releaseTracksService = require('../../../services/release-tracks/release-tracks-service'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const staticMarkingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; const supportedObjectTypes = Object.values(types); @@ -202,17 +203,7 @@ describe('Virtual release-track object-type filters API', function () { const mitigation = await post('/api/mitigations', buildMitigation('Pinned Type Member')); const matrix = await post('/api/matrices', buildMatrix('Excluded Type Member')); - await post( - `/api/release-tracks/${componentTrack.id}/contents?confirm_track_id=${componentTrack.id}`, - { - x_mitre_contents: [mitigation, matrix].map((object) => ({ - obj_ref: object.stix.id, - obj_modified: object.stix.modified, - })), - }, - 200, - ); - await post(`/api/release-tracks/${componentTrack.id}/snapshots/latest/release`, {}, 200); + await releaseExactMembers(app, passportCookie, componentTrack.id, [mitigation, matrix]); const newerMitigationRevision = cloneForCreate(mitigation); newerMitigationRevision.stix.modified = new Date(Date.now() + 1000).toISOString(); diff --git a/app/tests/api/release-tracks/virtual-quarantine.spec.js b/app/tests/api/release-tracks/virtual-quarantine.spec.js index d2e0a24d..f63e6c0c 100644 --- a/app/tests/api/release-tracks/virtual-quarantine.spec.js +++ b/app/tests/api/release-tracks/virtual-quarantine.spec.js @@ -7,6 +7,7 @@ const config = require('../../../config/config'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const staticMarkingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; @@ -73,15 +74,7 @@ describe('Virtual release-track quarantine API', function () { async function createReleasedComponent(name, member) { const track = await createTrack(name); - await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { - x_mitre_contents: [ - { - obj_ref: member.stix.id, - obj_modified: member.stix.modified, - }, - ], - }); - await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, {}); + await releaseExactMembers(app, passportCookie, track.id, [member]); return track; } diff --git a/app/tests/api/reports/reports.spec.js b/app/tests/api/reports/reports.spec.js index e3e696be..ae4085c4 100644 --- a/app/tests/api/reports/reports.spec.js +++ b/app/tests/api/reports/reports.spec.js @@ -4,6 +4,7 @@ const { expect } = require('expect'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const AttackObject = require('../../../models/attack-object-model'); +const Technique = require('../../../models/technique-model'); const config = require('../../../config/config'); const login = require('../../shared/login'); @@ -70,6 +71,20 @@ describe('Reports API', function () { // Check for a valid database configuration await databaseConfiguration.checkSystemConfiguration(); + const targetTimestamp = new Date(); + await Technique.create({ + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'attack-pattern', + spec_version: '2.1', + id: targetRef2, + created: targetTimestamp, + modified: targetTimestamp, + name: 'Report relationship target', + x_mitre_is_subtechnique: false, + }, + }); + // Enable ADM validation; the request payloads in this spec are ADM-compliant config.validateRequests.withAttackDataModel = true; config.validateRequests.withOpenApi = true; diff --git a/docs/README.md b/docs/README.md index b4e1ee03..b4e64a7e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -60,7 +60,8 @@ Configuration, deployment, and identity provider setup. - [Automation Run Audit Trail](admin/automation-runs.md): How to inspect migration and scheduler audit records - [Virtual Track Schedules](admin/virtual-track-schedules.md): UTC execution, restart recovery, retries, and observability - [Release-Track Membership Reconciliation](admin/release-track-reconciliation.md): Inspect and repair durable object-backref protection failures -- [Release-Track Destructive Audit Events](admin/release-track-audit.md): Inspect administrator member replacements and track deletions +- [Release-Track Destructive Audit Events](admin/release-track-audit.md): Inspect administrator track-deletion attempts +- [Release-Track Deterministic Graph Migration](admin/release-track-graph-migration.md): Preview and operate the relationship-pin and snapshot-manifest backfill ### Authentication diff --git a/docs/admin/release-track-audit.md b/docs/admin/release-track-audit.md index 548e7a68..c9634eb7 100644 --- a/docs/admin/release-track-audit.md +++ b/docs/admin/release-track-audit.md @@ -1,7 +1,7 @@ # Release-Track Destructive Audit Events -Workbench stores administrator-initiated member replacement and full-track -deletion attempts in `releaseTrackAuditEvents`. +Workbench stores administrator-initiated full-track deletion attempts in +`releaseTrackAuditEvents`. Each record contains: @@ -41,10 +41,8 @@ db.releaseTrackAuditEvents ``` A `pending` event can mean the process stopped after the audit insert or the -operation completed but the final audit update failed. Inspect the target -track before retrying. A failed member replacement may also have persisted a -new snapshot if backref reconciliation subsequently failed; correlate its -timestamp with `releaseTrackReconciliations`. +track was deleted but the final audit update failed. Confirm whether the track +still exists before retrying. These records have no automatic TTL. Establish retention and archive policy according to local audit requirements. diff --git a/docs/admin/release-track-graph-migration.md b/docs/admin/release-track-graph-migration.md new file mode 100644 index 00000000..227e562f --- /dev/null +++ b/docs/admin/release-track-graph-migration.md @@ -0,0 +1,53 @@ +# Release-Track Deterministic Graph Migration + +Release-track snapshot bundles depend on exact relationship endpoints and a +frozen snapshot graph manifest. The +`20260730180000-backfill-deterministic-snapshot-graphs` migration establishes +that data for an existing Workbench database. + +## Before deployment + +Run the read-only preview against the target database: + +```bash +DATABASE_URL='mongodb://host/database' \ + npm run preview:deterministic-snapshot-graphs +``` + +The report includes the latest relationship revisions scanned, endpoint pins +that would be written, release-track snapshots found, and baseline manifests +that would be created. No database writes or indexes are created by this +command. + +The preview fails if a latest relationship references a source or target +object that no longer exists. Repair those dangling endpoints before +deployment. Snapshot graph capture fails closed rather than silently producing +an incomplete deterministic baseline. + +## What the migration writes + +- Exact source and target revision metadata is added only to the latest + revision of each relationship in the underlying `relationships` collection. + `view.relationships.latest` may be used for discovery but is never written. +- Each existing release-track snapshot receives a graph manifest containing + its exact primary, relationship, secondary, supporting, and LinkById + dependencies. +- Backfilled manifests are marked `baseline_reconstruction: true`. They + reproduce the graph visible at migration time; the server cannot infer the + historically exact graph of snapshots created before endpoint pins existed. + +The migration is rerunnable. A complete manifest already linked to a snapshot +is reused, and a linked pending manifest left by an interrupted activation is +activated instead of duplicated. + +## Deployment behavior + +With `WB_REST_DATABASE_MIGRATION_ENABLE=true`, the migration runs during +normal server startup. If migrations are managed separately, run the standard +`migrate-mongo` workflow after reviewing the preview and before accepting +release-track traffic. + +After deployment, smoke-test one standard and one virtual snapshot with +`format=bundle`. Editing or hard-deleting a frozen secondary revision should +return `409 Conflict`; creating a new revision remains the supported update +path. diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index c46b2791..53920a37 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -21,8 +21,8 @@ Keep these rules in mind while updating the connector: namespace. Snapshot retrieval and release operations are shared. - New virtual-only operations include `/virtual/` in the path. - The current OpenAPI document is authoritative. Some older standard-only - workflow routes, such as `/candidates`, `/staged`, and `/contents`, predate - the namespace convention and do not currently include `/standard/`. + workflow routes, such as `/candidates` and `/staged`, predate the namespace + convention and do not currently include `/standard/`. - A release preview is a read-only `GET`. A release commit is a `POST`. ## P0 — Model draft revision selectors separately from released member pins @@ -93,9 +93,8 @@ interface MissingPrimaryRevisions { ``` - HTTP `400` means the current request selected a revision that does not - exist. Candidate add/version-update and direct standard member replacement - flows should keep the dialog open, identify the missing selections, and let - the operator correct them. + exist. Candidate add/version-update flows should keep the dialog open, + identify the missing selections, and let the operator correct them. - HTTP `409` means an existing draft or snapshot contains a dangling primary reference. Snapshot retrieval, release preview/commit, cloning, virtual materialization/quarantine promotion, and bundle export can return this @@ -109,13 +108,56 @@ Done when: - The release-track connector exposes `missing_references` on `400` and `409` responses instead of flattening the response to a generic message. -- Candidate and direct-content forms keep their input state after a `400` and - highlight the missing revisions. +- Candidate forms keep their input state after a `400` and highlight the + missing revisions. - Snapshot, release, clone, virtual-materialization, and export views present an actionable integrity error for `409`. - Tests cover multiple missing references and prove no partial snapshot or bundle is rendered. +### [ ] Explain snapshot-graph protection conflicts on object edits and deletes + +Release-track snapshots now freeze the exact relationships and secondary +objects needed to reproduce their bundle graph. If an object revision is a +protected dependency of any active or linked-pending snapshot manifest, an +in-place `PUT`, exact-revision `DELETE`, or full-lineage `DELETE` that would +invalidate that graph returns +`409 Conflict`: + +```ts +{ + message: string; + details?: string; + snapshot_graph_pins: Array<{ + track_id: string; + snapshot_modified: string; + kind: 'root' | 'relationship' | 'secondary' | 'supporting' | 'link_target'; + tier?: 'members' | 'staged' | 'candidates' | 'quarantine'; + }>; +} +``` + +This can occur from ordinary object-management screens, not only from the +release-track UI. Present it as a versioning constraint: the operator should +create a new object revision, or remove the draft snapshots that no longer +need the old revision. Do not offer a force-delete path; administrator +authorization does not bypass graph integrity. + +A standalone standard-track candidate or staged root remains editable through +the existing in-place review workflow. It becomes graph-protected only when +the same revision is also needed as a frozen dependency. Description-only +relationship corrections are allowed because older snapshots retain the +relationship payload captured in their manifests; relationship source, +target, and type changes are rejected as graph changes. + +Done when: + +- Shared object edit/delete error handling recognizes + `snapshot_graph_pins`. +- The message identifies the affected release track(s) and recommends a new + revision instead of a blind retry. +- The UI does not expose a force-delete action for graph-protected revisions. + ### [ ] Handle persisted mutations whose membership reconciliation failed A release-track mutation can persist its snapshot before a downstream object @@ -145,23 +187,71 @@ Done when: ## P0 — Align the Angular connector with the current routes -### [ ] Add administrator confirmation for destructive release-track actions +### [x] Remove direct snapshot mutation controls and client methods + +Persisted snapshot history is now immutable. The backend no longer exposes: + +```text +POST /api/release-tracks/:id/contents +POST /api/release-tracks/:id/snapshots/:modified/contents +POST /api/release-tracks/:id/snapshots/:modified/meta +``` -Full track deletion and both direct standard-track member replacement routes -are administrator-only. They now require the query parameter +Remove the corresponding connector methods, payload types, dialogs, buttons, +and tests. Standard-track content should move through candidates, staged, and +release. Virtual content should move through composition materialization and +quarantine resolution. Metadata can be changed only from the latest snapshot +via `POST /api/release-tracks/:id/meta`, which creates a new draft. + +Do not replace removed historical-edit actions with hidden calls or local +state edits. If an operator wants a different result, they should correct the +latest draft, delete it while deletion is still allowed, or create a newer +draft. + +Done when: + +- No Angular code calls or models any of the three removed routes. +- Snapshot history views are read-only except for supported release, clone, + and latest-draft deletion actions. +- Standard and virtual editors direct users to their respective supported + workflows. + +Completed 2026-07-30: the Angular connector methods, payload type, and +regression fixtures were removed. No component or menu called these methods, +so no UI control needed to be migrated. + +### [ ] Offer deletion only for the latest untagged draft + +`DELETE /api/release-tracks/:id/snapshots/:modified` is a narrow “undo latest +draft” operation. The server accepts it only when the selected snapshot is both +untagged and currently latest. Tagged releases and older drafts return `409` +because they are immutable history. + +In snapshot history, show Delete only on the latest item when `version == null`. +After a successful delete, refresh both the latest snapshot and the history; +the preceding snapshot becomes current. If a `409` occurs because another +operation created a newer draft, refresh instead of retrying the stale delete. + +Done when: + +- Tagged and historical rows never offer Delete. +- The confirmation explains that the track will revert to the preceding + snapshot. +- A stale `409` refreshes the view and preserves history. + +### [ ] Add administrator confirmation for full track deletion + +Full track deletion is administrator-only and requires the query parameter `confirm_track_id` to exactly equal the `:id` path parameter: ```text DELETE /api/release-tracks/:id?confirm_track_id=:id -POST /api/release-tracks/:id/contents?confirm_track_id=:id -POST /api/release-tracks/:id/snapshots/:modified/contents?confirm_track_id=:id ``` -Do not expose these actions to editors or team leads. Before sending a request, -show the track name and ID, explain that direct replacement bypasses the normal -candidate/staged workflow or that deletion removes all history, and require an -explicit confirmation interaction. A missing or stale ID returns `400`; a -non-administrator returns `401`. +Do not expose this action to editors or team leads. Before sending the request, +show the track name and ID, explain that deletion removes all history, and +require an explicit confirmation interaction. A missing or stale ID returns +`400`; a non-administrator returns `401`. Done when: @@ -427,21 +517,11 @@ Done when: - A `409` from preview/release explains that materialization is required instead of being swallowed as a null preview. -### [ ] Keep standard-only mutations out of virtual-track controls - -Direct contents replacement is now explicitly rejected for virtual tracks: - -```text -POST /api/release-tracks/:id/contents -POST /api/release-tracks/:id/snapshots/:modified/contents -``` +### [ ] Keep standard workflow controls out of virtual tracks Virtual membership has one authority: composition materialization followed by -optional quarantine resolution. Both contents endpoints return `400 Bad -Request` for a virtual track. - -Hide direct member replacement, candidate, and staged controls when -`type === 'virtual'`. Keep them available for standard tracks on their current +optional quarantine resolution. Hide candidate and staged controls when +`type === 'virtual'`; keep them available for standard tracks on their current routes. Done when: @@ -661,9 +741,9 @@ not mean “resolve every member to its latest object revision.” If the track not acquired another snapshot, repeated `/snapshots/latest` calls identify the same primary revision set. -Bundle downloads remain a documented exception: the backend appends secondary -relationships and supporting objects at request time, so the complete -`format=bundle` graph is not guaranteed to reproduce an earlier download. +Bundle downloads now replay the relationship/secondary graph captured when +the snapshot was created. The generated bundle-envelope ID may change, but +the object graph for a materialized virtual snapshot is stable. Done when: @@ -673,8 +753,8 @@ Done when: `object_modified`. - Tests prove that advancing a component after materialization does not change the displayed virtual member revision. -- User-facing export guidance does not promise byte-identical bundle - regeneration. +- User-facing export guidance distinguishes a stable snapshot object graph + from the intentionally variable bundle-envelope UUID. ## P1 — Submit mode-correct virtual snapshot schedules @@ -880,9 +960,14 @@ Minimum regression coverage: The following changes are useful context but should not create extra connector work: -- Snapshot bundle exports now include valid secondary relationships - dynamically. Existing bundle download code receives a more complete bundle - without changing its request. +- Snapshot bundle exports now include bounded secondary objects and their + relationships from a frozen graph manifest. Existing bundle download code + receives a more complete and reproducible bundle without changing its + request. A standard draft tier explicitly stored as `"latest"` remains + dynamic until release. +- Snapshot responses include an opaque, server-controlled + `graph_manifest_id`. The SPA does not need to send, interpret, or persist + this field; tolerate it in response models and omit it from request bodies. - Release-track object back-references are reconciled when snapshots change. Frontend object refreshes will see the updated membership metadata without a new endpoint. diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 04d21c2a..1ad59401 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -7,23 +7,54 @@ This branch implements the prioritized findings in recommendation is kept as a separate conventional commit so the merge request can be reviewed or reverted item by item. +### Current implementation slice — Immutable snapshot history + +- [x] Remove direct latest and historical snapshot member-replacement + endpoints (`POST /:id/contents` and + `POST /:id/snapshots/:modified/contents`) from routes, controllers, + services, validation, and OpenAPI. +- [x] Remove historical metadata rewriting + (`POST /:id/snapshots/:modified/meta`); retain latest metadata updates, + which create a new draft snapshot. +- [x] Permit snapshot deletion only for the latest untagged draft and return a + typed `409 Conflict` for tagged or historical snapshots. +- [x] Replace test setup that depended on direct member replacement with + supported bootstrap/candidate/promotion/release workflows, and add + regressions for the removed endpoints and deletion boundary. +- [x] Update user/developer/admin documentation and the Angular, Python, and + Bruno clients so no downstream surface suggests that persisted snapshot + history can be rewritten. +- [x] Run focused server/client checks, then the complete server `npm test` + suite, and record the results. + +Verification (2026-07-30): + +- Server: focused release-track regression files, OpenAPI validation, lint, + and the complete `npm test` suite passed. +- Angular: connector regression tests, changed-file formatting/lint checks, + and the complete frontend test suite passed. +- `internalattack`: release-track regression tests, changed-file Ruff checks, + and the complete Python test suite passed. +- Bruno: the release-track collection no longer contains the removed mutation + requests, and its modified request files pass scoped whitespace validation. + ### P0.1 — Enforce release version uniqueness - [x] Add a unique partial index for tagged `version` strings in every dynamic - release-track snapshot collection. + release-track snapshot collection. - [x] Convert duplicate-version races into a typed `409 Conflict` that - identifies the track and requested version. + identifies the track and requested version. - [x] Add a regression that releases two distinct drafts concurrently with the - same version and proves exactly one succeeds. + same version and proves exactly one succeeds. - [x] Adopt the pre-release reset policy for collections created with the - former non-unique index. No shared deployment retains beta release-track - data, so this change deliberately does not establish a permanent - migration contract for local development state. + former non-unique index. No shared deployment retains beta release-track + data, so this change deliberately does not establish a permanent + migration contract for local development state. - [x] Update release-version documentation and run focused, middleware, and - lint verification. + lint verification. - [ ] Obtain one clean aggregate `npm test` run for the migration cleanup. Three - attempts exposed the repository's roaming cross-spec isolation failure; - every affected spec passed immediately in isolation. + attempts exposed the repository's roaming cross-spec isolation failure; + every affected spec passed immediately in isolation. Original implementation verification (2026-07-30): @@ -47,22 +78,23 @@ Pre-release migration cleanup verification (2026-07-30): ### P0.2 — Make primary release membership fail closed - [x] Add one shared batch hydrator that resolves dynamic selectors, validates - every exact `(object_ref, object_modified)` pair, and reports all missing - primary revisions without swallowing repository failures. -- [x] Reject nonexistent exact candidate pins, candidate pin updates, direct - member replacement, track cloning, and virtual materialization before - snapshot persistence. + every exact `(object_ref, object_modified)` pair, and reports all missing + primary revisions without swallowing repository failures. +- [x] Reject nonexistent exact candidate pins, candidate pin updates, the + then-supported direct member replacement requests, track cloning, and + virtual materialization before snapshot persistence. Direct replacement + was subsequently removed by the immutable-history slice. - [x] Revalidate existing and promoted members at release-preview and - release-commit boundaries; return a typed `409 Conflict` for corrupt stored - drafts. + release-commit boundaries; return a typed `409 Conflict` for corrupt stored + drafts. - [x] Abort bundle import before creating a track when any authoritative - primary object failed to import or cannot be hydrated. + primary object failed to import or cannot be hydrated. - [x] Abort bundle/workbench export when selected primary revisions cannot be - hydrated; return every missing reference instead of a partial result. + hydrated; return every missing reference instead of a partial result. - [x] Add ingress, partial-import, deleted-staged-revision, virtual - materialization, and incomplete-export regressions. + materialization, and incomplete-export regressions. - [x] Update user/developer documentation and run focused, lint, OpenAPI, and - complete-suite verification. + complete-suite verification. Verification result (2026-07-30): @@ -80,17 +112,17 @@ Verification result (2026-07-30): ### P0.3 — Make tagged-content immutability authoritative and durable - [x] Guard object revision update/delete and delete-all by querying tagged - snapshot membership, even when `workspace.release_tracks` is missing or - stale. + snapshot membership, even when `workspace.release_tracks` is missing or + stale. - [x] Make release-track backref reconciliation failures propagate to the - triggering request so a release is never reported as fully successful when - protection writes failed. + triggering request so a release is never reported as fully successful when + protection writes failed. - [x] Persist every reconciliation attempt and its terminal outcome so - process crashes and partial listener failures remain operator-visible. + process crashes and partial listener failures remain operator-visible. - [x] Provide an idempotent repair command for failed/pending reconciliation - records and a full-scan mode for legacy drift. + records and a full-scan mode for legacy drift. - [x] Add failure-injection, missing-backref, repair, and historical-release - regressions; update user/developer/admin documentation. + regressions; update user/developer/admin documentation. - [x] Run focused, lint, OpenAPI, and complete-suite verification. Verification result (2026-07-30): @@ -111,17 +143,21 @@ Verification result (2026-07-30): ### P0.4 — Correct destructive authorization and add durable audit records +This records the earlier beta contract. The immutable-history slice later +removed both member-replacement routes and their audit action types; durable +auditing now applies only to full-track deletion. + - [x] Require administrator authorization for full track deletion and both - direct member-replacement routes. + direct member-replacement routes. - [x] Require an exact `confirm_track_id` precondition on each destructive - request so stale or accidental UI actions fail before persistence. + request so stale or accidental UI actions fail before persistence. - [x] Persist a durable, actor-attributed audit event before each operation - and record completion or failure without hiding partial persistence. + and record completion or failure without hiding partial persistence. - [x] Add an authorization matrix and operator-facing audit documentation. - [x] Update OpenAPI, frontend tasks, and Bruno requests for the confirmation - contract. + contract. - [x] Add admin/editor, missing/mismatched confirmation, success/failure - audit, lint, OpenAPI, focused, and complete-suite verification. + audit, lint, OpenAPI, focused, and complete-suite verification. Verification result (2026-07-30): @@ -136,27 +172,119 @@ Verification result (2026-07-30): - [ ] P0.5 — Complete the Angular contract migration and end-to-end smoke gate. - [ ] P0.6 — Finish scheduled-materialization fencing, retry bounds, and - operator intervention. + operator intervention. - [ ] P0.7 — Establish and enforce a safe storage operating envelope. - [ ] P0.8 — Harden deployment, database readiness, backup/restore, rollback, - and post-deploy verification. + and post-deploy verification. - [ ] Address P1 recommendations in documented criticality order. +## Current implementation slice — Deterministic snapshot bundle graphs + +This slice replaces export-time relationship and secondary-object discovery +with a frozen graph manifest for every persisted release-track snapshot. The +manifest is authoritative for bundle replay and for protecting every exact +revision on which that bundle depends. + +### Shared graph resolution + +- [x] Extract the bounded ATT&CK graph-selection rules from the mutable + `stix-bundles-service` singleton into a request-local resolver. +- [x] Preserve the existing one-hop and named special-case behavior for + `detects`, `attributed-to`, `revoked-by`, detection strategies, analytics, + and required supporting objects without introducing unrestricted graph + traversal. +- [x] Make the legacy/ephemeral exporter and release-track snapshot capture + use thin adapters around the same resolver. +- [x] Add parity and concurrent-request regression coverage before changing + release-track persistence. + +### Revision-pinned relationships + +- [x] Store server-controlled exact source and target revision pins under + `workspace.relationship_endpoints`; do not add non-ADM fields to emitted + STIX payloads. +- [x] Resolve endpoint revisions when relationships are created, including + bundle-import and automated relationship-creation paths, and fail closed + when an exact endpoint cannot be established. +- [x] Create new SRO revisions when a referenced SDO advances instead of + mutating an existing `(stix.id, stix.modified)` revision. +- [x] Reject in-place source, target, and relationship-type changes. + Description-only corrections remain allowed because manifests freeze the + relationship STIX payload used by existing snapshots. + +### Snapshot graph manifests + +- [x] Persist an exact, tier-aware manifest for every newly created standard + and virtual snapshot. Include primary roots, relationships, secondary + objects, identities, marking definitions, and LinkById render dependencies. +- [x] Store manifest entries in one indexed collection so exact revision + hydration and mutation-protection checks do not scan dynamic snapshot + collections. +- [x] Make snapshot capture fail closed and concurrency-safe. Complete + manifests are linked before activation; linked pending manifests remain + replayable and self-activate after an interrupted write. +- [x] Replace a standard draft's manifest from the resolved release plan in + the same conditional tag update, so staged `"latest"` selectors become + exact released members without creating a second snapshot timestamp. +- [x] Replay `format=bundle` entirely from the frozen manifest, with no live + relationship, secondary, supporting-object, or LinkById discovery. The + deliberate exception is an explicitly included standard draft tier whose + stored selector is `"latest"`; that tier remains dynamic until release. + +### Mutation and deletion protection + +- [x] Centralize exact graph-pin checks in the shared object service layer. +- [x] Return `409 Conflict` when a PUT or exact-revision delete would mutate + an emitted revision referenced by any active or pending manifest. +- [x] Reject lineage deletion when any revision in the lineage is manifest + referenced; administrator authorization must not bypass graph integrity. +- [x] Remove protection entries when their owning snapshot or track is + deleted, with idempotent reconciliation for interrupted cleanup. + +### Migration, documentation, and verification + +- [x] Add a dry-run-capable, idempotent migration that resolves endpoint pins + for the latest revision of each relationship directly from the underlying + collections. Do not write through `view.relationships.latest`. +- [x] Backfill existing snapshots with manifests reconstructed from the graph + visible at migration time and label them as baseline reconstructions rather + than historically exact captures. +- [x] Add regression coverage for newer SDO/SRO revisions, relationship + deprecation, missing dependencies, PUT/delete guards, standard and virtual + snapshots, query-tier filtering, migration reruns, and concurrent capture. +- [x] Update OpenAPI error contracts, user/developer/admin documentation, + frontend guidance, `internalattack`, and Bruno where the observable contract + changes. No request route or parameter changed, so generated clients and + Bruno request definitions require no transport change. +- [x] Run focused specs while iterating, then lint, OpenAPI validation, and the + complete `npm test` suite. +- [x] Propose conventional commits split by independently reviewable + architectural slice; do not commit until requested. + +Verification result (2026-07-30): + +- The focused deterministic-graph group passes all 101 cases; the final + compatibility group for relationship pagination, reports, and collection + imports passes all 27 cases. +- Lint, formatting, OpenAPI validation, and diff checks pass. +- The required clean full suite passes: OpenAPI 2, config 21, API 971, + middleware 29, and scheduler 10. + ## Current implementation slice — Scheduler regression and virtual schedule coverage - [x] Repair the legacy collection-index scheduler spec so it imports the - refactored `sync-collection-indexes-task` module without auto-registering - background jobs during the test. + refactored `sync-collection-indexes-task` module without auto-registering + background jobs during the test. - [x] Add virtual-track coverage proving reconciliation registers scheduled - cron jobs in UTC and removes jobs for tracks that no longer exist. + cron jobs in UTC and removes jobs for tracks that no longer exist. - [x] Add date-schedule boundary coverage for multiple due dates and future - dates. + dates. - [x] Add crash-window recovery coverage for a scheduled virtual snapshot that - was persisted before its occurrence ledger reached `completed`. + was persisted before its occurrence ledger reached `completed`. - [x] Add stale-claim recovery coverage and document any remaining - multi-process lease/fencing limitation. + multi-process lease/fencing limitation. - [x] Run the legacy scheduler spec, the virtual scheduler spec, the aggregate - scheduler suite, lint, and the complete `npm test` suite. + scheduler suite, lint, and the complete `npm test` suite. - [x] Record the coverage conclusion and propose a conventional commit message. Coverage conclusion (2026-07-30): @@ -183,36 +311,36 @@ Verification result (2026-07-30): ### Remaining scheduled-materialization hardening - [ ] Add an owner token (fencing token) to occurrence claims, make terminal - updates conditional on the active token, and renew leases for work that may - exceed the claim duration. Add a true multi-worker regression proving that - an expired worker cannot overwrite the succeeding worker's result. + updates conditional on the active token, and renew leases for work that may + exceed the claim duration. Add a true multi-worker regression proving that + an expired worker cannot overwrite the succeeding worker's result. - [ ] Decide and document an operator policy for permanent failures. If - indefinite one-minute retries are not acceptable, add bounded exponential - backoff plus a terminal/dead-letter state and operator-visible recovery - controls. + indefinite one-minute retries are not acceptable, add bounded exponential + backoff plus a terminal/dead-letter state and operator-visible recovery + controls. ## Current implementation slice — Deterministic standard releases - [x] Preserve `modified: "latest"` and omitted candidate selectors as dynamic - references through the candidate and staged tiers; preserve explicit - timestamps as exact revision pins. + references through the candidate and staged tiers; preserve explicit + timestamps as exact revision pins. - [x] Resolve every dynamic staged reference to the actual latest - `stix.modified` timestamp during standard release planning, before conflict - detection, preview rendering, or commit. + `stix.modified` timestamp during standard release planning, before conflict + detection, preview rendering, or commit. - [x] Ensure tagged members contain exact revisions only and that preview and - commit use the same release-planning rules. + commit use the same release-planning rules. - [x] Make dynamic candidate/staged references safe in tier comparison, - Workbench enrichment, bundle rendering, back-reference reconciliation, and - member-sync paths. + Workbench enrichment, bundle rendering, back-reference reconciliation, and + member-sync paths. - [x] Add regression coverage for dynamic and explicit candidate promotion, - release-time resolution after a newer revision is created, historical - release targeting, conflict handling, and member immutability. + release-time resolution after a newer revision is created, historical + release targeting, conflict handling, and member immutability. - [x] Update OpenAPI, user/developer documentation, frontend guidance, - `internalattack`, and Bruno as required by the corrected contract. + `internalattack`, and Bruno as required by the corrected contract. - [x] Run focused tests, lint, OpenAPI validation, and the complete `npm test` - suite. + suite. - [x] Apply logic review, inspect the final diff, and propose conventional - commit messages. + commit messages. Verification result (2026-07-30): @@ -240,78 +368,78 @@ completion backlog. ### P1 — Composition validation and deterministic resolution - [x] Make request validation strict so misspelled keys such as - `filters.domain` return 400 instead of silently disabling filtering. + `filters.domain` return 400 instead of silently disabling filtering. - [x] Validate component selectors according to `resolution_strategy`: - `specific_version` requires `version` and rejects `snapshot`; - `specific_snapshot` requires `snapshot` and rejects `version`; - `latest_tagged` rejects both selector fields. - [x] Make `priority` consistently required in Zod, Mongoose, OpenAPI, docs, - and examples; reject duplicate priorities at the request boundary. + and examples; reject duplicate priorities at the request boundary. - [x] Validate component existence, standard-track type, duplicate track IDs, - and duplicate priorities when a virtual track is initially created, not only - when composition is later updated or materialized. + and duplicate priorities when a virtual track is initially created, not only + when composition is later updated or materialized. - [x] Validate `snapshot_schedule` by mode: - `manual` rejects `cron` and `dates`; - `cron` requires `cron` and rejects `dates`; - `dates` requires at least one date and rejects `cron`. - [x] Constrain or document accepted `filters.object_types` values and add - direct regression coverage for exact-revision filtering. + direct regression coverage for exact-revision filtering. ### P1 — Deduplication correctness - [x] Treat the same exact object revision contributed by multiple components - as one duplicate, not a conflicting revision. + as one duplicate, not a conflicting revision. - [x] Ensure the `quarantine` strategy only quarantines genuinely different - revisions of the same object. + revisions of the same object. - [x] Attribute each surviving revision to one deterministic component so - `objects_contributed` totals cannot exceed `summary.total_objects`. + `objects_contributed` totals cannot exceed `summary.total_objects`. - [x] Add dedicated tests for all four strategies: - `prioritize_latest_object`, `prioritize_latest_snapshot`, - `prioritize_higher_priority`, and `quarantine`. + `prioritize_latest_object`, `prioritize_latest_snapshot`, + `prioritize_higher_priority`, and `quarantine`. ### P1 — Release provenance - [x] Populate virtual release `version_history[].component_versions` from the - materialized snapshot's immutable `composition_resolution`. + materialized snapshot's immutable `composition_resolution`. - [x] Define and test the provenance shape in Mongoose, OpenAPI, and user and - developer documentation. + developer documentation. ### P2 — Scheduled materialization - [x] Connect virtual `snapshot_schedule` metadata to the existing task - scheduler. This is required for virtual-track completion, not an optional - future enhancement. + scheduler. This is required for virtual-track completion, not an optional + future enhancement. - [x] Implement `cron` execution so each matching schedule occurrence - materializes a new virtual draft through the same lifecycle and validation - used by `POST /api/release-tracks/:id/virtual/snapshots/create`. + materializes a new virtual draft through the same lifecycle and validation + used by `POST /api/release-tracks/:id/virtual/snapshots/create`. - [x] Implement `dates` execution so every configured timestamp materializes - exactly one virtual draft, including deterministic handling for restart - recovery, missed timestamps, and duplicate-delivery prevention. + exactly one virtual draft, including deterministic handling for restart + recovery, missed timestamps, and duplicate-delivery prevention. - [x] Preserve `manual` semantics: store no executable schedule and create - drafts only through the explicit virtual snapshot-creation endpoint. + drafts only through the explicit virtual snapshot-creation endpoint. - [x] Define failure behavior when a component has no matching tagged - snapshot, including automation-run audit records and retry policy. + snapshot, including automation-run audit records and retry policy. - [x] Add scheduler integration tests for both `cron` and `dates`, including - successful execution, restart recovery, idempotency, component-resolution - failure, and retry behavior. + successful execution, restart recovery, idempotency, component-resolution + failure, and retry behavior. - [x] Add operational documentation covering scheduler activation, UTC - interpretation, observability, failures, and retries. + interpretation, observability, failures, and retries. ### Current implementation slice — Scheduled virtual materialization - [x] Add a scheduler reconciliation task for persisted virtual-track - `cron` and `dates` schedules while preserving explicit-only `manual` mode. + `cron` and `dates` schedules while preserving explicit-only `manual` mode. - [x] Persist schedule occurrences and claim them atomically so multiple - scheduler instances cannot concurrently process the same occurrence. + scheduler instances cannot concurrently process the same occurrence. - [x] Make snapshot persistence idempotent by recording the scheduled - occurrence on the resulting virtual draft. + occurrence on the resulting virtual draft. - [x] Recover missed `dates` occurrences and failed `cron` or `dates` - occurrences during reconciliation. + occurrences during reconciliation. - [x] Record every materialization attempt in the automation-run audit trail. - [x] Add scheduler integration coverage for success, restart recovery, - duplicate delivery, component failure, and retry. + duplicate delivery, component failure, and retry. - [x] Update OpenAPI, user/developer/operations documentation, frontend - guidance, and Bruno. + guidance, and Bruno. - [x] Run focused scheduler tests, lint, and the complete `npm test` suite. - [x] Review the final diff and propose conventional commit messages. @@ -346,40 +474,40 @@ Verification result (2026-07-29): ### P2 — Contract decisions - [x] Virtual tracks cannot compose virtual tracks. Components must be - standard tracks; revisit nesting only if a concrete future use case requires - it. + standard tracks; revisit nesting only if a concrete future use case requires + it. - [x] Do not implement the documented native-members/hybrid model. Virtual - tracks are purely compositional; content that is not already represented - belongs in a dedicated standard component track. + tracks are purely compositional; content that is not already represented + belongs in a dedicated standard component track. - [x] Do not implement `resolve=true` or `resolved_content`. Virtual - composition is resolved eagerly into exact object revisions when a draft is - materialized; retrieval must never re-resolve a persisted snapshot. + composition is resolved eagerly into exact object revisions when a draft is + materialized; retrieval must never re-resolve a persisted snapshot. - [x] Do not implement caching or component-release notifications without - measured scale or an approved operator workflow. Persisted snapshots already - avoid composition recomputation, and no notification recipient, channel, or - expected action has been defined. + measured scale or an approved operator workflow. Persisted snapshots already + avoid composition recomputation, and no notification recipient, channel, or + expected action has been defined. ### Current implementation slice — Deterministic virtual membership - [x] Resolve the `latest` request shorthand to the actual latest - `stix.modified` value before standard-track contents are persisted. + `stix.modified` value before standard-track contents are persisted. - [x] Defensively lock any unresolved component member to an exact revision - during virtual materialization, while preserving exact revisions already - frozen into tagged component snapshots. + during virtual materialization, while preserving exact revisions already + frozen into tagged component snapshots. - [x] Add regression coverage proving that component `track_latest` behavior - cannot move a materialized virtual member and repeated snapshot retrieval - returns the same exact revision set. + cannot move a materialized virtual member and repeated snapshot retrieval + returns the same exact revision set. - [x] Remove `resolve=true` and `resolved_content` from the documented - retrieval contract. -- [x] Clearly document that persisted primary member revisions are - deterministic while bundle-time secondary-object and relationship - expansion is not. + retrieval contract. +- [x] Initially document the distinction between deterministic primary + membership and the then-dynamic bundle graph; the later deterministic graph + manifest slice below supersedes that accepted limitation. - [x] Update OpenAPI, frontend guidance, and Bruno where the clarified - contract affects consumers. + contract affects consumers. - [x] Run focused tests, lint, OpenAPI validation, and the complete `npm test` - suite. + suite. - [x] Apply logic review, inspect the final diff, and propose conventional - commit messages. + commit messages. Verification result (2026-07-29): @@ -411,38 +539,38 @@ Verification result (2026-07-29): ```text docs(release-tracks): clarify snapshot determinism - Document exact virtual member pins and the bundle-time secondary-content - consistency boundary in the Bruno collection. + Document exact virtual member pins and the snapshot graph consistency + boundary in the Bruno collection. ``` ### Future architecture — Deterministic bundle graphs -- [ ] Design version-controlled STIX Relationship Objects whose source and - target references identify exact `(object_id, object_modified)` revisions - rather than an entire STIX object provenance chain. -- [ ] Evaluate cloning every affected SRO when a new SDO revision is created, - including atomicity, fan-out, concurrency, migration, and rollback behavior. -- [ ] Measure the resulting database-storage amplification and query/index - costs before approving implementation. -- [ ] Define and persist an export manifest that pins every secondary object, - supporting object, and relationship revision required to reproduce a bundle. -- [ ] Until that architecture is approved and implemented, preserve and - prominently document the accepted constraint that `format=bundle` output is - not graph- or byte-level deterministic. +- [x] Design version-controlled STIX Relationship Objects whose source and + target references identify exact `(object_id, object_modified)` revisions + rather than an entire STIX object provenance chain. +- [x] Evaluate cloning every affected SRO when a new SDO revision is created, + including atomicity, fan-out, concurrency, migration, and rollback behavior. +- [x] Avoid cloning SROs per snapshot by recording exact endpoint metadata on + each SRO revision and storing compact manifest references plus a frozen SRO + payload where description-only PUT compatibility requires it. +- [x] Define and persist an export manifest that pins every secondary object, + supporting object, and relationship revision required to reproduce a bundle. +- [x] Document the resulting guarantee: the emitted object graph is + deterministic, while the generated bundle-envelope UUID is not byte-stable. ### Current implementation slice — Pure standard-track composition - [x] Make standard component tracks a positive service-layer requirement, - preserving rejection during both virtual-track creation and composition - replacement. + preserving rejection during both virtual-track creation and composition + replacement. - [x] Reject unsupported top-level creation properties such as - `native_members` instead of silently stripping them. + `native_members` instead of silently stripping them. - [x] Add regression coverage for virtual-track nesting on both creation and - composition update, and for attempted native-member creation. + composition update, and for attempted native-member creation. - [x] Remove nesting and hybrid/native-member claims from OpenAPI, user and - developer documentation, frontend guidance, and Bruno. + developer documentation, frontend guidance, and Bruno. - [x] Run the focused virtual-composition spec, lint, and complete `npm test` - suite. + suite. - [x] Review the final diff and propose conventional commit messages. Verification result (2026-07-29): @@ -478,46 +606,46 @@ Verification result (2026-07-29): ### Documentation corrections - [ ] Replace `stix.type = "virtual"` with the top-level snapshot - `type: "virtual"`. + `type: "virtual"`. - [ ] Remove the nonexistent snapshot-level `snapshot_id`; retain - `version_history[].snapshot_id`. + `version_history[].snapshot_id`. - [ ] Correct response envelopes and the virtual-create response example. - [ ] Align `composition_resolution` examples with fields actually generated, - or implement the documented `by_type`, `by_tier`, and native statistics. + or implement the documented `by_type`, `by_tier`, and native statistics. - [ ] Align documented error envelopes with centralized error-handler output. - [x] Include required `priority` values in every composition example. - [x] Clearly distinguish configured composition from a materialized draft and - document scheduler activation, timing, recovery, and retry behavior. + document scheduler activation, timing, recovery, and retry behavior. ### Verified complete - [x] Composition changes invalidate inherited materialized contents and - require explicit rematerialization before release. + require explicit rematerialization before release. - [x] Generic contents replacement rejects virtual tracks. - [x] Exact-revision quarantine resolution is available at - `POST /api/release-tracks/:id/virtual/quarantine/promote`. + `POST /api/release-tracks/:id/virtual/quarantine/promote`. - [x] `filters.domains` hydrates and evaluates exact pinned revisions. - [x] Public domain names and STIX `*-attack` names are normalized. - [x] Multiple domain values are supported. - [x] Objects without domain metadata are excluded when a domain filter is set. - [x] Primary Enterprise, ICS, and Mobile matrices use their ATT&CK external ID - as the established domain fallback. + as the established domain fallback. - [x] Virtual tracks resolve only tagged snapshots and consume only component - `members`. + `members`. - [x] Virtual tracks maintain independent draft/release history and use the - shared snapshot retrieval and release endpoints after materialization. + shared snapshot retrieval and release endpoints after materialization. ### Current implementation slice — Strict composition contracts - [x] Add API regression coverage for unknown composition/filter keys on both - virtual-track creation and composition update. + virtual-track creation and composition update. - [x] Require the selector appropriate to each `resolution_strategy` and - reject selectors that do not apply to that strategy. + reject selectors that do not apply to that strategy. - [x] Make the composition, component, filter, and deduplication request - objects strict without changing persisted response shapes. + objects strict without changing persisted response shapes. - [x] Update OpenAPI, user/developer documentation, and Bruno examples. - [x] Run the focused regression spec, then lint and the complete `npm test` - suite. + suite. - [x] Review the final diff and propose a conventional commit message. Verification result (2026-07-29): @@ -542,15 +670,15 @@ Verification result (2026-07-29): ### Current implementation slice — Component identity and priority validation - [x] Add creation and composition-update regression coverage for required - priorities, duplicate priorities, and duplicate component track IDs. + priorities, duplicate priorities, and duplicate component track IDs. - [x] Reject missing component tracks and virtual component tracks before an - initial virtual track is persisted. + initial virtual track is persisted. - [x] Make component priority required and non-negative across Zod, Mongoose, - OpenAPI, user/developer documentation, and Bruno examples. + OpenAPI, user/developer documentation, and Bruno examples. - [x] Keep service-layer component validation as a defense for non-HTTP - callers while moving deterministic duplicates to request validation. + callers while moving deterministic duplicates to request validation. - [x] Run the focused regression specs, then lint and the complete `npm test` - suite. + suite. - [x] Review the final diff and propose a conventional commit message. Verification result (2026-07-29): @@ -585,17 +713,17 @@ Verification result (2026-07-29): ### Current implementation slice — Snapshot schedule contracts - [x] Add creation regressions for valid and invalid `manual`, `cron`, and - `dates` schedule payloads. + `dates` schedule payloads. - [x] Enforce a strict mode-discriminated request contract: - `manual` accepts only `mode`; - `cron` requires `cron` and rejects `dates`; - `dates` requires at least one date and rejects `cron`. - [x] Reject `snapshot_schedule` on standard-track creation instead of silently - dropping it. + dropping it. - [x] Repeat schedule invariants at the service and Mongoose boundaries for - non-HTTP callers. + non-HTTP callers. - [x] Align OpenAPI, user/developer documentation, frontend guidance, the - `internalattack` test fixture, and Bruno. + `internalattack` test fixture, and Bruno. - [x] Run focused regression specs, lint, and the complete `npm test` suite. - [x] Review the final diff and propose conventional commit messages. @@ -638,16 +766,16 @@ Verification result (2026-07-29): ### Current implementation slice — Object-type filter contracts - [x] Define `filters.object_types` against the canonical Workbench STIX type - vocabulary instead of accepting arbitrary strings. + vocabulary instead of accepting arbitrary strings. - [x] Reject empty arrays, duplicate values, malformed values, and unsupported - object types on both virtual-track creation and composition update. + object types on both virtual-track creation and composition update. - [x] Repeat the accepted-value constraint at the Mongoose persistence - boundary. + boundary. - [x] Add direct materialization coverage proving that object-type filtering - preserves the exact revision pinned by the tagged component snapshot rather - than resolving the latest database revision. + preserves the exact revision pinned by the tagged component snapshot rather + than resolving the latest database revision. - [x] Align OpenAPI, user/developer documentation, frontend guidance, and - Bruno; verify whether `internalattack` needs a typed client change. + Bruno; verify whether `internalattack` needs a typed client change. - [x] Run focused regression specs, lint, and the complete `npm test` suite. - [x] Review the final diff and propose conventional commit messages. @@ -686,23 +814,23 @@ Verification result (2026-07-29): ### Current implementation slice — Deterministic virtual deduplication - [x] Add materialization regressions for all four deduplication strategies - using both an exact revision shared by multiple components and genuinely - different revisions of the same STIX object. + using both an exact revision shared by multiple components and genuinely + different revisions of the same STIX object. - [x] Collapse repeated contributions of the same `(object_ref, - object_modified)` revision before applying conflict resolution. +object_modified)` revision before applying conflict resolution. - [x] Count an object contributed by multiple components once in - `duplicates_found`, but include it in `conflicts_resolved` only when multiple - distinct revisions remain after exact-revision collapse. + `duplicates_found`, but include it in `conflicts_resolved` only when multiple + distinct revisions remain after exact-revision collapse. - [x] Choose one deterministic source component for every surviving revision: - use the active strategy's ordering and use component priority as the stable - tie-breaker. + use the active strategy's ordering and use component priority as the stable + tie-breaker. - [x] Quarantine one entry per distinct conflicting revision and leave an - identical revision shared by multiple components in `members`. + identical revision shared by multiple components in `members`. - [x] Derive `objects_contributed` from explicit survivor attribution so its - component total equals `summary.total_objects`. + component total equals `summary.total_objects`. - [x] Align OpenAPI, user/developer documentation, frontend guidance, Bruno, - and `internalattack` if the clarified response semantics require downstream - changes. + and `internalattack` if the clarified response semantics require downstream + changes. - [x] Run focused regression specs, lint, and the complete `npm test` suite. - [x] Review the final diff and propose conventional commit messages. @@ -743,20 +871,20 @@ Verification result (2026-07-29): ### Current implementation slice — Virtual release provenance - [x] Add release preview and commit regressions proving that virtual - `version_history[].component_versions` comes from the selected draft's - immutable `composition_resolution`, even if a component is released again - before the virtual draft is tagged. + `version_history[].component_versions` comes from the selected draft's + immutable `composition_resolution`, even if a component is released again + before the virtual draft is tagged. - [x] Define `component_versions` as an optional object keyed by immutable - component track ID with tagged `MAJOR.MINOR` version values. + component track ID with tagged `MAJOR.MINOR` version values. - [x] Populate provenance only for virtual release history entries and leave - standard release history unchanged. + standard release history unchanged. - [x] Enforce the provenance value shape at the Mongoose persistence boundary - and describe it in OpenAPI. + and describe it in OpenAPI. - [x] Align user/developer documentation, frontend guidance, Bruno, and - `internalattack` if the response contract requires downstream changes. + `internalattack` if the response contract requires downstream changes. - [x] Run focused regression specs, lint, and the complete `npm test` suite. - [x] Apply logic and performance review checklists, inspect the final diff, - and propose conventional commit messages. + and propose conventional commit messages. Verification result (2026-07-29): @@ -800,19 +928,19 @@ Verification result (2026-07-29): - [x] Consolidate the virtual-track completion backlog into this section. - [x] Preserve completed implementation evidence in the dated records below. - [x] Move the downstream Angular handoff to - `docs/developer/FRONTEND_TODO.md`. + `docs/developer/FRONTEND_TODO.md`. - [x] Remove the superseded root-level tracker files. ## Document downstream frontend work - [x] Inventory the current release-track API contract and recent endpoint, - terminology, lifecycle, validation, and response-shape changes. + terminology, lifecycle, validation, and response-shape changes. - [x] Inspect the Angular release-track consumers so the handoff identifies - concrete downstream work instead of restating backend implementation notes. + concrete downstream work instead of restating backend implementation notes. - [x] Create `docs/developer/FRONTEND_TODO.md` with task-oriented guidance, - contextual explanations, and acceptance criteria. + contextual explanations, and acceptance criteria. - [x] Cross-check the handoff against OpenAPI, user/developer documentation, - Bruno, and the `internalattack` client. + Bruno, and the `internalattack` client. - [x] Review formatting and the final diff. Verification result (2026-07-29): @@ -833,13 +961,13 @@ Verification result (2026-07-29): ## Implement virtual quarantine resolution - [x] Add end-to-end regression coverage for exact-revision quarantine - promotion, snapshot immutability, back-reference reconciliation, validation, - and virtual-track type enforcement. + promotion, snapshot immutability, back-reference reconciliation, validation, + and virtual-track type enforcement. - [x] Add `POST /api/release-tracks/:id/virtual/quarantine/promote`. - [x] Promote the selected revision to members in a new draft and remove all - quarantined alternatives for the same object. + quarantined alternatives for the same object. - [x] Preserve the immutable composition-resolution record and historical - materialized snapshot. + materialized snapshot. - [x] Update OpenAPI, user/developer documentation, and Bruno. - [x] Run focused regression specs, then lint and the complete `npm test` suite. - [x] Review the final diff and propose a conventional commit message. @@ -868,9 +996,9 @@ Verification result (2026-07-29): ## Harden virtual materialization lifecycle - [x] Record the complete virtual-track audit in the dedicated virtual release - tracks section of this file. + tracks section of this file. - [x] Add regression coverage for stale composition state, unmaterialized - release attempts, and virtual use of standard contents endpoints. + release attempts, and virtual use of standard contents endpoints. - [x] Clear inherited materialized state when virtual composition changes. - [x] Require a materialized virtual draft for release preview and commit. - [x] Restrict generic contents replacement to standard tracks. @@ -896,15 +1024,15 @@ Verification result (2026-07-29): ## Consolidate virtual draft creation and shared release previews - [x] Move virtual-only composition and draft-creation operations under an - explicit `/virtual` capability namespace. + explicit `/virtual` capability namespace. - [x] Remove the standalone virtual snapshot-preview endpoint without an - alias. + alias. - [x] Enhance shared virtual release summaries to compare the persisted draft - with its preceding tagged release without recomputing composition. + with its preceding tagged release without recomputing composition. - [x] Add regression coverage for route removal, type enforcement, latest and - historical virtual previews, and release-preview non-persistence. + historical virtual previews, and release-preview non-persistence. - [x] Update OpenAPI, user/developer documentation, Bruno, and the - `internalattack` Python client. + `internalattack` Python client. - [x] Run focused regression specs, then the complete `npm test` suite. - [x] Review the final diff and propose a conventional commit message. @@ -923,32 +1051,32 @@ Verification result (2026-07-29): ## Bootstrap faster-release core, defense, and virtual tracks - [x] Reconcile the clarified ownership partition with the current release-track - and virtual-composition API. + and virtual-composition API. - [x] Add regression coverage for functional virtual domain filters and - relationship-complete snapshot bundle exports. + relationship-complete snapshot bundle exports. - [x] Implement virtual `filters.domains` using the established ATT&CK domain - inference rules. + inference rules. - [x] Reuse/extract existing bundle relationship logic so snapshot - `format=bundle` exports dynamically include valid secondary relationships. + `format=bundle` exports dynamically include valid secondary relationships. - [x] Inventory and report any additional release-track no-op placeholders. - [x] Update user/developer docs and OpenAPI for the effective contract change; - Bruno has no new or changed request parameter to mirror. + Bruno has no new or changed request parameter to mirror. - [x] Run focused release-track regression specs, then the complete `npm test` - suite. + suite. - [x] Scan all three ATT&CK v19.1 bundles and construct a disjoint exact-revision - partition for Enterprise Core, ICS Core, Mobile Core, and Defense. + partition for Enterprise Core, ICS Core, Mobile Core, and Defense. - [x] Assign the shared identity and marking definitions to Enterprise Core - using the representations supported by release-track snapshots. + using the representations supported by release-track snapshots. - [x] Preflight exact track names and refuse conflicting duplicate tracks. - [x] Create and verify the four v19.1-pinned standard tracks. - [x] Create and verify the three domain-filtered virtual track definitions. - [x] Verify that every in-scope v19.1 object is owned by exactly one standard - track and record intentional relationship/collection exclusions. CTI owns - `course-of-action`; ICS Core owns `x-mitre-asset`. + track and record intentional relationship/collection exclusions. CTI owns + `course-of-action`; ICS Core owns `x-mitre-asset`. - [x] Defer materializing virtual snapshots until the component standard tracks - have tagged releases; no release/tag action was authorized in this bootstrap. + have tagged releases; no release/tag action was authorized in this bootstrap. - [x] Review the final repository diff and propose a conventional commit - message. + message. Operational result (2026-07-28): @@ -982,11 +1110,11 @@ Operational result (2026-07-28): - [x] Scan the ATT&CK v19.1 ICS and Mobile bundles and report every object type. - [x] Preflight the production-mirroring Workbench API and existing tracks. - [x] Create the CTI standard track with the latest intrusion-set, malware, - tool, and campaign revisions as members. + tool, and campaign revisions as members. - [x] Verify the persisted CTI snapshot, object-type coverage, exact latest - revision pins, and counts. + revision pins, and counts. - [x] Record operational results and propose a conventional commit message for - the committable scratchpad update. + the committable scratchpad update. Operational result (2026-07-28): @@ -1002,66 +1130,66 @@ Operational result (2026-07-28): ## Harden release version selection - [x] Reject simultaneous `increment` and `version` selectors inside the - release planner, even when controller validation is bypassed. + release planner, even when controller validation is bypassed. - [x] Add regression coverage for planner-level mutual exclusivity. - [x] Make exact, incremental, default, and ambiguous selection behavior - explicit in OpenAPI, user/developer docs, and Bruno. + explicit in OpenAPI, user/developer docs, and Bruno. - [x] Run the focused release-track spec, lint, and complete `npm test` suite. - The focused release spec passes (10 tests), lint passes, and the complete - backend suite passes (OpenAPI: 2, config: 21, API: 907, middleware: 24). - Targeted frontend Prettier and ESLint pass; TypeScript remains blocked by - the checkout's existing Angular dependency-resolution and unrelated type - errors. + The focused release spec passes (10 tests), lint passes, and the complete + backend suite passes (OpenAPI: 2, config: 21, API: 907, middleware: 24). + Targeted frontend Prettier and ESLint pass; TypeScript remains blocked by + the checkout's existing Angular dependency-resolution and unrelated type + errors. - [x] Review the final diff and propose a conventional commit message. ## Release command and unified previews - [x] Replace bump routes and symbols with explicit release operations for - latest and historical snapshots. + latest and historical snapshots. - [x] Implement one pure release planner shared by summary, workbench, bundle, - and commit paths. + and commit paths. - [x] Remove `dry_run`, rename version `type` to `increment`, and reject - conflicting version-selection inputs. + conflicting version-selection inputs. - [x] Keep release targeting semantics explicit: `latest` resolves at request - time, while `:modified` pins a specific snapshot; no client precondition is - required. + time, while `:modified` pins a specific snapshot; no client precondition is + required. - [x] Add regression coverage for preview parity, non-persistence, conflicts, - formats, validation, historical releases, and removed bump routes. + formats, validation, historical releases, and removed bump routes. - [x] Update OpenAPI, user/developer documentation, Bruno, and frontend - consumers. + consumers. - [x] Run focused tests and frontend checks, then the complete `npm test` - backend suite. - Focused release-track suites pass (49 tests), and the affected backref suite - passes again in isolation (23 tests). The complete backend suite passes on - retry. Targeted frontend formatting and ESLint pass; frontend Vitest and - TypeScript startup remain blocked by the checkout's existing - ESM/dependency-resolution errors. + backend suite. + Focused release-track suites pass (49 tests), and the affected backref suite + passes again in isolation (23 tests). The complete backend suite passes on + retry. Targeted frontend formatting and ESLint pass; frontend Vitest and + TypeScript startup remain blocked by the checkout's existing + ESM/dependency-resolution errors. - [x] Review the final diff and propose a conventional commit message. ## Remove implicit latest-snapshot route - [x] Remove `GET /api/release-tracks/:id` while preserving track deletion. - [x] Make `/snapshots/latest` canonical across OpenAPI, tests, docs, Bruno, - and the frontend consumer. + and the frontend consumer. - [x] Add regression coverage proving the removed method returns 405. - [x] Run focused regression specs followed by the complete `npm test` suite. - The focused suites pass. The aggregate run reached 894 passing with three - unrelated documented roaming failures; all three affected specs pass - together in isolation (51 passing). + The focused suites pass. The aggregate run reached 894 passing with three + unrelated documented roaming failures; all three affected specs pass + together in isolation (51 passing). - [x] Review the final diff and propose a conventional commit message. ## Snapshot history collection endpoint - [x] Add `GET /api/release-tracks/:id/snapshots` with strict tagged filtering - and pagination, plus an explicit `/snapshots/latest` alias. + and pagination, plus an explicit `/snapshots/latest` alias. - [x] Return lightweight, type-oriented summaries: standard snapshots include - member/staged/candidate counts; virtual snapshots include member/quarantine - counts. + member/staged/candidate counts; virtual snapshots include member/quarantine + counts. - [x] Add regression coverage for defaults, filters, pagination, validation, - track types, and not-found behavior. + track types, and not-found behavior. - [x] Update OpenAPI, user/developer documentation, and Bruno requests. - [x] Run the focused regression spec followed by the complete `npm test` - suite. + suite. - [x] Review the final diff and propose a conventional commit message. ## Regression Tests @@ -1069,28 +1197,27 @@ Operational result (2026-07-28): - [ ] Implement regression tests - [x] **Investigate the recurring full-suite flake.** Two root causes found and fixed (2026-07-10) in `app/lib/database-in-memory.js`: - 1. *Port collision*: every spec file stopped and restarted the `mongodb-memory-server` instance, and a fresh mongod would intermittently fail with `Port already in use` — breaking that file's `before` hook (surfacing as `loginAnonymous` 404s) and cascading failures through the file. Fixed by reusing one mongod for all spec files in the process (`closeConnection` drops the database and disconnects but keeps the server running) plus `--exit` on the mocha scripts. - 2. *Vanishing unique indexes*: dropping the database between spec files also drops its indexes, and mongoose's per-model `init()` is memoized per process — so the `stix.id + stix.modified` unique index was intermittently missing for later files, letting duplicate-POST tests (and dependent count tests) fail in roaming pairs. Fixed by explicitly awaiting `createIndexes()` for all registered models after each reconnect. + 1. _Port collision_: every spec file stopped and restarted the `mongodb-memory-server` instance, and a fresh mongod would intermittently fail with `Port already in use` — breaking that file's `before` hook (surfacing as `loginAnonymous` 404s) and cascading failures through the file. Fixed by reusing one mongod for all spec files in the process (`closeConnection` drops the database and disconnects but keeps the server running) plus `--exit` on the mocha scripts. + 2. _Vanishing unique indexes_: dropping the database between spec files also drops its indexes, and mongoose's per-model `init()` is memoized per process — so the `stix.id + stix.modified` unique index was intermittently missing for later files, letting duplicate-POST tests (and dependent count tests) fail in roaming pairs. Fixed by explicitly awaiting `createIndexes()` for all registered models after each reconnect. Residual: rare (≈1 per run under heavy machine load) single-test failures of a different character (a count assertion, a 20s timeout in a pagination GET) still appear occasionally and pass in isolation — likely load-related; keep observing before chasing further. ## Release-track cross-tier revision uniqueness - [x] Read the release-track user and developer documentation and identify the - intended exact-revision invariant. + intended exact-revision invariant. - [x] Trace every standard/virtual tier ingress and transition path. - [x] Add regression coverage proving one `(stix.id, stix.modified)` revision - cannot occupy multiple tiers while different revisions of one ID can. + cannot occupy multiple tiers while different revisions of one ID can. - [x] Enforce the invariant for candidate adds, promotions, demotions, bulk - status transitions, release bumps, member sync, and quarantine workflows. + status transitions, release bumps, member sync, and quarantine workflows. - [x] Update user/developer documentation (and OpenAPI/Bruno only if the API - contract changes). + contract changes). - [x] Run focused specs and the complete `npm test` suite. The task-specific - and constituent suites pass; repeated aggregate runs each encountered one - unrelated roaming API failure that passed immediately in isolation. + and constituent suites pass; repeated aggregate runs each encountered one + unrelated roaming API failure that passed immediately in isolation. - [x] Review the final diff and propose a conventional commit message. - ## Snapshot Output Format **TASK Summary**: Implement support for the `bundle` output format for snapshots @@ -1117,8 +1244,8 @@ The following release-track snapshot retrieval endpoints support `include` and > [!Note] > The ephemeral bundle endpoint (`GET /api/release-tracks/ephemeral/{domain}`) supports `format`, but not tier `include`, because it does not read from a persisted release-track snapshot. Rather, it "blindly" includes all objects in the domain. - **Include Parameter** (controls which tiers are returned): + ``` GET /api/release-tracks/:id/snapshots/latest # Default: all tiers GET /api/release-tracks/:id/snapshots/latest?include=members # Members tier only @@ -1129,6 +1256,7 @@ GET /api/release-tracks/:id/snapshots/latest?include=all # All ti ``` **Format Parameter** (controls output format): + ``` GET /api/release-tracks/:id/snapshots/latest?format=workbench # Workbench snapshot with metadata (default) GET /api/release-tracks/:id/snapshots/latest?format=bundle # Standard STIX 2.1 bundle @@ -1136,6 +1264,7 @@ GET /api/release-tracks/:id/snapshots/latest?format=filesystemstore # Not im ``` **Combined Example:** + ``` GET /api/release-tracks/:id/snapshots/latest?include=all&format=workbench ``` @@ -1168,7 +1297,7 @@ Here is how each of the other query parameters should be handled/mapped to the n For release track retrieval requests that include the `format=bundle` query parameter, the following query parameters must be supported: - `include: ['candidate', 'staged']`: If specified, the value must be equal to an array of at least one value. The parameter acts as a filter, allowing users to specify whether release-track candidates and/or staged objects should be included in the bundle. If the `include` parameter is omitted, only members should be included. -- `state: ['work-in-progress', 'awaiting-review']`: If specified, the value must be equal to an array of at least one value. Notably, objects marked as `"reviewed"` are always included (by nature of all members being included —— all members are inherently "reviewed"), irrespective of this query parameter. The parameter acts as a union filter that logically combines with `include`. In other words, when `include` and `state` are both specified, `include` is applied first, then `state` is applied to the remaining `include`-filtered subset. (i.e., Of the candidates and/or staged objects that are ready to be included in the emitted bundle, only include the ones that are marked as "work-in-progress", "awaiting-review", or either). +- `state: ['work-in-progress', 'awaiting-review']`: If specified, the value must be equal to an array of at least one value. Notably, objects marked as `"reviewed"` are always included (by nature of all members being included —— all members are inherently "reviewed"), irrespective of this query parameter. The parameter acts as a union filter that logically combines with `include`. In other words, when `include` and `state` are both specified, `include` is applied first, then `state` is applied to the remaining `include`-filtered subset. (i.e., Of the candidates and/or staged objects that are ready to be included in the emitted bundle, only include the ones that are marked as "work-in-progress", "awaiting-review", or either). - `stixVersion` should be **preserved**. This parameter allows users to control which STIX version is used in the emitted bundle (`2.0` or `2.1`). It defaults to `2.1`. ### Fixing the /bump/preview endpoint @@ -1213,10 +1342,10 @@ For example: ```yaml workspace: - release_tracks: - - id: 'release-track--3a0e2537-1153-4b16-8ff5-1993f2d9cd7d' - phase: 'candidate' - status: 'work-in-progress' + release_tracks: + - id: 'release-track--3a0e2537-1153-4b16-8ff5-1993f2d9cd7d' + phase: 'candidate' + status: 'work-in-progress' stix: # ... ``` @@ -1228,20 +1357,20 @@ Object CRUD paths can mutate or destroy revisions that release tracks pin, witho - [x] **Reject revision re-keying on PUT.** `updateFull` merged body `stix.id`/`stix.modified` over the stored document, so a PUT could silently re-key a revision and strand any track pins. Now returns 400 when the body identity fields differ from the path parameters. Re-keying must go through POST (a new revision), which member sync captures. Tests: `app/tests/api/base-services/update-identity-guard.spec.js`. -- [x] **Capture in-place PUTs of pinned revisions.** Implemented 2026-07-13: `BaseService.updateFull` rejects (409, `MemberPinnedRevisionError`) when the revision is pinned in any track's `members` tier — released content is immutable in place; POST a new revision instead. `staged`/`candidates`-pinned revisions ride the `::updated` → revision-sync path and are marked with the server-assigned **`modified-in-place`** status (content changed with no revision history to diff — reviewers are told *that* something changed, not *what*; the marker is cleared via the review endpoint). Placement is centralized in the **workflow gate** (`app/lib/release-tracks/workflow-gate.js`): tier is decided against `candidacy_threshold`/`auto_promote` (`modified-in-place` ranks with `work-in-progress`), so permissive tracks keep in-place-edited staged entries staged while strict tracks demote them for re-review — and threshold-qualifying placements land directly in `staged` in a single snapshot (no more candidates bounce). Covers in-place deprecation (`x_mitre_deprecated` via PUT). The member-sync misfire (same-key duplicate cross-tier enrollment) is fixed by skipping enrollment of already-pinned revisions and skipping no-op snapshot clones. Future: an in-document changelog of in-place modifications would let the marker say *what* changed. Tests: `app/tests/api/release-tracks/release-tracks-change-capture.spec.js`. +- [x] **Capture in-place PUTs of pinned revisions.** Implemented 2026-07-13: `BaseService.updateFull` rejects (409, `MemberPinnedRevisionError`) when the revision is pinned in any track's `members` tier — released content is immutable in place; POST a new revision instead. `staged`/`candidates`-pinned revisions ride the `::updated` → revision-sync path and are marked with the server-assigned **`modified-in-place`** status (content changed with no revision history to diff — reviewers are told _that_ something changed, not _what_; the marker is cleared via the review endpoint). Placement is centralized in the **workflow gate** (`app/lib/release-tracks/workflow-gate.js`): tier is decided against `candidacy_threshold`/`auto_promote` (`modified-in-place` ranks with `work-in-progress`), so permissive tracks keep in-place-edited staged entries staged while strict tracks demote them for re-review — and threshold-qualifying placements land directly in `staged` in a single snapshot (no more candidates bounce). Covers in-place deprecation (`x_mitre_deprecated` via PUT). The member-sync misfire (same-key duplicate cross-tier enrollment) is fixed by skipping enrollment of already-pinned revisions and skipping no-op snapshot clones. Future: an in-document changelog of in-place modifications would let the marker say _what_ changed. Tests: `app/tests/api/release-tracks/release-tracks-change-capture.spec.js`. -- [x] **DELETE of tracked objects.** Implemented 2026-07-13 with a simplified decision: DELETE (single version or all versions) is *rejected* (409) when a revision is `members`-pinned, with guidance to retire the object via a new `x_mitre_deprecated` revision instead — members-pinned revisions are immutable and must never be deleted. (The earlier auto-convert-to-deprecation idea was dropped in favor of explicit rejection.) `candidates`/`staged`-pinned deletes remain allowed (the reconciler self-heals the dangling pin). Note: `CollectionsService` overrides `deleteVersionById`, so collections are not covered by the guard. Legacy delete controllers were migrated to the service-exception middleware (`next(err)`) so the 409 maps correctly. +- [x] **DELETE of tracked objects.** Implemented 2026-07-13 with a simplified decision: DELETE (single version or all versions) is _rejected_ (409) when a revision is `members`-pinned, with guidance to retire the object via a new `x_mitre_deprecated` revision instead — members-pinned revisions are immutable and must never be deleted. (The earlier auto-convert-to-deprecation idea was dropped in favor of explicit rejection.) `candidates`/`staged`-pinned deletes remain allowed unless the same revision is also a frozen graph dependency. The deterministic-graph slice extended the guard to secondary/supporting revisions and to `CollectionsService`'s custom exact, lineage, and `deleteAllContents` paths. Legacy delete controllers use the service-exception middleware (`next(err)`) so the 409 maps correctly. -- [x] **Revoke must reach member sync.** Implemented 2026-07-13: member sync subscribes to the 11 per-type `::revoked` events via a payload adapter (`handleStixObjectRevokedEvent`), so the revoked revision (`revoked: true`) is enrolled as a candidate in member tracks and candidate/staged pins move to it — treated exactly like any new revision. The revoke response's primary document carries the resulting backrefs. As decided, member sync is NOT extended to relationships: the revoked-by SRO and deprecation clones are pulled in dynamically at bundle export. +- [x] **Revoke must reach member sync.** Implemented 2026-07-13: member sync subscribes to the 11 per-type `::revoked` events via a payload adapter (`handleStixObjectRevokedEvent`), so the revoked revision (`revoked: true`) is enrolled as a candidate in member tracks and candidate/staged pins move to it — treated exactly like any new revision. The revoke response's primary document carries the resulting backrefs. Member sync is not extended to relationships; the bounded `revoked-by` edge is captured in each new snapshot's graph manifest and replayed from there. - [x] **Technique conversion should reach revision sync.** Implemented 2026-07-13 with the adapter approach (same pattern as `handleStixObjectRevokedEvent`): the `TECHNIQUE_CONVERTED_TO_SUBTECHNIQUE` / `SUBTECHNIQUE_CONVERTED_TO_TECHNIQUE` event payloads now carry the converted revision (`document`) and acting user, and member sync subscribes via `handleStixObjectConvertedEvent`, treating the conversion as a `new-revision` trigger through the workflow gate — candidate/staged pins move to the converted revision, member tracks enroll it as a candidate. The conversion responses refresh `workspace.release_tracks` after event processing (read-your-own-writes). Tests: conversion cases in `release-tracks-change-capture.spec.js` and the updated clone-strip test in `release-tracks-backrefs.spec.js`. ## Get Releases By Object -- [X] Implement `GET /api/release-tracks/objects/:objectRef/releases` so a - caller can retrieve every tagged snapshot whose `members` tier directly - contains the supplied STIX ID, across all object revisions and release - tracks. +- [x] Implement `GET /api/release-tracks/objects/:objectRef/releases` so a + caller can retrieve every tagged snapshot whose `members` tier directly + contains the supplied STIX ID, across all object revisions and release + tracks. ### Design @@ -1256,12 +1385,14 @@ Use `releaseTrackRegistry` as a compact global forward catalogue instead. Its single document per track gains a server-maintained `tagged_releases` array: ```javascript -tagged_releases: [{ - snapshot_modified: Date, // (track_id, snapshot_modified) identifies the snapshot - version: String, - tagged_at: Date, - tagged_by: String -}] +tagged_releases: [ + { + snapshot_modified: Date, // (track_id, snapshot_modified) identifies the snapshot + version: String, + tagged_at: Date, + tagged_by: String, + }, +]; ``` `tagged_release_count` is derived from `tagged_releases.length`. The actual @@ -1295,25 +1426,25 @@ therefore incur no index cost, and draft squashing does not affect the lookup. - Support `order=asc|desc` by `snapshot_modified`, plus `limit` and `offset`. - Return 200 with an empty result for a valid STIX ID with no tagged releases; malformed IDs return 400. -- Ascending order describes first *published/tagged* appearance, not the time +- Ascending order describes first _published/tagged_ appearance, not the time the object first entered an untagged draft. ### Checklist - [x] Registry schema/repository: add `tagged_releases`, reconciliation, and - derived count/latest-version maintenance. + derived count/latest-version maintenance. - [x] Dynamic snapshot schema/repository: add the tagged-member partial index - and a projected `findTaggedSnapshotsContainingObject` query. + and a projected `findTaggedSnapshotsContainingObject` query. - [x] Versioning: reconcile registry metadata after tagging and validate - version progression against track-wide tagged releases rather than a - potentially stale historical snapshot's embedded `version_history`. + version progression against track-wide tagged releases rather than a + potentially stale historical snapshot's embedded `version_history`. - [x] API: route, controller Zod validation, facade/service orchestration, - deterministic pagination, and OpenAPI contract. + deterministic pagination, and OpenAPI contract. - [x] Migration: backfill registry tagged-release refs and ensure the new index - on all existing dynamic track collections. + on all existing dynamic track collections. - [x] Regression tests: multiple tracks/releases/revisions, removal after an - earlier release, retroactive tag, virtual track, draft/non-member exclusion, - filtering/order/pagination, empty/malformed input, and backfill behavior. + earlier release, retroactive tag, virtual track, draft/non-member exclusion, + filtering/order/pagination, empty/malformed input, and backfill behavior. - [x] User/developer docs and Bruno request. - [ ] Verification: targeted spec first, then the complete `npm test` suite. - Targeted endpoint spec: 8 passing; release-track directory: 69 passing; @@ -1328,8 +1459,8 @@ therefore incur no index cost, and draft squashing does not affect the lookup. ## Snapshot Retention (Squash on Tag) - [ ] Implement draft-snapshot squashing so release cycles don't accumulate - unbounded snapshot storage. Design captured 2026-07-15; assessed as sound — - see analysis below. + unbounded snapshot storage. Design captured 2026-07-15; assessed as sound — + see analysis below. ### Why @@ -1356,7 +1487,7 @@ Mitigating facts (verified in code): docs, but no code change needed there. - `::created` events for brand-new objects are no-ops for member sync (`findTracksReferencingObject` only matches already-tracked `stix.id`s). - The O(N²) trap is bulk *re-imports/updates* of already-tracked objects + The O(N²) trap is bulk _re-imports/updates_ of already-tracked objects (e.g. re-importing a modified 20k-object bundle → 20k snapshots × MBs each). - `version_history` is embedded in and carried forward by every clone, so the release ledger survives squashing — tagged snapshots and the latest draft @@ -1414,36 +1545,36 @@ the latest snapshot. Like `git rebase --squash`ing the commits behind a tag. ### Alternatives considered -- *Amend-in-place* (member sync mutates the latest draft instead of cloning): +- _Amend-in-place_ (member sync mutates the latest draft instead of cloning): attacks the root cause but breaks the "every modification is a new snapshot" invariant, complicates concurrent reads, and silently degrades the audit trail for everyone. Rejected for now. -- *Delta/structural-sharing storage*: large refactor of the snapshot store; +- _Delta/structural-sharing storage_: large refactor of the snapshot store; revisit only if squash proves insufficient. -- *TTL/retention config* (e.g. `config.retention.auto_squash_on_tag`, +- _TTL/retention config_ (e.g. `config.retention.auto_squash_on_tag`, max-draft-age): natural follow-on once manual squash exists. ### Checklist - [ ] Repo: `deleteDraftSnapshotsBefore(trackId, boundary)` in - `release-track-dynamic.repository.js` (deleteMany on - `{ id, version: null, modified: { $lt: boundary } }`, excluding the latest - snapshot's `modified`). + `release-track-dynamic.repository.js` (deleteMany on + `{ id, version: null, modified: { $lt: boundary } }`, excluding the latest + snapshot's `modified`). - [ ] Service: squash logic in `versioning-service.js` (`squash` option on - `_doBump`) + standalone squash operation (probably `snapshot-service.js`); - reject for virtual tracks; return `squashed_count`. + `_doBump`) + standalone squash operation (probably `snapshot-service.js`); + reject for virtual tracks; return `squashed_count`. - [ ] Controller/routes: `squash` in the Zod bump body schema; new - `POST /api/release-tracks/:id/snapshots/squash` route with Zod-validated - optional `before`. + `POST /api/release-tracks/:id/snapshots/squash` route with Zod-validated + optional `before`. - [ ] OpenAPI: bump request body + new squash path. - [ ] Regression tests (`release-tracks-squash.spec.js`): squash-on-tag - deletes only pre-tag drafts; tagged snapshots survive; drafts newer than - the tagged snapshot survive; latest-draft never deleted by maintenance - squash; registry counters resync; backrefs untouched; virtual track - rejected; no-tagged-release + no `before` → 400; idempotent re-squash. + deletes only pre-tag drafts; tagged snapshots survive; drafts newer than + the tagged snapshot survive; latest-draft never deleted by maintenance + squash; registry counters resync; backrefs untouched; virtual track + rejected; no-tagged-release + no `before` → 400; idempotent re-squash. - [ ] Docs: `docs/user/release-tracks/versioning.md` (squash behavior + - the bulk-endpoints-vs-per-object-loop warning for initial population), - `docs/developer/release-tracks/` (why, trade-offs, provenance loss). + the bulk-endpoints-vs-per-object-loop warning for initial population), + `docs/developer/release-tracks/` (why, trade-offs, provenance loss). - [ ] Bruno: bump `.bru` gains `~squash` toggle; new squash request file. ### Future (not in scope) @@ -1459,18 +1590,21 @@ the latest snapshot. Like `git rebase --squash`ing the commits behind a tag. ## Small Fixes - [x] **Composition schema mismatch: `priority`.** Resolved 2026-07-29 by - requiring a unique, non-negative integer priority in request validation, - persistence, OpenAPI, documentation, and Bruno examples. + requiring a unique, non-negative integer priority in request validation, + persistence, OpenAPI, documentation, and Bruno examples. -- [ ] **`deleteSnapshot` lacks a tagged-release guard.** `DELETE /api/release-tracks/:id/snapshots/:modified` (`snapshot-service.deleteSnapshot`) deletes any snapshot, including tagged releases — contradicting the "immutable once set" versioning rule. Should 409 on `version != null` (a squash implementation must also filter `version: null`; see Snapshot Retention section). Found 2026-07-15 while designing squash. +- [x] **Restrict `deleteSnapshot` to the latest untagged draft.** + `DELETE /api/release-tracks/:id/snapshots/:modified` now returns `409` + for tagged releases and historical drafts, preserving immutable history. + Completed by the immutable-history slice. -- [ ] **`syncRegistryCounters` scales with snapshot count.** It fetches *all* snapshots (`getAllSnapshots` with projection) on every clone to recount — O(snapshot_count) reads per write, on the hottest path (member sync). Fine post-squash; consider a count query or incremental counters if draft accumulation between tags is large. +- [ ] **`syncRegistryCounters` scales with snapshot count.** It fetches _all_ snapshots (`getAllSnapshots` with projection) on every clone to recount — O(snapshot_count) reads per write, on the hottest path (member sync). Fine post-squash; consider a count query or incremental counters if draft accumulation between tags is large. ## Diffing Endpoint - [ ] Implement object diffing endpoints for snapshots. Users should be able to effectively preview changes to objects before tier transitions (candidates, staged, members). -### Idea 1 - Diffing endpoint specifically for release tracks +### Idea 1 - Diffing endpoint specifically for release tracks In this approach, we would implement a workflow-driven diffing endpoint that is specific to release tracks. The endpoint would allow users to diff objects in the candidate snapshot against their previous revisions in the staged or member snapshots. @@ -1483,12 +1617,12 @@ If `:objectRef` is a reference to an object that is not part of the candidate sn To clarify, snapshot objects transition linearly and unidirectionally through the following tier transitions: Candidate -> Staged -> Member -An object exists as a set of one or more revisions. An object is identified by its `stix.id` field, whereas an object revision is identified by its `stix.id` and `stix.modified` fields. +An object exists as a set of one or more revisions. An object is identified by its `stix.id` field, whereas an object revision is identified by its `stix.id` and `stix.modified` fields. A revision can exist in exactly one tier at a time. - If a revision exists in the candidate snapshot, it will not exist in the staged or member snapshots. -- If it exists in the staged snapshot, it will not exist in the candidate or member snapshots. +- If it exists in the staged snapshot, it will not exist in the candidate or member snapshots. - If it exists in the member snapshot, it will not exist in the candidate or staged snapshots. If a revision exists in the candidate snapshot, it will not exist in the staged or member snapshots. However, a _previous_ revision may exist in the staged or member tiers (though it is not guaranteed). Because the tier transitions are unidirectional, revisions must be temporally ordered as it relates to how they are distributed across the tiers. It should not be possible for a newer revision to exist in a previous tier. For example, if a revision exists in the candidate snapshot, it is not possible for a newer revision to exist in the staged or member snapshots. @@ -1508,6 +1642,7 @@ To stick with the example, if a revision exists as a candidate, a previous revis ### Idea 2 - Diffing endpoint for all objects (not just release tracks) Type-centric: + ``` GET /api/:type/:id/diff GET /api/:type/:id/modified/:modified/diff @@ -1515,7 +1650,8 @@ GET /api/:type/:id/modified/:modified/diff Type-agnostic: -Embed the +Embed the + ``` GET /api/attack-objects/:id/diff GET /api/attack-objects/:id/modified/:modified/diff @@ -1529,6 +1665,7 @@ GET /api/attack-objects/:id/modified/:modified/diff ``` Set up a diffing endpoint that is type-agnostic and allows users to compare any two revisions of an object. The endpoint should accept a request body that specifies the `compareTo` revision, and the endpoint should return a diff between the current revision and the specified `compareTo` revision. + ``` GET /api/compare { @@ -1545,8 +1682,6 @@ GET /api/compare } ``` - - ## Repurposing the `note` object - [ ] Implement support for tracking notes on snapshot objects (can be candidates, staged, or members). Notes should be stored in a separate Mongo collection and linked to the snapshot object via a reference field. Users should be able to add, edit, and delete notes via the API. Notably, we already have a notes service that can be leveraged for this purpose. However, it needs some modifications. The service was originally implemented with STIX in mind. The idea was to treat/represent notes as STIX objects and enable users to include them in emitted STIX bundles. However, the concept never really took off. We should modify the service to treat notes as second-class objects that are entirely separate from STIX, but rather as Workbench-native objects. Notes should be capable of being linked/attached to snapshot objects (candidates, staged, or members) as well as to objects independent of snapshots (documents in the `attackObjects` collection). @@ -1575,8 +1710,8 @@ Links/references between notes and snapshot objects will be one-to-many. A singl { "_id": "ObjectId", "workspace": { - "notes": ["ObjectId"] // Array of references to notes linked to this attack object + "notes": ["ObjectId"] // Array of references to notes linked to this attack object }, - "stix": "StixObject", + "stix": "StixObject" } ``` diff --git a/docs/developer/release-tracks/authorization.md b/docs/developer/release-tracks/authorization.md index 11fec021..0d6f62d1 100644 --- a/docs/developer/release-tracks/authorization.md +++ b/docs/developer/release-tracks/authorization.md @@ -2,8 +2,8 @@ Release-track access follows the existing Workbench roles. Read operations are available to visitors and higher. Normal draft workflow operations require an -editor, team lead, or administrator. Operations that can replace authoritative -membership or destroy history require an administrator. +editor, team lead, or administrator. Deleting an entire track and all of its +history requires an administrator. ## Authorization matrix @@ -13,23 +13,17 @@ membership or destroy history require an administrator. | Preview releases and export snapshots | Yes | Yes | Yes | | Create tracks and drafts; manage candidates/staged/config/composition | No | Yes | Yes | | Tag a standard or virtual snapshot | No | Yes | Yes | -| Delete an untagged individual snapshot | No | Yes | Yes | -| Replace standard-track members directly | No | No | Yes | +| Delete the latest untagged draft snapshot | No | Yes | Yes | | Delete an entire track and all snapshot history | No | No | Yes | -The two direct replacement routes and full-track deletion also require -`confirm_track_id` to equal the `:id` path parameter. Authorization runs before -the controller, and confirmation runs before request-body validation or -persistence. +Full-track deletion also requires `confirm_track_id` to equal the `:id` path +parameter. Authorization runs before the controller, and confirmation runs +before persistence. ## Audited destructive actions -The following actions create a `releaseTrackAuditEvents` record before their -business operation begins: - -- `replace_members_latest` -- `replace_members_historical` -- `delete_track` +The `delete_track` action creates a `releaseTrackAuditEvents` record before +the business operation begins. Each event records the authenticated actor, confirmation value, target track, request summary, timestamps, and a `pending`, `completed`, or `failed` status. diff --git a/docs/developer/release-tracks/backref-reconciliation.md b/docs/developer/release-tracks/backref-reconciliation.md index 6e2d3a9d..d5a98790 100644 --- a/docs/developer/release-tracks/backref-reconciliation.md +++ b/docs/developer/release-tracks/backref-reconciliation.md @@ -18,9 +18,9 @@ tracks follow that precedent but maintain the pointers event-driven. Membership changes through many routes: add/remove candidates, review, manual and auto promotion, demotion, release (staged → members), member sync, -`updateContents`, track cloning, bundle import, snapshot deletion, and track -deletion. Patching each route with a bespoke incremental backref update would -be error-prone and would drift. +track cloning, bundle import, latest-draft deletion, and track deletion. +Patching each route with a bespoke incremental backref update would be +error-prone and would drift. Instead, every route already funnels through a small set of persistence choke points, and each choke point triggers a full **snapshot-driven reconciliation**: @@ -34,7 +34,7 @@ self-healing — a missed or failed pass is corrected by the next one. ``` snapshot-service.cloneSnapshot ┐ (every tier/config/metadata mutation, snapshot-service._cloneToNewTrack │ member sync, auto-promotion, -snapshot-service.deleteSnapshot │ bundle import, updateContents, ...) +snapshot-service.deleteSnapshot │ bundle import, ...) snapshot-service.deleteTrack │ versioning-service.releaseLatest/releaseByModified ┘ (staged → members via tagSnapshotInPlace) │ diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index 5f9abb0c..81e984c7 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -100,22 +100,19 @@ Implemented in filter, mirroring the fact that members are inherently reviewed. `state` never affects members. `reviewed` is intentionally not a valid `state` value for this reason. -2. **Hydration** — any selected candidate/staged `"latest"` selectors are - resolved for this export request, then the concrete - `{object_ref, object_modified}` pairs are batch-fetched per STIX type via - each repository's `findManyByIdAndModified`. The stored draft selectors are - not mutated. Hydration is fail-closed: if any selected primary revision is - missing, the request returns `409` with `missing_references` and emits no - partial bundle. Database failures propagate as server errors. -3. **Relationships** — the relationship service fetches the latest active - relationship revisions whose `source_ref` and `target_ref` are both among - the selected objects. Deprecated data-component `detects` relationships - are excluded. Relationships remain indirect export-time content; they are - not added to the snapshot tiers. -4. **Supporting objects** — referenced identities and marking definitions - that are not themselves tier entries are fetched and appended. -5. **LinkById conversion** — same behavior as the legacy exporter, preferring - objects already in the export before falling back to a database lookup. +2. **Manifest replay** — the snapshot identifies an active graph manifest, or + a complete linked pending manifest recovering from an interrupted + activation, created at the same persistence boundary. The manifest records exact + primary, relationship, secondary, supporting, and LinkById dependency + revisions. Export hydrates those entries and performs no live graph + expansion. +3. **Bounded secondary selection** — replay starts from the requested primary + tiers, follows only dependency edges frozen in the manifest, and emits a + relationship only when both exact endpoint revisions are selected. +4. **Supporting objects** — only identities and marking definitions frozen in + the manifest and referenced by the selected graph are appended. +5. **LinkById conversion** — conversion uses only exact render targets frozen + in the manifest and never falls back to a current database lookup. 6. **Assembly** (Zod transform) — notes are dropped, objects are conformed to `stixVersion` via the shared `lib/stix-conformance.js` helpers, and the bundle envelope is emitted (with `spec_version: "2.0"` only when @@ -134,9 +131,10 @@ Implemented in - `x_mitre_contents`: every bundle object except marking definitions (which are recorded in `object_marking_refs`), sorted by `object_ref` -Because snapshot contents are explicitly curated, the export intentionally -does **not** apply the legacy attack-id / deprecated / revoked filters — if a -revision is in the snapshot, it is exported. +Because snapshot contents are explicitly curated, primary entries do **not** +receive the legacy attack-id / deprecated / revoked filters. Secondary graph +capture retains the established bounded ATT&CK expansion rules and freezes +the resulting graph at snapshot creation. #### Relationship and secondary-object consistency boundary @@ -146,47 +144,58 @@ Release-track snapshots distinguish **primary** and **secondary** content: record exact `(object_ref, object_modified)` revisions. Standard candidates and staged entries may instead store `"latest"` and are resolved just in time when a draft export includes those tiers. -- Secondary objects are not snapshot members. They are discovered because a - primary object references them through an embedded STIX ID, an SRO connects - two selected primary objects, or the bundle needs a supporting identity or - marking definition. +- Secondary objects are not snapshot members. They are discovered when the + snapshot is created because an exact-pinned SRO connects them to a primary, + the bounded ATT&CK rules identify a detection strategy, or the bundle needs + a supporting identity, marking definition, or LinkById render target. Tagged standard membership is deterministic because release planning resolves staged selectors before promoting them to members. Virtual materialization likewise copies exact member revisions from tagged component snapshots and -never follows a component's later `track_latest` candidate movement. Draft -exports that explicitly include dynamic candidate/staged tiers are snapshots -of the latest revisions at export time. Secondary content is also resolved -just in time during bundle generation. - -Relationships are the largest consistency boundary. Current SRO -`source_ref`/`target_ref` fields identify STIX object IDs, not exact -`(object_id, object_modified)` revisions. An SRO can consequently describe the -whole revision chain of each endpoint rather than one precise pair of SDO -entities. The exporter resolves the latest active relationship revisions when -the bundle is requested. This creates several tradeoffs: - -- exporting the same tagged snapshot at different times can produce different - relationship objects or TOC contents; -- relationship revisions are not represented in snapshot history, - release-track backrefs, or composition audit metadata; -- revoking a relationship can remove it from an older snapshot export, while - creating a relationship can add it to that export; -- each bundle request performs a relationship query, although the query is - constrained to relationships whose two endpoints are already selected. - -Consumers that require byte-for-byte or graph-level reproducibility must -archive the emitted bundle. - -Making bundle graphs deterministic requires a separate, high-risk data-model -change rather than virtual composition re-resolution. A future design must -version-control relationships, pin each SRO endpoint to an exact SDO revision, -and likely clone every affected SRO whenever a new endpoint revision is -created. It must also persist an export manifest containing the selected -relationship and other secondary-object revisions. That one-to-one SDO/SRO -model has significant migration, write-amplification, concurrency, and -database-storage costs and is deliberately deferred pending design and -measurement. +never follows a component's later `track_latest` candidate movement. + +Every relationship revision stores server-controlled exact source and target +pins under `workspace.relationship_endpoints`. These fields identify the +precise `(object_ref, object_modified)` pair represented by each side of the +SRO. They are not emitted because bundle output includes only the `stix` +object. When an endpoint advances, Workbench creates a new SRO revision with +updated pins rather than rewriting the older SRO. + +Each persisted snapshot references a tier-aware manifest. A pending manifest +and all of its entries are written before the snapshot is linked to it, then +activated after persistence succeeds. The snapshot link is the durable commit +record: replay can use and self-activate a complete linked pending manifest +after a process interruption. +A standard release replaces the draft manifest with one built from the +resolved release plan, so dynamic staged selectors become exact members. +Materialized virtual snapshots contain exact roots from the outset. + +Active and pending manifests protect their exact dependencies. In-place +updates and hard deletes that would invalidate a primary or secondary +revision return `409`; lineage deletion is rejected when any version is +protected. Relationship source, target, and type changes are rejected. +Description-only relationship corrections remain allowed because the +relationship STIX payload used by older snapshots is frozen in the manifest. +Deleting a draft snapshot or track removes its manifest and releases +protection that no other snapshot needs. + +Existing data is upgraded by an idempotent migration. Only the latest +revision of each legacy relationship can be endpoint-pinned truthfully. +Pre-existing snapshot manifests are labeled `baseline_reconstruction` +because they describe the graph visible during migration rather than an +unknowable historical graph. + +The deliberate exception is a standard draft export that explicitly includes +a candidate or staged entry stored as `"latest"`. That selector is defined to +move until release, so the selected draft graph is resolved for that request. +Release preview and commit resolve it again; a successful commit stores an +exact manifest. Members, tagged releases, materialized virtual snapshots, and +exact-selector draft tiers replay deterministically. + +The graph and object payload are reproducible, but the bundle is not promised +to be byte-for-byte identical: the bundle envelope receives a newly generated +bundle ID. Consumers should compare the emitted STIX object set and revisions, +not the envelope UUID. ### Where validation happens diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index abd4272d..e6794cb6 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -444,9 +444,13 @@ copies the exact member revisions from the selected tagged component snapshots, and later component activity cannot change the persisted virtual snapshot. -This deterministic guarantee covers primary snapshot membership. Secondary -objects and relationships discovered while rendering `format=bundle` remain -an export-time concern and can change between bundle requests. +Each snapshot also references an internal, tier-aware graph manifest. It +freezes the exact relationship endpoint revisions, bounded secondary objects, +supporting objects, and LinkById render targets needed by `format=bundle`. +Tagged standard snapshots, materialized virtual snapshots, and draft tiers +that use exact selectors therefore replay the same graph. A standard draft +tier explicitly stored as `"latest"` remains intentionally dynamic until the +release boundary. The three valid `snapshot_schedule` shapes are: diff --git a/docs/developer/release-tracks/error-handling.md b/docs/developer/release-tracks/error-handling.md index cc553b03..ebbd6cd8 100644 --- a/docs/developer/release-tracks/error-handling.md +++ b/docs/developer/release-tracks/error-handling.md @@ -14,7 +14,8 @@ } ``` -**Solution:** Create a new snapshot by modifying the collection, then release the new snapshot. +**Solution:** Create a new draft through a supported release-track workflow +operation, then release the new snapshot. ### InvalidVersionError @@ -52,6 +53,33 @@ Tagged snapshots are immutable release records. Create or modify a draft snapshot instead; deleting an entire release track remains a separate track-level operation. +### HistoricalSnapshotDeletionError + +**Thrown when:** Attempting to delete an untagged draft that is no longer the +latest snapshot. + +**HTTP Status:** 409 Conflict + +The response identifies both `snapshot_modified` and +`latest_snapshot_modified`. Refresh the track and continue from the latest +draft; historical drafts cannot be removed. + +### SnapshotGraphPinnedRevisionError + +**Thrown when:** An in-place update or hard delete would change an exact +primary, relationship, secondary, supporting, or LinkById dependency frozen +in a release-track snapshot graph. Full-lineage and collection +`deleteAllContents` operations are preflighted against the same invariant. + +**HTTP Status:** 409 Conflict + +The response includes `snapshot_graph_pins` entries identifying the track, +snapshot timestamp, manifest entry kind, and tier where applicable. Create a +new STIX revision instead. Administrator authorization is not a force-delete +override. Description-only relationship corrections remain allowed because +the older relationship payload is frozen inside each existing manifest; +source, target, and relationship-type changes return 400. + ### NotFoundError **Thrown when:** Collection with specified ID does not exist. diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index ec2db9dd..449bfbf7 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -52,7 +52,7 @@ silently dropping it. The error contract distinguishes who can correct the problem: - Request ingress returns `400` with `missing_references` when candidate - selection or direct member replacement names a revision that does not exist. + selection names a revision that does not exist. - Operations over already-persisted content return `409` with `missing_references` when a release preview/commit, track clone, virtual materialization, quarantine promotion, or bundle export encounters a @@ -162,10 +162,11 @@ a standard component track. Snapshot retrieval never re-runs composition, so there is no `resolve` query parameter or `resolved_content` response wrapper. Workbench retrieval returns -the persisted primary membership. Bundle export is a separate consistency -boundary: secondary relationships and supporting objects are discovered at -request time and are not deterministic until relationships become -version-controlled against exact endpoint revisions. +the persisted primary membership. Bundle export replays a graph manifest +captured with the snapshot. Relationship revisions carry server-controlled +exact endpoint pins in `workspace.relationship_endpoints`, and the manifest +freezes the bounded secondary/supporting graph without emitting those internal +fields in STIX output. Snapshot schedules use the same strict, mode-discriminated Zod schema at the controller and service boundaries. `manual` has no selector field, `cron` @@ -212,7 +213,8 @@ There is no side-effect-free virtual snapshot-creation preview. Once a virtual draft is persisted, it uses the same retrieval and release endpoints as a standard draft. Release planning never resolves composition and rejects a virtual draft without `composition_resolution` with `409 Conflict`. Generic -latest and historical `/contents` mutations are standard-only; virtual +snapshot member replacement is not supported for either track type. Standard +membership enters through the candidate/staged/release lifecycle; virtual membership has composition resolution as its sole authority. Quarantine promotion is a snapshot mutation, not a composition diff --git a/docs/developer/release-tracks/member-sync-strategies.md b/docs/developer/release-tracks/member-sync-strategies.md index c0288a9d..023a4f32 100644 --- a/docs/developer/release-tracks/member-sync-strategies.md +++ b/docs/developer/release-tracks/member-sync-strategies.md @@ -130,6 +130,10 @@ sync strategy determines what workflow action (if any) to take: > `BaseService` rejects `PUT`/`DELETE` of a members-pinned revision with > 409 (`MemberPinnedRevisionError`) — released content is immutable in > place. +> - A candidate or staged revision remains editable unless it is also a +> secondary/supporting dependency frozen in a snapshot graph manifest. In +> that case graph integrity takes precedence and the operation returns 409 +> (`SnapshotGraphPinnedRevisionError`). ### Relationship to Existing Features diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index f722075a..9a1bc26f 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -47,7 +47,6 @@ POST /api/release-tracks/new POST /api/release-tracks/new-from-bundle POST /api/release-tracks/import POST /api/release-tracks/:id/meta -POST /api/release-tracks/:id/contents?confirm_track_id=:id POST /api/release-tracks/:id/snapshots/latest/release POST /api/release-tracks/:id/clone DELETE /api/release-tracks/:id?confirm_track_id=:id @@ -59,7 +58,6 @@ DELETE /api/release-tracks/:id?confirm_track_id=:id GET /api/release-tracks/:id/snapshots GET /api/release-tracks/:id/snapshots/latest GET /api/release-tracks/:id/snapshots/:modified -POST /api/release-tracks/:id/snapshots/:modified/meta POST /api/release-tracks/:id/snapshots/:modified/release POST /api/release-tracks/:id/snapshots/:modified/clone DELETE /api/release-tracks/:id/snapshots/:modified @@ -462,46 +460,18 @@ Creates new snapshot with updated metadata. } ``` -### Update Contents +### Snapshot content is append-only -``` -POST /api/release-tracks/:id/contents -``` - -Creates new snapshot with updated member objects. **This is intended for retroactive hotfixes only.** The main workflow for enrolling new member objects into `x_mitre_contents` is through the candidate-staging promotion cycle described in [versioning.md](./versioning.md). - -This operation requires the administrator role. The -`confirm_track_id` query parameter must exactly equal the `:id` path -parameter. Every accepted attempt is recorded in the durable release-track -destructive audit trail. - -This operation is available only for standard tracks. Virtual membership is -computed from component releases and can only be updated by materializing a -virtual draft with `POST /api/release-tracks/:id/virtual/snapshots/create`. -Using either contents endpoint with a virtual track returns `400 Bad Request`. - -**Request Body:** - -```json -{ - "x_mitre_contents": [ - { - "obj_ref": "attack-pattern--uuid1", - "obj_modified": "2024-02-01T10:00:00.000Z" - }, - { - "obj_ref": "malware--uuid2", - "obj_modified": "latest" - } - ] -} -``` +There is no endpoint for replacing a persisted snapshot's `members` tier. +Standard tracks add or revise content through candidates, promote those +objects to staged, and freeze them into members during release. Virtual tracks +derive members only when a composition is materialized. -Every entry must include an object ID and either an ISO `obj_modified` -timestamp or the request-time shorthand `"latest"`. The server resolves -`"latest"` to the object's actual latest `stix.modified` value before -persisting the new standard-track snapshot. Snapshot members never store a -moving reference. +If an operator makes an unwanted draft, delete it while it is still the latest +untagged snapshot or continue with a newer corrective draft. Historical drafts +and tagged releases remain part of the immutable track history. Bootstrapping a +new track from a bundle is the supported way to start with an existing member +set. ### Release Latest Snapshot @@ -621,30 +591,6 @@ GET /api/release-tracks/:id/snapshots/2024-01-15T16:20:00.000Z?format=bundle GET /api/release-tracks/:id/snapshots/2024-01-15T16:20:00.000Z?format=bundle&include=staged ``` -### Update Metadata (Specific Snapshot) - -``` -POST /api/release-tracks/:id/snapshots/:modified/meta -``` - -Creates new snapshot with updated metadata. - -**Request Body:** Same as [Update Metadata](#update-metadata) for latest snapshot. - -### Update Contents (Specific Snapshot) - -``` -POST /api/release-tracks/:id/snapshots/:modified/contents?confirm_track_id=:id -``` - -Creates new snapshot with updated member objects. **This is intended for retroactive hotfixes only.** - -**Request Body:** Same as [Update Contents](#update-contents) for latest snapshot. - -Like the latest form, this operation is restricted to standard tracks, -requires the administrator role and exact track-ID confirmation, and creates -a durable audit event. - ### Release/Tag Specific Snapshot Converts a specific draft snapshot to a tagged release. Tags snapshot in-place (does not create new snapshot). @@ -665,12 +611,15 @@ POST /api/release-tracks/:id/snapshots/:modified/clone ### Delete Specific Snapshot -**TODO**: further consideration needs to be given here. We need to be careful to avoid breaking contextual continuity between snapshots. - ``` DELETE /api/release-tracks/:id/snapshots/:modified ``` +Deletes the selected snapshot only when it is both the latest snapshot and an +untagged draft. Deletion reverts the track to the immediately preceding +snapshot. Tagged releases and older drafts return `409 Conflict`; they cannot +be removed or rewritten. + --- ## Candidate Management @@ -1332,10 +1281,16 @@ retrieval never recomputes virtual composition. As long as the track does not acquire a newer snapshot, `/snapshots/latest` selects the same primary revision set, and `/snapshots/:modified` addresses that set explicitly. -This determinism does not extend to the complete `format=bundle` graph. -Secondary relationships, identities, marking definitions, and other supporting -objects are resolved during bundle generation and may change independently of -the primary snapshot members. +The server freezes the bounded `format=bundle` graph when it persists the +snapshot. Repeated exports reuse exact relationship, secondary, supporting, +and LinkById dependency revisions rather than discovering the current graph. +Hard deletes and unsafe in-place edits to those protected revisions return +`409 Conflict`. + +A standard draft remains intentionally dynamic only when the request includes +a candidate or staged entry stored with `object_modified: "latest"`. That +selector is resolved at request time until release. Tagged standard members +and all materialized virtual members are exact. `duplicates_found` counts object IDs contributed by more than one component, including repeated contributions of the same exact revision. diff --git a/docs/user/release-tracks/object-backrefs.md b/docs/user/release-tracks/object-backrefs.md index 7d960a68..4fff6916 100644 --- a/docs/user/release-tracks/object-backrefs.md +++ b/docs/user/release-tracks/object-backrefs.md @@ -76,8 +76,9 @@ An object referenced by multiple tracks carries one entry per track. Release tracks are never blind to changes in the objects they pin: -- **Members-pinned revisions are immutable in place.** `PUT` and `DELETE` - against a revision that any track pins in its `members` tier return +- **Released and graph-frozen revisions are immutable in place.** `PUT` and + `DELETE` against a revision that any track pins in its `members` tier, or + that a snapshot needs as a secondary/supporting graph dependency, return `409 Conflict` — released content cannot be changed or destroyed under the track. Make changes by creating a new revision (`POST`); retire an object by creating a new revision with `x_mitre_deprecated: true`. Revision sync @@ -86,6 +87,11 @@ Release tracks are never blind to changes in the objects they pin: protected when it belongs only to a historical tagged release, when a newer draft has removed it, or when a reconciliation failure temporarily omitted its backref. +- **Standalone candidate/staged roots remain editable.** Merely appearing in + a draft workflow tier does not create a graph-protection conflict, so the + existing in-place review workflow below still applies. If that same revision + is also a frozen secondary dependency of another selected root, graph + protection takes precedence and the edit returns `409`. - **Candidate/staged-pinned revisions can be edited in place, but the track sees it.** An in-place `PUT` (including one that only sets `x_mitre_deprecated`) marks the pinned entry `modified-in-place`: the @@ -104,8 +110,9 @@ Release tracks are never blind to changes in the objects they pin: workflow creates one new revision of the revoked object (`revoked: true`); revision sync enrolls it as a candidate in tracks where the object is a member and moves candidate/staged pins to it. The revoking - object and the `revoked-by` relationship are not tracked explicitly — - bundle export pulls secondary objects and their SROs in dynamically. + object and the `revoked-by` relationship are not direct track members. + Snapshot creation captures them as bounded secondary graph dependencies + when applicable; later bundle export replays that frozen graph. ## Lifecycle example diff --git a/docs/user/release-tracks/summary.md b/docs/user/release-tracks/summary.md index 55353ba9..336e067c 100644 --- a/docs/user/release-tracks/summary.md +++ b/docs/user/release-tracks/summary.md @@ -68,26 +68,25 @@ GET /api/release-tracks/:id/snapshots/latest POST /api/release-tracks/:id/config POST /api/release-tracks/:id/meta POST /api/release-tracks/:id/clone -PUT /api/release-tracks/:id/snapshots/latest/release -POST /api/release-tracks/:id/archive -DELETE /api/release-tracks/:id +POST /api/release-tracks/:id/snapshots/latest/release +DELETE /api/release-tracks/:id?confirm_track_id=:id # Candidate/workflow management POST /api/release-tracks/:id/candidates POST /api/release-tracks/:id/candidates/review +POST /api/release-tracks/:id/candidates/promote +POST /api/release-tracks/:id/staged/demote # Snapshot-specific operations GET /api/release-tracks/:id/snapshots/:modified -POST /api/release-tracks/:id/snapshots/:modified/config -POST /api/release-tracks/:id/snapshots/:modified/meta POST /api/release-tracks/:id/snapshots/:modified/clone DELETE /api/release-tracks/:id/snapshots/:modified -PUT /api/release-tracks/:id/snapshots/:modified/release +POST /api/release-tracks/:id/snapshots/:modified/release ``` ### 2. Git-Inspired Versioning -We borrow heavily concepts from git. Snapshots are sort of like commits and tagged releases are like git tags. A release track contains snapshots: delta permutations that can be linearly tracked to deduce how the release track has evolved over time. A snapshot is generated every time a change is made, whether that be adding/removing objects, updating the release track configuration, or renaming the release track altogether. +We borrow heavily concepts from git. Snapshots are sort of like commits and tagged releases are like git tags. A release track contains snapshots: delta permutations that can be linearly tracked to deduce how the release track has evolved over time. A snapshot is generated every time a supported draft operation changes state, such as adding or promoting candidates, updating release-track configuration, or renaming the release track. **Snapshots** (like Git commits) - Every modification creates a new snapshot @@ -130,6 +129,13 @@ and committing are separate operations, so a newer object revision created between them can legitimately produce a different plan; the committed release records the revision resolved by the commit itself. +At snapshot persistence, the server also freezes the bounded bundle graph: +exact relationship endpoint revisions, secondary objects, supporting objects, +and LinkById render targets. Tagged releases and materialized virtual +snapshots therefore reproduce the same STIX object graph on later +`format=bundle` retrievals. The generated bundle-envelope ID itself is not +stable. + Virtual snapshots are stricter still: they copy only exact member revisions from tagged standard component snapshots. They never inherit `track_latest`, and retrieving a persisted virtual snapshot does not re-resolve its component diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index 6885e5ae..fa06c205 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -954,10 +954,12 @@ Consequently, while the track does not acquire a newer snapshot, `latest` path segment selects the most recent snapshot; it is not a dynamic object-revision selector. -This guarantee applies to the persisted primary snapshot contents. -`format=bundle` also discovers secondary relationships and supporting objects -at export time, so the complete bundle graph is not currently reproducible. -See [Bundle Export](../../developer/release-tracks/bundle-export.md#relationship-and-secondary-object-consistency-boundary). +This guarantee also covers the bounded `format=bundle` object graph. +Relationship endpoint revisions, secondary objects, supporting objects, and +LinkById render targets are frozen in the snapshot's graph manifest. +Repeated exports may use a different bundle-envelope UUID, but replay the same +snapshot object graph. See +[Bundle Export](../../developer/release-tracks/bundle-export.md#relationship-and-secondary-object-consistency-boundary). ## Quarantine Management @@ -1151,8 +1153,7 @@ POST /api/release-tracks/:id/snapshots/:modified/release ``` If composition changes after materialization, repeat the create step. Direct -member replacement through either standard-track `/contents` endpoint is -rejected for virtual tracks. +member replacement is not supported. ### 2. Use Scheduled Snapshots for Consistency diff --git a/docs/user/release-tracks/workflow-examples.md b/docs/user/release-tracks/workflow-examples.md index d3e0c0fc..d1d796f4 100644 --- a/docs/user/release-tracks/workflow-examples.md +++ b/docs/user/release-tracks/workflow-examples.md @@ -8,76 +8,73 @@ POST /api/release-tracks/new { "name": "My Release", ... } # Creates: snapshot 1, x_mitre_version: null -# 2. Update contents -POST /api/release-tracks/release--123/contents -{ "x_mitre_contents": [...] } -# Creates: snapshot 2, x_mitre_version: null +# 2. Add objects as candidates +POST /api/release-tracks/release--123/candidates +{ "object_refs": [{ "id": "attack-pattern--...", "modified": "latest" }] } +# Creates: snapshot 2, version: null -# 3. Update metadata +# 3. Promote accepted candidates to staged +POST /api/release-tracks/release--123/candidates/promote +{ "object_refs": ["attack-pattern--..."] } +# Creates: snapshot 3, version: null + +# 4. Update metadata POST /api/release-tracks/release--123/meta { "description": "Updated description" } -# Creates: snapshot 3, x_mitre_version: null +# Creates: snapshot 4, version: null -# 4. Ready for first release - tag as v1.0 +# 5. Ready for first release - staged objects become members POST /api/release-tracks/release--123/snapshots/latest/release { "increment": "major" } -# Updates: snapshot 3, x_mitre_version: "1.0" (IN-PLACE) - -# 5. Continue development -POST /api/release-tracks/release--123/contents -{ "x_mitre_contents": [...] } -# Creates: snapshot 4, x_mitre_version: null - -# 6. Minor release -POST /api/release-tracks/release--123/snapshots/latest/release -{ "increment": "minor" } -# Updates: snapshot 4, x_mitre_version: "1.1" (IN-PLACE) +# Updates: snapshot 4, version: "1.0" (in place) -# 7. More changes -POST /api/release-tracks/release--123/contents -{ "x_mitre_contents": [...] } -# Creates: snapshot 5, x_mitre_version: null +# 6. Continue development through the same candidate workflow +POST /api/release-tracks/release--123/candidates +{ "object_refs": [{ "id": "malware--...", "modified": "latest" }] } +POST /api/release-tracks/release--123/candidates/promote +{ "object_refs": ["malware--..."] } +# Creates snapshots 5 and 6 -# 8. Another minor release +# 7. Minor release POST /api/release-tracks/release--123/snapshots/latest/release { "increment": "minor" } -# Updates: snapshot 5, x_mitre_version: "1.2" (IN-PLACE) +# Updates: snapshot 6, version: "1.1" (in place) ``` **Resulting Timeline:** ``` -snapshot 1: modified: T1, x_mitre_version: null -snapshot 2: modified: T2, x_mitre_version: null -snapshot 3: modified: T3, x_mitre_version: "1.0" ← RELEASE -snapshot 4: modified: T4, x_mitre_version: "1.1" ← RELEASE -snapshot 5: modified: T5, x_mitre_version: "1.2" ← RELEASE +snapshot 1: initial empty draft +snapshot 2: candidate added +snapshot 3: candidate staged +snapshot 4: version "1.0" ← RELEASE +snapshot 5: next candidate added +snapshot 6: version "1.1" ← RELEASE ``` ### Example 2: Selective Release Tagging ```bash -# Create several snapshots -POST /api/collections/collection--456/contents # snapshot 1 -POST /api/collections/collection--456/contents # snapshot 2 -POST /api/collections/collection--456/contents # snapshot 3 -POST /api/collections/collection--456/contents # snapshot 4 -POST /api/collections/collection--456/contents # snapshot 5 - -# Only tag snapshots 2 and 5 as releases -POST /api/collections/collection--456/modified//snapshots/latest/release +# Create several drafts through ordinary metadata/workflow changes +POST /api/release-tracks/release--456/meta # draft 2 +POST /api/release-tracks/release--456/meta # draft 3 +POST /api/release-tracks/release--456/meta # draft 4 +POST /api/release-tracks/release--456/meta # draft 5 + +# Tag draft 2 retroactively and then tag the latest draft +POST /api/release-tracks/release--456/snapshots//release { "version": "1.0" } -POST /api/collections/collection--456/snapshots/latest/release # Latest = snapshot 5 +POST /api/release-tracks/release--456/snapshots/latest/release { "version": "1.1" } ``` **Resulting Timeline:** ``` -snapshot 1: x_mitre_version: null (skipped) -snapshot 2: x_mitre_version: "1.0" ← RELEASE -snapshot 3: x_mitre_version: null (skipped) -snapshot 4: x_mitre_version: null (skipped) -snapshot 5: x_mitre_version: "1.1" ← RELEASE +snapshot 1: version: null (skipped) +snapshot 2: version: "1.0" ← RELEASE +snapshot 3: version: null (skipped) +snapshot 4: version: null (skipped) +snapshot 5: version: "1.1" ← RELEASE ``` This mirrors Git's ability to tag any commit, not just the latest. @@ -86,22 +83,22 @@ This mirrors Git's ability to tag any commit, not just the latest. ```bash # Tag latest snapshot -POST /api/collections/collection--789/snapshots/latest/release +POST /api/release-tracks/release--789/snapshots/latest/release { "version": "1.0" } # Success: snapshot tagged as v1.0 # Attempt to release the same snapshot again -POST /api/collections/collection--789/snapshots/latest/release +POST /api/release-tracks/release--789/snapshots/latest/release { "version": "1.1" } # Error: AlreadyReleasedError - "This snapshot has already been tagged as version 1.0" -# Solution: Make a change first (creates new snapshot) -POST /api/collections/collection--789/contents -{ "x_mitre_contents": [...] } +# Solution: Make a supported draft change first +POST /api/release-tracks/release--789/meta +{ "description": "Prepare the next release" } # Creates new snapshot # Now release the new snapshot -POST /api/collections/collection--789/snapshots/latest/release +POST /api/release-tracks/release--789/snapshots/latest/release { "version": "1.1" } # Success: new snapshot tagged as v1.1 ``` diff --git a/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js b/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js new file mode 100644 index 00000000..2643e96d --- /dev/null +++ b/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js @@ -0,0 +1,276 @@ +'use strict'; + +/** + * Backfill exact endpoint revision pins on the latest revision of each + * relationship, then reconstruct a baseline graph manifest for every + * pre-existing release-track snapshot. + * + * Historical relationships cannot be reconstructed truthfully because their + * endpoint revision was not recorded when they were created. Snapshot + * manifests produced here are therefore explicitly marked as baseline + * reconstructions of the graph visible at migration time. + */ + +const TRACK_COLLECTION_PATTERN = + /^release-track--[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +const CONCURRENCY = 4; + +async function mapWithConcurrency(items, mapper) { + let nextIndex = 0; + + async function worker() { + while (nextIndex < items.length) { + const index = nextIndex++; + await mapper(items[index]); + } + } + + await Promise.all(Array.from({ length: Math.min(CONCURRENCY, items.length) }, () => worker())); +} + +async function latestRelationships(db) { + const latestViewExists = await db + .listCollections({ name: 'view.relationships.latest' }, { nameOnly: true }) + .hasNext(); + if (latestViewExists) { + return db.collection('view.relationships.latest').find({}).toArray(); + } + + return db + .collection('relationships') + .aggregate([ + { $sort: { 'stix.id': 1, 'stix.modified': -1 } }, + { $group: { _id: '$stix.id', document: { $first: '$$ROOT' } } }, + { $replaceRoot: { newRoot: '$document' } }, + ]) + .toArray(); +} + +async function latestEndpoint(db, objectRef) { + return db + .collection('attackObjects') + .findOne( + { 'stix.id': objectRef }, + { projection: { 'stix.id': 1, 'stix.modified': 1 }, sort: { 'stix.modified': -1 } }, + ); +} + +async function buildRelationshipPinOperations(db) { + const relationships = await latestRelationships(db); + const endpointCache = new Map(); + const missing = []; + const operations = []; + + async function endpoint(objectRef) { + if (!endpointCache.has(objectRef)) { + endpointCache.set(objectRef, await latestEndpoint(db, objectRef)); + } + return endpointCache.get(objectRef); + } + + for (const relationship of relationships) { + const [source, target] = await Promise.all([ + endpoint(relationship.stix.source_ref), + endpoint(relationship.stix.target_ref), + ]); + if (!source || !target) { + missing.push({ + relationship_ref: relationship.stix.id, + relationship_modified: relationship.stix.modified, + missing_endpoints: [ + ...(!source ? [relationship.stix.source_ref] : []), + ...(!target ? [relationship.stix.target_ref] : []), + ], + }); + continue; + } + + operations.push({ + updateOne: { + filter: { _id: relationship._id }, + update: { + $set: { + 'workspace.relationship_endpoints': { + source: { + object_ref: source.stix.id, + object_modified: source.stix.modified, + }, + target: { + object_ref: target.stix.id, + object_modified: target.stix.modified, + }, + }, + }, + }, + }, + }); + } + + return { relationships, operations, missing }; +} + +async function findTrackIds(db) { + const [registeredTracks, collections] = await Promise.all([ + db.collection('releaseTrackRegistry').find({}).project({ track_id: 1, _id: 0 }).toArray(), + db.listCollections({}, { nameOnly: true }).toArray(), + ]); + return [ + ...new Set([ + ...registeredTracks.map((track) => track.track_id), + ...collections + .map((collection) => collection.name) + .filter((name) => TRACK_COLLECTION_PATTERN.test(name)), + ]), + ].sort(); +} + +async function ensureManifestIndexes(db) { + await Promise.all([ + db + .collection('releaseTrackGraphManifests') + .createIndex({ manifest_id: 1 }, { name: 'manifest_id_1', unique: true }), + db + .collection('releaseTrackGraphManifests') + .createIndex( + { track_id: 1, snapshot_modified: 1, state: 1 }, + { name: 'manifest_by_snapshot' }, + ), + db + .collection('releaseTrackGraphManifestEntries') + .createIndex( + { manifest_id: 1, revision_key: 1, kind: 1, tier: 1 }, + { name: 'unique_manifest_entry', unique: true }, + ), + db + .collection('releaseTrackGraphManifestEntries') + .createIndex( + { object_ref: 1, object_modified: 1, manifest_id: 1 }, + { name: 'manifest_revision_protection' }, + ), + db + .collection('releaseTrackGraphManifestEntries') + .createIndex({ manifest_id: 1, kind: 1, tier: 1 }, { name: 'manifest_id_1_kind_1_tier_1' }), + ]); +} + +async function backfillSnapshotManifests(db, options) { + const graphManifestService = require('../app/services/release-tracks/graph-manifest-service'); + const trackIds = await findTrackIds(db); + const report = { tracks: trackIds.length, snapshots: 0, manifests_created: 0 }; + + await mapWithConcurrency(trackIds, async (trackId) => { + const collectionExists = await db + .listCollections({ name: trackId }, { nameOnly: true }) + .hasNext(); + if (!collectionExists) return; + + const snapshots = await db.collection(trackId).find({}).toArray(); + report.snapshots += snapshots.length; + for (const snapshot of snapshots) { + if (snapshot.graph_manifest_id) { + const linkedManifest = await db.collection('releaseTrackGraphManifests').findOne({ + manifest_id: snapshot.graph_manifest_id, + state: { $in: ['pending', 'active'] }, + }); + if (linkedManifest) { + if (!options.dryRun && linkedManifest.state === 'pending') { + await graphManifestService.activate(linkedManifest.manifest_id); + } + continue; + } + } + if (options.dryRun) { + report.manifests_created++; + continue; + } + + const manifestId = await graphManifestService.prepare(snapshot, { + baselineReconstruction: true, + }); + try { + await db + .collection(trackId) + .updateOne({ _id: snapshot._id }, { $set: { graph_manifest_id: manifestId } }); + await graphManifestService.activate(manifestId); + report.manifests_created++; + } catch (err) { + await graphManifestService.discard(manifestId); + throw err; + } + } + }); + + return report; +} + +async function run(db, options = {}) { + const relationshipPins = await buildRelationshipPinOperations(db); + if (relationshipPins.missing.length > 0) { + const error = new Error( + 'Cannot pin latest relationship endpoints because one or more referenced objects are missing', + ); + error.missing_relationship_endpoints = relationshipPins.missing; + throw error; + } + + if (!options.dryRun && relationshipPins.operations.length > 0) { + await db.collection('relationships').bulkWrite(relationshipPins.operations, { + ordered: false, + }); + await ensureManifestIndexes(db); + } + const manifests = await backfillSnapshotManifests(db, options); + + return { + relationships_scanned: relationshipPins.relationships.length, + relationship_pins_written: relationshipPins.operations.length, + ...manifests, + dry_run: options.dryRun === true, + }; +} + +module.exports = { + async up(db) { + const report = await run(db); + console.log( + `Pinned ${report.relationship_pins_written} latest relationship revision(s) and ` + + `created ${report.manifests_created} baseline snapshot manifest(s)`, + ); + }, + + async down(db) { + const baselineManifests = await db + .collection('releaseTrackGraphManifests') + .find({ baseline_reconstruction: true }) + .project({ manifest_id: 1, track_id: 1, snapshot_modified: 1, _id: 0 }) + .toArray(); + + await mapWithConcurrency(baselineManifests, async (manifest) => { + if (await db.listCollections({ name: manifest.track_id }, { nameOnly: true }).hasNext()) { + await db.collection(manifest.track_id).updateOne( + { + modified: manifest.snapshot_modified, + graph_manifest_id: manifest.manifest_id, + }, + { $unset: { graph_manifest_id: '' } }, + ); + } + }); + const manifestIds = baselineManifests.map((manifest) => manifest.manifest_id); + if (manifestIds.length > 0) { + await db + .collection('releaseTrackGraphManifestEntries') + .deleteMany({ manifest_id: { $in: manifestIds } }); + await db + .collection('releaseTrackGraphManifests') + .deleteMany({ manifest_id: { $in: manifestIds } }); + } + }, + + _private: { + run, + latestRelationships, + buildRelationshipPinOperations, + findTrackIds, + }, +}; diff --git a/package.json b/package.json index deeb337a..bec0dca8 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "test:scheduler": "mocha --timeout 60000 --recursive ./app/tests/scheduler --exit", "test:file": "mocha --timeout 10000 --exit", "repair:release-track-backrefs": "node scripts/reconcileReleaseTrackBackrefs.js", + "preview:deterministic-snapshot-graphs": "node scripts/previewDeterministicSnapshotGraphMigration.js", "check:lockfile": "bash scripts/check-package-lock.sh" }, "dependencies": { diff --git a/scripts/previewDeterministicSnapshotGraphMigration.js b/scripts/previewDeterministicSnapshotGraphMigration.js new file mode 100644 index 00000000..348298e5 --- /dev/null +++ b/scripts/previewDeterministicSnapshotGraphMigration.js @@ -0,0 +1,22 @@ +'use strict'; + +const mongoose = require('mongoose'); +const database = require('../app/lib/database-connection'); +const migration = require('../migrations/20260730180000-backfill-deterministic-snapshot-graphs'); + +async function main() { + await database.initializeConnection(); + const report = await migration._private.run(mongoose.connection.db, { + dryRun: true, + }); + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); +} + +main() + .catch((err) => { + process.stderr.write(`${err.stack || err.message}\n`); + process.exitCode = 1; + }) + .finally(async () => { + await mongoose.disconnect(); + });