diff --git a/app/tests/api/release-tracks/deterministic-graph-migration.spec.js b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js index 7a149c2e..9e4689e2 100644 --- a/app/tests/api/release-tracks/deterministic-graph-migration.spec.js +++ b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js @@ -25,6 +25,9 @@ describe('Deterministic snapshot graph migration', function () { let group; let relationship; let trackId; + const deprecatedDanglingRelationshipId = 'relationship--f7a41277-6599-49df-9567-82c9227fb8b5'; + const activeDanglingRelationshipId = 'relationship--932fabf0-2868-46ed-9453-41e33dab7f39'; + const missingEndpointId = 'campaign--5f4e747c-11d7-49ae-a947-a0f436879d62'; before(async function () { await database.initializeConnection(); @@ -110,9 +113,65 @@ describe('Deterministic snapshot graph migration', function () { ReleaseTrackGraphManifest.deleteMany({ track_id: trackId }), ReleaseTrackGraphManifestEntry.deleteMany({ track_id: trackId }), ]); + await mongoose.connection.db.collection('relationships').insertOne({ + workspace: {}, + stix: { + type: 'relationship', + spec_version: '2.1', + id: deprecatedDanglingRelationshipId, + created: new Date(timestamp), + modified: new Date(timestamp), + relationship_type: 'uses', + source_ref: missingEndpointId, + target_ref: technique.stix.id, + revoked: false, + x_mitre_deprecated: true, + object_marking_refs: [markingDefinitionId], + }, + }); + }); + + it('fails closed when an active latest relationship has a dangling endpoint', async function () { + const timestamp = new Date(); + await mongoose.connection.db.collection('relationships').insertOne({ + workspace: {}, + stix: { + type: 'relationship', + spec_version: '2.1', + id: activeDanglingRelationshipId, + created: timestamp, + modified: timestamp, + relationship_type: 'uses', + source_ref: group.stix.id, + target_ref: missingEndpointId, + revoked: false, + x_mitre_deprecated: false, + object_marking_refs: [markingDefinitionId], + }, + }); + + try { + await expect( + migration._private.run(mongoose.connection.db, { + dryRun: true, + }), + ).rejects.toMatchObject({ + message: expect.stringContaining(activeDanglingRelationshipId), + missing_relationship_endpoints: [ + expect.objectContaining({ + relationship_ref: activeDanglingRelationshipId, + missing_endpoints: [missingEndpointId], + }), + ], + }); + } finally { + await mongoose.connection.db + .collection('relationships') + .deleteOne({ 'stix.id': activeDanglingRelationshipId }); + } }); - it('supports a non-mutating dry run', async function () { + it('supports a non-mutating dry run with unrelated deprecated dangling data', async function () { const report = await migration._private.run(mongoose.connection.db, { dryRun: true, }); @@ -145,6 +204,10 @@ describe('Deterministic snapshot graph migration', function () { object_ref: technique.stix.id, object_modified: new Date(technique.stix.modified), }); + const deprecatedDanglingRelationship = await mongoose.connection.db + .collection('relationships') + .findOne({ 'stix.id': deprecatedDanglingRelationshipId }); + expect(deprecatedDanglingRelationship.workspace.relationship_endpoints).toBeUndefined(); const manifests = await ReleaseTrackGraphManifest.find({ track_id: trackId, diff --git a/docs/admin/release-track-graph-migration.md b/docs/admin/release-track-graph-migration.md index 227e562f..91855ca5 100644 --- a/docs/admin/release-track-graph-migration.md +++ b/docs/admin/release-track-graph-migration.md @@ -14,21 +14,25 @@ DATABASE_URL='mongodb://host/database' \ npm run preview:deterministic-snapshot-graphs ``` -The report includes the latest relationship revisions scanned, endpoint pins -that would be written, release-track snapshots found, and baseline manifests -that would be created. No database writes or indexes are created by this -command. - -The preview fails if a latest relationship references a source or target -object that no longer exists. Repair those dangling endpoints before -deployment. Snapshot graph capture fails closed rather than silently producing -an incomplete deterministic baseline. +The report includes the latest active relationship revisions scanned, endpoint +pins that would be written, release-track snapshots found, and baseline +manifests that would be created. No database writes or indexes are created by +this command. + +The preview fails if an active latest relationship references a source or +target object that no longer exists. The error identifies the affected +relationship and missing endpoint IDs; repair those dangling endpoints before +deployment. Deprecated and revoked relationships are not eligible for bundle +graphs, so the migration leaves that inactive legacy history untouched. +Snapshot graph capture fails closed rather than silently producing an +incomplete deterministic baseline. ## What the migration writes -- Exact source and target revision metadata is added only to the latest - revision of each relationship in the underlying `relationships` collection. - `view.relationships.latest` may be used for discovery but is never written. +- Exact source and target revision metadata is added only to active latest + relationship revisions in the underlying `relationships` collection. + `view.relationships.latest.active` may be used for discovery but is never + written. - Each existing release-track snapshot receives a graph manifest containing its exact primary, relationship, secondary, supporting, and LinkById dependencies. diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 1ad59401..591677be 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -258,6 +258,29 @@ revision on which that bundle depends. Bruno request definitions require no transport change. - [x] Run focused specs while iterating, then lint, OpenAPI validation, and the complete `npm test` suite. + +### Production-shaped migration repair + +- [x] Scope legacy endpoint pinning to active latest relationships, matching + the relationship set eligible for deterministic snapshot graphs. +- [x] Preserve fail-closed behavior for active dangling relationships while + allowing deprecated or revoked dangling history to remain untouched. +- [x] Surface actionable missing-endpoint diagnostics in the migration preview + and startup failure. +- [x] Establish manifest indexes independently of whether relationship pin + updates happen to be required. +- [x] Add production-shaped migration regressions and update the operator + documentation. + +Verification result (2026-07-30): + +- The focused migration spec passes all 4 cases. +- A read-only preview against the restored production database scans 24,818 + active relationship revisions without encountering the 118 dangling + endpoints confined to deprecated relationship history. +- Lint, formatting, diff checks, and the complete `npm test` suite pass; the + API suite passes all 972 cases. + - [x] Propose conventional commits split by independently reviewable architectural slice; do not commit until requested. diff --git a/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js b/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js index 2643e96d..ff5ba792 100644 --- a/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js +++ b/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js @@ -14,6 +14,11 @@ const TRACK_COLLECTION_PATTERN = /^release-track--[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; const CONCURRENCY = 4; +const ACTIVE_RELATIONSHIP_FILTER = { + 'stix.x_mitre_deprecated': { $in: [null, false] }, + 'stix.revoked': { $in: [null, false] }, +}; +const ERROR_SAMPLE_LIMIT = 10; async function mapWithConcurrency(items, mapper) { let nextIndex = 0; @@ -29,11 +34,18 @@ async function mapWithConcurrency(items, mapper) { } async function latestRelationships(db) { + const latestActiveViewExists = await db + .listCollections({ name: 'view.relationships.latest.active' }, { nameOnly: true }) + .hasNext(); + if (latestActiveViewExists) { + return db.collection('view.relationships.latest.active').find({}).toArray(); + } + const latestViewExists = await db .listCollections({ name: 'view.relationships.latest' }, { nameOnly: true }) .hasNext(); if (latestViewExists) { - return db.collection('view.relationships.latest').find({}).toArray(); + return db.collection('view.relationships.latest').find(ACTIVE_RELATIONSHIP_FILTER).toArray(); } return db @@ -42,6 +54,7 @@ async function latestRelationships(db) { { $sort: { 'stix.id': 1, 'stix.modified': -1 } }, { $group: { _id: '$stix.id', document: { $first: '$$ROOT' } } }, { $replaceRoot: { newRoot: '$document' } }, + { $match: ACTIVE_RELATIONSHIP_FILTER }, ]) .toArray(); } @@ -109,6 +122,21 @@ async function buildRelationshipPinOperations(db) { return { relationships, operations, missing }; } +function missingEndpointError(missing) { + const sample = missing + .slice(0, ERROR_SAMPLE_LIMIT) + .map((entry) => `${entry.relationship_ref} -> ${entry.missing_endpoints.join(', ')}`) + .join('; '); + const remaining = missing.length - ERROR_SAMPLE_LIMIT; + const suffix = remaining > 0 ? `; and ${remaining} more` : ''; + const error = new Error( + `Cannot pin ${missing.length} active latest relationship(s) because referenced objects are ` + + `missing: ${sample}${suffix}`, + ); + error.missing_relationship_endpoints = missing; + return error; +} + async function findTrackIds(db) { const [registeredTracks, collections] = await Promise.all([ db.collection('releaseTrackRegistry').find({}).project({ track_id: 1, _id: 0 }).toArray(), @@ -206,17 +234,15 @@ async function backfillSnapshotManifests(db, options) { async function run(db, options = {}) { const relationshipPins = await buildRelationshipPinOperations(db); if (relationshipPins.missing.length > 0) { - const error = new Error( - 'Cannot pin latest relationship endpoints because one or more referenced objects are missing', - ); - error.missing_relationship_endpoints = relationshipPins.missing; - throw error; + throw missingEndpointError(relationshipPins.missing); } if (!options.dryRun && relationshipPins.operations.length > 0) { await db.collection('relationships').bulkWrite(relationshipPins.operations, { ordered: false, }); + } + if (!options.dryRun) { await ensureManifestIndexes(db); } const manifests = await backfillSnapshotManifests(db, options); @@ -233,7 +259,7 @@ module.exports = { async up(db) { const report = await run(db); console.log( - `Pinned ${report.relationship_pins_written} latest relationship revision(s) and ` + + `Pinned ${report.relationship_pins_written} active latest relationship revision(s) and ` + `created ${report.manifests_created} baseline snapshot manifest(s)`, ); }, diff --git a/scripts/previewDeterministicSnapshotGraphMigration.js b/scripts/previewDeterministicSnapshotGraphMigration.js index 348298e5..60ffb74b 100644 --- a/scripts/previewDeterministicSnapshotGraphMigration.js +++ b/scripts/previewDeterministicSnapshotGraphMigration.js @@ -15,6 +15,15 @@ async function main() { main() .catch((err) => { process.stderr.write(`${err.stack || err.message}\n`); + if (err.missing_relationship_endpoints) { + process.stderr.write( + `${JSON.stringify( + { missing_relationship_endpoints: err.missing_relationship_endpoints }, + null, + 2, + )}\n`, + ); + } process.exitCode = 1; }) .finally(async () => {