Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ describe('Deterministic snapshot graph migration', function () {
let group;
let relationship;
let trackId;
const deprecatedDanglingRelationshipId = 'relationship--f7a41277-6599-49df-9567-82c9227fb8b5';
const activeDanglingRelationshipId = 'relationship--932fabf0-2868-46ed-9453-41e33dab7f39';
const missingEndpointId = 'campaign--5f4e747c-11d7-49ae-a947-a0f436879d62';

before(async function () {
await database.initializeConnection();
Expand Down Expand Up @@ -110,9 +113,65 @@ describe('Deterministic snapshot graph migration', function () {
ReleaseTrackGraphManifest.deleteMany({ track_id: trackId }),
ReleaseTrackGraphManifestEntry.deleteMany({ track_id: trackId }),
]);
await mongoose.connection.db.collection('relationships').insertOne({
workspace: {},
stix: {
type: 'relationship',
spec_version: '2.1',
id: deprecatedDanglingRelationshipId,
created: new Date(timestamp),
modified: new Date(timestamp),
relationship_type: 'uses',
source_ref: missingEndpointId,
target_ref: technique.stix.id,
revoked: false,
x_mitre_deprecated: true,
object_marking_refs: [markingDefinitionId],
},
});
});

it('fails closed when an active latest relationship has a dangling endpoint', async function () {
const timestamp = new Date();
await mongoose.connection.db.collection('relationships').insertOne({
workspace: {},
stix: {
type: 'relationship',
spec_version: '2.1',
id: activeDanglingRelationshipId,
created: timestamp,
modified: timestamp,
relationship_type: 'uses',
source_ref: group.stix.id,
target_ref: missingEndpointId,
revoked: false,
x_mitre_deprecated: false,
object_marking_refs: [markingDefinitionId],
},
});

try {
await expect(
migration._private.run(mongoose.connection.db, {
dryRun: true,
}),
).rejects.toMatchObject({
message: expect.stringContaining(activeDanglingRelationshipId),
missing_relationship_endpoints: [
expect.objectContaining({
relationship_ref: activeDanglingRelationshipId,
missing_endpoints: [missingEndpointId],
}),
],
});
} finally {
await mongoose.connection.db
.collection('relationships')
.deleteOne({ 'stix.id': activeDanglingRelationshipId });
}
});

it('supports a non-mutating dry run', async function () {
it('supports a non-mutating dry run with unrelated deprecated dangling data', async function () {
const report = await migration._private.run(mongoose.connection.db, {
dryRun: true,
});
Expand Down Expand Up @@ -145,6 +204,10 @@ describe('Deterministic snapshot graph migration', function () {
object_ref: technique.stix.id,
object_modified: new Date(technique.stix.modified),
});
const deprecatedDanglingRelationship = await mongoose.connection.db
.collection('relationships')
.findOne({ 'stix.id': deprecatedDanglingRelationshipId });
expect(deprecatedDanglingRelationship.workspace.relationship_endpoints).toBeUndefined();

const manifests = await ReleaseTrackGraphManifest.find({
track_id: trackId,
Expand Down
28 changes: 16 additions & 12 deletions docs/admin/release-track-graph-migration.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,21 +14,25 @@ DATABASE_URL='mongodb://host/database' \
npm run preview:deterministic-snapshot-graphs
```

The report includes the latest relationship revisions scanned, endpoint pins
that would be written, release-track snapshots found, and baseline manifests
that would be created. No database writes or indexes are created by this
command.

The preview fails if a latest relationship references a source or target
object that no longer exists. Repair those dangling endpoints before
deployment. Snapshot graph capture fails closed rather than silently producing
an incomplete deterministic baseline.
The report includes the latest active relationship revisions scanned, endpoint
pins that would be written, release-track snapshots found, and baseline
manifests that would be created. No database writes or indexes are created by
this command.

The preview fails if an active latest relationship references a source or
target object that no longer exists. The error identifies the affected
relationship and missing endpoint IDs; repair those dangling endpoints before
deployment. Deprecated and revoked relationships are not eligible for bundle
graphs, so the migration leaves that inactive legacy history untouched.
Snapshot graph capture fails closed rather than silently producing an
incomplete deterministic baseline.

## What the migration writes

- Exact source and target revision metadata is added only to the latest
revision of each relationship in the underlying `relationships` collection.
`view.relationships.latest` may be used for discovery but is never written.
- Exact source and target revision metadata is added only to active latest
relationship revisions in the underlying `relationships` collection.
`view.relationships.latest.active` may be used for discovery but is never
written.
- Each existing release-track snapshot receives a graph manifest containing
its exact primary, relationship, secondary, supporting, and LinkById
dependencies.
Expand Down
23 changes: 23 additions & 0 deletions docs/developer/TODO.md
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,29 @@ revision on which that bundle depends.
Bruno request definitions require no transport change.
- [x] Run focused specs while iterating, then lint, OpenAPI validation, and the
complete `npm test` suite.

### Production-shaped migration repair

- [x] Scope legacy endpoint pinning to active latest relationships, matching
the relationship set eligible for deterministic snapshot graphs.
- [x] Preserve fail-closed behavior for active dangling relationships while
allowing deprecated or revoked dangling history to remain untouched.
- [x] Surface actionable missing-endpoint diagnostics in the migration preview
and startup failure.
- [x] Establish manifest indexes independently of whether relationship pin
updates happen to be required.
- [x] Add production-shaped migration regressions and update the operator
documentation.

Verification result (2026-07-30):

- The focused migration spec passes all 4 cases.
- A read-only preview against the restored production database scans 24,818
active relationship revisions without encountering the 118 dangling
endpoints confined to deprecated relationship history.
- Lint, formatting, diff checks, and the complete `npm test` suite pass; the
API suite passes all 972 cases.

- [x] Propose conventional commits split by independently reviewable
architectural slice; do not commit until requested.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@
const TRACK_COLLECTION_PATTERN =
/^release-track--[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
const CONCURRENCY = 4;
const ACTIVE_RELATIONSHIP_FILTER = {
'stix.x_mitre_deprecated': { $in: [null, false] },
'stix.revoked': { $in: [null, false] },
};
const ERROR_SAMPLE_LIMIT = 10;

async function mapWithConcurrency(items, mapper) {
let nextIndex = 0;
Expand All @@ -29,11 +34,18 @@ async function mapWithConcurrency(items, mapper) {
}

async function latestRelationships(db) {
const latestActiveViewExists = await db
.listCollections({ name: 'view.relationships.latest.active' }, { nameOnly: true })
.hasNext();
if (latestActiveViewExists) {
return db.collection('view.relationships.latest.active').find({}).toArray();
}

const latestViewExists = await db
.listCollections({ name: 'view.relationships.latest' }, { nameOnly: true })
.hasNext();
if (latestViewExists) {
return db.collection('view.relationships.latest').find({}).toArray();
return db.collection('view.relationships.latest').find(ACTIVE_RELATIONSHIP_FILTER).toArray();
}

return db
Expand All @@ -42,6 +54,7 @@ async function latestRelationships(db) {
{ $sort: { 'stix.id': 1, 'stix.modified': -1 } },
{ $group: { _id: '$stix.id', document: { $first: '$$ROOT' } } },
{ $replaceRoot: { newRoot: '$document' } },
{ $match: ACTIVE_RELATIONSHIP_FILTER },
])
.toArray();
}
Expand Down Expand Up @@ -109,6 +122,21 @@ async function buildRelationshipPinOperations(db) {
return { relationships, operations, missing };
}

function missingEndpointError(missing) {
const sample = missing
.slice(0, ERROR_SAMPLE_LIMIT)
.map((entry) => `${entry.relationship_ref} -> ${entry.missing_endpoints.join(', ')}`)
.join('; ');
const remaining = missing.length - ERROR_SAMPLE_LIMIT;
const suffix = remaining > 0 ? `; and ${remaining} more` : '';
const error = new Error(
`Cannot pin ${missing.length} active latest relationship(s) because referenced objects are ` +
`missing: ${sample}${suffix}`,
);
error.missing_relationship_endpoints = missing;
return error;
}

async function findTrackIds(db) {
const [registeredTracks, collections] = await Promise.all([
db.collection('releaseTrackRegistry').find({}).project({ track_id: 1, _id: 0 }).toArray(),
Expand Down Expand Up @@ -206,17 +234,15 @@ async function backfillSnapshotManifests(db, options) {
async function run(db, options = {}) {
const relationshipPins = await buildRelationshipPinOperations(db);
if (relationshipPins.missing.length > 0) {
const error = new Error(
'Cannot pin latest relationship endpoints because one or more referenced objects are missing',
);
error.missing_relationship_endpoints = relationshipPins.missing;
throw error;
throw missingEndpointError(relationshipPins.missing);
}

if (!options.dryRun && relationshipPins.operations.length > 0) {
await db.collection('relationships').bulkWrite(relationshipPins.operations, {
ordered: false,
});
}
if (!options.dryRun) {
await ensureManifestIndexes(db);
}
const manifests = await backfillSnapshotManifests(db, options);
Expand All @@ -233,7 +259,7 @@ module.exports = {
async up(db) {
const report = await run(db);
console.log(
`Pinned ${report.relationship_pins_written} latest relationship revision(s) and ` +
`Pinned ${report.relationship_pins_written} active latest relationship revision(s) and ` +
`created ${report.manifests_created} baseline snapshot manifest(s)`,
);
},
Expand Down
9 changes: 9 additions & 0 deletions scripts/previewDeterministicSnapshotGraphMigration.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,15 @@ async function main() {
main()
.catch((err) => {
process.stderr.write(`${err.stack || err.message}\n`);
if (err.missing_relationship_endpoints) {
process.stderr.write(
`${JSON.stringify(
{ missing_relationship_endpoints: err.missing_relationship_endpoints },
null,
2,
)}\n`,
);
}
process.exitCode = 1;
})
.finally(async () => {
Expand Down
Loading