retention interval

project/server/make/dev.mk

@@ -53,6 +53,7 @@ repl: build
 repl-nest: build
 	@ \
 		SIGNING_VOUCHER=$(shell $(call secret,voucher,signing-key)) \
+		SCHEDULER_ENABLE='false' \
 		node --enable-source-maps $(SERVER_OUT_REPL_NEST)
 
 start-container:

project/server/src/config/index.ts

@@ -52,6 +52,8 @@ export const config = () =>
 				/** how long a voucher is valid for — in seconds */
 				ttl: env.integer(required("SNAPSHOT_VOUCHER_TTL", floor(60 * 60 * 2))),
 			},
+			/** subsequent days with no submissions after which all data pertaining to a submitting instance is deleted */
+			revokeAfter: env.integer(required("SNAPSHOT_REVOKE_AFTER", floor(60))),
 			defer: {
 				target: env.choice(Schema.Enums(SnapshotDeferTarget))(
 					required("SNAPSHOT_DEFER_TARGET", SnapshotDeferTarget.None),

project/server/src/layer/scheduler/scheduled/compaction/optimize.module.ts

@@ -0,0 +1,11 @@
+import { Module } from "@nestjs/common";
+
+import { ModuleDatabase } from "../../../database/database.module";
+import { ServiceSchedulerScheduledCompactionOptimize } from "./optimize.service";
+
+@Module({
+	imports: [ModuleDatabase],
+	providers: [ServiceSchedulerScheduledCompactionOptimize],
+	exports: [ServiceSchedulerScheduledCompactionOptimize],
+})
+export class ModuleSchedulerScheduledCompactionOptimize {}

project/server/src/layer/scheduler/scheduled/compaction/optimize.service.ts

@@ -0,0 +1,56 @@
+import { Inject, Injectable } from "@nestjs/common";
+
+import { DatabaseStaging } from "../../../database/database.module";
+import { ServiceSchedulerScheduledCompactionOrphanedAttached } from "./orphaned/attached.service";
+import { ServiceSchedulerScheduledCompactionOrphanedSubmission } from "./orphaned/submission.service";
+
+import type { IDatabase } from "../../../../service/database";
+import type { SchedulerScheduled } from "../../../../service/scheduler/base";
+
+@Injectable()
+export class ServiceSchedulerScheduledCompactionOptimize
+	implements
+		SchedulerScheduled<typeof ServiceSchedulerScheduledCompactionOptimize>
+{
+	static readonly id = Symbol("ServiceSchedulerScheduledCompactionOptimize");
+
+	static readonly prerequisites = [
+		ServiceSchedulerScheduledCompactionOrphanedSubmission.id,
+		ServiceSchedulerScheduledCompactionOrphanedAttached.id,
+	];
+	static readonly schedule = {
+		minute: "0",
+		hour: "0",
+	} as const;
+
+	constructor(@Inject(DatabaseStaging) private db: IDatabase<"staging">) {}
+
+	async run(): Promise<void> {
+		await this.db.run({
+			query: "pragma wal_checkpoint(truncate);",
+			name: "Checkpoint",
+			parameters: [],
+			database: "staging",
+			connectionMode: "w",
+			integerMode: "number",
+			resultMode: "one",
+			rowMode: "object",
+		});
+
+		await this.db.run({
+			/*
+				https://sqlite.org/lang_analyze.html#periodically_run_pragma_optimize_
+				"The PRAGMA optimize command will normally only consider running ANALYZE on tables that have been previously queried by the same database connection or that do not have entries in the sqlite_stat1 table.
+        However, if the 0x10000 bit is added to the argument, PRAGMA optimize will examine all tables to see if they can benefit from ANALYZE, not just those that have been recently queried."
+        */
+			query: "pragma optimize=0x10000;",
+			name: "Optimize",
+			parameters: [],
+			database: "staging",
+			connectionMode: "w",
+			integerMode: "number",
+			resultMode: "one",
+			rowMode: "object",
+		});
+	}
+}

project/server/src/layer/scheduler/scheduled/compaction/orphaned/attached.module.ts

@@ -0,0 +1,11 @@
+import { Module } from "@nestjs/common";
+
+import { ModuleDatabase } from "../../../../database/database.module";
+import { ServiceSchedulerScheduledCompactionOrphanedAttached } from "./attached.service";
+
+@Module({
+	imports: [ModuleDatabase],
+	providers: [ServiceSchedulerScheduledCompactionOrphanedAttached],
+	exports: [ServiceSchedulerScheduledCompactionOrphanedAttached],
+})
+export class ModuleSchedulerScheduledCompactionOrphanedAttached {}

project/server/src/layer/scheduler/scheduled/compaction/orphaned/attached.service.test.ts

@@ -0,0 +1,200 @@
+import { randomBytes } from "node:crypto";
+import { type TestContext, test } from "node:test";
+
+import { testDatabase } from "../../../../../service/database/utility";
+import { StubIntrospection } from "../../../../../service/introspect/stub";
+import { Snapshot } from "../../../../../service/snapshot";
+import { Voucher } from "../../../../../service/voucher";
+import { floor } from "../../../../../type/codec/integer";
+import { uuid } from "../../../../../type/codec/uuid";
+import { unroll } from "../../../../../utility/iterable";
+import { omit } from "../../../../../utility/omit";
+import { ServiceSchedulerScheduledCompactionOrphanedAttached } from "./attached.service";
+
+import type { IDatabase } from "../../../../../service/database";
+
+const buildSnapshot = (database: IDatabase<"staging">) =>
+	new Snapshot(
+		database,
+		new StubIntrospection(),
+		new Voucher(randomBytes(64).toString()),
+		{
+			voucher: {
+				expectedAfter: floor(60 * 60 * 23),
+				ttl: floor(60 * 60 * 2),
+			},
+		},
+	);
+
+const device1 = {
+	entry_type: null,
+	has_configuration_url: false,
+	hw_version: null,
+	manufacturer: "Signify Netherlands B.V.",
+	model: "Hue white lamp",
+	model_id: "LWB010",
+	sw_version: "1.116.3",
+	via_device: null,
+} as const;
+
+const device2 = {
+	entry_type: null,
+	has_configuration_url: false,
+	hw_version: null,
+	manufacturer: "Signify Netherlands B.V.",
+	model: "Hue green lamp",
+	model_id: "LWB011",
+	sw_version: "1.116.4",
+	via_device: null,
+} as const;
+
+const entity1 = {
+	assumed_state: false,
+	domain: "light",
+	entity_category: null,
+	has_entity_name: true,
+	original_device_class: null,
+	unit_of_measurement: null,
+} as const;
+
+test("orphaned attachable", async (t: TestContext) => {
+	await using database = await testDatabase("staging", true);
+
+	const snapshot = buildSnapshot(database);
+	const service = new ServiceSchedulerScheduledCompactionOrphanedAttached(
+		database,
+	);
+
+	const subjectA = uuid();
+	{
+		const created = await snapshot.create(snapshot.voucher.initial(subjectA));
+		t.assert.strictEqual(created.kind, "success");
+
+		await snapshot.attach.device(created.handle, "foo", device1, [entity1]);
+		await snapshot.attach.device(created.handle, "bar", device2, [entity1]);
+
+		await snapshot.finalize(
+			created.handle,
+			{
+				version: 1,
+				hash: Buffer.alloc(32, 0xaa),
+			},
+			"2025.3.1",
+		);
+	}
+
+	const subjectB = uuid();
+	{
+		const created = await snapshot.create(snapshot.voucher.initial(subjectB));
+		t.assert.strictEqual(created.kind, "success");
+
+		await snapshot.attach.device(created.handle, "foo", device1, []);
+
+		await snapshot.finalize(
+			created.handle,
+			{
+				version: 1,
+				hash: Buffer.alloc(32, 0xab),
+			},
+			"2025.3.1",
+		);
+	}
+
+	t.assert.deepStrictEqual(
+		(await unroll(snapshot.staging.devices({ integration: "foo" }))).map(
+			(item) => omit(item, "id"),
+		),
+		[
+			{
+				integration: "foo",
+				manufacturer: device1.manufacturer,
+				model: device1.model,
+				modelId: device1.model_id,
+			},
+		],
+	);
+	t.assert.deepStrictEqual(
+		(await unroll(snapshot.staging.devices({ integration: "bar" }))).map(
+			(item) => omit(item, "id"),
+		),
+		[
+			{
+				integration: "bar",
+				manufacturer: device2.manufacturer,
+				model: device2.model,
+				modelId: device2.model_id,
+			},
+		],
+	);
+	t.assert.deepStrictEqual(
+		(await unroll(snapshot.staging.entities({ domain: "light" }))).map((item) =>
+			omit(item, "id"),
+		),
+		[
+			{
+				domain: entity1.domain,
+				assumedState: entity1.assumed_state,
+				hasName: entity1.has_entity_name,
+				category: entity1.entity_category ?? undefined,
+				originalDeviceClass: entity1.original_device_class ?? undefined,
+				unitOfMeasurement: entity1.unit_of_measurement ?? undefined,
+			},
+		],
+	);
+
+	let attributionA;
+	{
+		const all = await unroll(
+			snapshot.staging.attribution.submissions({ subject: subjectA }),
+		);
+		t.assert.deepEqual(all.length, 1);
+		attributionA = all[0];
+	}
+
+	await snapshot.delete(attributionA.submissionId);
+
+	await service.run();
+
+	t.assert.deepStrictEqual(
+		(await unroll(snapshot.staging.devices({ integration: "foo" }))).map(
+			(item) => omit(item, "id"),
+		),
+		[
+			{
+				integration: "foo",
+				manufacturer: device1.manufacturer,
+				model: device1.model,
+				modelId: device1.model_id,
+			},
+		],
+	);
+	t.assert.deepStrictEqual(
+		await unroll(snapshot.staging.devices({ integration: "bar" })),
+		[],
+	);
+	t.assert.deepStrictEqual(
+		await unroll(snapshot.staging.entities({ domain: "light" })),
+		[],
+	);
+
+	let attributionB;
+	{
+		const all = await unroll(
+			snapshot.staging.attribution.submissions({ subject: subjectB }),
+		);
+		t.assert.deepEqual(all.length, 1);
+		attributionB = all[0];
+	}
+	await snapshot.delete(attributionB.submissionId);
+
+	await service.run();
+
+	t.assert.deepStrictEqual(
+		await unroll(snapshot.staging.devices({ integration: "foo" })),
+		[],
+	);
+	t.assert.deepStrictEqual(
+		await unroll(snapshot.staging.devices({ integration: "bar" })),
+		[],
+	);
+});