From c2cc3d0ba1014d0cd2b42b6b9ad5b904b2268a88 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 12:09:13 +0200 Subject: [PATCH 01/41] feat(cmdb-import): read the archive sheet and let module and usage be archived Adds the OpenSpec change cmdb-import-archive-reconciliation (REQ-CMDB-015..017, REQ-CMDB-012 and REQ-CMDB-014 modified). The register fragment opts module (0.3.9) and usage (1.5.7) into OpenRegister's archive state, the import profile names the sheet "Gearchiveerde Applicaties" and the accepted missingRecords modes, and the workbook reader returns the APPIDs of that sheet, read under the same bounds as the CMDB sheets for its key column only. The archived-applications fixture covers it. First of three slices of #1281; nothing reads the archive sheet yet. Co-Authored-By: Claude Opus 5.5 --- lib/Service/Cmdb/CmdbImportProfile.php | 55 ++++- lib/Service/Cmdb/CmdbWorkbookReader.php | 151 ++++++++++--- lib/Settings/cmdb-import/topdesk-profile.json | 7 +- .../register.d/topdesk-cmdb-import.json | 8 +- .../.openspec.yaml | 2 + .../design.md | 207 ++++++++++++++++++ .../proposal.md | 95 ++++++++ .../specs/cmdb-export-import/spec.md | 191 ++++++++++++++++ .../tasks.md | 89 ++++++++ openspec/specs/cmdb-export-import/spec.md | 5 +- .../Unit/Fixtures/CmdbFixtureHygieneTest.php | 2 +- .../Service/Cmdb/CmdbImportProfileTest.php | 24 ++ .../Service/Cmdb/CmdbWorkbookReaderTest.php | 104 +++++++++ .../Settings/PublicationFieldRulesTest.php | 2 +- .../Unit/Settings/TopdeskCmdbFragmentTest.php | 30 ++- tests/fixtures/cmdb/README.md | 4 +- tests/fixtures/cmdb/build-fixtures.py | 46 ++++ .../cmdb/topdesk-archived-applications.xlsx | Bin 0 -> 151786 bytes 18 files changed, 970 insertions(+), 52 deletions(-) create mode 100644 openspec/changes/cmdb-import-archive-reconciliation/.openspec.yaml create mode 100644 openspec/changes/cmdb-import-archive-reconciliation/design.md create mode 100644 openspec/changes/cmdb-import-archive-reconciliation/proposal.md create mode 100644 openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md create mode 100644 openspec/changes/cmdb-import-archive-reconciliation/tasks.md create mode 100644 tests/fixtures/cmdb/topdesk-archived-applications.xlsx diff --git a/lib/Service/Cmdb/CmdbImportProfile.php b/lib/Service/Cmdb/CmdbImportProfile.php index 1e8805c90..183c02fe6 100644 --- a/lib/Service/Cmdb/CmdbImportProfile.php +++ b/lib/Service/Cmdb/CmdbImportProfile.php @@ -332,6 +332,42 @@ public function sheetNames(): array { return array_column($this->sheets(), 'name'); }//end sheetNames() + /** + * The name of the archive sheet, or null when the profile names none. + * + * The archive sheet lists the applications the municipality archived; the + * import reads only its key column (`archiveSheet.name` in the profile). + * + * @return string|null + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function archiveSheetName(): ?string { + $name = $this->profile()['archiveSheet']['name'] ?? null; + if (is_string($name) === false || $name === '' || in_array($name, $this->sheetNames(), true) === true) { + return null; + } + + return $name; + }//end archiveSheetName() + + /** + * Every sheet the reader may parse: the source sheets, then the archive sheet. + * + * @return array + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function readSheetNames(): array { + $names = $this->sheetNames(); + $archive = $this->archiveSheetName(); + if ($archive !== null) { + $names[] = $archive; + } + + return $names; + }//end readSheetNames() + /** * The rank of a sheet when an APPID is on more than one: lower wins. * @@ -574,19 +610,28 @@ public function neverWrittenOnUpdate(string $target): array { }//end neverWrittenOnUpdate() /** - * The accepted values of the missingRecords option. + * The accepted values of the missingRecords option, readable without validating the packs. + * + * The controller checks the option before the import runs, so, like + * maxFileBytes(), this must not depend on OpenRegister being available. * * @return array * - * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012 */ public function missingRecordsModes(): array { - $modes = $this->stringList(key: 'missingRecords'); - if ($modes === []) { + try { + $profile = $this->profile ?? $this->decodeFile(fileName: $this->profileFile); + } catch (CmdbImportException $e) { + return ['keep']; + } + + $modes = $profile['missingRecords'] ?? []; + if (is_array($modes) === false || $modes === []) { return ['keep']; } - return $modes; + return array_values(array_map('strval', $modes)); }//end missingRecordsModes() /** diff --git a/lib/Service/Cmdb/CmdbWorkbookReader.php b/lib/Service/Cmdb/CmdbWorkbookReader.php index 3ff3e9184..9bdf9b9a6 100644 --- a/lib/Service/Cmdb/CmdbWorkbookReader.php +++ b/lib/Service/Cmdb/CmdbWorkbookReader.php @@ -42,6 +42,9 @@ * beyond twice the row limit is `TOO_MANY_ROWS`. A read filter then materialises only the * header row and the resolved columns of the rows up to that bound * (CmdbReadFilter), which bounds the cell objects, not the parse. + * 7. The archive sheet ("Gearchiveerde Applicaties"), when the workbook has + * it, counts as a read sheet for every bound above, and only its key + * column is resolved and read (openspec/changes/cmdb-import-archive-reconciliation). * * @category Service * @package OCA\Stackiq\Service\Cmdb @@ -151,17 +154,21 @@ public function isAvailable(): bool { * The sheets are then read twice through a read filter: once for the * header row, once for the resolved columns of the data rows, so no other * cell is ever materialised. + * The archive sheet, when the workbook has it, is read under the same + * bounds as a source sheet, for its key column only. * * @param string $path The xlsx file, already checked by assertXlsx(). * @param CmdbImportProfile $profile The import profile. * * @return array `rows` (list of {sheet, row, cells, uncached}), `importWarnings` - * (list of {sheet, message}) and `date1904` (bool). + * (list of {sheet, message}), `date1904` (bool) and `archive` + * ({sheet, present, keyColumnMissing, appIds}). * * @throws CmdbImportException WORKBOOK_TOO_LARGE, READER_UNAVAILABLE, NOT_XLSX, NO_SOURCE_SHEET, * MISSING_COLUMN or TOO_MANY_ROWS. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 */ public function read(string $path, CmdbImportProfile $profile): array { $oversized = $this->assertUncompressedSize(path: $path, limit: $profile->maxUncompressedBytes(), partLimit: $profile->maxPartBytes()); @@ -177,7 +184,7 @@ public function read(string $path, CmdbImportProfile $profile): array { $blank = (new CmdbPartReferences())->assertPartSizes( path: $path, oversized: $oversized, - sourceSheets: $profile->sheetNames(), + sourceSheets: $profile->readSheetNames(), limit: $profile->maxPartBytes(), scanner: $scanner ); @@ -303,10 +310,81 @@ private function readBounded(string $path, CmdbImportProfile $profile, object $s $bounds->assertReferencedStringBytes( path: $path, limit: $profile->maxReferencedStringBytes(), - sheetCount: count($profile->sheetNames()), + sheetCount: count($profile->readSheetNames()), scanner: $scanner ); + ['present' => $present, 'available' => $available] = $this->sourceSheets(path: $path, profile: $profile); + + $archiveSheet = $profile->archiveSheetName(); + $loaded = $present; + if ($archiveSheet !== null && in_array($archiveSheet, $available, true) === true) { + $loaded[] = $archiveSheet; + } + + $limit = $profile->maxRowsPerSheet(); + $lastRow = self::lastReadableRow(limit: $limit); + $this->assertRowSpan(path: $path, sheetNames: $loaded, lastRow: $lastRow, limit: $limit); + + $headers = $this->load(path: $path, sheetNames: $loaded, filter: new CmdbReadFilter(lastRow: 1)); + try { + $resolved = $this->resolveSheets(spreadsheet: $headers, sheetNames: $present, profile: $profile); + $archiveColumns = []; + if ($loaded !== $present) { + // Only the key column of the archive sheet is read; the allowlist does not grow. + $archiveColumns = $this->resolveColumns(worksheet: $headers->getSheetByName($archiveSheet), referenced: [$profile->keyColumn()]); + } + } finally { + $headers->disconnectWorksheets(); + } + + // A sheet without the key column is not read; the import reports it as missing its key column. + $archive = [ + 'sheet' => (string)$archiveSheet, + 'present' => false, + 'keyColumnMissing' => ($loaded !== $present && $archiveColumns === []), + 'appIds' => [], + ]; + $letters = array_map(static fn (array $columns): array => array_keys($columns), $resolved['columns']); + $loaded = $present; + if ($archiveColumns !== []) { + $letters[$archiveSheet] = array_keys($archiveColumns); + $loaded[] = $archiveSheet; + } + + $spreadsheet = $this->load(path: $path, sheetNames: $loaded, filter: new CmdbReadFilter(lastRow: $lastRow, columns: $letters)); + try { + $rows = $this->readSourceRows(spreadsheet: $spreadsheet, sheetNames: $present, columnsPerSheet: $resolved['columns'], limit: $limit); + if ($archiveColumns !== []) { + $archiveRows = $this->readRows( + worksheet: $spreadsheet->getSheetByName($archiveSheet), + columns: $archiveColumns, + sheetName: (string)$archiveSheet, + limit: $limit + ); + $archive['present'] = true; + $archive['appIds'] = array_column(array_column($archiveRows, 'cells'), $profile->keyColumn()); + } + + $date1904 = self::usesDate1904(spreadsheet: $spreadsheet); + } finally { + $spreadsheet->disconnectWorksheets(); + } + + return ['rows' => $rows, 'importWarnings' => $resolved['warnings'], 'date1904' => $date1904, 'archive' => $archive]; + }//end readBounded() + + /** + * The profile's source sheets that the workbook has, and every sheet it has. + * + * @param string $path The xlsx file. + * @param CmdbImportProfile $profile The import profile. + * + * @return array{present: array, available: array} + * + * @throws CmdbImportException NOT_XLSX when the sheet list cannot be read, NO_SOURCE_SHEET when no source sheet is there. + */ + private function sourceSheets(string $path, CmdbImportProfile $profile): array { $reader = $this->newReader(sheetNames: null); try { $available = $reader->listWorksheetNames($path); @@ -328,41 +406,50 @@ private function readBounded(string $path, CmdbImportProfile $profile, object $s ); } - $limit = $profile->maxRowsPerSheet(); - $lastRow = self::lastReadableRow(limit: $limit); - $this->assertRowSpan(path: $path, sheetNames: $present, lastRow: $lastRow, limit: $limit); + return ['present' => $present, 'available' => $available]; + }//end sourceSheets() - $headers = $this->load(path: $path, sheetNames: $present, filter: new CmdbReadFilter(lastRow: 1)); - try { - $resolved = $this->resolveSheets(spreadsheet: $headers, sheetNames: $present, profile: $profile); - } finally { - $headers->disconnectWorksheets(); + /** + * The data rows of the source sheets, in sheet order. + * + * @param object $spreadsheet The spreadsheet, loaded through the data-pass read filter. + * @param array $sheetNames The source sheets the workbook has. + * @param array> $columnsPerSheet Sheet name => column letter => column name. + * @param int $limit The row limit per sheet. + * + * @return array> See read(). + * + * @throws CmdbImportException TOO_MANY_ROWS. + */ + private function readSourceRows(object $spreadsheet, array $sheetNames, array $columnsPerSheet, int $limit): array { + $rows = []; + foreach ($sheetNames as $sheetName) { + $sheetRows = $this->readRows( + worksheet: $spreadsheet->getSheetByName($sheetName), + columns: $columnsPerSheet[$sheetName], + sheetName: $sheetName, + limit: $limit + ); + array_push($rows, ...$sheetRows); } - $letters = array_map(static fn (array $columns): array => array_keys($columns), $resolved['columns']); - $spreadsheet = $this->load(path: $path, sheetNames: $present, filter: new CmdbReadFilter(lastRow: $lastRow, columns: $letters)); - try { - $rows = []; - foreach ($present as $sheetName) { - $sheetRows = $this->readRows( - worksheet: $spreadsheet->getSheetByName($sheetName), - columns: $resolved['columns'][$sheetName], - sheetName: $sheetName, - limit: $limit - ); - array_push($rows, ...$sheetRows); - } + return $rows; + }//end readSourceRows() - $date1904 = false; - if (method_exists($spreadsheet, 'getExcelCalendar') === true) { - $date1904 = ((int)$spreadsheet->getExcelCalendar() === 1904); - } - } finally { - $spreadsheet->disconnectWorksheets(); + /** + * Whether the workbook uses the 1904 date system. + * + * @param object $spreadsheet The spreadsheet. + * + * @return bool + */ + private static function usesDate1904(object $spreadsheet): bool { + if (method_exists($spreadsheet, 'getExcelCalendar') === false) { + return false; } - return ['rows' => $rows, 'importWarnings' => $resolved['warnings'], 'date1904' => $date1904]; - }//end readBounded() + return ((int)$spreadsheet->getExcelCalendar() === 1904); + }//end usesDate1904() /** * The last row number the data pass reads. diff --git a/lib/Settings/cmdb-import/topdesk-profile.json b/lib/Settings/cmdb-import/topdesk-profile.json index 79dc43253..a6b3df2d3 100644 --- a/lib/Settings/cmdb-import/topdesk-profile.json +++ b/lib/Settings/cmdb-import/topdesk-profile.json @@ -1,8 +1,8 @@ { "id": "topdesk-cmdb", "name": "TOPdesk CMDB export", - "version": "2.0.0", - "description": "How stackiq reads a TOPdesk CMDB export (xlsx): the two CMDB sheets, the key and required columns, date and id columns, values that mean empty, the pack per target and the limits. Columns that neither this profile nor a pack names are never read.", + "version": "2.1.0", + "description": "How stackiq reads a TOPdesk CMDB export (xlsx): the two CMDB sheets, the archive sheet (APPID column only), the key and required columns, date and id columns, values that mean empty, the pack per target, the accepted missingRecords modes and the limits. Columns that neither this profile nor a pack names are never read.", "maxFileBytes": 10485760, "maxRowsPerSheet": 10000, "maxUncompressedBytes": 52428800, @@ -21,6 +21,7 @@ } ], "sheetPrecedence": ["Beheerde Applicaties CMDB", "Onbeh Applicaties CMDB"], + "archiveSheet": { "name": "Gearchiveerde Applicaties" }, "keyColumn": "APPID", "nameColumn": "Applicatie Naam", "requiredColumns": ["APPID", "Applicatie Naam"], @@ -44,5 +45,5 @@ "neverWritten": { "module": ["publicationDate", "depublicationDate"] }, - "missingRecords": ["keep"] + "missingRecords": ["keep", "archive"] } diff --git a/lib/Settings/register.d/topdesk-cmdb-import.json b/lib/Settings/register.d/topdesk-cmdb-import.json index 5fb300099..977c5db24 100644 --- a/lib/Settings/register.d/topdesk-cmdb-import.json +++ b/lib/Settings/register.d/topdesk-cmdb-import.json @@ -2,7 +2,10 @@ "components": { "schemas": { "module": { - "version": "0.3.8", + "version": "0.3.9", + "configuration": { + "x-openregister-archive": { "enabled": true } + }, "properties": { "externalId": { "type": "string", @@ -72,8 +75,9 @@ } }, "usage": { - "version": "1.5.6", + "version": "1.5.7", "configuration": { + "x-openregister-archive": { "enabled": true }, "x-openregister-lifecycle": { "transitions": { "importAcquisition": { diff --git a/openspec/changes/cmdb-import-archive-reconciliation/.openspec.yaml b/openspec/changes/cmdb-import-archive-reconciliation/.openspec.yaml new file mode 100644 index 000000000..5c0f6674b --- /dev/null +++ b/openspec/changes/cmdb-import-archive-reconciliation/.openspec.yaml @@ -0,0 +1,2 @@ +schema: conduction +created: 2026-10-09 diff --git a/openspec/changes/cmdb-import-archive-reconciliation/design.md b/openspec/changes/cmdb-import-archive-reconciliation/design.md new file mode 100644 index 000000000..f71cfcce6 --- /dev/null +++ b/openspec/changes/cmdb-import-archive-reconciliation/design.md @@ -0,0 +1,207 @@ +# Design: cmdb-import-archive-reconciliation + +## Context + +The CMDB import (archived change `2026-10-05-cmdb-export-import`) matches rows on `module.externalKey` = `topdesk::` and leaves records that are missing from a newer export untouched (REQ-CMDB-012). The municipality's export carries, next to the two CMDB sheets, a sheet "Gearchiveerde Applicaties" (columns Applicatiecomponent, APPID, Applicatie Code, Applicatie Naam, Roepnaam, Applicatie Status, Bron, Datum Interface, Datum wijziging), derived from the raw "Invoer gearchiveerde appl" sheet. The municipality wants an application that moved there to be archived, and one that is on no sheet any more to be soft-deleted (2026-10-01). + +What OpenRegister offers (origin/beta `ff4dad5c`, identical to development for `lib/`): + +- **Soft delete.** `ObjectServiceInterface::deleteObject(uuid, register, schema, ...)` writes `@self.deleted` (`deletedBy`, `deletedAt`, `purgeDate` 30 days on). Lists exclude deleted objects unless `_includeDeleted=true`, but the list search cannot return one even then (D9). `MagicMapper::findDeletedAcrossAllMagicTables()` lists the trash; `MagicMapper::restoreObject(uuid)` clears the marker. `DeletedController` uses both. +- **Archive.** `ObjectEntity::archive()` / `unarchive()` write and clear `@self.archived` (`by`, `at`, `reason`). `Service\Object\ArchiveHandler::archive()` / `unarchive()` is the service path behind `POST` / `DELETE /api/objects/{register}/{schema}/{id}/archive`: it resolves the object, refuses a schema without `x-openregister-archive: {enabled: true}` (`ArchiveNotOfferedException`), checks `update` on the object for the session user, persists the marker and writes an audit entry. Lists exclude archived objects unless `_archived=true` (only the archive) or `_archived=any`. A read by identifier is unaffected. `SaveObject` refuses a data write to an archived object (`ObjectStateWriteException`). +- **Search lenses** combine in SQL: `_ids`, `_includeDeleted` and `_archived` apply together in `MagicSearchHandler`. Its row conversion, however, drops a soft-deleted row and hydrates no archive marker (D9). + +OpenCatalogi (beta `df25ff75`) passes list parameters through to OpenRegister and forces `_includeDeleted=false`; it has no archive filter of its own, so `_archived=true` reaches OpenRegister unchanged. Portaliq (beta `8287e390`) lists usages with neither lens. Both hide archived and deleted records as they are (decided 2026-10-09). + +## Goals / Non-Goals + +**Goals** + +- A re-import archives what moved to the archive sheet and soft-deletes what left the export entirely, for the imported municipality only. +- A returning application is revived, never duplicated. +- The admin sees and chooses the behaviour, and the report accounts for every archived, unarchived, deleted and restored application. +- Archived applications and usages can be listed in stackiq on request and stay out of OpenCatalogi and Portaliq without a change there. + +**Non-Goals** + +- Hard deletion, the trash's retention, contact persons, suppliers, modules another organisation imported (a module carrying this municipality's import key is this municipality's record, also when another organisation uses it; D5), changes to OpenCatalogi or Portaliq, an archive widget on the detail pages. + +## Architecture Overview + +``` +CmdbImportController::import() missingRecords: keep | archive (default archive); remove → 422 + ▼ +CmdbExportImportService::import() + ├─ resolve ArchiveHandler + MagicMapper (archive mode only; missing → 503 ARCHIVE_UNAVAILABLE) + ├─ lock the register + ├─ CmdbWorkbookReader::read() CMDB sheets as before + the archive sheet's APPID column + ├─ rows, each in its own boundary match module and usage: working, then _archived=true, then the trash (D9) + │ archived match → unarchive, deleted match → restore (and unarchive), then the usual update + └─ reconcile() (archive mode, not cancelled) phase `reconciling`, same lock + page usages (consumer = municipality): working, then _archived=true; deleted ones from the trash + load their modules in batches (_ids: working, then _archived=true; the rest from the trash) + per application: key on CMDB sheets → nothing + key on archive sheet → archive module + usage (restore first when deleted) + key nowhere → soft-delete module + usage, or archive when the sheet is absent + each in its own boundary → report rows archived | deleted | failed + ▼ +OpenRegister: module, usage (archived / deleted markers); contactPerson, organization untouched + ├─▶ OpenCatalogi: archived modules only with _archived=true + └─▶ Portaliq: archived and deleted usages disappear from "Software we use" +``` + +## Decisions + +### D1. "Archived" is OpenRegister's archive state on the module and on the usage + +Decided by the product owner on 2026-10-09. No new field, no new `usage.status` value, no `depublicationDate`. + +- **Alternative: `usage.status` `Archived`.** Rejected: OpenCatalogi drops `status === 'archived'` only on `/api/search`, so `/api/{catalogSlug}` and the federation route would need an OpenCatalogi change and a new beta. +- **Alternative: `module.depublicationDate`.** Rejected: the module read rule does not look at it, and the import never writes it (D6 of the first change). + +Both schemas opt in through the register fragment: `module` 0.3.8 → 0.3.9 and `usage` 1.5.6 → 1.5.7 get `configuration.x-openregister-archive: {"enabled": true}`. Fragments are merged by key union and the loader keeps the highest `version` per schema, so the bump deploys with the existing register import. The `SCHEMA_OUTDATED` check is not widened: an installation whose schemas predate the annotation gets `ArchiveNotOfferedException` per object, which the reconciliation reports as a failed row (D7), and the docs point to Force Update. + +### D2. The option: `keep | archive`, default `archive` + +`CmdbImportProfile::missingRecordsModes()` reads the accepted values from the profile (`["keep", "archive"]`) without validating the packs, like `maxFileBytes()`, because the controller checks the option before the import runs. The default lives in `CmdbExportImportService::DEFAULT_MISSING_RECORDS`. `remove` and `mark` stay reserved: 422 `MISSING_RECORDS_UNSUPPORTED` with `details.accepted` naming both accepted values. The section sends the admin's choice as `missingRecords` and defaults to `archive`. + +With `updateExisting=false`, a matched row is skipped as `exists` before anything is revived, as before: the admin asked to leave existing records alone. + +### D3. Reading the archive sheet + +The profile names the sheet (`archiveSheet.name`: "Gearchiveerde Applicaties"). The reader lists the workbook's sheets, and when the archive sheet is present it is loaded with the CMDB sheets under the same bounds: it counts as a read sheet for `CmdbPartReferences` (an oversized archive sheet is refused, not blanked) and for the referenced-string budget, its row span is bounded like a source sheet, and `TOO_MANY_ROWS` names it. Only its APPID column is resolved and read; the allowlist does not grow. The reader returns `archive: {present, keyColumnMissing, appIds}` next to `rows`. + +- Sheet absent, or present without an APPID column: `present` false (with `keyColumnMissing`), and the service adds a report warning. The reconciliation then archives every missing application and soft-deletes nothing: a workbook saved without the sheet must not empty the trash-bound set by accident. +- The archive sheet's other columns are not read; `Applicatie Status` there is always `Verwijderd` in the municipality's export and adds nothing. + +### D4. Matching widens to archived and deleted objects, and revives what it matches + +`matchModule()` looks the module up by `externalKey` in every state, as D9 reads it; `moduleBelongsTo()` looks up this municipality's usage the same way (a module whose usage is archived still belongs to the municipality), and keeps the working lens for "does any other organisation use it". `upsertUsage()` finds the usage in every state. Whatever is matched is revived before it is written: a soft-deleted object is restored (`MagicMapper::restoreObject()`), and unarchived as well when the restored object says it was archived; an archived one is unarchived (`ArchiveHandler::unarchive()`), because `SaveObject` refuses a write to an archived object. The update starts from the object `restoreObject()` answers, not from the trash listing's copy. The row outcome is `restored` when anything was restored, else `unarchived` when anything was unarchived, else the usual `created | updated | unchanged`. When more than one object matches, `findInAnyState()` takes a working one before an archived one, and an archived one before one in the trash: before this change a module deleted by hand was invisible to the match, so the next import created a second one with the same `externalKey`, and reviving the deleted one would make two live modules. Organisations and contact persons keep the working lens: a deleted supplier is never matched. + +### D5. The reconciliation step + +Runs inside `runImport()` after the row loop, only when the loop finished (not cancelled) and `missingRecords` is `archive`, under the register lock the import already holds. It checks for a cancel once before it loads the scope (a cancel that came in during the last row) and again before each application; a cancel stops it there, keeps what was already archived or deleted, and marks the report cancelled. An exception outside an object's boundary fails the run like any other (`IMPORT_FAILED`, operation `failed`), so a half-done reconciliation is visible. + +Scope: the usages whose `consumer` is the municipality, in every state as D9 reads it: paged with `PAGE_SIZE` without a lens and with `_archived=true`, plus the deleted ones from the trash. Their modules are loaded in chunks of `PAGE_SIZE` with `_ids`, without a lens and with `_archived=true`, and the rest from the trash. Every usage and module keeps the state D9 gave it; the table below decides on that state. Only a module whose `externalKey` starts with `topdesk::` is an imported application; the key after the last `:` is the APPID's match key. A usage without such a module (a hand-made usage, or a module another organisation imported) is left alone. + +Per application, with `processed` = every APPID match key on the CMDB sheets (whatever the row's outcome: a row that failed or was skipped is still in the export) and `archived` = the match keys on the archive sheet: + +| key is | module/usage state | action | outcome | +|---|---|---|---| +| on a CMDB sheet | any | nothing (the row handled it) | — | +| on the archive sheet | working | archive usage, then module | `archived` | +| on the archive sheet | archived | nothing | — | +| on the archive sheet | deleted | restore, then archive | `archived` | +| nowhere, sheet present | working or archived | soft-delete usage, then module | `deleted` | +| nowhere, sheet present | deleted | nothing | — | +| nowhere, sheet absent | working | archive usage, then module | `archived` | +| nowhere, sheet absent | archived or deleted | nothing | — | + +The usage is written before the module, so a failure between the two leaves the municipality's usage in the new state and the module visible, never the reverse. A module another municipality still uses is archived or deleted all the same when this municipality's usage says so: the module carries this municipality's import key, so it is this municipality's record (REQ-CMDB-006 allows a shared module only when the other organisation's usage was added by hand). The archive reason is `cmdb-import: on sheet "Gearchiveerde Applicaties"` or `cmdb-import: not in the export`, plus the operation id, so the audit trail says why. + +Each application runs in its own boundary: a failure is a `failed` row naming the step (`archive`, `restore`, `delete`) and the exception class, as row failures do, and the other applications continue. Contact persons are never touched. + +Progress: the operation gets the phase `reconciling` (added to `ProgressTracker::PHASES` with weight 0, so the ArchiMate import's weighted percentage is unchanged), `total_items` grows by the number of usages in scope, and `processed_items` advances per application; the section's own percentage comes from those counts. + +### D6. The report + +`CmdbImportReport` gains the outcomes `archived`, `unarchived`, `deleted` and `restored`; `summary()` counts them like the others. Reconciliation rows carry the archive sheet's name (for `archived` from the sheet) or an empty sheet, row 0, the APPID (`externalNumber`), the module name and both uuids: no person data. `toStoredArray()` ranks them with the rest. The section shows the four counts as summary tiles when they are non-zero, and the outcome filter offers them. + +### D7. Failure modes + +- `ArchiveHandler` or `MagicMapper` missing, or without a method the import calls (`archive`, `unarchive`, `restoreObject`, `findDeletedAcrossAllMagicTables`): 503 `ARCHIVE_UNAVAILABLE` before the lock is taken, only in archive mode. `keep` needs neither: without them the trash cannot be read, so a soft-deleted module or usage is not matched (the log says so) and a row whose match must be unarchived fails at step `module` or `usage`. +- A schema without `x-openregister-archive`: `ArchiveNotOfferedException` from OpenRegister per object → a failed row with the exception class; the docs say to run Force Update. +- A failure while reviving a matched row: the row fails at step `module` or `usage` like any other row failure. + +### D8. Stackiq UI + +The Applications (`Modules`, a `FacetedCatalogIndexView` that forwards `quickFilters` to `CnIndexPage`) and Applications in use (`Gebruik`) pages get the quick filter `{"label": "Archived", "filter": {"_archived": "true"}}`. `useSelfFetchList` spreads the active tab's filter into the fetch parameters as they are, so `_archived=true` reaches OpenRegister and the list shows the archive alone; the facet narrowing's `_ids` composes with it (AND). The icon `ArchiveOutline` is registered in `src/icons.js`. `@conduction/nextcloud-vue` 2.65 renders no `@self.archived` on a detail page or in the sidebar metadata, so the detail pages show no archive state yet; that is a library follow-up, not a stackiq widget. + +### D9. How the import reads the state of a stored object + +Found on a Nextcloud 35 rig with OpenRegister 2.1.37-beta (origin/beta `ff4dad5c`): the second import of the original export left APPID 1234 archived (`unchanged`) and created a second module for APPID 2. `ObjectService::searchObjects()` (via `QueryHandler::searchObjects()` → `MagicMapper::searchObjects()` → `MagicSearchHandler::searchObjects()`) applies the lenses in SQL (`buildFilteredQuery()` → `applyBasicFilters()`, `MagicSearchHandler.php` 2135–2153), but converts each row with `MagicSearchHandler::convertRowToObjectEntity()` (3780), which: + +- reads no `_archived` column, so every result keeps `ObjectEntity::$archived = []` (`ObjectEntity.php` 334) and looks working; +- turns the `_deleted` marker into `new DateTime(...)` (3896–3898), but `DeleteObject` writes it as a JSON object (`DeleteObject.php` 361), so the conversion throws, the row becomes null (4035) and `executeSearchQuery()` skips it (3714): `_includeDeleted=true` returns no soft-deleted object. + +So the import never reads the state from a search result: + +- **Working**: found by a search without a lens. **Archived**: found by a search with `_archived=true` (the archive alone, not deleted). An object is looked up working first, then archived. +- **Deleted**: from `MagicMapper::findDeletedAcrossAllMagicTables(_rbac: false, _multitenancy: false)`, the listing behind the Deleted page, which reads the rows itself (`MagicMapper.php` 7001). It scans every magic table of the instance and pages only after merging them, so the import reads it once per run, lazily, and keeps this register's modules and usages. Its objects carry data but no markers (`MagicMapper::rowToObjectEntity()`, 9131), and that data has every string that parses as JSON decoded ("2" becomes 2), so it is never written back. +- **Deleted and archived**: the trash does not say. `MagicMapper::restoreObject()` clears `_deleted` (10078) and answers the object as the mapper reads it by uuid, through `MagicStatisticsHandler::convertRowToObjectEntity()`, which does decode `_archived` (`MagicStatisticsHandler.php` 631–652). A returning row unarchives it when that object is archived, and updates from it. The reconciliation does not need to know: an object in the trash is either left there or restored and archived, and `ArchiveHandler::archive()` leaves an archived object as it is (`ArchiveHandler.php` 111). + +`ArchiveHandler::unarchive()` clears the marker to SQL NULL, so the object is back in the working lists, and does nothing for an object that is not archived (161). A lookup costs one search for a working match and two for an archived one or none, plus the one trash read per run. When OpenRegister hydrates both markers on search results, this can go back to one search with both lenses; the lens-based reading stays correct either way. + +## API Design + +`POST /api/cmdb-import`: `missingRecords` accepts `keep` and `archive` (default `archive`). The report's `summary` gains `archived`, `unarchived`, `deleted` and `restored`; `rows[].outcome` may be any of the four; reconciliation rows have `row: 0`. New error code 503 `ARCHIVE_UNAVAILABLE`. `MISSING_RECORDS_UNSUPPORTED` keeps its status and gains both accepted values in `details.accepted`. `openapi.json` is updated accordingly. + +## Database Changes + +None. The archive and delete markers are OpenRegister's metadata columns. + +## Mixed-spec rationale (ADR-032) + +The change is `kind: code`. Its only schema delta is the annotation `x-openregister-archive: {enabled: true}` on `module` and `usage` in the existing fragment `topdesk-cmdb-import.json`, with a version bump of both. The annotation is glue for this code: OpenRegister refuses to archive an object of a schema without it, and nothing else in stackiq reads it. It follows the fragment convention (ADR-037) and deploys with the existing register import. + +## Declarative-vs-imperative decision (ADR-031) + +- **Declarative:** which schemas may be archived (`x-openregister-archive` on `module` and `usage`), which sheet is the archive sheet and which option values are accepted (`topdesk-profile.json`), and which list shows the archive (the manifest quick filter `_archived=true`). +- **Imperative, because it is the external-integration exception the first change already claimed:** comparing an uploaded workbook with the register and deciding per application between nothing, archive, unarchive, soft-delete and restore. No `x-openregister-*` block can express "archive when the source lists it as archived and delete when the source no longer lists it": that rule depends on a file the register never sees. The state transitions themselves are OpenRegister's (`ArchiveHandler`, `deleteObject()`, `restoreObject()`); stackiq only decides when to call them. +- **Rule stated once, enforced in code:** the table in D5. + +## Nextcloud Integration + +- Controller: `CmdbImportController::import()` (unchanged auth: Nextcloud admins only, CSRF). +- Services: `CmdbExportImportService` (lenses, revival, reconciliation), `Cmdb\CmdbWorkbookReader` and `Cmdb\CmdbImportProfile` (archive sheet), `Cmdb\CmdbImportReport` (outcomes), `ProgressTracker` (phase). +- OpenRegister: `ObjectServiceInterface::searchObjects()` with `_archived` and `_ids`, `::deleteObject()` (contract); `Service\Object\ArchiveHandler::archive()` / `unarchive()` and `Db\MagicMapper::restoreObject()` / `findDeletedAcrossAllMagicTables()` (container, guarded, like the mapping engine). The handler checks `update` for the session user with RBAC on; the import is admin-only, so the check passes, and the audit entry names the admin. + +## Security Considerations + +- Still admin-only, CSRF-protected, with the file bounds of the first change; the archive sheet is read under the same bounds, APPID column only. +- Scope is the imported municipality's usages; a supplier's or another municipality's module is never archived or deleted through this municipality's import. +- Nothing is hard-deleted; every transition is an OpenRegister audit entry with a reason. +- The report and the logs carry APPIDs, module names and uuids, never owner data; contact persons are untouched. + +## NL Design System + +The section adds one radio group with a help text, built from `NcCheckboxRadioSwitch` like the existing switches, labelled and translated. The quick filter is the library's own chip. + +## File Structure + +``` +lib/Service/CmdbExportImportService.php lenses, revival, reconcile(), state helpers +lib/Service/Cmdb/CmdbWorkbookReader.php the archive sheet +lib/Service/Cmdb/CmdbImportProfile.php archiveSheetName(), readSheetNames(), missingRecordsModes() +lib/Service/Cmdb/CmdbImportReport.php archived | unarchived | deleted | restored +lib/Controller/CmdbImportController.php default archive, accepted values, ARCHIVE_UNAVAILABLE +lib/Exception/CmdbImportException.php ARCHIVE_UNAVAILABLE +lib/Service/ProgressTracker.php phase reconciling +lib/Settings/cmdb-import/topdesk-profile.json archiveSheet, missingRecords +lib/Settings/register.d/topdesk-cmdb-import.json x-openregister-archive, module 0.3.9, usage 1.5.7 +src/utils/cmdbImport.js, src/views/settings/sections/CmdbImport.vue +src/manifest.json, src/manifest.d/usages.json, src/icons.js +tests/Unit/..., tests/fixtures/cmdb/topdesk-archived-applications.xlsx, tests/e2e/spec-coverage/cmdb-import.spec.ts +docs/features/cmdb-import.md, l10n/en.json, l10n/nl.json +``` + +## Seed Data + +No seed objects change. The fragment's three seed modules stay as they are; the `module` and `usage` schemas gain only the archive annotation. + +## Risks / Trade-offs + +- [The default archives on the first re-import after the upgrade] → The section shows the choice with help text; the report lists every archived and deleted application; nothing is hard-deleted. +- [A workbook saved without the archive sheet] → archive only, never delete, plus a warning (D3). +- [A failure between the usage and the module] → the usage is written first, so the municipality's view is right and the module is at worst still visible (D5). +- [Reviving before writing adds calls per returning row] → only for rows whose match is archived or deleted; a row whose module is working costs the same searches as before, a new row one more search (the archive lens). +- [The trash listing scans every magic table of the instance] → read once per import, not per row (D9). +- [OpenRegister internals change shape] → guarded resolution, 503 before the file is read (D7). + +## Migration Plan + +No data migration. The fragment deploys with the existing register import (Force Update on an installation that already has the register). Rollback is a revert of the PR; archived or deleted objects keep their state and can be unarchived or restored by hand. + +## Open Questions + +- Should the library render `@self.archived` on detail pages, so an admin sees why an application is not in the list? (library follow-up) +- Should a module that no usage points at any more be archived as well? Out of scope here. diff --git a/openspec/changes/cmdb-import-archive-reconciliation/proposal.md b/openspec/changes/cmdb-import-archive-reconciliation/proposal.md new file mode 100644 index 000000000..7c27fdd85 --- /dev/null +++ b/openspec/changes/cmdb-import-archive-reconciliation/proposal.md @@ -0,0 +1,95 @@ +--- +kind: code +depends_on: [] +--- + +# Proposal: cmdb-import-archive-reconciliation + +## Summary + +A repeated CMDB import now reconciles the applications a municipality uses with the newer export. An application whose APPID is no longer on the two CMDB sheets but is listed on the export's sheet "Gearchiveerde Applicaties" is archived: its `module` and the municipality's `usage` get OpenRegister's archive state, so they drop out of every list and search until an admin asks for the archive. An application whose APPID is on no sheet at all is soft-deleted through OpenRegister's trash, from which it can be restored. An application that returns to the export is unarchived, or restored from the trash, and then updated as usual, so no duplicate is ever created. The import option `missingRecords` accepts `keep` and `archive`, and `archive` is the default; the settings section shows the choice. Stackiq's Applications and Applications in use pages get an "Archived" quick filter. The report gets the outcomes `archived`, `unarchived`, `deleted` and `restored`. + +## Motivation + +The first version of the import (openspec/changes/archive/2026-10-05-cmdb-export-import) left records that disappeared from a newer export untouched. The municipality delivers a new export every month and asked on 2026-10-01 (Jira WOO-587, parent WOO-586, epic WOO-281): an application that left the CMDB sheets but is on the archive sheet must be marked as archived and be findable in OpenCatalogi only with an explicit filter; an application that is on no sheet any more must be soft-deleted with the existing functionality. Without this, the catalogue keeps showing applications the municipality retired, and Portaliq keeps listing them under "Software we use". + +OpenRegister already has both states. Its archive state (`@self.archived`, `POST /api/objects/{register}/{schema}/{id}/archive`) keeps an object whole but out of every list unless `_archived=true` or `_archived=any` is asked; its soft delete (`@self.deleted`) puts an object in the trash for 30 days, from which it can be restored. OpenCatalogi passes the list parameters through to OpenRegister, and Portaliq lists usages without either lens, so both apps hide archived and deleted records with no change of their own (decided 2026-10-09). + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `cmdb-export-import`: `missingRecords` becomes `keep | archive` with `archive` as the default (REQ-CMDB-012 modified); the reconciliation of missing records against the archive sheet (REQ-CMDB-015), the return of an archived or deleted application (REQ-CMDB-016) and the archive lens in stackiq (REQ-CMDB-017) are added; the settings section shows the choice (REQ-CMDB-014 modified). The `module` and `usage` schemas opt into OpenRegister's archive state through the register fragment (see design.md, Mixed-spec rationale). + +## Affected Projects + +- [ ] Project: `stackiq`: import service, workbook reader and profile, controller option, register fragment, the "CMDB import" settings section, the manifest quick filters, tests, fixtures, administrator docs and translations. + +## Scope + +### In Scope + +- The sheet "Gearchiveerde Applicaties" (columns Applicatiecomponent, APPID, Applicatie Code, Applicatie Naam, Roepnaam, Applicatie Status, Bron, Datum Interface, Datum wijziging) is read for its APPID column only, when it is present. +- After every row of a completed import: the municipality's usages, with their modules, are compared with the APPIDs on the CMDB sheets and on the archive sheet. Missing and on the archive sheet: module and usage archived. Missing everywhere: module and usage soft-deleted. Without the archive sheet: archived only, with a warning in the report. +- A returning APPID: an archived module or usage is unarchived, a soft-deleted one is restored from the trash, and the row is then updated as before. +- `missingRecords`: `keep` leaves everything as it is (the behaviour so far), `archive` reconciles and is the default; `remove` stays reserved and is refused. +- The settings section offers the choice with help text; the report shows the new outcomes in its summary and rows. +- `x-openregister-archive` on the `module` and `usage` schemas through the register fragment, with a version bump. +- An "Archived" quick filter on the Applications and Applications in use pages. +- PHPUnit, Jest, a Playwright e2e for the re-import with the archive sheet, a fixture variant, docs and Dutch and English strings. + +### Out of Scope + +- Hard deletion of objects; the trash's own retention (30 days) stays OpenRegister's. +- Changes to the municipality's TOPdesk export, and the raw "Invoer gearchiveerde appl" sheet (the derived "Gearchiveerde Applicaties" sheet is the source). +- Contact persons: the owner of an archived or deleted application stays, because it is shared with the municipality's other applications. +- Modules only other organisations use, and organisations (suppliers) that no application names any more. +- An archive widget on the detail pages: `@conduction/nextcloud-vue` 2.65 does not render `@self.archived`; a follow-up once the library does. +- Changes to OpenCatalogi or Portaliq. + +## Approach + +`CmdbWorkbookReader` loads the archive sheet next to the two CMDB sheets when the workbook has it, under the same size and row bounds, and hands the import service the APPIDs it holds. `CmdbExportImportService` widens the match of a module (by import key) and of a usage (by consumer and module) to archived and soft-deleted objects, and revives what it matches before the usual update. After the last row, when the import was not cancelled and `missingRecords` is `archive`, a reconciliation step pages through the municipality's usages with no lens and with OpenRegister's `_archived=true` lens, reads the trash once through `MagicMapper::findDeletedAcrossAllMagicTables()` (a search with `_includeDeleted=true` returns no deleted rows on the current OpenRegister beta; design.md D9), loads their modules in batches, and archives, unarchives, soft-deletes or restores module and usage through OpenRegister's `ArchiveHandler`, `ObjectServiceInterface::deleteObject()` and `MagicMapper::restoreObject()`, each object in its own error boundary. The step runs under the import's register lock, reports its progress in a phase of its own, and adds its outcomes to the report. Details are in design.md. + +## New Dependencies + +None. `ArchiveHandler` and `MagicMapper` come from OpenRegister, which stackiq already requires; both are resolved through the container behind a guard, like the mapping engine. + +## Impact + +- **Backend**: `lib/Service/CmdbExportImportService.php` (match lenses, revival, reconciliation), `lib/Service/Cmdb/CmdbWorkbookReader.php` and `CmdbImportProfile.php` (archive sheet), `CmdbImportReport.php` (outcomes), `lib/Controller/CmdbImportController.php` (option default and accepted values), `lib/Service/ProgressTracker.php` (phase `reconciling`), `lib/Exception/CmdbImportException.php` (`ARCHIVE_UNAVAILABLE`). +- **Configuration**: `lib/Settings/cmdb-import/topdesk-profile.json` (archive sheet, accepted modes). +- **Schema**: `lib/Settings/register.d/topdesk-cmdb-import.json` declares `x-openregister-archive` on `module` (0.3.9) and `usage` (1.5.7). +- **Frontend**: `src/views/settings/sections/CmdbImport.vue`, `src/utils/cmdbImport.js`, `src/manifest.json`, `src/manifest.d/usages.json`, `src/icons.js`. +- **Data**: a re-import with the default option archives or soft-deletes modules and usages the export no longer lists. Nothing is hard-deleted; a soft-deleted object can be restored from OpenRegister's trash for 30 days. + +## Cross-Project Dependencies + +- **openregister** (consumed, not changed): `ObjectServiceInterface::deleteObject()` (contract), `Service\Object\ArchiveHandler` and `Db\MagicMapper::restoreObject()` and `findDeletedAcrossAllMagicTables()` (not public contracts, guarded), the `_archived` list lens, and the schema annotation `x-openregister-archive`. +- **opencatalogi** and **portaliq** (consumers, not changed): both list through OpenRegister's default lens, so archived and deleted applications disappear from them without a change; an OpenCatalogi caller finds archived applications with `_archived=true` on the existing public routes. + +## Risks + +### Risk 1: A re-import with the new default archives what an admin did not expect +**Severity:** Medium — **Mitigation:** only the municipality's own usages and the modules they point at are in scope, never a supplier's or another municipality's module; nothing is hard-deleted; the choice is visible in the section with help text, and the report lists every archived and deleted application. A cancelled or failed import never reconciles. + +### Risk 2: A workbook without the archive sheet soft-deletes everything that moved there +**Severity:** Medium — **Mitigation:** without the sheet, missing applications are archived, never soft-deleted, and the report warns that the sheet was not found. + +### Risk 3: The archive services are OpenRegister internals +**Severity:** Low — **Mitigation:** resolved through the container behind a guard; when missing, an import with `missingRecords=archive` is refused with 503 `ARCHIVE_UNAVAILABLE` before the file is read, and `keep` still works. + +### Risk 4: The schemas on an installation predate the annotation +**Severity:** Low — **Mitigation:** OpenRegister refuses to archive an object of a schema without `x-openregister-archive`; the import reports that object as failed and the docs point to Force Update, which deploys the fragment. + +## Rollback Strategy + +Revert the PR. Objects archived or soft-deleted by an import keep their state; an admin unarchives them through OpenRegister's object actions or restores them from the trash. The schema annotation may stay deployed without harm. + +## Open Questions + +- Should a module that no usage points at any more (every municipality archived it) be archived as well? Out of scope here; a supplier's module stays. diff --git a/openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md new file mode 100644 index 000000000..1ac827456 --- /dev/null +++ b/openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md @@ -0,0 +1,191 @@ +# cmdb-export-import Specification + +**Status**: in-progress +**Scope**: stackiq +**OpenSpec changes**: +- [cmdb-export-import](../../../archive/2026-10-05-cmdb-export-import/) _(archived 2026-10-05)_ +- [cmdb-import-archive-reconciliation](../../) + +## Purpose + +A repeated CMDB import reconciles the applications a municipality uses with the newer export. An application whose APPID left the two CMDB sheets is archived in OpenRegister's archive state when the export's sheet "Gearchiveerde Applicaties" lists it, and soft-deleted into OpenRegister's trash when no sheet lists it any more; one that returns is unarchived or restored and updated as usual, never duplicated. Archived and deleted applications disappear from OpenCatalogi and Portaliq without a change there, and stackiq lists the archive on request (Jira WOO-587). + +OpenRegister: `OCA\OpenRegister\Contract\ObjectServiceInterface` (`searchObjects()` with no lens and with the `_archived=true` and `_ids` lenses, `deleteObject()`), `Service\Object\ArchiveHandler` (`archive()`, `unarchive()`) and `Db\MagicMapper` (`findDeletedAcrossAllMagicTables()` for the trash, `restoreObject()`) (both resolved through the container behind a guard; why the trash is read that way: design.md D9), and the schema annotation `x-openregister-archive`. + +## ADDED Requirements + +### Requirement: An application missing from the CMDB sheets SHALL be archived when the archive sheet lists it and soft-deleted when no sheet does (REQ-CMDB-015) + +With `missingRecords` `archive`, after every row of an import that was not cancelled and did not fail, the import SHALL reconcile, under the register lock it already holds, every `usage` whose `consumer` is the imported municipality together with the `module` it points at, including archived and soft-deleted ones, and only those whose module carries this municipality's import key (`topdesk::`). It SHALL read the APPID column of the sheet "Gearchiveerde Applicaties" when the workbook has it, under the same size and row bounds as the CMDB sheets and without reading any other column of it. For an application whose APPID is on neither CMDB sheet: when the archive sheet lists the APPID, the import SHALL archive the usage and then the module through OpenRegister's archive state with a reason that names the import, restoring a soft-deleted one from the trash first; when no sheet lists it, the import SHALL soft-delete the usage and then the module through OpenRegister's trash. When the workbook has no archive sheet, or the sheet has no APPID column, the import SHALL archive and SHALL NOT soft-delete, and SHALL add a warning to the report. An application that is already in the state the export asks for SHALL be left alone and not reported. The import SHALL NOT touch contact persons, organisations, modules that carry another municipality's import key or no import key, or usages of another organisation. Each application SHALL be reconciled in its own error boundary: a failure SHALL be a `failed` report row naming the step and the exception class, and the other applications SHALL still be reconciled. The reconciliation SHALL report its progress as the phase `reconciling` of the import's operation. A cancel SHALL also be honoured after the last row and before each application of the reconciliation: the import SHALL stop there, SHALL keep what it archived or deleted so far, and SHALL report the run as cancelled. The report SHALL count `archived` and `deleted` in its summary and SHALL list each as a row with the APPID, the application name and both uuids, never person data. When OpenRegister's archive handler or object mapper is not available, an import with `missingRecords` `archive` SHALL be refused with 503 `ARCHIVE_UNAVAILABLE` before the file is read. + +#### Scenario: An application that moved to the archive sheet is archived +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** the modules with APPID `1234` and `2` were imported for "Gemeente Voorbeeldstad" with their usages +- **WHEN** an export is imported in which `1234` is only on "Gearchiveerde Applicaties" and `2` is on no sheet +- **THEN** the module and the usage of `1234` SHALL carry OpenRegister's archive state and the report row SHALL be `archived` +- **AND** the module and the usage of `2` SHALL be in OpenRegister's trash and the report row SHALL be `deleted` +- **AND** the Applications page SHALL list `1234` under the quick filter "Archived" and neither under "All" +- **AND** the owner contact person SHALL be unchanged + +#### Scenario: Without the archive sheet a missing application is archived, never deleted +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testWithoutTheArchiveSheetMissingApplicationsAreArchivedNotDeleted imports a workbook without the sheet and asserts the archive state, no delete, and the warning naming the sheet. + +- **GIVEN** the module with APPID `7` and its usage were imported +- **WHEN** a workbook without "Gearchiveerde Applicaties" that no longer lists `7` is imported +- **THEN** the module and the usage of `7` SHALL be archived and SHALL NOT be deleted +- **AND** the report SHALL carry a warning that the sheet "Gearchiveerde Applicaties" was not found + +#### Scenario: Keeping missing records, a cancelled import and another municipality leave everything as it is +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testRecordsMissingFromTheExportStayWhenKept, testACancelledImportDoesNotReconcile and testAnotherMunicipalitysUsagesAreNeverReconciled assert no archive, no delete and no reconciliation row. + +- **GIVEN** the modules with APPID `1` and `7` and their usages for "Gemeente Voorbeeldstad", and a usage of `7` for "Gemeente Anders" +- **WHEN** an export with only `1` is imported with `missingRecords` `keep`, or with `archive` but cancelled after row 1 +- **THEN** no module or usage SHALL be archived or deleted +- **AND** when it is imported with `archive` and completes, only the usage of "Gemeente Voorbeeldstad" for `7` and the module SHALL be archived; the usage of "Gemeente Anders" SHALL be untouched + +#### Scenario: A cancel after the last row or during the reconciliation stops it +@e2e exclude Timing-dependent; tests/Unit/Service/CmdbExportImportServiceTest.php testACancelDuringTheLastRowArchivesNothing and testACancelDuringTheReconciliationStopsBeforeTheNextApplication request the cancel at those two moments and assert the transitions and the cancelled report. + +- **GIVEN** the modules with APPID `1`, `7` and `8` and their usages for "Gemeente Voorbeeldstad" +- **WHEN** an export with only `1`, and `7` and `8` on the archive sheet, is imported with `archive` and a cancel arrives while row `1` is processed +- **THEN** no module or usage SHALL be archived or deleted, and the report SHALL be cancelled +- **AND** when the cancel arrives instead while `7` is archived, `7` SHALL stay archived, `8` SHALL be left as it is, and the report SHALL be cancelled + +#### Scenario: One application that cannot be archived does not stop the others +@e2e exclude Fault injection; tests/Unit/Service/CmdbExportImportServiceTest.php testAFailingArchiveIsAFailedRowAndTheOthersAreReconciled makes the archive of one application throw and asserts a failed row naming the step, the other application archived, and the log line without a name. + +- **GIVEN** the modules with APPID `7` and `8` left the export and are on the archive sheet +- **WHEN** archiving `7` throws +- **THEN** the report SHALL list `7` as `failed` with the step `archive` and the exception class +- **AND** `8` SHALL be archived + +#### Scenario: Without the archive services an archiving import is refused before reading +@e2e exclude Environment condition; tests/Unit/Service/CmdbExportImportServiceTest.php testMissingArchiveServicesRefuseAnArchivingImport asserts 503 ARCHIVE_UNAVAILABLE with no read and no save, and that keep still imports; tests/Unit/Controller/CmdbImportControllerTest.php asserts the status and the translated message. + +- **GIVEN** an OpenRegister without `ArchiveHandler` +- **WHEN** an export is imported with `missingRecords` `archive` +- **THEN** the endpoint SHALL answer 503 with error `ARCHIVE_UNAVAILABLE` and nothing SHALL be read or written +- **AND** the same export with `keep` SHALL be imported + +### Requirement: An application that returns to the export SHALL be unarchived or restored, never duplicated (REQ-CMDB-016) + +The import SHALL match a module by its import key, and a usage by consumer and module, among archived and soft-deleted objects as well as working ones. Before it updates a matched module or usage that is archived, it SHALL unarchive it through OpenRegister's archive state; before it updates one that is soft-deleted, it SHALL restore it from the trash. It SHALL then update the row as REQ-CMDB-006 and REQ-CMDB-009 describe, and SHALL NOT create a second module or usage. The row's outcome SHALL be `restored` when a module or usage was restored, else `unarchived` when one was unarchived, else the usual outcome; the summary SHALL count both. With `updateExisting` false the row SHALL be skipped as `exists` and nothing SHALL be revived. When more than one module or usage matches, the import SHALL take a working one before an archived one, and an archived one before one in the trash, so a module deleted by hand next to the one a later import created SHALL NOT be restored. Organisations and contact persons SHALL keep matching among working objects only. + +#### Scenario: An archived application returns to the export +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** the module and the usage of APPID `1234` are archived and those of APPID `2` are in the trash +- **WHEN** an export that lists both on the CMDB sheets is imported +- **THEN** the report SHALL show `1234` as `unarchived` and `2` as `restored` +- **AND** there SHALL still be one module and one usage for each +- **AND** both SHALL be listed again without a lens + +#### Scenario: A returning application is not revived when existing records are not updated +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testUpdateExistingFalseLeavesAnArchivedMatchArchived asserts the row is skipped as `exists`, the archive state stays and no object is created. + +- **GIVEN** the module and the usage of APPID `7` are archived +- **WHEN** a row with APPID `7` is imported with `updateExisting` false +- **THEN** the row SHALL be skipped with reason `exists` +- **AND** both SHALL stay archived, and no module or usage SHALL be created + +#### Scenario: A working match wins over a deleted one with the same import key +@e2e exclude Data condition from earlier imports; tests/Unit/Service/CmdbExportImportServiceTest.php testAWorkingMatchWinsOverADeletedOneWithTheSameKey stores a deleted module and usage ahead of the working ones and asserts the working ones are updated and nothing is restored. + +- **GIVEN** a module with APPID `7` in the trash and a working module with the same import key, each with a usage of "Gemeente Voorbeeldstad" +- **WHEN** a row with APPID `7` is imported +- **THEN** the working module and its usage SHALL be updated +- **AND** the module and the usage in the trash SHALL stay there, and nothing SHALL be restored + +### Requirement: Archived applications and usages SHALL be listable in stackiq and SHALL stay out of OpenCatalogi and Portaliq by default (REQ-CMDB-017) + +The `module` and `usage` schemas SHALL declare `x-openregister-archive` with `enabled` true through the register fragment `topdesk-cmdb-import.json`, as `module` 0.3.9 and `usage` 1.5.7, so OpenRegister offers the archive state on both. Stackiq's Applications and Applications in use pages SHALL offer a quick filter "Archived" that lists the archive alone (`_archived=true`). No change SHALL be made to OpenCatalogi or Portaliq: both list through OpenRegister's default lens, so an archived or deleted application is not in OpenCatalogi's search results or in Portaliq's "Software we use", and an OpenCatalogi caller finds archived applications with `_archived=true` on the existing routes. + +#### Scenario: The merged register offers the archive state on both schemas +@e2e exclude Configuration; tests/Unit/Settings/TopdeskCmdbFragmentTest.php testTheMergedSchemasOfferTheArchiveState asserts the versions and the annotation after merging every fragment. + +- **GIVEN** every register fragment merged in filename order +- **THEN** `module` SHALL be version 0.3.9 and `usage` 1.5.7 +- **AND** both SHALL declare `configuration.x-openregister-archive.enabled` true, and the usage lifecycle of 1.5.6 SHALL still be declared + +#### Scenario: The Applications page lists the archive on request +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** the module of APPID `1234` is archived +- **WHEN** an admin opens the Applications page +- **THEN** the module SHALL NOT be listed +- **AND** under the quick filter "Archived" it SHALL be listed + +## MODIFIED Requirements + +### Requirement: Records missing from a newer export SHALL be archived by default or kept on request (REQ-CMDB-012) + +The import SHALL accept `missingRecords` with the values `keep` and `archive`; `archive` is the default. With `keep` it SHALL NOT change, archive, depublish or delete a module, usage, organisation or contact person because its APPID is absent from the upload. With `archive` it SHALL reconcile as REQ-CMDB-015 describes. Any other value, including the reserved `mark` and `remove`, SHALL be refused with 422 `MISSING_RECORDS_UNSUPPORTED`, naming both accepted values in `details.accepted`. The accepted values SHALL be declared in the import profile. + +(Previously: `keep` was the only accepted value and the default; records missing from a newer export were always left untouched.) + +#### Scenario: An application dropped from the export stays when records are kept +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testRecordsMissingFromTheExportStayWhenKept imports two rows, then one with `missingRecords` `keep`, and asserts the other module and usage are unchanged. + +- **GIVEN** the modules with APPID `1` and `7` were imported for "Gemeente Voorbeeldstad" +- **WHEN** a newer export that only contains APPID `1` is imported with `missingRecords` `keep` +- **THEN** the module with APPID `7` and its usage SHALL be unchanged + +#### Scenario: Without the option a missing application is reconciled +@e2e exclude Validation; tests/Unit/Controller/CmdbImportControllerTest.php testMissingRecordsDefaultsToArchive asserts the service receives `archive` when the field is absent and the sent value otherwise. + +- **GIVEN** a valid export +- **WHEN** a Nextcloud admin posts it without `missingRecords` +- **THEN** the import SHALL run with `archive` + +#### Scenario: A reserved value is refused +@e2e exclude Validation; tests/Unit/Controller/CmdbImportControllerTest.php testAReservedMissingRecordsValueIsRefused. + +- **GIVEN** a valid export +- **WHEN** a Nextcloud admin posts it with `missingRecords=remove` +- **THEN** the endpoint SHALL answer 422 with error `MISSING_RECORDS_UNSUPPORTED` and `details.accepted` `["keep", "archive"]` +- **AND** no object SHALL be written + +### Requirement: The admin settings SHALL offer a CMDB import section (REQ-CMDB-014) + +Stackiq's admin settings page SHALL show a section "CMDB import", rendered by the settings page and not registered as an in-app route. The section SHALL let the admin choose an existing municipality or type the name of a new one, choose an `.xlsx` file, choose what happens to applications missing from the export (archive, the default, or keep) with help text that says what each does, and start the import. While the import runs it SHALL show a progress bar and a Cancel button. Afterwards it SHALL show the summary, including the archived, unarchived, deleted and restored counts when they are not zero, and a report table that can be filtered by outcome, including the four new ones; a reconciliation row SHALL show no row number. Every control SHALL have a visible label, and every string SHALL be translatable. + +(Previously: the section had no choice for missing records; the request always sent `keep`.) + +#### Scenario: The admin runs an import from the settings page +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** a Nextcloud admin on stackiq's admin settings page +- **WHEN** they choose "Gemeente Voorbeeldstad", choose the anonymised export and press "Import" +- **THEN** a progress bar SHALL appear while the import runs +- **AND** afterwards the summary and the report table SHALL be shown +- **AND** filtering the table on `created` SHALL show the two imported rows + +#### Scenario: The admin chooses to keep missing applications +@e2e exclude Covered by the Jest test; src/utils/cmdbImport.spec.js asserts buildImportForm() sends `archive` by default and `keep` when chosen, and src/views/settings/sections/CmdbImport.spec.js asserts the section's default. + +- **GIVEN** the section +- **WHEN** the admin selects "Keep them as they are" and starts the import +- **THEN** the request SHALL carry `missingRecords=keep` +- **AND** without a choice it SHALL carry `missingRecords=archive` + +## Non-Functional Requirements + +- **Performance:** the reconciliation SHALL page the municipality's usages with the import's page size and load their modules in one search per page, so an export of 1,100 rows adds a handful of searches, and SHALL make no write for an application that is already in the state the export asks for. +- **Security:** unchanged from the first change; the archive sheet is read under the same bounds, APPID column only; the scope is the imported municipality's usages; nothing is hard-deleted; every state change is an OpenRegister audit entry. +- **Privacy:** reconciliation rows and log lines carry APPIDs, module names and uuids only; contact persons are never touched. +- **Accessibility:** Target WCAG 2.2 AA. The new choice is a labelled radio group (SC 1.3.1, 3.3.2; gates `form-label-association`, `nc-input-labels`) with help text; the summary tiles and the outcome filter follow the existing pattern. New in 2.2: 2.4.11, 2.5.8 and 3.3.7 apply as before; 2.5.7, 3.2.6 and 3.3.8 do not apply (no dragging, no help mechanism, no authentication). +- **Internationalization:** Dutch and English MUST be supported (ADR-005) for the choice, the outcomes, the warning and the error message. + +## Acceptance Criteria + +- [ ] A re-import with the archive sheet archives the application that moved there and soft-deletes the one on no sheet; the report shows `archived` and `deleted`. +- [ ] Importing the original export again unarchives and restores them; no duplicate is created. +- [ ] With "Keep them as they are" nothing is archived or deleted. +- [ ] The Applications and Applications in use pages list archived records only under "Archived". +- [ ] OpenCatalogi's search and Portaliq's "Software we use" no longer show an archived application; `_archived=true` on OpenCatalogi's route finds it. + +## Notes + +- The decision table per application is in the change's [design.md](../../design.md), D5. +- The library does not render `@self.archived` on detail pages yet; a follow-up for `@conduction/nextcloud-vue`. diff --git a/openspec/changes/cmdb-import-archive-reconciliation/tasks.md b/openspec/changes/cmdb-import-archive-reconciliation/tasks.md new file mode 100644 index 000000000..4ae74b821 --- /dev/null +++ b/openspec/changes/cmdb-import-archive-reconciliation/tasks.md @@ -0,0 +1,89 @@ +# Tasks: cmdb-import-archive-reconciliation + +Spec: `openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md` (`SPEC` below). Design: `design.md` (the decision table in D5 is authoritative for what happens per application). + +## Implementation Tasks + +### Task 1: Schema opt-in and the import profile +- **spec_ref**: `SPEC#requirement-archived-applications-and-usages-shall-be-listable-in-stackiq-and-shall-stay-out-of-opencatalogi-and-portaliq-by-default-req-cmdb-017`, `SPEC#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012` +- **files**: `lib/Settings/register.d/topdesk-cmdb-import.json`, `lib/Settings/cmdb-import/topdesk-profile.json`, `lib/Service/Cmdb/CmdbImportProfile.php`, `tests/Unit/Settings/TopdeskCmdbFragmentTest.php`, `tests/Unit/Settings/PublicationFieldRulesTest.php`, `tests/Unit/Service/Cmdb/CmdbImportProfileTest.php` +- **acceptance_criteria**: + - GIVEN every register fragment merged in filename order THEN `module` is 0.3.9 and `usage` is 1.5.7, both with `configuration.x-openregister-archive.enabled` true, and the usage lifecycle of 1.5.6 is still there + - GIVEN the profile THEN `archiveSheetName()` is "Gearchiveerde Applicaties", `readSheetNames()` lists the two CMDB sheets plus the archive sheet, and `missingRecordsModes()` is `["keep", "archive"]` without OpenRegister +- [x] Implement +- [x] Test + +### Task 2: The reader reads the archive sheet's APPIDs +- **spec_ref**: `SPEC#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015` +- **files**: `lib/Service/Cmdb/CmdbWorkbookReader.php`, `tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php` +- **acceptance_criteria**: + - GIVEN the anonymised fixture THEN `read()` returns `archive.present` true and `archive.appIds` `["1198"]`, and the rows of the CMDB sheets are unchanged + - GIVEN a workbook without the sheet, or with the sheet but without an APPID column THEN `archive.present` is false (and `keyColumnMissing` says which), and the CMDB rows are still read + - GIVEN an archive sheet beyond the row limit THEN `TOO_MANY_ROWS` names it; only its APPID column is materialised +- [x] Implement +- [x] Test + +### Task 3: Matching revives archived and deleted applications +- **spec_ref**: `SPEC#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016` +- **files**: `lib/Service/CmdbExportImportService.php`, `lib/Service/Cmdb/CmdbImportReport.php`, `tests/Unit/Service/CmdbExportImportServiceTest.php`, `tests/Unit/Service/Cmdb/CmdbImportReportTest.php` +- **acceptance_criteria**: + - GIVEN an archived module and usage for an APPID WHEN a row with that APPID is imported THEN both are unarchived before the update, no second module or usage exists, and the row's outcome is `unarchived` + - GIVEN a soft-deleted module and usage WHEN the row returns THEN both are restored from the trash and the outcome is `restored` + - GIVEN only the usage archived or soft-deleted THEN the usage alone is unarchived or restored, with that outcome, and no second usage exists + - GIVEN a module and usage soft-deleted while archived THEN both are restored and unarchived, the outcome is `restored`, and the update starts from the restored object (design D9) + - GIVEN OpenRegister's search results (no archive marker; no soft-deleted row, `_includeDeleted` or not) THEN the state comes from the lens that found the object and from the trash listing, read once per import (design D9) + - GIVEN `updateExisting` false THEN the row is skipped as `exists` and nothing is revived + - GIVEN a deleted and a working module (and usage) sharing one import key THEN the working one is updated and nothing is restored +- [x] Implement +- [x] Test + +### Task 4: Reconciliation after the rows +- **spec_ref**: `SPEC#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015` +- **files**: `lib/Service/CmdbExportImportService.php`, `lib/Service/ProgressTracker.php`, `lib/Exception/CmdbImportException.php`, `tests/Unit/Service/CmdbExportImportServiceTest.php` +- **acceptance_criteria**: + - GIVEN APPIDs 1 and 7 imported WHEN an export with row 1 and 7 on the archive sheet is imported THEN module and usage of 7 are archived with a reason, the report row is `archived`, and the owner contact person is untouched + - GIVEN APPID 7 on no sheet THEN module and usage are soft-deleted and the row is `deleted`; GIVEN the archive sheet absent THEN they are archived instead and the report warns + - GIVEN APPID 7 soft-deleted and listed on the archive sheet THEN module and usage are restored and archived; GIVEN it already archived or, when on no sheet, already deleted THEN nothing is written + - GIVEN `missingRecords` `keep`, a cancelled import, or a usage of another municipality THEN nothing is archived or deleted + - GIVEN a cancel during the last row THEN nothing is archived or deleted; GIVEN a cancel during the reconciliation THEN it stops before the next application and the report is cancelled + - GIVEN the archive of one application throws THEN that application is a `failed` row and the others are reconciled; GIVEN no ArchiveHandler THEN 503 `ARCHIVE_UNAVAILABLE` before reading + - GIVEN the step runs THEN the operation is in phase `reconciling` and its progress advances per application +- [x] Implement +- [x] Test + +### Task 5: Controller option +- **spec_ref**: `SPEC#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012` +- **files**: `lib/Controller/CmdbImportController.php`, `tests/Unit/Controller/CmdbImportControllerTest.php`, `openapi.json`, `postman/stackiq-tests.json`, `l10n/en.json`, `l10n/nl.json` +- **acceptance_criteria**: + - GIVEN no `missingRecords` THEN the service receives `archive`; GIVEN `keep` or `archive` THEN it is passed through; GIVEN `remove` or `mark` THEN 422 `MISSING_RECORDS_UNSUPPORTED` with `details.accepted` `["keep", "archive"]` and no import + - GIVEN `ARCHIVE_UNAVAILABLE` from the service THEN 503 with its own translated message +- [x] Implement +- [x] Test + +### Task 6: Settings section, report words and the archive quick filter +- **spec_ref**: `SPEC#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014`, `SPEC#requirement-archived-applications-and-usages-shall-be-listable-in-stackiq-and-shall-stay-out-of-opencatalogi-and-portaliq-by-default-req-cmdb-017` +- **files**: `src/views/settings/sections/CmdbImport.vue`, `src/utils/cmdbImport.js`, `src/utils/cmdbImport.spec.js`, `src/views/settings/sections/CmdbImport.spec.js`, `tests/vitest/cmdbArchiveQuickFilter.spec.js`, `src/manifest.json`, `src/manifest.d/usages.json`, `src/icons.js`, `l10n/en.json`, `l10n/nl.json` +- **acceptance_criteria**: + - GIVEN the section THEN a labelled choice "Applications missing from the export" offers Archive (default) and Keep with help text, and `buildImportForm()` sends the choice as `missingRecords` + - GIVEN a report with the new outcomes THEN the summary shows their tiles, the outcome filter offers them, and a reconciliation row shows no row number + - GIVEN the Applications and Applications in use pages THEN a quick filter "Archived" sends `_archived=true` + - Every new string is in `l10n/en.json` and `l10n/nl.json`, and the generated `l10n/*.js` are current +- [x] Implement +- [x] Test + +### Task 7: Fixture, docs and e2e +- **spec_ref**: `SPEC#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015` +- **files**: `tests/fixtures/cmdb/build-fixtures.py`, `tests/fixtures/cmdb/topdesk-archived-applications.xlsx`, `tests/fixtures/cmdb/README.md`, `tests/Unit/Fixtures/CmdbFixtureHygieneTest.php`, `tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php`, `tests/Unit/Service/CmdbExportImportServiceTest.php`, `tests/e2e/spec-coverage/cmdb-import.spec.ts`, `docs/features/cmdb-import.md`, `docs/features/README.md`, `CHANGELOG.md` +- **acceptance_criteria**: + - GIVEN `build-fixtures.py` THEN it derives `topdesk-archived-applications.xlsx`: APPID 1234 on the archive sheet only, APPID 2 on no sheet, both CMDB sheets present and empty; the hygiene test passes on it + - GIVEN the e2e THEN after the first import, importing the variant archives 1234 and soft-deletes 2, the Applications page lists 1234 under "Archived", and importing the original again reports 1234 `unarchived` and 2 `restored` with the object counts back to the first import's + - GIVEN the docs THEN "Repeat imports" describes archive, soft delete, the archive sheet, the option and the return of an application, and the error table has `ARCHIVE_UNAVAILABLE` and the new meaning of `MISSING_RECORDS_UNSUPPORTED` +- [x] Implement +- [ ] Test + - Status: PHPUnit covers the fixture round trip (testTheArchivedApplicationsFixtureArchivesDeletesAndBringsBack); the Playwright test is written and not run yet, because no Nextcloud was reachable from the worktree. + +## Quality checklist + +- PHPUnit for every new rule in the service, the reader and the controller; the fixture hygiene test covers the new variant +- Jest for the section's request and the report words; Playwright for the re-import with the archive sheet +- `composer test:unit`, `composer phpcs`, `composer lint`, `npm run test:unit`, `npm run lint`, `npm run format`, `npm run test:l10n`, `npm run check:l10n-js`, `npm run check:manifest`, `npm run check:schema-l10n` and `npm run check:vue-demi` pass diff --git a/openspec/specs/cmdb-export-import/spec.md b/openspec/specs/cmdb-export-import/spec.md index d2f7fd4b5..927da9cfb 100644 --- a/openspec/specs/cmdb-export-import/spec.md +++ b/openspec/specs/cmdb-export-import/spec.md @@ -1,15 +1,16 @@ --- capability: cmdb-export-import -status: done +status: in-progress built_by: openspec/changes/archive/2026-10-05-cmdb-export-import --- # cmdb-export-import Specification -**Status**: done +**Status**: in-progress **Scope**: stackiq **OpenSpec changes**: - [cmdb-export-import](../../changes/archive/2026-10-05-cmdb-export-import/) _(archived 2026-10-05)_ — admin uploads a TOPdesk CMDB export (xlsx); stackiq upserts modules, vendor organisations, usages and owner contact persons for one municipality from the two CMDB sheets, matched on APPID, mapped by OpenRegister migration packs (kind: code) +- [cmdb-import-archive-reconciliation](../../changes/cmdb-import-archive-reconciliation/) — a re-import archives the applications that moved to the sheet "Gearchiveerde Applicaties" and soft-deletes the ones on no sheet, revives the ones that return, with `missingRecords: keep | archive` (default archive) and an "Archived" quick filter (kind: code) ## Purpose diff --git a/tests/Unit/Fixtures/CmdbFixtureHygieneTest.php b/tests/Unit/Fixtures/CmdbFixtureHygieneTest.php index 3da15838b..cc3dbd53a 100644 --- a/tests/Unit/Fixtures/CmdbFixtureHygieneTest.php +++ b/tests/Unit/Fixtures/CmdbFixtureHygieneTest.php @@ -188,7 +188,7 @@ private function parts(string $path): array { */ public function testTheFixturesExist(): void { $names = array_keys(self::fixtures()); - foreach (['topdesk-export-anonymised.xlsx', 'topdesk-missing-appid.xlsx', 'topdesk-shuffled-columns.xlsx', 'topdesk-formula-and-connection.xlsx'] as $expected) { + foreach (['topdesk-export-anonymised.xlsx', 'topdesk-missing-appid.xlsx', 'topdesk-shuffled-columns.xlsx', 'topdesk-formula-and-connection.xlsx', 'topdesk-archived-applications.xlsx'] as $expected) { $this->assertContains($expected, $names); } }//end testTheFixturesExist() diff --git a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php index 2d330258f..17e11067d 100644 --- a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php +++ b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php @@ -306,4 +306,28 @@ public function testTheUploadLimitNeedsNoOpenRegister(): void { $profile = new CmdbImportProfile(container: $this->emptyContainer(), directory: '/nonexistent'); $this->assertSame(CmdbImportProfile::DEFAULT_MAX_FILE_BYTES, $profile->maxFileBytes()); }//end testTheUploadLimitNeedsNoOpenRegister() + + /** + * The archive sheet is read after the source sheets, and the accepted missingRecords values need no OpenRegister. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012 + */ + public function testTheArchiveSheetAndTheMissingRecordsModes(): void { + $profile = new CmdbImportProfile(container: $this->emptyContainer()); + $this->assertSame(['keep', 'archive'], $profile->missingRecordsModes(), 'readable before load(), without the pack validator'); + + CmdbTestSupport::loadMigrationPack(); + $profile->load(); + $this->assertSame('Gearchiveerde Applicaties', $profile->archiveSheetName()); + $this->assertSame( + ['Onbeh Applicaties CMDB', 'Beheerde Applicaties CMDB', 'Gearchiveerde Applicaties'], + $profile->readSheetNames() + ); + $this->assertNotContains('Gearchiveerde Applicaties', $profile->sheetNames(), 'the archive sheet holds no rows to import'); + + $missing = new CmdbImportProfile(container: $this->emptyContainer(), directory: '/nonexistent'); + $this->assertSame(['keep'], $missing->missingRecordsModes(), 'without a profile only keep is safe'); + }//end testTheArchiveSheetAndTheMissingRecordsModes() }//end class diff --git a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php index 2e0e06736..b20bba79b 100644 --- a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php +++ b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php @@ -920,6 +920,110 @@ public function testOneRowOverTheLimitWithinTheSpanIsRefused(): void { } }//end testOneRowOverTheLimitWithinTheSpanIsRefused() + /** + * The archive sheet of the anonymised export yields its APPIDs; the CMDB rows are unchanged. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testTheArchiveSheetYieldsItsAppIds(): void { + $result = $this->read(name: 'topdesk-export-anonymised.xlsx'); + + $this->assertSame('Gearchiveerde Applicaties', $result['archive']['sheet']); + $this->assertTrue($result['archive']['present']); + $this->assertFalse($result['archive']['keyColumnMissing']); + $this->assertSame(['1198'], array_map('strval', $result['archive']['appIds'])); + $this->assertSame(['Onbeh Applicaties CMDB', 'Beheerde Applicaties CMDB'], array_column($result['rows'], 'sheet'), 'the archive sheet adds no rows to import'); + }//end testTheArchiveSheetYieldsItsAppIds() + + /** + * The later export moves APPID 1234 to the archive sheet and drops APPID 2 from every sheet. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testTheArchivedApplicationsVariantListsOnlyTheArchive(): void { + $result = $this->read(name: 'topdesk-archived-applications.xlsx'); + + $this->assertSame([], $result['rows']); + $this->assertTrue($result['archive']['present']); + $this->assertSame(['1234'], array_map('strval', $result['archive']['appIds'])); + }//end testTheArchivedApplicationsVariantListsOnlyTheArchive() + + /** + * Only the APPID column of the archive sheet is read; without the sheet or its APPID column it is absent. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testTheArchiveSheetIsOptionalAndReadForItsKeyOnly(): void { + $this->requireSpreadsheet(); + $cmdb = [['APPID', 'Applicatie Naam'], [1, 'Een']]; + $cases = [ + 'with the sheet' => [ + ['Beheerde Applicaties CMDB' => $cmdb, 'Gearchiveerde Applicaties' => [['APPID', 'Applicatie Naam', 'Bron'], [7, 'Zeven', 'P-0007'], [], [8, 'Acht', 'P-0008']]], + ['present' => true, 'keyColumnMissing' => false, 'appIds' => ['7', '8']], + ], + 'without the sheet' => [ + ['Beheerde Applicaties CMDB' => $cmdb], + ['present' => false, 'keyColumnMissing' => false, 'appIds' => []], + ], + 'without its APPID column' => [ + ['Beheerde Applicaties CMDB' => $cmdb, 'Gearchiveerde Applicaties' => [['Applicatie Code', 'Applicatie Naam'], ['APP-7', 'Zeven']]], + ['present' => false, 'keyColumnMissing' => true, 'appIds' => []], + ], + ]; + + foreach ($cases as $case => [$sheets, $expected]) { + $path = CmdbTestSupport::buildWorkbook(sheets: $sheets); + try { + $result = (new CmdbWorkbookReader())->read(path: $path, profile: $this->profile()); + } finally { + unlink($path); + } + + $archive = $result['archive']; + $archive['appIds'] = array_map('strval', $archive['appIds']); + unset($archive['sheet']); + $this->assertSame($expected, $archive, $case); + $this->assertSame([2], array_column($result['rows'], 'row'), $case); + $this->assertStringNotContainsString('P-000', (string)json_encode($result), $case); + $this->assertStringNotContainsString('Zeven', (string)json_encode($result), $case . ': only the key column is read'); + } + }//end testTheArchiveSheetIsOptionalAndReadForItsKeyOnly() + + /** + * An archive sheet with more rows than the profile allows is refused and named, like a source sheet. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testAnArchiveSheetOverTheRowLimitIsRefused(): void { + $this->requireSpreadsheet(); + $path = CmdbTestSupport::buildWorkbook( + sheets: [ + 'Beheerde Applicaties CMDB' => [['APPID', 'Applicatie Naam'], [1, 'Een']], + 'Gearchiveerde Applicaties' => [['APPID'], [7], [8]], + ] + ); + $directory = CmdbTestSupport::profileDirectory(overrides: ['maxRowsPerSheet' => 1]); + + try { + (new CmdbWorkbookReader())->read(path: $path, profile: $this->profile(directory: $directory)); + $this->fail('TOO_MANY_ROWS expected'); + } catch (CmdbImportException $e) { + $this->assertSame('TOO_MANY_ROWS', $e->getErrorCode()); + $this->assertSame(['sheet' => 'Gearchiveerde Applicaties', 'limit' => 1], $e->getDetails()); + } finally { + unlink($path); + CmdbTestSupport::removeDirectory(directory: $directory); + } + }//end testAnArchiveSheetOverTheRowLimitIsRefused() + /** * A text file named .xlsx, a .xlsm and a CSV are refused before PhpSpreadsheet is touched. * diff --git a/tests/Unit/Settings/PublicationFieldRulesTest.php b/tests/Unit/Settings/PublicationFieldRulesTest.php index f5ee7d5b7..84026c2a6 100644 --- a/tests/Unit/Settings/PublicationFieldRulesTest.php +++ b/tests/Unit/Settings/PublicationFieldRulesTest.php @@ -173,7 +173,7 @@ public function testAFragmentNeverLowersAVersion(): void { $this->assertSame('1.5.5', $merged['components']['schemas']['usage']['version']); $schemas = $this->register()['components']['schemas']; - $this->assertSame('1.5.6', $schemas['usage']['version'], 'topdesk-cmdb-import.json bumps it past publication-field-rules.json; value-assessment.json, sorting last, does not lower it'); + $this->assertSame('1.5.7', $schemas['usage']['version'], 'topdesk-cmdb-import.json bumps it past publication-field-rules.json; value-assessment.json, sorting last, does not lower it'); $this->assertSame('0.3.5', $schemas['connection']['version']); $this->assertSame('0.1.6', $schemas['moduleVersion']['version']); }//end testAFragmentNeverLowersAVersion() diff --git a/tests/Unit/Settings/TopdeskCmdbFragmentTest.php b/tests/Unit/Settings/TopdeskCmdbFragmentTest.php index 7ef15d1f9..df49a4556 100644 --- a/tests/Unit/Settings/TopdeskCmdbFragmentTest.php +++ b/tests/Unit/Settings/TopdeskCmdbFragmentTest.php @@ -58,14 +58,14 @@ private function mergedRegister(): array { }//end mergedRegister() /** - * The merged module is 0.3.8, carries the six optional, titled properties and allows BBN2+. + * The merged module is 0.3.9, carries the six optional, titled properties and allows BBN2+. * * @return void */ - public function testTheMergedModuleIsVersion038WithTheExternalIds(): void { + public function testTheMergedModuleIsVersion039WithTheExternalIds(): void { $module = $this->mergedRegister()['components']['schemas']['module']; - $this->assertSame('0.3.8', $module['version'], 'a fragment sorting after topdesk-cmdb-import.json overwrote the bump'); + $this->assertSame('0.3.9', $module['version'], 'a fragment sorting after topdesk-cmdb-import.json overwrote the bump'); foreach (self::PROPERTIES as $property) { $this->assertArrayHasKey($property, $module['properties']); $this->assertNotEmpty($module['properties'][$property]['title'] ?? '', $property); @@ -91,7 +91,7 @@ public function testTheMergedModuleIsVersion038WithTheExternalIds(): void { $this->assertSame(['BBN1', 'BBN2', 'BBN3', 'BBN2+'], $module['properties']['bbnLevel']['enum'], 'the fragment adds BBN2+ to the BIO levels'); $this->assertArrayHasKey('roadmapStatement', $module['properties'], 'the 0.3.4 fragment still applies'); $this->assertSame(['name'], $module['required']); - }//end testTheMergedModuleIsVersion038WithTheExternalIds() + }//end testTheMergedModuleIsVersion039WithTheExternalIds() /** * A CMDB import sets the usage status TOPdesk records from any state; only an administrator may. @@ -104,7 +104,7 @@ public function testTheMergedModuleIsVersion038WithTheExternalIds(): void { */ public function testAnAdministratorMayMoveAUsageToAnyStateTheSourceRecords(): void { $usage = $this->mergedRegister()['components']['schemas']['usage']; - $this->assertSame('1.5.6', $usage['version'], 'a lifecycle-only edit deploys only with a version bump'); + $this->assertSame('1.5.7', $usage['version'], 'a configuration-only edit deploys only with a version bump'); $lifecycle = $usage['configuration']['x-openregister-lifecycle']; $states = $usage['properties']['status']['enum']; @@ -167,4 +167,24 @@ public function testTheSeedModulesShowTheNewProperties(): void { // The base seeds are still there: the fragment appends, it does not replace. $this->assertGreaterThan(3, count($register['components']['objects'])); }//end testTheSeedModulesShowTheNewProperties() + + /** + * Module and usage offer OpenRegister's archive state, and the usage keeps its lifecycle. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-archived-applications-and-usages-shall-be-listable-in-stackiq-and-shall-stay-out-of-opencatalogi-and-portaliq-by-default-req-cmdb-017 + */ + public function testTheMergedSchemasOfferTheArchiveState(): void { + $schemas = $this->mergedRegister()['components']['schemas']; + + $this->assertSame('0.3.9', $schemas['module']['version']); + $this->assertSame('1.5.7', $schemas['usage']['version']); + foreach (['module', 'usage'] as $slug) { + $this->assertSame(['enabled' => true], $schemas[$slug]['configuration']['x-openregister-archive'] ?? null, $slug); + } + + $this->assertArrayHasKey('importAcquisition', $schemas['usage']['configuration']['x-openregister-lifecycle']['transitions'], 'the 1.5.6 lifecycle is still declared'); + $this->assertArrayNotHasKey('x-openregister-archive', $schemas['contactPerson']['configuration'] ?? [], 'contact persons are never archived by the import'); + }//end testTheMergedSchemasOfferTheArchiveState() }//end class diff --git a/tests/fixtures/cmdb/README.md b/tests/fixtures/cmdb/README.md index 415cef21b..219d5b379 100644 --- a/tests/fixtures/cmdb/README.md +++ b/tests/fixtures/cmdb/README.md @@ -1,6 +1,7 @@ # CMDB import fixtures -Test workbooks for the TOPdesk CMDB import (`openspec/changes/archive/2026-10-05-cmdb-export-import`). +Test workbooks for the TOPdesk CMDB import (`openspec/changes/archive/2026-10-05-cmdb-export-import`, +`openspec/changes/cmdb-import-archive-reconciliation`). They are used by the PHPUnit tests under `tests/Unit/` and by the Playwright test `tests/e2e/spec-coverage/cmdb-import.spec.ts`. @@ -19,6 +20,7 @@ in the script. The "Invoer" sheets are not read and stay as they are. | `topdesk-shuffled-columns.xlsx` | The same rows with the columns of both CMDB sheets in reverse order, and the header "Vendor" written as `Vendor⚡`. Reads to the same rows as the original. | | `topdesk-formula-and-connection.xlsx` | On "Beheerde", "Applicatie Naam" is a formula that would evaluate to `Evaluated` with the cached value `Rekenmodel`, "Roepnaam" is a formula without any cached value, and the package declares a synthetic external web connection to `https://example.invalid/`. | | `topdesk-no-source-sheet.xlsx` | A minimal workbook with only a sheet "Blad1". | +| `topdesk-archived-applications.xlsx` | A later export of the same municipality (`openspec/changes/cmdb-import-archive-reconciliation`): both CMDB sheets lose their data row, so APPID 1234 and APPID 2 are gone from them, and the one row of "Gearchiveerde Applicaties" caches APPID 1234. Importing it after `topdesk-export-anonymised.xlsx` archives 1234 and soft-deletes 2. | ## Placeholder data only diff --git a/tests/fixtures/cmdb/build-fixtures.py b/tests/fixtures/cmdb/build-fixtures.py index 84a4f72aa..74c870ee1 100644 --- a/tests/fixtures/cmdb/build-fixtures.py +++ b/tests/fixtures/cmdb/build-fixtures.py @@ -65,6 +65,7 @@ ONBEH_SHEET = 'xl/worksheets/sheet2.xml' # "Onbeh Applicaties CMDB" (from AIA) BEHEERDE_SHEET = 'xl/worksheets/sheet4.xml' # "Beheerde Applicaties CMDB" (from APP) +ARCHIVE_SHEET_NAME = 'Gearchiveerde Applicaties' # Placeholder cached values for formula cells of the CMDB sheets, per sheet and # cell. A cell that does not exist yet is appended to its row (columns are in @@ -367,6 +368,50 @@ def formula(data): return parts + [('xl/connections.xml', SYNTHETIC_CONNECTION.encode('utf-8'))] +def sheet_part(parts, sheet_name): + """The worksheet part of a named sheet, through xl/workbook.xml and its relationships.""" + parts_by_name = dict(parts) + workbook = text(parts_by_name['xl/workbook.xml']) + sheet = re.search(r']*name="%s"[^>]*r:id="([^"]+)"' % re.escape(sheet_name), workbook) + if sheet is None: + sys.exit('sheet "%s" not found' % sheet_name) + rels = text(parts_by_name['xl/_rels/workbook.xml.rels']) + target = re.search(r']*Id="%s"[^>]*Target="([^"]+)"' % sheet.group(1), rels) + if target is None: + target = re.search(r']*Target="([^"]+)"[^>]*Id="%s"' % sheet.group(1), rels) + return 'xl/' + target.group(1).lstrip('/').replace('xl/', '', 1) + + +def archived_applications(parts): + """A later export of the same municipality (cmdb-import-archive-reconciliation): + APPID 1234 moved from "Onbeh Applicaties CMDB" to "Gearchiveerde Applicaties", + and APPID 2 is on no sheet any more. Both CMDB sheets keep their header and + their empty rows but lose their data row; the archive sheet's one row gets + APPID 1234 as its cached value.""" + def drop_data_row(data): + xml = text(data) + new, count = re.subn(r']*?(?:/>|>.*?)', '', xml, count=1, flags=re.S) + if count != 1: + sys.exit('archived-applications: data row 2 not found') + return new.encode('utf-8') + + def archive_appid(data): + xml = text(data) + new, count = re.subn( + r'(]*>(?:[^<]*)?)[^<]*()', + r'\g<1>1234\g<2>', + xml, + count=1, + ) + if count != 1: + sys.exit('archived-applications: APPID cell B2 not found on %s' % ARCHIVE_SHEET_NAME) + return new.encode('utf-8') + + parts = replace_part(parts, ONBEH_SHEET, drop_data_row) + parts = replace_part(parts, BEHEERDE_SHEET, drop_data_row) + return replace_part(parts, sheet_part(parts, ARCHIVE_SHEET_NAME), archive_appid) + + def no_source_sheet(): """A minimal workbook with one sheet "Blad1" and neither CMDB sheet.""" main = 'http://schemas.openxmlformats.org/spreadsheetml/2006/main' @@ -416,6 +461,7 @@ def main(): 'topdesk-shuffled-columns.xlsx': shuffled_columns(base), 'topdesk-formula-and-connection.xlsx': formula_and_connection(base), 'topdesk-no-source-sheet.xlsx': no_source_sheet(), + 'topdesk-archived-applications.xlsx': archived_applications(base), } for name, parts in variants.items(): write_package(os.path.join(HERE, name), parts) diff --git a/tests/fixtures/cmdb/topdesk-archived-applications.xlsx b/tests/fixtures/cmdb/topdesk-archived-applications.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..68cd6a4d2c6dc17d18804d17268edd5747a8e105 GIT binary patch literal 151786 zcmeFa3piA7+dixinouFBn0BZnZG;LllqB0qsmN562&2e;W=bRqg^c~!Q$i(VGc!p@ zvfmqw{kAc-Gh?>@nNpeG^E~hOKHv9#@B4p$bsRS{>t5%5uJgLCdvmR|W*y5!pO;UZ zhlfXy{kO%ULF%PS$TA)tiFO@4%megj%Y|%M`jbu+b(r3}iOfONU#? zjvU7n?)7qYjBr%#0FTLqX(zglr&{Z6fPw4x>83QM%tuFQ3YFKxPpo^xCEhQ!)Z znIOAKK(SBFUa0bvzFM&Phg*$ngPS6pV)J~H@ih6>{S|)I6GT>Op5f8UDSoBzCYYyU zRAfh1n}AODuERRtt?LoqB)j&;X38TO&*B+Ju*v1-YqbQ|%2hY<4ht_~g0;Hw50*1G z^;&g3Z@yG_e0@ecbGj|So-rX`VHet%p=w?2sj>;pYr+-3yt8|^O|au;u$+en!2X2X zzGm;Nw1@LQE40S0Z11YO#QW&`4d_#yrR$T8l&tD;>pOUFsAjM|FnhFul(}STJa;>^4+o?6i_|pd3MR&jyQhnW#7j&t}TQb(lgzY zg*#l)A-m34G}osN0@oyuxhI`}XoEy-5E4tJuTu2f&ztu+=1ocERLrs#pXa1=N852n zOy;X*58B&C_e_OFmVMk4XM0aw-Zg^o+1~QD4mYeJ-uzOAgnq-zx+HR}ETzI~UhDXyjE!lo#0pR7Oaw7He-j}8GI z9#OVW-0hXzpto;cg+gy~`g7_tb7<#Y@k+cr;mCP-df^i%u`9b$JDzlA+g+fkMGg9% zztjIo#m}zO4X<~1bwZ4Zyw-{e^=9OSh$=5!amW9okXL1jFMOv{`YhIJp$oG}8X@xGz2Jh~s z*Nm<`C+@ae;fU;xH3l&gksBZKcIuqTzKrcXl@w9#Ud6w`p_t&Baqvaug5}foYxBjg zKiG)to3?%zUp&1V9k2Fs&eAm6n(<}!eD?D3BbK7huuO*nWwU4VU#!AbG!t-^Xp%5x z>%sLOh|eB9r~!WVIDQ34j&@3|vAVE)ZpVOmkg#cLN4K7(#N}x0Nt40bo_bfWub08k z^ofU+!(S|`1lnyd%Sij`rRg7TvCgk|%>x0$Y)AQl2Ssfr#g*fi>S?rr1jnPTnh`TTg;MNZ9wJKJ@`dX)v^tUer_XWrZ8aY698 zv4b1u1a;Te~JytpbNdd# zA3G5SSsNxQ!;C-s{9aV*2h!sQ8#@z99^hI$Z%yPIvNpf(y>dG3_Je_KOu$ODrbFI` zo2JJnU0uD>h&A(HN1V^k4|m%u+J79rm{;D3etI0i!#xDF-92~cv8VMi_AI~Q`w;j( zGb|kl-)0GPL^XSsV9)M;%=-D{817@lV`bPNQ6W@CRPer} zw!*%)(VntA@Wq$#=)()z6T6ZRG!Rp_YGoN%j%^hK#vuKpPHehYo2atkz-wLkbJeus z@{O%vc$h+Ft89->u|fDTLUOp#v-Z0aQS0(pZw}@!n@Lu*2G|Mkf7XgNNazVk_P}{@nK@%GPMD{+hy2O#!jD zPet~MJuZbfZ*m5MSz{LG)HF5k8xwVdE=WjqT|wLr%w^+8*EZ zMScI5SjH)>%v-YOzu>E=Zk0np?mJmcpEA=cXC%1;a2Kt0Hdcg($Iyj`XZ^1^$oa;# zYc9@8-#+$zn~dVj!#dBc^TVgxF?q|qHneIMxfaGHcDwd($9y!pDi4i3SG?1C?5=kV z1ho9(`$INR;p*ozsaN#%bsL%66R?NXDa>l`R+?#6YtvAFQb_I2G%DYI$A}BE3Z1>Hz>n0vC9WLSHL|05yIUEG$ z)J3?pBt7riZ8m$WTipT;d`UdlLh&qa^j3M&w|C>0$|fBI(g|}|*2=^AK%A*rbycND zUWVJ2OgokAnCTp7y4!?`gXSSwM@>r@#hu)3eF%GVYmcE#6Lmu6Ek4G$7KMTAHXCru zy^h6hmNNE)oT~xaGK*Xf;6{=aYtqA7Zlg<{*F_J~_Ei=`L_U|gzcCIZvr^V?uUSS> z?v8zbG~C zWOQX#5-rVg?M|Du8QPG%VuHO2Ea2_pC>3MjnEH9wdxoR$7Kmr&Goovvzh0l3z`Dge zty!^x);xg1WodM%e5^i8EGm*h#I}S)4F8~(l;OaIufv@3U=bQU{zhpiKo|N zq%XEXRNGI$Z(sHVM#?OIbYOcK^l|m%z#Eu_Wo*f9IQBLMa;UVSHZJmXSGqVYeCr|o z*+eTNj}DdEsro}|9u8k=lNrNqT=s4^Qsp0b{hC%qo^VcEjT_8|*Ilyd?YEc$Asp{rgnTK(Dyp+4{$xnt)Zx7|&v$&7*@)dF^QM4VwQ^(oQC>kZRSE0eo}=X)(t*F>Th#E- zEm^3xFV#NBKBhoRmm#%k;^~_wfQw@%JM{8!Wu zY9&%0BkX^7bb{4gk|rjv+037^nK@fo2-KLgnAfwJ_ph}BA9cUv-B~ze9CsGTCzKP7 zr3_8ISa^AW=3z6hV>5rvCQs~cg90#qRLElvF4@x z5*Eiw{Uj_-mtGUMxL6uv91}1|=!d^0hocP9SFn8j_K_XUH{a@?h*i9It&!`Ii!h3~gc zlD&S~K#t$k<#9^ni9nNveNV5h-5DGhd{12LgkHvIr0zbC@bJLkTjFB6RqV~<`)tJn zj|bR_haFG65x#U&w_)_^hB&?C=u5J+f1~}DoAX-^YJV?*L>S|-8iL%_jpa4B9?S{yY%5ri+LOND!n<68B z0ZuEY?F)LYXLYh6vO+mcE@-EY)rO|XXyr8VAX@_~yT-`x0LB^-9jh1K53GJv{0A*g zDPP|g6s2c%ydkn&`MO+?l#bQ9rpPGe>*7Hd46LpO$-}k^ZntQ6>$y z_5Bl{+Xe2Cj+s111FO6HuM{cruG*>W_Q-!MUu18}=r@q|`zOm?I3R^(M{9-F?>hay zK7aXx*mZkNOYSBru2^;Gz~zjLWk+MqI=!^3Fhag5d;Q|@5tYo|^DQsqL>vS*WtyD# ztJ$S??x1t}(e7oHFNP;wJ{@82{Te3%^Ph85zL#FsyG25#rf=+t`-w-oP44y~0TU4z> zAdtXoPF^-b1FwN!-U#Xq2kxj9wI36Ro#)k|-6JsWiXjprL9beXPLrbI3j+BJUL)4M ze%AYSci&5aO~g01i>y`?5)9<$Gv{4CvP^5uy~{D+UenDx_N=y-5{wn*(-H6?@ZJ;i zNm#G->e*%|qt)U&1@k5Ni~v6U{H5#ezmL)~Y2MuKxLWOyV4yUgx#;o{0j)K@mp5wn zif`EwyxLw*FjkgNN6d#HbWhALAyWI**)2}ltHmz}<}30UN&55yO4mJjFLl%;d`o-n zYBgKIz=M3|8<&p=YpwCW9CNg{dCQK;)%G5Ou^N0jKp%qWJ+X%g>yN#X*y<#*Mm#_; zUx&|Vn@|6m(scpvqmG%FZf)PQMlDJ((2&nue)))))|$Y}8+Cfax9%`nWB*(*_AH-{ zf)7FBp4g*=NS#+LTb&%&h!+axU*a=T^68f>UHACC)NvE>ZSBEp)T#snuko25SUw`9 zwI=9t%<*2+Z9B5p*mnrVI`HYJ`VcnW6AMmQulwrRHmBM(;$wpOcleBs`1DJct_yh| zrEAi>t$lKh+JaypjL#gjd<3Yq=E>!aCwj%V?+{sQFC-M}$ES1Dhp^?I*wci_6R*y0 zciOX7TuLZEh|fsZr+-`Ny3qGhdM4r9+l|($?Gy@p#%Hd-d_-1jP1xlaz24^SI~>>A z9}o@|A?FQ}9}GJ)!(mKBF@}{X0w7J$oN@(!^A@J$tR%1);z! zK6A6>BMMq;A}(*#?+usTQM=aORwy=)PsiMcpeQt=7|^d6@ZbDJ?Ofukl`&^)&DWNj zuf>_K{ZD@h4PNm7B=qpD)WgGRo41Z^Zj{?H9_{xq_Q8o5|4-MS#Ri;+3CytCv-S4= ztqv+l!DA2ApnBraD)!7Va=vAF`*>vd!`NphVj?oEEVtgiy4B%&QtU7b}&v$H(Nn_asum!KA0Myw!a z+6=VN#;$h9oK?l4?L;Vcg z!kkR%z$Kz%mpx&A4%^}y-8W>{x#}A2V2mWAc<6;?Q2M;9DNSiPi~{05qZMxH-B-!J zqE@c@FVATAbIxe9a1PE%=fnl3$Z=zCPVU$bFicxO$7{VNL!3hat(_!&Ds*q=)qCpm zO=O?tfg{$=Zx7HmS54QUo!}2w%I5$jXG>wBlD7Gq1nk%I-?u(gbA4>)`X+Cx`}F4( z_mdtYbKT<=!){H5i&E33hqr`mmEP0uBK1=2_=5iaHxm;s(Xlp7#X4_xw{7v*AEFOp z_N!cyFzmca4SlB*i$I&Bqpv@D7V_@oi0y)@|BCzNE-oIqUuGzFlOv2z*6Pyg!%R^! zLMJzxI^uQnVoH;`!kdZPqhYF6Jtz#|(t2ZD3S$AgNyFR-H$@Hu4PF-T&rzp9?6 zv$-eFXXE0|)Z~sPkJARQkOc|R!WXC0J4!NG(z_d1J|aG)yk@et#62q*xLcff_N3iS z)70VTcAKvIl%72uE@^uKX+#X%Z|;2obkE^x+dXfwPdKnfxP??TMa$!An0%-Dmf#Hu zU3{Na>La&=J=L`Lwi8S5R4ae*`B?w^g>l39N!N$!cD&Js3AVgZyaE)12i?}~d&)ND ziFL`<-^m?3>L_vMNO37f?TL=zHeH7lWnBlg*3Mr1WA8m_W0za=_MSOfwpCsP=l;_6 zO6$DS!3y5c=*erBXfvaN!Y^$hX$9s(f}$_SAgu@4r;`qtw3w*w__=)nhDUO<;OoF3 zF_C0L(Cm%On<6LeIwRdG-FF5@?SY0&ORw@0KiIRrZ~50zBv8QJ>OK?)4o;{WQr`cr z^IS3T==zIR_uW*!+%jxz@4Y_JLUKZvJS{B{nWMkcz}5GfD}OzeBb}7EC5doqQWxXh z_ORmNWmelqo%_dJPEc+sMJH)Uj)$vVb|b2%Pj4P{1p*H~Cg~gnC}0nUI>A0%K;_0) zUi0=0wGq8vVyX8!FjMuC{r36;=de95+)tm)(H%pap#bryOpKy#AQF5>Kt@-W3ov+PhT?+Uho9VUbZ!*$TYhn77*j@FC@UzBo2zi_@W zl~nLt?#Tr?&y6>iKfm?{q${{5jB!ZYHR@}~#M&3PFCFk@Uha_D$3OJYLvGv}5x6eD zNvtMqrB_vG_-=BTY{bhGRd%2YSE{1JUy{QPM%bUIx(@2NTon;+PYzRz_K^*4mGW`?vHMo|9#D>L`fzA<^cBe(S`uWiJK%oODS$z z{rOZPoAKKw|MyLc#_yXFi<^skZM6?=4o}uTWVl1X_qOSdbfKZUCc!HuloOY)-er>L zBWKE^&A-dE-ghN91MIgl+GJT=W0E$%VFr)yO4E!4A&I*t_x>ZaphSj=p4hHsiEAZx z*=oyfHcir&HO%1iJ!zVeBJ}L8NdR_&{aH^l`mw$8nR4RV)w@n5hRT^P*OuL7+UR={ zoN>(WWVDHp;Ij-99kCZYi7O>u+|ZWTY?`PoVVJSpcb92KlF-?^CVnf=DkrX7{bDNQ z0ekmc-|Aha4Zgd;8QOlkqD}Y(&;CQ}gN7L^eCngp$EQBK^j`o*clXgO0} z?Ss2an|$rS8OQzXqD_PaTPPz1?A=s8Nr@M>+6OnACTstt^+Dyt6|3z`5+BGl^Jok2 zYOeR)0M5|z+YoKSD|j%&}<6aH#a9~D;j3-`(8B7NEOPvYw~Dio^s;4)pn;6 zBjlQwYb!DhcNk{)VF15c>xzx@B#KCk-_Vxc+?=Q_ZJ5F9d)PE1S?JDP6aSTWloLf( zkDDY0%QY|4mfqFe;CmRHan$c{w26S=oeY!jjjFAo|mf)k_tU@3jA zXCF^rin^3CyT7W~{9aG9D^VCve3T6)t|3l38QVNL3+0Zqa^lS6J zZ;NIXq8O1Q7c9T-QE}T~xTnZGB~p_jw{zq1{6gXEKqtCRYTLp4;*)M@I5##gE- zuh7egbAMU-E9tIOF3IE#O57cTT#+pWk6FI|fC>UXO318SM;w@Z7h zwoSbwCwp>0WO>Y1ZJwBo!cQA<#ERzug%zNa21o1z=;zLztI57Nx+C{|2181yBuY@qaA~ax>Qe2MaLU#^mfnGzjWDSN5-18ZhyQZr2&bk4OYc2)Yzaf zBvll%QZffYZYj{mcrzB{A!r&E&m`sIu?#p)62&|;fuRgmBUmALR*o^=dz$8^z?{#m z1JTB@m8A~s~qZekv5Da>^Gs|WSWKUs{H4v8FOa2!J=3M0oX&IW|c6%;^DNG zVz!S?v)jG`uUG`(9P-4Gb% zEz27*-#5#wX0&)9?sQvbiQ?T0(XukwyxU&-kNtB_Izj^$rp^Z7HA#J5n-RBItsspO zdM>Rx1jv}B)lmnIcK58DolcyD`lVNjVz83&GVrg1%vi z_W2*1BtBf2fh)N+i@{hao&hC>hRo2Vqhh)h2d}th|qB)J|YA_+5gP% zARu#+pS+$x&N$ftA9)Q=VWrZH)X5|N9&(P!95Y?>Kbr`N$8{W z%5HLj4vaojf>xA4gf(EX2qhW#$2n$tE!^z%LQ}b2Q<)XgH!D#;_yi)`U>+XoR@2R=JZA zGH@qWzoy1SEU-osq5%qVMcausfs%DcML|zMSlFEq$$otZ6@r<&S~Azqf{m49J=uR4 z#z|05Gh&vI$)eU|ismo~ZdjMP0%%LwEVEl5MI%Byr-7E9(~|QzMCeY@X7k4NY=5 zF;?lKF0<`;rc90q%UKf-DVfU(MWHF9Q+Qg zy>eh41?i$IDPf8%J6ROhdppf4^MR)OK;UtNH=!0QWQQ-NKnE^&yif+)u$$*{*{nScKpwVU>lW~DoI%P z=@tlvGX97tz(|*bFj^kvU?}i;TWVAA@?g3+zGc*4$kdt+R# zhC^L`%}YaT$o8Tz!r+nerm>FrGj*gA4csir9`~j57)dATNQV*Tj5;cwFo9@@SRm`< zS`XLvz4!>J>^)o6M3vC>svN|QDbk)cN}e}A{K4rfGi)kC90kJx>Nw}dlZa@FU=%#_Wh)v!oA$+Wbu0=1fXu4~WB zt-O9&WZjVaNO*eJnMtHkhjARtj@~8aW@pTq`~+!>A5U9oJKf9#@V1Q9ue9t#(!Y1z$4ERm|cy=5S-S`AT1Ro14VV- zHZH_#-oe61&-23G??1Jd*ZEaOB`%%B*7MKkxBYYa@h|_JKDrn-{?88k@76~Y!#d%# zx7_ljp{V~C^&9B4T@=Z4Q_z>AkP8ji+$4mp4y!egnb$H+vMt1=6AFxRTwd+`pQ|5Z z>me2x?d?eg(bb8l|jyDjhi%qSS(aSwUP|QJ=B{PbL4lUO1IggFRBY*KC8+zAl-=*$4igZNQHRk!Ef^ zLN|1!W`3P>>ljnYZNgtq$JWA2VA);Y>|m>2GGSX(;zmAqbL?Q7nQQ!O)wxpwi)Y)| z1gEKU8*j{RvD58aEqmt@>!)4*FK;5&?7ew}R|B|`YSQ2zs@Xi3m{B)P@?e{9Oe4&` zg?HPX%mKbI(yxMe_OZQHRF%g6*`Z4 zfc)dh&Nr4UfFlEk=3kB?}oLnoElbkiP*{_pA(_a^!!`5qtjy;bQyQX{R0 zuLWr;g|8tO^2@z739lE7+dD~_Dk*dx!!1;f_#N-pEO_s`E^zIJM6n8KV7#iE8te#* zQbHT!<34m)%SW>(hTeI=Q+v4^bQpW{FDo~r9pe8>1KD!3C#TsB&T%!+uCAg5`ZSt` zr==*Xr0&}lbk_FENh>pI>MNHS%&zc=w0!0A)Yp50JZ!(}TV37&Xd0ot*iKXkG&plD zdVf=UR7JgVMB2WCL0MyxSDR!#Ag3IoJ)6X1EBck?(qA76YOsB2gnbgXhkcL_O{Vc_ zbISSY`!s{*ZNC^>A*g;lcl3J9!V}U~9Ms8ptsNv~_tnJe;RgL?rIZZ)8BEgpgDn|5 zb%QkQR-Lwbx*@E&Jq5nyWL(ziGZE26&Ejbl`wyl)e{C3a%kJeF?346b((|1rhJJP@ zO|6nQY->6Hyu#q1>kEkBSsVS9@E2+44wk=oeJ-fb?(12rR~sI+j9A~%Q^^Dz%LJHa zUilTi+ref?YjhD9Qv}9t7+x1yuYF2;o6CnCBaiRsxxNJWz66B7TnZPth_r4f0yh?c zo4DX_@95=c0X}8{nzNR|gDsHOLq*_WD!Myfa#CT}C9z#UdQ%u#f-eUkp0gA#0zq0Y z6oIKlU>X-YpR1nAE5NZ=0Ml1X;lWps*22YLkz%mu#--YKyXv{V2Kc@Pguh-27qLcK zZz%?EEe3Dnf}6VP<-Y-Zd;@5HvlJeD9cg{A7_3qZR^@_wyXuj00eo)(;%}G2MQoAQ zdd1+A#bA9dcnVv49^hCW9!K!#!z5*^B>wE}9uO_dVladY{)MeQAK;r02+!wYcSKs> zDhA&!20L@X=h)f{03Qng%>_&0!A?kPzhdx%Vz56Ke3P3V=^cQt5FlQ-6fWY7w2mqU zM;C)*xZua!^i+xf$BF=^MN8qqcaYXu#o+8>a1IyT-c8T77~oqB2rpg=7jZ{gmllIR z6obpS;Nfn1`6YmlC4lCVrSM=+q;*R%xV0GE#sx27Yexh4FaUAPQn&~VX+2g99xnz@ zaKQ)J+OdFRSb!;ZDLfc~v}P28nZ;lh7krhwp6h#n?|VS_`=xLZAEfn~67bp*u-K-h zT};_sFTWJnC1BSQup1Yg&emQD;QIs+|Fje?@)T(uPy!At0YBn`Z}-qs zsRA6U0+?1Ug$IWrt>a3-@g-ms7u>)@&-F9F_cI{;^HR7-1k(C-3HVJ3IF}3V;-Qyc z4ft3MXs%ug4~|4ymzRJmO2Cy|@K_H$(iZ?<4M4nRDO@B5Y28r*?koZ0xZoez+G_#F zY5}ITOX0zBNNYj~c(Mdc~T1%k8>(F3HF1Wg<9;pGq*9Z`ATnZOSMOyDfgLk393S4k&Pd$|;z_BKP zY12}8a5~aj0}a+hgF#&I$DVqw%>dtKKzQ>~xX25n^(i#?G#Y${3;vd^y#?^G1<>5G z6dwE%X?+b1zK#alaKXFS+FJp9Z2Lo1}XY02m2eru?4u<4=^2utM<);bIEJ)s=z?*u2idRe~9$-oJXr7K2xs3J+vNn2c$Y)^mFIBoAy&~=QVB!?sl*9# z3L9s_Ns9)8MeAZpauV=8iFlD;_XFpgv1oZ3L7K`bzitN}2a$1JZ#dZB2+r(rtOJlG z=5g$&Bo=SReskn+HY7Y-$4TGQ-9R4Z@-_R7LN6V8Inaf#cjzY z0aVPSMu!Wro>RE8h2IIT8EQyg#wklYV-*`Sxvj$mSkEa-JX29P_IHFQZzpCh=Uia2 zL;bv7^dm7ETU`DRc5^NN(UZKks_j^vrLy z{QsMo&VddWR6VCG>0-m|#E6|?3USKP9vL4q`FDi>2N#&4C;!#*pSUzJ`od0(EsDOMq}iJ> zxYmEVo<=ZxgNJKj89qyYJ$8%zW2nDL{r9Kj#tXQflAFURas4W;tq>2Fr4c0mzV(aU z2hQDn{=Q-V)ty<0Yi_TcZ5=mVr<~*KzUzH%d(bEFQ+~jf%hp${%ewjO^n?}0wK9V? z7g8VVW*wqZKo(A_vk69dFZeqw(D>D9P7&;z-4-MqfJ)$JxPDPQf8&Zv-=FF~KKauj z{)~qIMN@$#`QD~_47 z0XQ_0$azeaN{IZ~r(UujM@6%ruKZK|$0y$=nLi!k&uI8FL;P5B|NBM*P1?Rx8*$>> zsamUZ6Z(=$_r~Z+VMDiHXK?wyO*Q&INPi5;-_swvBChnO37t5$B>i!9EGno@^E&!_ zS1+bN(UV;1j|=oh(USCseG!iEW~`_F?^eB-{-klGKgyg@^=qlOA4&5nwl7lsQ~k#$ z-_oBy9pcYu_%lQNNPqrEMgyESf!f16iAPM8P=Z)aZ{(1B-zguyV-j{)wOa}6Wd~!G z`d5QA^OXyIOH&)3-;Zu)-+Wu>PM)BNYFaR-J5sA;SM3Qhl5S3^H6&LP;~{LOzL^lw zy@q?PIN7?4C&=A#EP}wvt7**pqDX{ypzUp;`})ae1P(rTZeq+7%Xg50-tXuG$p&Hi z>n%3K9nR2WFOf*RMlFWbTQ&ql*+g4>=xrQr)-oLLa>3Fkqeu(^;2svmq-k*N3gq#0 z5xdhPX|y3X(R?C^;Cjy6X7{XVl|6fB1p0QABt_Ra2>Lqmh({-487VP_^bkJp1j|P&NLc=RZxt z^wmE@49wm}_k~BKyZ-PDoA}Biu{9TQ)*^8s7qRKccVfR0E@IRoal#McOK_C*8 zICqiwEf;aeB5@ZNaS-jh;wZ00#ke8lYZ?*MuB1C@$=z|-UPdo6W4M@?vA&0o54ALW ziuCWyhrE~i!gP93@-&y^%0=Q&T*NyUiFb1m*DMlua1l4pE;@ORi@0HtxbX)uJABRG z!nb|?AHwIr_%7K4#wD4wD4E2?j9X;x`oa7yd>)IEy}2X{Grto{!Iu&@Qx+vJa1jqJ z5)X3`CoU4Fa1m!M5*Pj;W{0nV`7M0;kbejtaglk7i@9o%`3o1bA?v&33kWXBj77;z zF3Agv#8fWg=|$o>E@HwW@d6jIH-GQ9PSoVjld)u6Pifr!E(67vusEIzJaV?B1gE^<3o06^rEd+~k`U$vb|KvqK34QaCe>Hyr%eSWZd%EV6Iq zVwV(JWDn8tK>h4k1W`2{{M z>C%1Qd!xWwF7i0vMd>HG$R))V$(6aujTXrlevorQdg^;fk8AxyNHzS}?5w>DNBY*T zV5KZB%&dubu=YM@0Mwg#XY@+C(jJo|KXM3meHpvnGTT26g9a0REaaps{>FPtracN} zT&XL0^L``*IDcu)im8f`AR!fn+WWs`VQT-(zu%RxUAgx!DUH!TdcBxV z`X=KaqxA2vVCf{`^bpmOx#<-Kf5``b7>|%=#~LNRxEvs)3q+5=8u}!^4TMUi!zY#u zgt12=O0b5%I7aj9oQ^*dGs@k?*d3mlRHhGcc|S<>)n88!OIjQuIRm7;^g8vDxhR39 ziY~!vQT|BH*{MbwnZJ1FrZLLHee8fJcc1*HEbK|Bv&lxrlKSN@0|7>06DYxk|55w5 zqyD4zFJ(&X#YyxCd{~P!lf^Iz6?po*F-j9w4a6{}*e|#9XVFPR5D;qtm!5;AQpZGD zcRqna=pH0v7P-8AGAUztIE7kShgg`%VbaG$L99#6xgIKYAp}9Cfe^HrfjZV)6XGaqqPhSIVUqDI zf-wqCVWj7<=Aj^lC`lB8@utzR^kJwM`_+8~Ec&z=h&k1SA`ww6d^-EM=`0pwXkrA# z+(_%gjVK`&2ssG)%sj3QsEMF0fLd4!GxH!g6GuU^M7dvEoC?3Tmwt(dr_7M|U%k0d z`P*xYyUoL5E1#mK*>5g9^8Q^%rmV@xt`${5`(zVuniWToBPWgsZVD89x_gyU~SMIOo_+*(7;qjQ)Ngt1pRJd zcqkL(y+FY;=T#9nAjs2;MsHEA!(u(|Fop)OmRK0w4Zl8cx3;!|kYF?S zc~W&HzPks*q*sL%C}E13eG9lcM@tCyFf$CwMo!U5lmaMoYLGCY+E#}xr0IaXS=%*X z7g6k8Lz$IC%f4Av(9l6g41+ZEXadq*LTM6h%W=e`KgCQCQFxg;7O5DSgO7(0{3Ss{ zLvVNIa0tSaQJ~}qcZZdHBtqcq7trOnDN!j6qDq=5g!`S@;RPZVG|ANeR0mO~R>usm zJ`S+bW(x`@aBkGTp+~B!P$&k?`074Hv+1b=8drpkD9KpbVp;NNgHM$krv~&H!W3p| zf1P*M%+$qc#-mi!!qxe{^2P!7%e01vrp*mlHOZ=Nc=!Tkmp6RAcw(ilVSpOj?K zswY5LMlS?Zg!W+ICZJF(hRLK2OdzCXK#jm6Jid#5iZ!rMgA75*pR7rSW@BN@bRwH{feEsNJJY(6 zAZ&#z+hZl?`F`72^iTQOP|Fac^b~1aXH!DxbFUP;a++JxcbzHOs&i zv&d{;&69HmD#sTflS~!}>osi(a-S|JK(wIp>SaJw#yk#(FDXIO8YK~)US6|hj3!?i z5=Qo!X7Ue|0Uk^_TgrE1jo|Kan=~JxeNPVz-G>e8d}cffCyUO`yPLv#5w*{`m_dZNLg zL^0B_gJpFD0nod?+$X4PBF-3AI$P&(0_9#Bf%{X|1`n!p>(-IzimK&Bw;rP*QD z=mRpSIFS(w40B@k2@}f9=9qI2W<#Lvtl5|#(yy4UbE5a!s4Y(EAwek75y$4}tVOy{w}b)mdGY zWDxoFEROLSd5BeCNiD6mpf4M-!cytHG8>uW#@@u)x;tN4*Fe4_s0j*l=#77G#)`s&XpRwxv@^9&W5;ovIeq1DWGXH6e7Dx z0I_+p7ItC;SZNCKs4|8**MgGA?xPw|2Um~LXQPYene=A1^^s}Qkhy(3Q3!+CCKSs* z&6`>J1xUZV0Vn_>?PoXM1fuIv_>V?6l_{q)OGmnWh*3=P12Q z^4Z;@b4qjC)zDXn)SL_OM~scInwgldcp`9zH|2hjdL82Z#N#0^A$BdADTXm~FDhwb zjW8cImP;x^=UXH&%`?{_eZ&l7q}$B+O{$h>5q$3vNuh>#Z1b3pp4-Ncp_ z1~HdfSuTpP^C-xUZ*U%(uMmT`cn*Bbxrmw|j;Z#~9GPkX)o*+aZ-98xN@ zM{?^U+D4*rOow)5y#e-pyLv19=R^eW!F)$B3BSp#VS~l zJ5^EI7+WCr)}vt$N|PF|IcbcQEi{%i#-&`cGp*LqaT1$Rv&E?km zJ$xNqonLXC9ME>Qw#D$V4VZRt=wKUgn&Z-z12^PvxC~_ikK`# zj-t;$7405wU59Ut;3&HGQ_(u-n~$J3t2v6^{8aR)Ft5BIkDvKMo>Cp(o8y&)oeE%n zmZ$2Y{wsJn2iM~m@!it$uFi2>XA%40DV&a)X;W`hfBj%VJ~cm|y)FOzXH;^d&COxv z=A5WB|FX0uD+`mA#j$k!r=>54ThHNJTRE2A`Dy7n=bM?(n-d&MIS2bbbTSL`*fEu0 zp0uLq2leF~OvAQEGVQFW)j|KCd20&t95|NZe)hlXjt;X94xH`AUnAl`LHYT)Na<)@` zbyP=32eN|$XF6>BHPTcI@@e_`oRK#6%hDKg9LyZYiRs)gOKY+Sm@ERvQtD4jzYe#$ z;#)}^OJP4Pb#=bERMGdC78mAq7vxDZgD+Cd?fQt%Q?bt@nAsPpIJNL-L9vVakEP0qy@8)8V}Zu9SRN zHb`FTr=`x3=108Ek8oo8_@||ZzGO*bvZOhdR{XSd({SrjMc-p8;(XH|db5(_dyazd zmVTGXD-O84`~B<9pM16hl6Grey+ElTb+xw|dLj0ZX`?)?ZoCb8WD9E-eF_(Amp5d%jrk9?Hg zYw^`EXY<(d{*uSl5}18(^8AFv`%CKl#H&zG{s76Vpo z-dL-wJth`6B*AwTtbXd7jDV}^{1R5s+m1-H)sL4=AG>8)p>zATqpF_iu~ScV{MJj@ znSMG2`;yGA3N$@-`l+7Z4v810pH6q+q+@Z0`z0Dot!d%M}1OFFI+{bw`Gwm|0BU_S%SKfSEuDb!7iQiFM{m!DV%W-Yi!?#KJLceK;7Kg)+; z&M)sI0ef%Mh(5_OS!7EAtT&u{`H^mI0zUK7HGra{0SY3%fU8 zVH2)fr;PT=k5PMh+3S3%&{h7`uM_lm!~oHjS~$bq=U!cUxjGIfeWjTEElduoAH~YQ z#SSN|y^ehp>+eUYTA3dv5o>aeXD1+<@70@xokRLKLyL2-&@X%Wa+g26z&0_8-!LbE zAg7|x?s(4W!-ZvoyV=TXzbi*c*nyZ(=>XrWNHl1LlHP*b8TQz0!Asm2;wWM7gs+muL=v{KoMq-JDiVldgO z$&zU-jU{`7vCr6M`F}n$<@9Lz_MG#a&iVb%^LptX?z!f=-}klO*XJgaB3f2}(0kt1 zcDrx8V)Fsq#~>k~{ifVsM@CxL4oWnCWfO3vU=B>!CZh)tQ z03CoRwPS8VU{1&b62ms0gey!C7fk z#Gyw{Xq(G9wwZRq=S0z>03N|2&^8xyY||*sEbU`p z(cD)j0GLbWA7V*c0s6eeLt4P6yITH&;QT{#m!IUwzBqsq$Ri^xx{DWBDVej9%gG{I zae&ZbzSWn3m8>``HKd)uo-A4v%yXjzSV@($lCM!3Bkd!1>D+~S+@lc$g)dM3_rqbm$LLjX&$P()4Af80}=`0NJNA$M@}?4&4|l*F(CUgjzmIv zNMd}tC*&_|263L|;4BZIT;q}1Cb~-;#JR<0DtzcMWrLG^LyOVbfYWnI!KEaCr2uDF z6?O2GS7}S<8Ua@d=d2{fmje@x<}&8G2&}aE6=$UzJS1s8-7E4Jw1Ab2EIG1&6+pSk zBePF**E(P&IFv2>bju=-*8xHs_*UNrB(i-4TlQWVC#Efn7Tw_yd;>@%0g%W=R+c5b zVm*mO34Akm-x)w82#!cndFBaKtD;5sd2YM~mcny{A8nkbp81ixa_$u?U?p45N_FxL zDaL1ut>##61y%StnF&iWWWM zxlseam&EOxmRoem{&P4MIj9eCTELn=2&h6VD9C>3>S5ff~Z`~ z;@Tq)z-P0$CYfB6(KHpP_MOLZNHX-7L7?U;Wk|IK3Ou6{%Tdpv>NSY?B?dq+lT z1I+e=T+Qc@4!P_lAwqQxAs{o5>lkl?Q^?j2e}MDE@TJGm=yx8(cmg66V0IhiS}->! z>Q^9AUIZLh`j%sj$8EIeY>YP8*ND*ByUuFvL}||k`N8QNF@E;89hGzKW}^fW)bZ@u z`vwwgiU}ae<7C%%(dXN2}0K&uRIp!a?&wZ8_?XVrEB#T$^R4a3-_UKM5jj0yZq z7-gTD1{Qv&=9y9Vmu-J#K)*#hV0j7ZhErZTDbE-$xJSrrTl<$R0MDKRCeFN5VFH8 zn;Xbg`)3loCD!gW8MPja_;bG`_zfh)K_I~cA3Owd;3W9Q6q9p#c8h>&;nqt8P!@p+ z{!xXVm>=+g};bZ_y4%ykM5&v}k6Gm!0JPQp@1fZr?b>pSFyzM8q!YQ%wF+ zm|$Dv*9^Eqq!Xq8jpKetlmA(mU=!`fe)g3#e~ZC!qTugQy2tU|fS-O72!HCRe+bsU z2PW9i__g5rw+qcme*YDzs*#L31IA6@p!k8fHAE(4D=EdqBhPLHn5%%k04KpWdJr=R z2s{|^7k^3cZ-LwiG#S7XrsYh6&oRsHcOyc}`V)_sndRpz~$S-ZvP^07;%Z>*RTX(!zdBlGXg>*$L-x%NE*q0DX=I{~kbJRLkkg z13j0|UHKUk_^B`wT(N5%##=a@hs&QI8@Xctq3+My{;~?7^WUN!&JzjB{AtVhN<^QF`o9z==Op(l2Fy9hedD;_YVtn|lXFD+k)M4f&EH~h zoGAEvl>WIH@*AVpSJM1 zkQvC8o!~QphZ-(tQxaFhf6|4HC?f2D7&fmJD*UQGzi-WcI$nk|BhGy8DCs2QxzZ4j@i6_(p1f4WKWo z{Z22QV*)=FhE|V++A_Wt5$9a|m%`+nvU}Z#AhX9aHyO|r)Yov>O$K8}!{@+F29VjanHuC5AaYJW-*5yI!)%OfJ44g{tkoVM8Q8F z4!(l<%hLSZ4Edc=>z{%7xy$eiF#mR;zcL{Gx~Ko#h3o;PgI5Q{fjpfOs}DRjuJj;2 zARz3)1V8^vf_KP4*8wN^c|d~SHk05p&9b}Qh~NhwgB;jb6{9F9r6hD6*ko7&lnv|z z-w2$xftw5;~j%GxY@Eu`!l{at9JT*icwCli-1~ zF>pKCq}jS`i)9ruMFsSXy<#EEIGytCb_=Zw)U@B#^>7YU$=~}M8qlT|5BKr zPjbHi^YcmWM+W@g0`v0`=@(%B4uj)F!9O1k{wZm4njzl=!k;{1g;Lu0n+mdAWvhx_X3ZN`ALYCx`vM6HG%nG61;0e%7?RgcJqJ) z51l!|mv|5_6A&xG1P^jxCwPe*^!DhK4_rXCu=q=pp(o8OyUmRVGP9hy$q-CIElEOe z2b&BaS9X(OhlAnQa5%pYc@k?^LPjkCn7szMvTKlEfXF%he8Uk;3@c}21QWxO*}Dca z^yS$gLFb#jZFb2p8zty`A7<|xEEyaDNiM9S@c*C}&k@>p0CA$hH&Xj+0DV^N_j>Uh z6Zn}h_PRNF27dpNB}d&~w*8d>{TA(f=aMDIc)@aI`)phLmn?%*yZ!5y@s)@;Mg3n3 zlXH^$1(=+Z+&7N<9Zmj6VRDX0KlHP&r1@J6jvWR6d^q?D=C4bW(+v405dO?j{}8Ny z6HHDs6bnI=PqR8)f-40Ko)ha67G`r_gYkjwl>l;>N!$x@4MEY z#g}7GTY{=w7?|t^{*q7PU zoY`FV?8KMZ1kP+DdzSuXwu3V}3C<49;p$t|cf8j-Y*6zhL$M*H5jJ0~v`c(q^#y96 zyMlXe`J(J*RsdD};Rng~E~UWilj?`8;5{~&ChA;iuQw#^W}`9gScR~!#bR=_29 zK8$bIKR^!8Q1*h9UE(ry4;EjY{RMxYE0Ohr%**U;7L-pLb8)MLj0i^B4`p&~Jj>Q$ zejLxPfP~LHZ_Ez{ zLEQcJjkjMow|96z+QntzPSf708v|u?O0TsyYqq!FIGyvJD%~79Z|{mSY;b9r^b-dF zd7jvUh0qXBfvcgKyp2cdgWVMQ7oHX#lw2kjd?OtIKk4tXhd!psq%jR8ESj&=D2B+b z77Q;}M)y17p#S)%s2)r;s~btbqfkxM*9;%@Y6$NbdmVDe&i0dugUx_KPeV>*JVvJ)Lh3#|*TrId0niE+Vf`yM!Sc zriarm@ej2@7iqGY3_zfTgl5w~Q$5;;BaxU-^ZY0>h1%BKl(o+Kes+c04>R=NV?G^i z-PgLNPnB${mBMyR2=4$!)^`fR3oI6Qbl15*u!?it7Y2w)RD0Z17{(CQo`%Hh?SEP7 z#~?cca96aF2O!0I##*(2XhUyJLk`6Ghu&g{zO&x8`BNctOgYad^cs6#)@maIAnHbj zXcvIG(LeN)&>AjJ3*^+w<^t_?f z-yNy|;lyOb_0j(P3pjuMD;m`C-KHvFcc&d zr#czP5u#Ws4tw6+Z+fI`utjZ()=i|)yYUqjQWZ|Dsi`S^MXFO_;dyEYH43BXY)&5c zV$cQZSrhb1Df2?_=}*&r&V!PpDx;Nnn6D?Z9;2K($RxTD2A!Rpo&8l<*uj)Iq$^`D zt9f{&wWX;kmQ~5ktW?;{cj(Ba2ZIx_g0*gAgG!>p_ zj?e}gV6FL+9nH@y7|1{V#dvXrBSR63xVW^GxPJ5(iXfpvWixW9d2IP z^qg+gP5eaix{9kYp)o3vu2oTUGoJIVHngA9io-7L z9;=q>BYxT_ht6+!TVV-Z8JqAX_Y^I_6TiQzp1$^v_IcSEhY0%f#(q`bXMK4p}=Ggw52XoW1!Al|y%w7c}I?&-1%lCq{`*zgT72cP*@1H!@cvz5Dgkw*%cB zSEAR|1ApIFwK;4e+PYP)Z}_P~j&lZf=W7G>q)P6Zj(64ZMy(392wKXHU5rilZB174 z<3!6aRyAGcG-AomP!4w?;%oLkV^Z%>Vxncd4-t``nabhR_XH1(>GZ1|rxv_9w8O;8 zliYz~9`;4v?HU)zjy3s2vAxc;ob-)F@KwggdhgNhaMw>pHM0`jTdo|)yu)l@Fk=<_ zf+z1)ZHNxpwbYOC$=-k{cD^7}Q4*bJglx3X<}aj~e=6d0EZ7HqbU;vAJwNS|Se>`D z@3I@luL7956Ge*#qqz-a5sW6;%kv9Xg*Oycq^Z1Sp78RX5Tf;A! zF@C~*dGd#%w@9CoOSjWpmracBv6_!Hu86sJnQMeUn2x^1g%MMu=hdl&EX&c!YDsvPTTUcK|BK&gk=^N#jjOa=zBZBruF*8@zczA%L9V4 z=wtN>n-}u+6)f+AH9eHu={AoQy>4+{@qw&2rd72McPAIOt-yOMdYz(E`Dj{FW=l$nqvUH zR5!ZeI>YQKEq-%gef~t5QlOyMyCC&2!J3KM?eiClTU_K?|A4;CXr3@rBrE=&xu#rD zopMb`ytlonW>&1{iKn|%{g)(5^Y+e<a3izf1xmfd!4lYv3EO`hgPljo)c~^ zJ<7AKXbDYkpdLWj$CHyY@o4_s^x3ohHeY`6Bl$B_&7 zt=T1^wQJcNNj@!?&J`hZr9_YMXvoR(g?Ms*YPTudC8ndp<6g8S?(p(uQIDXTaa#`0 zFJC?X&EZcHr;X&7e@L5ADl00u-LwQ>7~OGI?H#vNUV|(4lb?~?0>v`nblm=jh9spK2%dNF$(CsC>rsy zwy36fnbFQY5<7$!R#cr7E33&_m*#Oc-^oZ-WoHdHk)QkMTzj1>I+`~HCB%gn6;(aj zcdW-3rAp3;d16l4#zWawUY~QMsb@ouh}7^7rI>T4yH3?LrCZ&qoSWF%w13+>J~#KL zt+l-6>z8=K7GB`xCDa9S`*&lL-~|0vXa9?R>l^QXc+%ig@W zp<cAhmoh)+yh#9+qjmmx$vC_Lt-81eJ zOiE!*%0%JI^M{3zx$!L08G6N;Yip8}G#v2y3Rf=p4w(`^HIb=71n;)2LN^nO9nW`3 z8)>`~Fd_EXmXCHhnK1$=b_Mnd$PuE4|9#IE=r&|}bRQ#oAuXA5k z>&gl8IGtXbB)cYQ)0!mt_Jtu!#}7Tt`dHkN3`svKu|-hlimuxH_J>=WV5-EWz?a3& z=Z3oIqpOBjzA$|e7MM7%b@i2)sVinvSDcS0ZucK_>z5ZBeomNrXY1Z8?a?dEY|q+B z6;uVkZm8h3mZ?71RayspRUNsha!|WnSo05G@*-C=kFC?)xUQa6kvEu{RQJ4ZEL}li z1QBz$Zd}9zlZz23+khS|JG3KDZoB(v;=Ja-`f8GcMvL(N^c}u2PjYFyaV9Avm6$i) z`JN39htJ>Yd)L))0@@p zG_J_@<{3p<3mE6Uebaqk;|ezZY2|Hl%b=qH%t?oItlA@r_0|JvVsR&z|IB#7qe(B* zJMQluOt^=A>e=i$^2n0 zw;inIVwxSDSN7a58y@HD_dHB}0!h2+71Q^ittRcbiu;k3HUx)Q(P_Kg2k%sdtwXJ2 z-0+`TrQUdJ!1aX&&n=_QYYP0YCrRlOB~5y)v0=KyAmdblaf-rqTE~eD9Oi|Jd5pYb ziv)#5Ctxg=O#4Jp=1>QmC#+`BURr4Th6M)2Im&y&OaXQ8urdMj&Twwe~qbV(qJ^U z^`G^p+a8&w<80vXC+6l`m=|1((`mDaCi*=HsxBF^kf=k7M@(18#>!WC_V<>zc)NIK zvD9MoQH8BAU)tD|slTy;hQokgfv0;wf%$X)g1q70P8%e&szJU=w8GGt+Tu&^VzgI> zW@aWZ3_H@Dm@ew>^m7tk6Si*{se@0DJ`5%j*P?11UDX=nh&1x^$~7uHk6l&Ga`LS+ z9Q<_qcB{Gk_KJ>;+ck#f5 z`p`-msf*^_H;k?HaA#U5PnS|2N3D1fb5U>WsdF`lWLA{l$X|6U?(!YjW2fNvNz!JX zo88Q)C1dy`H_Xwz>eT%of@@p($uOL3$(+jyJ9`piT6FLGKY&WWPeTuOkQZaNUar=G zW(U7@3Vu5vFj-vRIn*7E)o`?Uiy5`nt49m?^z{`eM~GkBy+o36Rn+@+A>P0 z8SEY%9UhX5fm&#gCr3&9gU72kb?hs|sot^>?GwPW)K_5CcP<(J2>0=oG>_5hc($#; z`*A_XBqFrhe`-3khs6lS8sfawG<+A2<3VFdHJy()apBN?`@;sD~=9q z<{#wmXQ~TyM{LDB$W83_sA@{NPkNSHuodInO|u(#e_z$HTbI~WbYFFw>jEXu?zb^Y zuGV%=@aUp?A6PW~V@hyB46MrBe2`(|_Uc{L;H2U_ip{Hc2It>B5z8n@YY)vSyfGO> zw8)xsS$=aecY@<)%fR8>;x@j$m=%}%H_x|sGd=Z6__S4+)9J;=Yd2KcWGU^byOOpS z1H3FWN+5|#+iRI{UW_>})?rTefyqk?l=7Z0&Dd+C6#vajd6wPVobo$LPY|bbl%6DS zc`ly$e9M_ITjIS=8aalox0gqk+u6%AFk0yHHAuZXN5{~getiN3Ye=(; zz7@XRGVCb2+}5n^%l~>{m(B+SwyWCLXHwe=-?O1Iukt`mZ zQ@*=S#Yl#Pfw7;na{#3Npo zo}MGrK6iuoe7M?*&C=UW#G5WjMOe9eoLrTPc;oJdZp(gCTE(9sWR6};=|8Lr{Mb~* z{{r~&+CYCA_~Td=f8pt=Rr9?(7E0b8+FX)3Z}?tF@D@pmSMepa-Rtyh-U!8ZEebfl z$3AvQMNhC=NPH|%#yj9fY|RrHwf90=)Iclsw%pj&X+3Rz;ZJ;?FCz(wk8|f6>^2E< zBF@v;9jIs1E)+YwD8P4*T-?w-J;5F!@u@%=zktJWHFYv-{X$yPf!!Ky=i^p?l*yV{ z6i(;f0(c?%i64ZI%hgF5(Rc*n@sJ zWOP_iOjvw=kj$lknuj&pWz{5vwH5?*?{1rXxO$gtmLPBVLY^&q9^E|bbwGBZ5HBUL z_|l%;j}O~y5{?z&4G7v}|7hsNVL?S<@x?(hmjiA*s6(v*4|{pbE|lV>1Q*ZK+Rd$Rvqv~qnm6Fe9=Z6T<@$m;!s5$_RzSN=We~?cJ98HYUQc5Z-{$J$>;* zUiyL-!s5$=WUd7WCe)P6s#ywatq8K(+qNTNb&YJ+Hr{Y?p7gzs()GPQ$SzdorGypF z)7f3BZ*xvKb_Z|3^*!i>p$2_HPhoM%Aer!hnuHpfteUT|mQ+x;PTOR{YOal0yLiJ_ z^K9AoXz39z-i-@2c_}xFFYVjC?ubo@aI7|Oz|B4OkB3x_2!;!buM3j76>#Hm&4!I? zcZ9Xp2U+cJ%YD3h^TsSVZ}eY~qDu^?3vC?2$_x zx_3k{NmzVSkj&kH!-+Kp8`YA9wd8`j544?6Tzz6=mJx5bJkOSckKP^evfH@Ol$R3m zF?YQnhWo$tHE>nE6jX6nY09pwj%8ZiN>>`RuXvaRy~HUg$C_%$#TSXeWtcl3Ac;K@w_YT^ciO0qxl;;B^f*Y)OX^C0(yxT%?A{fJ%nl%2yxrDjxaskrHB$BH%f-J9$k?q{#+ zOTN$dJpFE+hkUkGN_M%ZDJlKKgWM)@Q)kDD0Q>F`dxzIb|Bs01|2Mk+Z`sH*HO#S6 z$X$wnr>>3IDkzYX*#kL zbL<1;ZcjwWi^N)||3((%&!roPQ{;2$_W$BO{>v1i-BjDwO*xhBx23fzPCW)kN+~pp zbrkC>ptg_Mu+FQU7cs<|SGCGBw_D(h%^he}qDl4PN|$rZ3(xzEQ$5_zGl*XFR6`P_ zmp)KODx+Jb7v?DyLXyx#+CVv_NlJ|58R-)_iZ7e0=z@vqOm`eLeID7d^?`FFIy8ZS zCwgOJ6)_!8hndKf@+H=a3Xzy8urX#P*^2Gyz3*rA99iIRUd^2K|8?tZxk2by!o}rq z=7$|SWlqP=*!6?W+@QAUXAf8}&l0wZty+9NCeguj)&47fBGapi&cU;{EWY9B&sdnh zj+}43p&7A%u~eRgGzVui!<-WJ;u_V4UsHTSrU~t-t72bz_oGlI2KQ*_~Xu-%Jz$5$aaN3=`D}P z8zIVl6uZLe6sJ}(k0{5TIE|F7Jh?l%x>bDW$q^0K<`NR6`fYk)uCYR`O~FLkfx&Ts z+YSWmqy~xAOgWe7^T08C;y&46lrku_9j8!-KD50I+tqxI)um#qficN{UVyW3?in1& z=wKfA-C@zG;hYIe>>nf(tgd&!9Bi|XJJe&Tl}zpJqa7M7uL);fXWr4WQ8-MJrFZD4 z@BQXEQjx(z6Yi*5i|D@o^C`9UuXS1NmOa(CW4conQX(5ys2+dqNQlDchd2`-#CG!; zid7)?;>QRBQTW_>X`X%5!C<-UK0JN@_6mi9gJQu)W1_4rG@hLF_KZA4g>Ic13^ulq z+#s)3JwoY9YV7Ek9CUK6Cs)!-nhSBvV}~LKoyYI9GW(wjSdUStk+RJY=&8ob7I&)n zGZpGQu^qTv;>eKE>nm?zQaFDT^Fe3D(WFNI{DYp;Ub!%SMU8%1A913syuCwMKrT|! zP(eNQxG#es7pIZpJLTS$DQ2Cjs~g&QotY{UDwA3rN1O5*wgn!g7f{+?M;TThl~B+) zr<8X&>yhA>v zWTY{pu^_fN1AiH2E)v=!78D-rsHPySrr^e`+#^#RDx3jxgFpcWzJ0;qQWtAr_3!AHOol=s#yU9a`WxG=6`(@vDlG(@_ja*$h5KD1?K(KPf z9Jg$Zt~FC|&@NJB9nP6J$zrQFq5FP*+}UKNv!(#Nl-^eWoP-jMb~v*+%6|96Ih`3*0Iy>p7p9=-O_?ZQh>F7-Bkm*a{^_ACd6N}l3p&IY;lRXJpiUWxWTNgN{xSRZ zr&Fk6)TZQ_q1%|-0C5r_@@EEw0pbket6{WL7>&w6ZTj^g9VL!NeLXxlUv(QnG(v>^ z;t+!%8Y5;6&1!e6P`$~hUmC=dQAMb4hm-!ACEuC;9_ee;gJjhA23!B(iXXlAHN8nP zstomm;f#OW+aJC6j}KL%J|v@lI5?U=dHGXrgpAz6vdC91s5BBEA z%6?4cos6nL{hx!iIr@iG)u^dt)GrLq=5ybCmvouI-857nVtw+alZZv=14f8q)c2?E z8H#3pL#T8DG0)(ahs}yc*0bp~4U{Yqe`46SiW@DSfs&pXmZHVeQGaT<`P$xycm)ym zdANrNGbaAo;ZSxNsyx~5EjSdTHvBWgsZ+ZQlujaq43y3wj{cJa;a&!6Bl2XODm!7neY~Qf-;`F3L2CNZ;<*XXvMe|h zLV)A72vA}cd^?122p>p*c4fhLLI}F}QUa8k1=oNO^zhv8p*-2}JrKfSyw-cDNH%;g zgrJWPd=Fip4L<-O9Kn~qhbm>mbs>bKcy1z8I~#rkLO6!kB0>?_a03YNKM({Gp=Yw; zCJ=%FzLW@c&xV^r2!?p>I%rTf{1k*>gx9Kr-pz(vK?uh9z&dDRHrx(EFu|ABL9?^r zP7s1Ap1U4emJL4#AsokR)kBHda1RK<3?Enz?aGFGLkQ;h(t0R08}1JwSm3!EpgcM7 z00`j(UaJ8rk^{dCA)LeqHbB?sz(XK}Q~1&bs8SC6I)rc<&)o>s&Vk>85G?UpjZj1m zJOV;EgAZ(kp2>kffDo+krHxSc9C$2*V2$Vg01e84$3qAW+c8KnR|A?iOee2Hp-Kc;U5Lpm#Cw zZV2H#KClIvh=KP)2;TV87HBpGJ^&&3;JI6&Wf=Gfgy4(UYK0Opa2kZ*hYxIpc46RC z5Q0Cxv=vIlz?l#N63_h+%JTv~SAlQ=uk{fs@&Z0zfp8HY_z}AP1)Nub5YSh#oQ_ql zf-!D8Mz1PRd7a1TVJup8ST{69Qxvw3j@#N7 z`r-0hy9R2bIwbSo^yCOgjh@ZVw{X55$scE!)-qU+PO`_4={uVxQ_f(|WttDhrKnX! zJ)RbbYO*Tak9j=;BcIVw{Ydq=zp7{W=stzfD5?i8Msd7CK-G&GeFx2&`^T#sw>s7@ ziIm{tic^^LqieY;oP#0B>A0Z>a4_ULzaYTywYF(?{qa%DeRe9(m)ox19pEOjTfz33 z<Yi-NF+tAD(N=)y8i-WffcO(=9j&afZfc`IMPC#ijnS)$ROIim zuE12MnjbG@&|Xg>8Iygj#L31=N*PH^EH$-7GA^3so4QBT%O*j96I~ zuu3vLp?a))WRgm(B;rYmb+2)1-QjWmuwEUpYGl;d@F**J3;wvp1u2EW!V7~sVK|ov z)^_`RDJx`0+Jxi17?>Zu{Tw5txxm%K&v&wuj_$%+;~z2T?+6u1;gf^-j&S8DO!dGr zS|yR_={ZOrF1tNiva_>$VrqguVDIOV&KxQk7O1FCjf+MWG$oEHCgplMr>>P^Y`31K zrHv_Ha9s?8-V%zar?G+3%XJ+h+B@J26!{SoqYUaALA?xO?(J2K6X%i>|=f zB(X1wU{T$2yoGq?(Caciy{h{9-U_@JCN(yil#j+&dtNmjq4!61R8s0AF-cL?I9zUl z_b6<9Ji_8hb9K5!cZ0L*ROlJWQ;f$^wxsZTINwq_WvwXd6o0jeq<>A11~==>$7Kei zsp{8s##2kkiM}2#KEq?fRARVweNwf{=s^2`;iWEo?)c7>IO@QFp=acvRL9g-<-TD7 zJ(3h`s?CY1=*isQ^lW&jO6q;ImsijI7n$-@Qld^@2aB?Q@yNPzk9fa{p^8g-K0XEJ zKIgqhiDlH`vPnN7U5tX>B-75%NpWzmA2m1=SwP2+M=D2RiZk>yy`kStUIOh zq1FDmM@I-PjQ-Kk>D0!qW&u4GeXzQ4suej=VIf(3E&coosr)^A&81bl`p3Nb>nlb- zUAlOCarEGVs)CEU^QWdS7AkLk_0kKrnKpmJihF|#x?5hZnb*g7`C(;)6B5!%2^YP$ZU1uSV}&UNq3D9K5tvS@>+Xilq1!;dJO!X>D$l+{O?C3$-F9lV9H>b&n0 z8pdQje1xCsq?7ggX|jjIPZ1ItDEbd2%bw@}V%?=4awz;1%g3!h%8fKwOZgE6M(3*c zohZpJr|@M|S!MM%&64YT^_{%$VAT)2PiPvG_4K*(R413LKR}b!4?jmpXrk!HOLjfc z$z@TTXp8}ha{hNH9!V0P=wy=;_Nb!{h1+3tg1nDKNfMpHd8tpgtD~AFMSArey^mql zH@#2zF!t2L=h#!7Y_dLu_VjSL9UeYAlevehRd!NuU_SDno{ZpL+vi>0LseZUOA)$q$pCCznqEkRh;4qLn zdB50c1x<*bPnveIbl_PiWXB2oLN>IQ+n&v4jnzK0F0V5s3aK92n z(Ud0zT{(kXd+MazQoL81X;v#VMqK*;(;W=gMKD%f4T9{KCqb^BK^{7F(q|doLmwc$W7L2GNbefaBG-%N zKme}mBN%=FDO`ShrI6IM3)9{k_NLLA1~q**2Pk#oVJDNtPE?pBNV6af* zbl|?8OJNtQmrK`9p4k0y>)tDF8Vw63u7Ph)uu{X%-mF^ifwf|-)=sI9OzUfcWZx5$ zaa;C09-Y&SEMIt2*P871-J$9PdGXCZH&~M4H>>{CaEdH{v+7R`R%C~pRlh#;hZA=2 z%f@sJx$LZ%ljtg;q1&VK(CvCj1zBRjohp|Jm6(nTwLAEwl3XPY@2qI+aJxS5K#z)V zbd}34vPAIb53gq4%>bvbxGBiTgzoDpuFd_mZ$~U%ogp9B+PUChk04n>K1TDgYnmJX z&8ni>9slPf!D#29$6tmZktL_e!Uo%}B_1_;w>d)hq;`qX&Tl7AXSngoiQfi^?#ag# zhm>1_AKGv*DJ{Mz*=C`f_}h@ne}DT6a)4TCZ%BT1Pvm~un$rfB#Ua;>Z!VJiOD9KQ zJ73ps{P(W$@4)>($kPTY_nb^*B+8S$v z&f<`0%Z(y|YercW&l`9v;U9I~8=fRuPi*MFx-ZPTk-zN=EPJ2uh++bi+>%CA&PR&+5@Ty3BmSk}$ z?@<%}#dmeb%LMNkwH*KnZzO_*4QxU;Hen%~@Fbh>7Lf0r`yk&oIR4^p0sv;NkX$mG z@B^FBnN6s|1`rJrKGFsGo=pPzz9N7C`fYb?_TAYmrg`F25RA?7AxL=gFi3d#DM+aD z9wc062O<^C29bub`RYIx%s&Fk0iL}18A$k)O?aG52xSvSu?cnA0Fu}eI0o|FjOu!M zqu0%1@x`kmx@^K{Y(fMZ0Ghoe1V|`|2KnA2f&kvL`8s7wEI$rv29RkAdrO8O;Sif} zVI4@g*d9b0!sc7S=4;6YkP$JjB$?p1g?od67n@LvP56XONMsY*vjOC=C148jt$7Xt zn5a8<;-x!BsU2*>b8L>e>@ArAg#C4SlFj)$C7t8Un#|Swe1{nnVtr()b9lXVeMO|F zA&%a2hVepY5ZO-d@Dyv6WKr6uNO+90b!xbx9-%(k+`@mlw;to1+7i=ZB2}ksh`|^- z`}++Nhuh1Brv?hB**@pp-KYCh!v}hpWUpxkHq%pV@OnKmkB;lOJ*wnkSRdKv`r<`F zzV~z|y_ra*P|La~wK#)#Y-c7iL5Bj$9w855E=uxUVL0)s0J8ufiYi zJ2*U9C?5^E9TK`}x7SDMV8pXmUfBm~D3x@1t?RW}Z)~JDMLb;aq-JuA66Gb+SzxmD z$*R@IHtn{5-gmVR#IZ}83{z0+6Yjfj;*0cZX@G1@J*;`})zRf^ zs?Wbs(8}9roVfCeB@gii&SccYJqxPNV|e5|?pgH_{kjWABLRI2__THJ>!ujoz2RoO ztJ$ZY@V@cjgx~}}b~A1A_P+IciHFzJ&s&Yz0fWHKt&M!5PPS#poJ;sW9G z>+RoXzM8Y$cfCVX$z>aHYl)p!!a5mhySJ(t3wG+B?mJu2sJrU*^%bd!g5!I0-z@8# zvpYQ?@yR>>ARJ`6G-S>0NUJ?alMPq6+*xue19^Iai;J&Yd&fMq`FLAj@YH6{2sn9% z^1jZ;D-*RR6SfpXP!}g4mhUz9ZCjWow&W5)dN*H(#B_)}yvG-D#ei_nBPOHtc_jcJNzdCpTGrZ}jqTj?N!MlNfD$3^7Pevo+pO}E!Emt;aoV~ z+aplTBH(*JnyPBZ&(U6evrZv~Qp!7)=)krT{Rsh0M*LdF0oj@4JM%%Eou;+TH4 zYUE^FMJkhw6@U%0)S_`yQ<*IKa0yZhMz6M=!qa={cMDmxJ}Kk`IT*&GV+983>X7yS+Zbgu$7tdvbCjm0_K~Fr<+%_w)Bs|d^3|KD|O>o%~H%U zN^lrsL{VdUI10uZ9{o5py;?=0D!3XqfrZhhE=@V(Cd1=e7^Btx%(m1?Wh8S_s&H}u z3-dRZ(qQ#RMolM4`A^ham{&5#Dynhxp&XdsXtS~fD}w<%!yXap4@0 z4_%LI=~ThJ9v)I=eIz7F`944nb&X+5ChSy+N1Q){e;u;9WfMt}*>1t3<$S zYHVI?x769G5zp#K{3Q8&MX(2BOrgG2BWrZ9ssc})OzW^_(#dAAJ)S%A`&(M^Ny_KX zvo=)p;M+2+7X>ROnNE#h;%W&@f$q(%oz6OwZWEM`tUAp7l#!tUc^qAep7ms&I^E+D zU>;0h+G%WvbW<|iM}V<;RbLy%Uj4YIB7JkrP#b2YhQ&B5`pHU-62&SOFuMG-`Z7@>d)*QWDa!*u!bAMak{Lr`c(f()B2J@n_0(T|q}89^t4Sa7EWlN%IZM44lS*;bc-IjtwB$OjaB#Z9 zRDG&dHQoCK^6aVX?wYH_l(jM`XyvP`jGMLEOM5pTkuj*h>8SRq_h8XgVoXzv(Q~!d ziNl6*KIvOBrat1sZW4nH9Iqv7OnINSq!w*oxfZRL)4bf+FRQuyRd1#r`0Welm23MH zpN6m18x%N4B{MsKcL;tCz=xVo0v{q@U%9r2DmGqmKXQC)Bt>z6kOKTm9rW>K6{2J4 zjEb9t?s{sufCWXaX}K{)?!n%|dxh7P{omSMyt+1tIF|rm7_|l#=4sKm9u4!4i7U)c zMp~1+LWu#Rzlegz#lW!sX|vLk^qcTsjNz!Xm=v4J7wujxMiVc@9v?!Ib=C|;lBKl6Hme9mT&esF~s7hnwJ$e>7Li7^v zHODoxE_*F`Encf+Y5yVy25||P_FiM5nSS})5~6so(l&dW`kLo$?u^II_MfWbBGigt zDQ)SOPcG>Y570wTpB1%8R`{6FH&>yL??tlW3H7BZ(k7bumo1l&#WR$efLSBGPJ7w9 z!zqtUGz%_UEuo0#DUJPq?0tDyjo;dT$&`?<5E&BMl#+@(u|U5py3E*XhbNa zG~1}CXi`)%Rfca!(j-liDT+&CU^;brIMejcFrrJJ0^iK@z$ zy%nCRU%D+Sz_*X=JpOV_s)X5!+eaNnWqmo2`^C?FRG!$=g@!w?R1_!2Y+jLdwBl!Q zPD)JfRk*K@qX!RNCdEY#e2ydp4JV)Kd{M{5e`gI%JH#a*OHu@ zK6>B34v!EF8txY~cummATOMcJ9^cam7^mfDsC8k&<`#7wrA2+HwVE!xO?}m@ zh_Ksxr{3vl8FK&ay}eUz2K>}HuBml$mX`mvC(XfkbHU&J2zT#IuJ;-F@w;lYQA2diRXhcHLQ2U_K`|Bg?0(;;iGnWSPPYT*%{`uwJrk&Zx z_S$xLKJ@!yT69F~$lqFrVzrLG8a?}}^|IiZLkwn&H<&r9bjH;0PZe_|y5IhdkE^~^ ze~j=fO&g6_8e3;AH<-1;X_&zb#l;5GB@HANmQI%~mFV`_cAPm{<;#!lz7GZJzo{?T zM$w!xYt`6UD-}MtPM1(X2Pzdm?)OAsQs7^L@xG3Zo|Xnni^mCTu0)4;hYyju^vC2< zajhVxRGic1$2rsYM{7;fI`a%!QS2HUmsoY}SdAUA%gtj~&;B8FynqkT1sJy zP|z^#4?g4a5#_6&*+yACB34s6c9mG{$|bRCfzyJ9DNLU^R!69&x+{lof3*5Etw3xD zEo=yj{U4Qqlq6GXZSBo~_c==YV|Az52P0wu8TiZ?Vx<(e*zEuh$M0IyrKOZea(}0S zXME7|3DNR>p>;Sz>lkhI?U~=zhi=n6WN)>RTGz4qNMiu_NKs;}g#4N5QfDNxX4VEp zPLT>6n-C~wA2^{gut#fy=Cmi?C$;9wI_&(>dG=jU{JvNz1(6wljh!iYX2v8JmC|vM zeGQ0|+WTcm+rqq=x~*Cp&`@}%rxCqmDWCx-{Sa1#@iY_Jl zo2J~-5+QUc4Zhe?{#%AHpN_6#kY2_;5m{^%6R=eDXezVlvSYia`FRFw z=B~Y1M=5CuZcx^ysYudO#)8GU#`b{qq4u<{exwiT<93pShQf} z2PFM%SXwp?e1zl(+p~C%=#GqM#UGf47Un7Hw(4L}D;|$prBU|&$TBVeEkhuh^e}1i z$o{4wcefa|^39x9F3MqoavAr8Fcb8}O;F1#W(ev?vfng>?iLS19BH$i59UaWn822T zF`?7^jqikgF+&hrx&xkpriMnj40Yy?Jh-8hn>xh8&IimDF;_TE%v?2uP;>p0A@-=E zL;Tp>;4ZpE93A5PX^tU2tw0jok~-h#ZZSXP$pm+&xFZ5z#XyH3tdU$h0mh($!AFQS zLQPy}8zm8{hNzYdiUL+eVy_GR*;ub8J6!fJmlOqZ$JA0DvHYEBp(?hqEy}Rl!?=KnHj>?Y#Y7 zq_))$13K_8G@JHny0IO>`bPx-6xwcgF+j1I15hwrG)@;u+nv=PLY4N%>7p(Q;38_= z4EV2|%NmSOdcoZ8@7lSD-36e?<*r3X@ChxiI6LM@#y9s0b^3n{y|_Ql4jsv0L4PBa zpd*bhoV)Ueb_p_)D|iOFPot^@TRFMJ78WaW@Jb>xa}9p-<*Ye%JQH zhS203;v_o6_PYj?QVmz3LrBlWh9GJSODU|j7`s1C(?Ti;vG1!P{Q&=5gBd7-hf}+2 zFawRe;2D^M3i1mtR#j2O-nV&#( zi=6OF2a9g-ul^+n==vWc!4VH5pSSB^@euxnuj4HmK}WSj9&Oioh(u7#zi36Y#mtm; zK&h?kdTLR2@h6Lh-H2(yzfdEg$+1OiUJLVNj5(H*aZ76y>W zTjdN=NOf8?~aX&gKllZ|A|gKza(L|37_(ah2dMpu(2i#e`x z<5Up8f`6gWl@TjrJ~Nd?pp3-VLPjmViI0(@5dMY4N(M5P3sFPCU&q0iu5A4tgmRgQ zT6?No*X7eeJ!Hmx`%<=Prw;&57(U@J;FvDv%m@RnZ}I)!uF>(^ZIolfh66X+?xz08 zF7B{sa7TCXSNZ{L8;KC!t1*=wBHu3=ZTYp}u=pp@cKMP!0=uL`W=MsNl?qj8R4Tc% zVwZIIh=dU7gfOv$&?N~`4YiB4gzW=YNkxd>+Aq@i;O*m%a^1ku=SRe8`NwH$$LZYG z&A2I;c+pTR&TviU3$2tFYf@FasW0?0ZcdSoU7&So+_NvQ@1FCJ`u@J_v&-?Ib-o6( z@(gCg8Bl5|=9@;JH;|4sNQ^b#5o`Q8*7IVXuD(p%aCesd2x z!*`T95e5>W29o6lGcyflXVYoU25KemT&_fxPCsQZKlR`bmo~}5(?605Bz7zruN^dQ ztzU?CEpA=FnMHj0*Pjx!{sUQQXU?&k-r$b|<#~Vl) z8OSa)n7bFQieh(>Se2a14@#SF30M_%`YC)>B8D6KTa}?PxhjJ%#Hu!OtST{yb5$wK zRi#iDH8mv-18=;}d5W&8UHQJ@JaUGmNqW7MpCZ}0QyL^jmChORM#t*!4eQ!8O( z(|B}c^B#{7>LVu$KmF(Aj1jq{WtBUGZaO2;`JY>KmQCCci93q5^#7+}g2~lHt`W8=JscUrv6^7c)hJ3aS99C% z{d1iI3xcN=#_AyMd%#OXryg-dXbFN$OPHbTyw#R(nV9Fca1N+WdFmidKG40|L+k_z!y$&rSSluadevHTn7#&ZGK@5URRxzy zzT0KsLx>K!B+0om=A_CdfdPQn84i`1bE@o;BxwL}5HK?i1Hgu1N6In^J5rv9>tA|S z36WcS3z=G~i0t!ix*!NCUUOwku%=M4%1q_-%jcIegRu!(Zb?4RskmxHam`E3f!I1y zam4;q3v%pF@_FX|e4(sU+4(8IdjEA-1v0XRyCCt0?)O@pj}|!8KIG8;oz!7zijEg_ zsI$u`8)Cq3WQ1rXgsgFs!t#sVEZRC?;k&5*p`eV1OW5xUY;(ZDWxUR#9e0;j}g=GKPNi(=To}$To{9e9K z=LK6GG?r4w{LTiuHCu%F4rmeaF|$yhGIC52YK-yq(_GW{M!tSL@269u*f7j!eco#e$ zyYJdy3mG3IlE-brKv?@QzGblxguSl*$loO6Bf9oY-qbQuCC3oRCdF~u zJ>HMc<2M}?!{zag7u&r;jwj%A%F~@Vj1VG#;mO0I@89Gt`rVukoNwrkk~~cJn_#(NP&5V5@6xH@?+d=p1%YL^Ii`nK~ORV5GG#J&2 z+nxOsm74d#&8_)uTz4Y|8Kn5vHamA7VbzWdtBMO$Cy7OkJcSqs#k^}-eDhz3Kbb`J z=z?wE6yQ6Su&TYz7SEq+bZm=pzpnqg)d0&Kwgt;Ws%CSdWI~XEID8gZ;T)zFzU1HJ zHsNWK@An-s7{=+WQ;EPm7XY~86{M-#?O$kaaVm`4AQNRtQ$HDIa;d)D?$38{U*<9k z$J5B;fwHb{2{=NQn~mj{y@J8w!>yS)zLpxEt_UY(g64SOw&B)~`;~jl*c~XBZf(&V zVWKhs=usZ9Ka)H3U_Krqo4${S(ZsD%o->*-_Z^mf&o%aE>RZU7`$#c=b);l$fA(T8 zLfT8l{)LVfx>Q7WefB>>iM{_zTXd*J@rAZ1?1Fw-idAm_dv{X3r?KlxfW2+YKBn8_ z7?PUi$_a9J&L%-_=8vlC4u@vM(B|PKf$ZpeOvw3CNqCEGv-=HK)r~6La_4*|?uMCv zl5Eteup$)O1BmKr0=`0`3)<(h+%*6X7sxXOzZMo$^%ca5g3>PlKezT)JWID1Vd5Bw zq9c`}bb;n{F+}X?1AsO;98u_TSAh~J+<74H|Ee0W)etoT70NR3czNDy1zB(rg0L9T7r`qZphpO#N{PyIM=N_B>}fm2YYpHz>R$o^6&M zQ-1)`kgp*EAr1S)y<&NSFeI?*mk2-@f*}#Yu#X*kPmv<118B#V0bPr7m+-m7k~Y(~ zJk3~5Fln$n?W}$em^A+4X(*Gn$J0wBV>MO-Pf%WN#8`F^Rry+~G+1`+V51Ul%YGM4 zE_>HExa{MHxh;FlFy^vf@=tMnbpPL0ZkuOv}h(!2YK`_j0;Nq}3hx^r5x|1j=#C-d<{_5ZJZeB<7Wd393XR!ht_Kj+89 zlC@43@}IZuOItS00YxHxtcn>Q2&-~9J*?*aebm?cRzMhv+&6#0Gz=B?c|0DshOKyi zzxdBU>ePF?Z0(kciJd;WirhU;FYOdyrgehwJo@NTOe?rvLQ-mdeBz$PaJEm_$}O!H zI^Cx6`^UfIYw{)Ow6+JAvcK8M{r`OYSY-BsOU(zJ4pZkgdl?^c_F?xfNq(~vL}ss4 z(y8Wj52d*YlJewXCJ6>e%18Gv?fPbix*N>+O}+@%#y*xv_9x`vv4pE}-X)_}J0oPU zzB`e~T+5BoZ!&n&HVxhU>Wp%S=|1_1`>}fG$5Q06mTRKThkDE2U z;~RO3RM^o+!7;7dxLXtR@kFTM{&;}HT6&uW0H#_bvU?U00q@i<4L-m4`?i3kktn{v zI8cE01m>?MDVE5YdLN$zFGlED4!D!Ch>!Vrf`8)icxM7*u4Yz}M84nsA%Dt#h1FaO zHZ1V*CfwM84-#%htqw*@H;zXD6~}^c!p&@ak#SzS(f=gEZ8W)ElZF*Nx}Uy_D4L-bC|Mp^04142Ux=ZAOC-Z+iyW3V5qqBd6WaatDp6S`nwn;AW`Qo+9yBS;^g7q z)vFbt7`+v9B6|$8_JYT#RYG!Y4^CurHHw20Y>;$5``cEzyOlz}GYW7_;<+A>$mH3z zipaaXBm3kfxcV@YE+mdF`p|`au^~d5>w)J73%2GT`CTH1tYpgwJ>!+=5g8=s0Oj_( zRlq=(CQW({LC%3ka7%B@j6>vpNY1?r<{l@K5Mth_H6JpDTr<=WB-?7ZD@dqDaycge z(-$SDUnLaPG;Vlipzowgw;-k+m3>VuHNR0v`s8Mu5DJPi%L8NFN9mshZN^5Wv;taOy>AwJMc6 zG+IEq93ZGiohlj#zO!BNK0SsjvRE;b48t+4RmHy?27m|Ml@_Q@7ULH1=KuNhA&+OC z77rMm$cUK?DyaEmWYe>tBDf8|J5GhW4!$2%7{qkeTBq~*1<$8+Y=ykm?;OJdcsE5K zkfw^v?QU`Xq2mgfBftgyPG7Voz_MNv_*RI~8$jUOKjGUn5WWraW`GUjrnNMEa?{Nq zZzeZEn_HTY2_Zo}q@?Ok_;SV1%;^*O`9EO#4yVcK^VkEZeLi-UTjX69-kJFBaYr7% z4(%)7IkG>86LF7LM~=8Fgg}?qXm5)KF1U$LQlFsg2{LW$FG!|^sQ-pcgO$&rYlLAO zh+!8#;9drxesmgqxlupGLI~sKOE6w?Tp2HmIqd4*QjTD}4#aQ^=QItH@j3t-{`iZ9 z5XOr`PbUPTMHzi#kg@+GJ!K<$vLN(C)A)^^2yVN~CA-mi``C5|9kTG<_aV>ostleQVduo#R$z(P9l%X5CW_-aLQh z@mIYXukQ`lSbr=yVp^BLpj{F*mZ3>7b(GF3ES)jLKJh{OCfXX93mD3x{$I>QeHHZO^N=bl->zwOa|aFTvvS6~vm3G)uoy+g4Nn;@l? zl(ABK=$0u*TpI)Zf0WM~7C32`<;lZxkCtC$w^4p22;{S|X6CWeQ%WaDP4eC~raEqg z^mCItQ`BvyxXX{unK$&7`u({(Q-RDdCO)OzJKTBuMYio z7R>(C;Z!cOZ*1XxW19T~g=u*%AVSl3WczkTe9dL6)*UsbT8e4B@|N@=5%!fTI+S-IqSt z#fa68xhEX=E3aXE-P(8dXytY62p4!FGZ6)w%XMBW9DGa7{#Ix~ZkK_0V^QzUsw8-VxnNCthSqZPIPhATnT|m{ z^M=twiEwpGES`cz6pbrPGp{~t^61n($;gk-W0B&ml z$eO)|k580$f1oZt^P4r$PKI=y`y~gmOeAO(A*5;wBDbw7P`*@8EkUGI4Gb(_k^h_AS3RS9Dj#ibbE(AmPf43cPX`+}S*&iiv$4Od1iR7(5Uj<+?g_7* z&TTql$?2pI_B0)&Cgbx*(*cISb2gli?3m=YhXY6*|63+M>Wv~$z4;(D+25$`*O3kgA!!evG`>#*Oej4E$pyTA1na=gC@VwJN22xukCVO z*SX9dhgnU&mTeE@2BOOstbfq? z&FFvtV-4keIf_Z|vW!}55`UjB@YI_w?YVJ$Zi+uQA4r-(`at3@N29JAe>NT913YH~ zz`8W)z{&n~#(B<3zsp`QN{Ell(LK&oknsLne$S5la@6QjlHjHwJfCI>w$=> zBqO9pY*sFH`T|G-Ap?OA7z2eg}+(Y{6EDEI+^qgGfr&5&CSUMX5cc@kv7Mu9(r&gkmTfrJ^!fanI!j3cn= zlLFlj^-sLod#eU7E+rwYs=;-^5phUuAxnHfSTMAgyBZxWJ!;fw{R3>By;!ZSv^)c{ zP9$>zKOjmG2|48;QQ6+wURx2(wW)0CgszoOIUa`7Kkx(Sh|mVUU#!pV>~%O~*T}}e z&y6SKBb&{pT%Z+9H|N|ObLQrlQ1|7Z43mQmqmt<6 z!#t;gq6u2&g8)d_BbEt1U|VK7oD^6;86%-#f@X2tFu?~b z!`ucZ1=b&o+ocM;DafjMp=@wktKmQcTE2NVS_q_@)$WK16yz^6GHD)ko z0hydZEE5T0&3DaIoL{cPBPDSS7o|(U2kb}*N7$e4$`Ov6;W0jYCqaB}W`<+n`C(=% zb;;*BM&af8nLi4vJ8YUc0v~a~0}XuDNZ@nLAqK)d-&BPo@PQB5f$!yWpfGACN8lr_ zT(ANkgX}v~U#ZCu`&v-HO{qkg+CbF}!Q$l<>KqQMq0S-WC53Jy;*svhp*0a-4y}p! zl1WUn&f%i7(t3Pr94>M&S@_maq1v$huR6kg|J~D}+BlLJaornDVoW0o1W63;Bv9LI zJYc+B2iiW+BnCcUB(WVLEs>c&YO6b(u+0QXeiMFN)Z;XWNvJ{;1GX~dd<$30yx`e= z8XOyiSu?c;dDtq=I>(DG99nb1w8j=&%uqHIGh0_Dd$}cS1Ey>jnado?2G3#p0kz?d zF0GdG6hll$bjKmttSq%R2#^HuIfPhyy^BIY zt$`2NTGORHq|mKIu;n{Epl~i9i&CDZDr(8+(bb>P2l_eGAv91*mn~>Zw8!u?O-P74$G}{S5I>lwUmKrO%D#lEw=a7ntgLu#y_Do zdy(!8Zdzk2yFiLmwhP^Uad*Z!EG2|lFXmJ>GfELLah;m%;rM=2PTxcXbhIZtz*{>F+@Ce&X~WiDHWmC4~@ zilZ822A$jjA5TUh4fAhXxm@SxaF0+t|8~Mx;;_;-+}ciQ(jju_+bz4Vr(()hva&>! z&EBKAGUOhO=-lqxefb`f8u!r-ZxQi&Y3Ash!R z-&NLQh=m1oOxKY>2ks7-9tY4VfN$$T70tA2g5YwYZ(rPf9k*&FtKV8RfQRSJ*5%4x zuEWim6CN^BsaLoFlOKX`R|W``IpgNDZ84VR*eE%bM9ED_b#3L&ozhsChe-_wFnDsT zaiKeMCdUZQFt5YPa!xZ8gFUHhatmWQ!u&c;Fv?Umpog}>wKQgND4X%3Csf(ih*d|> z9d){u;<=gHvsfbX5~ilum=#H@(IK;FUsD5ju-OL`$>-}{K6l2E-UpK*e*ll2sSR1U zwZS~YVKr=ce9f=Dw|-%XS9?BXMt9W_bYIk{-#EA-Tai+RuMzZEHG>{=<)Fu!Iq9*- zBt2FkPtap~bZ~mCA%cS*10S%k-uERR33@D`5d(XJr^45)v8W3c*t>F`AqT+T_Z^>r zoXoLmOwMd{W`A)`V(6+k`x=AYkJHw`5`W5dJGW*EepEfIqV|TwNCU- z%Zb`>ESm}5kwe^t6`NUNFbVc6HsxO_24+o65{~xrt>~9;C-z2QlbrHpJ~By|a_`8I zDfesV)^?9;on&w#c-!{=QZ7@Bz@Yt!g;}FwpypGuDq&X2kXba_V<~$(b;4#8;QUbn z0YbGVC+49MXWQZln>jMU4RaFkr?izDckW2y0Q|&D0#RCG1jCRB==vbX+{!W~4YLSc zQ!B(G*!E|P*k5^0^$ttNOiZl4at9&?4GBDlZAiwICUcvT#)n~hBsf=wV~=oHw8PRP z^N(N=Q;gW79C}}|Fl!V|5qreMpb)xNnOHU_rE*|U3Wh=ZhpzpIxnN?jyZ|3CVs$yi z36!KZmf2M%Se&RG%sWn`zHUlV<_ZhgXyO5hCX&;X=Qb6ND8M3h!1xJB#ARg$hfI^qb!L>q0DOsZbrejrHhCyHN2L&t<2NTf#I_Y+m-D$CD~?s zP$hx`y#`Lsp`4)z$O}1SD#Jx#il{o0(w`z^Zj*{Q@wM>c!Sn_WMU)_lAWSI9ahaG3c>Pl6eb4^lN=$GQ-$RmK)IH4 zs)1<_1U;7KY+R{4R|bnIBF3g1F@?#Y5QiKdKMQl{HaLd@DHER5OJA{ms23L!l%}7{ z0YsO~Wh|!X0>F-X+YS@npRVH=KNiBFR@(O>59Ypn4Z!`g+ayPCK0?=b3_}2EN$MCJi z=`p}`q~_GX`H_3nx4Tf0!6Fddg*tR6iidJA=|6ad3ad>fYe|#V;O#q|BK4c9Us?TB0@w#N_u>Vn;(cx6h!Q5vcN0w|R5j+P*9D+&Yr`7V&o%qPi zieD#KHd9iu)ww6PMw595f#@lP3@qIV=ij=#ulaF0D2u-atOFph#a})b#>*NG*1`Tn zpEGEy$6aa2$r7&R3qd_SE`wk%DtI>*k&j{39bYJ1_VJiK!a()C@xVN_hBqFnxUT~R*0_qC@t6ZS4E@R$Y_+I;G`QUS0912{0= zgD~HbOw&5lsQ5y?GLG|J%*a?UN1ho7AlEW7I%;UK9v%p`Eym)&pNnCXI!o*tlNn?% zJ5ga}dMLBQNpeN{^?q)+>@l8sV&o`id!{eVi5rh1=(p?r+I4yS1eQ>$&G%3t8RoV<~w9s z-l&$_D2pP4+}xykoZtY5C0Oy`NlcK;2C$g}D<1k*0djMj!qF-Kn;Sq=L$|n)E*A;u zOG}Cgxqi?qBRnK{E(e^#X{YTv-3kcpL{xS<2xeoGsZ24VYKWls6$`UQ#R#ZJA)#L6 znq0_t8eo(AO@_x#^Kv1z%yF7`GJ(Tonba7(1dyZHhZ_SoKX@HV_<7c7<~m-)Hz#5Z z5=Pl*oJkErW}#ndu)cq>2nKi~#Ns_CZ^%8BfQ?o^aRV|MGzIVhyC8h@!rA(BmOR3f z_ca9_(iA4a$JaeF#I!9ugm~3Fi-BYX`G`#<*rQZ92Y<=RM}`4yZBuwy5C?JjdkYGl zOO`($=#DPkN(i+M#KNpy`ZK(LQtTm%{J(bYbZ@kh{&kf_7P3tLfwj0YWZc?sj16D;FR_7LRhxXn0uI_EY*p0etA z@F4c}D2Ci(4mlT04i@s@16CIO?*Ek_T`zt?OY5q^D@~Bn8j_hzl91?0Od`n4;vE<% zNsTT?NJ-e48vTH57)?qe8b+m)(JkI4VrYsqjIweRU^f}fHYNr8^OMu$j4MU)N-wwI zZ9k@gpI$+srW0s@hVFvVG%POzbPt<>yHPE=vH1jA_F~RVEt~cqYNM$Q?DqLLp^jI!6Wp2Y)Qy$wD#sOH%&N@(<+c3hxY}7@s4y!IC4U z!w^maadrm5Qb9jr^_3=w7=a3~!zGh~#v~Mo!`5T0j)(j$`%DJ3s#EAYL_9L_V{Ij? zKFt7F%aWzz#Z->w8_FLv-&o3CPUTG3*s(T@4S4fSk}j?MpJ={Gs;8ABXRV;im!tWH z1+FjY(~lyEsLTZ?o~uHnT#}`5&EFK#ETp1&SE@ z1+F*p-UkLSAnqgHdaZL+Ie0D^zxsZi*?$ASp5BM&?E7`5tr1U;e|<7R*%~=5?fZ2m z#Y8;)+-sT3%D+YK#N7FixqH7J8Lf<#6@8!1Y@Q*W?)n<`I3m4geZOwU=m;d{ZvOgY zm@?XJ-uLTm$PKGyuN_V+*KoZ`d4%=#QEy}wC2}Ii)ae{&#=Pz*1C^?j$tIWP)3qxK zcPWmGK5Af=EvxC?JM7kB?<0|Me(^%R>U_%U3@jzJvT0E?Nye4blu}A_soTZ6>@iEm zc3%*rtzHs0$EX8TG%o3)In_dz;K~={2dIW5g!csJFi#wfs$(}V89N@n^y>1sIO$w) zhs&w$fyu$QN!fu(j32q0{QIGGmYS`o!zYF@8R3gn`^BKYv~DLj#P1O*jm_^~-6^JRNXJRcd9zQ+UsZ8q-={{A&le zk4*e_my}laf?3iwTMr&#v;Z302fH@bfabRV%{&)y)fS3Dq(b%mUarxxBL))-18-nQ z#Iy_@z)$X+Hwa1W0(9!lNwSuw!cteQy~_A0twS%?uDG}rcE}UWruLyq_TgX<*xL^W zfzclH+A`adk+~^ZRdWuKK@JkBWT{I{jn~Bo0UeCwAUWWOuu;oU<}qaiF*i`9-m21M zhC6g_i){=LF*r9+W-#WK>G!x$hdw`8^;s;rqenG!K@WU573f&Nl(5(r(R59!+Su`m z^6?D&X|C6Cdi3I${7)+#Ovq-d4k`q<$AzH+xIJd;#NkLGh{av0gndH99T>K~QRk5M zCgM@4DGqW3sx%S62#~slN&~^3y`j1^D=37mUKfy49IN*uSFcoZ3<_ZjfvF2wJzNNL zFX2L}z?^^($3j3iC%%xKv&n@tyPK-O&Npl!n9;z60D_0L`&mSHF45FX?g~9y@G9=p z-LD5TV*o(hyTI$XBbDlOsHnx{LHdaRaNsp705AiG1-#=1e}hnXo}gUzk!t^=FeBfH z@#AiR2khZuGX#-vlLcO30L1D+-ysx(1T4`BO>_?IviHL?!6YSFqb~HQ+^b24DsR@rx(5Mh}#T4XbLJYL_ z?goT9ApAkQDJlk!5J@qpK!Bdu3WP@&;7kUa8)WT~CBeBd2O;Lx-yrZ<@;8Hkrf6_L zatC(LqNC6bifr;&!|1I_%2;z}&n#db4W~()v-HS<=K~zBV8ZDNsiZ5w6ALg%C=PG8~6!28s z>_FAUw(a0A$KXJy)q&vffz!6-84g55pm@)T8`MrF;Ddk_zg*}M%XOKu9MsN%g z!8u6%IK;U@ueKUEnnb-6cW&sVAm_0Mfd>n;hj;{m$%(^Db&)~*JS#QT94_L6fK)@S zgdIVcb0Z=M>?7x#8*J()#fS*P2#Q1qhG!+9HJn2MSWtu!BvXW;0MNt&uGBDFoDmK5 z0}W2Ym@tJI696yx$p-cwdsgG=5nfjz(<8kN^m+q#ox*fiLJamE!+u@VUF&&Z3O(2t z%#kCpWZ!=%(d+zt8IM5ygpy8aGL%?$XQX;o4=l`a4&qgcfhs?LwRK&#SE48;Ox@L3 zJpW+h(!Jo8=h;n|5~9E_bRuwUHNcyX3>!i&8rB1f>F>e_y1h>K0ziHfKgKyO)D4}7UlyRD-F3! zLhS2_`n$a?7B1ZfEoEkxq}K04M(Z{YFtKFtH_Y_!>8L@U$%f3L8+u=Z8_-)5$?|imHRDDe?`vLQYL$ed{H4zEDJl6&#<$5|e{5^!GCwC3}jqDRNhQa!cEMv&# zqqp~fJJ9Ocs2731iBz_zSUw2gncgQdu zOjJh#5UftXsU$2^iRxr%@ocynj!44KiMGoqWJsuXS8!B~RKf=V@Rd|FQ3CxNR`RbH zLoC3GeIpWmgBkVZU6j1~RqM*)s4cz+Xuzyz_G8q2fApcHvCF$OM#eQc1 znLJ8bz|CotJYrOHo^r#OSpec)GZo9^^Ku;J>0+!rWy55g({Q*b$5Eay=24#KQRrI- zR%~1L9fTJL@V|5r@Yam%{)nbJ@YW2B)&o|d?iI^M$2&hIW&S7!m+X-G+Tj6K%|iq_ zm5D%MeGT6n0Nl&{~H9sVgF)vf%lODwCASs@y*1$Zt z1r9VYwD%t|2(1~shyYY#0L>6c6m;r82QLe8YIe^8cejxc249Xs!dVb}4&xstR9*-(a7!Jg(3KP%m(K3AYHxnR&E=F+Z1qd|G z^BW2V5Iwu-GuO+DC^EQVf@h(0&ky$GJir1iZ+OiE?m4qdM4M7!#l~Zi9FvPgOT>R_ za@d&+AbB%?JcKVxz>Xy0+iMU)0-l~!lc$Ad@bwLS4t_u@EPjH;v;pEs5^g5wxkjm7 zU{rklQNm1eyvQT-ToGWP%XH}W7(33B5)Sq}yG9LMJjSv=cq(pgK=*8@sMvTJ4uld5 zV8uEpk;*d?cqyXqPwWVJxi89~v8X!mAlC5*oFJg;VjhrG9Uw^Iy3))8Fl(}dqe z`y7$+Ud^+DDbh9?6vl!;fB36{Sg@c-!N?crf_^nl59U+?AE^YPXhI46m`vqhenUhF zEP7ocSepox?d2{PoNP*GD>epWk+PxPg*!QS1L z@{DAW3mX^uQNfj8B9E{T23-k@MXHf&qVP@(qE~}koD7bn<#Wc#J^>Fqa8j+(j6x5b z@CF|GI>WBy%OQaoEz->Q4H>9ZhFVC{1H z4AK|>U=+EnKtnoLaK?%TOixe)(ZB+@f~aESiGf7}KKfQZ4|#AO42uTdoF>3WMuU;j zazl$nLEj!51jA?mu7ifPFN~JR65c!=Qn8;s#5O zvvC(5XX7X2f9CN#8^;dd`x%6S4bo?B;GXLNc6Tx2a=GB1D|#-v`eGf`yGM%+yC1zU zvHqL{u8fc;4IpmHbM}2$BZsv*9t&X`Gwwj38vyrGa6&rJLGb31%yT&$r`5bdfeE!C ziEAieIS{yd#z?22@g=N`*(i-Dz!bipkL;I$p%;%JnSMDO2k`wIvSAT~=iCN92>X8rygzvN?9kryT{tu21`f2RzM-BKEnF65=cwY{9ixOEDu@4+_80|bn z6QzlGTlukpbq9D<0wE)OAaLutpMgdQvAZ?Rti+8gkAZ6!*v5hvHQ*bIwIT2v2v(Dz z%L{^EqR{jBX=Mq7a9;ow2I&jzU79PyJ`T3=^)LmqGsRZ47`!upToF;WrjY}UZ?l3z z_&}g0AqOI|7ON2q&`O}Ci+LxXQ~=LIL>D^qQa+j)K%XfVv>9Yfn-R&Xz}Fdgq=VOr zAY!n+>|ZBlY)Xzcq7x`E9lWFs8|^`VqYX@qpniPGte4Dsw*%=+AQXU`J7m^lW8fr) z7{tI1nc#&Lu%90ln*v@~@n05$IrfG*p$Q_#vR`w7Xo8wnj%b2d5_4`(!1-dGLuSj3 zYBy|hea;s_J2HjYj!am2N?WaQrxSPy4nk=uStX%#2INeb&PVhKTh*ncwQyKCd@BZN z;q2gm`D{O8eH{b$K;|14u(?@s&JF8fu4BCW6>BK`gTTdniv(M&cMaZ_(u(!%AJsmxEQ5WqQ`+3INA+3!+|cXj6%0ZYW4*T${e9kSws3OUf}I1hSkLWx;!{> zJ%NVHT+1%3^8WKd97+5OU%>G3GI%ivnTY|&win@mB3S!V*8)e+vmEVDpy{6@3j7xa zad`K`tG)DNUFTL2KTpR8$lMw#D0ewpjdqHZlC4JB86B_-E>H>Yf(umYuM4g;o(+2} zs=HtpoE8;hC0)P^6iE(-RoH-cAF@08Q~i3fdu@KQo7ox#We;vLw{dLIFAkd0F#~|du)tj%OWx4IWfJOX_ z5VQoV*bg1dq$gp~+D_1uVJmQY68w$QlPo_PAmPdmj~V8~(qMSZHtqzwRuAqNHe_yGkQ36L(F-e^%`>ww zk4=x=t^WH*A%*TB;&B5zHwE{Wu)bu5>jDJ9gpS1J2whqg6eWW`$QAGn<4CT6&=I?t z1z5u(j*hgx&0_3(R>YZJRdt8Mz|QodQqC7~bjtzy3c;s+4-Wt9gY<(K8TYsd#8~z* z-Tp@vevhkb4v(O33#A5>S=^NzRjGfRS;{FEPVt4JWgkyqqwuUG?K-w*xXYX4W;x%S z)A=*Vc2xFnMPasoNfL2>X#1YneZKg?Tae?p-;?k4s_d`d1DD%0=+A zG^vcA3nEA9Q!X;jI{UPvrkqdZaIimWM`gTG0L%1Pw*mA^{I+O0d~*R7VR+sB?B^aY z9${E;0vvCXgwi_ZYYVWsVRa*%8x!&pg)@Bl0DT7nc?qP9-O7d04=Yal83cfQj==P9 zAB3#{7Ig>Sfbyy-Nx1Vf{8bL}02>@GalgU1*yE2`LQL0v?->u2j_fDcGMXTm<@rN+ zHyevjY9s#OTA{d2DMy=%X+WQoI{Z{CVvRW0`-{+Q*tf=si-;W8zq@DSr~T4Hw=j#i z%mV&bknn9lFP}B4HDay8>*e{FRT6kh!W(dMI8hzGbdPPveKiHHC&0D^_!hl>S`#^+ zU(&+QY(*wi3xAONdm(ro9?MKPzYpe9Gm`+A#C#6Ar!UK?nnvQwewJ&;$P;k6@ORh2 z@14&?Fj7fNufhksJsMy$<`~4$VxKYKTMsIyanw`b7cAL_7RGtXN$;peL1bZs-!KEY zh>xO4M3};`BsuHT}v8>8&_IEsIFP0NJU3tFQ zCt(#9Xc?kUGH2A({=CY@uMlRYGQgc?ut+6*Y z4>@CCY#!=g8lN%ODDM5?qx^PuV&g{;x!%^}kp8RPhBP)srq3C%g#J@mdJX4}QPZDCy!RGs(*zLJt`jo(-{y zTPY>AJVAFJXRy?L#_sX|DLj}50BND4f4Ikbw&R(NugjFq*=dAoq`zo~JKjzs_lg0orUFiG zV2;OoF?}3wbJDvz19ZGQ#(8cj{>Z-e7ZJ~nAx)L?x0~+qma}69WU4K|R5J%?s+ogx zc4DU5-^Pq(MZ}@szFk(UUF(zUHD4#XMtQQgiqOYn2yG5~bP}n_% z$`Qu*l+vaqADUYGXuFpFV3qq~5x1rVhwn<2?&-PTUJ!V7xZ8whKKs-7Cfui_iQir( zyff(Ia1OytEf1z(M1_moawxbnXt*FIcxq>mx?mr-aG&h1)Hk8o4qu1RB+@2^W6YIF7RhwZQC zp3C+6)~yh`>%sXUySGe}@m`rG=rDBSy|Jc4H{KsnIrPS1qqIvkH%BI1*fQHxK(WI5 zyTaGFqB^x-o|DF^y?uUY@qFP4_hwWMRyr)ZMNsB&%9bGx0yX!gD@z7w*EM1rUu`^{ z4@xC+w>@u9X}0Y-sd;#N()qd#xtpH)-0^JF%L-LhpX=dmER7gYKZXz;>d!WW8g21t zC>6@x4vMw&*mmF;CtM7ro^m$>ySS}vGq_PY%nnL%-Ih&5GYqb=tiWw!qd*3f;;}7a zSD4#!^W@_@qiBDLHOyR~&vjqIB#@v`JCr^gN?qZxFGG=i5%4$yrMT>i$5*AU=TvCe zV^~Wn*}ohtX=X<0YOYJ#owaLuT@4gmV8qY`6r_HhxuAUY08R9-vp{NKCOTaA=U+Tc zG#*jK29G20^x?WjJWO;0VxkGE8c>Rti6$T>YA7B9rFfaBA=5L5 zLp+s$`DqmJ)8zy6Q@vtsAjMS&H%H=rI@>M|Nb%4C%c!9aHa3c6lmaO(I)LHRJb?_K zDKtTIAZ5VSO~d(949W0mOdkxT5_q_&G0QgRz3*_E>v7bWJ~D-ipH8hky8XBZKN>$* zKq)T9sp>Y_KZ5XVmLV`4RB=JMEmuS6tAt7Bjfw+Q!pIcbq(4`J-1|S$!K3XQIfR!E zAeg{&NK{Oy>WYEcW)ZN>n1R`*tz8_D`cvE7H7V-rjZ$FDYCOEgYSLEAdq{QzF5`KK zn+*`__dx=by4rGq?!nwL0tAWb0!SG2@jNt?F32tKbiQ)@)jj?igg7i0z4YVIpDk(E zBs^jN*^;nu1GLsX7;_%AJ(GKKKYVb)fTilg)b~XIYwt2DHe=Io2f0g zH;1K%d-~$qjQCKq(&k!iKJ(q8LRRF9xKzBiC^LQOLdw zrH_D814FL9P>O3tUUH zpjes5zR0X-5+b3>DB377AT_Xk849J)eerBi>S}j%$-?-FsJ9iZwy=VgZn(@?*O+}q zZk>&jX=Ta-^yyJ_AT=OlM)U@`#D08(82^yWxBekwO411TPUy2qI5r@JP#xT$ACH#kE|&@uV|SCVeUPYkM|5dI;zszEL!OEO_^!h({HS z=^Vt~9wz}0ZzwgOq^b(Qj`zTls(7>ykmBX5EU-f)RpDZFAT=Og9cC8|qA zEFD*W8UjeC)5Vah;-&FK1jayG4@mLSIFk%ykWT+c2L3r22p!=Mw}r)5zf-sXs_Z`? z!tis1$qgb4if0ed69mP8)WFc?;88TLya7DJ;_w7OH4KoQP+czIacE!xDhdRs) zdd2EMikG`!s2?a8NbzzPtQZ^Ew%k1V2~=Z~#2OTN>*wX%uJ^%@d!3%Cd4+lPwx+7ZiFt)$_e5vdemDZckT5u18mEg=J4qksAH6T2EJvoMmp$_gpnk8hChDlv+2v z*NaAbtkzW@ZkglR^`>3Txx4X;S9iBCcxsDUx4BnOu3C42cyFud4NK4N4sq+Q)-Of5 zU5|6ST3d3v=szpOy?W?HQMBHU`ikD4MN!{#d(`yT3Je~{$HzAu{FB)9;8@M~$3yt| zDkk&siGWY`Imqp?+wQoX@xT1MeU1*zW_qV5FC3RUvvq3nt*qcRHm>POHc7fl_A#}t znbZkC8=fsvbXxj8sJTf=`Ng!Lz%fI1Er0ijpFh;xZ*9i{p_Tfvo%Ol zy-v|IH*tII)iUC>@n3#|gXo%0@59DRg*w&Tpjn9(6fBh#OdfUX__aeZ!<*ILh`gDs zc*i2iu~I(X&v)z;fts3!5h^UIJ_8gs}FIPHb z=IqWOkt(Y}Q>|ZKSNFW9NB`h+eau6H`x*s__2LDuZ*46)9$~&bspRK(y)`Cd2VXj{ zZuW!wCyN_zZKNJl?D=`9SK(?+v+M2WCoMUyzxD_v-Nu*xz>iyiYf%ds0xa?Tb zI`N)8^oEkghvK7-RO}l2`G{O@cT-WS+A6Q5-5=}U%RSqd@ZPKYZLpvA>hIa6@%ry` zU!E*e{j9lY-KiyIvr4atKl#gH{mIcc6{PPcO*(T*V)P=jNnh`TT=^0AbE?>?kE70{ zob;8F9G?2|{gF}fYo3~0CcKc>eJ)*dQlr+>#yZ1T3}nB)QB)4>9R z%v!H% z?7p?s?L$ICo<{|}`8+f6;-%r2rXK{;9FR zLF}1UU`O_r`J)Pak7=!1G=2NYDVHSHS?gvkv}{?ZTW$QlFg8VG*5-3}G|sv4uTPX+ znQyUfb(rDmhQS%%KVR^5KX)Tpf@b+`;^ljKHrs{L=i8g*NbBx*RFD&McpbLRG4B5O zzY+p>2p3$5ei|O1YJcWd&)%ZKu7Ai6otRj1zOc`sL5)mm}uu#=l zbIQv5f?{dc7X2NawXx}{xAr2X&t*Q|Bli}ZGAZm*5Sm~)EVRnUOziD-%EHV_(QjS* z<5#GbzDPG|=uy_S-xMwNx@M>LjomR-j)OZEjx-}^_8}t8Gc0>@WcLLB`1062ngJU}o^^bBb$Ml{u$$^v|L32Y+P2JlvP@6z-Hw%( zDkE;Z*74mvEpu;r-d^YCOG>_Tg`Va_|6{{jMTIZUon2!tp0=4PdFa%>1he=$2N%92 z$_w8s8236%UMReL_sY9+*E?REt2KSwl%Ch^VS8|> zzGt_brOS4Q-EzbqiZBJtgMSwWUE3ibKk{h1M%{h8mX0$qA!96;WcZHUtWD>8a5-Jq zp{jDnn%6n;`gYS-JYJ{KP;pr}qxQUkVU*#3ctJo#pygQjoniXAfwm_j z^Y2B9HO1FZn2d1>_ z?Jo3gJe1Iu?sGe;B6{>E4KJ%t+XcfeT<_NU+ok?U+j&FlH`d3t?wvJs?9n;)&_SE(7o9pewI}mh&c0W)A-G~Tp33yo1qsnjl8Z{+|;j=2M^tFAQOX)IN-mKB4r>s9j z%xh}wX-^MUpQ51^T|6jM>6+q?LS@CazuV$_x1U;3KdN+MkX`gy;iHN7p3GGB|0wXY zB_V0Ap`^|9@5{AjEYeq=Z=5^Qt2aeFU3Ykt=okgx6`HN;o9EwXYCO;+JAG)YfZ2oN zYJ$V_yjv;fzqFpwc=K38@YwC`6CI}tj7yNx5I(4@K0i7&DemK6cL$Fw)IVNHxo1R&iOpW_89}FwvM=jm%cq1V$x$&h|>Bj4agI2H68Ps-f*WR&%pB&zPdvkV@fZXbL zPd-UR1~{1xsjHyr%-VSQ+bBO;!b8uD2lH&My$dq>duU?j7lYCQ<)}|L9h0=hisf7P zpL6eB`EX*c9raYAxZywiM@;S!k(LuPIX`d0vXcjQb$uW6Si`bNYh)+CpUXUoxY?Sf zgQJE zu^Yw-Z=k$p|*V^_qr4>SDvMm)SbJmW=%ZRMvt>TWkgunCJBnf_aP@bOkF)3j;rx2CLM@g8=;#b`8?!g{jS%FVLi_m8oxbjb#mn8 zEu&VdZdzec-I4UB`o_Gmk2a*#^XEioj%%}iXL@6A;UZ)6I}TIa-d<0hyIOwkxr%i! zB@_$4hiuxn_uOBff8@2_P*qK=xoV|$g#Z2LafPDqo<1=!l{nd&_a?1n=KR@z4fpj) zIhvd>>&p+HrxUbfMwTfE8=u^k`v0ms>!_-}ZI9DZQUW5~AqWR)=@9Aeu7h;f0i==c zlm=-5X{4p3Lr_v0se`0+y(8Y=kL!6p?;ZEA`!)j@W3%Vy%(dp)d(ZEjYYdomk%8+q zl_4r3*=q6qR+{5|`9{yrV!J>_GNhSl;-m>278eg0KY`T8%{E_qh1c^9QIDP}G9%_$ zjus}BqRnWkfMYh1I=cg%!>VC|InyH&D!=xqhms%7a%wO?A-CAB^GZnIXAa#60&*T-KjYTDT(W=*71jzi&6>%>FU=B@~HKmmr- z3>R4e!cjDIWaa|E9mqdIWOm~;~p3005TSQiDl4M&(_H)xX3l&3AC&wX{qZ_#jt&> zjwx1`;pHOZm;>MS%5W}2JE$OBmlYTqkSa^Qc15djK28@MU)+tBU+m~zv^ghzS>8z3 zoQ9DUb;)k}tnI#dO$nn<-&LKXLY2?-S?(D_qQ{~6n_TO8I_tGnGcsoTe1<9;_0R8K zi0AbhW%}!3$GzmloX{M5vOO}0t96dS!+YFbpEIE6%-83b1uq zi0>F5R89mRRKSjW8TZH;FGtt6ao$6zx$J?JwAgE&j!-|C&Bval-HFwpNpyEHp-NZA zlPhv*4n4sf@;&P|MKKS~QSUvNLJdTJYM%AIa>#L?13jyyQ)+aIS7J%!!rAL(JRflB zJf4y#aS&r?Qa?>cmm_+dnr zjc(^VL3?x*hDNSnf(Izp?I8Hx8RtFstFBMG-oHL~DT)^11tFS5l@tc%hdhA@5l1sa z2cU_vlY_a<^P9(Sf~Ky+GAEALeDN39yr6hxJgM5{fn3_!=QLCM^p@J8dEsx4?oy`W zGT|)1WUWk1$8F8%8L^aeO~vI3OI79~6;17j4cfp&T%Dd-p6oAMCRWmzQN=e|q}Rx6 zqni{`QpCL(CmR#ql@5E*5A0N?Su8%plL0Rux_Z~Me|W6JIH+oAV||r9#=1!S*k=0S zosd|Kjd1M?Wjc~L%Y~R%@RLm6(lRqzz$Q99_-LvW-&Db~8|V0pxhG0Ot!EBiULPA; z*q6>m^;{45@F?owL=wK5y{t|m@pv(1zf{xb#@9qaRtMJ5e>y_$wZg}XE8nN>NWxJf zN}`=C8tBy*fo-3s+&WF4*Zb54h+D}0?mla|FokHCJ*-4XiiA*(ebBszk0MssgV0s- z;0;l8ijVDnsMTaH@+I~x2tR#PPUAtsn|Hu?GRk(ia19I(3>?}O(gQRR2 zqXY6lb=G@D6D7RK8zM{q;YUmyV&pGs`Gxf+A5&@(U>6g<%^!NBjJ&3lm1+?$6SGE; z?A|;~PW9AgkG?%{EP*484&I6>}#*f{h|7gd(%D-6Df`J{1aH72zrKGnuJ=V~bes{wsQ zZRx0fZw5~Qm{XkUJCoiO1y1#>o37XEb~vK4LlQ4Wla;7@sD4(nYm$Wck%1#=!R340 zZ0z~TYXFt^!CgqgQKNeg&2pH{Y=zoKC|yaA-WD?YevXdb^QV&4NX{$&c98f#yLgJzXWXpQKIGNc?o z7Bri(&F42UqvoWM`%t~YuF6?SamJt$tGUBHK*6oH-twN}LvuTfALuYwgAIg-UFoAj zQ{%O~NJ;O2Z+W5=|7tupa6L#DF}mXcOV<|oZM|y9D(jiZSK7P|K}DkLx#ust=Nyl5 zEBYnkJf>U5n$#t~V$O3f;gY&FEV<9>2|ZuOR1@ss;v1+Rw6P~}Zt;u%EZ9=!*gudE zic&g7i~5!*ffF@5Q_}h?mdPZ)MVI9h9mZkY{gwF7PCKM>pBYuEhKyPE{hwtI+vlrI z$8qx?p@gsx*|8h(bB;em7%ShomrPkh9H&Q+c`>EvHNqU}6F&%lhEbx6~rn%Z6SNiwjl)_2vLjrd&n^bAVw?KbXom zXxYNqOj$rIGGmJWhP#A%>GfBVi*AS1tcfF3qnq0y9m*Kw7sV0Kv$Hr(Y z1_7<}r7WU0qu_Z2v!17RReRxVEZnx4?!4@j>vu&0CfwF$=h3(Zywf$9bgw%wY%27; z?de^v1K7!)q$g+uA4Q(?s@gp{5ms--DXSxj8Ro#AdO`jkS1f<=WlAhL6a8jpizDsT z#r7pjMvs~=O?7uO2t-_UAg-LzxOC?^n~usT?$H>JaM@`1age@#YKDl5KBcB?@i)R6 zx@`jEx7P^2CLDJ_Y{;XKL_;3-Unt<4BIu0*ju>f^LSRLQ8EIS)jQ^584VUzvLEJKb!G6?NbTr z65?kfQs|b>zMcf>`sWo`3F1^-REV52ex_F8i-C#YFnQrk1+G_)I0)T#|>g+;|HywnNy0Ij5yA8PtDuV)fS zkcIPXo>}@39g=Of{D=Ot|9ARd06Jwlg3P@Y{J)rgj=%l?9M>iV&x-YO$8QH~b=y?? zX|7Wpf8mJ#{US_&$#d5Z+Blk!+85^-%pz#~>%t@2`nYN@tdEsBO&}8t4>D!81TC`eXJDBo5&9E|L+ZAUGgLKT&W#mbfvBiyE zRa3YRI7FnAeng*)$w>Oz99e)~IGk)Trh>&a_^9xkVeU@3=!}+VR9lnBlIJ4IEE>a; zet#iWmv}lWob4fu+V!3|l_G-zlCLs~(pUw?XAheL#62^PVp*SzBz3K|GHrisAjXf= z_RYWkB-|=++BqEl{CenQTb>A9MN$;?ABLUt?`ix0xoo-qpZm}K-|s&))`I8|Pu}GS7@$$0G=PL5hjd9`V*XtQI{4ZR*D z+3@p?H=HbAhxZHC?lfWNeG5i271Dh2n8^jMrJSJ4KTIwk#q}i*170@^-V5=stw_mRhFb z%*)vqZ@^nejZGk~DrI6#-Sdb=eogez9Gw0(yf^acBt;Q%O%nn3x9>^!oIQG1s{rPaI_AaLxqPQtCZ_Q*Dvx&Z8EY6vLKkp71gi55@t(_?L$R2edf?aHE zr^}WbYuAWHXkLiW%dFD$?g^^~g7@BIC#?5$IIXaS!qZ85fACNxHs0O}=mz1OSg4Mr zSVRl>I8T@1q?V`y``Sz%<)3V%0uBc*(;Um?4TQE8CkT(dscPDpwEZ+kq7kvH45abJ zWc+1iaj26Koi(vo(bRy#olF{WKmMePFE-$J_s2<2ps?TR%c3M+>-c;;kB8JA*JQ-x z=-3~S?*%ZlqZ>w+3u#c6V9#N5B5RMrG4$og#z--ZhCG$35Eqk>$7k5QAkghTbFLJn z*}$&IU!j)|4i7;OL0_j6&JC&*b`#a0D#6mj*2Buc&cI4UMhoD9+lcriu0=!gEhZ0#IbjBOo&KO&m0)MnETKyQ|x z7F4NEswU4a%6U3XD<_(QFqS`C*(oHRDi)Hxyz_io$zX|{;f`SI+Wqc5lQZRnkEEU> z<10)g)J$+xrK6_lRdrvPybKhD4CCobebY$D%gyb*xnh{(#yj5BVenZXWAnUVUA46j5$sXnd_hjPZ@Dq_<`cmk08&;*Aom9_A!WAZSvq)Mf-&WTr_6G_T} zJ{!*8x4=I>vj({Zl?#nv(_RoO;kxM0lUK+d`xtEqXB|HJ2v2x`k5O~@5X&)8BA;C1Ey?Iwjk7w6 zQWE7tnwMjqFh8kQP%DoVXc9b{o=TV&BEmq`J-Mq#O(%@IT>9Whds1k@3cc__t|Yo zV9F7G-f4da#?a30CN3(vwle^9PqOlBgZ1W^F^nz+dglF#Qbc@qv(fEWG3MHZlqj(l z)aQqF z6=&9=fL-Eca05fFOX(PA{9FzZMu~0e9Oh=LgV~(=n#a0o5qgeS z9eP306=ydM0euC7lbMYx2Kpnw&WI~euRU&N5>bq=^Z>vxj^{S8f#?LDqv<5E4xH%B z{20pqh9kTYFtD*tvq~3o9Jp#NK;|tpb>becmuG%XEzi?7@5M32q96?mhYNqZqvzv; z63!^d>d-;`xuXYi=iVGXkaI9F+M>2LPCy$ceHD-$&{6lta)Acar~;_KkV|DCZ)3C@ z)1N2$av0i>K*^4L9AY#Dc?0}-jsJD{zxx!v2nOBYdDKvCA}flZatpGMQna=B1e6x0D z?=O-M>~m_k9}@$;HxC*By{+RGp#Sg&@Oxo%yU7C@2)!NR7U;%!1H9cF0c{H1(YrO> z1^hIH?(abppi6YO1Vf%bjOMmr2MvTSaNPo(p8NrLTkiVzf$Y400N&Q0pn=dOpj!{W z{|R_o5P}B&-$Q2P|AU9OrILRi2>uiBws!IF1K$Yz!Nc2P!k>UY^!#6k53Hr&AAtX) z?*BymA+Y~Cd|;V`{y@By+t5I0#d{0f`U(7J{R?djt+Z~9D@1O*`9*g@TSKSyx7L1Q qaQ{2Whc<^!K5xy5#Q(|sAE~H5w-&mjvWd3-dp%rvT~z literal 0 HcmV?d00001 From 33585a1d5205818abc4bc12fdcb7659057daf478 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 12:09:14 +0200 Subject: [PATCH 02/41] feat(cmdb-import): archive or soft-delete applications that left the export With missingRecords archive, the default, a completed import reconciles the municipality's usages and the modules its import created: an APPID on the archive sheet is archived, one on no sheet is soft-deleted, and without the archive sheet missing applications are archived only, with a report warning. A returning application is unarchived or restored before it is updated, never duplicated. Object state is read per lens (working, then _archived=true) and from the trash, because OpenRegister's list search hydrates no archive marker and drops soft-deleted rows. The endpoint accepts missingRecords keep or archive and translates ARCHIVE_UNAVAILABLE; OpenAPI and Postman describe the new values and outcomes. Second of three slices of #1281, on top of the archive-sheet slice. Co-Authored-By: Claude Opus 5.5 --- l10n/en.js | 16 +- l10n/en.json | 16 +- l10n/nl.js | 16 +- l10n/nl.json | 16 +- lib/Controller/CmdbImportController.php | 22 +- lib/Exception/CmdbImportException.php | 2 + lib/Service/Cmdb/CmdbImportReport.php | 32 +- lib/Service/CmdbExportImportService.php | 937 ++++++++++++++++-- lib/Service/ProgressTracker.php | 5 + openapi.json | 40 +- postman/stackiq-tests.json | 3 +- .../Controller/CmdbImportControllerTest.php | 32 +- .../Service/Cmdb/CmdbImportReportTest.php | 23 + .../Service/CmdbExportImportServiceTest.php | 908 ++++++++++++++++- 14 files changed, 1964 insertions(+), 104 deletions(-) diff --git a/l10n/en.js b/l10n/en.js index 8dd84a2f4..8e95ef029 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1038,7 +1038,21 @@ OC.L10N.register( "The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.": "The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.", "Choose a municipality or enter the name of a new one.": "Choose a municipality or enter the name of a new one.", "No running CMDB import has this id.": "No running CMDB import has this id.", - "Only keeping records that are missing from the export is supported.": "Only keeping records that are missing from the export is supported.", + "Applications missing from the export can only be archived or kept.": "Applications missing from the export can only be archived or kept.", + "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.": "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.", + "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.": "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.", + "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.": "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.", + "Unarchived": "Unarchived", + "Restored": "Restored", + "Archived": "Archived", + "Deleted": "Deleted", + "Applications missing from the export": "Applications missing from the export", + "Archive or delete them": "Archive or delete them", + "Keep them as they are": "Keep them as they are", + "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.": "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.", + "Applications missing from the export cannot be archived.": "Applications missing from the export cannot be archived.", + "OpenRegister is missing or too old to archive objects. Choose \"Keep them as they are\" and import again, or update OpenRegister. Nothing was imported.": "OpenRegister is missing or too old to archive objects. Choose \"Keep them as they are\" and import again, or update OpenRegister. Nothing was imported.", + "Checking the applications missing from the export: {processed} of {total}": "Checking the applications missing from the export: {processed} of {total}", "Sheet \"%1$s\" has more than %2$s rows.": "Sheet \"%1$s\" has more than %2$s rows.", "Sheet \"%1$s\" has no column \"%2$s\".": "Sheet \"%1$s\" has no column \"%2$s\".", "Stackiq is not configured: the register or its schemas cannot be found.": "Stackiq is not configured: the register or its schemas cannot be found.", diff --git a/l10n/en.json b/l10n/en.json index 313f968ea..0d85cfbaf 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1037,7 +1037,21 @@ "The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.": "The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.", "Choose a municipality or enter the name of a new one.": "Choose a municipality or enter the name of a new one.", "No running CMDB import has this id.": "No running CMDB import has this id.", - "Only keeping records that are missing from the export is supported.": "Only keeping records that are missing from the export is supported.", + "Applications missing from the export can only be archived or kept.": "Applications missing from the export can only be archived or kept.", + "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.": "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.", + "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.": "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.", + "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.": "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.", + "Unarchived": "Unarchived", + "Restored": "Restored", + "Archived": "Archived", + "Deleted": "Deleted", + "Applications missing from the export": "Applications missing from the export", + "Archive or delete them": "Archive or delete them", + "Keep them as they are": "Keep them as they are", + "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.": "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.", + "Applications missing from the export cannot be archived.": "Applications missing from the export cannot be archived.", + "OpenRegister is missing or too old to archive objects. Choose \"Keep them as they are\" and import again, or update OpenRegister. Nothing was imported.": "OpenRegister is missing or too old to archive objects. Choose \"Keep them as they are\" and import again, or update OpenRegister. Nothing was imported.", + "Checking the applications missing from the export: {processed} of {total}": "Checking the applications missing from the export: {processed} of {total}", "Sheet \"%1$s\" has more than %2$s rows.": "Sheet \"%1$s\" has more than %2$s rows.", "Sheet \"%1$s\" has no column \"%2$s\".": "Sheet \"%1$s\" has no column \"%2$s\".", "Stackiq is not configured: the register or its schemas cannot be found.": "Stackiq is not configured: the register or its schemas cannot be found.", diff --git a/l10n/nl.js b/l10n/nl.js index 3151813d6..b03118333 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1106,7 +1106,21 @@ OC.L10N.register( "The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.": "Het stackiq-register of de schema's ervan zijn niet gevonden. Voer bovenaan deze pagina Auto Configure uit en probeer het daarna opnieuw.", "Choose a municipality or enter the name of a new one.": "Kies een gemeente of voer de naam van een nieuwe in.", "No running CMDB import has this id.": "Er loopt geen CMDB-import met deze id.", - "Only keeping records that are missing from the export is supported.": "Alleen het behouden van records die in de export ontbreken, wordt ondersteund.", + "Applications missing from the export can only be archived or kept.": "Applicaties die in de export ontbreken, kunnen alleen worden gearchiveerd of behouden.", + "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.": "Applicaties die in de export ontbreken, kunnen niet worden gearchiveerd: OpenRegister ontbreekt of is te oud. Kies ervoor ze te behouden, of werk OpenRegister bij.", + "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.": "Tabblad \"%1$s\" heeft geen kolom \"%2$s\", dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", + "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.": "Tabblad \"%s\" is niet gevonden, dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", + "Unarchived": "Uit archief gehaald", + "Restored": "Hersteld", + "Archived": "Gearchiveerd", + "Deleted": "Verwijderd", + "Applications missing from the export": "Applicaties die in de export ontbreken", + "Archive or delete them": "Archiveren of verwijderen", + "Keep them as they are": "Laat ze zoals ze zijn", + "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.": "Een applicatie van deze gemeente die op geen van beide CMDB-tabbladen meer staat, wordt gearchiveerd als het tabblad \"Gearchiveerde Applicaties\" haar noemt, en naar de prullenbak verplaatst als geen enkel tabblad dat doet. Beide verdwijnen uit de lijsten, uit OpenCatalogi en uit Portaliq; een applicatie die in een latere export terugkomt, wordt teruggezet. Staat dat tabblad niet in het bestand, dan worden applicaties alleen gearchiveerd. Kies Laat ze zoals ze zijn om ze ongemoeid te laten.", + "Applications missing from the export cannot be archived.": "Applicaties die in de export ontbreken, kunnen niet worden gearchiveerd.", + "OpenRegister is missing or too old to archive objects. Choose \"Keep them as they are\" and import again, or update OpenRegister. Nothing was imported.": "OpenRegister ontbreekt of is te oud om objecten te archiveren. Kies \"Laat ze zoals ze zijn\" en importeer opnieuw, of werk OpenRegister bij. Er is niets geïmporteerd.", + "Checking the applications missing from the export: {processed} of {total}": "Applicaties controleren die in de export ontbreken: {processed} van {total}", "Sheet \"%1$s\" has more than %2$s rows.": "Tabblad \"%1$s\" heeft meer dan %2$s rijen.", "Sheet \"%1$s\" has no column \"%2$s\".": "Tabblad \"%1$s\" heeft geen kolom \"%2$s\".", "Stackiq is not configured: the register or its schemas cannot be found.": "Stackiq is niet ingesteld: het register of de schema's ervan zijn niet gevonden.", diff --git a/l10n/nl.json b/l10n/nl.json index 680f6f853..92a16d73e 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1105,7 +1105,21 @@ "The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.": "Het stackiq-register of de schema's ervan zijn niet gevonden. Voer bovenaan deze pagina Auto Configure uit en probeer het daarna opnieuw.", "Choose a municipality or enter the name of a new one.": "Kies een gemeente of voer de naam van een nieuwe in.", "No running CMDB import has this id.": "Er loopt geen CMDB-import met deze id.", - "Only keeping records that are missing from the export is supported.": "Alleen het behouden van records die in de export ontbreken, wordt ondersteund.", + "Applications missing from the export can only be archived or kept.": "Applicaties die in de export ontbreken, kunnen alleen worden gearchiveerd of behouden.", + "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.": "Applicaties die in de export ontbreken, kunnen niet worden gearchiveerd: OpenRegister ontbreekt of is te oud. Kies ervoor ze te behouden, of werk OpenRegister bij.", + "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.": "Tabblad \"%1$s\" heeft geen kolom \"%2$s\", dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", + "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.": "Tabblad \"%s\" is niet gevonden, dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", + "Unarchived": "Uit archief gehaald", + "Restored": "Hersteld", + "Archived": "Gearchiveerd", + "Deleted": "Verwijderd", + "Applications missing from the export": "Applicaties die in de export ontbreken", + "Archive or delete them": "Archiveren of verwijderen", + "Keep them as they are": "Laat ze zoals ze zijn", + "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.": "Een applicatie van deze gemeente die op geen van beide CMDB-tabbladen meer staat, wordt gearchiveerd als het tabblad \"Gearchiveerde Applicaties\" haar noemt, en naar de prullenbak verplaatst als geen enkel tabblad dat doet. Beide verdwijnen uit de lijsten, uit OpenCatalogi en uit Portaliq; een applicatie die in een latere export terugkomt, wordt teruggezet. Staat dat tabblad niet in het bestand, dan worden applicaties alleen gearchiveerd. Kies Laat ze zoals ze zijn om ze ongemoeid te laten.", + "Applications missing from the export cannot be archived.": "Applicaties die in de export ontbreken, kunnen niet worden gearchiveerd.", + "OpenRegister is missing or too old to archive objects. Choose \"Keep them as they are\" and import again, or update OpenRegister. Nothing was imported.": "OpenRegister ontbreekt of is te oud om objecten te archiveren. Kies \"Laat ze zoals ze zijn\" en importeer opnieuw, of werk OpenRegister bij. Er is niets geïmporteerd.", + "Checking the applications missing from the export: {processed} of {total}": "Applicaties controleren die in de export ontbreken: {processed} van {total}", "Sheet \"%1$s\" has more than %2$s rows.": "Tabblad \"%1$s\" heeft meer dan %2$s rijen.", "Sheet \"%1$s\" has no column \"%2$s\".": "Tabblad \"%1$s\" heeft geen kolom \"%2$s\".", "Stackiq is not configured: the register or its schemas cannot be found.": "Stackiq is niet ingesteld: het register of de schema's ervan zijn niet gevonden.", diff --git a/lib/Controller/CmdbImportController.php b/lib/Controller/CmdbImportController.php index 7d1c00a8f..bc7869cbe 100644 --- a/lib/Controller/CmdbImportController.php +++ b/lib/Controller/CmdbImportController.php @@ -86,7 +86,8 @@ public function __construct( * Import a TOPdesk CMDB export for one municipality. * * Multipart fields: `cmdbFile`, `municipalityUuid` or `municipalityName`, - * `updateExisting` (default true), `publish` (default true), `missingRecords` (only `keep`) and + * `updateExisting` (default true), `publish` (default true), `missingRecords` (`keep` or `archive`, + * default `archive`) and * `operationId` (pattern `cmdb-` plus 8 to 64 letters, digits or hyphens). * * @return JSONResponse The report (200), or an error envelope with the contract code. @@ -94,6 +95,7 @@ public function __construct( * @auth admin-only the import writes with RBAC and multitenancy off, across tenants, so only a full Nextcloud admin may run it. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012 */ public function import(): JSONResponse { try { @@ -247,15 +249,21 @@ protected function iniBytes(string $name): ?int { * @return array{path: string, options: array}|JSONResponse The import input, or the first error. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012 */ private function readOptions(string $path, string $fileName): array|JSONResponse { - $missingRecords = $this->stringParam(name: 'missingRecords', default: 'keep'); + $missingRecords = $this->stringParam(name: 'missingRecords', default: CmdbExportImportService::DEFAULT_MISSING_RECORDS); if ($missingRecords === null) { return $this->invalidField(field: 'missingRecords'); } - if ($this->importService->supportsMissingRecords(mode: $missingRecords) === false) { - return $this->error(code: 'MISSING_RECORDS_UNSUPPORTED', status: Http::STATUS_UNPROCESSABLE_ENTITY, details: ['accepted' => ['keep']]); + $accepted = $this->importService->missingRecordsModes(); + if (in_array($missingRecords, $accepted, true) === false) { + return $this->error( + code: 'MISSING_RECORDS_UNSUPPORTED', + status: Http::STATUS_UNPROCESSABLE_ENTITY, + details: ['accepted' => $accepted] + ); } // An unrecognised value is refused rather than read as true: true is the mode that overwrites. @@ -293,6 +301,7 @@ private function readOptions(string $path, string $fileName): array|JSONResponse 'municipalityName' => $municipalityName, 'updateExisting' => $updateExisting, 'publish' => $publish, + 'missingRecords' => $missingRecords, 'operationId' => $this->request->getParam('operationId'), 'fileName' => basename(str_replace('\\', '/', $fileName)), ], @@ -394,7 +403,7 @@ private function message(string $code, array $details): string { 'NO_FILE_UPLOADED' => $this->l10n->t('No file was uploaded.'), 'NOT_XLSX' => $this->l10n->t('The file is not an Excel workbook (.xlsx).'), 'FILE_TOO_LARGE' => $this->l10n->t('The file is larger than the maximum of %s MB.', [$megabytes]), - 'MISSING_RECORDS_UNSUPPORTED' => $this->l10n->t('Only keeping records that are missing from the export is supported.'), + 'MISSING_RECORDS_UNSUPPORTED' => $this->l10n->t('Applications missing from the export can only be archived or kept.'), 'FIELD_INVALID' => $this->fieldMessage(field: (string)($details['field'] ?? ''), accepted: $accepted), 'MUNICIPALITY_REQUIRED' => $this->l10n->t('Choose a municipality or enter the name of a new one.'), 'MUNICIPALITY_INVALID' => $this->l10n->t('The chosen organisation is not a municipality.'), @@ -410,6 +419,9 @@ private function message(string $code, array $details): string { 'SCHEMA_OUTDATED' => $this->l10n->t('The stackiq register is out of date; import its configuration again.'), 'IMPORT_IN_PROGRESS' => $this->l10n->t('Another CMDB import is running; try again when it has finished.'), 'MUNICIPALITY_AMBIGUOUS' => $this->l10n->t('Several municipalities have this name; choose one from the list.'), + 'ARCHIVE_UNAVAILABLE' => $this->l10n->t( + 'Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.' + ), default => $this->l10n->t('The import failed. The details are in the Nextcloud log.'), }; }//end message() diff --git a/lib/Exception/CmdbImportException.php b/lib/Exception/CmdbImportException.php index 31404c505..65127a9fe 100644 --- a/lib/Exception/CmdbImportException.php +++ b/lib/Exception/CmdbImportException.php @@ -50,6 +50,7 @@ class CmdbImportException extends RuntimeException { public const SCHEMA_OUTDATED = 'SCHEMA_OUTDATED'; public const IMPORT_IN_PROGRESS = 'IMPORT_IN_PROGRESS'; public const MUNICIPALITY_AMBIGUOUS = 'MUNICIPALITY_AMBIGUOUS'; + public const ARCHIVE_UNAVAILABLE = 'ARCHIVE_UNAVAILABLE'; /** * HTTP status per error code. @@ -70,6 +71,7 @@ class CmdbImportException extends RuntimeException { self::SCHEMA_OUTDATED => 503, self::IMPORT_IN_PROGRESS => 409, self::MUNICIPALITY_AMBIGUOUS => 422, + self::ARCHIVE_UNAVAILABLE => 503, ]; /** diff --git a/lib/Service/Cmdb/CmdbImportReport.php b/lib/Service/Cmdb/CmdbImportReport.php index 730365fde..2615c8b07 100644 --- a/lib/Service/Cmdb/CmdbImportReport.php +++ b/lib/Service/Cmdb/CmdbImportReport.php @@ -36,6 +36,10 @@ class CmdbImportReport { public const UNCHANGED = 'unchanged'; public const SKIPPED = 'skipped'; public const FAILED = 'failed'; + public const ARCHIVED = 'archived'; + public const UNARCHIVED = 'unarchived'; + public const DELETED = 'deleted'; + public const RESTORED = 'restored'; /** * The row entries, in processing order. @@ -90,7 +94,7 @@ public function __construct( * Add one row outcome. * * @param string $sheet The sheet name. - * @param int $row The 1-based sheet row number. + * @param int $row The 1-based sheet row number, or 0 for an application the reconciliation archived or deleted. * @param string $appId The APPID ('' when missing). * @param string $name The application name ('' when missing). * @param string $outcome One of the outcome constants. @@ -180,14 +184,25 @@ public function markCancelled(): void { }//end markCancelled() /** - * The number of rows processed so far. + * Whether the run stopped on a cancel. + * + * @return bool + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function isCancelled(): bool { + return $this->cancelled; + }//end isCancelled() + + /** + * The number of sheet rows processed so far; reconciliation entries (row 0) are not sheet rows. * * @return int * * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 */ public function processed(): int { - return count($this->rows); + return count(array_filter($this->rows, static fn (array $row): bool => $row['row'] > 0)); }//end processed() /** @@ -195,21 +210,28 @@ public function processed(): int { * * `unpublished` counts the modules created without a publication date; * a row whose module was created but whose usage then failed counts too. + * `processed` counts sheet rows only; every outcome, also of a + * reconciliation entry, is counted under its own key. * * @return array{rowsRead: int, processed: int, created: int, updated: int, unchanged: int, skipped: int, failed: int, - * warnings: int, unpublished: int} + * archived: int, unarchived: int, deleted: int, restored: int, warnings: int, unpublished: int} * * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 */ public function summary(): array { $summary = [ 'rowsRead' => $this->rowsRead, - 'processed' => count($this->rows), + 'processed' => $this->processed(), self::CREATED => 0, self::UPDATED => 0, self::UNCHANGED => 0, self::SKIPPED => 0, self::FAILED => 0, + self::ARCHIVED => 0, + self::UNARCHIVED => 0, + self::DELETED => 0, + self::RESTORED => 0, 'warnings' => 0, 'unpublished' => $this->unpublished, ]; diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 1efc049d7..3c98a3f1e 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -30,7 +30,14 @@ * - `publicationDate` is set to the import's start on create, unless the * admin chose not to publish (`publish` false), and never written on * update; neither is `depublicationDate`. - * - Records missing from a newer export are left untouched. + * - Records missing from a newer export are left untouched with + * `missingRecords` `keep`. With `archive` (the default) the municipality's + * usages and their modules are reconciled after the last row: archived + * when the APPID is on the archive sheet, soft-deleted when it is on no + * sheet, archived only when the workbook has no archive sheet + * (openspec/changes/cmdb-import-archive-reconciliation, design D5). A + * module or usage matched among archived or soft-deleted objects is + * unarchived or restored before it is updated, never duplicated. * - Owners become contact persons, never Nextcloud user accounts, and no * report entry or log line carries an owner name or e-mail address. * - One import runs per register at a time: every match is find-then-create, @@ -55,6 +62,7 @@ use DateTimeImmutable; use DateTimeZone; +use JsonSerializable; use OCA\OpenRegister\Contract\ObjectServiceInterface; use OCA\Stackiq\Exception\CmdbImportException; use OCA\Stackiq\Service\Cmdb\CmdbImportProfile; @@ -68,6 +76,7 @@ use OCP\Lock\LockedException; use Psr\Container\ContainerInterface; use Psr\Log\LoggerInterface; +use RuntimeException; use Throwable; /** @@ -111,6 +120,58 @@ class CmdbExportImportService { */ public const SCHEMA_MAPPER_CLASS = 'OCA\OpenRegister\Db\SchemaMapper'; + /** + * OpenRegister's archive handler, behind POST and DELETE .../archive (not a public contract). + */ + public const ARCHIVE_HANDLER_CLASS = 'OCA\OpenRegister\Service\Object\ArchiveHandler'; + + /** + * OpenRegister's object mapper, whose restoreObject() empties the trash of one object (not a public contract). + */ + public const OBJECT_MAPPER_CLASS = 'OCA\OpenRegister\Db\MagicMapper'; + + /** + * What happens to records missing from the export when the request does not say. + */ + public const DEFAULT_MISSING_RECORDS = 'archive'; + + /** + * The start of every archive reason the import writes, so the audit trail says who archived and why. + */ + private const REASON_PREFIX = 'cmdb-import: '; + + /** + * The search lens that lists the archive alone (OpenRegister's `_archived=true`). + * + * The state of a stored object is read from the lens that found it, never + * from the archive marker of a search result: OpenRegister's list search + * does not hydrate that marker (design D9). + * + * @var array + */ + private const ARCHIVED_LENS = ['_archived' => 'true']; + + /** + * The state of an object found without a lens. + * + * @var array{archived: bool, deleted: bool} + */ + private const STATE_WORKING = ['archived' => false, 'deleted' => false]; + + /** + * The state of an object found with ARCHIVED_LENS. + * + * @var array{archived: bool, deleted: bool} + */ + private const STATE_ARCHIVED = ['archived' => true, 'deleted' => false]; + + /** + * The state of an object found in the trash; whether it was archived too is read when it is restored. + * + * @var array{archived: bool, deleted: bool} + */ + private const STATE_DELETED = ['archived' => false, 'deleted' => true]; + /** * The properties every match search and the module key rely on, per schema. * @@ -222,6 +283,20 @@ class CmdbExportImportService { */ private array $winningSheets = []; + /** + * OpenRegister's archive handler and object mapper, once resolved. + * + * @var array{handler: object, mapper: object}|null + */ + private ?array $archiveServices = null; + + /** + * The register's soft-deleted modules and usages, read once per run: schema key => uuid => entity. + * + * @var array{module: array, usage: array}|null + */ + private ?array $trash = null; + /** * Contact UIDs by e-mail or display name, per run. * @@ -300,17 +375,16 @@ public function assertXlsx(string $path, string $fileName): void { }//end assertXlsx() /** - * Whether a `missingRecords` value is supported (only `keep`). - * - * @param string $mode The requested value. + * The accepted `missingRecords` values: the profile's modes (`keep`, `archive`); any other is refused. * - * @return bool + * @return array * * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012 */ - public function supportsMissingRecords(string $mode): bool { - return $mode === 'keep'; - }//end supportsMissingRecords() + public function missingRecordsModes(): array { + return $this->profile->missingRecordsModes(); + }//end missingRecordsModes() /** * Ask a running import to stop between rows. @@ -349,19 +423,21 @@ public function requestCancel(string $operationId): bool { * * @param string $path The xlsx file, already checked by assertXlsx(). * @param array $options municipalityUuid, municipalityName, updateExisting, publish, - * operationId, and fileName (the upload's name, for the audit log line). + * missingRecords (keep or archive, default archive), operationId, + * and fileName (the upload's name, for the audit log line). * * @return array The report (contract.md). * * @throws CmdbImportException MAPPING_UNAVAILABLE, NOT_CONFIGURED, SCHEMA_OUTDATED, IMPORT_IN_PROGRESS, * WORKBOOK_TOO_LARGE, READER_UNAVAILABLE, NOT_XLSX, NO_SOURCE_SHEET, * MISSING_COLUMN, TOO_MANY_ROWS, MUNICIPALITY_REQUIRED, - * MUNICIPALITY_INVALID or MUNICIPALITY_AMBIGUOUS. + * MUNICIPALITY_INVALID, MUNICIPALITY_AMBIGUOUS or ARCHIVE_UNAVAILABLE. * @throws \Exception An unexpected OpenRegister error outside a row, such as * creating the municipality; rows catch their own. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 */ public function import(string $path, array $options): array { $this->resetRun(); @@ -371,6 +447,10 @@ public function import(string $path, array $options): array { $this->profile->load(); $this->engine = $this->resolveEngine(); $this->coordinates = $this->resolveCoordinates(); + if (self::missingRecordsFrom(options: $options) === 'archive') { + // Refused before the lock and the file: an import that cannot reconcile must not half-run. + $this->archiveServices(); + } $lock = $this->acquireImportLock(register: $this->coordinates['register']); try { @@ -442,29 +522,20 @@ private function acquireImportLock(int $register): string { * @return array The report. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 */ private function runImport(string $path, array $options, string $startedAt): array { $workbook = $this->reader->read(path: $path, profile: $this->profile); $municipality = $this->resolveMunicipality(options: $options); $operationId = $this->operationIdFrom(options: $options); + $missingRecords = self::missingRecordsFrom(options: $options); $rows = array_values($workbook['rows']); $report = new CmdbImportReport(operationId: $operationId, rowsRead: count($rows)); $report->setMunicipality(uuid: $municipality['uuid'], name: $municipality['name'], created: $municipality['created']); $report->addImportWarnings(warnings: $this->translateImportWarnings(warnings: $workbook['importWarnings'])); - if ($municipality['created'] === true) { - $report->addImportWarnings( - warnings: [ - [ - 'sheet' => '', - 'message' => $this->l10n->t( - 'No municipality named "%s" was found, so it was created. Check the name if you meant an existing one.', - [$municipality['name']] - ), - ], - ] - ); - } + $archive = $this->archiveSheet(workbook: $workbook); + $this->addRunWarnings(report: $report, municipality: $municipality, archive: $archive, missingRecords: $missingRecords); $this->progressTracker->startOperation( operationType: self::OPERATION_TYPE, @@ -488,6 +559,7 @@ private function runImport(string $path, array $options, string $startedAt): arr 'municipalityCreated' => $municipality['created'], 'updateExisting' => $updateExisting, 'publish' => $publish, + 'missingRecords' => $missingRecords, ]; $this->logger->info('CmdbExportImportService: import started', array_merge($audit, ['rows' => count($rows)])); try { @@ -507,6 +579,16 @@ private function runImport(string $path, array $options, string $startedAt): arr $this->progressTracker->updateProgress(processedItems: ($index + 1)); } + if ($missingRecords === 'archive' && $report->isCancelled() === false) { + $this->reconcile( + municipalityUuid: $municipality['uuid'], + archive: $archive, + operationId: $operationId, + processedItems: count($rows), + report: $report + ); + } + $result = $report->toArray(); $this->finishOperation(report: $report->toStoredArray(maxRows: self::STORED_REPORT_ROWS)); } catch (Throwable $e) { @@ -528,6 +610,39 @@ private function runImport(string $path, array $options, string $startedAt): arr return $result; }//end runImport() + /** + * Add the run's own warnings: a missing archive sheet on an archiving import, and a created municipality. + * + * @param CmdbImportReport $report The report. + * @param array{uuid: string, name: string, created: bool} $municipality The consumer, as resolveMunicipality() found it. + * @param array{sheet: string, present: bool, keyColumnMissing: bool, keys: array} $archive The archive sheet. + * @param string $missingRecords The import's `missingRecords` mode. + * + * @return void + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function addRunWarnings(CmdbImportReport $report, array $municipality, array $archive, string $missingRecords): void { + if ($missingRecords === 'archive' && $archive['present'] === false) { + $report->addImportWarnings(warnings: [['sheet' => $archive['sheet'], 'message' => $this->archiveSheetWarning(archive: $archive)]]); + } + + if ($municipality['created'] === true) { + $report->addImportWarnings( + warnings: [ + [ + 'sheet' => '', + 'message' => $this->l10n->t( + 'No municipality named "%s" was found, so it was created. Check the name if you meant an existing one.', + [$municipality['name']] + ), + ], + ] + ); + } + }//end addRunWarnings() + /** * Process one row in its own error boundary and add its outcome. * @@ -748,11 +863,20 @@ private function translateImportWarnings(array $warnings): array { * @param string $module The module outcome. * @param string $usage The usage outcome. * - * @return string created when the module was created, updated when anything was saved, else unchanged. + * @return string restored when either was restored from the trash, else unarchived when either was + * unarchived, else created when the module was created, updated when anything was saved, + * else unchanged. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 */ private static function rowOutcome(string $module, string $usage): string { + foreach ([CmdbImportReport::RESTORED, CmdbImportReport::UNARCHIVED] as $revived) { + if ($module === $revived || $usage === $revived) { + return $revived; + } + } + if ($module === CmdbImportReport::CREATED) { return CmdbImportReport::CREATED; } @@ -781,7 +905,7 @@ private function finishOperation(array $report): void { } $this->progressTracker->completeOperation(finalStatistics: ['report' => $report]); - // A cancel that came in after the last row's check has nothing left to stop. + // A cancel that came in after the last check has nothing left to stop. $this->progressTracker->clearCancelRequested(operationId: (string)$report['operationId']); }//end finishOperation() @@ -1028,19 +1152,19 @@ private function resolveManufacturer(array $values, int $rowNumber): ?string { * @param string $matchKey The APPID's match key. * @param array{updateExisting: bool} $options Whether a match is updated. * - * @return array{existing: object|null, uuid: string|null, skipReason: string|null} The stored module, the uuid - * to report for a skipped row, - * and why the row is skipped. + * @return array{existing: array{entity: object, state: array{archived: bool, deleted: bool}}|null, uuid: string|null, skipReason: string|null} + * The stored module with its state, the uuid to report for a skipped row, and why the row is skipped. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 */ private function matchModule(string $externalKey, string $municipalityUuid, string $matchKey, array $options): array { - $existing = $this->findOne(schemaKey: 'module', filters: ['externalKey' => $externalKey]); + $existing = $this->findInAnyState(schemaKey: 'module', filters: ['externalKey' => $externalKey]); if ($existing === null) { return ['existing' => null, 'uuid' => null, 'skipReason' => null]; } - $uuid = (string)$existing->getUuid(); + $uuid = (string)$existing['entity']->getUuid(); if ($this->moduleBelongsTo(moduleUuid: $uuid, municipalityUuid: $municipalityUuid) === false) { $this->logger->warning( 'CmdbExportImportService: import key belongs to a module another organisation uses; row not imported', @@ -1062,7 +1186,8 @@ private function matchModule(string $externalKey, string $municipalityUuid, stri * * @param array $data The mapped module fields. * @param string $externalKey The module's import key. - * @param object|null $existing The module matchModule() found, or null to create one. + * @param array{entity: object, state: array{archived: bool, deleted: bool}}|null $existing The module matchModule() + * found, or null to create one. * @param string|null $providerUuid The supplier, when there is one. * @param string|null $publicationDate ISO start time of the import for a module that is published * when created, or null to create it unpublished. @@ -1075,7 +1200,7 @@ private function matchModule(string $externalKey, string $municipalityUuid, stri private function importModule( array $data, string $externalKey, - ?object $existing, + ?array $existing, ?string $providerUuid, ?string $publicationDate, CmdbImportReport $report, @@ -1102,6 +1227,8 @@ private function importModule( * together with the usages: the module must already have a usage whose * consumer is this municipality, or, before the first import, no usage * at all. A module only another organisation uses is never taken over. + * This municipality's usage counts in every state: an archived or + * soft-deleted usage still says the module is the municipality's. * * @param string $moduleUuid The module found by its import key. * @param string $municipalityUuid The consumer of this import. @@ -1109,9 +1236,10 @@ private function importModule( * @return bool * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 */ private function moduleBelongsTo(string $moduleUuid, string $municipalityUuid): bool { - if ($this->findOne(schemaKey: 'usage', filters: ['consumer' => $municipalityUuid, 'module' => $moduleUuid]) !== null) { + if ($this->findInAnyState(schemaKey: 'usage', filters: ['consumer' => $municipalityUuid, 'module' => $moduleUuid]) !== null) { return true; } @@ -1123,19 +1251,22 @@ private function moduleBelongsTo(string $moduleUuid, string $municipalityUuid): * * @param array $data The mapped module fields. * @param string $externalKey The match key. - * @param object|null $existing The stored module, or null to create one. + * @param array{entity: object, state: array{archived: bool, deleted: bool}}|null $existing The stored module and its + * state, or null to create one. * @param string|null $providerUuid The supplier, when there is one. * @param string|null $publicationDate ISO start time of the import for a module that is published * when created, or null to create it unpublished. * - * @return array{uuid: string, outcome: string} Outcome created, updated or unchanged. + * @return array{uuid: string, outcome: string} Outcome created, updated or unchanged, or restored or + * unarchived when the stored module was revived first. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 */ private function upsertModule( array $data, string $externalKey, - ?object $existing, + ?array $existing, ?string $providerUuid, ?string $publicationDate, ): array { @@ -1154,14 +1285,15 @@ private function upsertModule( return ['uuid' => $this->save(schemaKey: 'module', data: $create, uuid: null), 'outcome' => CmdbImportReport::CREATED]; } - $uuid = (string)$existing->getUuid(); - $merged = $this->merge(target: 'module', stored: $existing->getObject(), mapped: $data); + $uuid = (string)$existing['entity']->getUuid(); + $revived = $this->revive(found: $existing); + $merged = $this->merge(target: 'module', stored: $revived['entity']->getObject(), mapped: $data); if ($merged === null) { - return ['uuid' => $uuid, 'outcome' => CmdbImportReport::UNCHANGED]; + return ['uuid' => $uuid, 'outcome' => ($revived['outcome'] ?? CmdbImportReport::UNCHANGED)]; } $this->save(schemaKey: 'module', data: $merged, uuid: $uuid); - return ['uuid' => $uuid, 'outcome' => CmdbImportReport::UPDATED]; + return ['uuid' => $uuid, 'outcome' => ($revived['outcome'] ?? CmdbImportReport::UPDATED)]; }//end upsertModule() /** @@ -1173,9 +1305,11 @@ private function upsertModule( * @param string|null $providerUuid The supplier, when there is one. * @param array{businessOwner: string|null} $owners The owner contact person. * - * @return array{uuid: string, outcome: string} + * @return array{uuid: string, outcome: string} Outcome created, updated or unchanged, or restored or + * unarchived when the stored usage was revived first. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 */ private function upsertUsage(array $data, string $municipalityUuid, string $moduleUuid, ?string $providerUuid, array $owners): array { if (isset($data['interneAnnotation']) === true && is_string($data['interneAnnotation']) === true) { @@ -1199,19 +1333,20 @@ private function upsertUsage(array $data, string $municipalityUuid, string $modu } } - $existing = $this->findOne(schemaKey: 'usage', filters: ['consumer' => $municipalityUuid, 'module' => $moduleUuid]); + $existing = $this->findInAnyState(schemaKey: 'usage', filters: ['consumer' => $municipalityUuid, 'module' => $moduleUuid]); if ($existing === null) { return ['uuid' => $this->save(schemaKey: 'usage', data: $data, uuid: null), 'outcome' => CmdbImportReport::CREATED]; } - $uuid = (string)$existing->getUuid(); - $merged = $this->merge(target: 'usage', stored: $existing->getObject(), mapped: $data); + $uuid = (string)$existing['entity']->getUuid(); + $revived = $this->revive(found: $existing); + $merged = $this->merge(target: 'usage', stored: $revived['entity']->getObject(), mapped: $data); if ($merged === null) { - return ['uuid' => $uuid, 'outcome' => CmdbImportReport::UNCHANGED]; + return ['uuid' => $uuid, 'outcome' => ($revived['outcome'] ?? CmdbImportReport::UNCHANGED)]; } $this->save(schemaKey: 'usage', data: $merged, uuid: $uuid); - return ['uuid' => $uuid, 'outcome' => CmdbImportReport::UPDATED]; + return ['uuid' => $uuid, 'outcome' => ($revived['outcome'] ?? CmdbImportReport::UPDATED)]; }//end upsertUsage() /** @@ -1633,7 +1768,7 @@ private function organisationsOfType(string $type): array { }//end organisationsOfType() /** - * The one object matching every filter, or null. + * The working object matching every filter, or null. * * @param string $schemaKey The coordinates key of the schema. * @param array $filters Field => exact value. @@ -1643,10 +1778,67 @@ private function organisationsOfType(string $type): array { * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 */ private function findOne(string $schemaKey, array $filters): ?object { + return $this->search(schemaKey: $schemaKey, filters: $filters, lens: []); + }//end findOne() + + /** + * The object matching every filter in any state, with that state, or null. + * + * A working object wins over an archived one, and an archived one over one + * in the trash: an object deleted by hand may share its import key with the + * one a later import created, and bringing the deleted one back would + * duplicate it. The state is the lens that found the object (design D9): + * a search without a lens finds only working objects, the archive lens only + * archived ones, and the trash is read from OpenRegister's trash listing, + * because its list search cannot return a soft-deleted object. + * + * @param string $schemaKey module or usage. + * @param array $filters Field => exact value. + * + * @return array{entity: object, state: array{archived: bool, deleted: bool}}|null + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + private function findInAnyState(string $schemaKey, array $filters): ?array { + $entity = $this->search(schemaKey: $schemaKey, filters: $filters, lens: []); + if ($entity !== null) { + return ['entity' => $entity, 'state' => self::STATE_WORKING]; + } + + $entity = $this->search(schemaKey: $schemaKey, filters: $filters, lens: self::ARCHIVED_LENS); + if ($entity !== null) { + return ['entity' => $entity, 'state' => self::STATE_ARCHIVED]; + } + + foreach (($this->trash()[$schemaKey] ?? []) as $entity) { + if (self::matchesFilters(entity: $entity, filters: $filters) === true) { + return ['entity' => $entity, 'state' => self::STATE_DELETED]; + } + } + + return null; + }//end findInAnyState() + + /** + * The first object a search with one lens returns that matches every filter, or null. + * + * The filters are checked again on the result: a filter OpenRegister cannot + * apply must not turn into a match. + * + * @param string $schemaKey The coordinates key of the schema. + * @param array $filters Field => exact value. + * @param array $lens No lens (working objects) or ARCHIVED_LENS. + * + * @return object|null The entity (ObjectEntityInterface). + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + private function search(string $schemaKey, array $filters, array $lens): ?object { $coordinates = $this->coordinates(); $results = $coordinates['objectService']->searchObjects( query: array_merge( ['@self' => ['register' => $coordinates['register'], 'schema' => $coordinates[$schemaKey]], '_limit' => 10], + $lens, $filters ), _rbac: false, @@ -1657,22 +1849,34 @@ private function findOne(string $schemaKey, array $filters): ?object { } foreach ($results as $entity) { - $data = $entity->getObject(); - $matches = true; - foreach ($filters as $field => $value) { - if (self::relationUuid(value: ($data[$field] ?? null)) !== $value) { - $matches = false; - break; - } - } - - if ($matches === true) { + if (self::matchesFilters(entity: $entity, filters: $filters) === true) { return $entity; } } return null; - }//end findOne() + }//end search() + + /** + * Whether an object's data has every filter's value. + * + * @param object $entity The entity (ObjectEntityInterface). + * @param array $filters Field => exact value. + * + * @return bool + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private static function matchesFilters(object $entity, array $filters): bool { + $data = $entity->getObject(); + foreach ($filters as $field => $value) { + if (self::relationUuid(value: ($data[$field] ?? null)) !== $value) { + return false; + } + } + + return true; + }//end matchesFilters() /** * Save an object through OpenRegister and return its uuid. @@ -1699,6 +1903,621 @@ private function save(string $schemaKey, array $data, ?string $uuid): string { return (string)$entity->getUuid(); }//end save() + /** + * The `missingRecords` mode of an import: the option, or DEFAULT_MISSING_RECORDS. + * + * @param array $options The import options. + * + * @return string + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012 + */ + private static function missingRecordsFrom(array $options): string { + $mode = ($options['missingRecords'] ?? null); + if (is_string($mode) === false || $mode === '') { + return self::DEFAULT_MISSING_RECORDS; + } + + return $mode; + }//end missingRecordsFrom() + + /** + * The archive sheet as the reader found it, with its APPIDs as match keys. + * + * @param array $workbook What the reader returned. + * + * @return array{sheet: string, present: bool, keyColumnMissing: bool, keys: array} + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function archiveSheet(array $workbook): array { + $archive = ($workbook['archive'] ?? []); + $key = $this->profile->keyColumn(); + $keys = []; + foreach (($archive['appIds'] ?? []) as $appId) { + $values = $this->normaliser->normalise(cells: [$key => $appId], dateColumns: [], idColumns: $this->profile->idColumns()); + $matchKey = self::matchKey(appId: ($values[$key] ?? '')); + if ($matchKey !== '') { + $keys[$matchKey] = true; + } + } + + return [ + 'sheet' => (string)($archive['sheet'] ?? ($this->profile->archiveSheetName() ?? '')), + 'present' => (($archive['present'] ?? false) === true), + 'keyColumnMissing' => (($archive['keyColumnMissing'] ?? false) === true), + 'keys' => $keys, + ]; + }//end archiveSheet() + + /** + * The report warning of an archiving import whose workbook lacks the archive sheet or its key column. + * + * @param array{sheet: string, keyColumnMissing: bool} $archive The archive sheet. + * + * @return string + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function archiveSheetWarning(array $archive): string { + if ($archive['keyColumnMissing'] === true) { + return $this->l10n->t( + 'Sheet "%1$s" has no column "%2$s", so applications missing from the export are archived, not deleted.', + [$archive['sheet'], $this->profile->keyColumn()] + ); + } + + return $this->l10n->t('Sheet "%s" was not found, so applications missing from the export are archived, not deleted.', [$archive['sheet']]); + }//end archiveSheetWarning() + + /** + * Resolve OpenRegister's archive handler and object mapper, failing closed. + * + * @return array{handler: object, mapper: object} The handler, with `archive()` and `unarchive()`, and the + * mapper, with `restoreObject()` and `findDeletedAcrossAllMagicTables()`. + * + * @throws CmdbImportException ARCHIVE_UNAVAILABLE. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function archiveServices(): array { + if ($this->archiveServices !== null) { + return $this->archiveServices; + } + + try { + $handler = $this->container->get(static::ARCHIVE_HANDLER_CLASS); + $mapper = $this->container->get(static::OBJECT_MAPPER_CLASS); + } catch (Throwable $e) { + $handler = null; + $mapper = null; + } + + if (is_object($handler) === false || method_exists($handler, 'archive') === false || method_exists($handler, 'unarchive') === false + || is_object($mapper) === false || method_exists($mapper, 'restoreObject') === false + || method_exists($mapper, 'findDeletedAcrossAllMagicTables') === false + ) { + throw new CmdbImportException( + errorCode: CmdbImportException::ARCHIVE_UNAVAILABLE, + message: 'OpenRegister ArchiveHandler or MagicMapper is not available' + ); + } + + $this->archiveServices = ['handler' => $handler, 'mapper' => $mapper]; + return $this->archiveServices; + }//end archiveServices() + + /** + * Whether an object as OpenRegister's mapper reads it by uuid is archived and whether it is in the trash. + * + * Read from its `@self` metadata. Only for an object the mapper read + * itself (the one restoreObject() answers): a list search result carries + * no archive marker (design D9). + * + * @param object $entity The entity (ObjectEntityInterface). + * + * @return array{archived: bool, deleted: bool} + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + private static function objectState(object $entity): array { + $self = []; + if ($entity instanceof JsonSerializable) { + $serialised = $entity->jsonSerialize(); + if (is_array($serialised) === true && is_array($serialised['@self'] ?? null) === true) { + $self = $serialised['@self']; + } + } + + return [ + 'archived' => (self::isEmptyValue(value: ($self['archived'] ?? null)) === false), + 'deleted' => (self::isEmptyValue(value: ($self['deleted'] ?? null)) === false), + ]; + }//end objectState() + + /** + * Bring a matched module or usage back before it is written: out of the trash, then out of the archive. + * + * OpenRegister refuses a data write to an archived object, and the + * archive handler cannot reach one in the trash, hence the order. The + * trash listing does not say whether an object was archived before it was + * deleted; the restored object, read by the mapper, does. The update then + * starts from that object: the trash listing decodes every string that + * parses as JSON, so its copy of the data must not be written back. + * + * @param array{entity: object, state: array{archived: bool, deleted: bool}} $found The matched entity and its state. + * + * @return array{outcome: string|null, entity: object} restored, unarchived, or null when it was neither, and the + * entity whose data the update starts from. + * + * @throws CmdbImportException ARCHIVE_UNAVAILABLE when the archive services are missing. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + private function revive(array $found): array { + $entity = $found['entity']; + $archived = $found['state']['archived']; + if ($found['state']['deleted'] === false && $archived === false) { + return ['outcome' => null, 'entity' => $entity]; + } + + $uuid = (string)$entity->getUuid(); + $outcome = null; + if ($found['state']['deleted'] === true) { + $restored = $this->restore(uuid: $uuid); + $outcome = CmdbImportReport::RESTORED; + if (is_object($restored) === true && method_exists($restored, 'getObject') === true) { + $entity = $restored; + $archived = self::objectState(entity: $restored)['archived']; + } + } + + if ($archived === true) { + $this->archiveServices()['handler']->unarchive(identifier: $uuid, reason: self::REASON_PREFIX . 'back in the export'); + $outcome = ($outcome ?? CmdbImportReport::UNARCHIVED); + } + + return ['outcome' => $outcome, 'entity' => $entity]; + }//end revive() + + /** + * Take one object out of the trash, and out of this run's copy of the trash. + * + * @param string $uuid The object. + * + * @return mixed What MagicMapper::restoreObject() answers: the object as it reads it. + * + * @throws CmdbImportException ARCHIVE_UNAVAILABLE when the archive services are missing. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + private function restore(string $uuid): mixed { + $restored = $this->archiveServices()['mapper']->restoreObject(uuid: $uuid); + if ($this->trash !== null) { + unset($this->trash['module'][$uuid], $this->trash['usage'][$uuid]); + } + + return $restored; + }//end restore() + + /** + * The register's soft-deleted modules and usages, read once per run. + * + * OpenRegister's list search cannot return a soft-deleted object, whatever + * `_includeDeleted` says, so the trash is read from the trash listing that + * the Deleted page uses (design D9). That listing scans every magic table + * of the instance, so it is read once, lazily, and only this register's + * modules and usages are kept. Its objects carry no archive or trash + * marker; everything in it is deleted. Without the archive services (a + * `keep` import on an OpenRegister without them) the trash cannot be read: + * nothing in it is matched, and the log says so. + * + * @return array{module: array, usage: array} Schema key => uuid => entity. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + private function trash(): array { + if ($this->trash !== null) { + return $this->trash; + } + + $this->trash = ['module' => [], 'usage' => []]; + try { + $services = $this->archiveServices(); + } catch (CmdbImportException $e) { + $this->logger->warning('CmdbExportImportService: OpenRegister trash is not readable; soft-deleted modules and usages are not matched'); + return $this->trash; + } + + $coordinates = $this->coordinates(); + $deleted = $services['mapper']->findDeletedAcrossAllMagicTables(limit: null, offset: null, _rbac: false, _multitenancy: false); + $this->trash = self::registerTrash(deleted: $deleted, coordinates: $coordinates); + + return $this->trash; + }//end trash() + + /** + * The modules and usages of this register among the instance's soft-deleted objects. + * + * @param mixed $deleted What MagicMapper::findDeletedAcrossAllMagicTables() answered. + * @param array $coordinates The register and schema ids, from coordinates(). + * + * @return array{module: array, usage: array} Schema key => uuid => entity. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + private static function registerTrash(mixed $deleted, array $coordinates): array { + $trash = ['module' => [], 'usage' => []]; + if (is_array($deleted) === false) { + return $trash; + } + + foreach ($deleted as $entity) { + if (is_object($entity) === false || (string)$entity->getRegister() !== (string)$coordinates['register'] || $entity->getUuid() === null) { + continue; + } + + foreach (['module', 'usage'] as $schemaKey) { + if ((string)$entity->getSchema() === (string)$coordinates[$schemaKey]) { + $trash[$schemaKey][(string)$entity->getUuid()] = $entity; + } + } + } + + return $trash; + }//end registerTrash() + + /** + * Archive or soft-delete every application of the municipality whose APPID left the CMDB sheets. + * + * Runs after the last row of an import that was not cancelled, under the + * same register lock. The whole scope is loaded before anything is + * written, so the paging is not shifted by the writes. Every application + * has its own error boundary. A cancel is honoured before the scope is + * loaded and before each application: what was archived or deleted so far + * stays, and the report says the run was cancelled. + * + * @param string $municipalityUuid The consumer of this import. + * @param array{sheet: string, present: bool, keyColumnMissing: bool, keys: array} $archive The archive sheet. + * @param string $operationId The operation id, for the archive reason. + * @param int $processedItems The rows already counted as processed. + * @param CmdbImportReport $report The report. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function reconcile(string $municipalityUuid, array $archive, string $operationId, int $processedItems, CmdbImportReport $report): void { + // A cancel that came in during the last row stops the import before anything is archived or deleted. + if ($this->progressTracker->isCancelRequested(operationId: $operationId) === true) { + $report->markCancelled(); + return; + } + + $scope = $this->reconciliationScope(municipalityUuid: $municipalityUuid); + $this->progressTracker->setPhase(phase: 'reconciling', data: ['total_items' => ($processedItems + count($scope))]); + + foreach ($scope as $index => $application) { + if ($this->progressTracker->isCancelRequested(operationId: $operationId) === true) { + $report->markCancelled(); + return; + } + + // Every APPID on a CMDB sheet was handled by its row, whatever the row's outcome. + if (isset($this->winningSheets[$application['key']]) === false) { + $this->reconcileApplication(application: $application, archive: $archive, operationId: $operationId, report: $report); + } + + $this->progressTracker->updateProgress(processedItems: ($processedItems + $index + 1)); + } + }//end reconcile() + + /** + * The municipality's usages, in every state, each with its module when that module carries this municipality's import key. + * + * A usage of a hand-made module, or of a module another organisation + * imported, is left out: only what this municipality's import created is + * reconciled. Working and archived usages are paged with one lens each, + * soft-deleted ones come from the trash (design D9); each object keeps the + * state its lens or the trash gave it. + * + * @param string $municipalityUuid The consumer of this import. + * + * @return array}> + * Per application, the APPID's match key, the usage, the module and the state of both by schema key. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function reconciliationScope(string $municipalityUuid): array { + $usages = $this->usagesInEveryState(municipalityUuid: $municipalityUuid); + + $owned = []; + foreach ($usages as $usage) { + $data = $usage['entity']->getObject(); + $moduleUuid = self::relationUuid(value: ($data['module'] ?? null)); + // The filter is checked again: a filter OpenRegister cannot apply must not widen the scope. + if (self::relationUuid(value: ($data['consumer'] ?? null)) === $municipalityUuid && $moduleUuid !== null && $usage['entity']->getUuid() !== null) { + $owned[] = $usage + ['module' => $moduleUuid]; + } + } + + $prefix = $this->profile->externalKeyPrefix() . ':' . $municipalityUuid . ':'; + $modules = $this->modulesByUuid(uuids: array_values(array_unique(array_column($owned, 'module')))); + $scope = []; + foreach ($owned as $usage) { + $module = ($modules[$usage['module']] ?? null); + if ($module === null) { + continue; + } + + $externalKey = (string)($module['entity']->getObject()['externalKey'] ?? ''); + if (str_starts_with($externalKey, $prefix) === true && strlen($externalKey) > strlen($prefix)) { + $scope[] = [ + 'key' => substr($externalKey, strlen($prefix)), + 'usage' => $usage['entity'], + 'module' => $module['entity'], + 'states' => ['usage' => $usage['state'], 'module' => $module['state']], + ]; + } + } + + return $scope; + }//end reconciliationScope() + + /** + * The usages that name this municipality as consumer, in every state, each with that state. + * + * Working and archived usages are paged with one lens each; soft-deleted + * ones come from the trash, which is not filtered by consumer, so the + * caller checks the consumer of every usage. + * + * @param string $municipalityUuid The consumer of this import. + * + * @return array + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function usagesInEveryState(string $municipalityUuid): array { + $coordinates = $this->coordinates(); + $usages = []; + foreach ([[[], self::STATE_WORKING], [self::ARCHIVED_LENS, self::STATE_ARCHIVED]] as [$lens, $state]) { + $offset = 0; + do { + $page = $coordinates['objectService']->searchObjects( + query: array_merge( + [ + '@self' => ['register' => $coordinates['register'], 'schema' => $coordinates['usage']], + 'consumer' => $municipalityUuid, + '_limit' => self::PAGE_SIZE, + '_offset' => $offset, + ], + $lens + ), + _rbac: false, + _multitenancy: false + ); + if (is_array($page) === false) { + $page = []; + } + + foreach ($page as $usage) { + $usages[] = ['entity' => $usage, 'state' => $state]; + } + + $offset += self::PAGE_SIZE; + $pageSize = count($page); + } while ($pageSize === self::PAGE_SIZE); + }//end foreach + + foreach ($this->trash()['usage'] as $usage) { + $usages[] = ['entity' => $usage, 'state' => self::STATE_DELETED]; + } + + return $usages; + }//end usagesInEveryState() + + /** + * The modules with these uuids, in every state, each with its state. + * + * Per chunk of PAGE_SIZE uuids one search without a lens and one with + * the archive lens; the rest is looked up in the trash. + * + * @param array $uuids The module uuids. + * + * @return array Uuid => entity and state. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function modulesByUuid(array $uuids): array { + $coordinates = $this->coordinates(); + $modules = []; + foreach (array_chunk($uuids, self::PAGE_SIZE) as $chunk) { + // The chunks are disjoint, so a uuid found in an earlier chunk is never overwritten. + $modules += $this->modulesInChunk(chunk: $chunk, coordinates: $coordinates); + } + + foreach ($uuids as $uuid) { + $trashed = ($this->trash()['module'][$uuid] ?? null); + if (isset($modules[$uuid]) === false && $trashed !== null) { + $modules[$uuid] = ['entity' => $trashed, 'state' => self::STATE_DELETED]; + } + } + + return $modules; + }//end modulesByUuid() + + /** + * The working and archived modules of one chunk of uuids; a module both lenses return keeps the first state. + * + * @param array $chunk At most PAGE_SIZE module uuids. + * @param array $coordinates The object service and register and schema ids, from coordinates(). + * + * @return array Uuid => entity and state. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function modulesInChunk(array $chunk, array $coordinates): array { + $modules = []; + foreach ([[[], self::STATE_WORKING], [self::ARCHIVED_LENS, self::STATE_ARCHIVED]] as [$lens, $state]) { + $results = $coordinates['objectService']->searchObjects( + query: array_merge( + ['@self' => ['register' => $coordinates['register'], 'schema' => $coordinates['module']], '_ids' => $chunk, '_limit' => count($chunk)], + $lens + ), + _rbac: false, + _multitenancy: false + ); + if (is_array($results) === false) { + continue; + } + + foreach ($results as $entity) { + $uuid = (string)$entity->getUuid(); + if (in_array($uuid, $chunk, true) === true && isset($modules[$uuid]) === false) { + $modules[$uuid] = ['entity' => $entity, 'state' => $state]; + } + } + } + + return $modules; + }//end modulesInChunk() + + /** + * Archive or soft-delete one application that left the CMDB sheets, usage first, in its own error boundary. + * + * The usage is written before the module, so a failure between the two + * leaves the municipality's view right and the module at worst visible. + * An application already in the state the export asks for is not reported. + * + * @param array{key: string, usage: object, module: object, states: array} $application Application. + * @param array{sheet: string, present: bool, keyColumnMissing: bool, keys: array} $archive The archive sheet. + * @param string $operationId The operation id, for the archive reason. + * @param CmdbImportReport $report The report. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function reconcileApplication(array $application, array $archive, string $operationId, CmdbImportReport $report): void { + $moduleData = $application['module']->getObject(); + $onArchiveSheet = ($archive['present'] === true && isset($archive['keys'][$application['key']]) === true); + $entry = [ + 'sheet' => '', + 'row' => 0, + 'appId' => (string)($moduleData['externalNumber'] ?? $application['key']), + 'name' => (string)($moduleData['name'] ?? ''), + ]; + + $target = CmdbImportReport::ARCHIVED; + $reason = self::REASON_PREFIX . 'not in the export (' . $operationId . ')'; + if ($onArchiveSheet === true) { + $entry['sheet'] = $archive['sheet']; + $reason = self::REASON_PREFIX . 'on sheet "' . $archive['sheet'] . '" (' . $operationId . ')'; + } elseif ($archive['present'] === true) { + $target = CmdbImportReport::DELETED; + } + + $uuids = ['module' => (string)$application['module']->getUuid(), 'usage' => (string)$application['usage']->getUuid()]; + $step = 'archive'; + $changed = false; + try { + foreach (['usage', 'module'] as $schemaKey) { + foreach (self::transitions(state: $application['states'][$schemaKey], target: $target, restoreDeleted: $onArchiveSheet) as $step) { + $this->transition(step: $step, schemaKey: $schemaKey, uuid: $uuids[$schemaKey], reason: $reason); + $changed = true; + } + } + } catch (Throwable $e) { + // The module's stored values are taken out of the log line, as a row's cells are. + $values = array_map('strval', array_filter($moduleData, static fn (mixed $value): bool => is_scalar($value))); + $this->failRow(report: $report, entry: $entry, step: $step, e: $e, warnings: [], uuids: [$uuids['module'], $uuids['usage']], values: $values); + return; + } + + if ($changed === true) { + $this->addRow(report: $report, entry: $entry, outcome: $target, moduleUuid: $uuids['module'], usageUuid: $uuids['usage']); + } + }//end reconcileApplication() + + /** + * The steps that bring one module or usage to the state the export asks for (design D5). + * + * Deleted: soft-delete what is not in the trash yet. Archived: archive + * what is working; an object in the trash is restored first only when the + * archive sheet lists it, so a workbook without the sheet never takes + * anything out of the trash. + * + * @param array{archived: bool, deleted: bool} $state The object's state. + * @param string $target archived or deleted. + * @param bool $restoreDeleted Whether an object in the trash is restored to be archived. + * + * @return array delete, restore and archive, in order; empty when nothing changes. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private static function transitions(array $state, string $target, bool $restoreDeleted): array { + if ($target === CmdbImportReport::DELETED) { + if ($state['deleted'] === true) { + return []; + } + + return ['delete']; + } + + if ($state['deleted'] === true) { + if ($restoreDeleted === true) { + return ['restore', 'archive']; + } + + return []; + } + + if ($state['archived'] === true) { + return []; + } + + return ['archive']; + }//end transitions() + + /** + * Run one step through OpenRegister: soft delete, restore from the trash, or archive. + * + * @param string $step delete, restore or archive. + * @param string $schemaKey The coordinates key of the schema. + * @param string $uuid The object. + * @param string $reason The archive reason. + * + * @return void + * + * @throws RuntimeException When OpenRegister does not delete the object. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function transition(string $step, string $schemaKey, string $uuid, string $reason): void { + if ($step === 'delete') { + $coordinates = $this->coordinates(); + $deleted = $coordinates['objectService']->deleteObject( + uuid: $uuid, + register: $coordinates['register'], + schema: $coordinates[$schemaKey], + _rbac: false, + _multitenancy: false + ); + if ($deleted !== true) { + throw new RuntimeException('OpenRegister did not delete the object'); + } + + return; + } + + if ($step === 'restore') { + $this->restore(uuid: $uuid); + return; + } + + $this->archiveServices()['handler']->archive(identifier: $uuid, reason: $reason); + }//end transition() + /** * Resolve OpenRegister's mapping engine. * @@ -1915,6 +2734,8 @@ private function resetRun(): void { $this->winningSheets = []; $this->contactUids = []; $this->contactPersons = []; + $this->archiveServices = null; + $this->trash = null; }//end resetRun() /** diff --git a/lib/Service/ProgressTracker.php b/lib/Service/ProgressTracker.php index b0c9ce229..5a9706ada 100644 --- a/lib/Service/ProgressTracker.php +++ b/lib/Service/ProgressTracker.php @@ -62,6 +62,11 @@ class ProgressTracker { 'processing_relationships' => ['weight' => 15, 'description' => 'Processing relationships'], 'processing_organizations' => ['weight' => 10, 'description' => 'Processing organizations'], 'processing_views' => ['weight' => 10, 'description' => 'Processing views'], + // The CMDB import's reconciliation of records missing from the source + // (openspec/changes/cmdb-import-archive-reconciliation). Weight 0, so the + // weighted percentage of the ArchiMate import's phases is unchanged; the + // CMDB section computes its own percentage from the item counts. + 'reconciling' => ['weight' => 0, 'description' => 'Reconciling records missing from the source'], 'finalizing' => ['weight' => 5, 'description' => 'Finalizing import'], 'completed' => ['weight' => 0, 'description' => 'Completed'], ]; diff --git a/openapi.json b/openapi.json index 7ee628b87..6469cf208 100644 --- a/openapi.json +++ b/openapi.json @@ -62,10 +62,11 @@ "missingRecords": { "type": "string", "enum": [ - "keep" + "keep", + "archive" ], - "default": "keep", - "description": "Only keep is accepted; mark and remove are reserved" + "default": "archive", + "description": "What happens to the municipality's applications whose APPID is no longer on either CMDB sheet. archive: after the last row, archive module and usage when the sheet \"Gearchiveerde Applicaties\" lists the APPID, soft-delete them when no sheet does, and archive only when the workbook has no archive sheet; an archived or deleted application that returns is unarchived or restored. keep: leave them as they are. mark and remove are reserved and refused with 422 MISSING_RECORDS_UNSUPPORTED" }, "operationId": { "type": "string", @@ -128,7 +129,7 @@ } }, "422": { - "description": "MISSING_RECORDS_UNSUPPORTED, MUNICIPALITY_REQUIRED, MUNICIPALITY_INVALID, MUNICIPALITY_AMBIGUOUS (more than one live Municipality has the typed name; details.matches lists their uuids), NO_SOURCE_SHEET, MISSING_COLUMN or TOO_MANY_ROWS", + "description": "MISSING_RECORDS_UNSUPPORTED (details.accepted lists keep and archive), MUNICIPALITY_REQUIRED, MUNICIPALITY_INVALID, MUNICIPALITY_AMBIGUOUS (more than one live Municipality has the typed name; details.matches lists their uuids), NO_SOURCE_SHEET, MISSING_COLUMN or TOO_MANY_ROWS", "content": { "application/json": { "schema": { @@ -148,7 +149,7 @@ } }, "503": { - "description": "MAPPING_UNAVAILABLE, READER_UNAVAILABLE, NOT_CONFIGURED, or SCHEMA_OUTDATED (a stackiq schema lacks a property the import matches on, or the module schema predates 0.3.8; details.schema names it and details.missing lists the properties, and externalKey.authorization.update for a missing admin-only write rule)", + "description": "MAPPING_UNAVAILABLE, READER_UNAVAILABLE, NOT_CONFIGURED, or SCHEMA_OUTDATED (a stackiq schema lacks a property the import matches on, or the module schema predates 0.3.8; details.schema names it and details.missing lists the properties, and externalKey.authorization.update for a missing admin-only write rule), or ARCHIVE_UNAVAILABLE (missingRecords=archive, but OpenRegister's archive handler or object mapper is missing; nothing is read or written)", "content": { "application/json": { "schema": { @@ -499,6 +500,10 @@ "unchanged", "skipped", "failed", + "archived", + "unarchived", + "deleted", + "restored", "warnings", "unpublished" ], @@ -524,6 +529,22 @@ "failed": { "type": "integer" }, + "archived": { + "type": "integer", + "description": "Applications whose module and usage this import archived because they left the CMDB sheets" + }, + "unarchived": { + "type": "integer", + "description": "Rows whose archived module or usage this import unarchived before updating it" + }, + "deleted": { + "type": "integer", + "description": "Applications whose module and usage this import soft-deleted because no sheet lists them any more" + }, + "restored": { + "type": "integer", + "description": "Rows whose soft-deleted module or usage this import restored from the trash before updating it" + }, "warnings": { "type": "integer" }, @@ -571,7 +592,8 @@ "type": "string" }, "row": { - "type": "integer" + "type": "integer", + "description": "The sheet row, or 0 for an application the reconciliation archived or deleted" }, "appId": { "type": "string" @@ -586,7 +608,11 @@ "updated", "unchanged", "skipped", - "failed" + "failed", + "archived", + "unarchived", + "deleted", + "restored" ] }, "reasons": { diff --git a/postman/stackiq-tests.json b/postman/stackiq-tests.json index 385214b7c..0da0499b8 100644 --- a/postman/stackiq-tests.json +++ b/postman/stackiq-tests.json @@ -23073,9 +23073,10 @@ "listen": "test", "script": { "exec": [ - "pm.test(\"Only missingRecords=keep is accepted\", function () {", + "pm.test(\"Only missingRecords=keep or archive is accepted\", function () {", " pm.response.to.have.status(422);", " pm.expect(pm.response.json().error).to.eql(\"MISSING_RECORDS_UNSUPPORTED\");", + " pm.expect(pm.response.json().details.accepted).to.eql([\"keep\", \"archive\"]);", "});" ], "type": "text/javascript" diff --git a/tests/Unit/Controller/CmdbImportControllerTest.php b/tests/Unit/Controller/CmdbImportControllerTest.php index a27b7cfbf..75af4b05f 100644 --- a/tests/Unit/Controller/CmdbImportControllerTest.php +++ b/tests/Unit/Controller/CmdbImportControllerTest.php @@ -114,7 +114,7 @@ private function controller( if ($service === null) { $service = $this->createMock(CmdbExportImportService::class); $service->method('maxFileBytes')->willReturn(10485760); - $service->method('supportsMissingRecords')->willReturnCallback(fn (string $mode): bool => $mode === 'keep'); + $service->method('missingRecordsModes')->willReturn(['keep', 'archive']); } $logger = ($logger ?? $this->createMock(LoggerInterface::class)); @@ -157,7 +157,7 @@ protected function iniBytes(string $name): ?int { private function service(): CmdbExportImportService|MockObject { $service = $this->createMock(CmdbExportImportService::class); $service->method('maxFileBytes')->willReturn(10485760); - $service->method('supportsMissingRecords')->willReturnCallback(fn (string $mode): bool => $mode === 'keep'); + $service->method('missingRecordsModes')->willReturn(['keep', 'archive']); return $service; }//end service() @@ -371,9 +371,30 @@ public function testAReservedMissingRecordsValueIsRefused(): void { $response = $this->controller(file: $this->file(path: $this->upload()), params: ['missingRecords' => $mode], service: $service)->import(); $this->assertSame(422, $response->getStatus(), $mode); $this->assertSame('MISSING_RECORDS_UNSUPPORTED', $response->getData()['error'], $mode); + $this->assertEquals((object)['accepted' => ['keep', 'archive']], $response->getData()['details'], $mode); + $this->assertSame('Applications missing from the export can only be archived or kept.', $response->getData()['message'], $mode); } }//end testAReservedMissingRecordsValueIsRefused() + /** + * Without missingRecords the import archives; keep and archive are passed through as sent. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012 + */ + public function testMissingRecordsDefaultsToArchive(): void { + foreach ([[[], 'archive'], [['missingRecords' => 'keep'], 'keep'], [['missingRecords' => 'archive'], 'archive']] as [$params, $expected]) { + $service = $this->service(); + $service->expects($this->once())->method('import') + ->with($this->anything(), $this->callback(static fn (array $options): bool => $options['missingRecords'] === $expected)) + ->willReturn(['success' => true]); + + $response = $this->controller(file: $this->file(path: $this->upload()), params: array_merge(['municipalityName' => 'Gemeente Voorbeeldstad'], $params), service: $service)->import(); + $this->assertSame(200, $response->getStatus(), $expected); + } + }//end testMissingRecordsDefaultsToArchive() + /** * Without a municipality the answer is 422 MUNICIPALITY_REQUIRED and nothing is imported. * @@ -407,6 +428,7 @@ public static function serviceErrors(): array { 'schema' => [CmdbImportException::SCHEMA_OUTDATED, 503, ['schema' => 'module', 'missing' => ['externalKey']]], 'running' => [CmdbImportException::IMPORT_IN_PROGRESS, 409, []], 'ambiguous' => [CmdbImportException::MUNICIPALITY_AMBIGUOUS, 422, ['matches' => ['00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000002']]], + 'archive' => [CmdbImportException::ARCHIVE_UNAVAILABLE, 503, []], ]; }//end serviceErrors() @@ -421,6 +443,10 @@ public static function engineMessages(): array { 'schema' => [CmdbImportException::SCHEMA_OUTDATED, 'The stackiq register is out of date; import its configuration again.'], 'running' => [CmdbImportException::IMPORT_IN_PROGRESS, 'Another CMDB import is running; try again when it has finished.'], 'ambiguous' => [CmdbImportException::MUNICIPALITY_AMBIGUOUS, 'Several municipalities have this name; choose one from the list.'], + 'archive' => [ + CmdbImportException::ARCHIVE_UNAVAILABLE, + 'Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.', + ], ]; }//end engineMessages() @@ -533,6 +559,7 @@ public function testAValidUploadReturnsTheReport(): void { 'municipalityName' => 'Gemeente Voorbeeldstad', 'updateExisting' => false, 'publish' => true, + 'missingRecords' => 'keep', 'operationId' => 'cmdb-00000000-0000-0000-0000-000000000000', 'fileName' => 'export.xlsx', ] @@ -567,6 +594,7 @@ public function testTheFieldsReachTheServiceAsSent(): void { 'municipalityName' => 'Gemeente Voorbeeldstad', 'updateExisting' => true, 'publish' => true, + 'missingRecords' => 'archive', 'operationId' => 'not-a-cmdb-id', 'fileName' => 'export.xlsx', ] diff --git a/tests/Unit/Service/Cmdb/CmdbImportReportTest.php b/tests/Unit/Service/Cmdb/CmdbImportReportTest.php index a3d7a576a..ef208a36f 100644 --- a/tests/Unit/Service/Cmdb/CmdbImportReportTest.php +++ b/tests/Unit/Service/Cmdb/CmdbImportReportTest.php @@ -69,4 +69,27 @@ public function testWithinTheLimitEveryRowIsStored(): void { $this->assertSame(array_merge($report->toArray(), ['rowsStored' => 5, 'rowsTruncated' => false]), $stored); }//end testWithinTheLimitEveryRowIsStored() + + /** + * The reconciliation outcomes are counted, and their entries (row 0) are not counted as processed sheet rows. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testTheReconciliationOutcomesAreCounted(): void { + $report = new CmdbImportReport(operationId: 'cmdb-report-02', rowsRead: 2); + $report->addRow(sheet: 'S', row: 2, appId: '1', name: 'Een', outcome: CmdbImportReport::UNARCHIVED); + $report->addRow(sheet: 'S', row: 3, appId: '2', name: 'Twee', outcome: CmdbImportReport::RESTORED); + $report->addRow(sheet: 'Gearchiveerde Applicaties', row: 0, appId: '7', name: 'Zeven', outcome: CmdbImportReport::ARCHIVED); + $report->addRow(sheet: '', row: 0, appId: '8', name: 'Acht', outcome: CmdbImportReport::DELETED); + + $summary = $report->summary(); + $this->assertSame(2, $summary['processed']); + $this->assertSame(2, $report->processed()); + $this->assertSame([1, 1, 1, 1], [$summary['archived'], $summary['unarchived'], $summary['deleted'], $summary['restored']]); + $this->assertFalse($report->isCancelled()); + $report->markCancelled(); + $this->assertTrue($report->isCancelled()); + }//end testTheReconciliationOutcomesAreCounted() }//end class diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index d81417ea7..7ac56ddc6 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -91,6 +91,41 @@ class CmdbExportImportServiceTest extends TestCase { */ private $beforeSave = null; + /** + * The archive and trash markers per schema id and uuid, as OpenRegister keeps them in `@self`. + * + * @var array|null, deleted?: array|null}>> + */ + private array $states = []; + + /** + * Every archive, unarchive, restore and delete: verb, schema id, uuid, reason. + * + * @var array + */ + private array $transitions = []; + + /** + * Called before every archive, unarchive, restore and delete with the verb and the uuid; may throw. + * + * @var callable|null + */ + private $beforeTransition = null; + + /** + * How often the trash listing was read. + * + * @var int + */ + private int $trashReads = 0; + + /** + * Whether the container knows OpenRegister's archive handler and object mapper. + * + * @var bool + */ + private bool $archiveServicesAvailable = true; + /** * Contacts in the fake address book: uid => name, email. * @@ -215,6 +250,11 @@ protected function setUp(): void { $this->store = [self::MODULE => [], self::ORGANIZATION => [], self::USAGE => [], self::CONTACT_PERSON => []]; $this->saves = []; $this->beforeSave = null; + $this->states = []; + $this->transitions = []; + $this->beforeTransition = null; + $this->trashReads = 0; + $this->archiveServicesAvailable = true; $this->contacts = []; $this->addressBooks = []; $this->contactsEnabled = true; @@ -341,20 +381,26 @@ private function noteScope(string $method, bool $rbac, bool $multitenancy): void * * @param string $uuid The uuid. * @param array $data The object data. + * @param array $state The archive and trash markers, serialised under `@self`. + * @param string|null $schema The schema id, as OpenRegister's own reads set it. * * @return ObjectEntityInterface */ - private function entity(string $uuid, array $data): ObjectEntityInterface { - return new class($uuid, $data) implements ObjectEntityInterface { + private function entity(string $uuid, array $data, array $state = [], ?string $schema = null): ObjectEntityInterface { + return new class($uuid, $data, $state, $schema) implements ObjectEntityInterface { /** * Constructor. * * @param string $uuid The uuid. * @param array $data The data. + * @param array $state The markers. + * @param string|null $schema The schema id. */ public function __construct( private string $uuid, private array $data, + private array $state, + private ?string $schema, ) { } @@ -371,7 +417,7 @@ public function getRegister(): ?string { } public function getSchema(): ?string { - return null; + return $this->schema; } public function getOrganisation(): ?string { @@ -383,7 +429,7 @@ public function getOwner(): ?string { } public function jsonSerialize(): array { - return $this->data; + return array_merge($this->data, ['@self' => array_merge(['archived' => null, 'deleted' => null], $this->state)]); } }; }//end entity() @@ -403,6 +449,11 @@ function (array $object, ?array $extend = [], $register = null, $schema = null, ($this->beforeSave)($schema, $object); } + // OpenRegister refuses a data write to an archived object (ObjectStateWriteException). + if ($uuid !== null && empty($this->states[$schema][$uuid]['archived']) === false) { + throw new RuntimeException('write to an archived object'); + } + $create = ($uuid === null); if ($create === true) { $uuid = sprintf('00000000-0000-4000-8000-%012d', count($this->saves) + 1); @@ -435,12 +486,32 @@ function (array $query = [], bool $_rbac = true, bool $_multitenancy = true): ar } } + $state = ($this->states[$schema][$uuid] ?? []); + // The list search loses every soft-deleted row, `_includeDeleted` or not: + // MagicSearchHandler::convertRowToObjectEntity() parses the JSON `_deleted` + // marker as a date, fails, and skips the row (OpenRegister beta ff4dad5c). + if (empty($state['deleted']) === false) { + continue; + } + + if (self::inLens(query: $query, uuid: (string)$uuid, state: $state) === false) { + continue; + } + + // The same converter hydrates no `_archived`: a search result never carries the archive marker. $found[] = $this->entity(uuid: $uuid, data: $data); } return array_slice($found, $offset, $limit); } ); + $service->method('deleteObject')->willReturnCallback( + function (string $uuid, $register = null, $schema = null, bool $_rbac = true, bool $_multitenancy = true): bool { + $this->noteScope(method: 'deleteObject', rbac: $_rbac, multitenancy: $_multitenancy); + $this->changeState(verb: 'delete', uuid: $uuid, reason: null); + return true; + } + ); $service->method('find')->willReturnCallback( function ($id, ?array $_extend = [], bool $files = false, $register = null, $schema = null, bool $_rbac = true, bool $_multitenancy = true): ?ObjectEntityInterface { $this->noteScope(method: 'find', rbac: $_rbac, multitenancy: $_multitenancy); @@ -459,6 +530,222 @@ function ($id, ?array $_extend = [], bool $files = false, $register = null, $sch return $service; }//end objectService() + /** + * Whether a working or archived object is in the list a query asks for, as OpenRegister's lenses decide. + * + * Archived objects are left out unless `_archived` asks for them (`true` + * only the archive, `any` both); `_ids` narrows to those uuids. + * + * @param array $query The search query. + * @param string $uuid The object. + * @param array $state Its markers. + * + * @return bool + */ + private static function inLens(array $query, string $uuid, array $state): bool { + if (isset($query['_ids']) === true && in_array($uuid, (array)$query['_ids'], true) === false) { + return false; + } + + $archived = (empty($state['archived']) === false); + $lens = ($query['_archived'] ?? null); + if ($lens === 'any') { + return true; + } + + if (filter_var($lens, FILTER_VALIDATE_BOOLEAN) === true) { + return $archived; + } + + return $archived === false; + }//end inLens() + + /** + * Archive, unarchive, restore or soft-delete a stored object, as OpenRegister's handlers do. + * + * @param string $verb archive, unarchive, restore or delete. + * @param string $uuid The object. + * @param string|null $reason The archive reason. + * + * @return array{uuid: string, archived: array|null} + */ + public function changeState(string $verb, string $uuid, ?string $reason): array { + if ($this->beforeTransition !== null) { + ($this->beforeTransition)($verb, $uuid); + } + + $schema = 0; + foreach ($this->store as $id => $objects) { + if (isset($objects[$uuid]) === true) { + $schema = $id; + } + } + + $this->assertNotSame(0, $schema, 'no stored object ' . $uuid); + $this->transitions[] = [$verb, $schema, $uuid, $reason]; + $state = ($this->states[$schema][$uuid] ?? []); + if ($verb !== 'restore' && empty($state['deleted']) === false) { + // The archive handler and the delete resolve the object without the trash. + throw new \OCP\AppFramework\Db\DoesNotExistException('object is in the trash'); + } + + match ($verb) { + 'archive' => $state['archived'] = ($state['archived'] ?? null) ?: ['by' => 'admin', 'at' => '2026-10-09T00:00:00+00:00', 'reason' => $reason], + 'unarchive' => $state['archived'] = null, + 'restore' => $state['deleted'] = null, + 'delete' => $state['deleted'] = ['deletedBy' => 'admin', 'deletedAt' => '2026-10-09T00:00:00+00:00'], + }; + $this->states[$schema][$uuid] = $state; + + return ['uuid' => $uuid, 'archived' => ($state['archived'] ?? null)]; + }//end changeState() + + /** + * OpenRegister's archive handler and object mapper over the in-memory store. + * + * @return array{0: object, 1: object} + */ + private function archiveDoubles(): array { + $test = $this; + $handler = new class($test) { + /** + * Constructor. + * + * @param CmdbExportImportServiceTest $test The test, which holds the markers. + */ + public function __construct( + private CmdbExportImportServiceTest $test, + ) { + } + + /** + * Archive an object. + * + * @param string $identifier The uuid. + * @param string|null $reason The reason. + * @param string|null $register Ignored. + * @param string|null $schema Ignored. + * + * @return array + */ + public function archive(string $identifier, ?string $reason = null, ?string $register = null, ?string $schema = null): array { + return $this->test->changeState(verb: 'archive', uuid: $identifier, reason: $reason); + } + + /** + * Unarchive an object. + * + * @param string $identifier The uuid. + * @param string|null $reason The reason. + * @param string|null $register Ignored. + * @param string|null $schema Ignored. + * + * @return array + */ + public function unarchive(string $identifier, ?string $reason = null, ?string $register = null, ?string $schema = null): array { + return $this->test->changeState(verb: 'unarchive', uuid: $identifier, reason: $reason); + } + }; + $mapper = new class($test) { + /** + * Constructor. + * + * @param CmdbExportImportServiceTest $test The test, which holds the markers. + */ + public function __construct( + private CmdbExportImportServiceTest $test, + ) { + } + + /** + * Take an object out of the trash and answer it as the mapper reads it, archive marker included. + * + * @param string $uuid The uuid. + * + * @return object + */ + public function restoreObject(string $uuid): object { + $this->test->changeState(verb: 'restore', uuid: $uuid, reason: null); + return $this->test->storedEntity(uuid: $uuid); + } + + /** + * The trash: every soft-deleted object, in every register and schema. + * + * @param int|null $limit Ignored. + * @param int|null $offset Ignored. + * @param bool $_rbac Must be false. + * @param bool $_multitenancy Must be false. + * + * @return array + */ + public function findDeletedAcrossAllMagicTables(?int $limit = null, ?int $offset = null, bool $_rbac = true, bool $_multitenancy = true): array { + return $this->test->trashListing(rbac: $_rbac, multitenancy: $_multitenancy); + } + }; + + return [$handler, $mapper]; + }//end archiveDoubles() + + /** + * A stored object as OpenRegister's mapper reads it by uuid: data plus both markers. + * + * @param string $uuid The object. + * + * @return ObjectEntityInterface + */ + public function storedEntity(string $uuid): ObjectEntityInterface { + foreach ($this->store as $schema => $objects) { + if (isset($objects[$uuid]) === true) { + return $this->entity(uuid: $uuid, data: $objects[$uuid], state: ($this->states[$schema][$uuid] ?? []), schema: (string)$schema); + } + } + + $this->fail('no stored object ' . $uuid); + }//end storedEntity() + + /** + * The trash as MagicMapper::findDeletedAcrossAllMagicTables() lists it. + * + * Its rows go through MagicMapper::rowToObjectEntity(): uuid, register, + * schema and the data, with every string that parses as JSON decoded + * ("2" becomes 2), and no archive or trash marker. + * + * @param bool $rbac The `_rbac` argument. + * @param bool $multitenancy The `_multitenancy` argument. + * + * @return array + */ + public function trashListing(bool $rbac, bool $multitenancy): array { + $this->noteScope(method: 'findDeletedAcrossAllMagicTables', rbac: $rbac, multitenancy: $multitenancy); + $this->trashReads++; + $listed = []; + foreach ($this->states as $schema => $states) { + foreach ($states as $uuid => $state) { + if (empty($state['deleted']) === true || isset($this->store[$schema][$uuid]) === false) { + continue; + } + + $data = array_map( + static function (mixed $value): mixed { + if (is_string($value) === true) { + $decoded = json_decode($value, true); + if (json_last_error() === JSON_ERROR_NONE) { + return $decoded; + } + } + + return $value; + }, + $this->store[$schema][$uuid] + ); + $listed[] = $this->entity(uuid: (string)$uuid, data: $data, schema: (string)$schema); + } + } + + return $listed; + }//end trashListing() + /** * OpenRegister's schema mapper over the declared properties. * @@ -674,19 +961,24 @@ function (&$value): void { /** * A reader that hands out given rows, for tests that do not need the fixture. * + * The workbook has the archive sheet, listing $archived; null is a workbook without the sheet. + * * @param array}> $rows The rows. + * @param array|null $archived The APPIDs on the archive sheet, or null when there is no such sheet. * * @return CmdbWorkbookReader */ - private function rowsReader(array $rows): CmdbWorkbookReader { - return new class($rows) extends CmdbWorkbookReader { + private function rowsReader(array $rows, ?array $archived = []): CmdbWorkbookReader { + return new class($rows, $archived) extends CmdbWorkbookReader { /** * Constructor. * * @param array> $rows The rows. + * @param array|null $archived The archive sheet's APPIDs, or null. */ public function __construct( private array $rows, + private ?array $archived, ) { } @@ -699,7 +991,8 @@ public function __construct( * @return array */ public function read(string $path, CmdbImportProfile $profile): array { - return ['rows' => $this->rows, 'importWarnings' => [], 'date1904' => false]; + $archive = ['sheet' => 'Gearchiveerde Applicaties', 'present' => ($this->archived !== null), 'keyColumnMissing' => false, 'appIds' => ($this->archived ?? [])]; + return ['rows' => $this->rows, 'importWarnings' => [], 'date1904' => false, 'archive' => $archive]; } }; }//end rowsReader() @@ -716,10 +1009,11 @@ public function read(string $path, CmdbImportProfile $profile): array { private function service(?CmdbWorkbookReader $reader = null, ?string $profileDir = null, array $config = ['register' => '20']): CmdbExportImportService { $objectService = $this->objectService(); $schemaMapper = $this->schemaMapper(); + [$archiveHandler, $objectMapper] = $this->archiveDoubles(); $container = $this->createMock(ContainerInterface::class); $container->method('has')->willReturn(false); $container->method('get')->willReturnCallback( - function (string $id) use ($objectService, $schemaMapper) { + function (string $id) use ($objectService, $schemaMapper, $archiveHandler, $objectMapper) { if ($id === ObjectServiceInterface::class) { return $objectService; } @@ -728,6 +1022,14 @@ function (string $id) use ($objectService, $schemaMapper) { return $schemaMapper; } + if ($this->archiveServicesAvailable === true && $id === CmdbExportImportService::ARCHIVE_HANDLER_CLASS) { + return $archiveHandler; + } + + if ($this->archiveServicesAvailable === true && $id === CmdbExportImportService::OBJECT_MAPPER_CLASS) { + return $objectMapper; + } + throw new RuntimeException('not in this container: ' . $id); } ); @@ -846,7 +1148,7 @@ public function testTheFixtureCreatesModulesUsagesAndSuppliers(): void { $this->assertTrue($report['success']); $this->assertFalse($report['cancelled']); $this->assertSame('cmdb-test-0001', $report['operationId']); - $this->assertSame(['rowsRead' => 2, 'processed' => 2, 'created' => 2, 'updated' => 0, 'unchanged' => 0, 'skipped' => 0, 'failed' => 0, 'warnings' => 0, 'unpublished' => 0], $report['summary']); + $this->assertSame(['rowsRead' => 2, 'processed' => 2, 'created' => 2, 'updated' => 0, 'unchanged' => 0, 'skipped' => 0, 'failed' => 0, 'archived' => 0, 'unarchived' => 0, 'deleted' => 0, 'restored' => 0, 'warnings' => 0, 'unpublished' => 0], $report['summary']); $this->assertSame('Gemeente Voorbeeldstad', $report['municipality']['name']); $this->assertTrue($report['municipality']['created']); $this->assertSame(['No municipality named "Gemeente Voorbeeldstad" was found, so it was created. Check the name if you meant an existing one.'], array_column($report['importWarnings'], 'message'), 'only the warning that the municipality was created'); @@ -1448,29 +1750,591 @@ public function testUpdateExistingFalseSkipsMatches(): void { }//end testUpdateExistingFalseSkipsMatches() /** - * A module missing from a newer export, and its usage, are left as they are. + * The uuid of the stored module with this APPID. + * + * @param string $appId The APPID. + * + * @return string + */ + private function moduleOf(string $appId): string { + foreach ($this->store[self::MODULE] as $uuid => $module) { + if (($module['externalNumber'] ?? null) === $appId) { + return (string)$uuid; + } + } + + $this->fail('no module with APPID ' . $appId); + }//end moduleOf() + + /** + * The uuid of the stored usage of a module by a consumer. + * + * @param string $moduleUuid The module. + * @param string $consumer The consumer. + * + * @return string + */ + private function usageOf(string $moduleUuid, string $consumer = 'muni-1'): string { + foreach ($this->store[self::USAGE] as $uuid => $usage) { + if (($usage['module'] ?? null) === $moduleUuid && ($usage['consumer'] ?? null) === $consumer) { + return (string)$uuid; + } + } + + $this->fail('no usage of ' . $moduleUuid . ' by ' . $consumer); + }//end usageOf() + + /** + * Whether a stored object carries the archive marker, and whether the trash marker. + * + * @param int $schema The schema id. + * @param string $uuid The object. + * + * @return array{0: bool, 1: bool} Archived, deleted. + */ + private function stateOf(int $schema, string $uuid): array { + $state = ($this->states[$schema][$uuid] ?? []); + return [empty($state['archived']) === false, empty($state['deleted']) === false]; + }//end stateOf() + + /** + * Import APPIDs 1, 7 and 8 for muni-1, as a first import would. * * @return void */ - public function testRecordsMissingFromTheExportStay(): void { + private function importOneSevenAndEight(): void { $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); - $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', row: 2), $this->row(appId: '7', row: 3, sheet: 'Onbeh Applicaties CMDB')])) - ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $rows = [ + $this->row(appId: '1', row: 2), + $this->row(appId: '7', row: 3, sheet: 'Onbeh Applicaties CMDB'), + $this->row(appId: '8', row: 4, cells: ['Applicatie Eigenaar (Persoon)' => 'Achternaam, Voornaam']), + ]; + $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1', 'missingRecords' => 'keep']); + $this->transitions = []; + }//end importOneSevenAndEight() + + /** + * A module missing from a newer export, and its usage, are left as they are when records are kept. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012 + */ + public function testRecordsMissingFromTheExportStayWhenKept(): void { + $this->importOneSevenAndEight(); $modules = $this->store[self::MODULE]; $usages = $this->store[self::USAGE]; - $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: ['Applicatie Naam' => 'Nieuw'])])) - ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: ['Applicatie Naam' => 'Nieuw'])])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'missingRecords' => 'keep']); foreach ($modules as $uuid => $module) { - if ($module['externalNumber'] === '7') { + if ($module['externalNumber'] !== '1') { $this->assertSame($module, $this->store[self::MODULE][$uuid]); } } $this->assertSame($usages, $this->store[self::USAGE]); - $this->assertCount(2, $this->store[self::MODULE]); - }//end testRecordsMissingFromTheExportStay() + $this->assertSame([], $this->transitions, 'nothing is archived or deleted'); + $this->assertSame([], $this->states); + $this->assertSame(['updated'], array_column($report['rows'], 'outcome')); + $this->assertSame([], $report['importWarnings'], 'keep does not need the archive sheet'); + }//end testRecordsMissingFromTheExportStayWhenKept() + + /** + * By default an application that moved to the archive sheet is archived and one on no sheet is soft-deleted, + * usage before module, and the owner contact person stays. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testMissingApplicationsAreArchivedOrDeletedByDefault(): void { + $this->importOneSevenAndEight(); + $contactPersons = $this->store[self::CONTACT_PERSON]; + $this->assertNotSame([], $contactPersons, 'APPID 8 has an owner'); + [$seven, $eight] = [$this->moduleOf(appId: '7'), $this->moduleOf(appId: '8')]; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7', '1198'])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'operationId' => 'cmdb-archive-01']); + + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $seven)); + $this->assertSame([true, false], $this->stateOf(schema: self::USAGE, uuid: $this->usageOf(moduleUuid: $seven))); + $this->assertSame([false, true], $this->stateOf(schema: self::MODULE, uuid: $eight)); + $this->assertSame([false, true], $this->stateOf(schema: self::USAGE, uuid: $this->usageOf(moduleUuid: $eight))); + $this->assertSame([false, false], $this->stateOf(schema: self::MODULE, uuid: $this->moduleOf(appId: '1'))); + $this->assertSame( + [ + ['archive', self::USAGE, $this->usageOf(moduleUuid: $seven)], + ['archive', self::MODULE, $seven], + ['delete', self::USAGE, $this->usageOf(moduleUuid: $eight)], + ['delete', self::MODULE, $eight], + ], + array_map(static fn (array $t): array => array_slice($t, 0, 3), $this->transitions), + 'the usage is written before its module' + ); + $this->assertSame('cmdb-import: on sheet "Gearchiveerde Applicaties" (cmdb-archive-01)', $this->transitions[0][3]); + $this->assertSame($contactPersons, $this->store[self::CONTACT_PERSON], 'contact persons are never touched'); + + $reconciled = array_values(array_filter($report['rows'], static fn (array $row): bool => $row['row'] === 0)); + $this->assertSame( + [ + ['sheet' => 'Gearchiveerde Applicaties', 'row' => 0, 'appId' => '7', 'name' => 'Applicatie 7', 'outcome' => 'archived', 'reasons' => [], 'warnings' => [], 'moduleUuid' => $seven, 'usageUuid' => $this->usageOf(moduleUuid: $seven)], + ['sheet' => '', 'row' => 0, 'appId' => '8', 'name' => 'Applicatie 8', 'outcome' => 'deleted', 'reasons' => [], 'warnings' => [], 'moduleUuid' => $eight, 'usageUuid' => $this->usageOf(moduleUuid: $eight)], + ], + $reconciled + ); + $this->assertSame(1, $report['summary']['processed'], 'reconciliation entries are not sheet rows'); + $this->assertSame([1, 1], [$report['summary']['archived'], $report['summary']['deleted']]); + $this->assertStringNotContainsString('Achternaam', (string)json_encode($report)); + $this->assertSame([], $report['importWarnings']); + + $this->transitions = []; + $again = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7'])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $this->assertSame([], $this->transitions, 'an application already in the state the export asks for is left alone'); + $this->assertSame(['unchanged'], array_column($again['rows'], 'outcome')); + }//end testMissingApplicationsAreArchivedOrDeletedByDefault() + + /** + * Without the archive sheet a missing application is archived, never deleted, and the report warns. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testWithoutTheArchiveSheetMissingApplicationsAreArchivedNotDeleted(): void { + $this->importOneSevenAndEight(); + $eight = $this->moduleOf(appId: '8'); + $this->states[self::MODULE][$eight] = ['deleted' => ['deletedBy' => 'admin']]; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: null)) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $this->moduleOf(appId: '7'))); + $this->assertSame([true, false], $this->stateOf(schema: self::USAGE, uuid: $this->usageOf(moduleUuid: $this->moduleOf(appId: '7')))); + $this->assertSame([false, true], $this->stateOf(schema: self::MODULE, uuid: $eight), 'what is in the trash stays there'); + $this->assertSame([true, false], $this->stateOf(schema: self::USAGE, uuid: $this->usageOf(moduleUuid: $eight))); + $this->assertNotContains('delete', array_column($this->transitions, 0)); + $this->assertNotContains('restore', array_column($this->transitions, 0)); + $this->assertSame('cmdb-import: not in the export (' . $report['operationId'] . ')', $this->transitions[0][3]); + $this->assertSame(['unchanged', 'archived', 'archived'], array_column($report['rows'], 'outcome')); + $this->assertSame( + [['sheet' => 'Gearchiveerde Applicaties', 'message' => 'Sheet "Gearchiveerde Applicaties" was not found, so applications missing from the export are archived, not deleted.']], + $report['importWarnings'] + ); + }//end testWithoutTheArchiveSheetMissingApplicationsAreArchivedNotDeleted() + + /** + * A cancelled import reconciles nothing. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testACancelledImportDoesNotReconcile(): void { + $this->importOneSevenAndEight(); + $service = null; + $this->beforeSave = function (int $schema) use (&$service): void { + if ($schema === self::MODULE) { + $service->requestCancel(operationId: 'cmdb-cancel-02'); + } + }; + $service = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: ['Applicatie Naam' => 'Nieuw']), $this->row(appId: '2', row: 3)])); + + $report = $service->import(path: '', options: ['municipalityUuid' => 'muni-1', 'operationId' => 'cmdb-cancel-02']); + + $this->assertTrue($report['cancelled']); + $this->assertSame([], $this->transitions); + $this->assertSame(['updated'], array_column($report['rows'], 'outcome')); + }//end testACancelledImportDoesNotReconcile() + + /** + * A cancel that comes in while the last row is processed stops the import before anything is archived or deleted. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testACancelDuringTheLastRowArchivesNothing(): void { + $this->importOneSevenAndEight(); + $service = null; + $this->beforeSave = function (int $schema) use (&$service): void { + if ($schema === self::MODULE) { + $this->assertTrue($service->requestCancel(operationId: 'cmdb-cancel-03')); + } + }; + $service = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: ['Applicatie Naam' => 'Nieuw'])], archived: ['7', '8'])); + + $report = $service->import(path: '', options: ['municipalityUuid' => 'muni-1', 'operationId' => 'cmdb-cancel-03']); + + $this->assertTrue($report['cancelled']); + $this->assertSame([], $this->transitions); + $this->assertSame(['updated'], array_column($report['rows'], 'outcome')); + $this->assertSame('cancelled', $this->cache['progress_cmdb-cancel-03']['status']); + $this->assertArrayNotHasKey('cancel_cmdb-cancel-03', $this->cache); + }//end testACancelDuringTheLastRowArchivesNothing() + + /** + * A cancel during the reconciliation stops it before the next application and keeps what was archived. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testACancelDuringTheReconciliationStopsBeforeTheNextApplication(): void { + $this->importOneSevenAndEight(); + [$seven, $eight] = [$this->moduleOf(appId: '7'), $this->moduleOf(appId: '8')]; + $service = null; + $this->beforeTransition = function (string $verb, string $uuid) use (&$service, $seven): void { + if ($uuid === $seven) { + $this->assertTrue($service->requestCancel(operationId: 'cmdb-cancel-04')); + } + }; + $service = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7', '8'])); + + $report = $service->import(path: '', options: ['municipalityUuid' => 'muni-1', 'operationId' => 'cmdb-cancel-04']); + + $this->assertTrue($report['cancelled']); + $this->assertSame([['archive', $this->usageOf(moduleUuid: $seven)], ['archive', $seven]], array_map(static fn (array $step): array => [$step[0], $step[2]], $this->transitions)); + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $seven), 'what was archived stays archived'); + $this->assertSame([false, false], $this->stateOf(schema: self::MODULE, uuid: $eight)); + $this->assertSame([false, false], $this->stateOf(schema: self::USAGE, uuid: $this->usageOf(moduleUuid: $eight))); + $this->assertSame(['unchanged', 'archived'], array_column($report['rows'], 'outcome')); + $this->assertSame('cancelled', $this->cache['progress_cmdb-cancel-04']['status']); + }//end testACancelDuringTheReconciliationStopsBeforeTheNextApplication() + + /** + * Only this municipality's usages and the modules its import created are reconciled. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testAnotherMunicipalitysUsagesAreNeverReconciled(): void { + $this->importOneSevenAndEight(); + $this->seedOrganisation(uuid: 'muni-2', name: 'Gemeente Anders', type: 'Municipality'); + $seven = $this->moduleOf(appId: '7'); + // Another municipality uses the same module (added by hand), and muni-1 uses a hand-made module. + $this->store[self::USAGE]['usage-other'] = ['id' => 'usage-other', 'consumer' => 'muni-2', 'module' => $seven]; + $this->store[self::MODULE]['module-manual'] = ['id' => 'module-manual', 'name' => 'Handmatig', 'externalKey' => 'topdesk:muni-2:9']; + $this->store[self::USAGE]['usage-manual'] = ['id' => 'usage-manual', 'consumer' => 'muni-1', 'module' => 'module-manual']; + + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1'), $this->row(appId: '8', row: 3)], archived: ['7'])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame([true, false], $this->stateOf(schema: self::USAGE, uuid: $this->usageOf(moduleUuid: $seven))); + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $seven), 'the module carries this municipality\'s import key'); + $this->assertSame([false, false], $this->stateOf(schema: self::USAGE, uuid: 'usage-other')); + $this->assertSame([false, false], $this->stateOf(schema: self::USAGE, uuid: 'usage-manual')); + $this->assertSame([false, false], $this->stateOf(schema: self::MODULE, uuid: 'module-manual')); + $this->assertSame(2, count($this->transitions)); + }//end testAnotherMunicipalitysUsagesAreNeverReconciled() + + /** + * One application that cannot be archived is a failed row; the others are still reconciled. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testAFailingArchiveIsAFailedRowAndTheOthersAreReconciled(): void { + $this->importOneSevenAndEight(); + $seven = $this->moduleOf(appId: '7'); + $this->beforeTransition = function (string $verb, string $uuid) use ($seven): void { + if ($uuid === $seven) { + throw new RuntimeException('Schema "Applicatie 7" does not declare x-openregister-archive'); + } + }; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7', '8'])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $rows = array_values(array_filter($report['rows'], static fn (array $row): bool => $row['row'] === 0)); + $this->assertSame(['failed', 'archived'], array_column($rows, 'outcome')); + $this->assertSame(['step "archive" failed (RuntimeException)'], $rows[0]['reasons']); + $this->assertSame([true, false], $this->stateOf(schema: self::USAGE, uuid: $this->usageOf(moduleUuid: $seven)), 'the usage went first'); + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $this->moduleOf(appId: '8'))); + $this->assertSame(1, $report['summary']['failed']); + $failed = array_values(array_filter($this->logLines, static fn (string $line): bool => str_starts_with($line, 'CmdbExportImportService: row failed'))); + $this->assertCount(1, $failed); + $this->assertStringNotContainsString('Applicatie 7', $failed[0]); + }//end testAFailingArchiveIsAFailedRowAndTheOthersAreReconciled() + + /** + * Without OpenRegister's archive services an archiving import is refused before the file is read; keep still imports. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testMissingArchiveServicesRefuseAnArchivingImport(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->archiveServicesAvailable = false; + $reader = new class extends CmdbWorkbookReader { + /** + * Constructor. + */ + public function __construct() { + } + + /** + * Never reached. + * + * @param string $path Ignored. + * @param CmdbImportProfile $profile Ignored. + * + * @return array + */ + public function read(string $path, CmdbImportProfile $profile): array { + throw new RuntimeException('the file was read'); + } + }; + + try { + $this->service(reader: $reader)->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $this->fail('ARCHIVE_UNAVAILABLE expected'); + } catch (CmdbImportException $e) { + $this->assertSame('ARCHIVE_UNAVAILABLE', $e->getErrorCode()); + $this->assertSame(503, $e->getHttpStatus()); + } + + $this->assertSame([], $this->saves); + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: ['municipalityUuid' => 'muni-1', 'missingRecords' => 'keep']); + $this->assertSame(['created'], array_column($report['rows'], 'outcome')); + }//end testMissingArchiveServicesRefuseAnArchivingImport() + + /** + * An archived application that returns is unarchived and a soft-deleted one restored, before the write; no duplicates. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + public function testAReturningApplicationIsUnarchivedOrRestoredNotDuplicated(): void { + $this->importOneSevenAndEight(); + [$seven, $eight] = [$this->moduleOf(appId: '7'), $this->moduleOf(appId: '8')]; + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7']))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $modules = count($this->store[self::MODULE]); + $usages = count($this->store[self::USAGE]); + $this->transitions = []; + + $rows = [ + $this->row(appId: '1'), + $this->row(appId: '7', row: 3, sheet: 'Onbeh Applicaties CMDB', cells: ['Applicatie Naam' => 'Terug']), + $this->row(appId: '8', row: 4), + ]; + $report = $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame(['unchanged', 'unarchived', 'restored'], array_column($report['rows'], 'outcome')); + $this->assertSame([1, 1], [$report['summary']['unarchived'], $report['summary']['restored']]); + $this->assertSame([$modules, $usages], [count($this->store[self::MODULE]), count($this->store[self::USAGE])], 'no second module or usage'); + foreach ([$seven, $eight] as $module) { + $this->assertSame([false, false], $this->stateOf(schema: self::MODULE, uuid: $module)); + $this->assertSame([false, false], $this->stateOf(schema: self::USAGE, uuid: $this->usageOf(moduleUuid: $module))); + } + + $this->assertSame('Terug', $this->store[self::MODULE][$seven]['name'], 'the row is updated after the unarchive'); + $this->assertSame(['unarchive', 'unarchive', 'restore', 'restore'], array_column($this->transitions, 0)); + }//end testAReturningApplicationIsUnarchivedOrRestoredNotDuplicated() + + /** + * A usage archived or soft-deleted on its own, next to a working module, is unarchived or restored when its row returns. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + public function testAReturningUsageIsUnarchivedOrRestoredOnItsOwn(): void { + $this->importOneSevenAndEight(); + [$seven, $eight] = [$this->moduleOf(appId: '7'), $this->moduleOf(appId: '8')]; + [$usageSeven, $usageEight] = [$this->usageOf(moduleUuid: $seven), $this->usageOf(moduleUuid: $eight)]; + $this->states[self::USAGE][$usageSeven] = ['archived' => ['by' => 'admin', 'at' => '2026-10-01T00:00:00+00:00', 'reason' => 'by hand']]; + $this->states[self::USAGE][$usageEight] = ['deleted' => ['deletedBy' => 'admin', 'deletedAt' => '2026-10-01T00:00:00+00:00']]; + $usages = array_keys($this->store[self::USAGE]); + + $rows = [$this->row(appId: '7', sheet: 'Onbeh Applicaties CMDB'), $this->row(appId: '8', row: 3)]; + $report = $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1', 'missingRecords' => 'keep']); + + $this->assertSame(['unarchived', 'restored'], array_column($report['rows'], 'outcome')); + $this->assertSame([$usageSeven, $usageEight], array_column($report['rows'], 'usageUuid')); + $this->assertSame($usages, array_keys($this->store[self::USAGE]), 'no second usage'); + $this->assertSame([false, false], $this->stateOf(schema: self::USAGE, uuid: $usageSeven)); + $this->assertSame([false, false], $this->stateOf(schema: self::USAGE, uuid: $usageEight)); + $this->assertSame([['unarchive', $usageSeven], ['restore', $usageEight]], array_map(static fn (array $t): array => [$t[0], $t[2]], $this->transitions)); + }//end testAReturningUsageIsUnarchivedOrRestoredOnItsOwn() + + /** + * An application soft-deleted while archived is restored and unarchived when it returns, and updated from the restored object. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + public function testAnApplicationDeletedWhileArchivedIsRestoredAndUnarchived(): void { + $this->importOneSevenAndEight(); + $seven = $this->moduleOf(appId: '7'); + $usage = $this->usageOf(moduleUuid: $seven); + $marker = [ + 'archived' => ['by' => 'admin', 'at' => '2026-10-01T00:00:00+00:00', 'reason' => 'cmdb-import: on sheet'], + 'deleted' => ['deletedBy' => 'admin', 'deletedAt' => '2026-10-02T00:00:00+00:00'], + ]; + $this->states[self::MODULE][$seven] = $marker; + $this->states[self::USAGE][$usage] = $marker; + $counts = [count($this->store[self::MODULE]), count($this->store[self::USAGE])]; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '7', sheet: 'Onbeh Applicaties CMDB', cells: ['Applicatie Naam' => 'Terug'])])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'missingRecords' => 'keep']); + + $this->assertSame(['restored'], array_column($report['rows'], 'outcome')); + $this->assertSame( + [['restore', $seven], ['unarchive', $seven], ['restore', $usage], ['unarchive', $usage]], + array_map(static fn (array $t): array => [$t[0], $t[2]], $this->transitions) + ); + $this->assertSame([false, false], $this->stateOf(schema: self::MODULE, uuid: $seven)); + $this->assertSame([false, false], $this->stateOf(schema: self::USAGE, uuid: $usage)); + $this->assertSame($counts, [count($this->store[self::MODULE]), count($this->store[self::USAGE])], 'no second module or usage'); + $this->assertSame('Terug', $this->store[self::MODULE][$seven]['name']); + $this->assertSame('7', $this->store[self::MODULE][$seven]['externalNumber'], 'the update starts from the restored object, not from the trash listing'); + }//end testAnApplicationDeletedWhileArchivedIsRestoredAndUnarchived() + + /** + * A soft-deleted application the archive sheet lists is restored and archived, and the trash is read once. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testADeletedApplicationOnTheArchiveSheetIsRestoredAndArchived(): void { + $this->importOneSevenAndEight(); + $seven = $this->moduleOf(appId: '7'); + $usage = $this->usageOf(moduleUuid: $seven); + $this->states[self::MODULE][$seven] = ['deleted' => ['deletedBy' => 'admin', 'deletedAt' => '2026-10-02T00:00:00+00:00']]; + $this->states[self::USAGE][$usage] = ['deleted' => ['deletedBy' => 'admin', 'deletedAt' => '2026-10-02T00:00:00+00:00']]; + $this->trashReads = 0; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1'), $this->row(appId: '8', row: 3)], archived: ['7'])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame( + [['restore', $usage], ['archive', $usage], ['restore', $seven], ['archive', $seven]], + array_map(static fn (array $t): array => [$t[0], $t[2]], $this->transitions) + ); + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $seven)); + $this->assertSame([true, false], $this->stateOf(schema: self::USAGE, uuid: $usage)); + $reconciled = array_values(array_filter($report['rows'], static fn (array $row): bool => $row['row'] === 0)); + $this->assertSame([['7', 'archived']], array_map(static fn (array $row): array => [$row['appId'], $row['outcome']], $reconciled)); + $this->assertSame(1, $this->trashReads, 'the trash is read once per import, not per row'); + }//end testADeletedApplicationOnTheArchiveSheetIsRestoredAndArchived() + + /** + * With updateExisting false a returning application is skipped as `exists` and stays archived. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + public function testUpdateExistingFalseLeavesAnArchivedMatchArchived(): void { + $this->importOneSevenAndEight(); + $seven = $this->moduleOf(appId: '7'); + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1'), $this->row(appId: '8', row: 3)], archived: ['7']))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $modules = $this->store[self::MODULE]; + $this->transitions = []; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '7', sheet: 'Onbeh Applicaties CMDB')])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'updateExisting' => false, 'missingRecords' => 'keep']); + + $this->assertSame(['skipped'], array_column($report['rows'], 'outcome')); + $this->assertSame(['exists'], $report['rows'][0]['reasons']); + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $seven)); + $this->assertSame([], $this->transitions); + $this->assertSame($modules, $this->store[self::MODULE]); + }//end testUpdateExistingFalseLeavesAnArchivedMatchArchived() + + /** + * A module and usage deleted by hand next to the working ones with the same import key stay in the trash. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + public function testAWorkingMatchWinsOverADeletedOneWithTheSameKey(): void { + $this->importOneSevenAndEight(); + $seven = $this->moduleOf(appId: '7'); + $usage = $this->usageOf(moduleUuid: $seven); + // Stored ahead of the working ones, so a first-hit match would find them first. + $this->store[self::MODULE] = ['module-deleted' => array_merge($this->store[self::MODULE][$seven], ['id' => 'module-deleted'])] + $this->store[self::MODULE]; + $this->store[self::USAGE] = ['usage-deleted' => array_merge($this->store[self::USAGE][$usage], ['id' => 'usage-deleted'])] + $this->store[self::USAGE]; + $this->states[self::MODULE]['module-deleted'] = ['deleted' => ['deletedBy' => 'admin', 'deletedAt' => '2026-10-01T00:00:00+00:00']]; + $this->states[self::USAGE]['usage-deleted'] = ['deleted' => ['deletedBy' => 'admin', 'deletedAt' => '2026-10-01T00:00:00+00:00']]; + $counts = [count($this->store[self::MODULE]), count($this->store[self::USAGE])]; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '7', sheet: 'Onbeh Applicaties CMDB', cells: ['Applicatie Naam' => 'Terug'])])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'missingRecords' => 'keep']); + + $this->assertSame(['updated'], array_column($report['rows'], 'outcome')); + $this->assertSame($seven, $report['rows'][0]['moduleUuid']); + $this->assertSame('Terug', $this->store[self::MODULE][$seven]['name']); + $this->assertSame([], $this->transitions, 'nothing is restored'); + $this->assertSame([false, true], $this->stateOf(schema: self::MODULE, uuid: 'module-deleted')); + $this->assertSame([false, true], $this->stateOf(schema: self::USAGE, uuid: 'usage-deleted')); + $this->assertSame($counts, [count($this->store[self::MODULE]), count($this->store[self::USAGE])], 'no module or usage is created'); + }//end testAWorkingMatchWinsOverADeletedOneWithTheSameKey() + + /** + * The fixture round trip of the e2e test: the later export archives 1234 and deletes 2, the original brings both back. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + public function testTheArchivedApplicationsFixtureArchivesDeletesAndBringsBack(): void { + $path = $this->fixture(); + $options = ['municipalityName' => 'Gemeente Voorbeeldstad']; + $this->service()->import(path: $path, options: $options); + $counts = [count($this->store[self::MODULE]), count($this->store[self::USAGE]), count($this->store[self::CONTACT_PERSON])]; + [$archived, $deleted] = [$this->moduleOf(appId: '1234'), $this->moduleOf(appId: '2')]; + + $report = $this->service()->import(path: CmdbTestSupport::fixtures() . '/topdesk-archived-applications.xlsx', options: $options); + + $this->assertSame(['archived', 'deleted'], array_column($report['rows'], 'outcome')); + $this->assertSame(['1234', '2'], array_column($report['rows'], 'appId')); + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $archived)); + $this->assertSame([false, true], $this->stateOf(schema: self::MODULE, uuid: $deleted)); + $this->assertSame([], $report['importWarnings']); + + $back = $this->service()->import(path: $path, options: $options); + + $this->assertSame(['unarchived', 'restored'], array_column($back['rows'], 'outcome')); + $this->assertSame($counts, [count($this->store[self::MODULE]), count($this->store[self::USAGE]), count($this->store[self::CONTACT_PERSON])]); + $this->assertSame([false, false], $this->stateOf(schema: self::MODULE, uuid: $archived)); + $this->assertSame([false, false], $this->stateOf(schema: self::MODULE, uuid: $deleted)); + }//end testTheArchivedApplicationsFixtureArchivesDeletesAndBringsBack() + + /** + * The reconciliation reports its own progress phase and advances per application. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testTheReconciliationHasItsOwnProgressPhase(): void { + $this->importOneSevenAndEight(); + $seen = []; + $this->beforeTransition = function () use (&$seen): void { + $progress = $this->tracker->getProgress(operationId: 'cmdb-reconcile-1'); + $seen[] = [$progress['phase'], $progress['processed_items'], $progress['total_items']]; + }; + + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7', '8'])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'operationId' => 'cmdb-reconcile-1']); + + // One row, then three usages in scope: APPID 1 (on a CMDB sheet), 7 and 8. + $this->assertSame([['reconciling', 2, 4], ['reconciling', 2, 4], ['reconciling', 3, 4], ['reconciling', 3, 4]], $seen); + $this->assertSame('completed', $this->cache['progress_cmdb-reconcile-1']['status']); + $this->assertSame(4, $this->cache['progress_cmdb-reconcile-1']['processed_items']); + }//end testTheReconciliationHasItsOwnProgressPhase() /** * An unknown Applicatie Status drops only that field and warns with column and value. @@ -1806,7 +2670,7 @@ public function testAnImportLeavesAnAuditRecord(): void { options: ['municipalityUuid' => 'muni-1', 'updateExisting' => false, 'publish' => false, 'operationId' => 'cmdb-audit-01', 'fileName' => 'C:\\Users\\beheer\\export.xlsx'] ); - $audit = '"operationId":"cmdb-audit-01","uid":"admin","fileName":"export.xlsx","municipality":"muni-1","municipalityCreated":false,"updateExisting":false,"publish":false'; + $audit = '"operationId":"cmdb-audit-01","uid":"admin","fileName":"export.xlsx","municipality":"muni-1","municipalityCreated":false,"updateExisting":false,"publish":false,"missingRecords":"archive"'; $started = array_values(array_filter($this->logLines, static fn (string $line): bool => str_starts_with($line, 'CmdbExportImportService: import started'))); $finished = array_values(array_filter($this->logLines, static fn (string $line): bool => str_starts_with($line, 'CmdbExportImportService: import finished'))); $this->assertCount(1, $started); @@ -1879,7 +2743,7 @@ public function testOneBadRowDoesNotStopTheOthers(): void { $this->assertSame(['created', 'failed', 'created'], array_column($report['rows'], 'outcome')); $this->assertStringStartsWith('step "module" failed', $report['rows'][1]['reasons'][0]); - $this->assertSame(['rowsRead' => 3, 'processed' => 3, 'created' => 2, 'updated' => 0, 'unchanged' => 0, 'skipped' => 0, 'failed' => 1, 'warnings' => 0, 'unpublished' => 0], $report['summary']); + $this->assertSame(['rowsRead' => 3, 'processed' => 3, 'created' => 2, 'updated' => 0, 'unchanged' => 0, 'skipped' => 0, 'failed' => 1, 'archived' => 0, 'unarchived' => 0, 'deleted' => 0, 'restored' => 0, 'warnings' => 0, 'unpublished' => 0], $report['summary']); $this->assertCount(2, $this->store[self::MODULE]); }//end testOneBadRowDoesNotStopTheOthers() @@ -2151,7 +3015,7 @@ public function testCancelNeedsARunningImport(): void { }//end testCancelNeedsARunningImport() /** - * A cancel that arrives after the last row's check leaves no flag that a later run with the same id would inherit. + * A cancel that arrives after the last row's check, with records kept, leaves no flag that a later run with the same id would inherit. * * @return void */ @@ -2165,7 +3029,7 @@ public function testACancelAfterTheLastCheckLeavesNoFlag(): void { }; $service = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', row: 2)])); - $report = $service->import(path: '', options: ['municipalityUuid' => 'muni-1', 'operationId' => 'cmdb-late-cancel-1']); + $report = $service->import(path: '', options: ['municipalityUuid' => 'muni-1', 'operationId' => 'cmdb-late-cancel-1', 'missingRecords' => 'keep']); $this->assertFalse($report['cancelled'], 'the only row was already done'); $this->assertSame('completed', $this->cache['progress_cmdb-late-cancel-1']['status']); From 7184a4a22df83553bdd36dacacb7f3d1eee1ffe7 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 12:09:15 +0200 Subject: [PATCH 03/41] feat(cmdb-import): offer the missing-records choice and an Archived quick filter The CMDB import section lets the admin archive or delete applications missing from the export, or keep them, and shows the archived, unarchived, deleted and restored outcomes. The Applications and Applications in use pages get an Archived quick filter. Vitest and the Playwright spec cover it; the feature docs and the changelog describe it. Third of three slices of #1281, on top of the reconciliation slice. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 ++ docs/features/README.md | 2 +- docs/features/cmdb-import.md | 90 +++++++++++-- src/icons.js | 2 + src/manifest.d/usages.json | 3 +- src/manifest.json | 7 + src/utils/cmdbImport.js | 72 +++++++++- src/utils/cmdbImport.spec.js | 58 +++++++- .../settings/sections/CmdbImport.spec.js | 58 ++++++++ src/views/settings/sections/CmdbImport.vue | 77 ++++++++++- tests/e2e/spec-coverage/cmdb-import.spec.ts | 125 +++++++++++++++++- tests/vitest/cmdbArchiveQuickFilter.spec.js | 62 +++++++++ 12 files changed, 540 insertions(+), 25 deletions(-) create mode 100644 tests/vitest/cmdbArchiveQuickFilter.spec.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 87505881a..69d2c85b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ sheets becomes or updates a module, its vendor, a usage for the municipality and a contact person for its owner; a repeat import matches on APPID, so it updates instead of duplicating. See `docs/features/cmdb-import.md`. +- CMDB import, repeat imports: applications that left the CMDB sheets are + archived when the sheet "Gearchiveerde Applicaties" lists them and moved to + OpenRegister's trash when no sheet does; one that returns is unarchived or + restored, never duplicated. The new option `missingRecords` (`archive`, the + default, or `keep`) is a choice in the CMDB import section, and the report + counts archived, unarchived, deleted and restored applications. The module + (0.3.9) and usage (1.5.7) schemas opt into OpenRegister's archive state, and + the Applications and Applications in use pages get an "Archived" quick + filter. (cmdb-import-archive-reconciliation) ### Changed - EOL feed: the register and schema slugs the feature reads from are corrected. diff --git a/docs/features/README.md b/docs/features/README.md index 7fa964f0a..c57ff8486 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -158,7 +158,7 @@ The ArchiMate integration maps GEMMA Softwarecatalogus objects to ArchiMate appl ## CMDB Import -Import a TOPdesk CMDB export (`.xlsx`) for one municipality from the **CMDB import** section of the admin settings. Every application row of the two CMDB sheets ("Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB") becomes or updates a module, its vendor as a Supplier organisation, a usage that links it to the municipality and records whether maintenance is arranged, and a contact person for its owner (identity in Nextcloud Contacts, never public). A repeat import matches on APPID per municipality, so it updates instead of duplicating, and leaves applications missing from the newer export as they are. The column mapping is declarative JSON executed by OpenRegister's mapping engine. +Import a TOPdesk CMDB export (`.xlsx`) for one municipality from the **CMDB import** section of the admin settings. Every application row of the two CMDB sheets ("Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB") becomes or updates a module, its vendor as a Supplier organisation, a usage that links it to the municipality and records whether maintenance is arranged, and a contact person for its owner (identity in Nextcloud Contacts, never public). A repeat import matches on APPID per municipality, so it updates instead of duplicating. By default it archives the applications that moved to the sheet "Gearchiveerde Applicaties" and moves those on no sheet any more to OpenRegister's trash, and brings either back when it returns; the admin can choose to keep them instead. The Applications and Applications in use pages list the archive under the quick filter "Archived". The column mapping is declarative JSON executed by OpenRegister's mapping engine. See [CMDB import](cmdb-import.md) for the steps, the expected file structure, the error codes and how to adjust the mapping. diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index b9d505abc..d4077f959 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -17,9 +17,11 @@ updates: in Nextcloud Contacts. Owners are never readable by the public. All of it is stored as OpenRegister objects in the stackiq register. Import a -newer export later and the same applications are updated, not duplicated. +newer export later and the same applications are updated, not duplicated; +applications that left the export are archived or moved to the trash (see +[Repeat imports](#repeat-imports)). -Specification: [`openspec/specs/cmdb-export-import/spec.md`](https://github.com/ConductionNL/stackiq/blob/development/openspec/specs/cmdb-export-import/spec.md) (change archived in [`openspec/changes/archive/2026-10-05-cmdb-export-import/`](https://github.com/ConductionNL/stackiq/tree/development/openspec/changes/archive/2026-10-05-cmdb-export-import)). +Specification: [`openspec/specs/cmdb-export-import/spec.md`](https://github.com/ConductionNL/stackiq/blob/development/openspec/specs/cmdb-export-import/spec.md) (change archived in [`openspec/changes/archive/2026-10-05-cmdb-export-import/`](https://github.com/ConductionNL/stackiq/tree/development/openspec/changes/archive/2026-10-05-cmdb-export-import); archiving and deleting on a repeat import in [`openspec/changes/cmdb-import-archive-reconciliation/`](https://github.com/ConductionNL/stackiq/tree/development/openspec/changes/cmdb-import-archive-reconciliation)). ## Who can import @@ -77,9 +79,15 @@ page in stackiq. and the summary counts them as *created unpublished*. Applications that were imported before keep their publication as it is, whichever you choose. -6. Press **Import**. A progress bar shows how many rows have been processed. - **Cancel import** stops the import before the next row; rows that were - already processed stay imported. The section says whether the server +6. **Applications missing from the export.** **Archive or delete them** is + the default: an application of this municipality that is on neither CMDB + sheet any more is archived or moved to the trash after the last row (see + [Repeat imports](#repeat-imports)). Choose **Keep them as they are** to + leave such applications untouched. +7. Press **Import**. A progress bar shows how many rows have been processed. + **Cancel import** stops the import before the next row, or, while the + missing applications are compared, before the next application; what + was already processed stays imported. The section says whether the server accepted the cancel; one pressed before the server has started on the rows cannot take effect yet, and the section says so. @@ -92,11 +100,15 @@ When the import finishes, the section shows: - the **summary**: rows read, created, updated, unchanged, skipped, failed and warnings, and, when publishing was off, how many applications were - created unpublished; + created unpublished; after a repeat import also how many applications + were archived, unarchived, deleted (moved to the trash) or restored, when + there are any; - **warnings for the whole file**, for example an optional column that is missing; - the **rows** table: sheet, row number, APPID, application, outcome, - and the reasons and warnings for that row. Filter it with **Show rows with + and the reasons and warnings for that row. An application archived or + deleted because it left the export has no row number; its sheet is + `Gearchiveerde Applicaties` when that sheet listed it. Filter it with **Show rows with outcome**, and sort it by clicking a column header. It shows 100 rows at a time; **Show 100 more rows** adds the next ones. The application name links to the module in stackiq. @@ -106,8 +118,10 @@ the same organisation. ## The file -The import reads the two CMDB sheets of the export and ignores all others, -including the `Invoer` sheets they are derived from: +The import reads the two CMDB sheets of the export, and of the sheet +`Gearchiveerde Applicaties` only the `APPID` column (see +[Repeat imports](#repeat-imports)). It ignores all other sheets, including +the `Invoer` sheets they are derived from: | Sheet | What it holds | Recorded on the usage | |---|---|---| @@ -213,9 +227,9 @@ colliding. - **Known APPID, nothing changed**: nothing is saved; the row is reported as *unchanged*. Importing the same export twice creates nothing the second time. -- **APPID missing from a newer export**: the application, its usage and - its contact persons are left as they are. They are not changed, depublished - or deleted. +- **APPID missing from a newer export**: see + [Applications that left the export](#applications-that-left-the-export) + below. - Each application keeps exactly one usage for the municipality. - **Known APPID, but the application belongs to another organisation**: an application found by its match key is only updated when the municipality @@ -243,6 +257,55 @@ OpenRegister refuses to save it, that row is reported as *failed* with the step that failed, and the other rows are imported. Importing again completes the failed row. +### Applications that left the export + +With **Archive or delete them** (the default; `missingRecords=archive` in +the API), the import compares the municipality's applications with the +export after the last row. It looks only at the municipality's own usages +and the applications its import created, never at an application only +another organisation uses or one added by hand. For an application whose +APPID is on neither CMDB sheet: + +| The APPID is | The application and its usage | +|---|---| +| on the sheet `Gearchiveerde Applicaties` | are **archived**: OpenRegister's archive state, with the reason `cmdb-import: on sheet "Gearchiveerde Applicaties"`. Reported as *archived*. | +| on no sheet at all | are **deleted**: moved to OpenRegister's trash, where they stay restorable for 30 days. Reported as *deleted*. | +| on no sheet, and the workbook has no sheet `Gearchiveerde Applicaties` (or it has no `APPID` column) | are **archived**, never deleted, and the result warns that the sheet was not found. A workbook saved without that sheet therefore never moves anything to the trash. | + +An archived or deleted application disappears from the stackiq lists, from +OpenCatalogi and from Portaliq's **Software we use**. In stackiq, the quick +filter **Archived** on the **Applications** and **Applications in use** pages +lists the archive. An OpenCatalogi caller finds archived applications by +adding `_archived=true` to its request. Contact persons are never archived +or deleted: an owner is shared with the municipality's other applications. +An application that is already in the state the export asks for is left +alone and not reported again. + +When an archived or deleted application **returns** in a later export, it is +taken out of the archive (*unarchived*) or out of the trash (*restored*), +and then updated as usual. No second application is created. With **Update +existing records** off, it is skipped as `exists` and stays archived or in +the trash. When an application was deleted by hand and a later import +created it again, the working one is updated and the deleted one stays in +the trash. + +The comparison runs only when the import went through all rows: an import +cancelled or failed before the comparison archives or deletes nothing. +**Cancel import** during the comparison stops it before the next +application; what was archived or deleted until then stays so, and the +result lists it. The progress bar shows the +step as *Checking the applications missing from the export*. Each +application is handled on its own; one that cannot be archived is reported +as *failed* with the step (`archive`, `restore` or `delete`) and the others +continue. Archiving needs module schema 0.3.9 and usage schema 1.5.7 or +later, which declare that their objects can be archived; on an older register configuration each application fails +with the step `archive`, and **Force Update** at the top of the stackiq admin +settings fixes it. + +With **Keep them as they are** (`missingRecords=keep`), the application, its +usage and its contact persons are left as they are: not changed, archived, +depublished or deleted. + ## Owners The owner becomes a **contact person of the municipality**, never a @@ -293,7 +356,8 @@ and the section shows the reason and the error code. | `IMPORT_IN_PROGRESS` | Another CMDB import is running. Only one import runs at a time. | Wait until it has finished and try again. | | `FIELD_INVALID` | A form field of the request has a value the import does not accept, for example an `updateExisting` or `publish` that is neither `true` nor `false`. The message names the field. | Not reachable from the section; reported for API callers. | | `UPLOAD_FAILED` | The file reached the server but could not be stored there. | Try again; the Nextcloud log has the details. | -| `MISSING_RECORDS_UNSUPPORTED` | The request asked to mark or remove records missing from the export. Only keeping them is supported. | Not reachable from the section; reported for API callers. | +| `MISSING_RECORDS_UNSUPPORTED` | The request asked for something other than `keep` or `archive` for records missing from the export, for example `mark` or `remove`. The details list the accepted values. | Not reachable from the section; reported for API callers. | +| `ARCHIVE_UNAVAILABLE` | **Archive or delete them** was chosen, but the OpenRegister on this server cannot archive or restore objects (it is missing or too old). Nothing was read or written. | Choose **Keep them as they are** and import again, or update OpenRegister. | | `MAPPING_UNAVAILABLE` | OpenRegister's mapping engine is missing, or one of the mapping files is invalid. | Update OpenRegister. If you changed a mapping file, check it against the Nextcloud log. | | `READER_UNAVAILABLE` | The Excel reader that ships with OpenRegister cannot be loaded. | Make sure OpenRegister is installed and enabled. | | `NOT_CONFIGURED` | The stackiq register or its schemas cannot be found. | Run **Auto Configure** at the top of the stackiq admin settings. | diff --git a/src/icons.js b/src/icons.js index 865c514c0..34ffbfdfb 100644 --- a/src/icons.js +++ b/src/icons.js @@ -17,6 +17,7 @@ import AccountGroup from 'vue-material-design-icons/AccountGroup.vue' import AccountMultiple from 'vue-material-design-icons/AccountMultiple.vue' import AlertCircle from 'vue-material-design-icons/AlertCircle.vue' import ApplicationOutline from 'vue-material-design-icons/ApplicationOutline.vue' +import ArchiveOutline from 'vue-material-design-icons/ArchiveOutline.vue' import ArrowRight from 'vue-material-design-icons/ArrowRight.vue' import BookOpenVariant from 'vue-material-design-icons/BookOpenVariant.vue' import BookOpenVariantOutline from 'vue-material-design-icons/BookOpenVariantOutline.vue' @@ -79,6 +80,7 @@ export default { AccountMultiple, AlertCircle, ApplicationOutline, + ArchiveOutline, ArrowRight, BookOpenVariant, BookOpenVariantOutline, diff --git a/src/manifest.d/usages.json b/src/manifest.d/usages.json index 66990ad7d..eec9d1242 100644 --- a/src/manifest.d/usages.json +++ b/src/manifest.d/usages.json @@ -35,7 +35,8 @@ { "label": "Planned", "filter": { "status": "Planned" } }, { "label": "In production", "filter": { "status": "In production" }, "icon": "CheckCircle" }, { "label": "To be phased out", "filter": { "status": "To be phased out" }, "icon": "AlertCircle" }, - { "label": "Phased out", "filter": { "status": "Phased out" } } + { "label": "Phased out", "filter": { "status": "Phased out" } }, + { "label": "Archived", "filter": { "_archived": "true" }, "icon": "ArchiveOutline" } ], "sidebar": { "enabled": true, "showMetadata": true }, "documentationUrl": "https://stackiq.conduction.nl" diff --git a/src/manifest.json b/src/manifest.json index 22b321684..7df63875f 100644 --- a/src/manifest.json +++ b/src/manifest.json @@ -662,6 +662,13 @@ "dpiaStatus": ["not required", "required"] }, "icon": "AlertCircle" + }, + { + "label": "Archived", + "filter": { + "_archived": "true" + }, + "icon": "ArchiveOutline" } ] } diff --git a/src/utils/cmdbImport.js b/src/utils/cmdbImport.js index ab7fa7080..64d7df36d 100644 --- a/src/utils/cmdbImport.js +++ b/src/utils/cmdbImport.js @@ -42,8 +42,27 @@ export const PROFILE_DEFAULTS = Object.freeze({ */ export const AS_TEXT = Object.freeze({ escape: false, sanitize: false }) -/** Every row outcome the report can carry, in display order. */ -export const OUTCOMES = ['created', 'updated', 'unchanged', 'skipped', 'failed'] +/** + * Every row outcome the report can carry, in display order: the outcomes of a + * sheet row, then those of an application that left the export or came back. + */ +export const OUTCOMES = [ + 'created', + 'updated', + 'unchanged', + 'skipped', + 'failed', + 'unarchived', + 'restored', + 'archived', + 'deleted', +] + +/** + * What the import may do with applications missing from the export (the + * profile's `missingRecords`), the default first. + */ +export const MISSING_RECORDS_MODES = Object.freeze(['archive', 'keep']) /** * The words for one row outcome. @@ -51,6 +70,7 @@ export const OUTCOMES = ['created', 'updated', 'unchanged', 'skipped', 'failed'] * @param {string} outcome The outcome key from the report * @return {string} The label * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-each-row-shall-be-processed-in-isolation-and-reported-with-its-outcome-req-cmdb-011 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014 */ export function outcomeLabel(outcome) { switch (outcome) { @@ -64,6 +84,14 @@ export function outcomeLabel(outcome) { return t('stackiq', 'Skipped') case 'failed': return t('stackiq', 'Failed') + case 'unarchived': + return t('stackiq', 'Unarchived') + case 'restored': + return t('stackiq', 'Restored') + case 'archived': + return t('stackiq', 'Archived') + case 'deleted': + return t('stackiq', 'Deleted') default: return String(outcome ?? '') } @@ -141,16 +169,19 @@ export function checkFile(file) { * @param {{uuid: string|null, name: string}} options.municipality The chosen municipality: an existing one has a uuid, a new one only a name * @param {boolean} options.updateExisting Whether matched rows are updated * @param {boolean} [options.publish] Whether the modules the import creates are published; true when left out + * @param {string} [options.missingRecords] What happens to applications missing from the export: 'archive' (when left out) or 'keep' * @param {string} options.operationId The progress operation id * @return {FormData} The body * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-every-import-shall-have-exactly-one-consuming-municipality-chosen-by-the-admin-req-cmdb-004 * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-a-newly-created-module-shall-get-a-publicationdate-when-the-admin-publishes-and-an-existing-one-shall-keep-its-own-req-cmdb-007 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012 */ export function buildImportForm({ file, municipality, updateExisting, publish = true, + missingRecords = MISSING_RECORDS_MODES[0], operationId, }) { const form = new FormData() @@ -162,7 +193,7 @@ export function buildImportForm({ } form.append('updateExisting', updateExisting ? 'true' : 'false') form.append('publish', publish ? 'true' : 'false') - form.append('missingRecords', 'keep') + form.append('missingRecords', missingRecords) form.append('operationId', operationId) return form } @@ -465,6 +496,7 @@ const KNOWN_ERRORS = new Set([ 'SCHEMA_OUTDATED', 'IMPORT_IN_PROGRESS', 'MUNICIPALITY_AMBIGUOUS', + 'ARCHIVE_UNAVAILABLE', 'OPERATION_NOT_FOUND', 'IMPORT_INTERRUPTED', 'NOT_SIGNED_IN', @@ -626,7 +658,7 @@ export function errorText(error) { return { title: t( 'stackiq', - 'Records missing from the export can only be kept.', + 'Applications missing from the export can only be archived or kept.', ), hint: '', } @@ -807,6 +839,17 @@ export function errorText(error) { 'Choose the municipality from the list instead of typing its name. Nothing was imported.', ), } + case 'ARCHIVE_UNAVAILABLE': + return { + title: t( + 'stackiq', + 'Applications missing from the export cannot be archived.', + ), + hint: t( + 'stackiq', + 'OpenRegister is missing or too old to archive objects. Choose "Keep them as they are" and import again, or update OpenRegister. Nothing was imported.', + ), + } case 'OPERATION_NOT_FOUND': return { title: t('stackiq', 'This import is no longer running.'), @@ -886,9 +929,13 @@ export function cancelFailureText(error) { * server has set them, because the tracker's own percentage is weighted by * the phases of the ArchiMate import. * + * In the phase `reconciling` the counts also hold the applications the + * import checks against the export after its rows. + * * @param {object|null} progress The snapshot from `GET /api/progress/{operationId}` * @return {{percentage: number, detail: string}|null} The view, or null before any progress * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-a-running-import-shall-report-its-progress-and-shall-stop-when-cancelled-req-cmdb-013 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 */ export function cmdbProgressView(progress) { if (!progress) { @@ -900,6 +947,17 @@ export function cmdbProgressView(progress) { total > 0 ? Math.min(100, Math.round((processed / total) * 100)) : Number(progress.percentage) || 0 + if (total > 0 && progress.phase === 'reconciling') { + return { + percentage, + detail: t( + 'stackiq', + 'Checking the applications missing from the export: {processed} of {total}', + { processed, total }, + AS_TEXT, + ), + } + } return { percentage, detail: @@ -960,9 +1018,13 @@ export function sortReportRows(rows, key, order) { /** * The rows of the report as the table shows them. * + * An application the import archived or deleted after its rows has row 0 + * in the report and no row number in the table. + * * @param {Array} rows The report's `rows` * @return {Array} The table rows * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-each-row-shall-be-processed-in-isolation-and-reported-with-its-outcome-req-cmdb-011 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014 */ export function reportRows(rows) { if (!Array.isArray(rows)) { @@ -971,7 +1033,7 @@ export function reportRows(rows) { return rows.map((row, index) => ({ key: `${row.sheet ?? ''}:${row.row ?? index}:${index}`, sheet: String(row.sheet ?? ''), - row: row.row ?? '', + row: Number(row.row) > 0 ? row.row : '', appId: String(row.appId ?? ''), name: String(row.name ?? ''), outcome: String(row.outcome ?? ''), diff --git a/src/utils/cmdbImport.spec.js b/src/utils/cmdbImport.spec.js index 8f566d6cb..9377f3b59 100644 --- a/src/utils/cmdbImport.spec.js +++ b/src/utils/cmdbImport.spec.js @@ -22,6 +22,7 @@ import { interruptedImportError, isKnownError, makeCmdbOperationId, + MISSING_RECORDS_MODES, municipalityOptions, normaliseError, outcomeLabel, @@ -58,6 +59,7 @@ const SERVER_CODES = [ 'SCHEMA_OUTDATED', 'IMPORT_IN_PROGRESS', 'MUNICIPALITY_AMBIGUOUS', + 'ARCHIVE_UNAVAILABLE', 'OPERATION_NOT_FOUND', 'IMPORT_INTERRUPTED', 'NOT_SIGNED_IN', @@ -198,10 +200,26 @@ describe('buildImportForm', () => { expect(form.get('municipalityUuid')).toBe('uuid-1') expect(form.has('municipalityName')).toBe(false) expect(form.get('updateExisting')).toBe('true') - expect(form.get('missingRecords')).toBe('keep') + expect(form.get('missingRecords')).toBe('archive') expect(form.get('operationId')).toBe('cmdb-abcdefgh') }) + it('sends the choice for applications missing from the export, archive by default', () => { + const options = { + file, + municipality: { uuid: 'uuid-1', name: 'Tilburg' }, + updateExisting: true, + operationId: 'cmdb-abcdefgh', + } + expect(MISSING_RECORDS_MODES).toEqual(['archive', 'keep']) + expect(buildImportForm(options).get('missingRecords')).toBe('archive') + expect( + buildImportForm({ ...options, missingRecords: 'keep' }).get( + 'missingRecords', + ), + ).toBe('keep') + }) + it('sends a typed new municipality by name, and updateExisting as an explicit false', () => { const form = buildImportForm({ file, @@ -642,6 +660,19 @@ describe('cmdbProgressView', () => { ).toEqual({ percentage: 25, detail: '1 of 4 rows processed' }) }) + it('says when the import checks the applications missing from the export', () => { + expect( + cmdbProgressView({ + phase: 'reconciling', + processed_items: 3, + total_items: 4, + }), + ).toEqual({ + percentage: 75, + detail: 'Checking the applications missing from the export: 3 of 4', + }) + }) + it('shows nothing before any progress', () => { expect(cmdbProgressView(null)).toBeNull() }) @@ -683,6 +714,31 @@ describe('the report table', () => { expect(reportRows(null)).toEqual([]) }) + it('shows no row number for an application archived or deleted after the rows', () => { + const [row] = reportRows([ + { + sheet: 'Gearchiveerde Applicaties', + row: 0, + appId: '7', + name: 'Zeven', + outcome: 'archived', + reasons: [], + warnings: [], + moduleUuid: 'm-7', + }, + ]) + expect(row.row).toBe('') + expect(outcomeLabel('archived')).toBe('Archived') + expect(OUTCOMES).toEqual( + expect.arrayContaining([ + 'archived', + 'unarchived', + 'deleted', + 'restored', + ]), + ) + }) + it('sorts numbers by value and keeps the report order for ties', () => { const rows = [ { key: 'a', row: 10, sheet: 'B' }, diff --git a/src/views/settings/sections/CmdbImport.spec.js b/src/views/settings/sections/CmdbImport.spec.js index 931f7edbc..7a16b9858 100644 --- a/src/views/settings/sections/CmdbImport.spec.js +++ b/src/views/settings/sections/CmdbImport.spec.js @@ -147,3 +147,61 @@ describe('CmdbImport municipality chooser', () => { expect(wrapper.vm.report).toBe(null) }) }) + +/** + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014 + */ +describe('CmdbImport applications missing from the export', () => { + afterEach(() => { + jest.clearAllMocks() + }) + + it('archives by default and posts the choice the admin made', async () => { + const wrapper = await mountSection() + axios.post.mockResolvedValue({ + data: { success: true, summary: {}, rows: [] }, + }) + wrapper.vm.municipality = wrapper.vm.municipalityOptions[0] + wrapper.vm.selectedFile = new File(['x'], 'export.xlsx') + + expect(wrapper.vm.missingRecords).toBe('archive') + await wrapper.vm.startImport() + wrapper.vm.missingRecords = 'keep' + await wrapper.vm.startImport() + + expect( + axios.post.mock.calls.map(([, form]) => form.get('missingRecords')), + ).toEqual(['archive', 'keep']) + }) + + it('shows the archived, unarchived, deleted and restored counts only when there are any', async () => { + const wrapper = await mountSection() + const tileKeys = () => wrapper.vm.summaryTiles.map((tile) => tile.key) + + wrapper.vm.report = { summary: { rowsRead: 2, created: 2 }, rows: [] } + expect(tileKeys()).not.toContain('archived') + + wrapper.vm.report = { + summary: { + rowsRead: 1, + archived: 1, + deleted: 1, + unarchived: 0, + restored: 2, + }, + rows: [], + } + expect(tileKeys()).toEqual( + expect.arrayContaining(['archived', 'deleted', 'restored']), + ) + expect(tileKeys()).not.toContain('unarchived') + expect(wrapper.vm.outcomeFilterOptions.map((option) => option.id)).toEqual( + expect.arrayContaining([ + 'archived', + 'unarchived', + 'deleted', + 'restored', + ]), + ) + }) +}) diff --git a/src/views/settings/sections/CmdbImport.vue b/src/views/settings/sections/CmdbImport.vue index f8bbc1e4b..00af68892 100644 --- a/src/views/settings/sections/CmdbImport.vue +++ b/src/views/settings/sections/CmdbImport.vue @@ -143,6 +143,40 @@ }}

+
+ + {{ t('stackiq', 'Applications missing from the export') }} + + + {{ t('stackiq', 'Archive or delete them') }} + + + {{ t('stackiq', 'Keep them as they are') }} + +

+ {{ + t( + 'stackiq', + 'An application of this municipality that is on neither CMDB sheet any more is archived when the sheet "Gearchiveerde Applicaties" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.', + ) + }} +

+
@@ -304,7 +338,7 @@ @@ -718,8 +727,7 @@ export default { const parts = [ [ 'archived', - (count) => - t('stackiq', '{count} archived', { count }, AS_TEXT), + (count) => t('stackiq', '{count} archived', { count }, AS_TEXT), ], [ 'unarchived', @@ -728,13 +736,11 @@ export default { ], [ 'deleted', - (count) => - t('stackiq', '{count} deleted', { count }, AS_TEXT), + (count) => t('stackiq', '{count} deleted', { count }, AS_TEXT), ], [ 'restored', - (count) => - t('stackiq', '{count} restored', { count }, AS_TEXT), + (count) => t('stackiq', '{count} restored', { count }, AS_TEXT), ], ] .map(([key, text]) => [Number(summary[key]) || 0, text]) From dca821194f2f61b1c769bcfc8e782bfe803b9550 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Tue, 13 Oct 2026 18:58:44 +0200 Subject: [PATCH 32/41] fix(cmdb-import): never soft-delete when a CMDB-sheet APPID is unreadable; usages before modules Closes the review findings: - "Unreadable CMDB-sheet APPID still soft-deletes": a CMDB-sheet row whose APPID is a formula without a cached value is skipped as empty, so its application looked absent. The run now notes such rows per sheet and, with any, archives instead of soft-deleting and warns naming the CMDB sheet and the rows (reusing the archive-sheet warning). New test testAnUncachedCmdbRowArchivesInsteadOfDeleting. - "Extras written after the module; no failing-extra test": itemsOf() now orders every usage (own and extra) before every module, as the docblock promises. New test testAFailingExtraUsageFailsTheApplicationBeforeItsModule. - archiveServices() swallowed the container exception: it is now logged (exception class only) and chained as previous. - trash() read for missingRecords=keep: kept on purpose (matching is mode independent, a hand-deleted application must be restored, not duplicated); the reason is now in its docblock. Class sweep: the rule is updated in the service docblocks, openapi.json (missingRecords description), and the change's spec.md (requirement and new scenario), design.md (D3 text, D5 table, risks) and proposal.md. docs/features/cmdb-import.md does not describe archive reconciliation yet, so it holds no copy of the rule. Co-Authored-By: Claude Opus 5.5 --- lib/Service/CmdbExportImportService.php | 81 ++++++++++++++++--- openapi.json | 2 +- .../design.md | 6 +- .../proposal.md | 4 +- .../specs/cmdb-export-import/spec.md | 10 ++- .../Service/CmdbExportImportServiceTest.php | 76 +++++++++++++++++ 6 files changed, 162 insertions(+), 17 deletions(-) diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 54c9e4b86..4c236d9ac 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -297,6 +297,13 @@ class CmdbExportImportService { */ private array $winningSheets = []; + /** + * The CMDB-sheet rows of this run whose key column the reader could not see (a formula without a cached value). + * + * @var array> Sheet name => row numbers. + */ + private array $unreadableKeyRows = []; + /** * OpenRegister's archive handler and object mapper, once resolved. * @@ -549,6 +556,7 @@ private function runImport(string $path, array $options, string $startedAt): arr $report->setMunicipality(uuid: $municipality['uuid'], name: $municipality['name'], created: $municipality['created']); $report->addImportWarnings(warnings: $this->translateImportWarnings(warnings: $workbook['importWarnings'])); $archive = $this->archiveSheet(workbook: $workbook); + $this->unreadableKeyRows = $this->unreadableKeyRows(rows: $rows); $this->addRunWarnings(report: $report, municipality: $municipality, archive: $archive, missingRecords: $missingRecords); $this->progressTracker->startOperation( @@ -625,7 +633,10 @@ private function runImport(string $path, array $options, string $startedAt): arr }//end runImport() /** - * Add the run's own warnings: an archive sheet that is missing, has no key column or has unreadable keys, and a created municipality. + * Add the run's own warnings. + * + * An archive sheet that is missing, has no key column or has unreadable keys, a CMDB sheet with + * unreadable keys, and a created municipality. * * @param CmdbImportReport $report The report. * @param array{uuid: string, name: string, created: bool} $municipality The consumer, as resolveMunicipality() found it. @@ -642,6 +653,13 @@ private function addRunWarnings(CmdbImportReport $report, array $municipality, a $report->addImportWarnings(warnings: [['sheet' => $archive['sheet'], 'message' => $this->archiveSheetWarning(archive: $archive)]]); } + if ($missingRecords === 'archive') { + foreach ($this->unreadableKeyRows as $sheet => $rowNumbers) { + $cmdbSheet = ['sheet' => $sheet, 'keyColumnMissing' => false, 'uncachedRows' => $rowNumbers]; + $report->addImportWarnings(warnings: [['sheet' => $sheet, 'message' => $this->archiveSheetWarning(archive: $cmdbSheet)]]); + } + } + if ($municipality['created'] === true) { $report->addImportWarnings( warnings: [ @@ -657,6 +675,32 @@ private function addRunWarnings(CmdbImportReport $report, array $municipality, a } }//end addRunWarnings() + /** + * The CMDB-sheet rows whose key column is a formula without a cached value, per sheet. + * + * Such a row reads as empty and is skipped, so its application may be + * missing from `winningSheets` although it is on a CMDB sheet. A run with + * one is not authoritative for a deletion, as with an archive sheet that + * has an unreadable key. + * + * @param array}> $rows The reader rows. + * + * @return array> Sheet name => row numbers. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function unreadableKeyRows(array $rows): array { + $key = $this->profile->keyColumn(); + $found = []; + foreach ($rows as $row) { + if (in_array($key, ($row['uncached'] ?? []), true) === true) { + $found[$row['sheet']][] = $row['row']; + } + } + + return $found; + }//end unreadableKeyRows() + /** * Process one row in its own error boundary and add its outcome. * @@ -1968,7 +2012,9 @@ private function archiveSheet(array $workbook): array { /** * The report warning of an archiving import whose archive sheet is absent or has no key column, or has unreadable keys. * - * @param array{sheet: string, keyColumnMissing: bool, uncachedRows: array} $archive The archive sheet. + * Also the warning of a CMDB sheet with unreadable keys: it is handed the sheet's name and rows in the archive sheet's shape. + * + * @param array{sheet: string, keyColumnMissing: bool, uncachedRows: array} $archive The archive sheet, or a CMDB sheet. * * @return string * @@ -2021,10 +2067,13 @@ private function archiveServices(): array { return $this->archiveServices; } + $previous = null; try { $handler = $this->container->get(static::ARCHIVE_HANDLER_CLASS); $mapper = $this->container->get(static::OBJECT_MAPPER_CLASS); } catch (Throwable $e) { + $this->logger->warning('CmdbExportImportService: OpenRegister archive services could not be resolved', ['exception' => get_class($e)]); + $previous = $e; $handler = null; $mapper = null; } @@ -2035,7 +2084,8 @@ private function archiveServices(): array { ) { throw new CmdbImportException( errorCode: CmdbImportException::ARCHIVE_UNAVAILABLE, - message: 'OpenRegister ArchiveHandler or MagicMapper is not available' + message: 'OpenRegister ArchiveHandler or MagicMapper is not available', + previous: $previous ); } @@ -2149,6 +2199,11 @@ private function restore(string $uuid): mixed { * nothing in it is matched, and the log says so. A listing that throws is * not cached, so the next row reads it again. * + * It is read for `keep` as well: matching is not tied to the mode (REQ-CMDB-016), + * and an application a person soft-deleted by hand must be restored, not + * created a second time. The listing cannot be narrowed, so the cost is one + * scan per run, paid on the first row no working or archived object matches. + * * @return array{module: array, usage: array} Schema key => uuid => entity. * * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 @@ -2545,8 +2600,9 @@ private function reconcileApplication(array $application, array $archive, string if ($onArchiveSheet === true) { $entry['sheet'] = $archive['sheet']; $reason = self::REASON_PREFIX . 'on sheet "' . $archive['sheet'] . '" (' . $operationId . ')'; - } elseif ($archive['present'] === true && $archive['uncachedRows'] === []) { - // A sheet with an unreadable key may list this application, so it is not authoritative for a deletion. + } elseif ($archive['present'] === true && $archive['uncachedRows'] === [] && $this->unreadableKeyRows === []) { + // A sheet with an unreadable key (the archive sheet or a CMDB sheet) may list this application, so the + // workbook is not authoritative for a deletion. $target = CmdbImportReport::DELETED; } @@ -2573,7 +2629,10 @@ private function reconcileApplication(array $application, array $archive, string }//end reconcileApplication() /** - * What one application brings to the state the export asks for: its usage and module, then the other usages and modules of its key. + * What one application brings to the state the export asks for: its usage and the other usages of its key, then its module and the other modules. + * + * Usages come before modules, so a failure part way leaves the + * municipality's view right and a module at worst visible. * * Only the application's own objects are restored from the trash, and only * when `restoreDeleted` says so; the extras never are. @@ -2588,16 +2647,18 @@ private function reconcileApplication(array $application, array $archive, string private static function itemsOf(array $application, bool $restoreDeleted): array { $items = []; foreach (['usage', 'module'] as $schemaKey) { + // Every usage is written before any module, whoever holds it. $items[] = [ 'schemaKey' => $schemaKey, 'uuid' => (string)$application[$schemaKey]->getUuid(), 'state' => $application['states'][$schemaKey], 'restore' => $restoreDeleted, ]; - } - - foreach (($application['extras'] ?? []) as $extra) { - $items[] = $extra + ['restore' => false]; + foreach (($application['extras'] ?? []) as $extra) { + if ($extra['schemaKey'] === $schemaKey) { + $items[] = $extra + ['restore' => false]; + } + } } return $items; diff --git a/openapi.json b/openapi.json index 4d4bb058b..f456fc209 100644 --- a/openapi.json +++ b/openapi.json @@ -66,7 +66,7 @@ "archive" ], "default": "archive", - "description": "What happens to the municipality's applications whose APPID is no longer on either CMDB sheet. archive: after the last row, archive module and usage when the sheet \"Gearchiveerde Applicaties\" lists the APPID, soft-delete them when no sheet does, and archive only when the workbook has no archive sheet, the sheet has no APPID column, or an APPID in it has no cached value; an archived or deleted application that returns is unarchived or restored. keep: leave them as they are. mark and remove are reserved and refused with 422 MISSING_RECORDS_UNSUPPORTED" + "description": "What happens to the municipality's applications whose APPID is no longer on either CMDB sheet. archive: after the last row, archive module and usage when the sheet \"Gearchiveerde Applicaties\" lists the APPID, soft-delete them when no sheet does, and archive only when the workbook has no archive sheet, the sheet has no APPID column, an APPID in it has no cached value, or an APPID on a CMDB sheet has no cached value (that application may be the one the reader could not see); an archived or deleted application that returns is unarchived or restored. keep: leave them as they are. mark and remove are reserved and refused with 422 MISSING_RECORDS_UNSUPPORTED" }, "operationId": { "type": "string", diff --git a/openspec/changes/cmdb-import-archive-reconciliation/design.md b/openspec/changes/cmdb-import-archive-reconciliation/design.md index a1bbbe62e..1fc7058ee 100644 --- a/openspec/changes/cmdb-import-archive-reconciliation/design.md +++ b/openspec/changes/cmdb-import-archive-reconciliation/design.md @@ -71,7 +71,7 @@ With `updateExisting=false`, a matched row is skipped as `exists` before anythin The profile names the sheet (`archiveSheet.name`: "Gearchiveerde Applicaties"). The reader lists the workbook's sheets, and when the archive sheet is present it is loaded with the CMDB sheets under the same bounds: it counts as a read sheet for `CmdbPartReferences` (an oversized archive sheet is refused, not blanked) and for the referenced-string budget, its row span is bounded like a source sheet, and `TOO_MANY_ROWS` names it. Only its APPID column is resolved and read; the allowlist does not grow. The reader returns `archive: {present, keyColumnMissing, appIds, uncachedRows}` next to `rows`. - Sheet absent, or present without an APPID column: `present` false (with `keyColumnMissing`), and the service adds a report warning. The reconciliation then archives every missing application and soft-deletes nothing: a workbook saved without the sheet must not empty the trash-bound set by accident. -- A row whose APPID is a formula without a cached value reads as empty, as on the CMDB sheets (REQ-CMDB-002), so the sheet would silently list one application fewer. The reader therefore lists such rows (their numbers) in `uncachedRows`, and the service treats the sheet like an absent one for the decision between archive and delete: it adds a warning naming the sheet and the rows, archives every missing application and soft-deletes nothing. A tool that saves formulas without cached values must not turn archived applications into trash. +- A row whose APPID is a formula without a cached value reads as empty, as on the CMDB sheets (REQ-CMDB-002), so the sheet would silently list one application fewer. The reader therefore lists such rows (their numbers) in `uncachedRows`, and the service treats the sheet like an absent one for the decision between archive and delete: it adds a warning naming the sheet and the rows, archives every missing application and soft-deletes nothing. A tool that saves formulas without cached values must not turn archived applications into trash. The same holds for a CMDB sheet: a row whose APPID has no cached value is skipped, so that application may be on a CMDB sheet and still look absent. The service notes such rows (`unreadableKeyRows`, per sheet) and, with any of them, archives instead of soft-deleting and warns naming the CMDB sheet and the rows, as for the archive sheet. - The archive sheet's other columns are not read; `Applicatie Status` there is always `Verwijderd` in the municipality's export and adds nothing. ### D4. Matching widens to archived and deleted objects, and revives what it matches @@ -94,7 +94,7 @@ Per application, with `processed` = every APPID match key on the CMDB sheets (wh | on the archive sheet | deleted | restore, then archive | `archived` | | nowhere, sheet present | working or archived | soft-delete usage, then module | `deleted` | | nowhere, sheet present | deleted | nothing | — | -| nowhere, sheet absent or incomplete (no key column, or `uncachedRows` not empty) | working | archive usage, then module | `archived` | +| nowhere, sheet absent or incomplete (no key column, `uncachedRows` not empty, or a CMDB-sheet row with an unreadable APPID) | working | archive usage, then module | `archived` | | nowhere, sheet absent or incomplete | archived or deleted | nothing | — | The table decides on one entry per key. When a municipality has several usages and modules under the same import key (D4 says earlier imports created exactly such duplicates, a module deleted by hand next to the one a later import created), the scope keeps the entry the import would match: the best module state wins (working, then archived, then deleted), the best usage state breaks a tie, the first listed breaks the rest. The other usages of the key follow the application, so a stray usage stays visible nowhere: a working or archived usage of a stale duplicate, a second usage of the same module, and a working or archived duplicate module are archived or soft-deleted as the key asks, once, in the same error boundary and the same report row (usages first, then modules). What is in the trash is left as it is: a stale trashed duplicate is not restored into the archive when the key is on the archive sheet, and is not touched when it is on no sheet. @@ -196,7 +196,7 @@ No seed objects change. The fragment's three seed modules stay as they are; the ## Risks / Trade-offs - [The default archives on the first re-import after the upgrade] → The section shows the choice with help text; the report lists every archived and deleted application; nothing is hard-deleted. -- [A workbook saved without the archive sheet, or whose archive sheet has no APPID column or an APPID without a cached value] → archive only, never delete, plus a warning (D3). +- [A workbook saved without the archive sheet, or whose archive sheet has no APPID column or an APPID without a cached value, or a CMDB sheet with an APPID without a cached value] → archive only, never delete, plus a warning (D3). - [A failure between the usage and the module] → the usage is written first, so the municipality's view is right and the module is at worst still visible (D5). - [Reviving before writing adds calls per returning row] → only for rows whose match is archived or deleted; a row whose module is working costs the same searches as before, a new row one more search (the archive lens). - [The trash listing scans every magic table of the instance] → read once per import, not per row (D9). diff --git a/openspec/changes/cmdb-import-archive-reconciliation/proposal.md b/openspec/changes/cmdb-import-archive-reconciliation/proposal.md index dd4017f30..d9685e710 100644 --- a/openspec/changes/cmdb-import-archive-reconciliation/proposal.md +++ b/openspec/changes/cmdb-import-archive-reconciliation/proposal.md @@ -34,7 +34,7 @@ None. ### In Scope - The sheet "Gearchiveerde Applicaties" (columns Applicatiecomponent, APPID, Applicatie Code, Applicatie Naam, Roepnaam, Applicatie Status, Bron, Datum Interface, Datum wijziging) is read for its APPID column only, when it is present. -- After every row of a completed import: the municipality's usages, with their modules, are compared with the APPIDs on the CMDB sheets and on the archive sheet. Missing and on the archive sheet: module and usage archived. Missing everywhere: module and usage soft-deleted. Without the archive sheet, or when it has no APPID column or an APPID the reader cannot see (a formula without a cached value): archived only, with a warning in the report. +- After every row of a completed import: the municipality's usages, with their modules, are compared with the APPIDs on the CMDB sheets and on the archive sheet. Missing and on the archive sheet: module and usage archived. Missing everywhere: module and usage soft-deleted. Without the archive sheet, or when it has no APPID column or an APPID the reader cannot see (a formula without a cached value, on the archive sheet or on a CMDB sheet): archived only, with a warning in the report. - A returning APPID: an archived module or usage is unarchived, a soft-deleted one is restored from the trash, and the row is then updated as before. - `missingRecords`: `keep` leaves everything as it is (the behaviour so far), `archive` reconciles and is the default; `remove` stays reserved and is refused. - The settings section offers the choice with help text; the report shows the new outcomes in its summary and rows. @@ -78,7 +78,7 @@ None. `ArchiveHandler` and `MagicMapper` come from OpenRegister, which stackiq a **Severity:** Medium — **Mitigation:** only the municipality's own usages and the modules they point at are in scope, never a supplier's or another municipality's module; nothing is hard-deleted; the choice is visible in the section with help text, and the report lists every archived and deleted application. A cancelled or failed import never reconciles. ### Risk 2: A workbook without a usable archive sheet soft-deletes everything that moved there -**Severity:** Medium — **Mitigation:** without the sheet, without its APPID column, or with an APPID the reader cannot see (a formula without a cached value), missing applications are archived, never soft-deleted, and the report warns, naming the sheet (and the rows). +**Severity:** Medium — **Mitigation:** without the sheet, without its APPID column, or with an APPID the reader cannot see on it or on a CMDB sheet (a formula without a cached value), missing applications are archived, never soft-deleted, and the report warns, naming the sheet (and the rows). ### Risk 3: The archive services are OpenRegister internals **Severity:** Low — **Mitigation:** resolved through the container behind a guard; when missing, an import with `missingRecords=archive` is refused with 503 `ARCHIVE_UNAVAILABLE` before the file is read, and `keep` still works. diff --git a/openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md index d03c04ee7..7f781f895 100644 --- a/openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md @@ -16,7 +16,7 @@ OpenRegister: `OCA\OpenRegister\Contract\ObjectServiceInterface` (`searchObjects ### Requirement: An application missing from the CMDB sheets SHALL be archived when the archive sheet lists it and soft-deleted when no sheet does (REQ-CMDB-015) -With `missingRecords` `archive`, after every row of an import that was not cancelled and did not fail, the import SHALL reconcile, under the register lock it already holds, every `usage` whose `consumer` is the imported municipality together with the `module` it points at, including archived and soft-deleted ones, and only those whose module carries this municipality's import key (`topdesk::`). It SHALL read the APPID column of the sheet "Gearchiveerde Applicaties" when the workbook has it, under the same size and row bounds as the CMDB sheets and without reading any other column of it. For an application whose APPID is on neither CMDB sheet: when the archive sheet lists the APPID, the import SHALL archive the usage and then the module through OpenRegister's archive state with a reason that names the import, restoring a soft-deleted one from the trash first; when no sheet lists it, the import SHALL soft-delete the usage and then the module through OpenRegister's trash. When the workbook has no archive sheet, or the sheet has no APPID column, or a row of the sheet holds an APPID formula without a cached value (read as empty, as REQ-CMDB-002 has it for the CMDB sheets, and listed by the reader in `archive.uncachedRows`), the import SHALL archive and SHALL NOT soft-delete, and SHALL add a warning to the report; for an uncached row the warning SHALL name the sheet and the row numbers. When a municipality has more than one module or usage under the same import key (a module deleted by hand next to the one a later import created), the import SHALL reconcile as the application the one it would match, a working one before an archived one and an archived one before one in the trash, and SHALL bring every other usage of that key, and every other module of it that is not in the trash, to the same state, so no stray usage stays visible; it SHALL leave what is in the trash as it is, so a stale duplicate is never restored into the archive. An application that is already in the state the export asks for SHALL be left alone and not reported. The import SHALL NOT touch contact persons, organisations, modules that carry another municipality's import key or no import key, or usages of another organisation. Each application SHALL be reconciled in its own error boundary: a failure SHALL be a `failed` report row naming the step and the exception class, and the other applications SHALL still be reconciled. The reconciliation SHALL report its progress as the phase `reconciling` of the import's operation. A cancel SHALL also be honoured after the last row and before each application of the reconciliation: the import SHALL stop there, SHALL keep what it archived or deleted so far, and SHALL report the run as cancelled. The report SHALL count `archived` and `deleted` in its summary and SHALL list each as a row with the APPID, the application name and both uuids, never person data. When OpenRegister's archive handler or object mapper is not available, an import with `missingRecords` `archive` SHALL be refused with 503 `ARCHIVE_UNAVAILABLE` before the file is read. +With `missingRecords` `archive`, after every row of an import that was not cancelled and did not fail, the import SHALL reconcile, under the register lock it already holds, every `usage` whose `consumer` is the imported municipality together with the `module` it points at, including archived and soft-deleted ones, and only those whose module carries this municipality's import key (`topdesk::`). It SHALL read the APPID column of the sheet "Gearchiveerde Applicaties" when the workbook has it, under the same size and row bounds as the CMDB sheets and without reading any other column of it. For an application whose APPID is on neither CMDB sheet: when the archive sheet lists the APPID, the import SHALL archive the usage and then the module through OpenRegister's archive state with a reason that names the import, restoring a soft-deleted one from the trash first; when no sheet lists it, the import SHALL soft-delete the usage and then the module through OpenRegister's trash. When the workbook has no archive sheet, or the sheet has no APPID column, or a row of the sheet holds an APPID formula without a cached value (read as empty, as REQ-CMDB-002 has it for the CMDB sheets, and listed by the reader in `archive.uncachedRows`), the import SHALL archive and SHALL NOT soft-delete, and SHALL add a warning to the report; for an uncached row the warning SHALL name the sheet and the row numbers. The same holds when a row of a CMDB sheet holds an APPID formula without a cached value (the row is skipped as empty, so its application may look absent): the import SHALL archive and SHALL NOT soft-delete, and the warning SHALL name that CMDB sheet and the rows. When a municipality has more than one module or usage under the same import key (a module deleted by hand next to the one a later import created), the import SHALL reconcile as the application the one it would match, a working one before an archived one and an archived one before one in the trash, and SHALL bring every other usage of that key, and every other module of it that is not in the trash, to the same state, so no stray usage stays visible; it SHALL leave what is in the trash as it is, so a stale duplicate is never restored into the archive. An application that is already in the state the export asks for SHALL be left alone and not reported. The import SHALL NOT touch contact persons, organisations, modules that carry another municipality's import key or no import key, or usages of another organisation. Each application SHALL be reconciled in its own error boundary: a failure SHALL be a `failed` report row naming the step and the exception class, and the other applications SHALL still be reconciled. The reconciliation SHALL report its progress as the phase `reconciling` of the import's operation. A cancel SHALL also be honoured after the last row and before each application of the reconciliation: the import SHALL stop there, SHALL keep what it archived or deleted so far, and SHALL report the run as cancelled. The report SHALL count `archived` and `deleted` in its summary and SHALL list each as a row with the APPID, the application name and both uuids, never person data. When OpenRegister's archive handler or object mapper is not available, an import with `missingRecords` `archive` SHALL be refused with 503 `ARCHIVE_UNAVAILABLE` before the file is read. #### Scenario: An application that moved to the archive sheet is archived @e2e tests/e2e/spec-coverage/cmdb-import.spec.ts @@ -53,6 +53,14 @@ With `missingRecords` `archive`, after every row of an import that was not cance - **THEN** the module and the usage of `7` SHALL be archived and SHALL NOT be deleted - **AND** the report SHALL carry a warning naming the sheet "Gearchiveerde Applicaties" and the row +#### Scenario: A CMDB row whose APPID has no cached value never turns an archive into a deletion +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testAnUncachedCmdbRowArchivesInsteadOfDeleting asserts the archive state, no delete, and the warning naming the CMDB sheet and the row. + +- **GIVEN** the module with APPID `8` and its usage were imported, `8` is on no readable row, and a row of a CMDB sheet holds its APPID in a formula that carries no cached value +- **WHEN** the workbook is imported with `archive` and a readable archive sheet that does not list `8` +- **THEN** the module and the usage of `8` SHALL be archived and SHALL NOT be deleted +- **AND** the report SHALL carry a warning naming the CMDB sheet and the row + #### Scenario: Keeping missing records, a cancelled import and another municipality leave everything as it is @e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testRecordsMissingFromTheExportStayWhenKept, testACancelledImportDoesNotReconcile and testAnotherMunicipalitysUsagesAreNeverReconciled assert no archive, no delete and no reconciliation row. diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 6fb99267f..c476b5b1f 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -2000,6 +2000,82 @@ public function testAnUncachedArchiveRowArchivesInsteadOfDeleting(): void { $this->assertSame([], $kept['importWarnings']); }//end testAnUncachedArchiveRowArchivesInsteadOfDeleting() + /** + * A CMDB-sheet row whose APPID the reader could not see is skipped as empty, so its application looks absent: + * the import archives what it would have deleted, and warns naming the CMDB sheet and the row. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testAnUncachedCmdbRowArchivesInsteadOfDeleting(): void { + $this->importOneSevenAndEight(); + [$seven, $eight] = [$this->moduleOf(appId: '7'), $this->moduleOf(appId: '8')]; + + // APPID 7 is on the readable archive sheet; APPID 8 is on no readable sheet, and row 5 of a CMDB sheet has an APPID formula without a cached value. + $report = $this->service( + reader: $this->rowsReader( + rows: [$this->row(appId: '1'), $this->row(appId: '', cells: ['APPID' => null], row: 5, uncached: ['APPID'])], + archived: ['7'] + ) + )->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertNotContains('delete', array_column($this->transitions, 0), 'nothing is soft-deleted'); + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $seven)); + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $eight), 'APPID 8 may be the unreadable row, so it is archived'); + $this->assertSame([true, false], $this->stateOf(schema: self::USAGE, uuid: $this->usageOf(moduleUuid: $eight))); + $this->assertSame([2, 0], [$report['summary']['archived'], $report['summary']['deleted']]); + $this->assertSame( + [['sheet' => 'Beheerde Applicaties CMDB', 'message' => 'Sheet "Beheerde Applicaties CMDB" has an APPID without a cached value in row 5, so applications missing from the export are archived, not deleted.']], + $report['importWarnings'] + ); + + // Keeping missing records needs no warning, and without the unreadable row the same workbook deletes. + $this->transitions = []; + $this->states = array_replace_recursive($this->states, [self::MODULE => [$eight => ['archived' => null]], self::USAGE => [$this->usageOf(moduleUuid: $eight) => ['archived' => null]]]); + $kept = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1'), $this->row(appId: '', cells: ['APPID' => null], row: 5, uncached: ['APPID'])], archived: ['7'])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'missingRecords' => 'keep']); + $this->assertSame([], $kept['importWarnings']); + $this->assertSame([], $this->transitions); + + $readable = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7'])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $this->assertSame([false, true], $this->stateOf(schema: self::MODULE, uuid: $eight), 'a readable workbook still soft-deletes what is on no sheet'); + $this->assertSame([], $readable['importWarnings']); + }//end testAnUncachedCmdbRowArchivesInsteadOfDeleting() + + /** + * An extra usage of a key that cannot be archived fails the application's row; its module, written after every usage, stays + * as it was, and the other applications are still reconciled. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testAFailingExtraUsageFailsTheApplicationBeforeItsModule(): void { + $this->importOneSevenAndEight(); + [$seven, $eight] = [$this->moduleOf(appId: '7'), $this->moduleOf(appId: '8')]; + $sevenUsage = $this->usageOf(moduleUuid: $seven); + $this->store[self::USAGE]['usage-second'] = array_merge($this->store[self::USAGE][$sevenUsage], ['id' => 'usage-second']); + $this->beforeTransition = static function (string $verb, string $uuid): void { + if ($uuid === 'usage-second') { + throw new RuntimeException('the extra usage cannot be archived'); + } + }; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7'])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $rows = array_values(array_filter($report['rows'], static fn (array $row): bool => $row['row'] === 0)); + $this->assertSame(['failed', 'deleted'], array_column($rows, 'outcome')); + $this->assertSame(['step "archive" failed (RuntimeException)'], $rows[0]['reasons']); + $this->assertSame([true, false], $this->stateOf(schema: self::USAGE, uuid: $sevenUsage), 'the application\'s own usage went first'); + $this->assertSame([false, false], $this->stateOf(schema: self::USAGE, uuid: 'usage-second'), 'the failing extra is unchanged'); + $this->assertSame([false, false], $this->stateOf(schema: self::MODULE, uuid: $seven), 'the module comes after every usage, so it was not reached'); + $this->assertSame([false, true], $this->stateOf(schema: self::MODULE, uuid: $eight), 'the next application is still reconciled'); + $this->assertSame(1, $report['summary']['failed']); + }//end testAFailingExtraUsageFailsTheApplicationBeforeItsModule() + /** * A cancelled import reconciles nothing. * From 18bd2224df881d3ed09c706d92f67ebc426de752 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Tue, 13 Oct 2026 18:58:44 +0200 Subject: [PATCH 33/41] docs(openspec): tick the reconciliation tasks that this PR implements Closes the merge artefact from the base fix, which unticked Tasks 3-7 and said their code lands in #1286/#1287: - Tasks 3, 4 and 5 (matching, reconciliation, controller option) are implemented and tested here: Implement and Test ticked again. - Tasks 6 and 7 stay unticked: the UI is #1287, and the docs and the e2e run of task 7 are not done. - The status line now says what is true for this PR. - Task 4's acceptance criterion also names the CMDB-sheet unreadable APPID case. Co-Authored-By: Claude Opus 5.5 --- .../cmdb-import-archive-reconciliation/tasks.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/openspec/changes/cmdb-import-archive-reconciliation/tasks.md b/openspec/changes/cmdb-import-archive-reconciliation/tasks.md index 9e1e85772..b75a1fb11 100644 --- a/openspec/changes/cmdb-import-archive-reconciliation/tasks.md +++ b/openspec/changes/cmdb-import-archive-reconciliation/tasks.md @@ -4,7 +4,7 @@ Spec: `openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-imp ## Implementation Tasks -Status in this PR (the archive-sheet reader slice): Tasks 1 and 2 are implemented here. Tasks 3-7 are unticked because their code lands in the follow-up PRs (reconciliation in #1286, UI in #1287). +Status in this PR (the reconciliation slice, on top of the reader slice): Tasks 1-5 are implemented here (the reader's tasks 1-2 come from the base PR). Tasks 6-7 stay unticked: the UI lands in #1287, and the docs and the e2e run of task 7 are not done yet. ### Task 1: Schema opt-in and the import profile - **spec_ref**: `SPEC#requirement-archived-applications-and-usages-shall-be-listable-in-stackiq-and-shall-stay-out-of-opencatalogi-and-portaliq-by-default-req-cmdb-017`, `SPEC#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012` @@ -37,23 +37,23 @@ Status in this PR (the archive-sheet reader slice): Tasks 1 and 2 are implemente - GIVEN OpenRegister's search results (no archive marker; no soft-deleted row, `_includeDeleted` or not) THEN the state comes from the lens that found the object and from the trash listing, read once per import (design D9) - GIVEN `updateExisting` false THEN the row is skipped as `exists` and nothing is revived - GIVEN a deleted and a working module (and usage) sharing one import key THEN the working one is updated and nothing is restored -- [ ] Implement -- [ ] Test +- [x] Implement +- [x] Test ### Task 4: Reconciliation after the rows - **spec_ref**: `SPEC#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015` - **files**: `lib/Service/CmdbExportImportService.php`, `lib/Service/ProgressTracker.php`, `lib/Exception/CmdbImportException.php`, `tests/Unit/Service/CmdbExportImportServiceTest.php` - **acceptance_criteria**: - GIVEN APPIDs 1 and 7 imported WHEN an export with row 1 and 7 on the archive sheet is imported THEN module and usage of 7 are archived with a reason, the report row is `archived`, and the owner contact person is untouched - - GIVEN APPID 7 on no sheet THEN module and usage are soft-deleted and the row is `deleted`; GIVEN the archive sheet absent, without an APPID column, or with an APPID that has no cached value THEN they are archived instead and the report warns + - GIVEN APPID 7 on no sheet THEN module and usage are soft-deleted and the row is `deleted`; GIVEN the archive sheet absent, without an APPID column, or with an APPID that has no cached value, or a CMDB-sheet row whose APPID has no cached value THEN they are archived instead and the report warns - GIVEN APPID 7 soft-deleted and listed on the archive sheet THEN module and usage are restored and archived; GIVEN it already archived or, when on no sheet, already deleted THEN nothing is written - GIVEN a working pair and a trashed pair under the same import key and the key on the archive sheet THEN the working pair is archived, nothing is restored and the trashed pair stays deleted; GIVEN a further working usage of the key (of the trashed module, or a second one of the working module) THEN it is archived or deleted with the application - GIVEN `missingRecords` `keep`, a cancelled import, or a usage of another municipality THEN nothing is archived or deleted - GIVEN a cancel during the last row THEN nothing is archived or deleted; GIVEN a cancel during the reconciliation THEN it stops before the next application and the report is cancelled - GIVEN the archive of one application throws THEN that application is a `failed` row and the others are reconciled; GIVEN no ArchiveHandler THEN 503 `ARCHIVE_UNAVAILABLE` before reading - GIVEN the step runs THEN the operation is in phase `reconciling` and its progress advances per application -- [ ] Implement -- [ ] Test +- [x] Implement +- [x] Test ### Task 5: Controller option - **spec_ref**: `SPEC#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012` @@ -61,8 +61,8 @@ Status in this PR (the archive-sheet reader slice): Tasks 1 and 2 are implemente - **acceptance_criteria**: - GIVEN no `missingRecords` THEN the service receives `archive`; GIVEN `keep` or `archive` THEN it is passed through; GIVEN `remove` or `mark` THEN 422 `MISSING_RECORDS_UNSUPPORTED` with `details.accepted` `["keep", "archive"]` and no import - GIVEN `ARCHIVE_UNAVAILABLE` from the service THEN 503 with its own translated message -- [ ] Implement -- [ ] Test +- [x] Implement +- [x] Test ### Task 6: Settings section, report words and the archive quick filter - **spec_ref**: `SPEC#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014`, `SPEC#requirement-archived-applications-and-usages-shall-be-listable-in-stackiq-and-shall-stay-out-of-opencatalogi-and-portaliq-by-default-req-cmdb-017` From 27a2110891ec39922e7e3c02a37e794e50bb3579 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Tue, 13 Oct 2026 19:05:06 +0200 Subject: [PATCH 34/41] fix(cmdb-import): see a fully empty CMDB row with an uncached APPID; describe what a failed application leaves Closes the pre-push findings: - "Fully-empty CMDB row with an uncached APPID stays invisible": the reader dropped a CMDB row whose only content is an APPID formula without a cached value before the service could see it, so its application could still be soft-deleted. read() now returns `uncachedKeyRows` (source sheet => dropped row numbers, collected only when the key column is the uncached one), and the service merges them with the rows that carry `uncached` into the same unreadable-key bookkeeping, so the run archives instead of deleting and warns. New tests: testAnEmptyCmdbRowWithAnUncachedKeyIsListed (reader) and testAFullyEmptyCmdbRowWithAnUncachedKeyArchivesInsteadOfDeleting (service). - "Failure-branch docblock overstates": the itemsOf() and reconcileApplication() docblocks and design.md D5 and Risks now state the order (own usage, extra usages, module, extra modules) and what a failure leaves: a failing extra usage leaves the own usage changed, that extra visible and the module unchanged, and the row only says failed. Class sweep: other callers of readRows() are the archive sheet (unchanged, no key filter) and readSourceRows(); the read() docblock names the new key. The spec and proposal speak of "a row of a CMDB sheet" in general and need no change. Co-Authored-By: Claude Opus 5.5 --- lib/Service/Cmdb/CmdbWorkbookReader.php | 59 ++++++++++++++++--- lib/Service/CmdbExportImportService.php | 32 ++++++++-- .../design.md | 4 +- .../Service/Cmdb/CmdbWorkbookReaderTest.php | 28 +++++++++ .../Service/CmdbExportImportServiceTest.php | 32 +++++++++- 5 files changed, 136 insertions(+), 19 deletions(-) diff --git a/lib/Service/Cmdb/CmdbWorkbookReader.php b/lib/Service/Cmdb/CmdbWorkbookReader.php index 3ba065a4d..4b80393b8 100644 --- a/lib/Service/Cmdb/CmdbWorkbookReader.php +++ b/lib/Service/Cmdb/CmdbWorkbookReader.php @@ -175,8 +175,11 @@ public function isAvailable(): bool { * @param CmdbImportProfile $profile The import profile. * * @return array `rows` (list of {sheet, row, cells, uncached}), `importWarnings` - * (list of {sheet, message}), `date1904` (bool) and `archive` - * ({sheet, present, keyColumnMissing, appIds, uncachedRows}). + * (list of {sheet, message}), `date1904` (bool), `archive` + * ({sheet, present, keyColumnMissing, appIds, uncachedRows}) and `uncachedKeyRows` + * (source sheet => numbers of the rows dropped as empty because their key is a + * formula without a cached value; a row with other content is in `rows`, its + * `uncached` lists the key). * * @throws CmdbImportException WORKBOOK_TOO_LARGE, READER_UNAVAILABLE, NOT_XLSX, NO_SOURCE_SHEET, * MISSING_COLUMN or TOO_MANY_ROWS. @@ -369,7 +372,15 @@ private function readBounded(string $path, CmdbImportProfile $profile, object $s $spreadsheet = $this->load(path: $path, sheetNames: $loaded, filter: new CmdbReadFilter(lastRow: $lastRow, columns: $letters)); try { - $rows = $this->readSourceRows(spreadsheet: $spreadsheet, sheetNames: $present, columnsPerSheet: $resolved['columns'], limit: $limit); + $uncachedKeyRows = []; + $rows = $this->readSourceRows( + spreadsheet: $spreadsheet, + sheetNames: $present, + columnsPerSheet: $resolved['columns'], + limit: $limit, + keyColumn: $profile->keyColumn(), + uncachedKeyRows: $uncachedKeyRows + ); if ($archiveColumns !== []) { $unseen = []; $archiveRows = $this->readRows( @@ -389,7 +400,13 @@ private function readBounded(string $path, CmdbImportProfile $profile, object $s $spreadsheet->disconnectWorksheets(); } - return ['rows' => $rows, 'importWarnings' => $resolved['warnings'], 'date1904' => $date1904, 'archive' => $archive]; + return [ + 'rows' => $rows, + 'importWarnings' => $resolved['warnings'], + 'date1904' => $date1904, + 'archive' => $archive, + 'uncachedKeyRows' => $uncachedKeyRows, + ]; }//end readBounded() /** @@ -434,20 +451,38 @@ private function sourceSheets(string $path, CmdbImportProfile $profile): array { * @param array $sheetNames The source sheets the workbook has. * @param array> $columnsPerSheet Sheet name => column letter => column name. * @param int $limit The row limit per sheet. + * @param string $keyColumn The key column's name. + * @param array> $uncachedKeyRows Filled, per sheet, with the number of each row that was + * dropped as empty only because its key is a formula + * without a cached value. * * @return array> See read(). * * @throws CmdbImportException TOO_MANY_ROWS. */ - private function readSourceRows(object $spreadsheet, array $sheetNames, array $columnsPerSheet, int $limit): array { + private function readSourceRows( + object $spreadsheet, + array $sheetNames, + array $columnsPerSheet, + int $limit, + string $keyColumn, + array &$uncachedKeyRows, + ): array { $rows = []; foreach ($sheetNames as $sheetName) { + $unseen = []; $sheetRows = $this->readRows( worksheet: $spreadsheet->getSheetByName($sheetName), columns: $columnsPerSheet[$sheetName], sheetName: $sheetName, - limit: $limit + limit: $limit, + unseen: $unseen, + keyColumn: $keyColumn ); + if ($unseen !== []) { + $uncachedKeyRows[$sheetName] = array_values((array)$unseen); + } + array_push($rows, ...$sheetRows); } @@ -732,12 +767,20 @@ private function resolveColumns(object $worksheet, array $referenced): array { * @param int $limit The maximum number of non-empty rows. * @param array|null $unseen Filled with the number of each row that read as empty only because * a formula in it has no cached value; null to not collect them. + * @param string|null $keyColumn When set, only a row whose uncached columns include this one is collected. * * @return array, uncached: array}> * * @throws CmdbImportException TOO_MANY_ROWS. */ - private function readRows(object $worksheet, array $columns, string $sheetName, int $limit, ?array &$unseen = null): array { + private function readRows( + object $worksheet, + array $columns, + string $sheetName, + int $limit, + ?array &$unseen = null, + ?string $keyColumn = null, + ): array { $rows = []; $lastRow = (int)$worksheet->getHighestDataRow(); for ($rowNumber = 2; $rowNumber <= $lastRow; $rowNumber++) { @@ -747,7 +790,7 @@ private function readRows(object $worksheet, array $columns, string $sheetName, rowNumber: $rowNumber ); if ($empty === true) { - if ($uncached !== [] && $unseen !== null) { + if ($uncached !== [] && $unseen !== null && ($keyColumn === null || in_array($keyColumn, $uncached, true) === true)) { $unseen[] = $rowNumber; } diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 4c236d9ac..1cce85f47 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -556,7 +556,7 @@ private function runImport(string $path, array $options, string $startedAt): arr $report->setMunicipality(uuid: $municipality['uuid'], name: $municipality['name'], created: $municipality['created']); $report->addImportWarnings(warnings: $this->translateImportWarnings(warnings: $workbook['importWarnings'])); $archive = $this->archiveSheet(workbook: $workbook); - $this->unreadableKeyRows = $this->unreadableKeyRows(rows: $rows); + $this->unreadableKeyRows = $this->unreadableKeyRows(rows: $rows, droppedRows: (array)($workbook['uncachedKeyRows'] ?? [])); $this->addRunWarnings(report: $report, municipality: $municipality, archive: $archive, missingRecords: $missingRecords); $this->progressTracker->startOperation( @@ -684,20 +684,31 @@ private function addRunWarnings(CmdbImportReport $report, array $municipality, a * has an unreadable key. * * @param array}> $rows The reader rows. + * @param array> $droppedRows The rows the reader dropped as empty because their key + * has no cached value (they are not in `$rows`), per sheet. * * @return array> Sheet name => row numbers. * * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 */ - private function unreadableKeyRows(array $rows): array { + private function unreadableKeyRows(array $rows, array $droppedRows): array { $key = $this->profile->keyColumn(); $found = []; + foreach ($droppedRows as $sheet => $rowNumbers) { + $found[(string)$sheet] = array_map('intval', array_values((array)$rowNumbers)); + } + foreach ($rows as $row) { if (in_array($key, ($row['uncached'] ?? []), true) === true) { $found[$row['sheet']][] = $row['row']; } } + foreach ($found as $sheet => $rowNumbers) { + sort($rowNumbers); + $found[$sheet] = $rowNumbers; + } + return $found; }//end unreadableKeyRows() @@ -2572,8 +2583,15 @@ private function modulesInChunk(array $chunk, array $coordinates): array { /** * Archive or soft-delete one application that left the CMDB sheets, usage first, in its own error boundary. * - * The usage is written before the module, so a failure between the two - * leaves the municipality's view right and the module at worst visible. + * The writes go in this order: the application's own usage, the other + * usages of its key, its module, the other modules. A failure stops the + * application there and reports one `failed` row (the step and the + * exception class, no list of what was already written). What it leaves: + * a failing own usage leaves everything as it was; a failing extra usage + * leaves the own usage in the new state, that extra and the module as + * they were (so the extra stays visible and the module stays working); + * a failing module leaves every usage in the new state and the module + * as it was. The next import reconciles the application again. * An application already in the state the export asks for is not reported. * * @param ScopeEntry $application The application. @@ -2631,8 +2649,10 @@ private function reconcileApplication(array $application, array $archive, string /** * What one application brings to the state the export asks for: its usage and the other usages of its key, then its module and the other modules. * - * Usages come before modules, so a failure part way leaves the - * municipality's view right and a module at worst visible. + * Own usage, extra usages, own module, extra modules. Usages come before + * modules, so a failure leaves usages in the new state and a module at + * worst unchanged; it does not leave a module changed under a usage that + * still shows. An extra usage that fails does hold back the module. * * Only the application's own objects are restored from the trash, and only * when `restoreDeleted` says so; the extras never are. diff --git a/openspec/changes/cmdb-import-archive-reconciliation/design.md b/openspec/changes/cmdb-import-archive-reconciliation/design.md index 1fc7058ee..5329cc153 100644 --- a/openspec/changes/cmdb-import-archive-reconciliation/design.md +++ b/openspec/changes/cmdb-import-archive-reconciliation/design.md @@ -99,7 +99,7 @@ Per application, with `processed` = every APPID match key on the CMDB sheets (wh The table decides on one entry per key. When a municipality has several usages and modules under the same import key (D4 says earlier imports created exactly such duplicates, a module deleted by hand next to the one a later import created), the scope keeps the entry the import would match: the best module state wins (working, then archived, then deleted), the best usage state breaks a tie, the first listed breaks the rest. The other usages of the key follow the application, so a stray usage stays visible nowhere: a working or archived usage of a stale duplicate, a second usage of the same module, and a working or archived duplicate module are archived or soft-deleted as the key asks, once, in the same error boundary and the same report row (usages first, then modules). What is in the trash is left as it is: a stale trashed duplicate is not restored into the archive when the key is on the archive sheet, and is not touched when it is on no sheet. -The usage is written before the module, so a failure between the two leaves the municipality's usage in the new state and the module visible, never the reverse. A module another municipality still uses is archived or deleted all the same when this municipality's usage says so: the module carries this municipality's import key, so it is this municipality's record (REQ-CMDB-006 allows a shared module only when the other organisation's usage was added by hand). The archive reason is `cmdb-import: on sheet "Gearchiveerde Applicaties"` or `cmdb-import: not in the export`, plus the operation id, so the audit trail says why. +The writes go in this order: the application's own usage, the other usages of its key, its module, the other modules. A failure stops that application and gives one `failed` report row (step and exception class only). A failing own usage leaves everything as it was; a failing extra usage leaves the own usage in the new state and that extra and the module as they were (the extra stays visible, the module stays working); a failing module leaves every usage in the new state and the module as it was. Never is a module changed while a usage of the key still shows. The next import reconciles the application again. A module another municipality still uses is archived or deleted all the same when this municipality's usage says so: the module carries this municipality's import key, so it is this municipality's record (REQ-CMDB-006 allows a shared module only when the other organisation's usage was added by hand). The archive reason is `cmdb-import: on sheet "Gearchiveerde Applicaties"` or `cmdb-import: not in the export`, plus the operation id, so the audit trail says why. Each application runs in its own boundary: a failure is a `failed` row naming the step (`archive`, `restore`, `delete`) and the exception class, as row failures do, and the other applications continue. Contact persons are never touched. @@ -197,7 +197,7 @@ No seed objects change. The fragment's three seed modules stay as they are; the - [The default archives on the first re-import after the upgrade] → The section shows the choice with help text; the report lists every archived and deleted application; nothing is hard-deleted. - [A workbook saved without the archive sheet, or whose archive sheet has no APPID column or an APPID without a cached value, or a CMDB sheet with an APPID without a cached value] → archive only, never delete, plus a warning (D3). -- [A failure between the usage and the module] → the usage is written first, so the municipality's view is right and the module is at worst still visible (D5). +- [A failure part way through an application] → usages are written before modules, so a module is never changed under a usage that still shows; but a failing extra usage leaves the own usage changed, that extra visible and the module unchanged, and the row only says `failed`. The next import retries (D5). - [Reviving before writing adds calls per returning row] → only for rows whose match is archived or deleted; a row whose module is working costs the same searches as before, a new row one more search (the archive lens). - [The trash listing scans every magic table of the instance] → read once per import, not per row (D9). - [OpenRegister internals change shape] → guarded resolution, 503 before the file is read (D7). diff --git a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php index b4d561809..fa5754636 100644 --- a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php +++ b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php @@ -1055,6 +1055,34 @@ public function testAnUncachedArchiveKeyIsListedNotDropped(): void { $this->assertSame([2, 5], $archive['uncachedRows'], 'the rows whose APPID formula has no cached value'); }//end testAnUncachedArchiveKeyIsListedNotDropped() + /** + * A CMDB row whose only content is an APPID formula without a cached value is dropped as empty, but its row + * number is listed per sheet, so a caller knows the sheet holds an application it could not see. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testAnEmptyCmdbRowWithAnUncachedKeyIsListed(): void { + $this->requireSpreadsheet(); + $path = CmdbTestSupport::buildWorkbook( + sheets: [ + 'Beheerde Applicaties CMDB' => [['APPID', 'Applicatie Naam'], [1, 'Een'], [['f' => 'Invoer!A2']], [2, 'Twee']], + 'Onbeh Applicaties CMDB' => [['APPID', 'Applicatie Naam'], [3, 'Drie']], + ] + ); + + try { + $result = (new CmdbWorkbookReader())->read(path: $path, profile: $this->profile()); + } finally { + unlink($path); + } + + $beheerd = array_filter($result['rows'], static fn (array $row): bool => $row['sheet'] === 'Beheerde Applicaties CMDB'); + $this->assertSame([2, 4], array_column($beheerd, 'row'), 'the empty row is not a row'); + $this->assertSame(['Beheerde Applicaties CMDB' => [3]], $result['uncachedKeyRows'], 'the dropped row is listed, only on its sheet'); + }//end testAnEmptyCmdbRowWithAnUncachedKeyIsListed() + /** * An archive sheet whose last used row lies beyond twice the row limit stops before any sheet is loaded. * diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index c476b5b1f..2cf2e04c8 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -994,19 +994,21 @@ function (&$value): void { * * @return CmdbWorkbookReader */ - private function rowsReader(array $rows, ?array $archived = [], array $uncachedRows = []): CmdbWorkbookReader { - return new class($rows, $archived, $uncachedRows) extends CmdbWorkbookReader { + private function rowsReader(array $rows, ?array $archived = [], array $uncachedRows = [], array $uncachedKeyRows = []): CmdbWorkbookReader { + return new class($rows, $archived, $uncachedRows, $uncachedKeyRows) extends CmdbWorkbookReader { /** * Constructor. * * @param array> $rows The rows. * @param array|null $archived The archive sheet's APPIDs, or null. * @param array $uncachedRows The archive sheet's rows without a cached APPID. + * @param array> $uncachedKeyRows The CMDB rows dropped as empty, per sheet. */ public function __construct( private array $rows, private ?array $archived, private array $uncachedRows, + private array $uncachedKeyRows = [], ) { } @@ -1020,7 +1022,7 @@ public function __construct( */ public function read(string $path, CmdbImportProfile $profile): array { $archive = ['sheet' => 'Gearchiveerde Applicaties', 'present' => ($this->archived !== null), 'keyColumnMissing' => false, 'appIds' => ($this->archived ?? []), 'uncachedRows' => $this->uncachedRows]; - return ['rows' => $this->rows, 'importWarnings' => [], 'date1904' => false, 'archive' => $archive]; + return ['rows' => $this->rows, 'importWarnings' => [], 'date1904' => false, 'archive' => $archive, 'uncachedKeyRows' => $this->uncachedKeyRows]; } }; }//end rowsReader() @@ -2044,6 +2046,30 @@ public function testAnUncachedCmdbRowArchivesInsteadOfDeleting(): void { $this->assertSame([], $readable['importWarnings']); }//end testAnUncachedCmdbRowArchivesInsteadOfDeleting() + /** + * A CMDB row whose only content is an uncached APPID is dropped by the reader before the service sees a row, and + * is handed over as a list of rows: the import still archives instead of deleting, and warns. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testAFullyEmptyCmdbRowWithAnUncachedKeyArchivesInsteadOfDeleting(): void { + $this->importOneSevenAndEight(); + $eight = $this->moduleOf(appId: '8'); + + $report = $this->service( + reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7'], uncachedKeyRows: ['Beheerde Applicaties CMDB' => [6]]) + )->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertNotContains('delete', array_column($this->transitions, 0)); + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $eight)); + $this->assertSame( + [['sheet' => 'Beheerde Applicaties CMDB', 'message' => 'Sheet "Beheerde Applicaties CMDB" has an APPID without a cached value in row 6, so applications missing from the export are archived, not deleted.']], + $report['importWarnings'] + ); + }//end testAFullyEmptyCmdbRowWithAnUncachedKeyArchivesInsteadOfDeleting() + /** * An extra usage of a key that cannot be archived fails the application's row; its module, written after every usage, stays * as it was, and the other applications are still reconciled. From c55a96d8810cd12c7e804406e0a81537649ea92b Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Tue, 13 Oct 2026 19:07:37 +0200 Subject: [PATCH 35/41] test(cmdb-import): wait on a rendered list, not networkidle, and fail loudly on a bad lens read Closes "e2e waits for networkidle, which never settles on Nextcloud (gate-58)": the archive test now waits for a working application row to be visible before asserting that the archived application is absent, so the negative assertion cannot pass on a list that has not rendered. No other networkidle wait remains in the touched spec (git grep over tests/e2e shows only comments explaining why it is avoided). Closes thread 4248187506 (e2e cleanup comment): the header and the helper doc no longer claim that an interrupted run leaves nothing behind or that _includeDeleted adds the trash (a soft-deleted row cannot be listed, design D9); objectsInLens now fails the test on a non-OK response instead of returning an empty list that made cleanup skip silently. Co-Authored-By: Claude Opus 5.5 --- tests/e2e/spec-coverage/cmdb-import.spec.ts | 31 +++++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/tests/e2e/spec-coverage/cmdb-import.spec.ts b/tests/e2e/spec-coverage/cmdb-import.spec.ts index c0bd8d3e3..e86047e41 100644 --- a/tests/e2e/spec-coverage/cmdb-import.spec.ts +++ b/tests/e2e/spec-coverage/cmdb-import.spec.ts @@ -33,8 +33,10 @@ * moved APPID 1234 to "Gearchiveerde Applicaties" and dropped APPID 2 archives * the one and moves the other to the trash, the Applications page lists 1234 * under "Archived", and the original export brings both back. The cleanup - * looks through OpenRegister's archive and trash lenses too, so an - * interrupted run leaves nothing archived behind. + * looks through OpenRegister's archive lens too, so an interrupted run + * leaves no archived application of the run's municipality behind; a + * soft-deleted one cannot be listed and stays in the trash until its purge + * date (design.md, D9). */ import type { APIRequestContext, Locator, Page, Response } from '@playwright/test' @@ -140,7 +142,8 @@ async function objectsOfMunicipality( /** * The objects of a schema that mention a municipality, through OpenRegister's * archive and trash lenses: `_archived` `true` lists the archive alone, `any` - * the archive and the working set; `_includeDeleted` adds the trash. + * the archive and the working set. `_includeDeleted` is sent for completeness: + * the list cannot return a soft-deleted row even then (design.md, D9). * * @param ctx The API context * @param schema The schema id @@ -158,9 +161,10 @@ async function objectsInLens( const res = await ctx.get( `/index.php/apps/openregister/api/objects/${config.register}/${schema}?_limit=5000&_archived=${lens.archived}&_includeDeleted=${lens.includeDeleted}`, ) - if (!res.ok()) { - return [] - } + expect( + res.ok(), + `Listing ${schema} with _archived=${lens.archived} failed with ${res.status()}`, + ).toBe(true) const body = await res.json() const list = body?.results ?? body ?? [] return (Array.isArray(list) ? list : []).filter((row) => @@ -358,7 +362,8 @@ test.describe.serial('CMDB import section', () => { config.contactpersoon_schema, config.module_schema, ]) { - // Archived and deleted objects too: the archive test may have stopped half-way. + // Archived objects too: the archive test may have stopped half-way. A + // soft-deleted one is not listable and stays in the trash until it is purged. for (const row of await objectsInLens( ctx, schema, @@ -560,8 +565,16 @@ test.describe.serial('CMDB import section', () => { // 2. The Applications page lists 1234 only under the quick filter "Archived". await gotoAppRoute(page, '/modules') await dismissSupportDialog(page) - // Not under the default tab: wait for the list to settle, then look. - await page.waitForLoadState('networkidle') + // Not under the default tab. Nextcloud keeps polls open, so no network + // idle wait: wait until the list shows a working application (the + // export brought more than the two this test took out), then look. + expect( + working.modules, + 'the working set must still hold an application the list can show', + ).toBeGreaterThan(0) + await expect(page.locator('tbody tr').first()).toBeVisible({ + timeout: 30000, + }) await expect(page.getByText(ARCHIVED_APPLICATION_NAME)).toHaveCount(0) await page.getByRole('tab', { name: 'Archived', exact: true }).click() await expect(page.getByText(ARCHIVED_APPLICATION_NAME).first()).toBeVisible({ From 4b70494f196b0827470980ea242d6df8c4a9db9a Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Tue, 13 Oct 2026 19:07:37 +0200 Subject: [PATCH 36/41] fix(applications): say on the Archived tab that facets and search are ignored Closes "Archived tab keeps facet sidebar and search active but ignores them": FacetedCatalogIndexView tracks the active quick-filter tab and shows a note under the toolbar while a tab whose filter carries the empty _ids override is active; en and nl strings and the generated l10n js are added. A mounted view test asserts the note appears on the Archived tab, is gone on another tab, and goes red with the note removed. Closes "Archived-tab facet override is tested against a local merge, not the view": the facet-override test now calls the library's own useSelfFetchList and reads the fixedFilters getter it hands the list composable, instead of restating the spread; only the list composable and store are replaced. It goes red when the _ids override is removed from the manifest. The source grep test that pinned the merge order is dropped as redundant. Class sweep: the Services page uses the same view but has no quick filter with the override, so it shows no note. Co-Authored-By: Claude Opus 5.5 --- l10n/en.js | 3 +- l10n/en.json | 3 +- l10n/nl.js | 3 +- l10n/nl.json | 3 +- src/views/FacetedCatalogIndexView.vue | 49 +++++++++- tests/vitest/cmdbArchiveQuickFilter.spec.js | 86 +++++++++++----- .../vitest/facetedCatalogArchiveNote.spec.js | 98 +++++++++++++++++++ 7 files changed, 214 insertions(+), 31 deletions(-) create mode 100644 tests/vitest/facetedCatalogArchiveNote.spec.js diff --git a/l10n/en.js b/l10n/en.js index 613cb76f2..c4ee74c0a 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1153,7 +1153,8 @@ OC.L10N.register( "{count} unarchived": "{count} unarchived", "{count} deleted": "{count} deleted", "{count} restored": "{count} restored", - "Applications: {changes}.": "Applications: {changes}." + "Applications: {changes}.": "Applications: {changes}.", + "This tab lists the whole archive. It ignores the GEMMA filters and the search term.": "This tab lists the whole archive. It ignores the GEMMA filters and the search term." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index 29d312f53..22795a655 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1152,6 +1152,7 @@ "{count} unarchived": "{count} unarchived", "{count} deleted": "{count} deleted", "{count} restored": "{count} restored", - "Applications: {changes}.": "Applications: {changes}." + "Applications: {changes}.": "Applications: {changes}.", + "This tab lists the whole archive. It ignores the GEMMA filters and the search term.": "This tab lists the whole archive. It ignores the GEMMA filters and the search term." } } diff --git a/l10n/nl.js b/l10n/nl.js index 1dc8b83d9..7407af21b 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1221,7 +1221,8 @@ OC.L10N.register( "{count} unarchived": "{count} uit het archief gehaald", "{count} deleted": "{count} verwijderd", "{count} restored": "{count} teruggezet", - "Applications: {changes}.": "Applicaties: {changes}." + "Applications: {changes}.": "Applicaties: {changes}.", + "This tab lists the whole archive. It ignores the GEMMA filters and the search term.": "Dit tabblad toont het hele archief. De GEMMA-filters en de zoekterm worden genegeerd." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index 7963d0ebc..26705400c 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1220,6 +1220,7 @@ "{count} unarchived": "{count} uit het archief gehaald", "{count} deleted": "{count} verwijderd", "{count} restored": "{count} teruggezet", - "Applications: {changes}.": "Applicaties: {changes}." + "Applications: {changes}.": "Applicaties: {changes}.", + "This tab lists the whole archive. It ignores the GEMMA filters and the search term.": "Dit tabblad toont het hele archief. De GEMMA-filters en de zoekterm worden genegeerd." } } diff --git a/src/views/FacetedCatalogIndexView.vue b/src/views/FacetedCatalogIndexView.vue index 427245c47..cfb2737b2 100644 --- a/src/views/FacetedCatalogIndexView.vue +++ b/src/views/FacetedCatalogIndexView.vue @@ -92,6 +92,18 @@ generic route-query-to-filter passthrough never sees it (see the
+

+ {{ + t( + 'stackiq', + 'This tab lists the whole archive. It ignores the GEMMA filters and the search term.', + ) + }} +

+
+ @view="openDetail" + @quickFilterChange="onQuickFilterChange" />
@@ -281,6 +294,7 @@ export default { showSaveViewModal: false, savingView: false, searchDebounceTimer: null, + activeQuickFilterIndex: null, } }, @@ -340,6 +354,20 @@ export default { ) }, + /** + * Whether the active quick-filter tab drops the facet id set and the + * search term: a tab whose filter carries the empty-string `_ids` + * override (the Archived tab, see the `quickFilters` prop). The facet + * sidebar and the search box stay on screen, so the page says so. + * + * @return {boolean} True while such a tab is active. + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-archived-applications-and-usages-shall-be-listable-in-stackiq-and-shall-stay-out-of-opencatalogi-and-portaliq-by-default-req-cmdb-017 + */ + archivedTabIgnoresFilters() { + const tab = (this.quickFilters ?? [])[this.activeQuickFilterIndex] + return tab?.filter?._ids === '' + }, + /** * Remount key for `CnIndexPage`: its self-fetch mode has no reactive * watcher on the `filter` prop (only route/quick-filter/tab changes @@ -375,6 +403,18 @@ export default { }, methods: { + /** + * Remember which quick-filter tab is active (single-select mode emits the + * tab index; multi-select emits an array, which never selects the + * Archived tab here). + * + * @param {number|number[]} index The tab index, or the selected indices + * @return {void} + */ + onQuickFilterChange(index) { + this.activeQuickFilterIndex = Number.isInteger(index) ? index : null + }, + /** * Open the detail page of a clicked row, or of the row whose View action was used. * @@ -538,6 +578,13 @@ export default { flex: 1; } +.faceted-catalog-index__archive-note { + color: var(--color-text-maxcontrast); + margin: 0; + padding: 0 calc(var(--default-grid-baseline, 8px) * 2) + calc(var(--default-grid-baseline, 8px) * 2); +} + .faceted-catalog-index__body { display: flex; align-items: flex-start; diff --git a/tests/vitest/cmdbArchiveQuickFilter.spec.js b/tests/vitest/cmdbArchiveQuickFilter.spec.js index ff4df73ee..f93a42ba1 100644 --- a/tests/vitest/cmdbArchiveQuickFilter.spec.js +++ b/tests/vitest/cmdbArchiveQuickFilter.spec.js @@ -18,12 +18,34 @@ import fs from 'fs' import path from 'path' import { fileURLToPath } from 'url' import { describe, expect, it, vi } from 'vitest' +import { effectScope } from 'vue' // resolveFilterTokens reads the current user at import time, which needs a // browser; the tokens this spec merges (`_archived`, `_ids`) resolve no user. vi.mock('@nextcloud/auth', () => ({ getCurrentUser: () => null })) +// useSelfFetchList is the real function; only the list composable it feeds its +// `fixedFilters` getter to is replaced, so the test can read the options it got. +const captured = vi.hoisted(() => ({ options: null })) +vi.mock( + '../../node_modules/@conduction/nextcloud-vue/src/composables/index.js', + () => ({ + useListView: (objectType, options) => { + captured.options = options + return { refresh: () => {}, searchTerm: { value: '' } } + }, + }), +) +vi.mock( + '../../node_modules/@conduction/nextcloud-vue/src/composables/useObjectSubscription.js', + () => ({ + useObjectSubscription: () => ({}), + }), +) +vi.mock('../../node_modules/@conduction/nextcloud-vue/src/store/index.js', () => ({ + useObjectStore: () => ({ registerObjectType: () => {} }), +})) +import { useSelfFetchList } from '../../node_modules/@conduction/nextcloud-vue/src/components/CnIndexPage/useSelfFetchList.js' // The REAL implementations from the installed package — not a local restatement. import { buildQueryString } from '../../node_modules/@conduction/nextcloud-vue/src/utils/headers.js' -import { resolveFilterMap } from '../../node_modules/@conduction/nextcloud-vue/src/utils/routeFilters.js' import { facetNarrowingFilter } from '../../src/utils/facetNarrowing.js' const here = path.dirname(fileURLToPath(import.meta.url)) @@ -74,41 +96,53 @@ describe('the archive quick filter', () => { // matched) and merges the active tab's filter over it. OpenRegister leaves // archived objects out of that id set, so `_ids` AND `_archived=true` lists // nothing: with a facet selected, the Archived tab must not send `_ids`. + // The merge is the library's own: useSelfFetchList hands useListView a + // `fixedFilters` getter, which is called here exactly as a fetch calls it. it('sends no _ids with the Archived tab when a facet narrowed the list', () => { const page = pages().find((candidate) => candidate.id === 'Modules') - const archived = page.config.quickFilters.find( + const quickFilters = page.config.quickFilters + const archivedIndex = quickFilters.findIndex( (tab) => tab.label === 'Archived', ) const facetFilter = facetNarrowingFilter(['live-1', 'live-2']) - // The merge useSelfFetchList#fixedFilters does in single-tab mode: - // the page's filter, then the active tab's filter spread last. - const archivedQuery = buildQueryString({ - ...facetFilter, - ...resolveFilterMap(archived.filter, {}, {}), - }) - const allQuery = buildQueryString({ - ...facetFilter, - ...resolveFilterMap({}, {}, {}), - }) + /** + * The query string the library builds for the list with one tab active. + * + * @param {number|null} tabIndex The active quick-filter tab + * @return {string} The query string + */ + function queryWithTab(tabIndex) { + const scope = effectScope() + let fixedFilters + captured.options = null + scope.run(() => { + const result = useSelfFetchList( + { + register: 'voorzieningen', + schema: 'module', + filter: facetFilter, + quickFilters, + columns: [], + }, + null, + () => null, + ) + result.activeQuickFilterIndex.value = tabIndex + fixedFilters = captured.options.fixedFilters + }) + const query = buildQueryString(fixedFilters()) + scope.stop() + return query + } + + const archivedQuery = queryWithTab(archivedIndex) + const otherQuery = queryWithTab(archivedIndex === 0 ? 1 : 0) expect(new URLSearchParams(archivedQuery).get('_archived')).toBe('true') expect(archivedQuery).not.toContain('_ids') // A tab without the override keeps the facet narrowing. - expect(allQuery).toContain('_ids') - }) - - it('still merges the active tab last, which the override above relies on', () => { - const source = fs.readFileSync( - path.join( - repoRoot, - 'node_modules/@conduction/nextcloud-vue/src/components/CnIndexPage/useSelfFetchList.js', - ), - 'utf8', - ) - expect(source).toMatch( - /\.\.\.base, \.\.\.resolveFilterMap\(tabFilter, params, ctx\)/, - ) + expect(otherQuery).toContain('_ids') }) it('the Modules page note names the Archived tab as the exception to the additive merge', () => { diff --git a/tests/vitest/facetedCatalogArchiveNote.spec.js b/tests/vitest/facetedCatalogArchiveNote.spec.js new file mode 100644 index 000000000..53f397737 --- /dev/null +++ b/tests/vitest/facetedCatalogArchiveNote.spec.js @@ -0,0 +1,98 @@ +// @vitest-environment jsdom +/** + * The Archived tab of the Applications page says it ignores the facets and + * the search term. + * + * The facet sidebar and the search box stay on screen on that tab, but its + * filter carries an empty `_ids` that drops the facet id set. Without a note + * an admin who picks a facet sees an unchanged list and no reason. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-archived-applications-and-usages-shall-be-listable-in-stackiq-and-shall-stay-out-of-opencatalogi-and-portaliq-by-default-req-cmdb-017 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +import { mount } from '@vue/test-utils' +import { createPinia, setActivePinia } from 'pinia' +import { describe, expect, it, vi } from 'vitest' +import { defineComponent, h } from 'vue' + +// CnIndexPage stands in as a component that emits what the real one emits +// when a quick-filter tab is chosen. +vi.mock('@conduction/nextcloud-vue', () => ({ + CnFacetSidebar: defineComponent({ + name: 'CnFacetSidebar', + render: () => h('div'), + }), + CnIndexPage: defineComponent({ + name: 'CnIndexPage', + emits: ['quick-filter-change'], + render() { + return h('div', { class: 'index-page-stub' }) + }, + }), +})) +vi.mock('../../src/modals/SaveFacetViewModal.vue', () => ({ + default: defineComponent({ name: 'SaveFacetViewModal', render: () => h('div') }), +})) + +import FacetedCatalogIndexView from '../../src/views/FacetedCatalogIndexView.vue' + +const NOTE = '.faceted-catalog-index__archive-note' +const QUICK_FILTERS = [ + { label: 'All', filter: {}, default: true }, + { label: 'Archived', filter: { _archived: 'true', _ids: '' } }, +] + +/** + * Mount the view with the facet store's network calls silenced. + * + * @return {object} The wrapper + */ +function mountView() { + setActivePinia(createPinia()) + return mount(FacetedCatalogIndexView, { + props: { + schema: 'module', + register: 'voorzieningen', + title: 'Applications', + quickFilters: QUICK_FILTERS, + }, + global: { + mocks: { + $route: { query: {} }, + $router: { push: vi.fn() }, + t: (app, text) => text, + }, + stubs: { + NcTextField: true, + NcButton: true, + NcActions: true, + NcActionButton: true, + NcActionCaption: true, + }, + plugins: [ + (app) => { + app.config.globalProperties.t = (a, text) => text + }, + ], + }, + }) +} + +describe('FacetedCatalogIndexView on the Archived tab', () => { + it('shows the note only while the Archived tab is active', async () => { + const wrapper = mountView() + const index = wrapper.findComponent({ name: 'CnIndexPage' }) + + expect(wrapper.find(NOTE).exists()).toBe(false) + + await index.vm.$emit('quick-filter-change', 1) + expect(wrapper.find(NOTE).exists()).toBe(true) + expect(wrapper.find(NOTE).text()).toContain('ignores the GEMMA filters') + + await index.vm.$emit('quick-filter-change', 0) + expect(wrapper.find(NOTE).exists()).toBe(false) + }) +}) From 3c17b477431b75c366a2ffc4b5665494817eac5f Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Tue, 13 Oct 2026 19:07:37 +0200 Subject: [PATCH 37/41] docs(cmdb-import): tick the UI tasks and document the archive tab note and unreadable CMDB APPIDs Makes tasks.md true for this PR after the merge of the reconcile base: the status line names #1286 for tasks 1-5 and this PR for task 6 (ticked, with the new Archived-tab note as an acceptance line); task 7 has its implementation ticked and its Playwright run still open. docs/features/cmdb-import.md already described archive and soft delete (the old "left as they are" sentence only remains for the Keep choice); it now also says that an APPID without a cached value on a CMDB sheet archives instead of deleting, and that the Archived tab shows a note about the ignored facets and search. Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 4 ++-- .../changes/cmdb-import-archive-reconciliation/tasks.md | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index c4d839903..e71756d73 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -277,13 +277,13 @@ APPID is on neither CMDB sheet: |---|---| | on the sheet `Gearchiveerde Applicaties` | are **archived**: OpenRegister's archive state, with the reason `cmdb-import: on sheet "Gearchiveerde Applicaties"`. Reported as *archived*. | | on no sheet at all | are **deleted**: moved to OpenRegister's trash, where they stay restorable for 30 days. Reported as *deleted*. | -| on no sheet, but the archive sheet cannot be relied on: the workbook has no sheet `Gearchiveerde Applicaties`, the sheet has no `APPID` column, or an `APPID` in it has no cached value (a formula whose result the file does not store) | are **archived**, never deleted. Each case has its own warning in the result: *sheet not found*, *no column APPID*, or *an APPID without a cached value in row …* (for more rows: *in N rows (…)*, naming the first ten). Such a workbook therefore never moves anything to the trash. | +| on no sheet, but the archive sheet cannot be relied on: the workbook has no sheet `Gearchiveerde Applicaties`, the sheet has no `APPID` column, or an `APPID` in it has no cached value (a formula whose result the file does not store), or an `APPID` on one of the two CMDB sheets has no cached value (the import cannot tell which application that row stands for) | are **archived**, never deleted. Each case has its own warning in the result: *sheet not found*, *no column APPID*, or *an APPID without a cached value in row …* (for more rows: *in N rows (…)*, naming the first ten; for a CMDB sheet the warning names that sheet). Such a workbook therefore never moves anything to the trash. | An archived or deleted application disappears from the stackiq lists, from OpenCatalogi and from Portaliq's **Software we use**. In stackiq, the quick filter **Archived** on the **Applications** and **Applications in use** pages lists the archive. On the **Applications** page the **Archived** tab lists -the whole archive: it ignores the GEMMA filters and the search box, because +the whole archive: it ignores the GEMMA filters and the search box (the page says so in a note under the search box while that tab is active), because their counts and matches describe the working set, which leaves archived applications out. An OpenCatalogi caller finds archived applications by adding `_archived=true` to its request. Contact persons are never archived diff --git a/openspec/changes/cmdb-import-archive-reconciliation/tasks.md b/openspec/changes/cmdb-import-archive-reconciliation/tasks.md index b75a1fb11..76d347861 100644 --- a/openspec/changes/cmdb-import-archive-reconciliation/tasks.md +++ b/openspec/changes/cmdb-import-archive-reconciliation/tasks.md @@ -4,7 +4,7 @@ Spec: `openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-imp ## Implementation Tasks -Status in this PR (the reconciliation slice, on top of the reader slice): Tasks 1-5 are implemented here (the reader's tasks 1-2 come from the base PR). Tasks 6-7 stay unticked: the UI lands in #1287, and the docs and the e2e run of task 7 are not done yet. +Status in this PR (the reconciliation slice, on top of the reader slice): Tasks 1-5 are implemented in #1286 (the reader's tasks 1-2 come from the base PR). Task 6 (the UI) is implemented in #1287; task 7 has its fixture, docs and e2e spec written there, and the Playwright run is the one thing not done yet. ### Task 1: Schema opt-in and the import profile - **spec_ref**: `SPEC#requirement-archived-applications-and-usages-shall-be-listable-in-stackiq-and-shall-stay-out-of-opencatalogi-and-portaliq-by-default-req-cmdb-017`, `SPEC#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012` @@ -71,9 +71,10 @@ Status in this PR (the reconciliation slice, on top of the reader slice): Tasks - GIVEN the section THEN a labelled choice "Applications missing from the export" offers Archive (default) and Keep with help text, and `buildImportForm()` sends the choice as `missingRecords` - GIVEN a report with the new outcomes THEN the summary shows their tiles, the outcome filter offers them, and a reconciliation row shows no row number - GIVEN the Applications and Applications in use pages THEN a quick filter "Archived" sends `_archived=true` + - GIVEN the Applications page with the "Archived" tab active THEN a note under the toolbar says the tab ignores the GEMMA filters and the search term, and it is gone on any other tab - Every new string is in `l10n/en.json` and `l10n/nl.json`, and the generated `l10n/*.js` are current -- [ ] Implement -- [ ] Test +- [x] Implement +- [x] Test ### Task 7: Fixture, docs and e2e - **spec_ref**: `SPEC#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015` @@ -82,7 +83,7 @@ Status in this PR (the reconciliation slice, on top of the reader slice): Tasks - GIVEN `build-fixtures.py` THEN it derives `topdesk-archived-applications.xlsx`: APPID 1234 on the archive sheet only, APPID 2 on no sheet, both CMDB sheets present and empty; the hygiene test passes on it - GIVEN the e2e THEN after the first import, importing the variant archives 1234 and soft-deletes 2, the Applications page lists 1234 under "Archived", and importing the original again reports 1234 `unarchived` and 2 `restored` with the object counts back to the first import's - GIVEN the docs THEN "Repeat imports" describes archive, soft delete, the archive sheet, the option and the return of an application, and the error table has `ARCHIVE_UNAVAILABLE` and the new meaning of `MISSING_RECORDS_UNSUPPORTED` -- [ ] Implement +- [x] Implement - [ ] Test - Status: PHPUnit covers the fixture round trip (testTheArchivedApplicationsFixtureArchivesDeletesAndBringsBack); the Playwright test is written and not run yet, because no Nextcloud was reachable from the worktree. From 1719b19ee8def2633ba9fc35e7fa4f8df10f2ad5 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Tue, 13 Oct 2026 19:17:39 +0200 Subject: [PATCH 38/41] fix(applications): keep the Archived-tab note in step with the list's tab The view started with no known tab and only learned of a change from the tab's click, but it remounts CnIndexPage whenever the facet or search key changes, which puts the page back on its default tab; the note then showed over the default list. The view now starts from the tab the library starts on (first quick filter marked default, else the first) and returns to it whenever the remount key changes. Tests: the note follows the default tab at mount, and disappears when a search change remounts the list while the Archived tab is active; the remount test goes red without the reset. The e2e wait for a listed application is scoped to the Applications list container, so a row of another table cannot satisfy it. Co-Authored-By: Claude Opus 5.5 --- src/views/FacetedCatalogIndexView.vue | 29 ++++++++++++++++++- tests/e2e/spec-coverage/cmdb-import.spec.ts | 4 ++- .../vitest/facetedCatalogArchiveNote.spec.js | 29 +++++++++++++++++-- 3 files changed, 58 insertions(+), 4 deletions(-) diff --git a/src/views/FacetedCatalogIndexView.vue b/src/views/FacetedCatalogIndexView.vue index cfb2737b2..6cb3f8aec 100644 --- a/src/views/FacetedCatalogIndexView.vue +++ b/src/views/FacetedCatalogIndexView.vue @@ -171,6 +171,21 @@ const DIMENSION_LABELS = { domain: () => t('stackiq', 'Domain'), } +/** + * The tab CnIndexPage starts on: the first quick filter marked `default`, + * else the first one (the library's own rule in `useSelfFetchList.js`). + * + * @param {Array|null} quickFilters The quick filters + * @return {number|null} The index, or null without quick filters + */ +function initialQuickFilterIndex(quickFilters) { + if (!Array.isArray(quickFilters) || quickFilters.length === 0) { + return null + } + const index = quickFilters.findIndex((tab) => tab && tab.default === true) + return index >= 0 ? index : 0 +} + /** Debounce delay (ms) between search keystrokes and the facets/list refetch. */ const SEARCH_DEBOUNCE_MS = 400 @@ -294,7 +309,7 @@ export default { showSaveViewModal: false, savingView: false, searchDebounceTimer: null, - activeQuickFilterIndex: null, + activeQuickFilterIndex: initialQuickFilterIndex(this.quickFilters), } }, @@ -383,6 +398,18 @@ export default { }, }, + watch: { + /** + * CnIndexPage is remounted when the key changes and starts on its + * default tab again, so the tab this view remembers goes back to it. + * + * @return {void} + */ + indexPageKey() { + this.activeQuickFilterIndex = initialQuickFilterIndex(this.quickFilters) + }, + }, + created() { this.facetStore.setFiltersFromQuery(this.schema, this.$route.query) }, diff --git a/tests/e2e/spec-coverage/cmdb-import.spec.ts b/tests/e2e/spec-coverage/cmdb-import.spec.ts index e86047e41..b10deda29 100644 --- a/tests/e2e/spec-coverage/cmdb-import.spec.ts +++ b/tests/e2e/spec-coverage/cmdb-import.spec.ts @@ -572,7 +572,9 @@ test.describe.serial('CMDB import section', () => { working.modules, 'the working set must still hold an application the list can show', ).toBeGreaterThan(0) - await expect(page.locator('tbody tr').first()).toBeVisible({ + await expect( + page.locator('.faceted-catalog-index__list tbody tr').first(), + ).toBeVisible({ timeout: 30000, }) await expect(page.getByText(ARCHIVED_APPLICATION_NAME)).toHaveCount(0) diff --git a/tests/vitest/facetedCatalogArchiveNote.spec.js b/tests/vitest/facetedCatalogArchiveNote.spec.js index 53f397737..23060dc1e 100644 --- a/tests/vitest/facetedCatalogArchiveNote.spec.js +++ b/tests/vitest/facetedCatalogArchiveNote.spec.js @@ -50,14 +50,14 @@ const QUICK_FILTERS = [ * * @return {object} The wrapper */ -function mountView() { +function mountView(quickFilters = QUICK_FILTERS) { setActivePinia(createPinia()) return mount(FacetedCatalogIndexView, { props: { schema: 'module', register: 'voorzieningen', title: 'Applications', - quickFilters: QUICK_FILTERS, + quickFilters, }, global: { mocks: { @@ -95,4 +95,29 @@ describe('FacetedCatalogIndexView on the Archived tab', () => { await index.vm.$emit('quick-filter-change', 0) expect(wrapper.find(NOTE).exists()).toBe(false) }) + + it('follows the default tab at mount, before any tab event', () => { + expect(mountView().find(NOTE).exists()).toBe(false) + + const archivedFirst = [ + { label: 'Archived', filter: { _archived: 'true', _ids: '' } }, + { label: 'All', filter: {} }, + ] + // No default marked: CnIndexPage starts on the first tab. + expect(mountView(archivedFirst).find(NOTE).exists()).toBe(true) + }) + + it('drops the note when a facet or search change remounts the list on its default tab', async () => { + const wrapper = mountView() + const index = wrapper.findComponent({ name: 'CnIndexPage' }) + await index.vm.$emit('quick-filter-change', 1) + expect(wrapper.find(NOTE).exists()).toBe(true) + + // A search term changes the key CnIndexPage is mounted under. + wrapper.vm.facetStore.module.search = 'mail' + await wrapper.vm.$nextTick() + await wrapper.vm.$nextTick() + + expect(wrapper.find(NOTE).exists()).toBe(false) + }) }) From 7d5aa915946947f1dd6f592e8226b7ee98ed3b5c Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Thu, 15 Oct 2026 10:37:37 +0200 Subject: [PATCH 39/41] refactor(cmdb-import): choose a missing application's target and reason in their own method reconcileApplication() reached the phpmd cyclomatic complexity threshold of 10 once the archive-sheet branch landed. The choice between archiving and soft-deleting, with its archive reason, moves to reconcileTarget(); the writes and their error boundary stay where they were. Co-Authored-By: Claude Opus 5.5 --- lib/Service/CmdbExportImportService.php | 61 +++++++++++++++++++------ 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 1cce85f47..24e6329cd 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -2605,25 +2605,19 @@ private function modulesInChunk(array $chunk, array $coordinates): array { */ private function reconcileApplication(array $application, array $archive, string $operationId, CmdbImportReport $report): void { $moduleData = $application['module']->getObject(); - $onArchiveSheet = ($archive['present'] === true && isset($archive['keys'][$application['key']]) === true); + [ + 'target' => $target, + 'reason' => $reason, + 'sheet' => $sheet, + 'onArchiveSheet' => $onArchiveSheet, + ] = $this->reconcileTarget(key: $application['key'], archive: $archive, operationId: $operationId); $entry = [ - 'sheet' => '', + 'sheet' => $sheet, 'row' => 0, 'appId' => (string)($moduleData['externalNumber'] ?? $application['key']), 'name' => (string)($moduleData['name'] ?? ''), ]; - $target = CmdbImportReport::ARCHIVED; - $reason = self::REASON_PREFIX . 'not in the export (' . $operationId . ')'; - if ($onArchiveSheet === true) { - $entry['sheet'] = $archive['sheet']; - $reason = self::REASON_PREFIX . 'on sheet "' . $archive['sheet'] . '" (' . $operationId . ')'; - } elseif ($archive['present'] === true && $archive['uncachedRows'] === [] && $this->unreadableKeyRows === []) { - // A sheet with an unreadable key (the archive sheet or a CMDB sheet) may list this application, so the - // workbook is not authoritative for a deletion. - $target = CmdbImportReport::DELETED; - } - $uuids = ['module' => (string)$application['module']->getUuid(), 'usage' => (string)$application['usage']->getUuid()]; $step = 'archive'; $changed = false; @@ -2646,6 +2640,47 @@ private function reconcileApplication(array $application, array $archive, string } }//end reconcileApplication() + /** + * Where an application that left the CMDB sheets goes, and the archive reason it gets. + * + * Listed on the archive sheet: archived, with that sheet as the reason. + * Otherwise soft-deleted, but only when the workbook is authoritative for + * a deletion: it has the archive sheet and no sheet has a row whose key + * could not be read. In every other case it is archived. + * + * @param string $key The application key. + * @param ArchiveSheet $archive The archive sheet. + * @param string $operationId The operation id, for the archive reason. + * + * @return array{target: string, reason: string, sheet: string, onArchiveSheet: bool} + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function reconcileTarget(string $key, array $archive, string $operationId): array { + if ($archive['present'] === true && isset($archive['keys'][$key]) === true) { + return [ + 'target' => CmdbImportReport::ARCHIVED, + 'reason' => self::REASON_PREFIX . 'on sheet "' . $archive['sheet'] . '" (' . $operationId . ')', + 'sheet' => $archive['sheet'], + 'onArchiveSheet' => true, + ]; + } + + $target = CmdbImportReport::ARCHIVED; + if ($archive['present'] === true && $archive['uncachedRows'] === [] && $this->unreadableKeyRows === []) { + // A sheet with an unreadable key (the archive sheet or a CMDB sheet) may list this application, so the + // workbook is not authoritative for a deletion. + $target = CmdbImportReport::DELETED; + } + + return [ + 'target' => $target, + 'reason' => self::REASON_PREFIX . 'not in the export (' . $operationId . ')', + 'sheet' => '', + 'onArchiveSheet' => false, + ]; + }//end reconcileTarget() + /** * What one application brings to the state the export asks for: its usage and the other usages of its key, then its module and the other modules. * From 363c470881dbb5ad0d854c7f5e0ada3ee52e0a0f Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Thu, 15 Oct 2026 10:37:37 +0200 Subject: [PATCH 40/41] docs(applications): tag the quick-filter tab tracking with the archived-list requirement The indexPageKey watcher and onQuickFilterChange() keep the Archived-tab note in step with the active tab; gate 16 needs their @spec tag. Co-Authored-By: Claude Opus 5.5 --- src/views/FacetedCatalogIndexView.vue | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/views/FacetedCatalogIndexView.vue b/src/views/FacetedCatalogIndexView.vue index 6cb3f8aec..6ca68fd61 100644 --- a/src/views/FacetedCatalogIndexView.vue +++ b/src/views/FacetedCatalogIndexView.vue @@ -404,6 +404,7 @@ export default { * default tab again, so the tab this view remembers goes back to it. * * @return {void} + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-archived-applications-and-usages-shall-be-listable-in-stackiq-and-shall-stay-out-of-opencatalogi-and-portaliq-by-default-req-cmdb-017 */ indexPageKey() { this.activeQuickFilterIndex = initialQuickFilterIndex(this.quickFilters) @@ -437,6 +438,7 @@ export default { * * @param {number|number[]} index The tab index, or the selected indices * @return {void} + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-archived-applications-and-usages-shall-be-listable-in-stackiq-and-shall-stay-out-of-opencatalogi-and-portaliq-by-default-req-cmdb-017 */ onQuickFilterChange(index) { this.activeQuickFilterIndex = Number.isInteger(index) ? index : null From e0aba3dffefe4cdcb0380cae611cbf3423f63427 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Thu, 15 Oct 2026 11:00:45 +0200 Subject: [PATCH 41/41] style(test): prettier on the archive quick filter spec Co-Authored-By: Claude Opus 5.5 --- tests/vitest/cmdbArchiveQuickFilter.spec.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/vitest/cmdbArchiveQuickFilter.spec.js b/tests/vitest/cmdbArchiveQuickFilter.spec.js index a708a8050..7b2c06cd5 100644 --- a/tests/vitest/cmdbArchiveQuickFilter.spec.js +++ b/tests/vitest/cmdbArchiveQuickFilter.spec.js @@ -150,7 +150,9 @@ describe('the archive quick filter', () => { expect(page._note).toContain('additive AND') expect(page._note).toMatch(/exception is the Archived tab/) - expect(page._note).toMatch(/ignores the GEMMA facets; the search term still applies/) + expect(page._note).toMatch( + /ignores the GEMMA facets; the search term still applies/, + ) }) it('uses an icon the app registers', () => {