From 361ccd3a8c2320966bc8173d7a3a0c1970789d4c Mon Sep 17 00:00:00 2001 From: aarroyo Date: Mon, 3 Aug 2026 15:09:15 -0500 Subject: [PATCH] test(robosoft): the same Core signal blocks, or does not, by tenant configuration Advances CP-10: criteria 3 and 4 closed; 1 partial and 2 unstarted, both said why. `core-sdlc-parity` asserts the property no other robot covers and that four cards were built for: the SAME attached deposit contributes nothing to the verdict with no matrix, contributes `core-signal:architecture` when the tenant marks it blocking, and contributes nothing again when marked advisory. Between the three evaluations nothing about the deposit changed -- only the configuration. VERIFIED FOR REAL, NOT IN CI. The full local stack was brought up (`local-test.sh up`: kind + build + Helm) and the robot ran against it: 27 checks, 0 failures. The nine robots on the CI list ran against that same deployment: 211 checks, 0 failures. I had earlier written this row off as unverifiable. That was not a measurement -- docker, kind, kubectl and helm were all installed and a cluster already existed. The user asked whether I had tried; I had not. ONE TENANT, TWO CONFIGURATIONS -- stated rather than implied. The criterion says "a tenant ... and another"; this runs it as ONE tenant reconfigured between evaluations, because a deposit is scoped by the machine key's tenant and a second one would need another key plus its own seeded initiative. The code path is identical -- `CoreSignalRule` is stored per tenant AND phase -- but what it does NOT prove is isolation between two tenants' matrices, which `tenant-isolation` covers. WHY IT IS NOT ON THE CI LIST: it needs a CoreMachine key bound to the tenant the robot reads as, and `deploy-check.yml` configures none -- the same reason `core-evidence-ingest`, `core-integration` and `runtime-approvals` are excluded. Without the key it SOFT-SKIPS VISIBLY: the summary reports `0 ok`, not a silent PASS. A robot claiming PASS having exercised nothing is the vacuous green this repository keeps finding. A defect in the robot itself, corrected: its first version asserted "the gate is NOT blocked by the deposit" by checking the state was not RETURNED. The gate was RETURNED over an unresolved approval unrelated to the deposit, so the assertion measured something else. It now compares the unmet reasons with and without the matrix: with the signal advisory the gate is EXACTLY as it was with no matrix. Co-Authored-By: Claude Opus 5 --- docs/audit/tracker-gap-reference-catalog.md | 16 +- docs/audit/tracker-gap-tracking.md | 2 +- robosoft/robots/core-sdlc-parity.robot.mjs | 254 ++++++++++++++++++++ 3 files changed, 267 insertions(+), 5 deletions(-) create mode 100644 robosoft/robots/core-sdlc-parity.robot.mjs diff --git a/docs/audit/tracker-gap-reference-catalog.md b/docs/audit/tracker-gap-reference-catalog.md index bcd54ce0..51a575de 100644 --- a/docs/audit/tracker-gap-reference-catalog.md +++ b/docs/audit/tracker-gap-reference-catalog.md @@ -279,10 +279,18 @@ No se reutilizó `CoreEvaluationDeposit`: es la forma de la INGESTA y exige viol - **Criticality:** P1 · **Complexity:** M - **Proposed fix:** Añadir un robot `core-sdlc-parity` que ejecute una iniciativa multi-fase, consuma formatos vivos, produzca artefactos, sincronice/evalúe con Core y afirme decisiones tenant. - **Acceptance criteria:** - - [ ] El robot crea una iniciativa y recorre al menos discovery→design→construction con artefactos Core. - - [ ] Verifica recomendaciones y acciones requeridas visibles para el usuario. - - [ ] Prueba un tenant donde una señal es advisory y otro donde la misma señal bloquea. - - [ ] Emite evidencia JSON enlazable para cerrar los CP. + - [ ] El robot crea una iniciativa y recorre al menos discovery→design→construction con artefactos Core. **Parcial:** `core-sdlc-parity` monta la iniciativa y abre la compuerta de construction; NO recorre discovery→design→construction como jornada, que es lo que `governance-journey` ya cubre por separado. + - [ ] Verifica recomendaciones y acciones requeridas visibles para el usuario. **Sin empezar:** el Core no se despliega en este pipeline, así que no hay recomendaciones reales que enseñar; las del depósito se conservan (CP-05) pero no hay superficie que las muestre (CP-11). + - [x] Prueba un tenant donde una señal es advisory y otro donde la misma señal bloquea — **la aserción que sostiene el robot**, verificada contra un despliegue vivo: el MISMO depósito adjunto no aporta nada al veredicto sin matriz, aporta `core-signal:architecture` cuando el tenant lo marca `blocking`, y vuelve a no aportar nada al marcarlo `advisory`. Entre las tres evaluaciones no cambió nada del depósito: sólo la configuración. + - [x] Emite evidencia JSON enlazable — `robosoft/.evidence/robosoft-*.json`, como todos los robots. + +**Verificado de verdad, no en CI (2026-08-03).** Se levantó el stack local completo (`local-test.sh up`: kind + build + Helm) y el robot corrió contra él: **27 checks, 0 fallos**. Los nueve robots de la lista de CI corrieron contra ese mismo despliegue: **211 checks, 0 fallos**. + +**Una tenencia, dos configuraciones — dicho en vez de insinuado.** El criterio pide «un tenant … y otro»; el robot lo ejecuta como UN tenant reconfigurado entre evaluaciones, porque un depósito está ligado al tenant de la clave de máquina y un segundo tenant necesitaría otra clave más su propia iniciativa sembrada. El camino de código es idéntico —`CoreSignalRule` se guarda por tenant Y fase, así que «la matriz de otro tenant» es la misma búsqueda contra otra fila— pero lo que NO prueba es el aislamiento entre matrices de dos tenants; esa clase la cubre `tenant-isolation`. + +**Por qué NO entra en la lista de CI.** Necesita una clave `CoreMachine` ligada al mismo tenant que lee el robot, y `deploy-check.yml` no configura ninguna — la misma razón por la que `core-evidence-ingest`, `core-integration` y `runtime-approvals` están fuera de `ROBOSOFT_ONLY`. Sin la clave **salta en blando y se ve**: el resumen reporta `0 ok`, no un PASS silencioso. Un robot que dijera PASS sin haber ejercitado nada sería el verde vacuo que este repositorio lleva toda la campaña encontrando. + +**Un fallo del propio robot, corregido:** su primera versión afirmaba «la compuerta NO está bloqueada por el depósito» comprobando que el estado no fuera `RETURNED`. La compuerta estaba `RETURNED` por una aprobación pendiente ajena al depósito, así que la aserción medía otra cosa. Ahora compara los motivos de incumplimiento con y sin matriz: con la señal advisory la compuerta queda EXACTAMENTE como sin matriz. - **Dependencies:** CP-02, CP-03, CP-04, CP-05, CP-06. #### CP-11 diff --git a/docs/audit/tracker-gap-tracking.md b/docs/audit/tracker-gap-tracking.md index bdba4331..e53524f6 100644 --- a/docs/audit/tracker-gap-tracking.md +++ b/docs/audit/tracker-gap-tracking.md @@ -24,7 +24,7 @@ This board is the single source of truth for Tracker technical debt, gaps, oppor | [`CP-04`](./tracker-gap-reference-catalog.md#cp-04) | El contexto que el Tracker envía al Core es demasiado pobre para evaluar artefactos, evidencia, autoría y revisión de repositorio | El Core recibe un gate casi sin insumos tipados y sólo puede emitir un veredicto básico, no inteligencia útil sobre el paquete de fase | Ya viajan tipados `schemaVersion`, `requester` y `repositoryRevision` — este último sale del `passthrough` que el Core no interpreta; y los artefactos de la fase viajan tipados con lo exigido y lo presentado, armados por el servidor; la evidencia de gate sigue siendo otro agregado y no se manda | `Integration` | Cross | P1 | M | `PENDING` | | [`CP-05`](./tracker-gap-reference-catalog.md#cp-05) | El resultado canónico del Core se reduce al mínimo y se pierden recomendaciones, acciones requeridas, señales y trazabilidad | La persona o agente que completa una fase no recibe inteligencia accionable, sólo un pass/fail parcial | `ParseVerdict` conserva `overallVerdict`, `outcome` y `results.gate`; descarta `rulesExecuted`, `policiesApplied`, `gaps`, `risks`, `recommendations`, `requiredActions` y otros result kinds | `Backend/WEB` | Cross | P1 | M | `PENDING` | | [`CP-07`](./tracker-gap-reference-catalog.md#cp-07) | El ingest Core→Tracker ya existe, pero sus depósitos no quedan adjuntos al expediente SDLC de iniciativa, fase o artefacto | Una evaluación producida por CLI, runtime o MCP queda en el ledger técnico, pero no aparece como recomendación o evidencia del gate que el usuario está trabajando | El vínculo tipado ya existe y se adjunta por API con permiso propio, sin copiar ni reinterpretar el veredicto; y el depósito adjunto ya pasa por la matriz de señales del tenant dentro de la decisión de la compuerta; falta la pantalla de fase y la cobertura del robot | `Integration/Governance` | Cross | P1 | M | `PENDING` | -| [`CP-10`](./tracker-gap-reference-catalog.md#cp-10) | No hay robot de paridad SDLC que conduzca una iniciativa y pruebe formatos, evaluación Core, recomendaciones y reglas tenant por fase | Podemos cerrar piezas individuales sin demostrar que juntas sostienen el flujo principal del producto | RoboSoft prueba gobierno, scorecard y Core integration por separado, pero no un journey que use los formatos vivos del Core en cada gate | `Infra/Quality` | Cross | P1 | M | `PENDING` | +| [`CP-10`](./tracker-gap-reference-catalog.md#cp-10) | No hay robot de paridad SDLC que conduzca una iniciativa y pruebe formatos, evaluación Core, recomendaciones y reglas tenant por fase | Podemos cerrar piezas individuales sin demostrar que juntas sostienen el flujo principal del producto | El robot core-sdlc-parity prueba contra un despliegue vivo que la MISMA señal del Core bloquea o no según la matriz del tenant; falta la jornada multi-fase y las recomendaciones visibles, que dependen de desplegar el Core | `Infra/Quality` | Cross | P1 | M | `PENDING` | | [`CP-11`](./tracker-gap-reference-catalog.md#cp-11) | No hay un modelo de interacción advisory que ordene chat, nudges, acciones on-demand y adjuntos al expediente SDLC | La inteligencia consultiva existe en varias puertas, pero el usuario no sabe cuándo hablar con el asistente, cuándo aceptar una sugerencia o cuándo ejecutar una consulta Core | Hay `AssistantPanel` con prompts fijos y algunas pantallas publican contexto; Design y Intake tienen acciones advisory, pero no una matriz de triggers por fase/capacidad/tenant | `WEB/Integration` | Cross | P1 | M | `PENDING` | | [`CP-14`](./tracker-gap-reference-catalog.md#cp-14) | Los artefactos SDLC no tienen versionado documental con comparación, restauración y línea base aprobada | Se puede actualizar un artefacto, pero no queda claro qué cambió, qué versión fue evaluada o cuál quedó aprobada | El historial por artefacto ya existe con autor, fecha, checksum, comparación y restauración, y lo aprobado no se modifica en sitio; y la aprobación de la compuerta sella qué versiones aprobó, con el vínculo del depósito nombrando la versión evaluada | `Backend/Artifacts` | Cross | P1 | M | `PENDING` | | [`CP-16`](./tracker-gap-reference-catalog.md#cp-16) | Falta una bitácora SDLC por iniciativa que una artefactos, ediciones, aprobaciones, agentes, evaluaciones Core y decisiones | El seguimiento queda repartido entre pantallas y logs técnicos, sin una línea de tiempo entendible para auditoría o gestión | La bitácora existe como PROYECCIÓN de auditoría, entregas de compuerta y depósitos adjuntos, filtrable por fase y artefacto y clasificando advisory, cumplido, bloqueo y excepción; con las versiones de artefacto ya enlazadas; sólo faltan los exports, que aún no se registran en ningún sitio | `Governance/Audit` | Cross | P1 | M | `PENDING` | diff --git a/robosoft/robots/core-sdlc-parity.robot.mjs b/robosoft/robots/core-sdlc-parity.robot.mjs new file mode 100644 index 00000000..0210d2ed --- /dev/null +++ b/robosoft/robots/core-sdlc-parity.robot.mjs @@ -0,0 +1,254 @@ +// ----------------------------------------------------------------------------- +// RoboSoft agent · core-sdlc-parity (CP-10) +// +// The property no other robot covers: **the SAME Core signal blocks or does not +// block depending on the tenant's configuration** (CP-06's matrix), fed by a Core +// deposit attached to a phase (CP-07), decided inside the real gate. +// +// It proves end to end, against a live tracker-api, what four cards built: +// CP-05 the canonical result is preserved instead of dying at the ACL +// CP-06 the tenant decides which signals block +// CP-07 a deposit hangs off the SDLC record and runs through that matrix +// CP-02 the phase catalogue the record is contrasted against +// +// WHAT IT NEEDS, AND WHY IT IS NOT IN THE CI LIST +// ---------------------------------------------- +// A CoreMachine key on the deployed tracker-api bound to the SAME tenant this +// robot reads as. `deploy-check.yml` does not configure one — the same reason +// `core-evidence-ingest`, `core-integration` and `runtime-approvals` are kept out +// of `ROBOSOFT_ONLY`. Run it locally with: +// +// kubectl set env deploy/tracker-api-evolith-tracker-api \ +// CoreMachine__Keys__0__TenantId= \ +// CoreMachine__Keys__0__Key= +// ROBOSOFT_CORE_MACHINE_KEY= node robosoft/run.mjs +// +// Without the key it SOFT-SKIPS with the reason. It does not pass quietly: a +// robot that reports PASS when it never exercised anything is the vacuous green +// this repository keeps finding. +// +// ONE TENANT, TWO CONFIGURATIONS — stated rather than implied +// ---------------------------------------------------------- +// CP-10's criterion says "a tenant where a signal is advisory and another where +// the same signal blocks". This runs it as ONE tenant reconfigured between the +// two evaluations, because a deposit is tenant-scoped by the machine key and a +// second tenant would need a second key plus its own seeded initiative. The code +// path is identical — `CoreSignalRule` is stored per tenant AND phase, so "another +// tenant's matrix" is the same lookup against a different row. What this does NOT +// prove is the isolation between two tenants' matrices; `tenant-isolation` covers +// that class separately. +// ----------------------------------------------------------------------------- + +import { config } from '../lib/config.mjs'; + +const RUN = Date.now().toString(36); + +export default { + name: 'core-sdlc-parity', + description: + 'Assert the SAME Core signal blocks or not according to the tenant matrix (CP-06), fed by an attached deposit (CP-07).', + + async run(ctx) { + const { api, step, check, info } = ctx; + + const machineKey = process.env.ROBOSOFT_CORE_MACHINE_KEY; + if (!machineKey) { + check( + 'ROBOSOFT_CORE_MACHINE_KEY is set (the deposit path needs a CoreMachine key)', + false, + { + soft: true, + detail: `not set — bind a key to tenant ${config.tenantId} and re-run. See this file's header.`, + }, + ); + return; + } + + const code = (p) => `${p}-${RUN}`.toUpperCase().slice(0, 20); + const idOf = (r) => r?.body?.id || r?.body?.initiativeId || r?.body; + const need = async (label, p) => { + const r = await p; + if (!check(label, r.ok, { detail: r.error ? r.error : `status=${r.status}` })) { + throw new Error(`precondition failed: ${label} (status=${r.status})`); + } + return r; + }; + + // The deposit route is machine-authenticated, and the shared client cannot send + // an arbitrary header — so this one call goes out raw, with its own key. + const deposit = async (payload) => { + const res = await fetch(`${config.apiBase}${config.apiPrefix}/core-evaluation-transactions`, { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': machineKey }, + body: JSON.stringify(payload), + }); + const raw = await res.text(); + let body = raw; + try { body = raw ? JSON.parse(raw) : null; } catch { /* keep raw */ } + return { ok: res.ok, status: res.status, body }; + }; + + const OWNER = 'po-parity'; + const productCode = code('PAR'); + const correlationId = `parity-${RUN}`; + + // ── 1. An initiative with an OPEN construction gate ────────────────────── + step('Set up an initiative with an open construction gate'); + + await need('PUT /gate-policies/discovery', api.put('/gate-policies/discovery', { + mode: 'SIMPLE', requiresCoreVerdict: false, strategy: 'SINGLE', stages: 'parallel', + requiredEvidence: [], approvers: [], + })); + await need('PUT /gate-policies/construction (no matrix yet)', api.put('/gate-policies/construction', { + mode: 'SIMPLE', requiresCoreVerdict: false, strategy: 'SINGLE', stages: 'parallel', + requiredEvidence: [], approvers: [], coreSignalRules: [], + })); + + const prod = await need('POST /products', api.post('/products', { + code: productCode, name: 'Parity Probe', description: 'CP-10 parity probe.', + })); + const productId = idOf(prod); + await need('POST /products/{id}/owner', api.post(`/products/${productId}/owner`, { ownerIdentity: OWNER })); + + const opp = await need('POST /opportunities', api.post('/opportunities', { + code: code('OPP'), title: 'Parity probe', description: 'CP-10', source: 'stakeholder-request', + problem: 'n/a', expectedValue: 'n/a', sponsor: `sponsor-${productCode}`, + targetProductCode: productCode, priority: 'medium', + })); + const oppId = idOf(opp); + await need('POST /opportunities/{id}/triage', api.post(`/opportunities/${oppId}/triage`, { priority: 'high' })); + const initiativeId = idOf(await need('promote', api.post(`/opportunities/${oppId}/promote`))); + if (!initiativeId) throw new Error('no initiative id'); + + const sub = await need('POST /gate-submissions (construction)', api.post('/gate-submissions', { + initiativeId, phase: 'construction', requestedBy: `construction@${productCode}`, + })); + const subId = idOf(sub); + info(`construction gate ${subId} open on initiative ${initiativeId}`); + + // ── 2. A Core deposit, ATTACHED to that phase ──────────────────────────── + step('Deposit a Core verdict and attach it to the phase'); + + const dep = await deposit({ + schemaVersion: '1.0.0', + correlationId, + producer: { surface: 'cli', version: 'robosoft' }, + evaluatedAt: new Date(0).toISOString(), + overallVerdict: 'FAIL', + outcome: 'rejected', + rulesExecuted: [{ ruleId: 'ARCH-1', engine: 'opa', verdict: 'FAIL' }], + violations: [{ + ruleId: 'ARCH-1', tool: 'evolith', file: 'src/x.cs', severity: 'error', + message: 'capa cruzada', category: 'architecture', fingerprint: `fp-${RUN}`, + }], + accountableOwners: [], + blockingViolationCount: 1, + versions: { core: '1.0.0' }, + }); + + if (!check('POST /core-evaluation-transactions (machine key)', dep.ok, { + detail: `status=${dep.status}. A 401 means the key is not bound to tenant ${config.tenantId}.`, + })) { + throw new Error('the deposit was refused — cannot exercise the matrix'); + } + + const rows = (await need('GET /core-evaluation-transactions', api.get('/core-evaluation-transactions'))).body; + const row = (Array.isArray(rows) ? rows : []).find((r) => r?.correlationId === correlationId); + if (!check('the deposit is readable in the ledger', Boolean(row), { detail: `rows=${rows?.length}` })) { + throw new Error('no ledger row for this run'); + } + + const attached = await need('POST /core-evaluation-transactions/{id}/attachment', + api.post(`/core-evaluation-transactions/${row.id}/attachment`, { + initiativeId, phase: 'construction', artifactKind: 'architecture-review', artifactVersion: 1, + })); + check('the attachment names the phase and the version', + attached.body?.attachment?.phase === 'construction' && attached.body?.attachment?.artifactVersion === 1, { + detail: JSON.stringify(attached.body?.attachment ?? null), + }); + check('attaching did NOT rewrite the verdict', attached.body?.overallVerdict === 'FAIL', { + detail: `overallVerdict=${attached.body?.overallVerdict}`, + }); + + // ── 3. With NO matrix, the signal does not block ───────────────────────── + let sinMatriz = []; + step('With no tenant matrix, the same signal is advisory'); + { + const ev = await need('POST /gate-submissions/{id}/evaluate', api.post(`/gate-submissions/${subId}/evaluate`)); + const unmet = ev.body?.unmet ?? []; + check('no core-signal reason among the unmet', !unmet.some((u) => String(u).startsWith('core-signal:')), { + detail: `unmet=${JSON.stringify(unmet)}`, + }); + // NOT "the gate is approved": this gate has its own unrelated unmet reasons + // (an unresolved approval). What is asserted is that the DEPOSIT contributes + // none of them — conflating the two would make this check pass or fail for + // reasons that have nothing to do with the matrix. + sinMatriz = unmet; + info(`unmet without a matrix: ${JSON.stringify(unmet)}`); + } + + // ── 4. The tenant marks it blocking — the SAME deposit now blocks ──────── + step('The tenant marks `architecture` blocking — the same deposit now blocks'); + { + await need('PUT /gate-policies/construction (blocking matrix)', api.put('/gate-policies/construction', { + mode: 'SIMPLE', requiresCoreVerdict: false, strategy: 'SINGLE', stages: 'parallel', + requiredEvidence: [], approvers: [], + coreSignalRules: [{ signal: 'architecture', disposition: 'blocking' }], + })); + + const ev = await need('POST /gate-submissions/{id}/evaluate', api.post(`/gate-submissions/${subId}/evaluate`)); + const unmet = ev.body?.unmet ?? []; + + // THE load-bearing assertion of this robot. Nothing about the deposit changed + // between step 3 and here — only the tenant's configuration did. + check('the SAME signal now blocks the gate', unmet.includes('core-signal:architecture'), { + detail: `unmet=${JSON.stringify(unmet)}`, + }); + check('the gate is not APPROVED while a blocking signal stands', ev.body?.state !== 'APPROVED', { + detail: `state=${ev.body?.state}`, + }); + } + + // ── 5. Back to advisory — and it stops blocking again ──────────────────── + step('Set the same signal to advisory — it stops blocking'); + { + await need('PUT /gate-policies/construction (advisory matrix)', api.put('/gate-policies/construction', { + mode: 'SIMPLE', requiresCoreVerdict: false, strategy: 'SINGLE', stages: 'parallel', + requiredEvidence: [], approvers: [], + coreSignalRules: [{ signal: 'architecture', disposition: 'advisory' }], + })); + + const ev = await need('POST /gate-submissions/{id}/evaluate', api.post(`/gate-submissions/${subId}/evaluate`)); + const unmet = ev.body?.unmet ?? []; + check('advisory does NOT block', !unmet.includes('core-signal:architecture'), { + detail: `unmet=${JSON.stringify(unmet)}`, + }); + // The sharpest form of the property: with the signal advisory the gate is in + // EXACTLY the state it was in with no matrix at all. The deposit added + // nothing and removed nothing — only the configuration ever changed. + check('advisory leaves the gate exactly as it was with no matrix', + JSON.stringify([...unmet].sort()) === JSON.stringify([...sinMatriz].sort()), { + detail: `sin matriz=${JSON.stringify(sinMatriz)} advisory=${JSON.stringify(unmet)}`, + }); + } + + // ── 6. The journey is readable in the ledger (CP-16) ───────────────────── + step('The initiative ledger shows the deposit and its phase'); + { + const led = await need('GET /initiatives/{id}/ledger', + api.get(`/initiatives/${initiativeId}/ledger?phase=construction`)); + const entries = led.body?.entries ?? []; + check('the attached deposit appears in the ledger', + entries.some((e) => e.source === 'core-deposit' && e.phase === 'construction'), { + detail: `entries=${entries.length}`, + }); + check('the ledger classifies what blocked and what only advised', + entries.every((e) => typeof e.kind === 'string' && e.kind.length > 0), { + detail: `kinds=${JSON.stringify([...new Set(entries.map((e) => e.kind))])}`, + }); + } + + step('parity verified'); + info(`the same deposit blocked and did not block on tenant ${config.tenantId}, by configuration alone`); + }, +};