From a49e929743c533d6a19949589b83a0ff9ab4356b Mon Sep 17 00:00:00 2001 From: davidamacey Date: Thu, 15 Oct 2026 02:39:15 -0400 Subject: [PATCH 1/5] chore(codegen): fail comments that cite a file that is not in the tree Adds check_cited_paths.py (pre-commit, shrink-only baseline for the frontend citations of design notes that left the tree) and rewords the 30 citations outside the frontend. --- .pre-commit-config.yaml | 7 + CONTRIBUTING.md | 1 + docker/trainer/job_protocol.py | 2 +- docker/trainer/job_spec.py | 6 +- docker/trainer/trainer.py | 4 +- export/preflight.py | 2 +- scripts/codegen/baselines/cited_paths.txt | 193 ++++++++++++++++++ scripts/codegen/check_cited_paths.py | 95 +++++++++ scripts/curation/bakeoff/bakeoff_runner.py | 2 +- scripts/curation/worker/fairness.py | 2 +- scripts/lib/model_setup.sh | 2 +- scripts/release/build_and_publish.sh | 5 +- setup-openprocessor.sh | 2 +- src/config/curation.py | 2 +- src/config/curation_accessors.py | 2 +- src/config/project_context.py | 2 +- src/config/projects.py | 2 +- src/routers/curation/_mounting.py | 2 +- src/routers/curation/bakeoff.py | 2 +- .../curation/dataset_import/__init__.py | 2 +- src/services/projects/__init__.py | 2 +- src/services/projects/busy.py | 3 +- src/services/projects/capacity.py | 2 +- src/services/projects/guard.py | 2 +- src/services/projects/lifecycle.py | 2 +- src/services/projects/registry.py | 2 +- src/services/training/job_models.py | 2 +- src/services/training/triton_promote.py | 2 +- tests/codegen/test_check_cited_paths.py | 62 ++++++ tests/curation/test_probe_models.py | 2 +- tests/curation/test_region_fields.py | 3 +- .../test_region_merge_concurrent_edit.py | 3 +- .../test_region_not_visible_terminal.py | 3 +- tests/curation/test_region_write_occ.py | 3 +- .../curation/test_search_no_legacy_tokens.py | 4 +- tests/projects/test_config_view.py | 2 +- tests/projects/test_project_context.py | 2 +- .../test_export_hints_reference_real_paths.py | 3 +- 38 files changed, 395 insertions(+), 46 deletions(-) create mode 100644 scripts/codegen/baselines/cited_paths.txt create mode 100755 scripts/codegen/check_cited_paths.py create mode 100644 tests/codegen/test_check_cited_paths.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5124f13d4..9394fd628 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -139,6 +139,13 @@ repos: pass_filenames: false always_run: true + - id: check-cited-paths + name: Comments and docstrings cite files that exist + entry: python3 scripts/codegen/check_cited_paths.py + language: system + pass_filenames: false + always_run: true + - id: check-vulture name: No new unused code (vulture) entry: bash -c 'py=python3; [ -x .venv/bin/python ] && py=.venv/bin/python; exec "$py" scripts/codegen/check_vulture.py' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index df4008fdc..d4f539820 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -132,6 +132,7 @@ Never add a line to hide a new finding; fix it. | `check_layering.py` | routers import services and clients, never the reverse; clients never import services; `src/domain`, `config`, `utils` and `schemas` import neither; only routers import `fastapi`. No baseline: the one permanent exception (the guarded OpenSearch client factory) is listed with its reason in the script | | `check_ruff_ratchet.py` | no function above complexity 15, no new `except Exception` | | `check_process_ids.py` | no wave, round or work-package ids, and no history names, in comments, docstrings or file names | +| `check_cited_paths.py` | a comment or docstring that cites a `docs/`, `scripts/` or similar file cites one that exists | | `check_vulture.py` | no unused code; a reviewed false positive goes in `vulture_whitelist.py` with a reason | | `check_naming_leaks.py` | no private product or domain names in the tracked tree | | ruff `TID251` | `src/` reads the environment only through `src/config/env.py` (`env_str`, `env_int`, `env_float`, `env_bool`, `env_path`); a malformed value raises naming the variable | diff --git a/docker/trainer/job_protocol.py b/docker/trainer/job_protocol.py index 09c53e387..594188fac 100644 --- a/docker/trainer/job_protocol.py +++ b/docker/trainer/job_protocol.py @@ -102,7 +102,7 @@ def _is_terminal(status_path: Path) -> bool: def list_pending_jobs(jobs_dir: Path) -> list[Path]: """Find ``*.job.json`` files that haven't reached a terminal state, across every project's own dir under ``/projects/`` - (docs/design/openprocessor_internal/projects_plan.md §5.3; + (the project-isolation design §5.3; P1R §6.1/D-A: ``default`` nests the same as every other project -- see ``src.config.projects.resources_for_new``). The top-level ``jobs_dir`` glob is kept for a job file dropped directly there by diff --git a/docker/trainer/job_spec.py b/docker/trainer/job_spec.py index 4abb29e48..92dbefad4 100644 --- a/docker/trainer/job_spec.py +++ b/docker/trainer/job_spec.py @@ -71,7 +71,7 @@ class JobSpec: augmentation: dict[str, Any] mlflow_run_name: str submitted_at: str - # Project isolation (docs/design/openprocessor_internal/projects_plan.md + # Project isolation (the project-isolation design # §5.3). ``project`` names which project submitted the run; # ``project_export_root`` gates ``dataset_export_dir`` -- a job whose # export escapes its own project's export tree is refused rather than @@ -166,8 +166,8 @@ class JobValidationError(ValueError): class ExportOutsideProjectError(JobValidationError): """``dataset_export_dir`` is not a subpath of the job's own - ``project_export_root`` (docs/design/openprocessor_internal/ - projects_plan.md §5.3) -- refused rather than trained, since it would + ``project_export_root`` (the project-isolation design, §5.3) + -- refused rather than trained, since it would otherwise let one project's training run read another project's dataset.""" diff --git a/docker/trainer/trainer.py b/docker/trainer/trainer.py index 6e23eb0fe..70ad5e9a4 100644 --- a/docker/trainer/trainer.py +++ b/docker/trainer/trainer.py @@ -184,8 +184,8 @@ def run_job(spec: JobSpec) -> None: 'MLFLOW_TRACKING_URI', os.environ.get('MLFLOW_TRACKING_URI', 'http://mlflow:5000') ) os.environ['MLFLOW_RUN'] = spec.mlflow_run_name - # The job's own project experiment (docs/design/ - # openprocessor_internal/projects_plan.md §5.3) takes priority + # The job's own project experiment (the project-isolation design, + # §5.3) takes priority # over any env-var-only experiment name -- one trainer process # serves every project's jobs, so a stale process-wide env var # must never leak one project's runs into another's experiment. diff --git a/export/preflight.py b/export/preflight.py index 5341d0796..8eda4bc68 100644 --- a/export/preflight.py +++ b/export/preflight.py @@ -1,6 +1,6 @@ """Image self-check run once, right after pull, before any export step. -Installer plan (docs/design/openprocessor_internal/one_line_installer_plan.md +Installer plan (the installer design section 3.2): a stale ``:latest`` API image once shipped without ``perception_models`` importable. This script is invoked as diff --git a/scripts/codegen/baselines/cited_paths.txt b/scripts/codegen/baselines/cited_paths.txt new file mode 100644 index 000000000..cba47897f --- /dev/null +++ b/scripts/codegen/baselines/cited_paths.txt @@ -0,0 +1,193 @@ +frontend/e2e/conftest.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/fixtures/wire.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/harness/stub.py 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/live/test_data_agreement.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_assist_scope.py 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_bakeoff.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_cluster_discard.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_cluster_move_undo.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_clusters_display_order_representatives.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_curation_settings.py 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_dashboard_stats.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_detector_seed.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_ingest.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_ingest_policy.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_keymap.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_labeling_flow.py 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_no_region_profile.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_region_stage.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_review_actions.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_review_crop_viewport.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_review_deep_link_locate.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_source_overlay.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_tier2_profile.py 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_train_gpus.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_visual_audit_review.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_visual_fix_pages_narrow.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/e2e/stubbed/test_vlm_job_poll_by_id.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/scripts/capture_docs_screenshots.py 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/scripts/docs_screenshot_fixtures.py 4 cites a design note that left the tree; drop the citation when the file is next touched +frontend/scripts/repro_chromium_network_change.py 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/scripts/run-e2e.mjs 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/annotations/cohorts.ts 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/annotations/config/allowLists.ts 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/annotations/config/parseSlotConfig.ts 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/annotations/config/roundTrip.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/annotations/cropSlots.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/annotations/datasetExport.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/annotations/multiBox.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/annotations/profiles.falsification.test.ts 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/annotations/readSlot.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/annotations/registeredSlots.noProfile.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/annotations/registeredSlots.ts 3 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/annotations/registry.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/annotations/secondSlotIntegration.test.ts 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/annotations/servedRegionSlot.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/annotations/servedRegionSlot.ts 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/annotations/types.ts 3 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/api.autoLabel.test.ts 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/api.clusters.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/api.ingest.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/api.mapRawCrop.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/api.settings.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/api.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/apiPrefixScan.test.ts 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/api_autoLabel.ts 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/api_clusters.ts 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/api_cropMap.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/api_ingest.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/api_models.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/api_scores.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/api_settings.ts 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/api_viz.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/assistScope.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/bakeoff/bakeoffController.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/classPicker.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/clusters/clusterController.discardUndo.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/clusters/clusterController.move.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/clusters/clusterController.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/clusters/clusterController.vlm.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/clusters/clusterDetailController.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/clusters/clusterListController.svelte.ts 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/clusters/displayOrderRepresentatives.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/combine/combineWizardController.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/AssistScopeBar.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/AutoLabelPanel.gate.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/AutoLabelPanel.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/AutoLabelPanel.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/CropCard.svelte 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/CropCard.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/CropMetaPanel.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/CropMetaPanel.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/CropMetaPanel.visualAudit.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/DatasetStats.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/DatasetStats.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/EmbeddingPlot.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/ProvenanceChip.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/ScoreChip.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/ScoresCard.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/SlotBboxEditor.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/SlotCard.svelte 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/SlotCard.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/SourceImageOverlay.projectCache.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/SourceImageOverlay.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/SourceImageOverlay.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/StrategyBar.appliedSort.test.ts 3 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/StrategyBar.svelte 3 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/TrainForm.gpuPicker.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/classes/ClassRegistryTable.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/cropMeta/CropMetaSlots.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/export/ExportClassTable.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/ingest/ClusteringHandoff.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/ingest/IngestBatchPanel.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/ingest/IngestBatchPanel.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/ingest/IngestDropZone.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/ingest/IngestRunPanel.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/ingest/IngestStatusTable.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/ingest/RegionDrainPanel.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/review/ReviewItemMeta.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/review/ReviewSlotPanel.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/settings/KeymapCard.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/settings/KeymapCard.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/slots/SlotGallery.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/components/slots/SlotGallery.unclustered.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/contract/endpointCatalog.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/contract/ingestContract.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/contract/wireKeys.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/curationSettings.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/dashboard/classBalance.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/datasets/importWizardController.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/domainNeutral.scan.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/embeddingPlot.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/ingest/fileSource.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/ingest/ingestConfig.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/ingest/ingestResults.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/ingest/ingestRunController.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/ingest/uploadPlanner.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/modelUnload.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/modelUnload.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/packs/packEditorController.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/packs/packListController.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/profiles/profileActive.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/profiles/profileEditorController.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/profiles/profileFields.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/profiles/profileListController.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/projectChange.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/review/abortRegistry.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/review/cropDisplaySize.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/review/reviewController.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/review/reviewController.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/review/reviewCopy.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/review/reviewQueueLoader.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/review/reviewSlotActions.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/review/reviewSlotEditController.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/review/reviewTabsController.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/review/slotKeymap.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/review/slotPanel.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/review/slotQueueOps.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/review/slotTabGuard.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/reviewTabs.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/scores.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/slots/slotGalleryController.saveBox.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/slots/slotGalleryController.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/sse.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/stores/keyboard.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/stores/projects.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/stores/regionProfile.svelte.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/stores/regionProfile.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/strategies.ts 4 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/strategyBar.svelte.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/strategyBar.svelte.ts 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/test/fixtures/aircraftTailNumberSlot.ts 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/test/fixtures/defectCodeSlot.ts 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/test/fixtures/methodsPayloads.ts 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/test/makeItem.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/testing/sourceScan.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/train/trainSingleClassExport.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/types.ts 6 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/types_combine.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/types_ingest.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/types_itemFilter.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/types_models.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/types_projects.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/types_review.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/vlm/vlmActive.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/vlm/vlmEndpointCreateController.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/vlm/vlmEndpointEditorController.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/vlm/vlmFields.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/lib/vlm/vlmModelsController.svelte.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/routes/p/[project]/+layout.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/routes/p/[project]/bakeoff/+page.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/routes/p/[project]/classes/visualAudit.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/routes/p/[project]/clusters/+page.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/routes/p/[project]/clusters/[id]/clusterMoveRace.test.ts 3 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/routes/p/[project]/clusters/dqP2GalleryFooter.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/routes/p/[project]/datasets/imports/[id]/+page.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/routes/p/[project]/ingest/+page.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/routes/p/[project]/models/+page.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/routes/p/[project]/review/slotReviewCharacterization.test.ts 2 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/routes/p/[project]/settings/+page.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/routes/p/[project]/settings/models/new-endpoint/+page.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/routes/p/[project]/settings/models/vlm/[name]/+page.svelte 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/routes/p/[project]/settings/settingsPage.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched +frontend/src/routes/p/[project]/train/datasetExportGate.test.ts 1 cites a design note that left the tree; drop the citation when the file is next touched diff --git a/scripts/codegen/check_cited_paths.py b/scripts/codegen/check_cited_paths.py new file mode 100755 index 000000000..7a4b039ce --- /dev/null +++ b/scripts/codegen/check_cited_paths.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Pre-commit guard: a comment or docstring that cites a repo file cites one that exists. + +A citation such as "see the plan in the design directory" goes stale when the document is +renamed or removed, and then points a reader at nothing. This reads the comments and +docstrings the process-id guard reads (same scope) and fails on a cited path that +starts with a top-level documentation or tooling directory (``docs``, ``scripts``, +``examples``, ``docker``, ``export``, ``config_templates``, ``monitoring``, +``contracts``) and is not a file in the tree. Globs, ```` and URLs are +not paths and are skipped. + +Today's hits live in ``baselines/cited_paths.txt`` (``pathhitsreason``) +and can only shrink (see ``_baseline.py``). +""" + +from __future__ import annotations + +import argparse +import re +import sys +import tokenize +from pathlib import Path + + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _baseline +import check_process_ids as ids + + +REPO_ROOT = Path(__file__).resolve().parents[2] +BASELINE_PATH = REPO_ROOT / 'scripts' / 'codegen' / 'baselines' / 'cited_paths.txt' + +CITATION = re.compile( + r'(? int: + source = (repo_root / rel).read_text(encoding='utf-8') + texts = ( + ids._comment_texts_python(source, rel) + if rel.endswith('.py') + else ids._comment_texts_other(source, rel) + ) + return sum( + 1 for text in texts for cited in CITATION.findall(text) if not (repo_root / cited).is_file() + ) + + +def scan(repo_root: Path) -> tuple[dict[str, int], list[str]]: + found: dict[str, int] = {} + broken: list[str] = [] + for rel in ids.tracked_files(repo_root): + if not ids.in_content_scope(rel): + continue + try: + n = missing_citations(repo_root, rel) + except (SyntaxError, tokenize.TokenError, UnicodeDecodeError, OSError): + broken.append(rel) + continue + if n: + found[rel] = n + return found, broken + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--baseline', type=Path, default=BASELINE_PATH) + parser.add_argument('--tighten', action='store_true', help='Lower baseline values, drop stale.') + parser.add_argument('--root', type=Path, default=REPO_ROOT) + args = parser.parse_args(argv) + + found, broken = scan(args.root) + baseline = _baseline.load(args.baseline) + errors = [f'{rel}: cannot be read, so its comments are unchecked' for rel in broken] + errors += _baseline.compare(found, baseline, unit='missing citation(s)') + if args.tighten: + changed = _baseline.tighten(args.baseline, found, baseline) + sys.stderr.write(f'tightened {changed} baseline line(s)\n') + errors = [e for e in errors if 'new violation' in e or 'grew past' in e or 'read' in e] + if errors: + sys.stderr.write( + 'ERROR: a comment cites a file that is not in the tree. Cite the file that exists, ' + 'or say what the comment needs to say in words.\n' + ) + for e in errors: + sys.stderr.write(f' {e}\n') + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/curation/bakeoff/bakeoff_runner.py b/scripts/curation/bakeoff/bakeoff_runner.py index db431e729..ec7ee2e08 100644 --- a/scripts/curation/bakeoff/bakeoff_runner.py +++ b/scripts/curation/bakeoff/bakeoff_runner.py @@ -383,7 +383,7 @@ def finish(state: str, error: str | None = None) -> dict[str, Any]: def _pending_job_files(jobs_dir: Path) -> list[Path]: """``*.job.json`` in ``jobs_dir`` plus every project's own bake-off job - dir (docs/design/openprocessor_internal/projects_plan.md §5.3). + dir (the project-isolation design §5.3). The API writes every project's jobs, ``default`` included, to ``/projects//bakeoff_jobs`` diff --git a/scripts/curation/worker/fairness.py b/scripts/curation/worker/fairness.py index df8b04cda..0e155ce08 100644 --- a/scripts/curation/worker/fairness.py +++ b/scripts/curation/worker/fairness.py @@ -1,7 +1,7 @@ """Multi-project fairness scheduling for the detection worker (``scripts/curation/worker/runner.py`` and its stage modules). -See ``docs/design/openprocessor_internal/projects_plan.md`` §5.1. +See the project-isolation design §5.1. This module is deliberately self-contained and asyncio-free at the top level so the deficit round-robin math is directly unit-testable diff --git a/scripts/lib/model_setup.sh b/scripts/lib/model_setup.sh index 481b07b34..f3582246c 100755 --- a/scripts/lib/model_setup.sh +++ b/scripts/lib/model_setup.sh @@ -4,7 +4,7 @@ # ============================================================================= # Shared by setup-openprocessor.sh and the `openprocessor models` CLI. One # implementation, per the installer plan -# (docs/design/openprocessor_internal/one_line_installer_plan.md section 4). +# (the installer design section 4). # # Every compose invocation here goes through the caller's dc() wrapper; # this file never calls `docker compose` directly diff --git a/scripts/release/build_and_publish.sh b/scripts/release/build_and_publish.sh index 73aa0d0a1..d46c4ba1a 100755 --- a/scripts/release/build_and_publish.sh +++ b/scripts/release/build_and_publish.sh @@ -1,7 +1,7 @@ #!/bin/bash # Local release script for OpenProcessor's published images. # -# Owner decision (docs/design/openprocessor_internal/one_line_installer_plan.md +# Owner decision (the installer design # §11.1 item 2): no CI image builds. This script, run by hand on this host # (or via `make release`), builds every published image with the Docker # build cache, gates on a Trivy CRITICAL scan, and -- only with --push -- @@ -15,8 +15,7 @@ # It never touches release-manifest.txt: that is the installer's file list, # read by scripts/release/build_deploy_bundle.sh. # -# Modeled on Cropwright's local multi-arch release script -# (scripts/release.sh and scripts/release/*.sh in that repo), trimmed to +# Modeled on Cropwright's local multi-arch release scripts, trimmed to # this project's single-arch (linux/amd64, GPU) image set. # # Usage: diff --git a/setup-openprocessor.sh b/setup-openprocessor.sh index 35089e9d6..9f98bc084 100755 --- a/setup-openprocessor.sh +++ b/setup-openprocessor.sh @@ -8,7 +8,7 @@ # # Installs a pinned OpenProcessor release into ./openprocessor/ with no git # clone, no local image build and no host Python. Design: -# docs/design/openprocessor_internal/one_line_installer_plan.md +# the installer design # # The whole script is one function (__op_define) that only defines things, # followed by a single line that calls it and then main. A truncated diff --git a/src/config/curation.py b/src/config/curation.py index c1b2c1b6b..d196c9074 100644 --- a/src/config/curation.py +++ b/src/config/curation.py @@ -243,7 +243,7 @@ class CurationConfig: # --- Project-scoped fields (see ``PROJECT_SCOPED_FIELDS`` in # ``src.config.curation_accessors`` and - # docs/design/openprocessor_internal/projects_plan.md §2.2/§3.3). + # the project-isolation design §2.2/§3.3). # No env var sets any project-scoped field: ``get_curation_config()`` # resolves them from the bound project's persisted resources. The # defaults only shape an explicitly constructed ``CurationConfig`` diff --git a/src/config/curation_accessors.py b/src/config/curation_accessors.py index abb4bf3f1..e157f9cfb 100644 --- a/src/config/curation_accessors.py +++ b/src/config/curation_accessors.py @@ -16,7 +16,7 @@ # Fields resolved from the *bound project*'s ``ProjectResources`` rather # than the process-global env-built instance (see -# docs/design/openprocessor_internal/projects_plan.md §2.2/§3.3). Every +# the project-isolation design §2.2/§3.3). Every # other ``CurationConfig`` field is global. ``test_config_view.py`` # fails if a new dataclass field is added to neither this set nor # treated as global -- keep it in sync with the dataclass above. diff --git a/src/config/project_context.py b/src/config/project_context.py index 0431dab3d..7ff9aed5f 100644 --- a/src/config/project_context.py +++ b/src/config/project_context.py @@ -1,5 +1,5 @@ """The bound-project request context (see -``docs/design/openprocessor_internal/projects_plan.md`` §3.3). +the project-isolation design §3.3). A project is *always* bound while curation code runs -- there is no ``if project is None`` branch anywhere. Unbound access raises diff --git a/src/config/projects.py b/src/config/projects.py index 436a6db3e..0f9ed31e7 100644 --- a/src/config/projects.py +++ b/src/config/projects.py @@ -1,5 +1,5 @@ """Project resource naming and records (see -``docs/design/openprocessor_internal/projects_plan.md`` §2.2). +the project-isolation design §2.2). A *project* is a named, isolated dataset workspace. ``default`` is an ordinary project, created at bootstrap with the same naming as every diff --git a/src/routers/curation/_mounting.py b/src/routers/curation/_mounting.py index 37cc18635..4f8f38a25 100644 --- a/src/routers/curation/_mounting.py +++ b/src/routers/curation/_mounting.py @@ -1,5 +1,5 @@ """Mount the curation routers (see -``docs/design/openprocessor_internal/projects_plan.md`` §3.2 and its +the project-isolation design §3.2 and its "fresh build, no backwards compatibility" owner decision). Every curation route lives under ``{api_prefix}/projects/{project}`` and diff --git a/src/routers/curation/bakeoff.py b/src/routers/curation/bakeoff.py index 9d2bf5270..74fb87ff5 100644 --- a/src/routers/curation/bakeoff.py +++ b/src/routers/curation/bakeoff.py @@ -75,7 +75,7 @@ logger = get_logger(__name__) -# Project-scoped (docs/design/openprocessor_internal/projects_plan.md +# Project-scoped (the project-isolation design # §5.3): each project's bake-off jobs/outputs live under its own # ``CurationConfig.bakeoff_jobs_dir`` (a PROJECT_SCOPED_FIELDS entry), not # a single global dir, so no project's queue or results leak into diff --git a/src/services/curation/dataset_import/__init__.py b/src/services/curation/dataset_import/__init__.py index ba035bd2e..0260d3c91 100644 --- a/src/services/curation/dataset_import/__init__.py +++ b/src/services/curation/dataset_import/__init__.py @@ -1,7 +1,7 @@ """Dataset import: bring an already-labeled dataset (YOLO or COCO format) into a project, mapping its classes onto the registry by name (never by index — see the class-identity invariant in -``docs/design/openprocessor_internal/any_domain_plan.md``). +the any-domain design). Submodules: diff --git a/src/services/projects/__init__.py b/src/services/projects/__init__.py index 45d027b29..d10b6b928 100644 --- a/src/services/projects/__init__.py +++ b/src/services/projects/__init__.py @@ -1,7 +1,7 @@ """Project registry: the ``op_projects`` index, its in-process snapshot, and startup bootstrap of the ``default`` project record. -See ``docs/design/openprocessor_internal/projects_plan.md`` §2/§4. +See the project-isolation design §2/§4. """ from __future__ import annotations diff --git a/src/services/projects/busy.py b/src/services/projects/busy.py index ef4b27a5c..c91cb5000 100644 --- a/src/services/projects/busy.py +++ b/src/services/projects/busy.py @@ -1,5 +1,4 @@ -"""Per-project job inventory (docs/design/openprocessor_internal/ -projects_plan.md §5.4): what's currently running for one project, across +"""Per-project job inventory (the project-isolation design, §5.4): what's currently running for one project, across every job-producing subsystem. Each source is its own small function so a unit test can fake one at a diff --git a/src/services/projects/capacity.py b/src/services/projects/capacity.py index d99db0a54..94c7defd6 100644 --- a/src/services/projects/capacity.py +++ b/src/services/projects/capacity.py @@ -1,6 +1,6 @@ """Read-only OpenSearch shard/heap capacity check, served on ``GET /projects`` so Cropwright can show it before create (see -docs/design/openprocessor_internal/projects_plan.md §2.3). +the project-isolation design §2.3). P1 only serves this; P3 adds the create-time 409 enforcement. There is no fixed project-count cap (owner decision D4) -- the real limits are diff --git a/src/services/projects/guard.py b/src/services/projects/guard.py index d4a5adee9..9fc42efb1 100644 --- a/src/services/projects/guard.py +++ b/src/services/projects/guard.py @@ -1,7 +1,7 @@ """The OpenSearch project guard: a transport-level wrapper that refuses every request that is not one of the shapes the codebase really sends, aimed at the bound project's concrete index names (see -``docs/design/openprocessor_internal/projects_plan.md`` §2.4). +the project-isolation design §2.4). It fails **closed by construction**: there is no list of bad shapes to keep up to date. A request passes only if it matches an allowlisted shape diff --git a/src/services/projects/lifecycle.py b/src/services/projects/lifecycle.py index 7f951eccf..5db4f911c 100644 --- a/src/services/projects/lifecycle.py +++ b/src/services/projects/lifecycle.py @@ -1,7 +1,7 @@ """Project lifecycle service: create / patch / archive / unarchive / clone_settings / delete (dry-run and guarded). -See ``docs/design/openprocessor_internal/projects_plan.md`` §4, §7, the +See the project-isolation design §4, §7, the owner decisions in §13 (D4 capacity, D5 ``default`` is undeletable), and the Cropwright rev-3 deltas 2 (envelope + revision), 3 (list membership, implemented in P1's projects wire models), 10 (delete fits a proxy diff --git a/src/services/projects/registry.py b/src/services/projects/registry.py index 732f1bd8f..f7cf20483 100644 --- a/src/services/projects/registry.py +++ b/src/services/projects/registry.py @@ -2,7 +2,7 @@ counter, and an in-process snapshot refreshed on a cheap poll (one GET of the counter; a ``_search`` only when it changed). -See ``docs/design/openprocessor_internal/projects_plan.md`` §4. +See the project-isolation design §4. """ from __future__ import annotations diff --git a/src/services/training/job_models.py b/src/services/training/job_models.py index 354d5fb73..0761aea7a 100644 --- a/src/services/training/job_models.py +++ b/src/services/training/job_models.py @@ -189,7 +189,7 @@ class TrainJobSpec(BaseModel): registry_sha: str | None = None registry_snapshot_path: str | None = None - # Project isolation (docs/design/openprocessor_internal/projects_plan.md + # Project isolation (the project-isolation design # §5.3) -- filled in by write_job() from the bound project, never set by # API callers directly. ``project_export_root`` lets the trainer refuse # a job whose ``dataset_export_dir`` escapes the project's own export diff --git a/src/services/training/triton_promote.py b/src/services/training/triton_promote.py index a7571bc85..83d468e1b 100644 --- a/src/services/training/triton_promote.py +++ b/src/services/training/triton_promote.py @@ -283,7 +283,7 @@ async def promote( 'triton_name': triton_name, 'version': str(next_version), 'promoted_at': datetime.now(tz=UTC).isoformat(), - # docs/design/openprocessor_internal/projects_plan.md §5.3: the + # the project-isolation design §5.3: the # project that submitted this promote, so an operator (or # DELETE /models/{name}'s ownership check) can trace a served # model back to its owning project without re-deriving it from diff --git a/tests/codegen/test_check_cited_paths.py b/tests/codegen/test_check_cited_paths.py new file mode 100644 index 000000000..a04371725 --- /dev/null +++ b/tests/codegen/test_check_cited_paths.py @@ -0,0 +1,62 @@ +"""check_cited_paths.py: a comment or docstring that cites a repo file must cite one that exists.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import check_cited_paths as guard +import pytest + + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + from conftest import ScratchRepo + + +def run(repo: ScratchRepo, baseline: Path, *extra: str) -> int: + return guard.main(['--root', str(repo.root), '--baseline', str(baseline), *extra]) + + +@pytest.mark.parametrize( + ('path', 'source'), + [ + ('src/a.py', '# see docs/design/gone.md for why\nx = 1\n'), + ('src/a.py', '"""Mirrors scripts/lib/missing.sh."""\n'), + ('scripts/a.sh', '# see docker/nowhere/x.py\necho hi\n'), + ('frontend/src/lib/a.ts', '// see docs/design/gone-plan.md\nexport const a = 1;\n'), + ], +) +def test_a_citation_of_a_missing_file_fails( + scratch_repo: ScratchRepo, baseline_file: Callable[[str], Path], path: str, source: str +) -> None: + scratch_repo.write(path, source) + assert run(scratch_repo, baseline_file('')) == 1 + + +@pytest.mark.parametrize( + ('path', 'source'), + [ + ('src/a.py', '# see docs/real.md\nx = 1\n'), + ('src/a.py', "x = 'docs/design/gone.md is data, not a comment'\n"), + ('src/a.py', '# a glob like docs/design/*.md or docs/.md is not a path\n'), + ('src/a.py', '# see https://example.test/docs/design/gone.md\n'), + ], +) +def test_real_paths_data_and_globs_pass( + scratch_repo: ScratchRepo, baseline_file: Callable[[str], Path], path: str, source: str +) -> None: + scratch_repo.write('docs/real.md', '# real\n') + scratch_repo.write(path, source) + assert run(scratch_repo, baseline_file('')) == 0 + + +def test_a_baselined_citation_passes_and_a_fixed_one_must_be_removed( + scratch_repo: ScratchRepo, baseline_file: Callable[[str], Path] +) -> None: + scratch_repo.write('src/a.py', '# see docs/design/gone.md\n') + base = baseline_file('src/a.py\t1\tcites a plan that left the tree\n') + assert run(scratch_repo, base) == 0 + scratch_repo.write('src/a.py', '# no citation now\n') + assert run(scratch_repo, base) == 1 diff --git a/tests/curation/test_probe_models.py b/tests/curation/test_probe_models.py index feb16c72f..005328b3c 100644 --- a/tests/curation/test_probe_models.py +++ b/tests/curation/test_probe_models.py @@ -3,7 +3,7 @@ The registry is append-only and gapped by design (deprecated/merged classes keep their old id, per the class identity invariant in -``docs/design/openprocessor_internal/any_domain_plan.md``). A checkpoint's +the any-domain design). A checkpoint's dense output index 0..nc-1 has no relationship to those ids -- only to the checkpoint's own embedded ``names`` metadata. These tests pin that: a gapped registry must not affect which name a given model output index diff --git a/tests/curation/test_region_fields.py b/tests/curation/test_region_fields.py index 62fe9ce32..7c4195192 100644 --- a/tests/curation/test_region_fields.py +++ b/tests/curation/test_region_fields.py @@ -6,8 +6,7 @@ ``RegionFields`` instance) is covered separately, alongside ``curation_opensearch/bodies_core.py`` (and its mapping builder). -This file is one of the two hardcoded exemptions in -``scripts/codegen/check_no_literal_region_fields.py`` — it legitimately +This file legitimately constructs a ``roi_*``-named instance to prove overridability without a reindex. """ diff --git a/tests/curation/test_region_merge_concurrent_edit.py b/tests/curation/test_region_merge_concurrent_edit.py index 0aee607c2..6f7645a91 100644 --- a/tests/curation/test_region_merge_concurrent_edit.py +++ b/tests/curation/test_region_merge_concurrent_edit.py @@ -2,8 +2,7 @@ its own VLM re-verification call must never be silently reverted or resurrected by that pass's now-stale merge. -Reproduces the review's exact probes (docs/design/openprocessor_internal/ -w8_pipeline_review_2026-09-27.md, "Re-review 2026-09-27" section, finding +Reproduces the review's exact probes (the 2026-09-27 pipeline review, "Re-review 2026-09-27" section, finding R-M3): the B1 fix's ``merge_boxes_for_write`` only guarded the ITEM-level ``region_revision`` (via the status staleness check in ``bulk_writer._merge``), never the specific box a pass's own verdict diff --git a/tests/curation/test_region_not_visible_terminal.py b/tests/curation/test_region_not_visible_terminal.py index 35c3255c8..1e57b397f 100644 --- a/tests/curation/test_region_not_visible_terminal.py +++ b/tests/curation/test_region_not_visible_terminal.py @@ -1,8 +1,7 @@ """R-B1 regression: a re-verified box the VLM says isn't visible must reach a TERMINAL outcome, never stay indefinitely re-queued. -Reproduces the review's exact probe (docs/design/openprocessor_internal/ -w8_pipeline_review_2026-09-27.md, "Re-review 2026-09-27" section, finding +Reproduces the review's exact probe (the 2026-09-27 pipeline review, "Re-review 2026-09-27" section, finding R-B1): the B1 fix made Path 1 (``pending_verification``) merge its resolved verdict back into the stored box list instead of replacing it outright. But when the VLM answers ``region_visible=False``, the pre-fix diff --git a/tests/curation/test_region_write_occ.py b/tests/curation/test_region_write_occ.py index 59019e01f..7c19542a1 100644 --- a/tests/curation/test_region_write_occ.py +++ b/tests/curation/test_region_write_occ.py @@ -1,8 +1,7 @@ """Regression: the box-list write must never use a stale ``region_revision`` / ``region_box_seq`` snapshot. -Reproduces the review's exact probe (docs/design/openprocessor_internal/ -w8_pipeline_review_2026-09-27.md, finding M1): something else (a human +Reproduces the review's exact probe (the 2026-09-27 pipeline review, finding M1): something else (a human PUT) bumps the item's stored ``region_revision`` / ``region_box_seq`` and replaces its box list while the worker's own VLM call is still in flight. Before the M1 fix, the worker's write built ``region_revision`` / diff --git a/tests/curation/test_search_no_legacy_tokens.py b/tests/curation/test_search_no_legacy_tokens.py index 5f5a571f5..e33e4f32a 100644 --- a/tests/curation/test_search_no_legacy_tokens.py +++ b/tests/curation/test_search_no_legacy_tokens.py @@ -1,9 +1,7 @@ """Guard test: the semantic-search files never reintroduce the legacy ``/search/*`` surface, ``visual_search_*``, or ``MobileCLIP``. -Mirrors what ``scripts/codegen/check_no_legacy_search.py`` / -``check_no_mobileclip.py`` enforce at pre-commit time (neither guard is -wired up here — this test stands in for them at file scope), scoped +Scoped explicitly to the files this task added — an independent, in-repo double-check that ``/curation/search/text`` is the only ``/search/*``-shaped route string diff --git a/tests/projects/test_config_view.py b/tests/projects/test_config_view.py index 33a00f099..d4ca424bb 100644 --- a/tests/projects/test_config_view.py +++ b/tests/projects/test_config_view.py @@ -1,5 +1,5 @@ """P1 commit 2: CurationConfigView resolves project-scoped fields at -access time (docs/design/openprocessor_internal/projects_plan.md §3.3).""" +access time (the project-isolation design §3.3).""" from __future__ import annotations diff --git a/tests/projects/test_project_context.py b/tests/projects/test_project_context.py index af3d6bce3..00b3568b4 100644 --- a/tests/projects/test_project_context.py +++ b/tests/projects/test_project_context.py @@ -1,5 +1,5 @@ """P1 commit 1: bound-project context (see -docs/design/openprocessor_internal/projects_plan.md §10 P1 test list).""" +the project-isolation design §10 P1 test list).""" from __future__ import annotations diff --git a/tests/test_export_hints_reference_real_paths.py b/tests/test_export_hints_reference_real_paths.py index e1d827002..06244b7c2 100644 --- a/tests/test_export_hints_reference_real_paths.py +++ b/tests/test_export_hints_reference_real_paths.py @@ -4,8 +4,7 @@ ``export_mobileclip_image_encoder.py`` / ``export_mobileclip_text_encoder.py`` told an operator with a missing checkpoint to run -``bash scripts/track_e/setup_mobileclip_env.sh`` -- a directory that has -never existed in this repo (``scripts/track_e/`` is not a real path). +a script in a ``track_e`` directory that has never existed in this repo. The actual downloader is ``make download-models`` (``scripts/lib/download.sh``). Both files' module docstrings also told an operator to run the export scripts themselves from a stale From 56bbbaed7085e7d5e9b4a962feb0689f6f2433c7 Mon Sep 17 00:00:00 2001 From: davidamacey Date: Thu, 15 Oct 2026 02:40:19 -0400 Subject: [PATCH 2/5] feat(tooling): public-data guard for screenshots, wider docs checks, bulk_ingest default, docs-site typecheck - capture_docs_screenshots.py: required public-project allowlist; listing pages refused while the backend holds a project outside it (#284) - check_docs_vs_code.py: CLI commands and flags, make targets, screenshot files, NAME= lines; version test scans frontend docs; docs.yml runs on docs/** and type-checks; weekly external link check (#287) - bulk_ingest.py: --api-base defaults to localhost:8000 inside the container (#288) - docs-site: drop deprecated baseUrl, add @types/react-dom, .nvmrc follows the Dockerfile (#293, #289) --- .github/workflows/docs-links.yml | 28 +++++ .github/workflows/docs.yml | 5 + docs-site/.nvmrc | 2 +- docs-site/package-lock.json | 11 ++ docs-site/package.json | 1 + docs-site/tsconfig.json | 5 +- frontend/scripts/capture_docs_screenshots.py | 34 +++++- .../scripts/docs_screenshots/public_data.py | 53 ++++++++ scripts/curation/bulk_ingest.py | 21 ++-- scripts/docs/check_docs_vs_code.py | 115 +++++++++++++++++- tests/curation/test_script_api_prefix.py | 15 ++- .../test_docs_screenshot_public_data_guard.py | 71 +++++++++++ tests/test_docs_vs_code.py | 58 ++++++++- tests/test_version_consistency.py | 15 +++ 14 files changed, 417 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/docs-links.yml create mode 100644 frontend/scripts/docs_screenshots/public_data.py create mode 100644 tests/test_docs_screenshot_public_data_guard.py diff --git a/.github/workflows/docs-links.yml b/.github/workflows/docs-links.yml new file mode 100644 index 000000000..0087cbf7f --- /dev/null +++ b/.github/workflows/docs-links.yml @@ -0,0 +1,28 @@ +name: Docs External Links + +# External links rot without a code change, so this runs on a schedule and by hand rather +# than on every PR. Relative links and anchors are checked offline by +# scripts/docs/check_docs_vs_code.py (pre-commit and the test suite). + +on: + schedule: + - cron: '0 7 * * 1' + workflow_dispatch: + +permissions: + contents: read + +jobs: + external-links: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Check external links + uses: lycheeverse/lychee-action@v2 + with: + args: >- + --no-progress --scheme https --max-retries 2 --timeout 20 + --exclude '^https?://(localhost|127\.0\.0\.1|0\.0\.0\.0|example\.)' + README.md INSTALLATION.md SECURITY.md CONTRIBUTING.md docs docs-site/docs + fail: true diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index fd1952524..8f8b05bd7 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -19,6 +19,7 @@ on: - master paths: - 'docs-site/**' + - 'docs/**' - '.github/workflows/docs.yml' pull_request: branches: @@ -26,6 +27,7 @@ on: - master paths: - 'docs-site/**' + - 'docs/**' - '.github/workflows/docs.yml' workflow_dispatch: @@ -59,6 +61,9 @@ jobs: - name: Install dependencies run: npm ci + - name: Type-check + run: npm run typecheck + - name: Build documentation run: npm run build diff --git a/docs-site/.nvmrc b/docs-site/.nvmrc index 209e3ef4b..6f4247a62 100644 --- a/docs-site/.nvmrc +++ b/docs-site/.nvmrc @@ -1 +1 @@ -20 +26 diff --git a/docs-site/package-lock.json b/docs-site/package-lock.json index 247022318..12d24b070 100644 --- a/docs-site/package-lock.json +++ b/docs-site/package-lock.json @@ -24,6 +24,7 @@ "@docusaurus/module-type-aliases": "3.10.2", "@docusaurus/tsconfig": "3.10.2", "@docusaurus/types": "3.10.2", + "@types/react-dom": "^19.3.0", "typescript": "~6.0.3" }, "engines": { @@ -6659,6 +6660,16 @@ "csstype": "^3.2.2" } }, + "node_modules/@types/react-dom": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.3.0" + } + }, "node_modules/@types/react-router": { "version": "5.1.20", "resolved": "https://registry.npmjs.org/@types/react-router/-/react-router-5.1.20.tgz", diff --git a/docs-site/package.json b/docs-site/package.json index 445b514a0..71c71931c 100644 --- a/docs-site/package.json +++ b/docs-site/package.json @@ -30,6 +30,7 @@ "@docusaurus/module-type-aliases": "3.10.2", "@docusaurus/tsconfig": "3.10.2", "@docusaurus/types": "3.10.2", + "@types/react-dom": "^19.3.0", "typescript": "~6.0.3" }, "browserslist": { diff --git a/docs-site/tsconfig.json b/docs-site/tsconfig.json index 2a8fa095d..63825c5d8 100644 --- a/docs-site/tsconfig.json +++ b/docs-site/tsconfig.json @@ -1,7 +1,10 @@ { "extends": "@docusaurus/tsconfig", "compilerOptions": { - "baseUrl": "." + "baseUrl": null, + "paths": { + "@site/*": ["./*"] + } }, "exclude": [".docusaurus", "build", "node_modules"] } diff --git a/frontend/scripts/capture_docs_screenshots.py b/frontend/scripts/capture_docs_screenshots.py index 8d8463dd7..d7b4c5cb8 100755 --- a/frontend/scripts/capture_docs_screenshots.py +++ b/frontend/scripts/capture_docs_screenshots.py @@ -53,7 +53,8 @@ import sys from pathlib import Path -from docs_screenshots.ctx import Ctx, _settle +from docs_screenshots import public_data +from docs_screenshots.ctx import Ctx, _settle, api_get from docs_screenshots.guard import _read_only from docs_screenshots.registry import STATE_HEIGHT, STATES from playwright.sync_api import sync_playwright @@ -94,6 +95,13 @@ def main() -> int: help="Project slug holding the public sample data. Without it bare routes " "redirect to the default project.", ) + parser.add_argument( + "--public-projects", + default=os.environ.get("OP_CROPWRIGHT_PUBLIC_PROJECTS"), + help="Required. Comma-separated slugs of every project on the backend that is public. " + "--project must be one of them, and the pages that list all projects are refused " + "while the backend holds any project outside this list.", + ) parser.add_argument("--out", type=Path, default=OUT_DIR) parser.add_argument( "--only", @@ -113,6 +121,11 @@ def main() -> int: ) args = parser.parse_args() args.out = args.out.resolve() + try: + allowed = public_data.parse_allowlist(args.public_projects) + public_data.require_allowlisted_project(args.project, allowed) + except ValueError as e: + sys.exit(str(e)) routes = [] if args.skip_routes else load_routes() states = dict(STATES) @@ -121,8 +134,23 @@ def main() -> int: states = {k: v for k, v in states.items() if k in args.only} if not routes and not states: sys.exit(f"no matching routes or states for --only {args.only}") - elif args.project is None: - states = {} # states need a project + + listing = api_get(args.base_url, "/curation/projects?include_archived=true") + refused = public_data.listing_pages_to_refuse( + [r["name"] for r in routes if r["route"].startswith("/projects")] + + [r["name"] for r in routes] + + list(states), + listing, + allowed, + ) + if refused: + foreign = public_data.foreign_projects(listing, allowed) + print( + f" refusing pages that list every project ({sorted(set(refused))}): the backend also " + f"holds {len(foreign)} project(s) outside the public allowlist." + ) + routes = [r for r in routes if r["name"] not in refused and not r["route"].startswith("/projects")] + states = {k: v for k, v in states.items() if k not in refused} args.out.mkdir(parents=True, exist_ok=True) diff --git a/frontend/scripts/docs_screenshots/public_data.py b/frontend/scripts/docs_screenshots/public_data.py new file mode 100644 index 000000000..f15362d06 --- /dev/null +++ b/frontend/scripts/docs_screenshots/public_data.py @@ -0,0 +1,53 @@ +"""The public-data guard of the docs screenshot capture: refuse pages that would show a +project the operator has not declared public.""" + +from __future__ import annotations + + +# Pages that render a list of every project the backend holds (the project list, the combine +# wizard and its job, copy-settings and clone pickers, model sharing's consumer list). +LISTING_STATES = frozenset( + { + "projects", + "combine-wizard", + "combine-job", + "projects-delete-dry-run", + "projects-copy-settings", + "models-sharing", + "models-unshare-force", + "clone-from-project", + } +) + + +def parse_allowlist(raw: str | None) -> set[str]: + slugs = {s.strip() for s in (raw or "").split(",") if s.strip()} + if not slugs: + raise ValueError( + "a public-project allowlist is required: --public-projects slug[,slug...] " + "(or OP_CROPWRIGHT_PUBLIC_PROJECTS), naming every project on the backend that is public" + ) + return slugs + + +def lists_all_projects(name: str) -> bool: + return name in LISTING_STATES + + +def require_allowlisted_project(project: str | None, allowed: set[str]) -> None: + if project is None: + raise ValueError("--project is required: the capture only ever reads one named public project") + if project not in allowed: + raise ValueError(f"project {project!r} is not on the public allowlist {sorted(allowed)}") + + +def foreign_projects(listing: dict, allowed: set[str]) -> list[str]: + """Slugs in a ``GET /curation/projects`` payload that the allowlist does not name.""" + return [p["slug"] for p in listing.get("projects", []) if p["slug"] not in allowed] + + +def listing_pages_to_refuse(names: list[str], listing: dict, allowed: set[str]) -> list[str]: + """The pages among ``names`` that list all projects, when any listed project is foreign.""" + if not foreign_projects(listing, allowed): + return [] + return [n for n in names if lists_all_projects(n)] diff --git a/scripts/curation/bulk_ingest.py b/scripts/curation/bulk_ingest.py index a6889c0cf..a2cc8c815 100755 --- a/scripts/curation/bulk_ingest.py +++ b/scripts/curation/bulk_ingest.py @@ -25,23 +25,22 @@ Ctrl-C asks the server to cancel (images already stored stay stored) and waits for it to stop. -Usage (inside the ``api`` container the API listens on port 8000, so pass -``--api-base``; on the host the default ``localhost:4603`` works): +Usage (the default ``--api-base`` is ``localhost:8000`` inside the ``api`` container, where +the API listens, and ``localhost:4603`` on the host, where compose publishes it): docker compose exec api python scripts/curation/bulk_ingest.py \\ - --folder /data/source/my_dataset --project cars \\ - --api-base http://localhost:8000/curation + --folder /data/source/my_dataset --project cars docker compose exec api python scripts/curation/bulk_ingest.py \\ --manifest /data/source/list.txt --batch-size 16 --concurrency 8 \\ - --source census --project cars --api-base http://localhost:8000/curation + --source census --project cars docker compose exec api python scripts/curation/bulk_ingest.py \\ - --s3 s3://my-bucket/incoming/ --project cars --api-base http://localhost:8000/curation + --s3 s3://my-bucket/incoming/ --project cars docker compose exec api python scripts/curation/bulk_ingest.py --project cars \\ - --status bulk_20261011T120000_ab12cd34 --api-base http://localhost:8000/curation + --status bulk_20261011T120000_ab12cd34 docker compose exec api python scripts/curation/bulk_ingest.py --project cars \\ - --cancel bulk_20261011T120000_ab12cd34 --api-base http://localhost:8000/curation + --cancel bulk_20261011T120000_ab12cd34 Exit status: 0 completed, 1 completed with failed images, 2 failed (the run stopped), 130 cancelled. @@ -72,6 +71,10 @@ REQUEST_TIMEOUT_S = 60.0 +def _in_container() -> bool: + return Path('/.dockerenv').exists() + + class ApiError(Exception): pass @@ -89,7 +92,7 @@ def build_parser() -> argparse.ArgumentParser: what.add_argument('--cancel', metavar='RUN_ID', help='Ask a live run to stop and exit') parser.add_argument( '--api-base', - default=f'http://localhost:4603{get_curation_config().api_prefix}', + default=f'http://localhost:{8000 if _in_container() else 4603}{get_curation_config().api_prefix}', help='Curation API mount (no trailing slash); requests go to ' '/projects/<--project>/...', ) diff --git a/scripts/docs/check_docs_vs_code.py b/scripts/docs/check_docs_vs_code.py index e26d8092a..61f0c898f 100755 --- a/scripts/docs/check_docs_vs_code.py +++ b/scripts/docs/check_docs_vs_code.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Check the documentation against the code. -Five checks, each reporting every miss (not just the first): +Nine checks, each reporting every miss (not just the first): * routes - every ``METHOD /path`` written in a doc exists in the generated OpenAPI (``contracts/openapi/*.json``) or, for non-curation routes, in the @@ -13,6 +13,13 @@ Names the installer and CLI refuse (``RETIRED_ENV_NAMES``) are exempt. * template - every non-``OP_`` setting ``env.template`` assigns is read by a compose file, script or code, so nothing advertised does nothing. +* cli - every ``./openprocessor `` in a doc is a command of the CLI, and a + ``--flag`` written in the same code span appears in the CLI sources. +* make - every ``make `` in a code span or fenced block is a Makefile target. +* screenshots - every ```` names a file under + ``docs-site/static/img/screenshots/``. +* envfiles - every ``NAME=`` line in a fenced block names a setting some compose file, script or + code reads (a name the block itself expands as ``$NAME`` is a local shell variable). * links - every relative link resolves to a file, and every ``#anchor`` that points into a markdown file matches a heading in that file. @@ -347,11 +354,117 @@ def check_links() -> list[str]: return errors +_DESIGN_DOCS = 'docs/design/' +_CLI_CMD_RE = re.compile(r'(?:\./|`)openprocessor[ \t]+([a-z][a-z-]*)') +_FLAG_RE = re.compile(r'(?]*?\bname="([^"]+)"') +_FENCE_RE = re.compile(r'```[^\n]*\n(.*?)```', re.S) +_SPAN_RE = re.compile(r'`([^`\n]+)`') +_SHELL_ASSIGN_RE = re.compile(r'^[ \t]*(?:export[ \t]+)?([A-Z][A-Z0-9]*(?:_[A-Z0-9]+)*)=', re.M) + + +def _user_docs() -> list[Path]: + """Docs a reader follows (markdown only; design notes describe plans, not the product).""" + return [ + d + for d in doc_files() + if d.suffix in _DOC_SUFFIXES and not _rel(d).as_posix().startswith(_DESIGN_DOCS) + ] + + +def _cli_commands() -> set[str]: + main = (REPO_ROOT / 'openprocessor').read_text(encoding='utf-8').split('main() {', 1)[1] + labels = re.findall(r'^ {8}([a-z\-|]+)\)$', main, re.M) + return {w for label in labels for w in label.split('|') if w[0].isalpha()} + + +def _cli_corpus() -> str: + paths = [REPO_ROOT / 'openprocessor', REPO_ROOT / 'scripts' / 'setup.sh'] + paths += sorted((REPO_ROOT / 'scripts' / 'lib').glob('*.sh')) + return '\n'.join(p.read_text(encoding='utf-8', errors='replace') for p in paths if p.is_file()) + + +def check_cli() -> list[str]: + commands = _cli_commands() + corpus = _cli_corpus() + errors: list[str] = [] + for doc in _user_docs(): + for line in doc.read_text(encoding='utf-8', errors='replace').splitlines(): + for m in _CLI_CMD_RE.finditer(line): + if m.group(1) not in commands: + errors.append( + f'{_rel(doc)}: ./openprocessor {m.group(1)} is not a command' + ) # formatted message built next to its check + span = line[m.end() :].split('`', 1)[0] + errors.extend( + f'{_rel(doc)}: {flag} is not a flag of ./openprocessor' + for flag in _FLAG_RE.findall(span) + if flag not in corpus + ) + return errors + + +def _code_texts(text: str) -> list[str]: + return _FENCE_RE.findall(text) + _SPAN_RE.findall(re.sub(_FENCE_RE, '', text)) + + +def check_make_targets() -> list[str]: + targets = set(re.findall(r'^([A-Za-z][\w-]*):', (REPO_ROOT / 'Makefile').read_text(), re.M)) + errors: list[str] = [] + for doc in _user_docs(): + text = doc.read_text(encoding='utf-8', errors='replace') + for code in _code_texts(text): + errors.extend( + f'{_rel(doc)}: make {name} is not a Makefile target' + for name in _MAKE_RE.findall(code) + if name not in targets + ) + return errors + + +def check_screenshots() -> list[str]: + root = REPO_ROOT / 'docs-site' / 'static' / 'img' / 'screenshots' + errors: list[str] = [] + for doc in _user_docs(): + for name in _SCREENSHOT_RE.findall(doc.read_text(encoding='utf-8', errors='replace')): + if '<' not in name and not (root / name).is_file(): + errors.append(f'{_rel(doc)}: screenshot {name} has no file under {_rel(root)}') # noqa: PERF401 # formatted message built next to its check + return errors + + +def check_envfiles() -> list[str]: + corpus = _env_corpus() + live = REPO_ROOT / 'tests' / 'live' + if live.is_dir(): + corpus += '\n'.join( + p.read_text(encoding='utf-8', errors='replace') for p in live.rglob('*.py') + ) + errors: list[str] = [] + for doc in _user_docs(): + for block in _FENCE_RE.findall(doc.read_text(encoding='utf-8', errors='replace')): + for name in sorted(set(_SHELL_ASSIGN_RE.findall(block))): + local = re.search(rf'\$\{{?{name}\b', block) + if ( + not name.startswith('OP_') + and not local + and not re.search(rf'\b{name}\b', corpus) + ): + errors.append( + f'{_rel(doc)}: {name}= is not read by any code' + ) # formatted message built next to its check + return errors + + CHECKS = { 'routes': check_routes, 'env': check_env, 'compose': check_compose_vars, 'template': check_template_names, + 'cli': check_cli, + 'make': check_make_targets, + 'screenshots': check_screenshots, + 'envfiles': check_envfiles, 'links': check_links, } diff --git a/tests/curation/test_script_api_prefix.py b/tests/curation/test_script_api_prefix.py index 43af55e67..1cd3cad65 100644 --- a/tests/curation/test_script_api_prefix.py +++ b/tests/curation/test_script_api_prefix.py @@ -169,7 +169,6 @@ class _Cfg: ('module', 'argv'), [ ('scripts.curation.ingest_upload', ['--root', '/tmp']), - ('scripts.curation.bulk_ingest', ['--folder', '/tmp']), ('scripts.curation.import_labeled_dataset', ['--dataset', '/tmp']), ], ) @@ -180,3 +179,17 @@ def test_ingest_tools_default_api_base_follows_prefix( monkeypatch.setattr(mod, 'get_curation_config', _Cfg) args = mod.build_parser().parse_args(argv) assert args.api_base == 'http://localhost:4603/custom-mount' + + +@pytest.mark.parametrize( + ('in_container', 'expected'), + [(False, 'http://localhost:4603/custom-mount'), (True, 'http://localhost:8000/custom-mount')], +) +def test_bulk_ingest_default_api_base_works_in_the_container_and_on_the_host( + monkeypatch: pytest.MonkeyPatch, in_container: bool, expected: str +) -> None: + """Inside ``api`` the API listens on 8000; the published 4603 is the host's view (#288).""" + mod = importlib.import_module('scripts.curation.bulk_ingest') + monkeypatch.setattr(mod, 'get_curation_config', _Cfg) + monkeypatch.setattr(mod, '_in_container', lambda: in_container) + assert mod.build_parser().parse_args(['--folder', '/tmp']).api_base == expected diff --git a/tests/test_docs_screenshot_public_data_guard.py b/tests/test_docs_screenshot_public_data_guard.py new file mode 100644 index 000000000..b646d5b7f --- /dev/null +++ b/tests/test_docs_screenshot_public_data_guard.py @@ -0,0 +1,71 @@ +"""The docs screenshot capture refuses to run against private data (issue #284). + +Pages that render a list of every project (the project list, combine, copy-from, sharing) are +captured only when each project the backend lists is on the operator's public allowlist; the +project being captured must be on it too. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + + +SCRIPTS = Path(__file__).resolve().parents[1] / 'frontend' / 'scripts' +sys.path.insert(0, str(SCRIPTS)) + +from docs_screenshots import public_data # noqa: E402 # path set above + + +def test_allowlist_is_required_and_parsed() -> None: + assert public_data.parse_allowlist('a, b ,,c') == {'a', 'b', 'c'} + with pytest.raises(ValueError, match='allowlist'): + public_data.parse_allowlist('') + with pytest.raises(ValueError, match='allowlist'): + public_data.parse_allowlist(None) + + +@pytest.mark.parametrize( + 'name', + [ + 'projects', + 'combine-wizard', + 'combine-job', + 'projects-delete-dry-run', + 'projects-copy-settings', + 'models-sharing', + 'models-unshare-force', + 'clone-from-project', + ], +) +def test_pages_that_list_every_project_are_recognised(name: str) -> None: + assert public_data.lists_all_projects(name) + + +@pytest.mark.parametrize('name', ['dashboard', 'review-regions-multibox', 'settings']) +def test_per_project_pages_are_not_flagged(name: str) -> None: + assert not public_data.lists_all_projects(name) + + +def test_project_to_capture_must_be_allowlisted() -> None: + with pytest.raises(ValueError, match='private-one'): + public_data.require_allowlisted_project('private-one', {'sample-coco'}) + public_data.require_allowlisted_project('sample-coco', {'sample-coco'}) + with pytest.raises(ValueError, match='--project'): + public_data.require_allowlisted_project(None, {'sample-coco'}) + + +def test_foreign_projects_are_the_listed_slugs_outside_the_allowlist() -> None: + payload = {'projects': [{'slug': 'default'}, {'slug': 'sample-coco'}, {'slug': 'client-x'}]} + assert public_data.foreign_projects(payload, {'sample-coco', 'default'}) == ['client-x'] + assert public_data.foreign_projects({'projects': []}, {'a'}) == [] + + +def test_names_to_skip_drops_listing_pages_when_a_foreign_project_exists() -> None: + payload = {'projects': [{'slug': 'sample-coco'}, {'slug': 'client-x'}]} + names = ['projects', 'dashboard', 'combine-wizard', 'models-sharing'] + skipped = public_data.listing_pages_to_refuse(names, payload, {'sample-coco'}) + assert skipped == ['projects', 'combine-wizard', 'models-sharing'] + assert public_data.listing_pages_to_refuse(names, payload, {'sample-coco', 'client-x'}) == [] diff --git a/tests/test_docs_vs_code.py b/tests/test_docs_vs_code.py index 37581111e..ea6f0dfeb 100644 --- a/tests/test_docs_vs_code.py +++ b/tests/test_docs_vs_code.py @@ -31,7 +31,9 @@ def _reset_only(): checker.set_only(None) -@pytest.mark.parametrize('name', ['routes', 'env', 'compose', 'template']) +@pytest.mark.parametrize( + 'name', ['routes', 'env', 'compose', 'template', 'cli', 'make', 'screenshots', 'envfiles'] +) def test_real_docs_match_the_code(name: str) -> None: errors = checker.CHECKS[name]() assert errors == [], f'{name} check found {len(errors)} problem(s):\n' + '\n'.join(errors) @@ -166,3 +168,57 @@ def git(*args: str) -> None: names = {p.name for p in checker.doc_files()} assert names == {'tracked.md'} + + +def test_cli_check_rejects_an_unknown_subcommand_and_flag(tmp_path: Path) -> None: + assert _check_one(tmp_path, 'Run `./openprocessor curation up --help`.\n', 'cli') == [] + bad = _check_one(tmp_path, 'Run `./openprocessor frobnicate` first.\n', 'cli') + assert len(bad) == 1 + assert 'frobnicate' in bad[0] + bad = _check_one(tmp_path, '`./openprocessor logs --no-such-flag`\n', 'cli') + assert len(bad) == 1 + assert '--no-such-flag' in bad[0] + + +def test_cli_check_reads_a_flag_only_up_to_the_end_of_its_code_span(tmp_path: Path) -> None: + body = '`./openprocessor upgrade`, and `./setup-openprocessor.sh --rollback`\n' + assert _check_one(tmp_path, body, 'cli') == [] + + +def test_make_check_rejects_an_unknown_target_in_code_only(tmp_path: Path) -> None: + assert _check_one(tmp_path, 'Run `make up` then\n```bash\nmake status\n```\n', 'make') == [] + assert _check_one(tmp_path, 'We make the API send requests.\n', 'make') == [] + assert _check_one(tmp_path, 'Globs like `make clone-refs-*` are fine.\n', 'make') == [] + bad = _check_one(tmp_path, 'Run `make no-such-target`.\n', 'make') + assert len(bad) == 1 + assert 'no-such-target' in bad[0] + bad = _check_one(tmp_path, '```bash\n$ make also-missing\n```\n', 'make') + assert len(bad) == 1 + + +def test_screenshot_check_rejects_a_name_with_no_file(tmp_path: Path) -> None: + assert ( + _check_one(tmp_path, '\n', 'screenshots') + == [] + ) + real = next((checker.REPO_ROOT / 'docs-site' / 'static' / 'img' / 'screenshots').rglob('*.png')) + name = real.relative_to(checker.REPO_ROOT / 'docs-site' / 'static' / 'img' / 'screenshots') + assert ( + _check_one(tmp_path, f'\n', 'screenshots') + == [] + ) + bad = _check_one( + tmp_path, '\n', 'screenshots' + ) + assert len(bad) == 1 + assert 'no-such-1600.png' in bad[0] + + +def test_envfiles_check_rejects_an_assigned_name_nothing_reads(tmp_path: Path) -> None: + ok = '```bash\nCOMPOSE_PROJECT_NAME=mine\n```\n' + assert _check_one(tmp_path, ok, 'envfiles') == [] + local = '```bash\nBK=~/backup\nmkdir -p "$BK"\n```\n' + assert _check_one(tmp_path, local, 'envfiles') == [] + bad = _check_one(tmp_path, '```bash\nNOT_A_REAL_SETTING=1\n```\n', 'envfiles') + assert len(bad) == 1 + assert 'NOT_A_REAL_SETTING' in bad[0] diff --git a/tests/test_version_consistency.py b/tests/test_version_consistency.py index 8fada2e88..11c9cc116 100644 --- a/tests/test_version_consistency.py +++ b/tests/test_version_consistency.py @@ -22,6 +22,13 @@ *ROOT.glob('docs/*.md'), *(ROOT / 'docs-site' / 'docs').rglob('*.md'), *(ROOT / 'docs-site' / 'docs').rglob('*.mdx'), + # The frontend's own docs. CHANGELOG.md and docs/releases/ are history: they name old releases. + *(p for p in (ROOT / 'frontend').glob('*.md') if p.name != 'CHANGELOG.md'), + *( + p + for p in (ROOT / 'frontend' / 'docs').rglob('*.md') + if 'releases' not in p.relative_to(ROOT / 'frontend' / 'docs').parts + ), ) # A release mention ends at a word character or a version dot, not at a sentence period # ("shipped in v0.4.0." is a mention; "v0.4.0.1" and "v0.4.0rc1" are not this pattern). @@ -83,3 +90,11 @@ def test_release_mention_pattern_catches_a_mention_at_a_sentence_end() -> None: def test_release_mention_pattern_ignores_longer_versions_and_words() -> None: assert RELEASE_MENTION.findall('v0.9.9.1 v0.9.9rc1 dev0.9.9') == [] + + +def test_the_scan_covers_the_frontend_docs_but_not_their_release_history() -> None: + names = {p.relative_to(ROOT).as_posix() for p in DOC_ROOTS} + assert 'frontend/README.md' in names + assert 'frontend/docs/FEATURES.md' in names + assert 'frontend/CHANGELOG.md' not in names + assert not [n for n in names if n.startswith('frontend/docs/releases/')] From 3f36a3b2ac1390ae803644b98f22679cda2c8e78 Mon Sep 17 00:00:00 2001 From: davidamacey Date: Thu, 15 Oct 2026 02:41:35 -0400 Subject: [PATCH 3/5] fix(cli): apply the GPU profile, load docker-compose.local.yml, start a subset of curation services, default the project name - profile: sets GPU_PROFILE and OP_INGEST_GPU_DECODE in an existing .env; the unread TRITON_WORKERS / MAX_BATCH_SIZE / SHM_SIZE keys and the never-loaded generated override are removed (#281) - ./openprocessor and make load an optional docker-compose.local.yml (#283) - curation up [service...] and make curation-up SERVICES=...; curation-evaluator gets its own evaluator profile (#285) - a checkout without COMPOSE_PROJECT_NAME uses the project openprocessor with a warning (#291) --- .gitignore | 3 +- CHANGELOG.md | 29 +++++ CLAUDE.md | 3 +- INSTALLATION.md | 65 +++++++---- Makefile | 14 ++- README.md | 7 +- config_templates/profiles/full.json | 2 - config_templates/profiles/minimal.json | 2 - config_templates/profiles/standard.json | 2 - docker-compose.yml | 7 +- docker/evaluator/Dockerfile | 6 +- docs-site/docs/configuration/basic.mdx | 11 +- docs-site/docs/deployment/gpu-sizing.mdx | 32 +++--- .../docs/getting-started/compose-profiles.mdx | 19 ++-- .../docs/getting-started/quick-start.mdx | 15 ++- .../docs/getting-started/second-stack.mdx | 6 +- docs-site/docs/guides/ingest-large-folder.mdx | 36 +++---- docs-site/docs/guides/ingest-sources.mdx | 7 +- docs-site/docs/guides/pipeline-modes.mdx | 39 +++---- .../docs/operations/training-and-promote.mdx | 2 +- docs-site/docs/operations/workers.mdx | 2 +- docs/ARCHITECTURE.md | 2 +- docs/CURATION.md | 2 +- env.template | 21 +--- openprocessor | 12 ++- scripts/lib/cli_core.sh | 27 ++++- scripts/lib/cli_dev.sh | 5 +- scripts/lib/config.sh | 93 ++++------------ scripts/lib/gpu.sh | 4 +- scripts/setup.sh | 3 - .../test_service_profiles_and_healthchecks.py | 6 ++ tests/installer/test_cli_compose_files.py | 94 ++++++++++++++++ tests/installer/test_cli_profile_command.py | 101 ++++++++++++++++++ tests/test_makefile_curation_targets.py | 59 ++++++++++ tests/test_opensearch_heap_sizing.py | 11 -- 35 files changed, 493 insertions(+), 256 deletions(-) create mode 100644 tests/installer/test_cli_compose_files.py create mode 100644 tests/installer/test_cli_profile_command.py create mode 100644 tests/test_makefile_curation_targets.py diff --git a/.gitignore b/.gitignore index 1e240db4d..07c81780b 100644 --- a/.gitignore +++ b/.gitignore @@ -180,7 +180,8 @@ docker-compose.*.yml # (not generated) -- restores the checkout hot-reload workflow that the # now deploy-safe docker-compose.yml no longer carries by default. !docker-compose.dev.yml -# docker-compose.override.yml is generated by setup.sh - do not commit +# Your own compose overrides, loaded by ./openprocessor and make (docs-site: ingest-large-folder) +docker-compose.local.yml # Environment template output (auto-generated by setup) # Note: env.template is committed, .env is generated diff --git a/CHANGELOG.md b/CHANGELOG.md index 87a17492f..273ce3b08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,21 @@ history of this codebase and was never published. This release is `[0.5.0]`. ### Added +- `./openprocessor` and `make` load an optional, git-ignored `docker-compose.local.yml` after the + stock compose files: a second read-only source root, a changed `shm_size` or a mounted credentials + file no longer need hand-built `docker compose` commands (#283). +- `./openprocessor curation up [service...]` and `make curation-up SERVICES="..."` start a subset of the + curation services; a name that is not a curation service is refused (#285). +- `capture_docs_screenshots.py` requires `--public-projects` (`OP_CROPWRIGHT_PUBLIC_PROJECTS`), refuses + a `--project` outside it, and refuses the pages that list every project (project list, combine, + copy-settings, clone-from, model sharing) while the backend holds any project outside it (#284). +- `scripts/docs/check_docs_vs_code.py` checks `./openprocessor` commands and flags, `make` targets, + `` file names and `NAME=` lines in fenced blocks against the code; the version test + scans the frontend docs; `docs.yml` also runs on `docs/**` and type-checks the site; a weekly + `docs-links.yml` checks external links (#287, #293). +- `scripts/codegen/check_cited_paths.py` (pre-commit): a comment or docstring that cites a `docs/`, + `scripts/` or similar file cites one that exists. Citations of design notes that left the tree + in the frontend sit in a shrink-only baseline (#289). - `docs/design/engineering_standards.md`: the portable standards and lessons from the repo cleanup (principles, the lint, type and test config that worked, every guard with its baseline format, `--tighten` workflow and plant-a-violation proof, an ordered port checklist for a sister repo, @@ -143,6 +158,18 @@ history of this codebase and was never published. This release is `[0.5.0]`. ### Changed +- `./openprocessor profile ` now sets `GPU_PROFILE` and `OP_INGEST_GPU_DECODE` in an existing + `.env` and names the real restart command. `setup.sh` no longer writes `docker-compose.override.yml`, + and `TRITON_WORKERS`, `MAX_BATCH_SIZE` and `SHM_SIZE` (read by nothing) are gone from `.env`, + `env.template`, the profile JSON and the docs (#281). +- `curation-evaluator` moved from the `curation` profile to its own `evaluator` profile, so + `--profile curation up -d` no longer reserves a GPU for it. Start it with + `./openprocessor curation up curation-evaluator` or `docker compose run --rm curation-evaluator` (#285). +- In a checkout, `./openprocessor` with no `COMPOSE_PROJECT_NAME` in `.env` uses the documented default + project `openprocessor` and says so, instead of exiting 3. An install directory still refuses (#291). +- `bulk_ingest.py` defaults `--api-base` to `localhost:8000` inside the container and `localhost:4603` + on the host (#288). +- `docs-site/.nvmrc` follows the Node version of `docs-site/Dockerfile` (#289). - Repo cleanup, PR 9a (typing): mypy now runs with `disallow_untyped_defs`, `disallow_incomplete_defs` and `warn_return_any` over `src scripts docker export` (`tests/` is outside its scope), and the nine-error-code relax override for the ML modules is gone. 328 errors were fixed with real types @@ -408,6 +435,8 @@ history of this codebase and was never published. This release is `[0.5.0]`. ### Fixed +- `docs-site` `npm run typecheck` passes: `baseUrl` (deprecated, TS5101) is dropped for `paths` alone, + and the missing `@types/react-dom` is added (#293). - Visual-search ingest (#318): a face response with fewer landmark entries than faces no longer raises `IndexError` and fails the whole batch. Only the affected image is refused, with the reason `face_landmarks_length_mismatch`; `detect_faces`, `recognize_faces` and the single-image face paths diff --git a/CLAUDE.md b/CLAUDE.md index 5bd7e0090..67ca09ae0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,7 +82,8 @@ Docker socket (read its header first). | `triton-server` | none | 4600 HTTP, 4601 gRPC, 4602 metrics | TensorRT models, explicit model control | | `api` | none | 4603 | FastAPI, all routes | | `opensearch` | none | 4607 | k-NN store and the curation datastore | -| `curation-detection-worker`, `curation-vlm-worker`, `curation-auto-label-worker`, `curation-cluster-refresh`, `curation-evaluator` | `curation` | | Region cascade, VLM loop, auto-label driver, clustering refresh, bake-off runner | +| `curation-detection-worker`, `curation-vlm-worker`, `curation-auto-label-worker`, `curation-cluster-refresh` | `curation` | | Region cascade, VLM loop, auto-label driver, clustering refresh | +| `curation-evaluator` | `evaluator` | | Bake-off runner, on demand (`./openprocessor curation up curation-evaluator`) | | `segmenter` | `segmenter` | 4611 | Region proposals from a text prompt | | `vlm` | `vlm` | 4612 | Local vLLM serving a catalog model | | `curation-trainer`, `curation-mlflow` | `training` | 4609 (MLflow) | Training jobs through a file protocol, tracking | diff --git a/INSTALLATION.md b/INSTALLATION.md index 3d97a9e76..4f9620e69 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -199,7 +199,37 @@ To change it, edit `.env`, then recreate the three services with `./openprocessor restart` does not re-read `.env`. Paths you pass to ingest are container paths under `/data/source`. `OP_SOURCE_PATH_ALIASES` adds more named roots, but compose does not mount them for you: bind-mount each one into the -same three services with your own compose override file. +same three services in your own `docker-compose.local.yml` (next). + +### More than one source root and your own compose overrides + +`./openprocessor` and `make` load `docker-compose.local.yml` (next to `docker-compose.yml`, +ignored by git) after the stock files, when it exists. Put anything of your own in it: a second +read-only source root, a changed `shm_size`, a mounted credentials file. A file named +`docker-compose.override.yml` is not read. Example for a second root: + +```yaml +# docker-compose.local.yml (next to docker-compose.yml; git ignores it) +services: + api: + volumes: + - /srv/nas/archive:/data/archive:ro + curation-detection-worker: + volumes: + - /srv/nas/archive:/data/archive:ro + curation-auto-label-worker: + volumes: + - /srv/nas/archive:/data/archive:ro +``` + +and in `.env`: + +```bash +OP_SOURCE_PATH_ALIASES=archive=/data/archive +``` + +Then `./openprocessor start` (or `make up`) recreates the services. Check the merged result with +`docker compose -p -f docker-compose.yml -f docker-compose.local.yml config`. ### The `openprocessor` CLI @@ -403,9 +433,9 @@ git clone https://github.com/davidamacey/OpenProcessor.git && cd OpenProcessor & In a checkout, manage the stack with `make` (`make up`, `make curation-up`, `make down`, `make status`). The `./openprocessor` CLI (and its shim -`./scripts/openprocessor.sh`) refuses to run until `.env` has a -`COMPOSE_PROJECT_NAME` line; the `.env` that `setup.sh` writes has none, so -add `COMPOSE_PROJECT_NAME=openprocessor` first. When you run `docker compose` +`./scripts/openprocessor.sh`) uses the project `openprocessor` and warns when `.env` has no +`COMPOSE_PROJECT_NAME` line (the `.env` that `setup.sh` writes has none); set the line to +choose another name. When you run `docker compose` by hand, pass `-p ` so you never act on another stack that uses the default name. @@ -520,10 +550,9 @@ and **standard**. Set it yourself in `.env` to override; `openprocessor gpu plan - The Triton load list is fixed in `docker-compose.yml`. A model that was never exported stays unloaded and Triton keeps running. `scripts/setup.sh --profile=minimal` exports the essential models only, with no OCR. -- `.env` also gets `TRITON_WORKERS`, `MAX_BATCH_SIZE` and `SHM_SIZE`, but nothing reads them. - The API runs 32 uvicorn workers and each service has `shm_size: 8g` in `docker-compose.yml`. -- `setup.sh` writes `docker-compose.override.yml`, but it is never loaded: `setup.sh`, - `./openprocessor` and `make` pass explicit `-f` files. +- The API runs 32 uvicorn workers and each service has `shm_size: 8g` in `docker-compose.yml`, + whatever the profile says. To change either, use your own `docker-compose.local.yml` (see + [More than one source root and your own compose overrides](#more-than-one-source-root-and-your-own-compose-overrides)). ### Selecting a Profile @@ -532,20 +561,20 @@ and **standard**. Set it yourself in `.env` to override; `openprocessor gpu plan ./scripts/setup.sh --profile=minimal ``` -Switching later with `./openprocessor profile ` (a checkout only) is incomplete -(issue #281). It rewrites the Triton configs but, when `.env` already exists, leaves -`GPU_PROFILE` and `OP_INGEST_GPU_DECODE` unchanged. Switch by hand: +Switch later with `./openprocessor profile ` (a checkout only). It rewrites the Triton +configs and sets `GPU_PROFILE` and `OP_INGEST_GPU_DECODE` in the existing `.env`, leaving every +other line alone: ```bash -./openprocessor profile standard # rewrites models/*/config.pbtxt -# edit .env: GPU_PROFILE=standard and OP_INGEST_GPU_DECODE=0 (or 1) +./openprocessor profile standard # rewrites models/*/config.pbtxt, updates .env ./openprocessor restart triton-server ``` Do not pass the optional second argument (a GPU id); it is written into every `config.pbtxt`. -A `.env` made by `scripts/setup.sh` or copied from `env.template` must also have -`COMPOSE_PROJECT_NAME` set, or `./openprocessor` refuses to run (see -[Configure the environment](#4-configure-environment)). +In a checkout, a `.env` without `COMPOSE_PROJECT_NAME` runs the default project `openprocessor` +and says so; set `COMPOSE_PROJECT_NAME` to use another (see +[Configure the environment](#4-configure-environment)). An install directory still refuses to run +without it. See [Deployment and GPU sizing](docs-site/docs/deployment/gpu-sizing.mdx) for the details. --- @@ -591,7 +620,7 @@ cp env.template .env # Edit .env to adjust settings ``` -The `./openprocessor` CLI also needs the compose project name in `.env`: +To name the compose project yourself (the `./openprocessor` CLI otherwise uses `openprocessor`): ```bash echo 'COMPOSE_PROJECT_NAME=openprocessor' >> .env @@ -804,7 +833,7 @@ docker compose down --rmi all # Remove generated files rm -rf pytorch_models/*.pt pytorch_models/*.onnx rm -rf models/*/1/model.plan -rm -rf .env docker-compose.override.yml +rm -rf .env docker-compose.local.yml rm -rf cache/ logs/ outputs/ test_results/ ``` diff --git a/Makefile b/Makefile index 769de140c..7eabfd079 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,10 @@ SHELL := /bin/bash # checkout, so it always adds that overlay -- detected by src/main.py # existing next to the compose file (same check scripts/setup.sh and # scripts/openprocessor.sh use). -DEV_COMPOSE := docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.gpu-arbiter.yml --profile curation --profile segmenter --profile vlm --profile training -COMPOSE := docker compose $(if $(wildcard src/main.py),-f docker-compose.yml -f docker-compose.dev.yml,-f docker-compose.yml) +DEV_DEV_COMPOSE := docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.gpu-arbiter.yml --profile curation --profile segmenter --profile vlm --profile training +# The curation services `curation-up SERVICES=...` accepts (the evaluator is on demand). +CURATION_SERVICES := curation-detection-worker curation-vlm-worker curation-auto-label-worker curation-cluster-refresh curation-evaluator +COMPOSE := docker compose $(if $(wildcard src/main.py),-f docker-compose.yml -f docker-compose.dev.yml,-f docker-compose.yml)$(if $(wildcard docker-compose.local.yml), -f docker-compose.local.yml) V := .venv/bin # Prefer the project venv's interpreter when it exists; fall back to # system python3 for a fresh checkout that hasn't created .venv yet. @@ -1125,11 +1127,12 @@ docs-build: ## Build docs-site (npm ci + docusaurus build; broken links and anch cd docs-site && npm ci && npm run build DOCS_PROJECT ?= coco-bench-2k +DOCS_PUBLIC_PROJECTS ?= $(DOCS_PROJECT) .PHONY: docs-screenshots docs-screenshots: ## Capture the Cropwright docs screenshots (OP_CROPWRIGHT_URL=... required; public sample project only) @test -n "$(OP_CROPWRIGHT_URL)" || { echo "Set OP_CROPWRIGHT_URL to a Cropwright instance that holds only public sample data."; exit 2; } - cd frontend && e2e/.venv/bin/python scripts/capture_docs_screenshots.py --base-url $(OP_CROPWRIGHT_URL) --project $(DOCS_PROJECT) + cd frontend && e2e/.venv/bin/python scripts/capture_docs_screenshots.py --base-url $(OP_CROPWRIGHT_URL) --project $(DOCS_PROJECT) --public-projects $(DOCS_PUBLIC_PROJECTS) .PHONY: docs docs: info ## Alias for info @@ -1178,9 +1181,10 @@ clone-ref: ## Clone a specific reference repo (usage: make clone-ref REPO=ultral # never touch them. Use the targets below instead. .PHONY: curation-up -curation-up: ## Start the curation worker services (experimental, opt-in) +curation-up: ## Start the curation workers (all, or SERVICES="curation-vlm-worker ...") + $(if $(filter-out $(CURATION_SERVICES),$(SERVICES)),$(error Not a curation service: $(filter-out $(CURATION_SERVICES),$(SERVICES)). Choose from: $(CURATION_SERVICES))) @echo "Starting curation subsystem (profile: curation)..." - $(COMPOSE) --profile curation up -d + $(COMPOSE) --profile curation up -d $(SERVICES) @echo "" @echo "Curation services starting. Check status with: make curation-status" diff --git a/README.md b/README.md index 07aeea493..86c0d759d 100644 --- a/README.md +++ b/README.md @@ -255,8 +255,7 @@ make up && make curation-up # 3. --folder is a container path under the /data/source mount, not a # host-relative one: docker compose -p openprocessor exec api python scripts/curation/bulk_ingest.py \ - --folder /data/source/coco_va_readme/images --project cars \ - --api-base http://localhost:8000/curation + --folder /data/source/coco_va_readme/images --project cars ``` An installed directory already has `OP_SOURCE_ROOT_HOST=./data`, so skip step 2 @@ -569,8 +568,8 @@ Client (Port 4603) - `triton-server` (4600-4602): Triton with TensorRT models. - `opensearch` (4607): vector database and the curation datastore. - `curation-detection-worker`, `curation-vlm-worker`, - `curation-auto-label-worker`, `curation-cluster-refresh`, - `curation-evaluator` [`curation`]. + `curation-auto-label-worker`, `curation-cluster-refresh` [`curation`]; + `curation-evaluator` [`evaluator`, on demand]. - `segmenter` (4611) [`segmenter`], `vlm` (4612) [`vlm`]. - `curation-trainer`, `curation-mlflow` (4609) [`training`]. - Prometheus (4604), Grafana (4605), Loki (4606), OpenSearch Dashboards (4608), diff --git a/config_templates/profiles/full.json b/config_templates/profiles/full.json index 9d6983870..d96f9f7a4 100644 --- a/config_templates/profiles/full.json +++ b/config_templates/profiles/full.json @@ -4,8 +4,6 @@ "instance_count": 4, "max_batch_size": 64, "preferred_batch_sizes": [8, 16, 32, 64], - "shm_size": "8g", - "workers": 32, "ocr_pipeline_instances": 4, "ingest_gpu_decode": true, "python_backend_budget_mib": 2048, diff --git a/config_templates/profiles/minimal.json b/config_templates/profiles/minimal.json index 287e66d4e..58dccb804 100644 --- a/config_templates/profiles/minimal.json +++ b/config_templates/profiles/minimal.json @@ -4,8 +4,6 @@ "instance_count": 1, "max_batch_size": 16, "preferred_batch_sizes": [4, 8, 16], - "shm_size": "4g", - "workers": 8, "ocr_pipeline_instances": 1, "ingest_gpu_decode": false, "python_backend_budget_mib": 512, diff --git a/config_templates/profiles/standard.json b/config_templates/profiles/standard.json index 2799536ac..e2d24dd7e 100644 --- a/config_templates/profiles/standard.json +++ b/config_templates/profiles/standard.json @@ -4,8 +4,6 @@ "instance_count": 2, "max_batch_size": 32, "preferred_batch_sizes": [8, 16, 32], - "shm_size": "6g", - "workers": 16, "ocr_pipeline_instances": 2, "ingest_gpu_decode": false, "python_backend_budget_mib": 1024, diff --git a/docker-compose.yml b/docker-compose.yml index c3080ba32..394cf9de2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -606,14 +606,15 @@ services: networks: - triton_net - # On-demand only (no `restart:` — see docker/evaluator/Dockerfile): - # `docker compose --profile curation run --rm curation-evaluator`. + # On-demand only (no `restart:` — see docker/evaluator/Dockerfile): its own + # profile, so `--profile curation up -d` does not reserve a GPU for it. + # `docker compose run --rm curation-evaluator`. curation-evaluator: image: ${OP_EVALUATOR_IMAGE:-${OP_IMAGE_REPO:-davidamacey}/openprocessor-evaluator:${OP_IMAGE_TAG:-0.5.0}} container_name: ${COMPOSE_PROJECT_NAME:-openprocessor}-evaluator logging: *default-logging profiles: - - curation + - evaluator env_file: - path: ./.env required: false diff --git a/docker/evaluator/Dockerfile b/docker/evaluator/Dockerfile index da0e65f88..0446c7c98 100644 --- a/docker/evaluator/Dockerfile +++ b/docker/evaluator/Dockerfile @@ -13,12 +13,12 @@ # build wheel dependencies stay in the builder and never reach the runtime image. # # Build: -# docker compose --profile curation build curation-evaluator +# docker compose build curation-evaluator # # Run on demand (watch mode; scans every projects/*/bakeoff_jobs under the state dir): -# docker compose --profile curation run --rm curation-evaluator +# docker compose run --rm curation-evaluator # One-shot job: -# docker compose --profile curation run --rm curation-evaluator \ +# docker compose run --rm curation-evaluator \ # python -m scripts.curation.bakeoff.bakeoff_runner --job /eval_jobs/x.job.json # ============================================================================= diff --git a/docs-site/docs/configuration/basic.mdx b/docs-site/docs/configuration/basic.mdx index 2589420b1..a62b578a1 100644 --- a/docs-site/docs/configuration/basic.mdx +++ b/docs-site/docs/configuration/basic.mdx @@ -60,10 +60,8 @@ profile (`minimal`, `standard` or `full`; see [GPU sizing](../deployment/gpu-siz The profile is applied **when it is chosen**: it sets the Triton model instance counts and batch size in the model configs, and decides whether GPU decode is on (`OP_INGEST_GPU_DECODE`, the one profile-driven key the app -acts on). The other keys below record the choice; `docker-compose.yml` -does not read `TRITON_WORKERS`, `MAX_BATCH_SIZE` or `SHM_SIZE` (the API -runs 32 uvicorn workers and every service that sets `shm_size` uses `8g`), so -editing them changes nothing. +acts on). The API runs 32 uvicorn workers and every service that sets +`shm_size` uses `8g`, whatever the profile. | Variable | Default | Meaning | |---|---|---| @@ -75,8 +73,9 @@ editing them changes nothing. Choose the profile at install time: pass `--profile NAME` to the installer, or run `scripts/setup.sh --profile=NAME --force` in a checkout (`--force` rewrites `.env`, keeping your `OPENSEARCH_HEAP`). Then check the result with -`./openprocessor gpu plan`. `./openprocessor profile ` does not switch -an existing install: it leaves `GPU_PROFILE` in `.env` unchanged. +`./openprocessor gpu plan`. In a checkout, `./openprocessor profile ` +switches an existing install: it rewrites the Triton configs and sets +`GPU_PROFILE` and `OP_INGEST_GPU_DECODE` in `.env`. ## GPU ids diff --git a/docs-site/docs/deployment/gpu-sizing.mdx b/docs-site/docs/deployment/gpu-sizing.mdx index 38a0b53f8..93c48de63 100644 --- a/docs-site/docs/deployment/gpu-sizing.mdx +++ b/docs-site/docs/deployment/gpu-sizing.mdx @@ -37,33 +37,26 @@ What a profile changes, and what it does not: PaddleOCR models and `ocr_pipeline`: instance count, batch sizes and the GPU id (`gpus: [ 0 ]`). These are tracked files, so switching a profile in a checkout shows up in `git status`. The PE encoders are not touched. Triton reads them when it starts. -- **It writes `GPU_PROFILE` and `OP_INGEST_GPU_DECODE` to `.env`**, but only when there is no - `.env` yet. The installer uses `GPU_PROFILE` for its model setup, and the API reads +- **It writes `GPU_PROFILE` and `OP_INGEST_GPU_DECODE` to `.env`**: the installer and + `scripts/setup.sh` when there is no `.env` yet, `./openprocessor profile` into the existing one. The installer uses `GPU_PROFILE` for its model setup, and the API reads `OP_INGEST_GPU_DECODE`. - **It does not change the model list.** The Triton load list is fixed in `docker-compose.yml`. A model that was never exported stays unloaded and the server keeps running (`--exit-on-error=false`). `scripts/setup.sh --profile=minimal` exports the essential models only, with no OCR. -- **`TRITON_WORKERS`, `MAX_BATCH_SIZE` and `SHM_SIZE` in `.env` are not read by anything.** - The API runs 32 uvicorn workers and every service has `shm_size: 8g` in - `docker-compose.yml`, whatever the profile says. To change either, edit the compose - file or use your own override file. -- **The `docker-compose.override.yml` that `setup.sh` writes is not loaded.** `setup.sh`, - `./openprocessor` and `make` all pass explicit `-f` files, so Compose does not pick the - file up automatically (a bare `docker compose up` in the directory would). `./openprocessor` - never loads override files you add. +- **It does not set the API worker count or `shm_size`.** The API runs 32 uvicorn workers and + every service has `shm_size: 8g` in `docker-compose.yml`, whatever the profile says. To + change either, put the change in `docker-compose.local.yml`, which `./openprocessor` and + `make` load after the stock files (see + [Ingest a large folder](../guides/ingest-large-folder.mdx#3-more-than-one-share-op_source_path_aliases)). ### Switching profile later -`./openprocessor profile ` does not switch profiles cleanly yet: -(issue #281). It needs a checkout. It regenerates the Triton configs, which works, -but with an existing `.env` it leaves `GPU_PROFILE` and `OP_INGEST_GPU_DECODE` alone, and writes -an override file the CLI never loads. Until it is fixed, switch by hand: +`./openprocessor profile ` needs a checkout. It regenerates the Triton configs and sets +`GPU_PROFILE` and `OP_INGEST_GPU_DECODE` in the existing `.env`, leaving the other lines alone: ```bash -# not run on a stack. The generation step below was run on a scratch checkout with no .env. -./openprocessor profile standard # rewrites models/*/config.pbtxt -# edit .env: set GPU_PROFILE=standard and OP_INGEST_GPU_DECODE=0 (or 1) +./openprocessor profile standard # rewrites models/*/config.pbtxt, updates .env ./openprocessor restart triton-server ./openprocessor gpu plan # shows the GPU decode state and the VRAM it needs ``` @@ -159,9 +152,8 @@ spread the load: All default to `0`. Set each explicitly: `.env` values are not shell-expanded, so one can't reference another. The trainer accepts a single id from the variable; a multi-GPU trainer (`OP_TRAIN_GPU_ORDER=0,2`) -also needs a compose override file listing the same `device_ids` in the -same order. The `docker-compose.override.yml` that `setup.sh` writes is never -loaded (see [GPU profiles](#gpu-profiles)), so it does not affect these variables. +also needs a `docker-compose.local.yml` listing the same `device_ids` in the +same order. Example layout on a three-GPU host (one card per heavy tenant): diff --git a/docs-site/docs/getting-started/compose-profiles.mdx b/docs-site/docs/getting-started/compose-profiles.mdx index 310be3cb8..c3b178d5b 100644 --- a/docs-site/docs/getting-started/compose-profiles.mdx +++ b/docs-site/docs/getting-started/compose-profiles.mdx @@ -12,7 +12,8 @@ exactly Triton, the API and OpenSearch. Everything else is opt-in. | Profile | Services | Needs a GPU | Start with | |---|---|---|---| | *(none)* | `triton-server`, `api`, `opensearch` | Triton, and the API for torch fallbacks | `docker compose up -d` / `make up` | -| `curation` | `curation-detection-worker`, `curation-vlm-worker`, `curation-auto-label-worker`, `curation-cluster-refresh`, `curation-evaluator` | only the evaluator | `docker compose --profile curation up -d` / `make curation-up` | +| `curation` | `curation-detection-worker`, `curation-vlm-worker`, `curation-auto-label-worker`, `curation-cluster-refresh` | no | `docker compose --profile curation up -d` / `make curation-up` | +| `evaluator` | `curation-evaluator` (bake-off runner, on demand) | yes | `./openprocessor curation up curation-evaluator` / `docker compose run --rm curation-evaluator` | | `segmenter` | `segmenter` (SAM 3 behind the generic segment protocol) | yes, plus `HF_TOKEN` for the gated weights | `docker compose --profile curation --profile segmenter up -d` | | `vlm` | `vlm` (vLLM, OpenAI-compatible) | yes | `docker compose --profile vlm up -d` | | `training` | `curation-trainer`, `curation-mlflow` | the trainer | `docker compose --profile training up -d` | @@ -36,15 +37,15 @@ Notes: [VLM selection](../guides/vlm-selection.mdx). `./secrets/vlm` is mounted read-only into `api` and the workers so endpoints can reference keys as `secret:`. -- **`curation-evaluator` is documented as run on demand.** It watches the - bake-off job directories, has no restart policy, and reserves the card - named by `EVALUATOR_GPU_ID`. The compose file runs it with - `docker compose --profile curation run --rm curation-evaluator`, which - blocks your shell while it watches. `--profile curation up -d` and - `./openprocessor curation up` also start it. +- **`curation-evaluator` runs on demand.** It watches the bake-off job + directories, has no restart policy, and reserves the card named by + `EVALUATOR_GPU_ID`, so it has its own `evaluator` profile and + `--profile curation up -d` does not start it. Start it with + `./openprocessor curation up curation-evaluator`, or run it in the + foreground with `docker compose run --rm curation-evaluator`. - **Starting only some curation services.** Name them: - `docker compose -p --profile curation up -d curation-detection-worker`. - The CLI has no subset command (issue #285). Which workers a + `./openprocessor curation up curation-detection-worker` or + `make curation-up SERVICES=curation-detection-worker`. Which workers a workflow needs is in [Pipeline modes](../guides/pipeline-modes.mdx). - **In an installed directory the profiles come from `.env`.** The installer writes `COMPOSE_PROFILES` from the tiers you chose, and `./openprocessor diff --git a/docs-site/docs/getting-started/quick-start.mdx b/docs-site/docs/getting-started/quick-start.mdx index ba7bac4e2..a5d77fcff 100644 --- a/docs-site/docs/getting-started/quick-start.mdx +++ b/docs-site/docs/getting-started/quick-start.mdx @@ -125,11 +125,11 @@ make down ``` :::note[`./openprocessor` in a checkout] -The `openprocessor` CLI refuses to run until `.env` names its compose project -(exit code 3). The `.env` that `scripts/setup.sh` writes does not, so add -`COMPOSE_PROJECT_NAME=openprocessor` to `.env` before using -`./openprocessor` or `./scripts/openprocessor.sh` in a checkout. The `make` -targets do not need it. When you run `docker compose` by hand, always pass +In a checkout the `openprocessor` CLI uses the compose project `openprocessor` +when `.env` does not name one (the `.env` that `scripts/setup.sh` writes does +not) and prints a warning; add `COMPOSE_PROJECT_NAME=` to `.env` to +choose another. An install directory still refuses to run without it (exit +code 3). When you run `docker compose` by hand, always pass `-p ` so you never act on another stack that uses the default name. ::: @@ -253,11 +253,10 @@ SRC=/data/source/samples/coco_va_readme/images # installed directory # SRC=/data/source/coco_va_readme/images # source checkout docker compose -p openprocessor exec api python scripts/curation/bulk_ingest.py \ - --project $P --folder $SRC --api-base http://localhost:8000/curation + --project $P --folder $SRC ``` -Inside the `api` container the API listens on port 8000, so `--api-base` is -required there; the script's default targets host port 4603. The script shows +The script shows live progress and exits 0 when the run completes, 1 when some images failed, 2 when the run failed. diff --git a/docs-site/docs/getting-started/second-stack.mdx b/docs-site/docs/getting-started/second-stack.mdx index bc666089e..670972caf 100644 --- a/docs-site/docs/getting-started/second-stack.mdx +++ b/docs-site/docs/getting-started/second-stack.mdx @@ -98,6 +98,6 @@ curl http://localhost:4703/health Compose reads `COMPOSE_PROJECT_NAME` from the clone's `.env`. From any other directory, pass `-p opdev` on every `docker compose` command. The -`./openprocessor` CLI also reads the name from `.env`; in a checkout it exits -with code 3 until `COMPOSE_PROJECT_NAME` is set there, which this checklist -does. +`./openprocessor` CLI also reads the name from `.env`; in a checkout without +it the CLI falls back to the project `openprocessor`, which this checklist +avoids by setting `COMPOSE_PROJECT_NAME`. diff --git a/docs-site/docs/guides/ingest-large-folder.mdx b/docs-site/docs/guides/ingest-large-folder.mdx index 4b4813c9b..fb1a9918e 100644 --- a/docs-site/docs/guides/ingest-large-folder.mdx +++ b/docs-site/docs/guides/ingest-large-folder.mdx @@ -124,7 +124,8 @@ OP_SOURCE_PATH_ALIASES={"archive": "/data/archive"} An alias name cannot contain `/`, and a malformed value is rejected with an error instead of being dropped. Compose does **not** mount alias paths. Add the bind mount to the same three services -in an override file next to `docker-compose.yml`, here `docker-compose.nas.yml`: +in `docker-compose.local.yml`, next to `docker-compose.yml`. `./openprocessor` and +`make` load that file after the stock files when it exists, and git ignores it: ```yaml services: @@ -139,20 +140,17 @@ services: - /srv/nas/archive:/data/archive:ro ``` -Start the stack with the override listed after the base files (add -`-f docker-compose.dev.yml` in a checkout, and `--env-file .env` if you run from -another directory): +Recreate the services so they pick up the mount: ```bash -docker compose -p -f docker-compose.yml -f docker-compose.nas.yml \ - up -d api curation-detection-worker curation-auto-label-worker +./openprocessor start ``` -:::caution -`./openprocessor start` builds its own list of compose files and does not read your -override. Running it recreates the services without the alias mount. Use the -`docker compose` command above for as long as you use the alias, or you will see 404 -thumbnails again. +:::note +Only the name `docker-compose.local.yml` is read. A `docker-compose.override.yml`, or +any other file name, is not. When you run `docker compose` yourself, list the file +after the base files: `-f docker-compose.yml -f docker-compose.local.yml` (add +`-f docker-compose.dev.yml` in a checkout, before the local file). ::: Ingest paths under the alias like any other (`--folder /data/archive/2023`). Images @@ -198,7 +196,7 @@ The simplest trial is a small sub-folder: ```bash docker compose -p exec api python scripts/curation/bulk_ingest.py \ --project nas-photos --folder /data/source/2024-03-sat \ - --api-base http://localhost:8000/curation --source nas-trial + --source nas-trial ``` Or a manifest. It is a text file with one container path per line (or JSON Lines with @@ -223,7 +221,7 @@ path in the first line differs from yours: ```bash docker compose -p exec api python scripts/curation/bulk_ingest.py \ --project nas-photos --manifest /data/source/trial.txt \ - --api-base http://localhost:8000/curation --source nas-trial + --source nas-trial ``` ```text @@ -270,14 +268,12 @@ Projects page and start again, since a policy change does not rewrite stored ite ```bash docker compose -p exec api python scripts/curation/bulk_ingest.py \ - --project nas-photos --folder /data/source \ - --api-base http://localhost:8000/curation + --project nas-photos --folder /data/source ``` -Pass `--api-base http://localhost:8000/curation` whenever you run the script inside -`api`: the API listens on port 8000 in the container, while the script's default -points at host port 4603. On the host (with the repo's Python environment and -`OPENSEARCH_URL` set) the default works. +The default `--api-base` follows where the script runs: `http://localhost:8000/curation` +inside `api`, where the API listens, and `http://localhost:4603/curation` on the host +(with the repo's Python environment and `OPENSEARCH_URL` set). `--folder` and `--manifest` are paths inside the API container and must lie under a source root. The script prints a live progress line, then a summary, and exits with @@ -471,5 +467,5 @@ df -h /var/lib/docker | Thumbnails 404 and the VLM stage predicts 0 | A worker is missing the source mount (step 2) or the alias mount (step 3). | | 409 `bulk_ingest_busy` | Another run of the project is live. Watch it with `--status`, or cancel it. | | Fewer files discovered than on the share | Symbolic links are not followed and unreadable directories are skipped (step 8). | -| Script cannot connect to `localhost:4603` | Inside `api` pass `--api-base http://localhost:8000/curation`. | +| Script cannot connect | Check `--api-base`: the default is `localhost:8000` inside `api` and `localhost:4603` on the host. Pass it when the API is elsewhere. | | Run `failed` right after the start | Five all-failed batches in a row. Read `error` in `GET .../ingest/bulk/`, fix the detector or OpenSearch, start the same source again. | diff --git a/docs-site/docs/guides/ingest-sources.mdx b/docs-site/docs/guides/ingest-sources.mdx index ada046acf..2677d39ed 100644 --- a/docs-site/docs/guides/ingest-sources.mdx +++ b/docs-site/docs/guides/ingest-sources.mdx @@ -82,10 +82,10 @@ never returns them. boto3 finds them the usual way, in the **`api` container**: `.env` into the container, so putting them there works; keep `.env` out of version control). - A mounted credentials file: set `AWS_SHARED_CREDENTIALS_FILE` to its in-container - path and mount it read-only with a compose override. + path and mount it read-only in `docker-compose.local.yml`. - An instance or task role on AWS. -A mounted file, as `docker-compose.override.yml`: +A mounted file, as `docker-compose.local.yml`: ```yaml services: @@ -114,8 +114,7 @@ run it inside the container (the API listens on port 8000 there): ```bash docker compose exec api python scripts/curation/bulk_ingest.py \ - --s3 s3://my-bucket/incoming/ --project my-project \ - --api-base http://localhost:8000/curation + --s3 s3://my-bucket/incoming/ --project my-project ``` Add `--batch-size` and `--concurrency` to tune it. The command shows progress and exits diff --git a/docs-site/docs/guides/pipeline-modes.mdx b/docs-site/docs/guides/pipeline-modes.mdx index 935efc63d..7147c3e25 100644 --- a/docs-site/docs/guides/pipeline-modes.mdx +++ b/docs-site/docs/guides/pipeline-modes.mdx @@ -108,18 +108,18 @@ Clustering runs from two triggers. Switch off each one you do not want: The VLM is a third source of work. Deactivate it per project (mode (b), step 2). -With only some workers running, start them by name. There is no CLI for a subset: -`./openprocessor curation up` and `make curation-up` start all five curation services. -(issue #285) +With only some workers running, start them by name: ```bash -# not run (installed directory: .env holds COMPOSE_PROFILES) -docker compose -p --profile curation up -d curation-detection-worker -# not run (checkout: add the dev overlay) -docker compose -p -f docker-compose.yml -f docker-compose.dev.yml \ - --profile curation up -d curation-detection-worker +./openprocessor curation up curation-detection-worker +make curation-up SERVICES="curation-detection-worker curation-auto-label-worker" ``` +`curation up` with no names starts the four curation workers. `curation-evaluator` (a +bake-off watcher with no restart policy that needs the NVIDIA runtime) has its own +`evaluator` profile and starts only when you name it, or with +`docker compose run --rm curation-evaluator`. + `curation-detection-worker` runs the region cascade. Without a region profile it idles, so for plain item detection you do not need it at all. @@ -137,10 +137,9 @@ if one is configured. ./openprocessor curation status ``` -`curation up` starts all five curation services, including `curation-evaluator`, which -needs the NVIDIA runtime. The evaluator is a bake-off watcher with no restart policy. -The compose file documents it as run on demand with -`docker compose --profile curation run --rm curation-evaluator`. +`curation up` starts the four curation workers. `curation-evaluator` is not among them: it needs +the NVIDIA runtime and is a bake-off watcher with no restart policy, so it has its own +`evaluator` profile (`./openprocessor curation up curation-evaluator`). ### 2. Decide about the VLM now @@ -211,11 +210,12 @@ changes them: | `--project` | every active project | | `--once` | one pass, then exit | -To change a flag, replace the service `command` in a compose override file. The merged -result was checked with `docker compose config`: +To change a flag, replace the service `command` in `docker-compose.local.yml`, which +`./openprocessor` and `make` load after the stock files. The merged result was checked +with `docker compose config`: ```yaml -# docker-compose.cluster.yml +# docker-compose.local.yml services: curation-cluster-refresh: command: @@ -227,16 +227,9 @@ services: ``` ```bash -# not run -docker compose -p -f docker-compose.yml -f docker-compose.cluster.yml \ - --profile curation up -d curation-cluster-refresh +./openprocessor curation up curation-cluster-refresh ``` -`./openprocessor` never loads your own override files, so a later `./openprocessor -start` recreates the service with the stock command. Use `docker compose` directly for -this service. Running it directly skips the CLI's checks that the project belongs to -this directory, so pass the same `-p` name the CLI uses. - The second trigger lives in the auto-label worker. While idle it checks each project every `OP_IVF_RETRAIN_CHECK_S` seconds (default 1800). It retrains the centroids when the pool has grown to `OP_IVF_RETRAIN_GROWTH` times (default 1.5) the trained size and at diff --git a/docs-site/docs/operations/training-and-promote.mdx b/docs-site/docs/operations/training-and-promote.mdx index 1f3fc7854..c7016fe8c 100644 --- a/docs-site/docs/operations/training-and-promote.mdx +++ b/docs-site/docs/operations/training-and-promote.mdx @@ -129,7 +129,7 @@ model version registered under `OP_MLFLOW_MODEL_NAME` (default caption="Bake-off setup over the bakeoff routes (captured from Cropwright, a frontend for this backend)" /> -The evaluator (`curation-evaluator`, `--profile curation`) scores trained +The evaluator (`curation-evaluator`, profile `evaluator`) scores trained runs and baseline models **per class** on the test split of any of the project's exports, or on frozen external sets under the project's `bakeoff_eval` directory (`{OP_PROJECTS_DATA_ROOT}/{slug}/bakeoff_eval/`, diff --git a/docs-site/docs/operations/workers.mdx b/docs-site/docs/operations/workers.mdx index 6400e5457..15ef1be8f 100644 --- a/docs-site/docs/operations/workers.mdx +++ b/docs-site/docs/operations/workers.mdx @@ -34,7 +34,7 @@ make curation-down | `curation-vlm-worker` | `python -m scripts.curation.vlm_worker --continuous` | Continuously asks each project's active VLM for class suggestions on unvalidated items that have a `pe_embedding`, on its own schedule. | | `curation-auto-label-worker` | `scripts.curation.auto_label_worker` | Claims jobs queued by `POST /curation/projects/{project}/pipeline/auto_label/start` from the shared `/jobs` volume and runs them one at a time across projects, oldest first: clustering, and VLM labeling only when the job says `run_vlm=true`. Can also trigger a centroid retrain when a project's pool has grown. | | `curation-cluster-refresh` | `scripts/curation/cluster_refresh_daemon.py` | Polls each project's items index (every 120 s by default). Once a project has grown by 200 items it calls that project's `POST /curation/projects/{project}/clusters/auto_promote`, then queues the auto-label job (no VLM stage) as the centroid retrain. The retrain is on by default (`--no-auto-label` turns it off). The flags are command-line only; see [Pipeline modes](../guides/pipeline-modes.mdx#3-what-the-cluster-refresh-daemon-does). | -| `curation-evaluator` | its own image | Watches each project's `bakeoff_jobs` directory and runs model comparisons. It has no restart policy, and the compose file documents it as run on demand: `docker compose --profile curation run --rm curation-evaluator`. `--profile curation up -d` (and `./openprocessor curation up`) also starts it. It needs the NVIDIA runtime and the card named by `EVALUATOR_GPU_ID`. | +| `curation-evaluator` | its own image | Watches each project's `bakeoff_jobs` directory and runs model comparisons. It has no restart policy and its own `evaluator` profile, so `--profile curation up -d` and `./openprocessor curation up` do not start it. Start it on purpose with `./openprocessor curation up curation-evaluator` or `docker compose run --rm curation-evaluator`. It needs the NVIDIA runtime and the card named by `EVALUATOR_GPU_ID`. | `api`, `curation-detection-worker` and `curation-auto-label-worker` must mount the **same** source directory at the same container path diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 7b0a93cf5..19602512e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -82,7 +82,7 @@ checkout. GPU placement comes from `.env` keys, not from the compose files. | `curation-vlm-worker` | `curation` | | VLM class and verification loop | | `curation-auto-label-worker` | `curation` | | Drives the auto-label job protocol | | `curation-cluster-refresh` | `curation` | | Schedules the residual-clustering retrain as an auto-label job, when ingest is quiet | -| `curation-evaluator` | `curation` | | Runs bake-off job specs | +| `curation-evaluator` | `evaluator` | | Runs bake-off job specs | | `segmenter` | `segmenter` | 4611 | Region proposals from a text prompt, `docker/segmenter/` | | `curation-trainer` | `training` | | Runs training jobs from `job.json` files, `docker/trainer/` | | `curation-mlflow` | `training` | 4609 | Experiment tracking, optional | diff --git a/docs/CURATION.md b/docs/CURATION.md index c69303acc..94713fe89 100644 --- a/docs/CURATION.md +++ b/docs/CURATION.md @@ -140,7 +140,7 @@ inside the `api` container, which listens on port 8000 there: ```bash docker compose -p exec api python scripts/curation/bulk_ingest.py \ - --project --folder /data/source/ --api-base http://localhost:8000/curation + --project --folder /data/source/ ``` Defaults are batch size 16 and concurrency 16 diff --git a/env.template b/env.template index f990c82ba..a341d6286 100644 --- a/env.template +++ b/env.template @@ -22,20 +22,6 @@ GPU_PROFILE=standard # Use nvidia-smi to see available GPUs GPU_ID=0 -# ============================================================================= -# Performance Settings (auto-configured based on GPU_PROFILE) -# ============================================================================= - -# Number of uvicorn workers for the FastAPI service -# Adjust based on CPU cores and memory -# Recommended: minimal=8, standard=16, full=32 -TRITON_WORKERS=16 - -# Maximum batch size for inference -# Lower values use less VRAM but reduce throughput -# Recommended: minimal=16, standard=32, full=64 -MAX_BATCH_SIZE=32 - # Largest single uploaded file, in MB. The cap counts each file of a multipart request, not # the request: /curation/projects/{project}/ingest/upload takes up to 128 images, and a request # of 12-24 MP photos (median 9 MB, up to 35 MB each) is limited by the largest photo. A file over @@ -44,11 +30,6 @@ MAX_BATCH_SIZE=32 # (512 MiB) is refused before it is read. # MAX_FILE_SIZE_MB=50 -# Shared memory size for Docker containers -# Used for inter-process communication -# Recommended: minimal=4g, standard=6g, full=8g -SHM_SIZE=6g - # ============================================================================= # Advanced: Triton model-name overrides (src/config/settings.py # TritonModelConfig) -- swap a served model without a code change, e.g. a @@ -1031,7 +1012,7 @@ VLM_PORT=4612 # SEGMENTER_COMPILE=0 # SEGMENTER_ENABLE_MASKS=1 -# --- Projects (docs/design/openprocessor_internal/projects_plan.md) --- +# --- Projects (the project-isolation design) --- # Isolated dataset workspaces. `default` is created at first boot exactly # like any other project (indexes `op_prj_default__`, data under # OP_PROJECTS_DATA_ROOT/default); it can be archived, never deleted. diff --git a/openprocessor b/openprocessor index dcca52205..67967f943 100755 --- a/openprocessor +++ b/openprocessor @@ -46,6 +46,11 @@ fi if [[ -f "$PROJECT_DIR/docker-compose.cpu.yml" ]]; then DC_FILES+=(-f "$PROJECT_DIR/docker-compose.cpu.yml") fi +# The user's own overrides (extra bind mounts, a second source root), last so +# they win. docker-compose.override.yml is not read: the file name is yours. +if [[ -f "$PROJECT_DIR/docker-compose.local.yml" ]]; then + DC_FILES+=(-f "$PROJECT_DIR/docker-compose.local.yml") +fi _env_value() { local key="$1" file="${2:-$PROJECT_DIR/.env}" @@ -86,8 +91,11 @@ resolve_project() { fi OP_PROJECT="$from_env" elif [[ -z "${OP_PROJECT:-}" ]]; then - log_error "no compose project: set COMPOSE_PROJECT_NAME in ${PROJECT_DIR}/.env (or OP_PROJECT)" - exit 3 + # Only a checkout gets here: an install dir exited above. docker-compose.yml + # names the project this way too; assert_project_owned still refuses a + # project whose containers came from another directory. + log_warn "no COMPOSE_PROJECT_NAME in ${PROJECT_DIR}/.env: using the default project 'openprocessor' (set COMPOSE_PROJECT_NAME or OP_PROJECT to choose another)" + OP_PROJECT="openprocessor" fi if [[ ! "$OP_PROJECT" =~ ^[a-z0-9][a-z0-9_-]*$ ]]; then log_error "invalid compose project name '${OP_PROJECT}'" diff --git a/scripts/lib/cli_core.sh b/scripts/lib/cli_core.sh index 5c8a81fc5..321f86f10 100755 --- a/scripts/lib/cli_core.sh +++ b/scripts/lib/cli_core.sh @@ -166,6 +166,10 @@ cmd_download() { # curation-cluster-refresh) all carry `profiles: [curation]` in # docker-compose.yml, so `start`/`stop` above never touch them. +# Every service in the curation profile, plus the on-demand evaluator (profile +# `evaluator`, so a bare `curation up` leaves it alone). +CURATION_SERVICES="curation-detection-worker curation-vlm-worker curation-auto-label-worker curation-cluster-refresh curation-evaluator" + cmd_curation() { local subcommand="${1:-status}" shift 2>/dev/null || true @@ -174,8 +178,23 @@ cmd_curation() { case "$subcommand" in up) - log_step "Starting curation subsystem (profile: curation)..." - dc --profile curation up -d + if (( $# > 0 )); then + local svc + for svc in "$@"; do + case " $CURATION_SERVICES " in + *" $svc "*) ;; + *) + log_error "Not a curation service: $svc" + echo "Curation services: $CURATION_SERVICES" + return 1 + ;; + esac + done + log_step "Starting curation services: $*..." + else + log_step "Starting curation subsystem (profile: curation)..." + fi + dc --profile curation up -d "$@" log_success "Curation services starting. Check with: $0 curation status" ;; down) @@ -200,7 +219,7 @@ cmd_curation() { ;; *) log_error "Unknown curation subcommand: $subcommand" - echo "Available: up, down, logs, status, seed" + echo "Available: up [service...], down, logs, status, seed" return 1 ;; esac @@ -271,7 +290,7 @@ Testing: Curation (experimental, opt-in compose profile): curation [action] Manage the curation worker services - Options: up, down, logs, status (default), seed + Options: up [service...], down, logs, status (default), seed Maintenance: clean [target] Clean files diff --git a/scripts/lib/cli_dev.sh b/scripts/lib/cli_dev.sh index 22bcb6424..f37122160 100755 --- a/scripts/lib/cli_dev.sh +++ b/scripts/lib/cli_dev.sh @@ -50,11 +50,10 @@ cmd_profile() { # Generate new configs generate_all_configs "$profile" "$gpu_id" - generate_env_file "$profile" "$gpu_id" - generate_compose_override "$profile" "$gpu_id" + apply_profile_to_env "$profile" log_success "Profile switched to: $profile" - log_info "Restart services to apply: ./scripts/openprocessor.sh restart" + log_info "Restart services to apply: ./openprocessor restart triton-server" } # ============================================================================= diff --git a/scripts/lib/config.sh b/scripts/lib/config.sh index fe49f3a34..cecc9418e 100755 --- a/scripts/lib/config.sh +++ b/scripts/lib/config.sh @@ -286,11 +286,6 @@ GPU_ID=$gpu_id # GPU decode: 1 only where the profile's card has ~8.5 GB of room beyond its models OP_INGEST_GPU_DECODE=$gpu_decode -# Performance (auto-configured based on GPU_PROFILE) -TRITON_WORKERS=$PROFILE_WORKERS -MAX_BATCH_SIZE=$PROFILE_MAX_BATCH -SHM_SIZE=$PROFILE_SHM_SIZE - # OpenSearch JVM heap: host RAM/2, clamped to 2g..30g (scripts/lib/opensearch_heap.sh) OPENSEARCH_HEAP=$heap @@ -309,76 +304,30 @@ EOF } # ============================================================================= -# Docker Compose Override Generation +# Switching the profile of an existing .env # ============================================================================= -# Generate docker-compose.override.yml for profile-specific settings -generate_compose_override() { +# apply_profile_to_env PROFILE -- set the two profile keys something reads +# (GPU_PROFILE, OP_INGEST_GPU_DECODE) in the existing .env, leaving every other +# line alone. generate_env_file never rewrites an existing .env. +apply_profile_to_env() { local profile="${1:-standard}" - local gpu_id="${2:-0}" - - local override_file="$PROJECT_DIR/docker-compose.override.yml" + local env_file="$PROJECT_DIR/.env" - # Load profile load_profile "$profile" || return 1 - - log_step "Generating docker-compose.override.yml..." - - cat > "$override_file" << EOF -# ============================================================================= -# docker-compose.override.yml -# Generated by setup.sh - Profile: $profile -# ============================================================================= -# This file overrides settings in docker-compose.yml based on your GPU profile. -# Regenerate with: ./scripts/openprocessor.sh profile $profile -# ============================================================================= - -services: - triton-server: - shm_size: ${PROFILE_SHM_SIZE} - deploy: - resources: - reservations: - devices: - - driver: nvidia - device_ids: ['$gpu_id'] - capabilities: - - gpu - - api: - shm_size: ${PROFILE_SHM_SIZE} - command: - - uvicorn - - src.main:app - - --host=0.0.0.0 - - --port=8000 - - --workers=${PROFILE_WORKERS} - - --limit-max-requests=10000 - - --loop=uvloop - - --http=httptools - - --backlog=4096 - - --limit-concurrency=512 - - --timeout-keep-alive=75 - - --timeout-graceful-shutdown=30 - - --access-log - - --log-level=info - deploy: - resources: - reservations: - devices: - - driver: nvidia - device_ids: ['$gpu_id'] - capabilities: - - gpu - - opensearch: - environment: - - discovery.type=single-node - - bootstrap.memory_lock=true - - "OPENSEARCH_JAVA_OPTS=-Xms\${OPENSEARCH_HEAP:-2g} -Xmx\${OPENSEARCH_HEAP:-2g}" - - DISABLE_SECURITY_PLUGIN=true -EOF - - log_success "docker-compose.override.yml created" - return 0 + [[ -f "$env_file" ]] || { log_error "no .env at $env_file to update"; return 1; } + + local key value + for key in GPU_PROFILE OP_INGEST_GPU_DECODE; do + case "$key" in + GPU_PROFILE) value="$profile" ;; + *) value="$PROFILE_INGEST_GPU_DECODE" ;; + esac + if grep -q "^${key}=" "$env_file"; then + sed -i "s|^${key}=.*|${key}=${value}|" "$env_file" + else + printf '%s=%s\n' "$key" "$value" >> "$env_file" + fi + done + log_success ".env updated: GPU_PROFILE=$profile OP_INGEST_GPU_DECODE=$PROFILE_INGEST_GPU_DECODE" } diff --git a/scripts/lib/gpu.sh b/scripts/lib/gpu.sh index 760107e3f..2580bdff8 100755 --- a/scripts/lib/gpu.sh +++ b/scripts/lib/gpu.sh @@ -149,13 +149,11 @@ load_profile() { PROFILE_NAME=$(grep -o '"name"[[:space:]]*:[[:space:]]*"[^"]*"' "$profile_file" | cut -d'"' -f4) PROFILE_INSTANCE_COUNT=$(grep -o '"instance_count"[[:space:]]*:[[:space:]]*[0-9]*' "$profile_file" | grep -o '[0-9]*$') PROFILE_MAX_BATCH=$(grep -o '"max_batch_size"[[:space:]]*:[[:space:]]*[0-9]*' "$profile_file" | grep -o '[0-9]*$') - PROFILE_SHM_SIZE=$(grep -o '"shm_size"[[:space:]]*:[[:space:]]*"[^"]*"' "$profile_file" | cut -d'"' -f4) - PROFILE_WORKERS=$(grep -o '"workers"[[:space:]]*:[[:space:]]*[0-9]*' "$profile_file" | grep -o '[0-9]*$') PROFILE_OCR_PIPELINE_INSTANCES=$(grep -o '"ocr_pipeline_instances"[[:space:]]*:[[:space:]]*[0-9]*' "$profile_file" | grep -o '[0-9]*$') PROFILE_INGEST_GPU_DECODE=$(profile_ingest_gpu_decode "$profile_file") - export PROFILE_NAME PROFILE_INSTANCE_COUNT PROFILE_MAX_BATCH PROFILE_SHM_SIZE PROFILE_WORKERS PROFILE_OCR_PIPELINE_INSTANCES PROFILE_INGEST_GPU_DECODE + export PROFILE_NAME PROFILE_INSTANCE_COUNT PROFILE_MAX_BATCH PROFILE_OCR_PIPELINE_INSTANCES PROFILE_INGEST_GPU_DECODE log_debug "Loaded profile: $PROFILE_NAME (instances=$PROFILE_INSTANCE_COUNT, batch=$PROFILE_MAX_BATCH)" return 0 diff --git a/scripts/setup.sh b/scripts/setup.sh index 39d80d0bd..b20c1f29d 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -316,9 +316,6 @@ generate_configs_step() { # --force was passed) generate_env_file "$SELECTED_PROFILE" "$GPU_ID" "$FORCE_ENV" - # Generate docker-compose.override.yml - generate_compose_override "$SELECTED_PROFILE" "$GPU_ID" - log_success "Configuration generated" } diff --git a/tests/compose/test_service_profiles_and_healthchecks.py b/tests/compose/test_service_profiles_and_healthchecks.py index 752f29e5e..082462a3a 100644 --- a/tests/compose/test_service_profiles_and_healthchecks.py +++ b/tests/compose/test_service_profiles_and_healthchecks.py @@ -206,3 +206,9 @@ def test_api_has_a_healthcheck() -> None: def test_curation_mlflow_has_a_healthcheck() -> None: services = _services() assert services['curation-mlflow'].get('healthcheck'), 'curation-mlflow needs a healthcheck' + + +def test_evaluator_is_on_demand_not_in_the_curation_profile() -> None: + """It reserves a GPU and has no restart policy, so `--profile curation up -d` must skip it (#285).""" + profiles = _services()['curation-evaluator'].get('profiles') or [] + assert profiles == ['evaluator'] diff --git a/tests/installer/test_cli_compose_files.py b/tests/installer/test_cli_compose_files.py new file mode 100644 index 000000000..cbf77d426 --- /dev/null +++ b/tests/installer/test_cli_compose_files.py @@ -0,0 +1,94 @@ +"""The deploy CLI's compose file list and fresh-checkout project name, and the subset form of +``curation up`` (issues #283, #285, #291).""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest +import test_cli_project +from test_cli_project import cli, set_env_project + + +if TYPE_CHECKING: + from pathlib import Path + + from installer_harness import Shimmed + +inst = test_cli_project.inst + + +def _make_checkout(inst: Path) -> None: + (inst / 'src').mkdir() + (inst / 'src' / 'main.py').write_text('') + (inst / 'docker-compose.dev.yml').write_text('services: {}\n') + + +def test_a_local_override_file_is_loaded_after_the_stock_files( + shimmed: Shimmed, inst: Path +) -> None: + shimmed.flag('allow_mutations') + (inst / 'docker-compose.local.yml').write_text('services: {}\n') + done = cli(shimmed, inst, 'stop') + assert done.returncode == 0, done.stderr + down = [ln for ln in shimmed.mutating_docker_calls() if ln.endswith(' down')] + assert down, shimmed.mutating_docker_calls() + line = down[0] + assert f'-f {inst}/docker-compose.local.yml' in line + assert line.index('docker-compose.yml') < line.index('docker-compose.local.yml') + + +def test_without_a_local_file_the_file_list_is_unchanged(shimmed: Shimmed, inst: Path) -> None: + shimmed.flag('allow_mutations') + assert cli(shimmed, inst, 'stop').returncode == 0 + assert all('docker-compose.local.yml' not in ln for ln in shimmed.mutating_docker_calls()) + + +def test_a_fresh_checkout_without_a_project_name_uses_the_documented_default( + shimmed: Shimmed, inst: Path +) -> None: + shimmed.flag('allow_mutations') + _make_checkout(inst) + set_env_project(inst, None) + done = cli(shimmed, inst, 'stop') + assert done.returncode == 0, done.stderr + assert 'COMPOSE_PROJECT_NAME' in done.stdout + done.stderr + assert any(' -p openprocessor ' in ln for ln in shimmed.mutating_docker_calls()) + + +def test_an_install_dir_without_a_project_name_still_refuses(shimmed: Shimmed, inst: Path) -> None: + shimmed.flag('allow_mutations') + set_env_project(inst, None) + done = cli(shimmed, inst, 'stop') + assert done.returncode == 3 + assert shimmed.mutating_docker_calls() == [] + + +def test_curation_up_without_names_starts_the_curation_profile( + shimmed: Shimmed, inst: Path +) -> None: + shimmed.flag('allow_mutations') + done = cli(shimmed, inst, 'curation', 'up') + assert done.returncode == 0, done.stderr + ups = [ln for ln in shimmed.mutating_docker_calls() if ' up -d' in ln] + assert ups, shimmed.mutating_docker_calls() + assert ups[0].endswith(' --profile curation up -d') + + +def test_curation_up_accepts_service_names(shimmed: Shimmed, inst: Path) -> None: + shimmed.flag('allow_mutations') + done = cli(shimmed, inst, 'curation', 'up', 'curation-detection-worker', 'curation-evaluator') + assert done.returncode == 0, done.stderr + ups = [ln for ln in shimmed.mutating_docker_calls() if ' up -d' in ln] + assert ups[0].endswith(' up -d curation-detection-worker curation-evaluator') + + +@pytest.mark.parametrize('bad', ['api', 'triton-server', 'nonsense']) +def test_curation_up_refuses_a_name_that_is_not_a_curation_service( + shimmed: Shimmed, inst: Path, bad: str +) -> None: + shimmed.flag('allow_mutations') + done = cli(shimmed, inst, 'curation', 'up', bad) + assert done.returncode != 0 + assert bad in done.stderr + done.stdout + assert shimmed.mutating_docker_calls() == [] diff --git a/tests/installer/test_cli_profile_command.py b/tests/installer/test_cli_profile_command.py new file mode 100644 index 000000000..7e2292f37 --- /dev/null +++ b/tests/installer/test_cli_profile_command.py @@ -0,0 +1,101 @@ +"""``openprocessor profile `` applies the profile it names (issue #281). + +It rewrites the Triton model configs and records the choice in an existing ``.env`` +(GPU_PROFILE and OP_INGEST_GPU_DECODE, the two profile keys something reads). It writes no +``docker-compose.override.yml`` and no TRITON_WORKERS / MAX_BATCH_SIZE / SHM_SIZE: nothing read them. +""" + +from __future__ import annotations + +import re +import shutil +import subprocess +from typing import TYPE_CHECKING + +from installer_harness import REPO_ROOT + + +if TYPE_CHECKING: + from pathlib import Path + + +def _project(tmp_path: Path, env: str) -> Path: + proj = tmp_path / 'proj' + shutil.copytree(REPO_ROOT / 'config_templates', proj / 'config_templates') + for cfg in (REPO_ROOT / 'models').glob('*/config.pbtxt'): + dest = proj / 'models' / cfg.parent.name + dest.mkdir(parents=True) + shutil.copy(cfg, dest / 'config.pbtxt') + (proj / '.env').write_text(env) + return proj + + +def _profile(proj: Path, *args: str) -> subprocess.CompletedProcess: + script = ( + f'PROJECT_DIR="{proj}"; SCRIPT_DIR="{REPO_ROOT}/scripts"; ' + 'source "$SCRIPT_DIR/lib/colors.sh"; source "$SCRIPT_DIR/lib/gpu.sh"; ' + 'load_config_lib() { source "$SCRIPT_DIR/lib/config.sh"; }; ' + 'source "$SCRIPT_DIR/lib/cli_dev.sh"; cmd_profile ' + ' '.join(args) + ) + return subprocess.run( + ['bash', '-c', script], + check=False, + capture_output=True, + text=True, + env={'PATH': '/usr/bin:/bin', 'LANG': 'C.UTF-8'}, + timeout=60, + ) + + +def test_profile_updates_the_profile_keys_of_an_existing_env(tmp_path: Path) -> None: + proj = _project(tmp_path, 'GPU_PROFILE=minimal\nOP_INGEST_GPU_DECODE=0\nOPENSEARCH_HEAP=5g\n') + done = _profile(proj, 'full', '0') + assert done.returncode == 0, done.stderr + env = (proj / '.env').read_text() + assert re.search(r'^GPU_PROFILE=full$', env, re.MULTILINE), env + assert re.search(r'^OP_INGEST_GPU_DECODE=1$', env, re.MULTILINE), env + assert re.search(r'^OPENSEARCH_HEAP=5g$', env, re.MULTILINE), env + assert 'GPU_PROFILE=minimal' not in env + + +def test_profile_appends_the_keys_when_the_env_lacks_them(tmp_path: Path) -> None: + proj = _project(tmp_path, 'API_PORT=4603\n') + assert _profile(proj, 'standard', '0').returncode == 0 + env = (proj / '.env').read_text() + assert re.search(r'^GPU_PROFILE=standard$', env, re.MULTILINE), env + assert re.search(r'^API_PORT=4603$', env, re.MULTILINE), env + + +def test_profile_writes_no_override_file_and_no_dead_keys(tmp_path: Path) -> None: + proj = _project(tmp_path, 'GPU_PROFILE=minimal\n') + assert _profile(proj, 'full', '0').returncode == 0 + assert not (proj / 'docker-compose.override.yml').exists() + assert not re.search(r'^(TRITON_WORKERS|MAX_BATCH_SIZE|SHM_SIZE)=', (proj / '.env').read_text()) + + +def test_profile_restart_hint_names_the_real_command(tmp_path: Path) -> None: + proj = _project(tmp_path, 'GPU_PROFILE=minimal\n') + done = _profile(proj, 'full', '0') + assert './openprocessor restart triton-server' in done.stdout + done.stderr + assert 'scripts/openprocessor.sh' not in done.stdout + done.stderr + + +def test_generated_env_and_template_carry_no_dead_profile_keys(tmp_path: Path) -> None: + proj = tmp_path / 'p' + shutil.copytree(REPO_ROOT / 'config_templates', proj / 'config_templates') + script = ( + f'PROJECT_DIR="{proj}"; source "{REPO_ROOT}/scripts/lib/config.sh"; ' + 'generate_env_file standard 0' + ) + done = subprocess.run( + ['bash', '-c', script], check=False, capture_output=True, text=True, timeout=60 + ) + assert done.returncode == 0, done.stderr + dead = re.compile(r'^#?\s*(TRITON_WORKERS|MAX_BATCH_SIZE|SHM_SIZE)=', re.MULTILINE) + assert not dead.search((proj / '.env').read_text()) + assert not dead.search((REPO_ROOT / 'env.template').read_text()) + assert 'generate_compose_override' not in (REPO_ROOT / 'scripts/lib/config.sh').read_text() + for name in ('minimal', 'standard', 'full'): + text = (REPO_ROOT / f'config_templates/profiles/{name}.json').read_text() + assert '"workers"' not in text + assert '"shm_size"' not in text diff --git a/tests/test_makefile_curation_targets.py b/tests/test_makefile_curation_targets.py new file mode 100644 index 000000000..f92da19ff --- /dev/null +++ b/tests/test_makefile_curation_targets.py @@ -0,0 +1,59 @@ +"""``make curation-up`` takes the same optional service names as ``./openprocessor curation up`` (#285), +and every compose call from the Makefile loads the user's docker-compose.local.yml (#283).""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def _dry_run(*args: str, cwd: Path = REPO_ROOT) -> str: + done = subprocess.run( + ['make', '-n', '-s', *args], + cwd=cwd, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + assert done.returncode == 0, done.stderr + return done.stdout + + +def test_curation_up_without_services_starts_the_whole_profile() -> None: + out = _dry_run('curation-up') + assert '--profile curation up -d' in [ + ln.strip()[-len('--profile curation up -d') :] for ln in out.splitlines() + ] + + +def test_curation_up_with_services_starts_only_those() -> None: + out = _dry_run('curation-up', 'SERVICES=curation-detection-worker curation-vlm-worker') + assert '--profile curation up -d curation-detection-worker curation-vlm-worker\n' in out + + +def test_curation_up_refuses_a_service_that_is_not_a_curation_one() -> None: + done = subprocess.run( + ['make', '-n', '-s', 'curation-up', 'SERVICES=api'], + cwd=REPO_ROOT, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + assert done.returncode != 0 + + +@pytest.mark.parametrize('present', [True, False]) +def test_makefile_loads_a_local_override_only_when_it_exists(tmp_path: Path, present: bool) -> None: + for name in ('Makefile', 'docker-compose.yml'): + (tmp_path / name).write_text((REPO_ROOT / name).read_text()) + if present: + (tmp_path / 'docker-compose.local.yml').write_text('services: {}\n') + out = _dry_run('curation-up', cwd=tmp_path) + assert ('-f docker-compose.local.yml' in out) is present diff --git a/tests/test_opensearch_heap_sizing.py b/tests/test_opensearch_heap_sizing.py index 71415d9fd..21e8fee21 100644 --- a/tests/test_opensearch_heap_sizing.py +++ b/tests/test_opensearch_heap_sizing.py @@ -142,17 +142,6 @@ def test_config_sh_forced_regen_keeps_a_user_heap(tmp_path: Path) -> None: assert 'OPENSEARCH_HEAP=4g' not in env -def test_config_sh_override_interpolates_opensearch_heap(tmp_path: Path) -> None: - proj = _project(tmp_path) - result = _generate(proj, _meminfo(tmp_path, 32), 'generate_compose_override standard 0') - assert result.returncode == 0, result.stderr - override = (proj / 'docker-compose.override.yml').read_text() - assert ( - '"OPENSEARCH_JAVA_OPTS=-Xms${OPENSEARCH_HEAP:-2g} -Xmx${OPENSEARCH_HEAP:-2g}"' in override - ) - assert not re.search(r'-Xm[sx][0-9]', override) - - def test_compose_interpolates_heap_and_hard_codes_no_xmx() -> None: compose = (REPO_ROOT / 'docker-compose.yml').read_text() assert '"OPENSEARCH_JAVA_OPTS=-Xms${OPENSEARCH_HEAP:-2g} -Xmx${OPENSEARCH_HEAP:-2g}"' in compose From 239399f1874db838cad9bbf0f5dedb622b1a5cd0 Mon Sep 17 00:00:00 2001 From: davidamacey Date: Thu, 15 Oct 2026 02:43:26 -0400 Subject: [PATCH 4/5] docs(embedding): say which paths write not_selected and deferred --- src/domain/embedding_state.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/domain/embedding_state.py b/src/domain/embedding_state.py index a7e90344d..b4602e889 100644 --- a/src/domain/embedding_state.py +++ b/src/domain/embedding_state.py @@ -18,10 +18,10 @@ EMBEDDED = 'embedded' -# Reserved for the selective-embedding policy; written by no path yet. +# Ingest policy mode ``selected``: this detection was left out of the embedding. NOT_SELECTED = 'not_selected' -# No vector yet by design: the vector was dropped because the target project -# could not use it. +# No vector yet by design: ingest policy mode ``lazy`` stores no vector at ingest, and a +# vector dropped because the target project could not use it is stored this way too. DEFERRED = 'deferred' # The encoder raised for this item at ingest. FAILED = 'failed' From 2673a5c2a68a9a827df2b0d98ed16e8749ad1aea Mon Sep 17 00:00:00 2001 From: davidamacey Date: Thu, 15 Oct 2026 02:52:50 -0400 Subject: [PATCH 5/5] docs(frontend): changelog entry for the screenshot capture guard --- frontend/CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frontend/CHANGELOG.md b/frontend/CHANGELOG.md index f06ca715e..d898b167c 100644 --- a/frontend/CHANGELOG.md +++ b/frontend/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- The docs screenshot capture (`scripts/capture_docs_screenshots.py`) now requires `--public-projects` and + refuses any project outside that list, and refuses pages that list every project while the backend holds + another project, so a capture can never show private data. + ### Added - `/ingest` has a "Bulk ingest (folder or manifest)" panel, shown with the server-path ingest it sits beside: start a