diff --git a/services/api/requirements.in b/services/api/requirements.in index 63b91368..2c58bb07 100644 --- a/services/api/requirements.in +++ b/services/api/requirements.in @@ -21,6 +21,15 @@ orjson>=3.12.0 # RT-ORJSON: Rust JSON for ORJSONResponse + hot blob boto3>=1.43.73 # MinIO/S3 object storage (range reads); lazy-imported psycopg[binary]>=3.1 # Postgres driver (prod). Locked here so `--require-hashes` covers it too. +# Outbound HTTP client. Imported at MODULE SCOPE by `bsdd.py` (bSDD API v1) and `site_context.py` +# (elevation / place lookups), so it is a direct dependency of shipped code — but until 2026-08-28 it +# was declared nowhere and reached the lock on a single transitive edge, `# via anthropic`. The +# anthropic 1.x SDK moved its HTTP layer from httpx to httpx2, so recompiling the lock would have +# dropped httpx outright. The failure would not have been loud: both consumers are imported inside +# functions, so the service still boots and only the bSDD lookup and site-context routes 500. +# A package our own source imports is a DIRECT dependency however else it happens to arrive. +httpx>=0.28.1 + # --- analysis / QA + export bridge to the data service (services/data/src) --- ifcopenshell>=0.8.5,<0.9 openpyxl>=3.1 @@ -64,7 +73,12 @@ diskcache>=5.6 # Apache-2.0, pure-Python, no C extension # --- optional (lazy-imported, only when the matching env var is set) --- redis>=8.1.0 # shared rate-limit + login-lockout counters (AEC_REDIS_URL) -anthropic>=0.120.0 # optional AI assists (Draft RFI) — ANTHROPIC_API_KEY +anthropic>=1.0.0 # optional AI assists (Draft RFI) — ANTHROPIC_API_KEY. 1.x floor: the SDK + # dropped long-deprecated surface and moved httpx -> httpx2. Every call site + # here is plain `client.messages.create(...)`; inventoried against the 1.x + # migration list and none of the removed surface is used — no + # with_raw_response, no Text Completions, no temperature/top_p/top_k, no + # output_format, no Bedrock/Vertex. The only fallout was httpx, declared above. sentry-sdk[fastapi]>=2.68.1 # external error alerting (sentry.py); no-op unless AEC_SENTRY_DSN is set # --- OpenTelemetry distributed tracing (otel.py); no-op unless OTEL_EXPORTER_OTLP_ENDPOINT is set --- @@ -87,6 +101,12 @@ rdflib>=7.6.0 # 200 MB+ against onnxruntime's ~50 MB, and the service never trains. Detection stays inert without # a model file (AEC_PHOTO_MODEL), so installing this does not by itself turn anything on. onnxruntime>=1.23 +# +# Pillow is Tier 1's decoder — `photo_cv.py` does `from PIL import Image, ImageOps` at module scope, +# and `photo_detect.py` calls it "a hard dep" in a comment. It was nonetheless undeclared, arriving +# only `# via reportlab`. Same shape as the httpx line above and found by the same sweep: one +# transitive edge away from breaking site-photo QA. Floor matches services/data/requirements.txt. +pillow>=12.3.0 # --- SCOPE-DOCX: Exhibit A as an editable Word document (scope_docx.py) --- # MIT, recorded in docs/ATTRIBUTIONS.md, approved by the user 2026-08-07. Pinned in the change that diff --git a/services/api/requirements.lock b/services/api/requirements.lock index d23e0fe4..53cd74d7 100644 --- a/services/api/requirements.lock +++ b/services/api/requirements.lock @@ -16,9 +16,9 @@ annotated-types==0.7.0 \ --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 # via pydantic -anthropic==0.120.2 \ - --hash=sha256:0f0bc2b381dc0eb41c8d886b815d79c2041cd2374f83aed36f574b6dc9c579c1 \ - --hash=sha256:9722efc10c27a30a69f5338ddacdb35bc6a64297a4e4ba729bf83af873d5fb3a +anthropic==1.2.0 \ + --hash=sha256:12f8eedee7b7fb5685837b1371b7bfae1b281703f62355f4632598ec2fc53b34 \ + --hash=sha256:b60642b3e3cd6b8e3e328a2d3f2863ad2b6e743f1037e42cc0143f7df99f63c6 # via -r services/api/requirements.in anyio==4.14.1 \ --hash=sha256:4e5533c5b8ff0a24f5d7a176cbe6877129cd183893f66b537f8f227d10527d72 \ @@ -26,6 +26,7 @@ anyio==4.14.1 \ # via # anthropic # httpx + # httpx2 # starlette # watchfiles asgiref==3.12.1 \ @@ -332,10 +333,6 @@ diskcache==5.6.3 \ --hash=sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc \ --hash=sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19 # via -r services/api/requirements.in -distro==1.9.0 \ - --hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \ - --hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2 - # via anthropic docstring-parser==0.18.0 \ --hash=sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015 \ --hash=sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b @@ -539,11 +536,16 @@ h11==0.16.0 \ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 # via # httpcore + # httpcore2 # uvicorn httpcore==1.0.9 \ --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 # via httpx +httpcore2==2.12.0 \ + --hash=sha256:7e04258ce01013d7d615e5b910a3b27fac937d7a95038227e79652b4ba3b4ceb \ + --hash=sha256:9293522bba0aa7c4c8e9e3f040c16575bd8868e155a77fa30c7a9085a5eae648 + # via httpx2 httptools==0.8.0 \ --hash=sha256:0770728beb05094c809b98e814edff5fef69d26ad7d21185f2f6d5884a0ba683 \ --hash=sha256:0ea897f0c729581ebf72131a438a7932d9b14efef72d75ada966700cac3caaeb \ @@ -599,6 +601,10 @@ httptools==0.8.0 \ httpx==0.28.1 \ --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via -r services/api/requirements.in +httpx2==2.12.0 \ + --hash=sha256:7631fe9887a8a2275f4a2540e053aa670fcc50742864a9ae7c66e609fdcf12cf \ + --hash=sha256:cc8b6eecb8661c146b8f89a60e97456ee086e91a784ed31ac450c3a9e613dd36 # via anthropic idna==3.18 \ --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ @@ -606,6 +612,7 @@ idna==3.18 \ # via # anyio # httpx + # httpx2 # requests ifcopenshell==0.8.5 \ --hash=sha256:005dced5aaa45064e73287c75ed9cf5f8ba54bce80c496fd76404313833cd62e \ @@ -1398,7 +1405,9 @@ pillow==12.3.0 \ --hash=sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d \ --hash=sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198 \ --hash=sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7 - # via reportlab + # via + # -r services/api/requirements.in + # reportlab protobuf==7.35.1 \ --hash=sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799 \ --hash=sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87 \ @@ -2003,6 +2012,12 @@ trimesh==5.0.0 \ --hash=sha256:0195003198baaf2550aebe612254ba2f01c385cec46c37ecc5beca8291f79a13 \ --hash=sha256:51ec67d7f9f74b918f2a695da5fd511b9c084e96307648ae83f45b58f13f8009 # via -r services/api/requirements.in +truststore==0.10.4 \ + --hash=sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301 \ + --hash=sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981 + # via + # httpcore2 + # httpx2 typing-extensions==4.16.0 \ --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 @@ -2012,6 +2027,7 @@ typing-extensions==4.16.0 \ # anyio # ezdxf # fastapi + # httpx2 # ifcopenshell # opentelemetry-api # opentelemetry-exporter-otlp-proto-http diff --git a/services/api/run_tests.py b/services/api/run_tests.py index 7c3ed697..a0d62948 100644 --- a/services/api/run_tests.py +++ b/services/api/run_tests.py @@ -84,7 +84,7 @@ "test_markup", "test_route_authz", "test_resource_id_authz", "test_route_reachability", "test_resumable_upload", "test_model_align", "test_ifc_parse_gate", "test_plugin_isolation", "test_body_pid_authz", "test_global_authz", "test_protected_prefix_coverage", "test_baseline", "test_global_mutating_authz", "test_ref_counter", "test_audit_coverage", "test_bsdd", "test_openbim_registry", "test_waterfall", "test_waterfall_cents", "test_sessions", "test_mfa", "test_stored_ids", "test_cobie", "test_fts_index", "test_scim", "test_scim_provision_race", "test_saml", "test_responsibility", "test_array_live", "test_assemblies", "test_dxf_takeoff", "test_qto_class_match", "test_georef", "test_scene_package", "test_clash_bvh", "test_model_qa", "test_model_health", "test_roundtrip_qa", "test_stakeholder", "test_prioritization", "test_ai_readiness", - "test_scan_deviation", "test_plan_to_bim", "test_errorlog", "test_import_cycles", "test_tenant_scoping", "test_schedule_risk_single", "test_carbon_compliance", "test_permit_check", "test_drawing_qa", "test_element_5d", "test_authoring_matrix", "test_option_missing", "test_option_score", "test_plugin_registry", "test_jobs", "test_clash_federated_job", "test_inbox_jobs", "test_job_kind_labels", "test_worker_split", "test_job_orphan_scope", "test_job_stall", "test_pid_lock_xproc", "test_pid_lock_surface", "test_sheet_layout", "test_dim_component", "test_sheet_recover", "test_firm_standards", "test_site_context", "test_risk_board", "test_env_wind", "test_model_options", "test_doc_text", "test_escalation", "test_query_dsl", "test_rule_library", "test_schedule_baselines", "test_model_ci", "test_xlsx_roundtrip", "test_geometric_rules", "test_rebar_rules", "test_cx", "test_distwaterfall", "test_license_cloud", "test_smart_views", "test_view_delete", "test_version_approve_identity", "test_upload_streaming", "test_lod_aspects", "test_lod_element_table", "test_publish_reconvert", "test_model_cache_seed", "test_model_cache_mutation", "test_mutating_readers", "test_adopt_guid", "test_ifcpatch", "test_bcf_api", "test_coordination_fresh", "test_assemblies_cost", "test_fem_export", "test_subset_export", "test_norm_valid", "test_schema_diag", "test_revision_delta", "test_bep", "test_pm_close", "test_itp", "test_quality_chain", "test_quality_chain_route", "test_meeting_links", "test_est_bands", "test_scope_gap", "test_golden_thread", "test_clash_xml_import", "test_gis_out", "test_cbs", "test_mep_graph", "test_model_warnings", "test_schedule_options", "test_master_builder", "test_master_builder_scope", "test_get_commits", "test_project_pulse", "test_client_portal", "test_selections", "test_margin", "test_model_assets", "test_macros", "test_layout_options", "test_equipment", "test_space_util", "test_design_metrics", "test_mep_fittings", "test_prod_actuals", "test_pipeline_allocate", "test_production", "test_procure_level", "test_adjacency", "test_supply_chain", "test_invisible_unicode", "test_cited_answer", "test_est_confidence", "test_buyout_schedule", "test_scope_register", "test_permit_timeline", "test_absorption", "test_progress_rollup", "test_fill_matrix", "test_parcel_geometry", "test_assembly_thermal", "test_portal_txn", "test_persona_answer", "test_boe_ledger", "test_assumption_provenance", "test_assumption_provenance_route", "test_concept_budget", "test_topic_board", "test_roof_window", "test_topic_lifecycle", "test_calc_fields", "test_constraints", "test_element_lookup", "test_cli", "test_view_templates", "test_type_catalogs", "test_password_policy", "test_stepup_single_verifier", "test_fin_gov", "test_fin_calc", "test_fin_ingest", "test_fin_portfolio", "test_level_move", "test_instance_props", "test_roundtrip", "test_wall_joins", "test_composite_family", "test_shared_params", "test_version_values", "test_ifcpatch_transforms", "test_bcf3", "test_energy_export", "test_net_effective", "test_cre_deal_desk", "test_cre_governance", "test_cre_tier3", "test_family_geometry", "test_demo_seed", "test_cost_spine", "test_commercial_drift", "test_family_shapes", "test_workflow_config", "test_option_takeoff", "test_option_carbon", "test_option_carbon_route", "test_option_economics", "test_option_economics_route", "test_option_object", "test_option_object_route", "test_family_coverage", "test_section_annotation", "test_lod500_readiness", "test_scan_to_lod500", "test_egress_routes", "test_status_workflow_parity", "test_section_hatch", "test_section_keynotes", "test_detail_refs", "test_vg_overrides", "test_revit_export_cfg", "test_soft_clash", "test_sequence_clash", "test_element_tags", "test_cost_ifc", "test_fived", "test_health_consistency", "test_module_rooms", "test_modules_response_complete", "test_lifecycle_strip", "test_family_merge", "test_element_facts", "test_consistency", "test_work_queue", "test_task_bind", "test_qto_wire", "test_estimate_diff", "test_dim_constraints", "test_sov_build", "test_takeoff_scope", "test_r37_wire_routes", "test_claim_type", "test_risk_calibrate", "test_schedule_status", "test_engine_routes", "test_reachable", "test_money_wire", "test_license_gate", "test_license_lock_gate", "test_lock_advisories", "test_npm_advisories", "test_perf_budget", "test_perf_rate", "test_cache_key", "test_oauth_providers", "test_qto_measured_area", "test_lod_census", "test_lod_proxy", "test_model_ensure", "test_support_graph", "test_export_colour_stable", "test_stair_ramp", "test_profile_dims", "test_eot", "test_eot_methods", "test_eot_sourced", "test_shared_model", "test_plan_identity", "test_axon_view", "test_view_kind_dispatch", "test_photo_cv", "test_photo_detect", "test_photo_duplicate", "test_pipeline_scales", "test_plan_pins", "test_plan_cut_quality", "test_pins_unified", "test_index_freshness", "test_bake_budget", "test_geom_slots", "test_bake_shared", "test_geo_ref", "test_file_sizes", "test_delete_ratchet", "test_doc_substance", "test_claude_md_gates", "test_cors_expose_headers", "test_open_redirect", "test_mp_engine", "test_upload_cap", "test_vitals", "test_samples", "test_bundle_index", + "test_scan_deviation", "test_plan_to_bim", "test_errorlog", "test_import_cycles", "test_tenant_scoping", "test_schedule_risk_single", "test_carbon_compliance", "test_permit_check", "test_drawing_qa", "test_element_5d", "test_authoring_matrix", "test_option_missing", "test_option_score", "test_plugin_registry", "test_jobs", "test_clash_federated_job", "test_inbox_jobs", "test_job_kind_labels", "test_worker_split", "test_job_orphan_scope", "test_job_stall", "test_pid_lock_xproc", "test_pid_lock_surface", "test_sheet_layout", "test_dim_component", "test_sheet_recover", "test_firm_standards", "test_site_context", "test_risk_board", "test_env_wind", "test_model_options", "test_doc_text", "test_escalation", "test_query_dsl", "test_rule_library", "test_schedule_baselines", "test_model_ci", "test_xlsx_roundtrip", "test_geometric_rules", "test_rebar_rules", "test_cx", "test_distwaterfall", "test_license_cloud", "test_smart_views", "test_view_delete", "test_version_approve_identity", "test_upload_streaming", "test_lod_aspects", "test_lod_element_table", "test_publish_reconvert", "test_model_cache_seed", "test_model_cache_mutation", "test_mutating_readers", "test_adopt_guid", "test_ifcpatch", "test_bcf_api", "test_coordination_fresh", "test_assemblies_cost", "test_fem_export", "test_subset_export", "test_norm_valid", "test_schema_diag", "test_revision_delta", "test_bep", "test_pm_close", "test_itp", "test_quality_chain", "test_quality_chain_route", "test_meeting_links", "test_est_bands", "test_scope_gap", "test_golden_thread", "test_clash_xml_import", "test_gis_out", "test_cbs", "test_mep_graph", "test_model_warnings", "test_schedule_options", "test_master_builder", "test_master_builder_scope", "test_get_commits", "test_project_pulse", "test_client_portal", "test_selections", "test_margin", "test_model_assets", "test_macros", "test_layout_options", "test_equipment", "test_space_util", "test_design_metrics", "test_mep_fittings", "test_prod_actuals", "test_pipeline_allocate", "test_production", "test_procure_level", "test_adjacency", "test_supply_chain", "test_invisible_unicode", "test_cited_answer", "test_est_confidence", "test_buyout_schedule", "test_scope_register", "test_permit_timeline", "test_absorption", "test_progress_rollup", "test_fill_matrix", "test_parcel_geometry", "test_assembly_thermal", "test_portal_txn", "test_persona_answer", "test_boe_ledger", "test_assumption_provenance", "test_assumption_provenance_route", "test_concept_budget", "test_topic_board", "test_roof_window", "test_topic_lifecycle", "test_calc_fields", "test_constraints", "test_element_lookup", "test_cli", "test_view_templates", "test_type_catalogs", "test_password_policy", "test_stepup_single_verifier", "test_fin_gov", "test_fin_calc", "test_fin_ingest", "test_fin_portfolio", "test_level_move", "test_instance_props", "test_roundtrip", "test_wall_joins", "test_composite_family", "test_shared_params", "test_version_values", "test_ifcpatch_transforms", "test_bcf3", "test_energy_export", "test_net_effective", "test_cre_deal_desk", "test_cre_governance", "test_cre_tier3", "test_family_geometry", "test_demo_seed", "test_cost_spine", "test_commercial_drift", "test_family_shapes", "test_workflow_config", "test_option_takeoff", "test_option_carbon", "test_option_carbon_route", "test_option_economics", "test_option_economics_route", "test_option_object", "test_option_object_route", "test_family_coverage", "test_section_annotation", "test_lod500_readiness", "test_scan_to_lod500", "test_egress_routes", "test_status_workflow_parity", "test_section_hatch", "test_section_keynotes", "test_detail_refs", "test_vg_overrides", "test_revit_export_cfg", "test_soft_clash", "test_sequence_clash", "test_element_tags", "test_cost_ifc", "test_fived", "test_health_consistency", "test_module_rooms", "test_modules_response_complete", "test_lifecycle_strip", "test_family_merge", "test_element_facts", "test_consistency", "test_work_queue", "test_task_bind", "test_qto_wire", "test_estimate_diff", "test_dim_constraints", "test_sov_build", "test_takeoff_scope", "test_r37_wire_routes", "test_claim_type", "test_risk_calibrate", "test_schedule_status", "test_engine_routes", "test_reachable", "test_money_wire", "test_license_gate", "test_license_lock_gate", "test_lock_advisories", "test_npm_advisories", "test_perf_budget", "test_perf_rate", "test_cache_key", "test_oauth_providers", "test_qto_measured_area", "test_lod_census", "test_lod_proxy", "test_model_ensure", "test_support_graph", "test_export_colour_stable", "test_stair_ramp", "test_profile_dims", "test_eot", "test_eot_methods", "test_eot_sourced", "test_shared_model", "test_plan_identity", "test_axon_view", "test_view_kind_dispatch", "test_photo_cv", "test_photo_detect", "test_photo_duplicate", "test_pipeline_scales", "test_plan_pins", "test_plan_cut_quality", "test_pins_unified", "test_index_freshness", "test_bake_budget", "test_geom_slots", "test_bake_shared", "test_geo_ref", "test_file_sizes", "test_declared_imports", "test_delete_ratchet", "test_doc_substance", "test_claude_md_gates", "test_cors_expose_headers", "test_open_redirect", "test_mp_engine", "test_upload_cap", "test_vitals", "test_samples", "test_bundle_index", # R41-TEST-RESIDUE — the residue sweep must never propose a database it does not own: "test_sweep_guard", # R23-DIGEST — the deterministic model digest and its two routes: diff --git a/services/api/test_declared_imports.py b/services/api/test_declared_imports.py new file mode 100644 index 00000000..24b8d326 --- /dev/null +++ b/services/api/test_declared_imports.py @@ -0,0 +1,169 @@ +""" +Every third-party package our source imports at module scope must be DECLARED, not inherited. + +Found on 2026-08-28 while bumping `anthropic` across its 0.x -> 1.x major. Two packages that shipped +code imports directly were declared in no requirements file at all, and reached the lock on a single +transitive edge each: + + httpx <- only `# via anthropic` (bsdd.py, site_context.py) + pillow <- only `# via reportlab` (photo_cv.py, and photo_detect.py calls it "a hard dep") + +The anthropic bump is exactly the event that collects on that debt: 1.x moved its HTTP layer from +`httpx` to `httpx2`, so recompiling the lock would have removed `httpx` from the install set while +two modules still did `import httpx` at the top of the file. + +**The failure would have been quiet, which is the part worth a gate.** Both consumers are imported +inside functions, so the service boots normally, the health check passes, and the whole suite is +green -- right up until someone opens the bSDD lookup or a site-context route and gets a 500. Nothing +about "we upgraded an unrelated AI SDK" points at those routes. + +So the rule is not "pin more things". It is: **a package that appears in our own `import` statements +is a direct dependency, whatever else happens to pull it in.** Transitive availability is a fact +about somebody else's metadata, and it can change in a release we do not review. + +WHAT IS DELIBERATELY NOT COVERED -- the exemptions are structural, not a name list: + + * **function-local imports.** `from massingifc_ifc import convert_ifc` inside a function, or + `import pye57` under `try/except ImportError`, is the established way this codebase says + "optional, supplied by the deployment". Those must stay legal, and they fail loudly at call time + rather than silently, which is the difference that matters. + * **first-party**, including the vendored trees on `src/` (`massingplan`, `massingcapture`) and + `aec_data` -- derived from the source layout, never listed here, so vendoring a fourth package + does not need an edit to this file. + * **stdlib**, from `sys.stdlib_module_names`. + +An import that is neither installed nor exempt fails too. At module scope that is not a style +question: the module cannot be imported at all, and something else is hiding it. +""" +import ast +import os +import re +import sys +from importlib.metadata import packages_distributions + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +FAILED = [] + + +def check(label, ok, detail=""): + print((" ok " if ok else " FAIL ") + label + (f" -- {detail}" if detail and not ok else "")) + if not ok: + FAILED.append(label) + + +def norm(name): + """PEP 503 normalisation, so `Pillow`, `pillow` and `sentry_sdk` compare equal to their pins.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def declared_in(path): + """Top-level requirement names from a pip-compile input, extras and specifiers stripped.""" + out = set() + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.split("#")[0].strip() + if not line or line.startswith("-"): + continue + m = re.match(r"^([A-Za-z0-9._-]+)", line) + if m: + out.add(norm(m.group(1))) + return out + + +def module_scope_imports(tree): + """Top-level import names, skipping function/class bodies and try/except ImportError guards. + + Walking with `ast.walk` would be shorter and wrong: it flattens the tree, so a deliberate lazy + import inside a function is indistinguishable from one at the top of the file -- and that + distinction is the entire point of this gate. + """ + names = [] + + def guarded(node): + for h in node.handlers: + t = h.type + if isinstance(t, ast.Name) and t.id == "ImportError": + return True + if isinstance(t, ast.Tuple) and any( + isinstance(e, ast.Name) and e.id == "ImportError" for e in t.elts): + return True + return False + + def visit(body): + for node in body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + continue # lazy/optional by construction + if isinstance(node, ast.Try): + if guarded(node): + continue # declared-optional by construction + visit(node.body) + for h in node.handlers: + visit(h.body) + visit(node.orelse) + visit(node.finalbody) + elif isinstance(node, ast.Import): + names.extend(a.name.split(".")[0] for a in node.names) + elif isinstance(node, ast.ImportFrom): + if not node.level and node.module: # level>0 is a relative (first-party) import + names.append(node.module.split(".")[0]) + elif isinstance(node, (ast.If, ast.With, ast.For, ast.While)): + visit(node.body) + visit(getattr(node, "orelse", [])) + visit(tree.body) + return names + + +API_SRC = os.path.join(ROOT, "services", "api", "src") +DATA_SRC = os.path.join(ROOT, "services", "data", "src") +# First-party is DERIVED from the layout, not listed: every top-level package on either src root. +FIRST_PARTY = {d for root in (API_SRC, DATA_SRC) if os.path.isdir(root) for d in os.listdir(root) + if os.path.isdir(os.path.join(root, d))} + +# One lock covers both services -- requirements.in says so in its own header ("the data service's +# runtime deps are a strict subset"). So both trees are measured against that one file, which also +# keeps the header's claim honest rather than merely written down. +DECLARED = declared_in(os.path.join(ROOT, "services", "api", "requirements.in")) +PKG2DIST = packages_distributions() + +print(f"first-party packages (derived from src roots): {', '.join(sorted(FIRST_PARTY))}") +print(f"declared in services/api/requirements.in: {len(DECLARED)}") + +sites = {} +for tree_root in (API_SRC, DATA_SRC): + for dirpath, _, files in os.walk(tree_root): + for fname in files: + if not fname.endswith(".py"): + continue + path = os.path.join(dirpath, fname) + with open(path, encoding="utf-8") as fh: + try: + parsed = ast.parse(fh.read()) + except SyntaxError: + continue + for mod in module_scope_imports(parsed): + if mod and mod not in FIRST_PARTY and mod not in sys.stdlib_module_names: + sites.setdefault(mod, set()).add(os.path.relpath(path, ROOT).replace("\\", "/")) + +print(f"distinct third-party module-scope imports: {len(sites)}\n") + +for mod in sorted(sites): + where = sorted(sites[mod]) + extra = f" (+{len(where) - 1} more)" if len(where) > 1 else "" + dists = PKG2DIST.get(mod, []) + if not dists: + check(f"`import {mod}` resolves to an installed distribution", False, + f"not installed, yet imported at module scope by {where[0]}{extra}") + continue + provider = "/".join(dists) + check(f"`import {mod}` is declared ({provider})", + any(norm(d) in DECLARED for d in dists), + f"provided by {provider}, which requirements.in does not declare -- imported at module " + f"scope by {where[0]}{extra}. Add it to requirements.in, or make the import " + f"function-local if it is genuinely optional") + +print() +if FAILED: + print("FAILED:", ", ".join(FAILED)) + sys.exit(1) +print("test_declared_imports OK")