From 46a180c423e1722651d457b55d218bac92b03a77 Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Fri, 6 Nov 2026 03:44:49 +0100 Subject: [PATCH 1/2] refactor(background_tasks): follow the module layout (moves only) Constants, settings, worker engine settings, config guards, health, lifecycle, the invalidation transport, the sync session factory and tracking -> core/; the Celery app factory, runner, tenant task bases, routing, heartbeats, signals, worker settings and fleet inspection -> worker/; enqueue after commit, tenant context, the admin listing service, filters and retries -> executions/; the outbox_* files -> outbox/; retained logs -> logs/; deps and endpoints -> web/; page components and TS helpers -> ui/. tasks.py stays at the root and Celery task names are unchanged; no durable event or handler moved. Ten old paths (runner, celery_app, sync_db, tenant_context, settings, config_guards, task_routing, worker_settings, outbox_transport, outbox_worker) stay as deprecated sys.modules aliases (remove_in 0.0.46) because worker scripts and smpy_modules import them. Logger names that come from __name__ change with the move. One test-only change: a signals test now releases the tenant it claimed, so the new collection order no longer leaks the task owner into later tests. Claude-Session: https://claude.ai/code/session_01FXHA3ZicrtZ4C5QPEgxfgq --- CHANGELOG.md | 1 + docs/framework/leases.md | 2 +- docs/framework/multi-tenancy.md | 2 +- docs/module-layout.md | 6 +- docs/modules/background_tasks.md | 4 +- .../_optional/background_tasks/run_worker.py | 6 +- framework/cli/tests/test_cli_recipes.py | 2 +- .../hosting/tests/test_invalidation_wiring.py | 12 +- .../testing/simple_module_test/plugin.py | 4 +- host/tests/test_module_layout.py | 15 +- modules/background_tasks/README.md | 2 +- .../background_tasks/__init__.py | 18 +- .../background_tasks/celery_app.py | 223 +--------------- .../background_tasks/config_guards.py | 223 +--------------- .../background_tasks/contracts/outbox.py | 2 +- .../background_tasks/contracts/schemas.py | 2 +- .../background_tasks/core/__init__.py | 7 + .../background_tasks/{ => core}/_lifecycle.py | 34 +-- .../background_tasks/core/config_guards.py | 220 +++++++++++++++ .../background_tasks/{ => core}/constants.py | 0 .../{ => core}/db_settings.py | 0 .../background_tasks/{ => core}/health.py | 0 .../{ => core}/invalidation.py | 0 .../background_tasks/{ => core}/services.py | 4 +- .../background_tasks/core/settings.py | 175 ++++++++++++ .../{ => core}/settings_celery.py | 8 +- .../{ => core}/settings_env_checks.py | 2 +- .../{ => core}/settings_logs.py | 2 +- .../background_tasks/core/sync_db.py | 246 +++++++++++++++++ .../background_tasks/{ => core}/tracking.py | 8 +- .../background_tasks/executions/__init__.py | 5 + .../{ => executions}/after_commit.py | 0 .../{ => executions}/filters.py | 2 +- .../{ => executions}/retry_service.py | 20 +- .../{ => executions}/service.py | 6 +- .../executions/tenant_context.py | 220 +++++++++++++++ .../background_tasks/logs/__init__.py | 4 + .../{log_purge.py => logs/purge.py} | 0 .../{log_service.py => logs/service.py} | 2 +- .../{log_sink.py => logs/sink.py} | 10 +- .../background_tasks/models.py | 2 +- .../background_tasks/module.py | 26 +- .../background_tasks/outbox/__init__.py | 5 + .../{outbox_claim.py => outbox/claim.py} | 4 +- .../{outbox_clock.py => outbox/clock.py} | 0 .../dispatch.py} | 8 +- .../{outbox_guard.py => outbox/guard.py} | 0 .../{outbox_loop.py => outbox/loop.py} | 0 .../{outbox_service.py => outbox/service.py} | 6 +- .../{outbox_support.py => outbox/support.py} | 4 +- .../{outbox_sweep.py => outbox/sweep.py} | 6 +- .../background_tasks/outbox/transport.py | 198 ++++++++++++++ .../background_tasks/outbox/worker.py | 188 +++++++++++++ .../background_tasks/outbox_transport.py | 203 +------------- .../background_tasks/outbox_worker.py | 193 +------------- .../background_tasks/pages/Detail.tsx | 18 +- .../background_tasks/pages/Index.tsx | 25 +- .../background_tasks/pages/Logs.tsx | 8 +- .../background_tasks/pages/Outbox.tsx | 6 +- .../background_tasks/pages/Workers.tsx | 13 +- .../background_tasks/runner.py | 88 +----- .../background_tasks/settings.py | 178 +------------ .../background_tasks/sync_db.py | 251 +----------------- .../background_tasks/task_routing.py | 161 +---------- .../background_tasks/tasks.py | 16 +- .../background_tasks/tenant_context.py | 223 +--------------- .../{pages => ui}/components/DetailFacts.tsx | 2 +- .../{pages => ui}/components/ExecutionRow.tsx | 4 +- .../components/ExecutionsTable.tsx | 2 +- .../components/LogLevelSelect.tsx | 2 +- .../{pages => ui}/components/LogTable.tsx | 4 +- .../{pages => ui}/components/LogsPanel.tsx | 2 +- .../{pages => ui}/components/OutboxTable.tsx | 4 +- .../{pages => ui}/components/PayloadCards.tsx | 2 +- .../{pages => ui}/components/ProgressCard.tsx | 2 +- .../components/RetryAllDialog.tsx | 2 +- .../components/RetryConfirmDialog.tsx | 2 +- .../{pages => ui}/components/RoutesTable.tsx | 2 +- .../{pages => ui}/components/StatusPill.tsx | 2 +- .../{pages => ui}/components/StatusStrip.tsx | 2 +- .../{pages => ui}/components/TaskFilters.tsx | 2 +- .../{pages => ui}/components/TasksEmpty.tsx | 4 +- .../components/TracebackCard.tsx | 0 .../{pages => ui}/components/WorkerCard.tsx | 2 +- .../components/WorkerEmptyStates.tsx | 0 .../components/WorkerHealthBanner.tsx | 6 +- .../{pages => ui/utils}/constants.ts | 0 .../{pages => ui/utils}/logs.ts | 2 +- .../{pages => ui/utils}/outbox.ts | 2 +- .../{pages => ui/utils}/retry.ts | 2 +- .../{pages => ui/utils}/worker-health.ts | 0 .../background_tasks/web/__init__.py | 1 + .../background_tasks/{ => web}/deps.py | 8 +- .../{ => web}/endpoints/__init__.py | 0 .../{ => web}/endpoints/api_admin.py | 20 +- .../{ => web}/endpoints/api_status.py | 2 +- .../{ => web}/endpoints/logs_api.py | 13 +- .../{ => web}/endpoints/logs_views.py | 6 +- .../{ => web}/endpoints/outbox_api.py | 8 +- .../{ => web}/endpoints/outbox_views.py | 6 +- .../{ => web}/endpoints/views.py | 12 +- .../background_tasks/worker/__init__.py | 6 + .../{ => worker}/_signal_bus.py | 0 .../{ => worker}/_signal_support.py | 10 +- .../background_tasks/worker/celery_app.py | 224 ++++++++++++++++ .../fleet_state.py} | 2 +- .../inspector.py} | 0 .../{ => worker}/log_context.py | 0 .../background_tasks/worker/runner.py | 85 ++++++ .../background_tasks/worker/settings.py | 84 ++++++ .../background_tasks/{ => worker}/signals.py | 36 +-- .../{ => worker}/task_heartbeat.py | 10 +- .../{ => worker}/task_resource.py | 4 +- .../background_tasks/worker/task_routing.py | 158 +++++++++++ .../{ => worker}/tenant_celery.py | 2 +- .../background_tasks/worker_settings.py | 89 +------ .../background_tasks_DetailPage.test.tsx | 2 +- .../background_tasks_IndexPage.test.tsx | 2 +- .../background_tasks_LogsTab.test.tsx | 2 +- .../background_tasks_OutboxPage.test.tsx | 2 +- ...ckground_tasks_WorkerHealthBanner.test.tsx | 2 +- .../background_tasks_WorkersPage.test.tsx | 2 +- .../background_tasks/tests-js/outbox.test.ts | 2 +- .../background_tasks/tests-js/retry.test.ts | 2 +- .../tests-js/task-formatters.test.ts | 2 +- modules/background_tasks/tests/conftest.py | 4 +- .../test_background_tasks_redis_health.py} | 2 +- .../test_background_tasks_invalidation.py} | 8 +- ...background_tasks_invalidation_degraded.py} | 8 +- ...st_background_tasks_invalidation_redis.py} | 2 +- .../test_background_tasks_beat_schedule.py} | 6 +- .../test_background_tasks_config_guards.py} | 10 +- .../test_background_tasks_db_timeouts.py} | 7 +- .../test_background_tasks_redis_url.py} | 4 +- .../test_background_tasks_settings.py | 2 +- .../test_background_tasks_settings_env.py} | 4 +- ...t_background_tasks_startup_log_masking.py} | 0 .../sync_db/test_background_tasks_sync_db.py} | 2 +- .../test_background_tasks_sync_db_fork.py} | 2 +- ...est_background_tasks_retry_failed_bulk.py} | 2 +- ...t_background_tasks_retry_failed_guards.py} | 4 +- ..._background_tasks_retry_sweep_ordering.py} | 6 +- .../test_background_tasks_after_commit.py | 9 +- .../test_background_tasks_eager_bindings.py | 10 +- ..._background_tasks_index_has_executions.py} | 2 +- ...est_background_tasks_list_item_payload.py} | 2 +- ...t_background_tasks_newest_first_status.py} | 4 +- .../test_background_tasks_order_index.py | 2 +- ...est_background_tasks_publish_validation.py | 6 +- .../test_background_tasks_read_projection.py | 4 +- ...test_background_tasks_retention_retries.py | 5 +- ...test_background_tasks_savepoint_publish.py | 3 +- .../service/test_background_tasks_service.py} | 4 +- .../test_background_tasks_tz_columns.py | 4 +- .../purge/test_background_tasks_log_purge.py} | 6 +- .../test_background_tasks_log_context.py} | 4 +- .../sink/test_background_tasks_log_sink.py} | 7 +- .../test_background_tasks_log_sink_loop.py} | 5 +- .../test_background_tasks_log_sink_nul.py} | 2 +- .../test_background_tasks_outbox_admin.py} | 6 +- .../test_background_tasks_outbox_tenancy.py} | 2 +- .../test_background_tasks_delivery_context.py | 4 +- .../test_background_tasks_inprocess_sweep.py} | 10 +- ...st_background_tasks_outbox_concurrency.py} | 6 +- .../test_background_tasks_outbox_delivery.py} | 6 +- ...background_tasks_outbox_eager_platform.py} | 6 +- ...round_tasks_outbox_eager_serialization.py} | 2 +- ...st_background_tasks_outbox_fence_guard.py} | 6 +- ...ackground_tasks_outbox_guard_savepoint.py} | 6 +- ..._background_tasks_outbox_publish_scope.py} | 6 +- ...t_background_tasks_outbox_retry_policy.py} | 10 +- ...ckground_tasks_outbox_swallowed_cancel.py} | 6 +- ...t_background_tasks_outbox_worker_chain.py} | 12 +- ...st_background_tasks_outbox_worker_task.py} | 9 +- ...background_tasks_outbox_loop_interrupt.py} | 2 +- ...t_background_tasks_outbox_thread_loops.py} | 5 +- .../api/test_background_tasks_admin_api.py} | 2 +- .../test_background_tasks_detail_props.py} | 2 +- .../test_background_tasks_index_filters.py} | 2 +- .../api/test_background_tasks_log_api.py} | 0 .../api/test_background_tasks_page_clamp.py} | 2 +- ...est_background_tasks_pagination_bounds.py} | 0 .../test_background_tasks_task_status_api.py} | 2 +- .../test_background_tasks_users_ordering.py} | 0 ...background_tasks_index_worker_presence.py} | 2 +- ...test_background_tasks_worker_inspector.py} | 14 +- ...est_background_tasks_workers_endpoints.py} | 6 +- .../test_background_tasks_workers_uptime.py} | 14 +- ...t_background_tasks_routes_override_api.py} | 2 +- .../test_background_tasks_runner_settings.py} | 16 +- .../test_background_tasks_task_routing.py} | 10 +- .../test_background_tasks_worker_settings.py} | 4 +- .../test_background_tasks_argument_json.py | 8 +- ...est_background_tasks_execution_bindings.py | 8 +- .../test_background_tasks_heartbeat.py | 7 +- ...ackground_tasks_quiet_sweep_unrecorded.py} | 10 +- .../test_background_tasks_resource_status.py | 2 +- ...est_background_tasks_resource_transport.py | 12 +- .../test_background_tasks_result_json.py | 9 +- ...t_background_tasks_signal_tenant_stamp.py} | 6 +- .../signals/test_background_tasks_signals.py} | 16 +- .../test_background_tasks_sweep_stuck.py} | 4 +- .../test_background_tasks_task_progress.py} | 11 +- ...t_background_tasks_task_progress_audit.py} | 7 +- .../test_background_tasks_tracking_modes.py | 22 +- .../test_background_tasks_worker_audit.py} | 6 +- ...kground_tasks_worker_audit_bookkeeping.py} | 6 +- .../test_background_tasks_custom_bases.py | 6 +- ...est_background_tasks_execution_tenancy.py} | 10 +- .../test_background_tasks_tenant_context.py} | 2 +- .../test_background_tasks_worker_tenancy.py | 2 +- ...ackground_tasks_worker_tenancy_context.py} | 4 +- .../file_storage/file_storage/core/worker.py | 4 +- .../file_storage/core/worker_support.py | 2 +- modules/file_storage/file_storage/tasks.py | 8 +- .../tests/_file_storage_worker_support.py | 5 +- .../test_file_storage_tenant_deletion.py | 8 +- .../worker/test_file_storage_object_reaper.py | 2 +- .../worker/test_file_storage_upload_tasks.py | 4 +- .../core/worker/test_file_storage_worker.py | 2 +- .../test_file_storage_worker_failures.py | 2 +- ...st_file_storage_checksum_reconciliation.py | 2 +- .../test_settings_default_clears_override.py | 2 +- ...tings_lenient_related_fields_validators.py | 2 +- .../hydrate/test_settings_hydrate.py | 2 +- .../test_settings_module_api_json_fields.py | 2 +- .../tests/_tenants_deletion_support.py | 2 +- .../test_tenants_teardown_integration.py | 4 +- .../users/tests/test_users_deletion_events.py | 2 +- scripts/run_worker.py | 6 +- 230 files changed, 2477 insertions(+), 2263 deletions(-) create mode 100644 modules/background_tasks/background_tasks/core/__init__.py rename modules/background_tasks/background_tasks/{ => core}/_lifecycle.py (85%) create mode 100644 modules/background_tasks/background_tasks/core/config_guards.py rename modules/background_tasks/background_tasks/{ => core}/constants.py (100%) rename modules/background_tasks/background_tasks/{ => core}/db_settings.py (100%) rename modules/background_tasks/background_tasks/{ => core}/health.py (100%) rename modules/background_tasks/background_tasks/{ => core}/invalidation.py (100%) rename modules/background_tasks/background_tasks/{ => core}/services.py (92%) create mode 100644 modules/background_tasks/background_tasks/core/settings.py rename modules/background_tasks/background_tasks/{ => core}/settings_celery.py (97%) rename modules/background_tasks/background_tasks/{ => core}/settings_env_checks.py (97%) rename modules/background_tasks/background_tasks/{ => core}/settings_logs.py (97%) create mode 100644 modules/background_tasks/background_tasks/core/sync_db.py rename modules/background_tasks/background_tasks/{ => core}/tracking.py (91%) create mode 100644 modules/background_tasks/background_tasks/executions/__init__.py rename modules/background_tasks/background_tasks/{ => executions}/after_commit.py (100%) rename modules/background_tasks/background_tasks/{ => executions}/filters.py (96%) rename modules/background_tasks/background_tasks/{ => executions}/retry_service.py (94%) rename modules/background_tasks/background_tasks/{ => executions}/service.py (97%) create mode 100644 modules/background_tasks/background_tasks/executions/tenant_context.py create mode 100644 modules/background_tasks/background_tasks/logs/__init__.py rename modules/background_tasks/background_tasks/{log_purge.py => logs/purge.py} (100%) rename modules/background_tasks/background_tasks/{log_service.py => logs/service.py} (97%) rename modules/background_tasks/background_tasks/{log_sink.py => logs/sink.py} (93%) create mode 100644 modules/background_tasks/background_tasks/outbox/__init__.py rename modules/background_tasks/background_tasks/{outbox_claim.py => outbox/claim.py} (98%) rename modules/background_tasks/background_tasks/{outbox_clock.py => outbox/clock.py} (100%) rename modules/background_tasks/background_tasks/{outbox_dispatch.py => outbox/dispatch.py} (98%) rename modules/background_tasks/background_tasks/{outbox_guard.py => outbox/guard.py} (100%) rename modules/background_tasks/background_tasks/{outbox_loop.py => outbox/loop.py} (100%) rename modules/background_tasks/background_tasks/{outbox_service.py => outbox/service.py} (97%) rename modules/background_tasks/background_tasks/{outbox_support.py => outbox/support.py} (95%) rename modules/background_tasks/background_tasks/{outbox_sweep.py => outbox/sweep.py} (90%) create mode 100644 modules/background_tasks/background_tasks/outbox/transport.py create mode 100644 modules/background_tasks/background_tasks/outbox/worker.py rename modules/background_tasks/background_tasks/{pages => ui}/components/DetailFacts.tsx (99%) rename modules/background_tasks/background_tasks/{pages => ui}/components/ExecutionRow.tsx (97%) rename modules/background_tasks/background_tasks/{pages => ui}/components/ExecutionsTable.tsx (98%) rename modules/background_tasks/background_tasks/{pages => ui}/components/LogLevelSelect.tsx (93%) rename modules/background_tasks/background_tasks/{pages => ui}/components/LogTable.tsx (96%) rename modules/background_tasks/background_tasks/{pages => ui}/components/LogsPanel.tsx (98%) rename modules/background_tasks/background_tasks/{pages => ui}/components/OutboxTable.tsx (98%) rename modules/background_tasks/background_tasks/{pages => ui}/components/PayloadCards.tsx (96%) rename modules/background_tasks/background_tasks/{pages => ui}/components/ProgressCard.tsx (96%) rename modules/background_tasks/background_tasks/{pages => ui}/components/RetryAllDialog.tsx (97%) rename modules/background_tasks/background_tasks/{pages => ui}/components/RetryConfirmDialog.tsx (98%) rename modules/background_tasks/background_tasks/{pages => ui}/components/RoutesTable.tsx (96%) rename modules/background_tasks/background_tasks/{pages => ui}/components/StatusPill.tsx (96%) rename modules/background_tasks/background_tasks/{pages => ui}/components/StatusStrip.tsx (98%) rename modules/background_tasks/background_tasks/{pages => ui}/components/TaskFilters.tsx (99%) rename modules/background_tasks/background_tasks/{pages => ui}/components/TasksEmpty.tsx (97%) rename modules/background_tasks/background_tasks/{pages => ui}/components/TracebackCard.tsx (100%) rename modules/background_tasks/background_tasks/{pages => ui}/components/WorkerCard.tsx (99%) rename modules/background_tasks/background_tasks/{pages => ui}/components/WorkerEmptyStates.tsx (100%) rename modules/background_tasks/background_tasks/{pages => ui}/components/WorkerHealthBanner.tsx (96%) rename modules/background_tasks/background_tasks/{pages => ui/utils}/constants.ts (100%) rename modules/background_tasks/background_tasks/{pages => ui/utils}/logs.ts (97%) rename modules/background_tasks/background_tasks/{pages => ui/utils}/outbox.ts (98%) rename modules/background_tasks/background_tasks/{pages => ui/utils}/retry.ts (99%) rename modules/background_tasks/background_tasks/{pages => ui/utils}/worker-health.ts (100%) create mode 100644 modules/background_tasks/background_tasks/web/__init__.py rename modules/background_tasks/background_tasks/{ => web}/deps.py (89%) rename modules/background_tasks/background_tasks/{ => web}/endpoints/__init__.py (100%) rename modules/background_tasks/background_tasks/{ => web}/endpoints/api_admin.py (94%) rename modules/background_tasks/background_tasks/{ => web}/endpoints/api_status.py (97%) rename modules/background_tasks/background_tasks/{ => web}/endpoints/logs_api.py (86%) rename modules/background_tasks/background_tasks/{ => web}/endpoints/logs_views.py (88%) rename modules/background_tasks/background_tasks/{ => web}/endpoints/outbox_api.py (92%) rename modules/background_tasks/background_tasks/{ => web}/endpoints/outbox_views.py (89%) rename modules/background_tasks/background_tasks/{ => web}/endpoints/views.py (95%) create mode 100644 modules/background_tasks/background_tasks/worker/__init__.py rename modules/background_tasks/background_tasks/{ => worker}/_signal_bus.py (100%) rename modules/background_tasks/background_tasks/{ => worker}/_signal_support.py (95%) create mode 100644 modules/background_tasks/background_tasks/worker/celery_app.py rename modules/background_tasks/background_tasks/{workers_state.py => worker/fleet_state.py} (94%) rename modules/background_tasks/background_tasks/{worker_inspector.py => worker/inspector.py} (100%) rename modules/background_tasks/background_tasks/{ => worker}/log_context.py (100%) create mode 100644 modules/background_tasks/background_tasks/worker/runner.py create mode 100644 modules/background_tasks/background_tasks/worker/settings.py rename modules/background_tasks/background_tasks/{ => worker}/signals.py (94%) rename modules/background_tasks/background_tasks/{ => worker}/task_heartbeat.py (94%) rename modules/background_tasks/background_tasks/{ => worker}/task_resource.py (94%) create mode 100644 modules/background_tasks/background_tasks/worker/task_routing.py rename modules/background_tasks/background_tasks/{ => worker}/tenant_celery.py (97%) rename modules/background_tasks/tests/{test_bg_redis_health.py => core/health/test_background_tasks_redis_health.py} (98%) rename modules/background_tasks/tests/{test_bg_invalidation.py => core/invalidation/test_background_tasks_invalidation.py} (97%) rename modules/background_tasks/tests/{test_bg_invalidation_degraded.py => core/invalidation/test_background_tasks_invalidation_degraded.py} (96%) rename modules/background_tasks/tests/{test_bg_invalidation_redis.py => core/invalidation/test_background_tasks_invalidation_redis.py} (99%) rename modules/background_tasks/tests/{test_beat_schedule.py => core/settings/test_background_tasks_beat_schedule.py} (96%) rename modules/background_tasks/tests/{test_bg_config_guards.py => core/settings/test_background_tasks_config_guards.py} (94%) rename modules/background_tasks/tests/{test_bg_db_timeouts.py => core/settings/test_background_tasks_db_timeouts.py} (92%) rename modules/background_tasks/tests/{test_bg_redis_url.py => core/settings/test_background_tasks_redis_url.py} (97%) rename modules/background_tasks/tests/{ => core/settings}/test_background_tasks_settings.py (98%) rename modules/background_tasks/tests/{test_bg_settings_env.py => core/settings/test_background_tasks_settings_env.py} (97%) rename modules/background_tasks/tests/{test_bg_startup_log_masking.py => core/settings/test_background_tasks_startup_log_masking.py} (100%) rename modules/background_tasks/tests/{test_sync_db.py => core/sync_db/test_background_tasks_sync_db.py} (97%) rename modules/background_tasks/tests/{test_sync_db_fork.py => core/sync_db/test_background_tasks_sync_db_fork.py} (98%) rename modules/background_tasks/tests/{test_retry_failed_bulk.py => executions/retry/test_background_tasks_retry_failed_bulk.py} (99%) rename modules/background_tasks/tests/{test_retry_failed_guards.py => executions/retry/test_background_tasks_retry_failed_guards.py} (98%) rename modules/background_tasks/tests/{test_retry_sweep_ordering.py => executions/retry/test_background_tasks_retry_sweep_ordering.py} (97%) rename modules/background_tasks/tests/{ => executions/service}/test_background_tasks_after_commit.py (96%) rename modules/background_tasks/tests/{ => executions/service}/test_background_tasks_eager_bindings.py (84%) rename modules/background_tasks/tests/{test_bg_index_has_executions.py => executions/service/test_background_tasks_index_has_executions.py} (97%) rename modules/background_tasks/tests/{test_bg_list_item_payload.py => executions/service/test_background_tasks_list_item_payload.py} (97%) rename modules/background_tasks/tests/{test_bg_newest_first_status.py => executions/service/test_background_tasks_newest_first_status.py} (93%) rename modules/background_tasks/tests/{ => executions/service}/test_background_tasks_order_index.py (98%) rename modules/background_tasks/tests/{ => executions/service}/test_background_tasks_publish_validation.py (93%) rename modules/background_tasks/tests/{ => executions/service}/test_background_tasks_read_projection.py (96%) rename modules/background_tasks/tests/{ => executions/service}/test_background_tasks_retention_retries.py (96%) rename modules/background_tasks/tests/{ => executions/service}/test_background_tasks_savepoint_publish.py (95%) rename modules/background_tasks/tests/{test_bg_service.py => executions/service/test_background_tasks_service.py} (98%) rename modules/background_tasks/tests/{ => executions/service}/test_background_tasks_tz_columns.py (93%) rename modules/background_tasks/tests/{test_bg_log_purge.py => logs/purge/test_background_tasks_log_purge.py} (86%) rename modules/background_tasks/tests/{test_log_context.py => logs/sink/test_background_tasks_log_context.py} (97%) rename modules/background_tasks/tests/{test_bg_log_sink.py => logs/sink/test_background_tasks_log_sink.py} (93%) rename modules/background_tasks/tests/{test_bg_log_sink_loop.py => logs/sink/test_background_tasks_log_sink_loop.py} (92%) rename modules/background_tasks/tests/{test_bg_log_sink_nul.py => logs/sink/test_background_tasks_log_sink_nul.py} (93%) rename modules/background_tasks/tests/{test_outbox_admin.py => outbox/admin/test_background_tasks_outbox_admin.py} (96%) rename modules/background_tasks/tests/{test_outbox_tenancy.py => outbox/admin/test_background_tasks_outbox_tenancy.py} (97%) rename modules/background_tasks/tests/{ => outbox/delivery}/test_background_tasks_delivery_context.py (95%) rename modules/background_tasks/tests/{test_bg_inprocess_sweep.py => outbox/delivery/test_background_tasks_inprocess_sweep.py} (94%) rename modules/background_tasks/tests/{test_outbox_concurrency.py => outbox/delivery/test_background_tasks_outbox_concurrency.py} (97%) rename modules/background_tasks/tests/{test_outbox_delivery.py => outbox/delivery/test_background_tasks_outbox_delivery.py} (97%) rename modules/background_tasks/tests/{test_bg_outbox_eager_platform.py => outbox/delivery/test_background_tasks_outbox_eager_platform.py} (95%) rename modules/background_tasks/tests/{test_bg_outbox_eager_serialization.py => outbox/delivery/test_background_tasks_outbox_eager_serialization.py} (97%) rename modules/background_tasks/tests/{test_bg_outbox_fence_guard.py => outbox/delivery/test_background_tasks_outbox_fence_guard.py} (97%) rename modules/background_tasks/tests/{test_bg_outbox_guard_savepoint.py => outbox/delivery/test_background_tasks_outbox_guard_savepoint.py} (96%) rename modules/background_tasks/tests/{test_bg_outbox_publish_scope.py => outbox/delivery/test_background_tasks_outbox_publish_scope.py} (96%) rename modules/background_tasks/tests/{test_bg_outbox_retry_policy.py => outbox/delivery/test_background_tasks_outbox_retry_policy.py} (95%) rename modules/background_tasks/tests/{test_bg_outbox_swallowed_cancel.py => outbox/delivery/test_background_tasks_outbox_swallowed_cancel.py} (91%) rename modules/background_tasks/tests/{test_bg_outbox_worker_chain.py => outbox/delivery/test_background_tasks_outbox_worker_chain.py} (94%) rename modules/background_tasks/tests/{test_outbox_worker_task.py => outbox/delivery/test_background_tasks_outbox_worker_task.py} (94%) rename modules/background_tasks/tests/{test_bg_outbox_loop_interrupt.py => outbox/loop/test_background_tasks_outbox_loop_interrupt.py} (94%) rename modules/background_tasks/tests/{test_bg_outbox_thread_loops.py => outbox/loop/test_background_tasks_outbox_thread_loops.py} (94%) rename modules/background_tasks/tests/{test_admin_api.py => web/api/test_background_tasks_admin_api.py} (99%) rename modules/background_tasks/tests/{test_detail_props.py => web/api/test_background_tasks_detail_props.py} (97%) rename modules/background_tasks/tests/{test_index_filters.py => web/api/test_background_tasks_index_filters.py} (99%) rename modules/background_tasks/tests/{test_bg_log_api.py => web/api/test_background_tasks_log_api.py} (100%) rename modules/background_tasks/tests/{test_bg_page_clamp.py => web/api/test_background_tasks_page_clamp.py} (97%) rename modules/background_tasks/tests/{test_bg_pagination_bounds.py => web/api/test_background_tasks_pagination_bounds.py} (100%) rename modules/background_tasks/tests/{test_bg_task_status_api.py => web/api/test_background_tasks_task_status_api.py} (98%) rename modules/background_tasks/tests/{test_bg_users_ordering.py => web/api/test_background_tasks_users_ordering.py} (100%) rename modules/background_tasks/tests/{test_bg_index_worker_presence.py => worker/inspector/test_background_tasks_index_worker_presence.py} (99%) rename modules/background_tasks/tests/{test_worker_inspector.py => worker/inspector/test_background_tasks_worker_inspector.py} (93%) rename modules/background_tasks/tests/{test_workers_endpoints.py => worker/inspector/test_background_tasks_workers_endpoints.py} (96%) rename modules/background_tasks/tests/{test_workers_uptime.py => worker/inspector/test_background_tasks_workers_uptime.py} (95%) rename modules/background_tasks/tests/{test_bg_routes_override_api.py => worker/runner/test_background_tasks_routes_override_api.py} (96%) rename modules/background_tasks/tests/{test_bg_runner_settings.py => worker/runner/test_background_tasks_runner_settings.py} (93%) rename modules/background_tasks/tests/{test_bg_task_routing.py => worker/runner/test_background_tasks_task_routing.py} (97%) rename modules/background_tasks/tests/{test_bg_worker_settings.py => worker/runner/test_background_tasks_worker_settings.py} (94%) rename modules/background_tasks/tests/{ => worker/signals}/test_background_tasks_argument_json.py (94%) rename modules/background_tasks/tests/{ => worker/signals}/test_background_tasks_execution_bindings.py (92%) rename modules/background_tasks/tests/{ => worker/signals}/test_background_tasks_heartbeat.py (96%) rename modules/background_tasks/tests/{test_bg_quiet_sweep_unrecorded.py => worker/signals/test_background_tasks_quiet_sweep_unrecorded.py} (85%) rename modules/background_tasks/tests/{ => worker/signals}/test_background_tasks_resource_status.py (99%) rename modules/background_tasks/tests/{ => worker/signals}/test_background_tasks_resource_transport.py (91%) rename modules/background_tasks/tests/{ => worker/signals}/test_background_tasks_result_json.py (90%) rename modules/background_tasks/tests/{test_signal_tenant_stamp.py => worker/signals/test_background_tasks_signal_tenant_stamp.py} (95%) rename modules/background_tasks/tests/{test_signals.py => worker/signals/test_background_tasks_signals.py} (93%) rename modules/background_tasks/tests/{test_sweep_stuck.py => worker/signals/test_background_tasks_sweep_stuck.py} (97%) rename modules/background_tasks/tests/{test_bg_task_progress.py => worker/signals/test_background_tasks_task_progress.py} (92%) rename modules/background_tasks/tests/{test_bg_task_progress_audit.py => worker/signals/test_background_tasks_task_progress_audit.py} (89%) rename modules/background_tasks/tests/{ => worker/signals}/test_background_tasks_tracking_modes.py (92%) rename modules/background_tasks/tests/{test_bg_worker_audit.py => worker/signals/test_background_tasks_worker_audit.py} (96%) rename modules/background_tasks/tests/{test_bg_worker_audit_bookkeeping.py => worker/signals/test_background_tasks_worker_audit_bookkeeping.py} (95%) rename modules/background_tasks/tests/{ => worker/tenancy}/test_background_tasks_custom_bases.py (93%) rename modules/background_tasks/tests/{test_execution_tenancy.py => worker/tenancy/test_background_tasks_execution_tenancy.py} (95%) rename modules/background_tasks/tests/{test_tenant_context.py => worker/tenancy/test_background_tasks_tenant_context.py} (97%) rename modules/background_tasks/tests/{ => worker/tenancy}/test_background_tasks_worker_tenancy.py (96%) rename modules/background_tasks/tests/{test_worker_tenancy.py => worker/tenancy/test_background_tasks_worker_tenancy_context.py} (96%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 526d280d9..dd951856c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,7 @@ All notable changes to this project are documented in this file. The format is b - `mail` follows the module layout: `mail.core.{constants,settings,state,data_compatibility}` and `mail.delivery.{sender,runtime}`. `mail.settings` and `mail.constants` still import, with a `DeprecationWarning`, until 0.0.46. - `site_lock` follows the module layout: `site_lock.core.{constants,settings,state}` and `site_lock.web.{middleware,page,rate_limit}` (the unlock template moves with the page). `site_lock.settings` still imports, with a `DeprecationWarning`, until 0.0.46. - `sharing` follows the module layout: `sharing.core.{constants,settings,services,service,policy,audience,tenant_cleanup,user_cleanup}`, `sharing.web.{deps,present,endpoints}`, and the page components and types move to `ui/`. `sharing.policy`, `sharing.constants` and `sharing.endpoints.api` still import, with a `DeprecationWarning`, until 0.0.46; new code imports `sharing.core.policy`. +- `background_tasks` follows the module layout: `background_tasks.core.*` (constants, settings, services, config guards, health, invalidation, `sync_db`, tracking), `background_tasks.worker.*` (`celery_app`, `runner`, signals, task routing, heartbeats, worker settings, fleet inspection), `background_tasks.{executions,outbox,logs,web}.*`; the page components and TS helpers move to `ui/`. `background_tasks.{runner,celery_app,settings,sync_db,tenant_context,config_guards,task_routing,worker_settings,outbox_transport,outbox_worker}` still import, with a `DeprecationWarning`, until 0.0.46, so existing `scripts/run_worker.py` files keep working. Celery task names, durable event and handler names and the package-root exports are unchanged; module-name loggers move with their modules. - `dashboard` follows the module layout: `dashboard.core.constants`, `dashboard.web.{endpoints,stats,doctor}`, and the page components move from `pages/components/` to `ui/components/`. Nothing outside the module imported the old paths, so no aliases remain. - `feature_flags` follows the module layout: `feature_flags.core.{constants,service,invalidation,tenant_cleanup}`, `feature_flags.web.{deps,endpoints}`, and the page components and types move to `ui/`. `feature_flags.constants` still imports, with a `DeprecationWarning`, until 0.0.46. - `keycloak` follows the module layout: `keycloak.core.{constants,settings,state}`, `keycloak.identity.{provider,oidc,renewal,directory,persistence,tokens,jwks}` and `keycloak.web.endpoints`. `keycloak.{constants,settings,provider,jwks,tokens}` still import, with a `DeprecationWarning`, until 0.0.46. diff --git a/docs/framework/leases.md b/docs/framework/leases.md index c8e27737b..94015409f 100644 --- a/docs/framework/leases.md +++ b/docs/framework/leases.md @@ -45,7 +45,7 @@ A lease does not refresh itself. Pick a TTL longer than the work, or call `refre ## Backends -The backend is process-global because a Celery worker has no `app.state`. `create_app` installs it when the app is built, and the worker bootstrap (`background_tasks.runner.build_runner_celery`) installs the same kind from the same settings: +The backend is process-global because a Celery worker has no `app.state`. `create_app` installs it when the app is built, and the worker bootstrap (`background_tasks.worker.runner.build_runner_celery`) installs the same kind from the same settings: | Configured | Backend | Notes | |---|---|---| diff --git a/docs/framework/multi-tenancy.md b/docs/framework/multi-tenancy.md index 29357f185..6990eaf27 100644 --- a/docs/framework/multi-tenancy.md +++ b/docs/framework/multi-tenancy.md @@ -300,7 +300,7 @@ An unscoped event's handler runs with no tenant bound — never as the platform — so a public webhook's payload cannot drive a cross-tenant read. See [background_tasks](../modules/background_tasks.md). -The worker never builds the app, so `background_tasks.sync_db` attaches the +The worker never builds the app, so `background_tasks.core.sync_db` attaches the same listeners to its own session class and reads `multi_tenant` from the host settings (`scripts/run_worker.py`): task bodies get the same fail-closed rules as request code. It also passes `default_tenant`, so an unbound insert in a diff --git a/docs/module-layout.md b/docs/module-layout.md index 6084d5658..1718844df 100644 --- a/docs/module-layout.md +++ b/docs/module-layout.md @@ -32,8 +32,8 @@ checkers, not a framework runtime lookup. | `py.typed` | PEP 561 consumers; `framework/cli/simple_module_cli/templates/module/pyproject.toml.tpl:33` selects the whole package for the wheel, including this marker. | | `module.py` | `framework/core/simple_module_core/discovery.py:80` loads the `simple_module` entry point; the scaffold declares its target in `framework/cli/simple_module_cli/templates/module/pyproject.toml.tpl:15`. | | `models.py` or `models/` | `framework/db/simple_module_db/migrations.py:74` imports the module's models for metadata; `framework/core/simple_module_core/diagnostics/_tenancy.py:94` and `framework/core/simple_module_core/diagnostics/_expression_index.py:52` do the same. A `models/` package must load its tables. | -| `tasks.py` | `modules/background_tasks/background_tasks/celery_app.py:215` autodiscovers with `related_name="tasks"`; `framework/core/simple_module_core/diagnostics/_heavy_task_imports.py:65` checks this root file (SM028). | -| `task_config.py` | `modules/background_tasks/background_tasks/task_routing.py:46` imports `.task_config` for a module's Celery routes and beat schedule. | +| `tasks.py` | `modules/background_tasks/background_tasks/worker/celery_app.py:215` autodiscovers with `related_name="tasks"`; `framework/core/simple_module_core/diagnostics/_heavy_task_imports.py:65` checks this root file (SM028). | +| `task_config.py` | `modules/background_tasks/background_tasks/worker/task_routing.py:46` imports `.task_config` for a module's Celery routes and beat schedule. | | `contracts/` | Public imports for sibling modules; the scaffold consumer is `framework/cli/simple_module_cli/templates/module/__PACKAGE__/web/service.py.tpl:15`. Keep public SQLModel DTOs, events and real extension contracts here. | | `pages/` | `framework/hosting/simple_module_hosting/manifest.py:97` discovers pages; `framework/hosting/simple_module_hosting/page_globs.py:34` emits their Vite glob; `framework/core/simple_module_core/diagnostics/_pages.py:170` checks page names. | | `pages/api.ts`, `pages/api.generated.ts` | `framework/cli/simple_module_cli/openapi_types.py:130` selects the root pages directory for generation; `framework/cli/simple_module_cli/templates/module/__PACKAGE__/pages/api.ts.tpl:8` consumes the generated types. | @@ -215,7 +215,7 @@ when a later module PR moves files. | api_tokens | 3 (+4 shims) | 0 | follows this layout | | audit_log | 3 | 0 | follows this layout | | auth | 3 (+4 shims) | 0 | follows this layout | -| background_tasks | 49 | 83 | flat | +| background_tasks | 4 (+10 shims) | 0 | follows this layout | | branding | 2 (+6 shims) | 0 | follows this layout | | dashboard | 2 | 0 | follows this layout | | feature_flags | 3 (+1 shim) | 0 | follows this layout | diff --git a/docs/modules/background_tasks.md b/docs/modules/background_tasks.md index 23a933f5a..7aeee49e6 100644 --- a/docs/modules/background_tasks.md +++ b/docs/modules/background_tasks.md @@ -226,7 +226,7 @@ Entry values are plain Celery [beat entries](https://docs.celeryq.dev/en/stable/ Beat persists its schedule (`celery.beat:PersistentScheduler`) so it survives restarts. -The worker and beat read the same DB-backed settings the web process does: `scripts/run_worker.py` (and the copy `smpy new` scaffolds) calls `background_tasks.runner.build_runner_celery()`, which hydrates `BackgroundTasksSettings` from the database at boot exactly as the web lifespan does. Except for the DB-only `execution_tracking` and `track_publish` fields, precedence is env → DB → default: an `SM_BG_TASKS_*` variable that is set wins over a stored value for the same field, and the database supplies everything the environment leaves unset. So an admin-edited queue or time limit reaches the worker after a restart, unless the deployment pins that field in its environment. Because the web process behaves identically, this adds no new trust boundary; still, treat write access to these settings as control of the workers. If the database cannot be read the worker starts from env and defaults and logs a warning. +The worker and beat read the same DB-backed settings the web process does: `scripts/run_worker.py` (and the copy `smpy new` scaffolds) calls `background_tasks.worker.runner.build_runner_celery()`, which hydrates `BackgroundTasksSettings` from the database at boot exactly as the web lifespan does. Except for the DB-only `execution_tracking` and `track_publish` fields, precedence is env → DB → default: an `SM_BG_TASKS_*` variable that is set wins over a stored value for the same field, and the database supplies everything the environment leaves unset. So an admin-edited queue or time limit reaches the worker after a restart, unless the deployment pins that field in its environment. Because the web process behaves identically, this adds no new trust boundary; still, treat write access to these settings as control of the workers. If the database cannot be read the worker starts from env and defaults and logs a warning. ## Routing and queues @@ -260,7 +260,7 @@ A publisher that never imported a task can still enqueue it by name: `enqueue_af ## Worker tuning Task bodies can inspect the host's configured tenancy with the public -`background_tasks.sync_db.worker_tenancy() -> WorkerTenancy | None` (#685). +`background_tasks.core.sync_db.worker_tenancy() -> WorkerTenancy | None` (#685). `WorkerTenancy` is an immutable SQLModel snapshot with `multi_tenant: bool` and `default_tenant: str | None`. It returns `None` until `set_database_url` pins a URL, including when a URL is only available from the environment. A configured diff --git a/framework/cli/simple_module_cli/templates/host/_optional/background_tasks/run_worker.py b/framework/cli/simple_module_cli/templates/host/_optional/background_tasks/run_worker.py index 7b57d8f1a..cdde7f8e9 100644 --- a/framework/cli/simple_module_cli/templates/host/_optional/background_tasks/run_worker.py +++ b/framework/cli/simple_module_cli/templates/host/_optional/background_tasks/run_worker.py @@ -6,17 +6,17 @@ uv run celery -A scripts.run_worker:celery beat -l info Both the web process and the worker go through the same -:func:`background_tasks.celery_app.build_celery` factory, so the broker +:func:`background_tasks.worker.celery_app.build_celery` factory, so the broker config, autodiscovered tasks, and signal handlers stay in lockstep. The bootstrap around it (``.env``, host settings, DB-hydrated module settings) -lives in :mod:`background_tasks.runner`, shared with scaffolded apps. +lives in :mod:`background_tasks.worker.runner`, shared with scaffolded apps. """ from __future__ import annotations from pathlib import Path -from background_tasks.runner import build_runner_celery +from background_tasks.worker.runner import build_runner_celery # Module-level name ``celery`` is what ``celery -A scripts.run_worker:celery`` # looks for. Keep it stable. The fallback is the repo-root ``.env`` (next to diff --git a/framework/cli/tests/test_cli_recipes.py b/framework/cli/tests/test_cli_recipes.py index e592139f3..27e55c634 100644 --- a/framework/cli/tests/test_cli_recipes.py +++ b/framework/cli/tests/test_cli_recipes.py @@ -34,7 +34,7 @@ def test_recipe_writes_run_worker_script(tmp_path: Path) -> None: text = script.read_text() # GH #490: the shared runner, which hydrates DB-backed settings — not a # local copy built from the environment alone. - assert "from background_tasks.runner import build_runner_celery" in text + assert "from background_tasks.worker.runner import build_runner_celery" in text assert "celery = build_runner_celery(" in text assert "BackgroundTasksSettings()" not in text diff --git a/framework/hosting/tests/test_invalidation_wiring.py b/framework/hosting/tests/test_invalidation_wiring.py index 5a203c6c9..022159983 100644 --- a/framework/hosting/tests/test_invalidation_wiring.py +++ b/framework/hosting/tests/test_invalidation_wiring.py @@ -107,7 +107,7 @@ class _StubState: class _StubApp: def __init__(self, bus: InvalidationBus, settings) -> None: - from background_tasks.services import BackgroundTasksServices + from background_tasks.core.services import BackgroundTasksServices self.state = _StubState() self.state.sm = _StubSm(bus) @@ -116,8 +116,8 @@ def __init__(self, bus: InvalidationBus, settings) -> None: class TestShutdown: async def test_shutdown_detaches_and_stops_the_transport(self, monkeypatch): + from background_tasks.core.settings import BackgroundTasksSettings from background_tasks.module import BackgroundTasksModule - from background_tasks.settings import BackgroundTasksSettings stopped: list[bool] = [] @@ -134,7 +134,7 @@ async def publish(self, message: str) -> None: async def stop(self) -> None: stopped.append(True) - import background_tasks.invalidation as bg_invalidation + import background_tasks.core.invalidation as bg_invalidation monkeypatch.setattr(bg_invalidation, "RedisInvalidationTransport", _FakeTransport) @@ -164,8 +164,8 @@ class TestStartupAgainstADeadBroker: async def test_an_unreachable_url_does_not_fail_the_boot(self): import socket + from background_tasks.core.settings import BackgroundTasksSettings from background_tasks.module import BackgroundTasksModule - from background_tasks.settings import BackgroundTasksSettings with socket.socket() as probe: probe.bind(("127.0.0.1", 0)) @@ -193,8 +193,8 @@ async def test_publishing_through_a_dead_broker_still_evicts_locally(self): """And the app keeps working afterwards, degraded to per-process.""" import socket + from background_tasks.core.settings import BackgroundTasksSettings from background_tasks.module import BackgroundTasksModule - from background_tasks.settings import BackgroundTasksSettings with socket.socket() as probe: probe.bind(("127.0.0.1", 0)) @@ -223,7 +223,7 @@ async def test_publishing_through_a_dead_broker_still_evicts_locally(self): class TestSettingsValidation: def test_an_empty_channel_name_is_rejected(self): """Subscribing to "" would silently hear nothing.""" - from background_tasks.settings import BackgroundTasksSettings + from background_tasks.core.settings import BackgroundTasksSettings from pydantic import ValidationError with pytest.raises(ValidationError): diff --git a/framework/testing/simple_module_test/plugin.py b/framework/testing/simple_module_test/plugin.py index 1cb870961..22725866f 100644 --- a/framework/testing/simple_module_test/plugin.py +++ b/framework/testing/simple_module_test/plugin.py @@ -92,8 +92,8 @@ def _bootstrap_eager_celery() -> None: """ os.environ.setdefault("SM_BG_TASKS_BROADCAST_INVALIDATIONS", "false") with contextlib.suppress(ImportError): - from background_tasks.celery_app import build_celery - from background_tasks.settings import BackgroundTasksSettings + from background_tasks.core.settings import BackgroundTasksSettings + from background_tasks.worker.celery_app import build_celery build_celery(BackgroundTasksSettings(task_always_eager=True, task_eager_propagates=True)) diff --git a/host/tests/test_module_layout.py b/host/tests/test_module_layout.py index 7cdd36b9e..8813197f8 100644 --- a/host/tests/test_module_layout.py +++ b/host/tests/test_module_layout.py @@ -36,6 +36,7 @@ "api_tokens": frozenset({"credentials"}), "audit_log": frozenset({"entries"}), "auth": frozenset({"oidc_common"}), + "background_tasks": frozenset({"executions", "logs", "outbox", "worker"}), "branding": frozenset({"assets"}), "file_storage": frozenset({"backends", "files", "multipart"}), "keycloak": frozenset({"identity"}), @@ -55,20 +56,6 @@ # Frozen inventory, not a computed baseline: moves must remove their old entries. LEGACY_FLAT: dict[str, frozenset[str]] = { - "background_tasks": frozenset( - """ - _lifecycle.py _signal_bus.py _signal_support.py after_commit.py celery_app.py - config_guards.py constants.py db_settings.py deps.py endpoints filters.py health.py - invalidation.py log_context.py log_purge.py log_service.py log_sink.py - outbox_claim.py outbox_clock.py outbox_dispatch.py outbox_guard.py outbox_loop.py - outbox_service.py outbox_support.py outbox_sweep.py outbox_transport.py - outbox_worker.py retry_service.py runner.py service.py services.py settings.py - settings_celery.py settings_env_checks.py settings_logs.py signals.py sync_db.py - task_heartbeat.py task_resource.py task_routing.py tenant_celery.py - tenant_context.py tracking.py worker_inspector.py worker_settings.py - workers_state.py - """.split() - ), "users": frozenset( """ admin administrators.py audit.py auth_local backend.py bootstrap.py constants.py diff --git a/modules/background_tasks/README.md b/modules/background_tasks/README.md index 8851473d0..407266ee2 100644 --- a/modules/background_tasks/README.md +++ b/modules/background_tasks/README.md @@ -14,7 +14,7 @@ Requires a Redis broker. The broker / result-backend URLs are module settings (` - Zero-config task discovery — any installed module that ships a `tasks.py` has its tasks autodiscovered (`celery.autodiscover_tasks` imports `.tasks` for every installed module). No per-module registration hook. - Admin UI at `/admin/background-tasks` — list recent runs, retry failed, inspect tracebacks (gated by the `background_tasks.view` permission). -- `build_celery(settings)` factory in `background_tasks.celery_app`, plus `bind_task_context` / `get_log_context` / `install_log_filter` exported from the package root (import name `background_tasks`, distribution name `simple_module_background_tasks`). +- `build_celery(settings)` factory in `background_tasks.worker.celery_app`, plus `bind_task_context` / `get_log_context` / `install_log_filter` exported from the package root (import name `background_tasks`, distribution name `simple_module_background_tasks`). - `enqueue_after_commit(db, task, *args, **kwargs)` — publish a task only once the session's transaction commits, never on rollback. - `heartbeat()` — called from a long task body, keeps the stuck sweep from flagging it; `@shared_task(stuck_after=seconds)` sets a per-task stuck threshold. See the module docs for both. diff --git a/modules/background_tasks/background_tasks/__init__.py b/modules/background_tasks/background_tasks/__init__.py index 7379b357f..bb3bef53a 100644 --- a/modules/background_tasks/background_tasks/__init__.py +++ b/modules/background_tasks/background_tasks/__init__.py @@ -2,19 +2,19 @@ from typing import TYPE_CHECKING, Any -from background_tasks.after_commit import enqueue_after_commit -from background_tasks.constants import SM_TRACKING_HEADER -from background_tasks.log_context import ( +from background_tasks.core.constants import SM_TRACKING_HEADER +from background_tasks.core.tracking import TrackingMode +from background_tasks.executions.after_commit import enqueue_after_commit +from background_tasks.worker.log_context import ( bind_task_context, get_log_context, install_log_filter, ) -from background_tasks.task_resource import with_task_resource -from background_tasks.tracking import TrackingMode +from background_tasks.worker.task_resource import with_task_resource if TYPE_CHECKING: - from background_tasks.task_heartbeat import heartbeat - from background_tasks.worker_settings import settings_for + from background_tasks.worker.settings import settings_for + from background_tasks.worker.task_heartbeat import heartbeat _LAZY_HEARTBEAT = "heartbeat" _LAZY_SETTINGS = "settings_for" @@ -22,11 +22,11 @@ def __getattr__(name: str) -> Any: if name == _LAZY_HEARTBEAT: - from background_tasks.task_heartbeat import heartbeat + from background_tasks.worker.task_heartbeat import heartbeat return heartbeat if name == _LAZY_SETTINGS: - from background_tasks.worker_settings import settings_for + from background_tasks.worker.settings import settings_for return settings_for raise AttributeError(name) diff --git a/modules/background_tasks/background_tasks/celery_app.py b/modules/background_tasks/background_tasks/celery_app.py index 489c48861..e1bab7cd0 100644 --- a/modules/background_tasks/background_tasks/celery_app.py +++ b/modules/background_tasks/background_tasks/celery_app.py @@ -1,224 +1,13 @@ -"""Build and configure the Celery application. - -Invoked from two places: - -- :meth:`BackgroundTasksModule.on_startup` in the web process, so the API - can enqueue tasks via ``app.state.background_tasks.celery.send_task(...)``. -- ``scripts/run_worker.py`` at worker boot, so the worker process uses an - identical config (broker URL, signal handlers, autodiscovered tasks). - -Task discovery uses :py:meth:`celery.Celery.autodiscover_tasks` with the -top-level package name of every installed module (enumerated via the -``simple_module`` entry-point group). Any module that ships a ``tasks.py`` -is picked up automatically — no framework hook, no per-module registration. -""" +"""Moved to :mod:`background_tasks.worker.celery_app`; this alias goes in 0.0.46 (module layout).""" from __future__ import annotations import importlib -import logging -from importlib.metadata import entry_points +import sys -from celery import Celery -from celery.schedules import schedule -from simple_module_core.discovery import ENTRY_POINT_GROUP +from simple_module_core.deprecation import deprecated_module -from background_tasks.config_guards import celery_overrides_from_env -from background_tasks.constants import ( - BEAT_PURGE_OLD, - BEAT_SWEEP_OUTBOX, - BEAT_SWEEP_STUCK, - INTERNAL_TASK_PURGE_OLD, - INTERNAL_TASK_SWEEP_OUTBOX, - INTERNAL_TASK_SWEEP_STUCK, - MODULE_NAME, +deprecated_module( + "background_tasks.celery_app", remove_in="0.0.46", use="background_tasks.worker.celery_app" ) -from background_tasks.settings import BackgroundTasksSettings -from background_tasks.task_routing import load_task_configs, merge_routes, select_packages -from background_tasks.tenant_celery import TenantCelery - -logger = logging.getLogger(__name__) - - -def _discover_task_packages() -> list[str]: - """Return the top-level package name of every installed simple_module. - - Celery's ``autodiscover_tasks`` imports ``.tasks`` for each name - returned here, so any module that ships a ``tasks.py`` registers its - tasks without a per-module hook. - """ - packages: set[str] = set() - for ep in entry_points(group=ENTRY_POINT_GROUP): - module_path = ep.value.split(":", 1)[0] - top_level = module_path.split(".", 1)[0] - packages.add(top_level) - return sorted(packages) - - -def _collect_module_beat_schedules(packages: list[str]) -> dict: - """Merge every installed module's optional ``tasks.BEAT_SCHEDULE`` mapping. - - A module registers periodic work the same way it registers tasks — by - shipping a ``tasks.py`` — and additionally exporting a module-level - ``BEAT_SCHEDULE`` dict of ``{entry_name: celery beat entry}``. Because - ``build_celery`` runs identically in the web and worker processes, this is - worker-safe and deterministic, unlike runtime registration via - ``add_periodic_task`` / the ``on_after_finalize`` signal (and unlike the - commonly-cited ``on_after_configure``, which never fires here — see GH #199). - - Importing ``.tasks`` is idempotent (autodiscover imports the same - modules), so doing it eagerly here has no extra side effects. - """ - merged: dict = {} - for pkg in packages: - try: - tasks_mod = importlib.import_module(f"{pkg}.tasks") - except ModuleNotFoundError as exc: - # The module simply ships no tasks.py — skip. But a tasks.py that - # *exists* and fails to import (its own dependency is missing) must - # surface loudly, not be mistaken for "no tasks.py" and silently - # drop the module's periodic work. - if exc.name in (pkg, f"{pkg}.tasks"): - continue - raise - schedule_dict = getattr(tasks_mod, "BEAT_SCHEDULE", None) - if not isinstance(schedule_dict, dict): - continue - for name, entry in schedule_dict.items(): - if name in merged: - logger.warning( - "Beat entry %r from %s.tasks overrides an earlier module's entry", name, pkg - ) - merged[name] = entry - return merged - - -# setting name -> Celery conf key (they differ only for the memory cap, which -# Celery names without the unit although it is KiB). -_TUNING_KEYS = { - "worker_max_tasks_per_child": "worker_max_tasks_per_child", - "worker_max_memory_per_child_kb": "worker_max_memory_per_child", - "task_reject_on_worker_lost": "task_reject_on_worker_lost", - "task_soft_time_limit": "task_soft_time_limit", - "task_time_limit": "task_time_limit", - "worker_concurrency": "worker_concurrency", -} - - -def _tuning_conf(settings: BackgroundTasksSettings) -> dict: - """Celery conf for the optional tuning fields; ``None`` keeps Celery's default.""" - conf = { - key: getattr(settings, name) - for name, key in _TUNING_KEYS.items() - if getattr(settings, name) is not None - } - if settings.visibility_timeout_seconds is not None: - conf["broker_transport_options"] = { - "visibility_timeout": settings.visibility_timeout_seconds - } - return conf - - -def describe_routes(settings: BackgroundTasksSettings) -> list[dict[str, str]]: - """Effective ``[{pattern, queue, module}]`` routes, for the Workers admin page.""" - configs = load_task_configs(_discover_task_packages()) - return merge_routes(configs, settings.task_routes_override)[1] - - -def build_celery(settings: BackgroundTasksSettings) -> Celery: - """Construct a Celery app wired to the project's Redis broker.""" - celery = TenantCelery(MODULE_NAME) - - all_packages = _discover_task_packages() - # Routes and beat entries come from the light ``task_config`` of *every* - # module; only the heavy ``tasks.py`` imports are narrowed to this - # process's subset (GH #450). - configs = load_task_configs(all_packages) - packages = select_packages(all_packages, configs, settings) - task_routes, _ = merge_routes(configs, settings.task_routes_override) - - # The internal entries always ship; modules contribute more via their - # own ``tasks.BEAT_SCHEDULE``. ``setdefault`` keeps the built-ins authoritative - # if a module reuses one of their entry names. See GH #199. - beat_schedule = { - BEAT_SWEEP_STUCK: { - "task": INTERNAL_TASK_SWEEP_STUCK, - "schedule": schedule(settings.stuck_sweep_interval_seconds), - }, - BEAT_PURGE_OLD: { - "task": INTERNAL_TASK_PURGE_OLD, - "schedule": schedule(settings.purge_interval_seconds), - }, - # Delivers durable events whose post-commit dispatch never ran (a crash - # between commit and publish) and retries failed ones (GH #459). - BEAT_SWEEP_OUTBOX: { - "task": INTERNAL_TASK_SWEEP_OUTBOX, - "schedule": schedule(settings.outbox_sweep_interval_seconds), - }, - } - builtin_beat = frozenset(beat_schedule) - for name, entry in _collect_module_beat_schedules(packages).items(): - if name in beat_schedule: - logger.warning( - "Module beat entry %r clashes with a built-in entry; the built-in wins", name - ) - continue - beat_schedule[name] = entry - # ``task_config.BEAT_SCHEDULE`` wins over the same entry in ``tasks.py``. - for pkg in sorted(configs): - for name, entry in configs[pkg].beat_schedule.items(): - if name in builtin_beat: - logger.warning( - "task_config beat entry %r clashes with a built-in; built-in wins", name - ) - continue - beat_schedule[name] = entry - - celery.conf.update( - sm_execution_tracking=settings.execution_tracking, - sm_track_publish=settings.track_publish, - broker_url=settings.broker_url, - result_backend=settings.result_backend, - task_default_queue=settings.task_default_queue, - # Run tasks synchronously inside the calling process. Tests - # toggle this on (via ``SM_BG_TASKS_TASK_ALWAYS_EAGER=true``) so - # ``task.delay()`` doesn't reach for a real broker. - task_always_eager=settings.task_always_eager, - task_eager_propagates=settings.task_eager_propagates, - # ``task_track_started`` gives us the ``STARTED`` state so - # ``task_prerun`` can flip our row to ``running``. - task_track_started=True, - # ``task_acks_late`` + ``worker_prefetch_multiplier=1`` gives us - # at-least-once semantics, which matches how the admin UI asks users - # to think about retries — duplicates beat lost work. - task_acks_late=True, - worker_prefetch_multiplier=1, - task_serializer="json", - result_serializer="json", - accept_content=["json"], - timezone="UTC", - enable_utc=True, - broker_connection_retry_on_startup=True, - beat_schedule=beat_schedule, - beat_scheduler="celery.beat:PersistentScheduler", - task_routes=task_routes, - ) - celery.conf.update(_tuning_conf(settings)) - overrides = celery_overrides_from_env() - if overrides: - logger.info("Applying celery_overrides: %s", sorted(overrides)) - celery.conf.update(overrides) - - # Autodiscover across every installed simple_module — a module just ships - # a `tasks.py` and the worker picks it up. - logger.info("Celery autodiscover_tasks across: %s", packages) - celery.autodiscover_tasks(packages, related_name="tasks", force=True) - - # Side-effect import: connects signal handlers to this Celery instance's - # ``celery.signals.*`` globals. Safe to import repeatedly. - from background_tasks import signals # noqa: F401 - from background_tasks.log_context import install_log_filter - - install_log_filter() - - return celery +sys.modules[__name__] = importlib.import_module("background_tasks.worker.celery_app") diff --git a/modules/background_tasks/background_tasks/config_guards.py b/modules/background_tasks/background_tasks/config_guards.py index 801cb966b..6b920fbcf 100644 --- a/modules/background_tasks/background_tasks/config_guards.py +++ b/modules/background_tasks/background_tasks/config_guards.py @@ -1,220 +1,17 @@ -"""Guards for operator-supplied Celery configuration. - -Celery conf can name serializers (``pickle``), dotted paths it imports on boot -(``imports``, ``worker_pool``, ``beat_scheduler`` ...) and signing material, so -a raw conf dict is a code-execution surface for whoever can write it. Two -inputs reach ``build_celery``: - -- ``SM_BG_TASKS_CELERY_OVERRIDES`` (JSON object): environment only, so it is - controlled by whoever deploys the process and never by an admin-UI editor. - It is read here rather than being a ``BackgroundTasksSettings`` field, because - every field of that class is stored in, and hydrated from, the database. - Dangerous keys are refused even from here. -- ``SM_BG_TASKS_WORKER_QUEUES`` / ``_INCLUDE_MODULES`` / ``_EXCLUDE_MODULES`` - (comma-separated): which queues and modules *this one worker process* - serves. Environment only for the same reason: one stored value would - re-scope every worker. All three go through the helpers below. -- ``task_routes_override`` (DB-backed, admin-editable): only names queues, and - is reduced to plain queue names and route dicts with a closed key set. A route - dict is merged into ``apply_async`` options, so ``serializer``, ``link`` and - friends must not be expressible. +"""Moved to :mod:`background_tasks.core.config_guards`; +this alias goes in 0.0.46 (module layout). """ from __future__ import annotations -import json -import logging -import os -import re -from dataclasses import dataclass -from typing import Any - -from background_tasks.constants import ENV_PREFIX - -logger = logging.getLogger(__name__) +import importlib +import sys -ENV_CELERY_OVERRIDES = f"{ENV_PREFIX}CELERY_OVERRIDES" -ENV_WORKER_QUEUES = f"{ENV_PREFIX}WORKER_QUEUES" -ENV_INCLUDE_MODULES = f"{ENV_PREFIX}INCLUDE_MODULES" -ENV_EXCLUDE_MODULES = f"{ENV_PREFIX}EXCLUDE_MODULES" +from simple_module_core.deprecation import deprecated_module -# Conf keys that choose a (de)serializer or load code by dotted path / file. -_DENIED_KEYS = frozenset( - { - "accept_content", - "task_serializer", - "result_serializer", - "result_accept_content", - "event_serializer", - "imports", - "include", - "worker_pool", - "worker_autoscaler", - "worker_consumer", - "worker_timer", - "beat_scheduler", - "beat_schedule", - "beat_schedule_filename", - "task_routes", - "task_annotations", - "task_queues", - "task_cls", - "broker_transport", - "result_backend_transport_options", - "worker_state_db", - } +deprecated_module( + "background_tasks.config_guards", + remove_in="0.0.46", + use="background_tasks.core.config_guards", ) -_DENIED_SUBSTRINGS = ("hijack", "serializer", "accept_content", "pickle") -_DENIED_PREFIXES = ("security_", "worker_pool_", "pool_") -_DENIED_SUFFIXES = ("_cls", "_class") - - -def is_denied_key(key: str) -> bool: - """True for a conf key that must not be settable from configuration.""" - # Celery also resolves the old ``CELERY_*`` spellings. - k = key.strip().lower().removeprefix("celery_") - return ( - k in _DENIED_KEYS - or k.startswith(_DENIED_PREFIXES) - or k.endswith(_DENIED_SUFFIXES) - or any(s in k for s in _DENIED_SUBSTRINGS) - ) - - -def filter_celery_overrides(raw: object, *, source: str = "celery_overrides") -> dict[str, Any]: - """Drop (and log) denied keys; non-dict input yields ``{}``.""" - if not isinstance(raw, dict): - if raw not in (None, {}): - logger.error("%s must be a JSON object; ignoring it", source) - return {} - safe: dict[str, Any] = {} - for key, value in raw.items(): - if not isinstance(key, str) or is_denied_key(key): - logger.error( - "%s: refusing Celery conf key %r (serializer, import and code-loading " - "keys cannot be set from configuration); ignoring it", - source, - key, - ) - continue - safe[key] = value - return safe - - -def _env_text(name: str) -> str: - return os.environ.get(name, "").strip() - - -def env_csv(name: str) -> list[str]: - """A comma-separated environment variable as a list (``[]`` when unset).""" - return [part.strip() for part in _env_text(name).split(",") if part.strip()] - - -def env_json_object(name: str) -> dict[str, Any]: - """A JSON-object environment variable (``{}`` when unset or unusable, with a log).""" - raw = _env_text(name) - if not raw: - return {} - try: - parsed = json.loads(raw) - except ValueError as exc: - logger.error("%s is not valid JSON (%s); ignoring it", name, exc) - return {} - if not isinstance(parsed, dict): - logger.error("%s must be a JSON object; ignoring it", name) - return {} - return parsed - - -def celery_overrides_from_env() -> dict[str, Any]: - """The vetted ``SM_BG_TASKS_CELERY_OVERRIDES`` mapping (``{}`` when unset).""" - return filter_celery_overrides( - env_json_object(ENV_CELERY_OVERRIDES), source=ENV_CELERY_OVERRIDES - ) - - -@dataclass(frozen=True) -class WorkerScope: - """What this process serves; see :func:`worker_scope_from_env`.""" - - queues: frozenset[str] = frozenset() - include_modules: frozenset[str] = frozenset() - exclude_modules: frozenset[str] = frozenset() - - -def worker_scope_from_env() -> WorkerScope: - """Per-process queue/module subset (GH #450), from the environment only.""" - return WorkerScope( - queues=frozenset(env_csv(ENV_WORKER_QUEUES)), - include_modules=frozenset(env_csv(ENV_INCLUDE_MODULES)), - exclude_modules=frozenset(env_csv(ENV_EXCLUDE_MODULES)), - ) - - -_QUEUE_RE = re.compile(r"^[A-Za-z0-9_.:\-]{1,255}$") -_ROUTE_KEYS = frozenset({"queue", "exchange", "routing_key", "priority"}) - - -def _safe_route(value: object) -> object | None: - if isinstance(value, str): - return value if _QUEUE_RE.match(value) else None - if not isinstance(value, dict) or set(value) - _ROUTE_KEYS: - return None - queue = value.get("queue") - if not isinstance(queue, str) or not _QUEUE_RE.match(queue): - return None - for key in ("exchange", "routing_key"): - if key in value and not (isinstance(value[key], str) and _QUEUE_RE.match(value[key])): - return None - priority = value.get("priority") - if "priority" in value and (isinstance(priority, bool) or not isinstance(priority, int)): - return None - return dict(value) - - -def sanitize_task_routes(raw: object) -> dict[str, Any]: - """Keep only ``{pattern: queue-name | {queue, exchange, routing_key, priority}}``.""" - if not isinstance(raw, dict): - return {} - clean: dict[str, Any] = {} - for pattern, value in raw.items(): - route = _safe_route(value) - if not isinstance(pattern, str) or not pattern.strip() or route is None: - logger.error( - "task_routes_override: refusing entry %r (needs a task pattern and a queue " - "name or a route dict using only %s); ignoring it", - pattern, - sorted(_ROUTE_KEYS), - ) - continue - clean[pattern] = route - return clean - - -def parse_task_routes(raw: object) -> dict[str, Any]: - """Strict form of :func:`sanitize_task_routes` for user input. - - Accepts a mapping or a JSON string of one (what the admin settings text box - submits) and raises ``ValueError`` naming the first bad entry instead of - silently dropping it, so the editor can show why a save was refused. - """ - if isinstance(raw, str): - if not raw.strip(): - return {} - try: - raw = json.loads(raw) - except ValueError as exc: - raise ValueError(f"task_routes_override is not valid JSON: {exc}") from exc - if raw is None: - return {} - if not isinstance(raw, dict): - raise ValueError( - 'task_routes_override must be a JSON object such as {"reports.*": "heavy"}' - ) - for pattern, value in raw.items(): - if not isinstance(pattern, str) or not pattern.strip() or _safe_route(value) is None: - raise ValueError( - f"task_routes_override entry {pattern!r} needs a task pattern and a queue " - f"name or a route dict using only {sorted(_ROUTE_KEYS)}" - ) - return dict(raw) +sys.modules[__name__] = importlib.import_module("background_tasks.core.config_guards") diff --git a/modules/background_tasks/background_tasks/contracts/outbox.py b/modules/background_tasks/background_tasks/contracts/outbox.py index de699bce0..870a3b79c 100644 --- a/modules/background_tasks/background_tasks/contracts/outbox.py +++ b/modules/background_tasks/background_tasks/contracts/outbox.py @@ -9,7 +9,7 @@ from pydantic import ConfigDict from sqlmodel import SQLModel -from background_tasks.constants import OutboxStatus +from background_tasks.core.constants import OutboxStatus class OutboxItem(SQLModel): diff --git a/modules/background_tasks/background_tasks/contracts/schemas.py b/modules/background_tasks/background_tasks/contracts/schemas.py index c461a8a64..78e13d5eb 100644 --- a/modules/background_tasks/background_tasks/contracts/schemas.py +++ b/modules/background_tasks/background_tasks/contracts/schemas.py @@ -9,7 +9,7 @@ from pydantic import ConfigDict, field_validator from sqlmodel import SQLModel -from background_tasks.constants import TaskStatus +from background_tasks.core.constants import TaskStatus class TaskExecutionListItem(SQLModel): diff --git a/modules/background_tasks/background_tasks/core/__init__.py b/modules/background_tasks/background_tasks/core/__init__.py new file mode 100644 index 000000000..60cf02f51 --- /dev/null +++ b/modules/background_tasks/background_tasks/core/__init__.py @@ -0,0 +1,7 @@ +"""What every background_tasks feature builds on. + +Constants, the module settings and their Celery, log and env-check parts, +worker engine settings, the app.state container, config guards, health, +lifecycle wiring, the Redis invalidation transport, the sync DB session factory +and tracking policy. +""" diff --git a/modules/background_tasks/background_tasks/_lifecycle.py b/modules/background_tasks/background_tasks/core/_lifecycle.py similarity index 85% rename from modules/background_tasks/background_tasks/_lifecycle.py rename to modules/background_tasks/background_tasks/core/_lifecycle.py index a6ce6cf26..48613ea54 100644 --- a/modules/background_tasks/background_tasks/_lifecycle.py +++ b/modules/background_tasks/background_tasks/core/_lifecycle.py @@ -22,12 +22,12 @@ async def startup(app: FastAPI, module_name: str) -> None: from simple_module_hosting.log_sinks import redact - from background_tasks.celery_app import build_celery - from background_tasks.db_settings import worker_database_options - from background_tasks.log_sink import install_log_sink - from background_tasks.signals import bind_event_bus - from background_tasks.sync_db import configure_database, set_database_url - from background_tasks.tenant_context import set_default_tenant + from background_tasks.core.db_settings import worker_database_options + from background_tasks.core.sync_db import configure_database, set_database_url + from background_tasks.executions.tenant_context import set_default_tenant + from background_tasks.logs.sink import install_log_sink + from background_tasks.worker.celery_app import build_celery + from background_tasks.worker.signals import bind_event_bus services = app.state.background_tasks # Pin the sync engine to the same URL the host's async settings @@ -62,7 +62,7 @@ async def startup(app: FastAPI, module_name: str) -> None: # settings changes rather than pinning the boot-time instance. from simple_module_core.health import HealthCheck - from background_tasks.health import CHECK_REDIS, build_redis_check + from background_tasks.core.health import CHECK_REDIS, build_redis_check app.state.sm.health_registry.add( HealthCheck( @@ -87,10 +87,10 @@ async def startup(app: FastAPI, module_name: str) -> None: def install_durable_transport(app: FastAPI) -> None: """Make ``bus.publish_durable`` write to this module's outbox (GH #459).""" - from background_tasks.constants import OutboxMode - from background_tasks.outbox_sweep import start_sweep - from background_tasks.outbox_transport import OutboxTransport - from background_tasks.outbox_worker import use_bus + from background_tasks.core.constants import OutboxMode + from background_tasks.outbox.sweep import start_sweep + from background_tasks.outbox.transport import OutboxTransport + from background_tasks.outbox.worker import use_bus bus = app.state.sm.event_bus services = app.state.background_tasks @@ -122,7 +122,7 @@ async def start_invalidation_transport(app: FastAPI) -> None: documented cross-process limit as ``bind_event_bus`` above: a task that invalidates a cache should do it through an API call or accept the TTL. """ - from background_tasks.invalidation import RedisInvalidationTransport + from background_tasks.core.invalidation import RedisInvalidationTransport services = app.state.background_tasks s = services.settings @@ -146,11 +146,11 @@ async def start_invalidation_transport(app: FastAPI) -> None: async def shutdown(app: FastAPI) -> None: - from background_tasks.log_sink import uninstall_log_sink - from background_tasks.outbox_sweep import stop_sweep - from background_tasks.outbox_worker import reset_worker_state - from background_tasks.signals import unbind_event_bus - from background_tasks.sync_db import dispose_sync_engine + from background_tasks.core.sync_db import dispose_sync_engine + from background_tasks.logs.sink import uninstall_log_sink + from background_tasks.outbox.sweep import stop_sweep + from background_tasks.outbox.worker import reset_worker_state + from background_tasks.worker.signals import unbind_event_bus await stop_sweep(app.state.background_tasks) bus = getattr(app.state.sm, "event_bus", None) diff --git a/modules/background_tasks/background_tasks/core/config_guards.py b/modules/background_tasks/background_tasks/core/config_guards.py new file mode 100644 index 000000000..2214f2127 --- /dev/null +++ b/modules/background_tasks/background_tasks/core/config_guards.py @@ -0,0 +1,220 @@ +"""Guards for operator-supplied Celery configuration. + +Celery conf can name serializers (``pickle``), dotted paths it imports on boot +(``imports``, ``worker_pool``, ``beat_scheduler`` ...) and signing material, so +a raw conf dict is a code-execution surface for whoever can write it. Two +inputs reach ``build_celery``: + +- ``SM_BG_TASKS_CELERY_OVERRIDES`` (JSON object): environment only, so it is + controlled by whoever deploys the process and never by an admin-UI editor. + It is read here rather than being a ``BackgroundTasksSettings`` field, because + every field of that class is stored in, and hydrated from, the database. + Dangerous keys are refused even from here. +- ``SM_BG_TASKS_WORKER_QUEUES`` / ``_INCLUDE_MODULES`` / ``_EXCLUDE_MODULES`` + (comma-separated): which queues and modules *this one worker process* + serves. Environment only for the same reason: one stored value would + re-scope every worker. All three go through the helpers below. +- ``task_routes_override`` (DB-backed, admin-editable): only names queues, and + is reduced to plain queue names and route dicts with a closed key set. A route + dict is merged into ``apply_async`` options, so ``serializer``, ``link`` and + friends must not be expressible. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +from dataclasses import dataclass +from typing import Any + +from background_tasks.core.constants import ENV_PREFIX + +logger = logging.getLogger(__name__) + +ENV_CELERY_OVERRIDES = f"{ENV_PREFIX}CELERY_OVERRIDES" +ENV_WORKER_QUEUES = f"{ENV_PREFIX}WORKER_QUEUES" +ENV_INCLUDE_MODULES = f"{ENV_PREFIX}INCLUDE_MODULES" +ENV_EXCLUDE_MODULES = f"{ENV_PREFIX}EXCLUDE_MODULES" + +# Conf keys that choose a (de)serializer or load code by dotted path / file. +_DENIED_KEYS = frozenset( + { + "accept_content", + "task_serializer", + "result_serializer", + "result_accept_content", + "event_serializer", + "imports", + "include", + "worker_pool", + "worker_autoscaler", + "worker_consumer", + "worker_timer", + "beat_scheduler", + "beat_schedule", + "beat_schedule_filename", + "task_routes", + "task_annotations", + "task_queues", + "task_cls", + "broker_transport", + "result_backend_transport_options", + "worker_state_db", + } +) +_DENIED_SUBSTRINGS = ("hijack", "serializer", "accept_content", "pickle") +_DENIED_PREFIXES = ("security_", "worker_pool_", "pool_") +_DENIED_SUFFIXES = ("_cls", "_class") + + +def is_denied_key(key: str) -> bool: + """True for a conf key that must not be settable from configuration.""" + # Celery also resolves the old ``CELERY_*`` spellings. + k = key.strip().lower().removeprefix("celery_") + return ( + k in _DENIED_KEYS + or k.startswith(_DENIED_PREFIXES) + or k.endswith(_DENIED_SUFFIXES) + or any(s in k for s in _DENIED_SUBSTRINGS) + ) + + +def filter_celery_overrides(raw: object, *, source: str = "celery_overrides") -> dict[str, Any]: + """Drop (and log) denied keys; non-dict input yields ``{}``.""" + if not isinstance(raw, dict): + if raw not in (None, {}): + logger.error("%s must be a JSON object; ignoring it", source) + return {} + safe: dict[str, Any] = {} + for key, value in raw.items(): + if not isinstance(key, str) or is_denied_key(key): + logger.error( + "%s: refusing Celery conf key %r (serializer, import and code-loading " + "keys cannot be set from configuration); ignoring it", + source, + key, + ) + continue + safe[key] = value + return safe + + +def _env_text(name: str) -> str: + return os.environ.get(name, "").strip() + + +def env_csv(name: str) -> list[str]: + """A comma-separated environment variable as a list (``[]`` when unset).""" + return [part.strip() for part in _env_text(name).split(",") if part.strip()] + + +def env_json_object(name: str) -> dict[str, Any]: + """A JSON-object environment variable (``{}`` when unset or unusable, with a log).""" + raw = _env_text(name) + if not raw: + return {} + try: + parsed = json.loads(raw) + except ValueError as exc: + logger.error("%s is not valid JSON (%s); ignoring it", name, exc) + return {} + if not isinstance(parsed, dict): + logger.error("%s must be a JSON object; ignoring it", name) + return {} + return parsed + + +def celery_overrides_from_env() -> dict[str, Any]: + """The vetted ``SM_BG_TASKS_CELERY_OVERRIDES`` mapping (``{}`` when unset).""" + return filter_celery_overrides( + env_json_object(ENV_CELERY_OVERRIDES), source=ENV_CELERY_OVERRIDES + ) + + +@dataclass(frozen=True) +class WorkerScope: + """What this process serves; see :func:`worker_scope_from_env`.""" + + queues: frozenset[str] = frozenset() + include_modules: frozenset[str] = frozenset() + exclude_modules: frozenset[str] = frozenset() + + +def worker_scope_from_env() -> WorkerScope: + """Per-process queue/module subset (GH #450), from the environment only.""" + return WorkerScope( + queues=frozenset(env_csv(ENV_WORKER_QUEUES)), + include_modules=frozenset(env_csv(ENV_INCLUDE_MODULES)), + exclude_modules=frozenset(env_csv(ENV_EXCLUDE_MODULES)), + ) + + +_QUEUE_RE = re.compile(r"^[A-Za-z0-9_.:\-]{1,255}$") +_ROUTE_KEYS = frozenset({"queue", "exchange", "routing_key", "priority"}) + + +def _safe_route(value: object) -> object | None: + if isinstance(value, str): + return value if _QUEUE_RE.match(value) else None + if not isinstance(value, dict) or set(value) - _ROUTE_KEYS: + return None + queue = value.get("queue") + if not isinstance(queue, str) or not _QUEUE_RE.match(queue): + return None + for key in ("exchange", "routing_key"): + if key in value and not (isinstance(value[key], str) and _QUEUE_RE.match(value[key])): + return None + priority = value.get("priority") + if "priority" in value and (isinstance(priority, bool) or not isinstance(priority, int)): + return None + return dict(value) + + +def sanitize_task_routes(raw: object) -> dict[str, Any]: + """Keep only ``{pattern: queue-name | {queue, exchange, routing_key, priority}}``.""" + if not isinstance(raw, dict): + return {} + clean: dict[str, Any] = {} + for pattern, value in raw.items(): + route = _safe_route(value) + if not isinstance(pattern, str) or not pattern.strip() or route is None: + logger.error( + "task_routes_override: refusing entry %r (needs a task pattern and a queue " + "name or a route dict using only %s); ignoring it", + pattern, + sorted(_ROUTE_KEYS), + ) + continue + clean[pattern] = route + return clean + + +def parse_task_routes(raw: object) -> dict[str, Any]: + """Strict form of :func:`sanitize_task_routes` for user input. + + Accepts a mapping or a JSON string of one (what the admin settings text box + submits) and raises ``ValueError`` naming the first bad entry instead of + silently dropping it, so the editor can show why a save was refused. + """ + if isinstance(raw, str): + if not raw.strip(): + return {} + try: + raw = json.loads(raw) + except ValueError as exc: + raise ValueError(f"task_routes_override is not valid JSON: {exc}") from exc + if raw is None: + return {} + if not isinstance(raw, dict): + raise ValueError( + 'task_routes_override must be a JSON object such as {"reports.*": "heavy"}' + ) + for pattern, value in raw.items(): + if not isinstance(pattern, str) or not pattern.strip() or _safe_route(value) is None: + raise ValueError( + f"task_routes_override entry {pattern!r} needs a task pattern and a queue " + f"name or a route dict using only {sorted(_ROUTE_KEYS)}" + ) + return dict(raw) diff --git a/modules/background_tasks/background_tasks/constants.py b/modules/background_tasks/background_tasks/core/constants.py similarity index 100% rename from modules/background_tasks/background_tasks/constants.py rename to modules/background_tasks/background_tasks/core/constants.py diff --git a/modules/background_tasks/background_tasks/db_settings.py b/modules/background_tasks/background_tasks/core/db_settings.py similarity index 100% rename from modules/background_tasks/background_tasks/db_settings.py rename to modules/background_tasks/background_tasks/core/db_settings.py diff --git a/modules/background_tasks/background_tasks/health.py b/modules/background_tasks/background_tasks/core/health.py similarity index 100% rename from modules/background_tasks/background_tasks/health.py rename to modules/background_tasks/background_tasks/core/health.py diff --git a/modules/background_tasks/background_tasks/invalidation.py b/modules/background_tasks/background_tasks/core/invalidation.py similarity index 100% rename from modules/background_tasks/background_tasks/invalidation.py rename to modules/background_tasks/background_tasks/core/invalidation.py diff --git a/modules/background_tasks/background_tasks/services.py b/modules/background_tasks/background_tasks/core/services.py similarity index 92% rename from modules/background_tasks/background_tasks/services.py rename to modules/background_tasks/background_tasks/core/services.py index 12796a738..8f5c19eea 100644 --- a/modules/background_tasks/background_tasks/services.py +++ b/modules/background_tasks/background_tasks/core/services.py @@ -22,8 +22,8 @@ from celery import Celery from background_tasks.contracts.schemas import WorkerSnapshot - from background_tasks.invalidation import RedisInvalidationTransport - from background_tasks.settings import BackgroundTasksSettings + from background_tasks.core.invalidation import RedisInvalidationTransport + from background_tasks.core.settings import BackgroundTasksSettings @dataclass diff --git a/modules/background_tasks/background_tasks/core/settings.py b/modules/background_tasks/background_tasks/core/settings.py new file mode 100644 index 000000000..74ce1fa97 --- /dev/null +++ b/modules/background_tasks/background_tasks/core/settings.py @@ -0,0 +1,175 @@ +"""BackgroundTasks module settings (DB-backed). + +Values come from defaults at boot, then get hydrated from the DB by the +hosting lifespan before module ``on_startup`` runs. Runtime changes go +through ``settings.module_settings.reload.apply_changes_and_reload``. + +The broker and result-backend URLs are deployment plumbing rather than module +config, so they stay env-readable through ``SM_REDIS_URL`` — one variable that +seeds both. They have to work *before* any DB row exists: the production +validator below rejects the localhost defaults, so without them a containerised +app can't boot far enough to hydrate settings, and a worker process (which +never sees ``app.state``) has no other source at all. Like every env var it +beats a stored value (env → DB → default): ``env_seeded_fields`` reports both +fields to the hydrator, since ``SM_REDIS_URL`` is read by their default +factories where pydantic's env source cannot see it. With it unset, the two +fields keep separate DB overrides for anyone who wants the broker and the +backend on different Redis databases. + +``SM_BG_TASKS_BROKER_URL`` and ``SM_BG_TASKS_RESULT_BACKEND`` still work and +take precedence over ``SM_REDIS_URL`` — a deployment that set them meant them — +but log a deprecation warning. Being real env reads, they also beat a stored +value (env → DB → default, GH #486). They are answered by pydantic's own env source +now that ``env_prefix`` is set (GH #283), which is also what stops every other +field being readable from its bare, unprefixed name. + +The other env read is ``SM_ENVIRONMENT``, consulted by the +``@model_validator`` to refuse a localhost broker in production — that's a +host-level setting, not a background_tasks-module field. + +Process wiring, including ``outbox_mode`` and sweep cadence, requires a restart; +``json_schema_extra`` exposes that requirement in the settings UI. +""" + +from __future__ import annotations + +import logging +from typing import Literal + +from pydantic import Field, ValidationInfo, field_validator, model_validator +from pydantic_core import PydanticCustomError +from simple_module_core.settings_hydrate import RELATED_FIELDS_CTX + +from background_tasks.core.constants import ( + DEFAULT_OUTBOX_LEASE_SECONDS, + DEFAULT_OUTBOX_MAX_ATTEMPTS, + DEFAULT_OUTBOX_SWEEP_INTERVAL_SECONDS, + DEFAULT_RETENTION_DAYS, + ENV_PREFIX, + max_outbox_handler_timeout, +) +from background_tasks.core.settings_celery import _CELERY_RESTART, CelerySettingsFields +from background_tasks.core.settings_env_checks import ( + forbid_localhost_broker_in_production, + warn_on_legacy_redis_vars, +) +from background_tasks.core.tracking import TrackingMode + +__all__ = ["DEFAULT_RETENTION_DAYS", "ENV_PREFIX", "BackgroundTasksSettings"] + +logger = logging.getLogger(__name__) + + +class BackgroundTasksSettings(CelerySettingsFields): + """Configuration for the Celery + Redis task runner.""" + + execution_tracking: TrackingMode = Field( + default=TrackingMode.FULL, json_schema_extra=_CELERY_RESTART + ) + track_publish: bool = Field(default=True, json_schema_extra=_CELERY_RESTART) + + @classmethod + def settings_customise_sources( + cls, settings_cls, init_settings, env_settings, dotenv_settings, file_secret_settings + ): + # Tracking controls are DB-backed; preserve env wiring for older fields. + def db_only(source): + def read(): + return { + k: v + for k, v in source().items() + if k not in {"execution_tracking", "track_publish"} + } + + return read + + return ( + init_settings, + db_only(env_settings), + db_only(dotenv_settings), + db_only(file_secret_settings), + ) + + # ── Durable event outbox (GH #459) ────────────────────── + outbox_mode: Literal["celery", "inprocess"] = Field( + default="celery", json_schema_extra=_CELERY_RESTART + ) + outbox_max_attempts: int = Field( + default=DEFAULT_OUTBOX_MAX_ATTEMPTS, + ge=1, + le=100, + description="Delivery attempts before a durable event is marked dead (1-100).", + ) + outbox_sweep_interval_seconds: int = Field( + default=DEFAULT_OUTBOX_SWEEP_INTERVAL_SECONDS, + ge=1, + json_schema_extra=_CELERY_RESTART, + ) + outbox_lease_seconds: int = Field( + default=DEFAULT_OUTBOX_LEASE_SECONDS, + ge=10, + le=86400, + description="Seconds a claimed delivery may run before another worker reclaims it.", + ) + # Empty: the lease minus a safety margin (10 %, at least 5 s), the most allowed. + outbox_handler_timeout_seconds: int | None = Field( + default=None, ge=1, description="Seconds a durable handler may run per attempt." + ) + + @field_validator("task_time_limit") + @classmethod + def _soft_limit_below_hard(cls, hard: int | None, info: ValidationInfo) -> int | None: + """A soft limit at or above the hard one can never fire (#548). + + Attached to the later-declared hard limit so the error carries that + field's ``loc`` and the settings form shows it under the field. It names + the soft limit as related so boot hydration can drop a stored soft limit + when the hard one is pinned by env. + """ + soft = info.data.get("task_soft_time_limit") + if hard is not None and soft is not None and soft >= hard: + raise PydanticCustomError( + "time_limit_order", + "task_time_limit ({hard}) must be greater than task_soft_time_limit ({soft}); " + "otherwise the soft limit can never fire", + {"hard": hard, "soft": soft, RELATED_FIELDS_CTX: ["task_soft_time_limit"]}, + ) + return hard + + @field_validator("outbox_handler_timeout_seconds") + @classmethod + def _handler_timeout_fits_the_lease( + cls, timeout: int | None, info: ValidationInfo + ) -> int | None: + """A handler outliving its lease would be reclaimed and run twice at once. + + A field validator (not a model one) so the error carries the field's + ``loc`` and the settings form shows it under that field. The lease is + declared first, so it is already in ``info.data`` unless it was invalid + itself, in which case that error is the one to fix first. + """ + lease = info.data.get("outbox_lease_seconds") + if timeout is None or lease is None: + return timeout + ceiling = max_outbox_handler_timeout(lease) + if timeout > ceiling: + raise PydanticCustomError( + "handler_timeout_exceeds_lease", + "outbox_handler_timeout_seconds ({timeout}) must leave a safety margin " + "inside outbox_lease_seconds ({lease}): at most " + "{ceiling}. Raise the lease or lower the timeout.", + { + "timeout": timeout, + "lease": lease, + "ceiling": f"{ceiling:g}", + RELATED_FIELDS_CTX: ["outbox_lease_seconds"], + }, + ) + return timeout + + @model_validator(mode="after") + def _check_broker_env(self) -> BackgroundTasksSettings: + """Deprecation warnings and the production localhost-broker refusal.""" + warn_on_legacy_redis_vars(self.broker_url, self.result_backend) + forbid_localhost_broker_in_production(self.broker_url, self.result_backend) + return self diff --git a/modules/background_tasks/background_tasks/settings_celery.py b/modules/background_tasks/background_tasks/core/settings_celery.py similarity index 97% rename from modules/background_tasks/background_tasks/settings_celery.py rename to modules/background_tasks/background_tasks/core/settings_celery.py index 980011606..6bb45c2ba 100644 --- a/modules/background_tasks/background_tasks/settings_celery.py +++ b/modules/background_tasks/background_tasks/core/settings_celery.py @@ -15,8 +15,8 @@ from pydantic_settings import SettingsConfigDict from simple_module_core.settings_hydrate import RELATED_FIELDS_CTX -from background_tasks.config_guards import parse_task_routes -from background_tasks.constants import ( +from background_tasks.core.config_guards import parse_task_routes +from background_tasks.core.constants import ( DEFAULT_BROKER_URL, DEFAULT_INVALIDATION_CHANNEL, DEFAULT_MAX_RETRIES, @@ -29,8 +29,8 @@ ENV_PREFIX, ENV_REDIS_URL, ) -from background_tasks.db_settings import WorkerDatabaseSettings -from background_tasks.settings_logs import LogSinkFields +from background_tasks.core.db_settings import WorkerDatabaseSettings +from background_tasks.core.settings_logs import LogSinkFields _CELERY_RESTART = {"requires_restart": True, "group": "Celery"} diff --git a/modules/background_tasks/background_tasks/settings_env_checks.py b/modules/background_tasks/background_tasks/core/settings_env_checks.py similarity index 97% rename from modules/background_tasks/background_tasks/settings_env_checks.py rename to modules/background_tasks/background_tasks/core/settings_env_checks.py index ca5a7c4d3..57de91f47 100644 --- a/modules/background_tasks/background_tasks/settings_env_checks.py +++ b/modules/background_tasks/background_tasks/core/settings_env_checks.py @@ -11,7 +11,7 @@ from simple_module_core.environments import NON_PROD_ENVIRONMENTS -from background_tasks.constants import ENV_PREFIX, ENV_REDIS_URL +from background_tasks.core.constants import ENV_PREFIX, ENV_REDIS_URL logger = logging.getLogger("background_tasks.settings") diff --git a/modules/background_tasks/background_tasks/settings_logs.py b/modules/background_tasks/background_tasks/core/settings_logs.py similarity index 97% rename from modules/background_tasks/background_tasks/settings_logs.py rename to modules/background_tasks/background_tasks/core/settings_logs.py index a3dccbc0f..003c38852 100644 --- a/modules/background_tasks/background_tasks/settings_logs.py +++ b/modules/background_tasks/background_tasks/core/settings_logs.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, Field, field_validator -from background_tasks.constants import ( +from background_tasks.core.constants import ( DEFAULT_LOG_RETENTION_DAYS, DEFAULT_LOG_SINK_LEVEL, LOG_LEVEL_NAMES, diff --git a/modules/background_tasks/background_tasks/core/sync_db.py b/modules/background_tasks/background_tasks/core/sync_db.py new file mode 100644 index 000000000..a3e8b3fd9 --- /dev/null +++ b/modules/background_tasks/background_tasks/core/sync_db.py @@ -0,0 +1,246 @@ +"""Sync SQLAlchemy session factory for Celery signal handlers. + +Celery signals are called synchronously from inside the worker / web-process +hot path. Building an event-loop just to await a query is both ugly and +deadlock-prone (web-process signals can fire from inside an already-running +loop). We instead maintain a second, sync engine pointed at the same DB URL +and borrow a short-lived session from it whenever a signal fires. + +The engine is lazily built on first use and process-global. It's cheap — +SQLAlchemy's connection pool is per-process so the signal path reuses +connections after the first signal. +""" + +from __future__ import annotations + +import logging +import os +from collections.abc import Generator +from contextlib import contextmanager + +from celery.signals import worker_process_init +from simple_module_db import DEFAULT_TENANT_ID +from simple_module_db.audit_sinks import discover_audit_callback +from simple_module_db.engine_options import connect_args +from simple_module_db.listeners import attach_session_listeners +from simple_module_db.provider import DatabaseProvider, detect_provider +from simple_module_db.query_filter import EngineTenancy, bind_engine_policy +from sqlalchemy import create_engine +from sqlalchemy.engine import Engine +from sqlalchemy.orm import Session, sessionmaker +from sqlmodel import SQLModel + +logger = logging.getLogger(__name__) + +_engine: Engine | None = None +# PID that built ``_engine``. A prefork Celery child inherits the parent's +# engine (and its sockets) across fork(); using them would interleave two +# processes on one Postgres connection. +_engine_pid: int | None = None +_session_factory: sessionmaker[Session] | None = None +_url_override: str | None = None +_database_options: dict[str, int] = {} +_tenant_strict: bool = False +_default_tenant: str | None = None + + +class WorkerTenancy(SQLModel, frozen=True): + """Snapshot of the host tenancy configured for this worker (#685).""" + + multi_tenant: bool + default_tenant: str | None + + +def worker_tenancy() -> WorkerTenancy | None: + """Return configured tenancy, or ``None`` when no database URL is pinned.""" + if _url_override is None: + return None + return WorkerTenancy(multi_tenant=_tenant_strict, default_tenant=_default_tenant) + + +class WorkerSession(Session): + """This engine's own session class, so the entity listeners attached to it + never double up with a ``DatabaseState``'s (each state has its own + ``Session`` subclass; listeners on the base class would fire for both).""" + + +def _sync_url(async_url: str) -> str: + """Convert an async SQLAlchemy URL to its sync driver equivalent. + + Mirrors ``host/migrations/env.py`` so signals use the same URL shape + Alembic does. + """ + return async_url.replace("+aiosqlite", "").replace("+asyncpg", "+psycopg2") + + +def set_database_url( + url: str | None, *, tenant_strict: bool = False, default_tenant: str | None = None +) -> None: + """Pin the URL used to build the sync engine. + + The web process loads ``.env`` via pydantic-settings, but those values + never land in ``os.environ`` — so reading ``SM_DATABASE_URL`` directly + from the env can silently drop us back to the SQLite default while the + rest of the app uses Postgres. ``BackgroundTasksModule.on_startup`` + calls this with the resolved ``settings.database_url`` so signals use + the same DB the app is on. Pass ``None`` to clear the override (used in + tests + on shutdown). + + ``tenant_strict`` mirrors the host's ``multi_tenant``: task bodies get the + same fail-closed tenant rules as request code. + + ``default_tenant`` mirrors the host's ``default_tenant`` (ignored when + strict): an insert with no tenant bound — an ``all_tenants()`` block in a + task body — is stamped with it, as ``DatabaseState.default_tenant_id`` does + in the web process, instead of landing in ``DEFAULT_TENANT_ID`` where the + install's own scoped reads never see it. + """ + global _url_override, _tenant_strict, _default_tenant + default_tenant = None if tenant_strict else (default_tenant or None) + if (_url_override, _tenant_strict, _default_tenant) == (url, tenant_strict, default_tenant): + return + _url_override = url + _tenant_strict = tenant_strict + _default_tenant = default_tenant + _drop_engine() + + +def _drop_engine() -> None: + """Dispose the cached engine, never touching sockets inherited via fork.""" + global _engine, _session_factory, _engine_pid + if _engine is not None: + # In a forked child the pooled sockets belong to the parent: close=False + # forgets them without sending a protocol-level close down the shared fd. + _engine.dispose(close=_engine_pid == os.getpid()) + _engine = None + _session_factory = None + _engine_pid = None + + +def _resolve_url() -> str: + if _url_override is not None: + return _url_override + return os.environ.get("SM_DATABASE_URL", "sqlite:///./app.db") + + +def async_database_url() -> str: + """The same database as :func:`sync_session`, spelled for an async driver. + + For the outbox dispatcher, whose durable handlers take an ``AsyncSession``. + """ + url = _resolve_url() + if "+aiosqlite" in url or "+asyncpg" in url: + return url + if url.startswith("sqlite://"): + return url.replace("sqlite://", "sqlite+aiosqlite://", 1) + for prefix in ("postgresql+psycopg2://", "postgresql+psycopg://", "postgresql://"): + if url.startswith(prefix): + return url.replace(prefix, "postgresql+asyncpg://", 1) + return url + + +def engine_tenancy() -> EngineTenancy: + """The policy worker engines run with (mirrors the host's). + + Includes the installed audit sink, so a task's writes are audited exactly + as a request's are when ``audit_log`` is installed (#498). It comes from + the ``simple_module.audit_sinks`` entry point, not an import: this module + does not depend on ``audit_log``. + """ + return EngineTenancy( + tenant_strict=_tenant_strict, + default_tenant_id=_default_tenant or DEFAULT_TENANT_ID, + audit_callback=discover_audit_callback(), + ) + + +def configure_database(options: dict[str, int]) -> None: + """Pin resolved settings before building either worker engine.""" + global _database_options + if options != _database_options: + _database_options = dict(options) + _drop_engine() + + +def database_options() -> dict[str, int]: + """A copy for the outbox engine and its cache key.""" + return dict(_database_options) + + +def _build_engine() -> Engine: + sync_url = _sync_url(_resolve_url()) + # Small pool — signals fire sequentially per worker process. + options = database_options() + args = connect_args( + sync_url, + connect_timeout=options.pop("connect_timeout", 10), + statement_timeout_ms=options.pop("statement_timeout_ms", 0), + idle_in_tx_timeout_ms=options.pop("idle_in_tx_timeout_ms", 0), + ) + if detect_provider(sync_url) == DatabaseProvider.SQLITE: + options = {} + else: + options = {"pool_size": 2, "max_overflow": 3, "pool_recycle": 1800} | options + engine = create_engine(sync_url, pool_pre_ping=True, connect_args=args, **options) + # A worker never runs create_app, so without this its sessions would have + # no tenant filter at all: the tenant restored around a task body would + # scope nothing (#371). + attach_session_listeners(WorkerSession) + bind_engine_policy(engine, engine_tenancy()) + return engine + + +def get_sync_session_factory() -> sessionmaker[Session]: + """Return the process-global sync session factory, building it once.""" + global _engine, _session_factory, _engine_pid + if _session_factory is not None and _engine_pid != os.getpid(): + _drop_engine() + if _session_factory is None: + _engine = _build_engine() + _engine_pid = os.getpid() + _session_factory = sessionmaker(bind=_engine, class_=WorkerSession, expire_on_commit=False) + return _session_factory + + +def dispose_sync_engine() -> None: + """Release pooled connections and drop the cached engine. + + Called from :meth:`BackgroundTasksModule.on_shutdown` so lifespan + restarts within one process (test runners, uvicorn dev reload) don't + accumulate engines against the old DB URL. + """ + global _url_override, _tenant_strict, _default_tenant + _drop_engine() + _url_override = None + configure_database({}) + _tenant_strict = False + _default_tenant = None + + +@worker_process_init.connect +def _dispose_inherited_engine(**_: object) -> None: + """Forget the parent's engine in a fresh prefork child (rebuilt lazily).""" + _drop_engine() + + +@worker_process_init.connect +def _init_sentry(**_: object) -> None: + """Start Sentry in each prefork child (no-op without ``SM_SENTRY_DSN``).""" + from simple_module_hosting.sentry import PROCESS_WORKER, init_sentry + + init_sentry(PROCESS_WORKER) + + +@contextmanager +def sync_session() -> Generator[Session]: + """Open a short-lived sync session; commit on success, rollback on error.""" + factory = get_sync_session_factory() + session = factory() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() diff --git a/modules/background_tasks/background_tasks/tracking.py b/modules/background_tasks/background_tasks/core/tracking.py similarity index 91% rename from modules/background_tasks/background_tasks/tracking.py rename to modules/background_tasks/background_tasks/core/tracking.py index f8866b86a..9b7b3efe6 100644 --- a/modules/background_tasks/background_tasks/tracking.py +++ b/modules/background_tasks/background_tasks/core/tracking.py @@ -3,8 +3,12 @@ from enum import StrEnum from typing import Any -from background_tasks.constants import INTERNAL_TASK_SWEEP_OUTBOX, SM_TRACKING_HEADER, TaskStatus -from background_tasks.tenant_context import header_of_request +from background_tasks.core.constants import ( + INTERNAL_TASK_SWEEP_OUTBOX, + SM_TRACKING_HEADER, + TaskStatus, +) +from background_tasks.executions.tenant_context import header_of_request class TrackingMode(StrEnum): diff --git a/modules/background_tasks/background_tasks/executions/__init__.py b/modules/background_tasks/background_tasks/executions/__init__.py new file mode 100644 index 000000000..93255fa12 --- /dev/null +++ b/modules/background_tasks/background_tasks/executions/__init__.py @@ -0,0 +1,5 @@ +"""Task executions as the app sees them. + +Enqueue after commit, carrying the request tenant, the admin listing service, +filters and retries. +""" diff --git a/modules/background_tasks/background_tasks/after_commit.py b/modules/background_tasks/background_tasks/executions/after_commit.py similarity index 100% rename from modules/background_tasks/background_tasks/after_commit.py rename to modules/background_tasks/background_tasks/executions/after_commit.py diff --git a/modules/background_tasks/background_tasks/filters.py b/modules/background_tasks/background_tasks/executions/filters.py similarity index 96% rename from modules/background_tasks/background_tasks/filters.py rename to modules/background_tasks/background_tasks/executions/filters.py index 0dc6e8f8c..7be70f6d0 100644 --- a/modules/background_tasks/background_tasks/filters.py +++ b/modules/background_tasks/background_tasks/executions/filters.py @@ -12,7 +12,7 @@ from sqlalchemy import ColumnElement, select from sqlmodel import col -from background_tasks.constants import PLATFORM_TENANT_FILTER, RETRYABLE_STATUSES, TaskStatus +from background_tasks.core.constants import PLATFORM_TENANT_FILTER, RETRYABLE_STATUSES, TaskStatus from background_tasks.models import TaskExecution Conditions = list[ColumnElement[bool]] diff --git a/modules/background_tasks/background_tasks/retry_service.py b/modules/background_tasks/background_tasks/executions/retry_service.py similarity index 94% rename from modules/background_tasks/background_tasks/retry_service.py rename to modules/background_tasks/background_tasks/executions/retry_service.py index d03b7fe99..d1b290968 100644 --- a/modules/background_tasks/background_tasks/retry_service.py +++ b/modules/background_tasks/background_tasks/executions/retry_service.py @@ -1,6 +1,6 @@ """Re-enqueueing work: one execution, or a filtered sweep of them. -Split from :mod:`background_tasks.service` because re-enqueueing is the only +Split from :mod:`background_tasks.executions.service` because re-enqueueing is the only thing in this module that writes to the broker, and it is where every correctness question lives — what may be retried, how many at once, and what must not happen twice. The read paths have none of those questions. @@ -20,17 +20,17 @@ from sqlalchemy import Select, func, select, update from sqlmodel import col -from background_tasks._signal_support import now_utc -from background_tasks.after_commit import after_commit -from background_tasks.constants import RETRY_ALL_BATCH, SM_TRACKING_HEADER, TaskStatus from background_tasks.contracts.events import TaskRetried from background_tasks.contracts.schemas import RetryFailedResult -from background_tasks.filters import Conditions, bulk_retry_conditions +from background_tasks.core.constants import RETRY_ALL_BATCH, SM_TRACKING_HEADER, TaskStatus +from background_tasks.core.sync_db import sync_session +from background_tasks.core.tracking import TrackingMode +from background_tasks.executions.after_commit import after_commit +from background_tasks.executions.filters import Conditions, bulk_retry_conditions +from background_tasks.executions.tenant_context import publish_as_owner from background_tasks.models import TaskExecution -from background_tasks.sync_db import sync_session -from background_tasks.task_resource import resource_headers -from background_tasks.tenant_context import publish_as_owner -from background_tasks.tracking import TrackingMode +from background_tasks.worker._signal_support import now_utc +from background_tasks.worker.task_resource import resource_headers if TYPE_CHECKING: from celery import Celery @@ -43,7 +43,7 @@ def claim_query(conditions: Conditions, batch: int) -> Select[tuple[TaskExecutio ``FOR UPDATE SKIP LOCKED`` is what makes two simultaneous sweeps disjoint. The guard that is *supposed* to keep a row out of a second sweep — - :func:`~background_tasks.filters.bulk_retry_conditions` excluding parents + :func:`~background_tasks.executions.filters.bulk_retry_conditions` excluding parents that already have a child — only starts working once the child row exists, and a concurrent sweep reads the same batch long before that. Locking the parents makes the second reader skip them instead of publishing the same diff --git a/modules/background_tasks/background_tasks/service.py b/modules/background_tasks/background_tasks/executions/service.py similarity index 97% rename from modules/background_tasks/background_tasks/service.py rename to modules/background_tasks/background_tasks/executions/service.py index 2b3302910..87d782040 100644 --- a/modules/background_tasks/background_tasks/service.py +++ b/modules/background_tasks/background_tasks/executions/service.py @@ -12,16 +12,16 @@ from simple_module_db.pagination import total_pages from sqlalchemy import func, select -from background_tasks.constants import RETRYABLE_STATUSES, TaskStatus from background_tasks.contracts.schemas import ( RetryFailedResult, TaskExecutionDetail, TaskExecutionListItem, TaskExecutionListResponse, ) -from background_tasks.filters import execution_filters +from background_tasks.core.constants import RETRYABLE_STATUSES, TaskStatus +from background_tasks.executions.filters import execution_filters +from background_tasks.executions.retry_service import RetryCoordinator from background_tasks.models import TaskExecution, dto_columns, newest_first -from background_tasks.retry_service import RetryCoordinator if TYPE_CHECKING: from celery import Celery diff --git a/modules/background_tasks/background_tasks/executions/tenant_context.py b/modules/background_tasks/background_tasks/executions/tenant_context.py new file mode 100644 index 000000000..710afd2cc --- /dev/null +++ b/modules/background_tasks/background_tasks/executions/tenant_context.py @@ -0,0 +1,220 @@ +"""Carry the enqueuing request's tenant into the Celery task that runs later. + +A task body runs in a worker with no request, so ``current_tenant_id`` is +unset there — under strict isolation any tenant-scoped query would raise, and +without it the query would read every tenant. The tenant is stamped onto the +message headers at publish time and restored around the task body. + +Platform tasks that legitimately span tenants (sweeps, purges) wrap their +work in ``simple_module_db.all_tenants()`` instead. +""" + +from __future__ import annotations + +import logging +from collections.abc import Generator +from contextlib import contextmanager +from contextvars import ContextVar, Token +from typing import Any + +from simple_module_db import TenantIsolationError, current_tenant_id, is_valid_tenant_id +from simple_module_db.listeners import current_user_id + +from background_tasks.core.constants import OWNER_ID_MAX_LENGTH + +TENANT_HEADER = "sm_tenant_id" +OWNER_HEADER = "sm_owner_id" + +_log = logging.getLogger(__name__) +_tokens: ContextVar[tuple[tuple[str, Token[str | None], Token[str | None]], ...]] = ContextVar( + "sm_task_binding_tokens", default=() +) +# The owner of the task body running on this context, so tasks it enqueues +# carry the same owner. Kept apart from ``current_user_id`` on purpose: that +# one also stamps ``created_by`` on audited rows. +_task_owner: ContextVar[str | None] = ContextVar("sm_task_owner", default=None) +# Single-tenant hosts (``default_tenant``): tasks with no tenant on the message +# — beat jobs, tasks enqueued outside a request — run as that tenant. +_default_tenant: str | None = None + + +def set_default_tenant(tenant_id: str | None) -> None: + global _default_tenant + _default_tenant = tenant_id or None + + +def default_tenant() -> str | None: + """The tenant platform work runs as on a single-tenant host, else ``None``.""" + return _default_tenant + + +def stamp_tenant(headers: dict[str, Any] | None) -> None: + """Record the current tenant on an outgoing message (publish side). + + An explicit ``headers={"sm_tenant_id": ...}`` passed to ``send_task`` is + kept only when no tenant is bound (platform code enqueueing work for a + tenant) or it names the bound tenant: request code must not be able to + schedule work as another tenant. + """ + if headers is None: + return + tenant_id = current_tenant_id.get() + explicit = headers.get(TENANT_HEADER) + if explicit: + if not is_valid_tenant_id(explicit): + raise TenantIsolationError(f"Invalid tenant id on task headers: {explicit!r}") + if tenant_id is not None and explicit != tenant_id: + raise TenantIsolationError( + f"Cannot enqueue a task for tenant '{explicit}' in context of tenant '{tenant_id}'" + ) + return + if tenant_id is not None: + headers[TENANT_HEADER] = tenant_id + + +def stamp_owner(headers: dict[str, Any] | None) -> None: + """Record the user whose request enqueues this message (publish side). + + A task that enqueues a task has no ``current_user_id`` in its worker, so + :func:`restore_tenant` binds the incoming message's owner for the body and + this stamps it onward; :func:`publish_as_owner` does the same for a retry, + which must keep the original owner rather than adopt the operator. Failing + both, the bound user is the owner. A header already on the message is + ignored when a user is bound (request code must not name another owner). + """ + if headers is None: + return + user_id = _task_owner.get() + if user_id is None: + user_id = current_user_id.get() + if user_id: + headers[OWNER_HEADER] = str(user_id)[:OWNER_ID_MAX_LENGTH] + + +@contextmanager +def publish_as_owner(owner_user_id: str | None) -> Generator[None]: + """Publish tasks as *owner_user_id* (``None``: no owner), whoever is bound.""" + token = _task_owner.set(owner_user_id or "") + try: + yield + finally: + _task_owner.reset(token) + + +def published_owner(headers: dict[str, Any] | None) -> str | None: + """The owner an outgoing message carries (call after :func:`stamp_owner`).""" + value = (headers or {}).get(OWNER_HEADER) + return str(value)[:OWNER_ID_MAX_LENGTH] if value else None + + +def published_tenant(headers: dict[str, Any] | None) -> str | None: + """The tenant an outgoing message is for: its header, else the publisher's. + + Call after :func:`stamp_tenant`. The header may also have been set by + platform code naming a tenant while none is bound. + """ + return (headers or {}).get(TENANT_HEADER) or current_tenant_id.get() + + +def _tenant_of(task: Any) -> str | None: + return _tenant_of_request(getattr(task, "request", None)) + + +def header_of_request(request: Any, name: str) -> Any: + """A message header off a running task's request (``Request`` or publish-side object).""" + value = getattr(request, name, None) + # A worker ``Request`` (revoked signal) keeps headers in a dict instead. + for attr in ("headers", "request_dict"): + if value is None and isinstance(getattr(request, attr, None), dict): + value = getattr(request, attr).get(name) + return value + + +def message_owner(*, task: Any = None, request: Any = None) -> str | None: + """The owner named on a running task's message header, if any.""" + request = request if request is not None else getattr(task, "request", None) + if request is None: + return None + value = header_of_request(request, OWNER_HEADER) + return str(value)[:OWNER_ID_MAX_LENGTH] if value else None + + +def _tenant_of_request(request: Any) -> str | None: + if request is None: + return None + value = header_of_request(request, TENANT_HEADER) + if not value: + return None + if not is_valid_tenant_id(value): + # Never run a task body as a malformed tenant; it runs unscoped and, + # under strict mode, fails closed on its first tenant-scoped query. + _log.warning("Ignoring invalid tenant id on task message: %r", str(value)[:80]) + return None + return str(value) + + +def message_tenant(*, task: Any = None, request: Any = None) -> str | None: + """The tenant named on a running task's message header, if any. + + Signal handlers stamp it onto the ``TaskExecution`` row: the publish signal + may never have written the row (it fires in the publisher's process, which + can have a different DB, or none bound), so the worker-side upsert must + carry the tenant itself. The configured default tenant is deliberately not + returned — a platform message stays a platform row. + """ + return _tenant_of_request(request) if request is not None else _tenant_of(task) + + +def restore_tenant(*, task_id: str | None, task: Any) -> None: + """Enter the message's tenant for the task body (prerun).""" + if not task_id: + return + # Always set both, so a headerless worker task never inherits the previous + # binding. An in-process run (``apply()``, eager mode) is its caller's own + # context, so without headers it keeps the caller's tenant and owner. + eager = bool(getattr(getattr(task, "request", None), "is_eager", False)) + tenant = _tenant_of(task) or (current_tenant_id.get() if eager else _default_tenant) + owner = message_owner(task=task) or (_task_owner.get() if eager else "") # "": no owner + owner_token = _task_owner.set(owner) + tenant_token = current_tenant_id.set(tenant) + _tokens.set((*_tokens.get(), (task_id, tenant_token, owner_token))) + + +def release_tenant(*, task_id: str | None) -> None: + """Leave the binding :func:`restore_tenant` entered for *task_id* (postrun).""" + stack = _tokens.get() + ids = [entry[0] for entry in stack] + if task_id not in ids: + return + depth = len(ids) - 1 - ids[::-1].index(task_id) + if depth != len(stack) - 1: + # A prerun without its postrun: unwind it too rather than strand it. + _log.warning("Unwinding %d unfinished task binding(s)", len(stack) - 1 - depth) + _, tenant_token, owner_token = stack[depth] + _tokens.set(stack[:depth]) + try: + _task_owner.reset(owner_token) + current_tenant_id.reset(tenant_token) + except ValueError: + # Token from another context (exotic pool patching): clear instead, so + # the next task on this worker never inherits a tenant or owner. + _task_owner.set(None) + current_tenant_id.set(None) + + +__all__ = [ + "OWNER_HEADER", + "TENANT_HEADER", + "default_tenant", + "header_of_request", + "message_owner", + "message_tenant", + "publish_as_owner", + "published_owner", + "published_tenant", + "release_tenant", + "restore_tenant", + "set_default_tenant", + "stamp_owner", + "stamp_tenant", +] diff --git a/modules/background_tasks/background_tasks/logs/__init__.py b/modules/background_tasks/background_tasks/logs/__init__.py new file mode 100644 index 000000000..44d4674ff --- /dev/null +++ b/modules/background_tasks/background_tasks/logs/__init__.py @@ -0,0 +1,4 @@ +"""Retained task log lines. + +The database sink, reading them for the admin screens, and retention. +""" diff --git a/modules/background_tasks/background_tasks/log_purge.py b/modules/background_tasks/background_tasks/logs/purge.py similarity index 100% rename from modules/background_tasks/background_tasks/log_purge.py rename to modules/background_tasks/background_tasks/logs/purge.py diff --git a/modules/background_tasks/background_tasks/log_service.py b/modules/background_tasks/background_tasks/logs/service.py similarity index 97% rename from modules/background_tasks/background_tasks/log_service.py rename to modules/background_tasks/background_tasks/logs/service.py index 8555f3819..3f7be8c49 100644 --- a/modules/background_tasks/background_tasks/log_service.py +++ b/modules/background_tasks/background_tasks/logs/service.py @@ -11,8 +11,8 @@ from sqlalchemy import false, select from sqlmodel import col -from background_tasks.constants import LOG_LEVEL_NAMES from background_tasks.contracts.logs import LogItem, LogListResponse +from background_tasks.core.constants import LOG_LEVEL_NAMES from background_tasks.models import RetainedLog _LEVEL_NAMES = {number: name for name, number in logging.getLevelNamesMapping().items()} diff --git a/modules/background_tasks/background_tasks/log_sink.py b/modules/background_tasks/background_tasks/logs/sink.py similarity index 93% rename from modules/background_tasks/background_tasks/log_sink.py rename to modules/background_tasks/background_tasks/logs/sink.py index 1d1f857ad..dbb401b9e 100644 --- a/modules/background_tasks/background_tasks/log_sink.py +++ b/modules/background_tasks/background_tasks/logs/sink.py @@ -24,7 +24,7 @@ uninstall_sink, ) -from background_tasks.constants import ( +from background_tasks.core.constants import ( LOG_EXTRA_MAX_KEYS, LOG_EXTRA_VALUE_MAX_LENGTH, LOG_ID_MAX_LENGTH, @@ -32,10 +32,10 @@ LOG_NAME_MAX_LENGTH, TENANT_ID_MAX_LENGTH, ) -from background_tasks.log_context import LogContextFilter +from background_tasks.core.settings import BackgroundTasksSettings +from background_tasks.core.sync_db import sync_session from background_tasks.models import RetainedLog -from background_tasks.settings import BackgroundTasksSettings -from background_tasks.sync_db import sync_session +from background_tasks.worker.log_context import LogContextFilter _log = logging.getLogger(__name__) @@ -131,6 +131,6 @@ def _reinstall_in_child(**_: object) -> None: if handler is None: return logging.getLogger().removeHandler(handler) - from background_tasks.worker_settings import load_runner_settings + from background_tasks.worker.settings import load_runner_settings install_log_sink(load_runner_settings()) diff --git a/modules/background_tasks/background_tasks/models.py b/modules/background_tasks/background_tasks/models.py index d30dc69b0..fb5020742 100644 --- a/modules/background_tasks/background_tasks/models.py +++ b/modules/background_tasks/background_tasks/models.py @@ -13,7 +13,7 @@ from sqlalchemy.orm import load_only from sqlmodel import Field, SQLModel -from background_tasks.constants import ( +from background_tasks.core.constants import ( DEFAULT_QUEUE, LOG_ID_MAX_LENGTH, LOG_MESSAGE_MAX_LENGTH, diff --git a/modules/background_tasks/background_tasks/module.py b/modules/background_tasks/background_tasks/module.py index 1116d8ca1..693191131 100644 --- a/modules/background_tasks/background_tasks/module.py +++ b/modules/background_tasks/background_tasks/module.py @@ -15,7 +15,7 @@ from simple_module_core.module_settings import register_module_settings from simple_module_core.permissions import PermissionRegistry -from background_tasks.constants import ( +from background_tasks.core.constants import ( _MODULE_SETTINGS, API_PREFIX, LOGS_MENU_ICON, @@ -80,8 +80,8 @@ class BackgroundTasksModule(ModuleBase): ) def register_settings(self, app: FastAPI) -> None: - from background_tasks.services import BackgroundTasksServices - from background_tasks.settings import BackgroundTasksSettings + from background_tasks.core.services import BackgroundTasksServices + from background_tasks.core.settings import BackgroundTasksSettings register_module_settings( app, @@ -141,13 +141,13 @@ def register_menu_items(self, registry: MenuRegistry) -> None: ) def register_routes(self, api_router: APIRouter, view_router: APIRouter) -> None: - from background_tasks.endpoints.api_admin import router as api_admin - from background_tasks.endpoints.api_status import router as api_status - from background_tasks.endpoints.logs_api import router as logs_api - from background_tasks.endpoints.logs_views import router as logs_views - from background_tasks.endpoints.outbox_api import router as outbox_api - from background_tasks.endpoints.outbox_views import router as outbox_views - from background_tasks.endpoints.views import router as views + from background_tasks.web.endpoints.api_admin import router as api_admin + from background_tasks.web.endpoints.api_status import router as api_status + from background_tasks.web.endpoints.logs_api import router as logs_api + from background_tasks.web.endpoints.logs_views import router as logs_views + from background_tasks.web.endpoints.outbox_api import router as outbox_api + from background_tasks.web.endpoints.outbox_views import router as outbox_views + from background_tasks.web.endpoints.views import router as views api_router.include_router(api_admin) api_router.include_router(api_status) @@ -164,17 +164,17 @@ def locale_dirs(self) -> dict[str, Path]: async def on_startup(self, app: FastAPI) -> None: """Build the Celery app, install signal handlers, hand-off to worker.""" - from background_tasks._lifecycle import startup + from background_tasks.core._lifecycle import startup await startup(app, self.meta.name) async def on_shutdown(self, app: FastAPI) -> None: - from background_tasks._lifecycle import shutdown + from background_tasks.core._lifecycle import shutdown await shutdown(app) @staticmethod async def _start_invalidation_transport(app: FastAPI) -> None: - from background_tasks._lifecycle import start_invalidation_transport + from background_tasks.core._lifecycle import start_invalidation_transport await start_invalidation_transport(app) diff --git a/modules/background_tasks/background_tasks/outbox/__init__.py b/modules/background_tasks/background_tasks/outbox/__init__.py new file mode 100644 index 000000000..fe5f57378 --- /dev/null +++ b/modules/background_tasks/background_tasks/outbox/__init__.py @@ -0,0 +1,5 @@ +"""Durable event delivery. + +The transport, claiming, dispatch, the database clock, commit guard, worker +loops, recovery sweeps and admin operations. +""" diff --git a/modules/background_tasks/background_tasks/outbox_claim.py b/modules/background_tasks/background_tasks/outbox/claim.py similarity index 98% rename from modules/background_tasks/background_tasks/outbox_claim.py rename to modules/background_tasks/background_tasks/outbox/claim.py index 65c820dc5..08a8bb19b 100644 --- a/modules/background_tasks/background_tasks/outbox_claim.py +++ b/modules/background_tasks/background_tasks/outbox/claim.py @@ -22,7 +22,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from sqlmodel import col -from background_tasks.constants import ( +from background_tasks.core.constants import ( OUTBOX_BACKOFF_BASE_SECONDS, OUTBOX_BACKOFF_CAP_SECONDS, OUTBOX_CLAIM_BATCH, @@ -30,7 +30,7 @@ max_outbox_handler_timeout, ) from background_tasks.models import OutboxMessage -from background_tasks.outbox_clock import db_now +from background_tasks.outbox.clock import db_now logger = logging.getLogger(__name__) diff --git a/modules/background_tasks/background_tasks/outbox_clock.py b/modules/background_tasks/background_tasks/outbox/clock.py similarity index 100% rename from modules/background_tasks/background_tasks/outbox_clock.py rename to modules/background_tasks/background_tasks/outbox/clock.py diff --git a/modules/background_tasks/background_tasks/outbox_dispatch.py b/modules/background_tasks/background_tasks/outbox/dispatch.py similarity index 98% rename from modules/background_tasks/background_tasks/outbox_dispatch.py rename to modules/background_tasks/background_tasks/outbox/dispatch.py index e35894ea2..2bfb30d00 100644 --- a/modules/background_tasks/background_tasks/outbox_dispatch.py +++ b/modules/background_tasks/background_tasks/outbox/dispatch.py @@ -47,13 +47,13 @@ from simple_module_db.callbacks import discard_on_commit_callbacks, run_on_commit_callbacks from sqlalchemy import update -from background_tasks.constants import ( +from background_tasks.core.constants import ( OUTBOX_DISPATCH_LIMIT, OUTBOX_ERROR_MAX_LENGTH, OutboxStatus, ) from background_tasks.models import OutboxMessage -from background_tasks.outbox_claim import ( +from background_tasks.outbox.claim import ( ClaimedRow, OutboxPolicy, SessionFactory, @@ -61,8 +61,8 @@ claim, reclaim_expired, ) -from background_tasks.outbox_guard import handler_guard -from background_tasks.outbox_support import ( +from background_tasks.outbox.guard import handler_guard +from background_tasks.outbox.support import ( Clock, DispatchResult, describe, diff --git a/modules/background_tasks/background_tasks/outbox_guard.py b/modules/background_tasks/background_tasks/outbox/guard.py similarity index 100% rename from modules/background_tasks/background_tasks/outbox_guard.py rename to modules/background_tasks/background_tasks/outbox/guard.py diff --git a/modules/background_tasks/background_tasks/outbox_loop.py b/modules/background_tasks/background_tasks/outbox/loop.py similarity index 100% rename from modules/background_tasks/background_tasks/outbox_loop.py rename to modules/background_tasks/background_tasks/outbox/loop.py diff --git a/modules/background_tasks/background_tasks/outbox_service.py b/modules/background_tasks/background_tasks/outbox/service.py similarity index 97% rename from modules/background_tasks/background_tasks/outbox_service.py rename to modules/background_tasks/background_tasks/outbox/service.py index e60d7f1b2..a86c74d76 100644 --- a/modules/background_tasks/background_tasks/outbox_service.py +++ b/modules/background_tasks/background_tasks/outbox/service.py @@ -11,15 +11,15 @@ from sqlalchemy import delete, false, func, select, update from sqlmodel import col -from background_tasks.constants import OutboxStatus from background_tasks.contracts.outbox import OutboxItem, OutboxListResponse +from background_tasks.core.constants import OutboxStatus from background_tasks.models import OutboxMessage -from background_tasks.outbox_clock import db_now +from background_tasks.outbox.clock import db_now if TYPE_CHECKING: from simple_module_db import RequestSession - from background_tasks.outbox_transport import OutboxTransport + from background_tasks.outbox.transport import OutboxTransport # What the screen is for: deliveries that have not happened yet. Delivered rows # are history and only wait for the purge. diff --git a/modules/background_tasks/background_tasks/outbox_support.py b/modules/background_tasks/background_tasks/outbox/support.py similarity index 95% rename from modules/background_tasks/background_tasks/outbox_support.py rename to modules/background_tasks/background_tasks/outbox/support.py index 8d492a0e1..efb65696a 100644 --- a/modules/background_tasks/background_tasks/outbox_support.py +++ b/modules/background_tasks/background_tasks/outbox/support.py @@ -13,8 +13,8 @@ from simple_module_db import all_tenants, no_tenant, tenant_context -from background_tasks import tenant_context as task_tenant -from background_tasks.constants import OUTBOX_ERROR_MAX_LENGTH, OutboxScope +from background_tasks.core.constants import OUTBOX_ERROR_MAX_LENGTH, OutboxScope +from background_tasks.executions import tenant_context as task_tenant logger = logging.getLogger(__name__) diff --git a/modules/background_tasks/background_tasks/outbox_sweep.py b/modules/background_tasks/background_tasks/outbox/sweep.py similarity index 90% rename from modules/background_tasks/background_tasks/outbox_sweep.py rename to modules/background_tasks/background_tasks/outbox/sweep.py index 0231b7857..f5ef38b3d 100644 --- a/modules/background_tasks/background_tasks/outbox_sweep.py +++ b/modules/background_tasks/background_tasks/outbox/sweep.py @@ -9,9 +9,9 @@ from simple_module_core.events import EventBus -from background_tasks.outbox_claim import OutboxPolicy, SessionFactory -from background_tasks.outbox_dispatch import dispatch -from background_tasks.services import BackgroundTasksServices +from background_tasks.core.services import BackgroundTasksServices +from background_tasks.outbox.claim import OutboxPolicy, SessionFactory +from background_tasks.outbox.dispatch import dispatch logger = logging.getLogger(__name__) JITTER_FRACTION = 0.1 diff --git a/modules/background_tasks/background_tasks/outbox/transport.py b/modules/background_tasks/background_tasks/outbox/transport.py new file mode 100644 index 000000000..99d9ae39f --- /dev/null +++ b/modules/background_tasks/background_tasks/outbox/transport.py @@ -0,0 +1,198 @@ +"""The ``DurableTransport`` background_tasks installs on an ``EventBus``. + +``enqueue`` writes one ``OutboxMessage`` per durable handler in the publisher's +own session, so the event exists exactly when the publisher's transaction +commits. After the commit it asks a worker to deliver those rows right away +(``dispatch_outbox``). That hand-off is best-effort: if the process dies between +the commit and the publish, or the broker is down, the rows are still +``pending`` and the ``sweep_outbox`` beat task delivers them within one sweep +interval. + +Two installations: + +- :class:`OutboxTransport` on the web app's bus. In eager mode + (``task_always_eager``, tests and single-process dev) there is no worker: + rows are delivered by running the dispatcher on the app's own engine once the + publisher commits — from ``on_commit`` for a request's session, and from the + session's ``after_commit`` event for any other (a CLI script, a startup hook, + a durable handler's ``ctx.db``). ``send_task`` would be no use there: Celery + ignores ``task_always_eager`` for it, so the row would wait forever. +- :class:`WorkerOutboxTransport` on the bus a Celery worker builds, so a + durable handler (or any task) can publish durably through ``ctx.db``. Its + rows are handed to ``dispatch_outbox`` after that session commits — for a + handler, after the transaction that also marks its own delivery done. +""" + +from __future__ import annotations + +import asyncio +import logging +import uuid +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any + +from simple_module_core.durable_events import event_key, serialize_event +from simple_module_db import RequestSession, current_tenant_id +from simple_module_db.tenancy import is_all_tenants +from simple_module_db.transaction import SESSION_START_KEY +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import Session + +from background_tasks.core.constants import INTERNAL_TASK_DISPATCH_OUTBOX, OutboxMode, OutboxScope +from background_tasks.executions.after_commit import after_commit, current_app +from background_tasks.models import OutboxMessage +from background_tasks.outbox.claim import OutboxPolicy +from background_tasks.outbox.clock import db_now +from background_tasks.outbox.dispatch import dispatch + +if TYPE_CHECKING: + from simple_module_core.events import EventBus + from simple_module_db import DatabaseState + + from background_tasks.core.services import BackgroundTasksServices + +logger = logging.getLogger(__name__) + + +class _OutboxWriter: + """Writes the rows; subclasses decide how they reach a dispatcher.""" + + async def enqueue( + self, event: Any, handler_names: Sequence[str], db: AsyncSession | Session + ) -> None: + payload = serialize_event(event) + key = event_key(type(event)) + # The database's clock, the one the claim compares against: a + # publisher whose clock ran ahead would make its rows look not yet + # due to the worker that dispatches them right after the commit. + now = await db_now(db) + # The publisher's scope, so the handler runs in it: its tenant, the + # platform (inside ``all_tenants()``), or none bound — which must not + # be mistaken for the platform, or a public route's event would be + # handled with access to every tenant. + tenant_id = current_tenant_id.get() + if tenant_id is not None: + scope = OutboxScope.TENANT + elif is_all_tenants(): + scope = OutboxScope.PLATFORM + else: + scope = OutboxScope.UNSCOPED + rows = [ + OutboxMessage( + event_key=key, + payload=payload, + handler_name=name, + tenant_id=tenant_id, + scope=scope, + next_attempt_at=now, + created_at=now, + ) + for name in handler_names + ] + if isinstance(db, AsyncSession): + db.add_all(rows) + await db.flush() + else: + db.add_all(rows) + db.flush() + self.schedule(db, [row.id for row in rows]) + + def schedule(self, db: AsyncSession | Session, ids: Sequence[uuid.UUID]) -> None: + raise NotImplementedError + + +class WorkerOutboxTransport(_OutboxWriter): + """Durable publishing from inside a Celery worker (GH #459).""" + + def schedule(self, db: AsyncSession | Session, ids: Sequence[uuid.UUID]) -> None: + """Hand *ids* to ``dispatch_outbox`` once *db* commits; nothing if it rolls back.""" + if not ids: + return + id_strs = [str(i) for i in ids] + after_commit( + db, [lambda: current_app.send_task(INTERNAL_TASK_DISPATCH_OUTBOX, args=[id_strs])] + ) + + +class OutboxTransport(_OutboxWriter): + """Outbox-backed durable delivery for the web app (GH #459).""" + + def __init__( + self, + *, + services: BackgroundTasksServices, + bus: EventBus, + db_state: DatabaseState, + mode: OutboxMode = OutboxMode.CELERY, + ) -> None: + self._mode = mode + self._services = services + self._bus = bus + self._db_state = db_state + # Eager in-process deliveries still running (kept so they aren't + # garbage-collected mid-flight, and so ``drain`` can await them). + self._tasks: set[asyncio.Task[Any]] = set() + # Chained eager publishes must not share a StaticPool transaction (#655). + self._dispatch_lock = asyncio.Lock() + + def policy(self) -> OutboxPolicy: + s = self._services.settings + return OutboxPolicy( + max_attempts=s.outbox_max_attempts, + lease_seconds=s.outbox_lease_seconds, + handler_timeout_seconds=s.outbox_handler_timeout_seconds, + ) + + def schedule(self, db: AsyncSession | Session, ids: Sequence[uuid.UUID]) -> None: + """Deliver *ids* once *db* commits; nothing if it rolls back.""" + if not ids: + return + id_list = list(ids) + if self._mode == OutboxMode.INPROCESS or self._services.settings.task_always_eager: + if isinstance(db, RequestSession) and SESSION_START_KEY in db.info: + db.on_commit(lambda: self._dispatch(id_list)) + else: + after_commit(db, [lambda: self._dispatch_soon(id_list)]) + return + id_strs = [str(i) for i in id_list] + after_commit(db, [lambda: self._send(id_strs)]) + + async def _dispatch(self, ids: list[uuid.UUID]) -> Any: + async with self._dispatch_lock: + return await dispatch(self._db_state.session_factory, self._bus, self.policy(), ids=ids) + + def _dispatch_soon(self, ids: list[uuid.UUID]) -> None: + """Eager delivery from a session's ``after_commit`` event. + + That event fires inside the commit, so the delivery is started as a + task on the running loop rather than awaited. With no loop (a sync + session in a plain script) it runs to completion right here. + """ + try: + loop = asyncio.get_running_loop() + except RuntimeError: + asyncio.run(self._dispatch(ids)) + return + task = loop.create_task(self._dispatch(ids)) + self._tasks.add(task) + task.add_done_callback(self._finished) + + def _finished(self, task: asyncio.Task[Any]) -> None: + self._tasks.discard(task) + if not task.cancelled() and task.exception() is not None: + logger.error("outbox: eager delivery failed", exc_info=task.exception()) + + async def drain(self) -> None: + """Wait for eager deliveries started from ``after_commit`` (tests, shutdown).""" + while self._tasks: + await asyncio.gather(*list(self._tasks), return_exceptions=True) + + def _send(self, ids: list[str]) -> None: + celery = self._services.celery + if celery is None: + logger.warning("outbox: Celery not started; %d row(s) wait for the sweep", len(ids)) + return + celery.send_task(INTERNAL_TASK_DISPATCH_OUTBOX, args=[ids]) + + +__all__ = ["OutboxTransport", "WorkerOutboxTransport"] diff --git a/modules/background_tasks/background_tasks/outbox/worker.py b/modules/background_tasks/background_tasks/outbox/worker.py new file mode 100644 index 000000000..0e1b1e4b3 --- /dev/null +++ b/modules/background_tasks/background_tasks/outbox/worker.py @@ -0,0 +1,188 @@ +"""What the outbox Celery tasks need in a worker process: an async DB and a bus. + +**Database.** Durable handlers take an ``AsyncSession`` (``ctx.db``), the same +kind request code uses, so module service code runs unchanged. Celery tasks are +synchronous, so each dispatch runs on the calling thread's long-lived outbox +event loop (:mod:`.outbox_loop`), and the engine — built once per loop from the +URL :mod:`.sync_db` was pinned to — keeps a small connection pool on that loop. +Under the ``threads`` pool every Celery thread therefore has its own loop and +engine (a pool of one, plus two overflow). +Its session class gets the framework's entity listeners and the host's tenancy +policy, like the sync engine's. + +**Bus.** A worker never builds the app, so it builds its own ``EventBus`` once +per process by calling every installed module's +``register_event_handlers(bus, None)``. That is where durable subscriptions +come from. It carries a :class:`~.outbox_transport.WorkerOutboxTransport`, so +a durable handler can itself ``publish_durable`` through ``ctx.db``. In the web +process ``on_startup`` hands over the app's own bus instead, which already has +both. +""" + +from __future__ import annotations + +import inspect +import logging +import threading +from typing import Any + +from simple_module_core.events import EventBus +from simple_module_db import init_db +from simple_module_db.listeners import attach_session_listeners +from simple_module_db.query_filter import bind_engine_policy +from simple_module_db.tenant_cleanup import register_tenant_cleanup +from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker + +from background_tasks.core import sync_db +from background_tasks.core.constants import MODULE_NAME +from background_tasks.core.settings import BackgroundTasksSettings +from background_tasks.outbox.claim import OutboxPolicy +from background_tasks.outbox.dispatch import dispatch +from background_tasks.outbox.loop import run_on_worker_loop, stop_worker_loop, worker_loop_cache +from background_tasks.outbox.transport import WorkerOutboxTransport + +logger = logging.getLogger(__name__) + +_lock = threading.Lock() +_ENGINE_KEY = "outbox_worker_engine" +_bus: EventBus | None = None +_bus_override: EventBus | None = None + + +def worker_session_factory() -> async_sessionmaker[Any]: + """The calling thread's async session factory on the worker's database. + + Cached on the thread's outbox loop: its pooled connections can only be + used there, and :func:`run_dispatch` runs on that same loop. + """ + url = sync_db.async_database_url() + policy = sync_db.engine_tenancy() + options = sync_db.database_options() + key = (url, policy.tenant_strict, policy.default_tenant_id, tuple(sorted(options.items()))) + cache = worker_loop_cache() + cached: tuple[Any, async_sessionmaker[Any], AsyncEngine] | None = cache.get(_ENGINE_KEY) + if cached is not None and cached[0] == key: + return cached[1] + if cached is not None: + _dispose_soon(cached[2]) + # Small pool of its own: one thread delivers one row at a time. Timeouts and + # recycle are shared with the sync workers. + state = init_db( + url, + pool_size=1, + max_overflow=2, + pool_recycle=options.get("pool_recycle", 1800), + pool_timeout=options.get("pool_timeout", 30), + connect_timeout=options.get("connect_timeout", 10), + statement_timeout_ms=options.get("statement_timeout_ms", 0), + idle_in_tx_timeout_ms=options.get("idle_in_tx_timeout_ms", 0), + ) + attach_session_listeners(state.sync_session_class) + bind_engine_policy(state.engine.sync_engine, policy) + cache[_ENGINE_KEY] = (key, state.session_factory, state.engine) + return state.session_factory + + +def _dispose_soon(engine: AsyncEngine) -> None: + try: + run_on_worker_loop(engine.dispose) + except Exception: + logger.warning("outbox: disposing a reconfigured worker engine failed", exc_info=True) + + +async def _dispose_cached(cache: dict[str, Any]) -> None: + cached = cache.pop(_ENGINE_KEY, None) + if cached is not None: + await cached[2].dispose() + + +def use_bus(bus: EventBus | None) -> None: + """Dispatch with *bus* (the web app's own) instead of building one; ``None`` clears.""" + global _bus_override + _bus_override = bus + + +def _register(module: Any, bus: EventBus) -> None: + hook = module.register_event_handlers + params = inspect.signature(hook).parameters + accepts_app = "app" in params or any( + p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values() + ) + if accepts_app: + hook(bus, app=None) + else: + hook(bus) + + +def build_worker_bus() -> EventBus: + """An ``EventBus`` carrying every installed module's subscriptions.""" + from simple_module_core.discovery import discover_modules, topological_sort + + bus = EventBus() + register_tenant_cleanup(bus) + bus.durable_transport = WorkerOutboxTransport() + # Lenient on purpose: the worker delivers whatever is installed, and a + # module set the web process refuses is reported there, not here. + for module in topological_sort(discover_modules()): + try: + _register(module, bus) + except Exception: + # One module's handler wiring needing the app must not stop every + # other module's durable events from being delivered. Logged once + # per worker process (the bus is built once). + logger.exception( + "outbox.worker_bus: %s.register_event_handlers failed without an app; " + "durable handlers it subscribes after that point are unknown to this " + "worker, and their events will retry until they go dead. Subscribe " + "durable handlers before anything that needs the app.", + module.meta.name, + ) + logger.info("outbox.worker_bus: %d durable subscription(s)", len(bus.durable)) + return bus + + +def worker_bus() -> EventBus: + global _bus + if _bus_override is not None: + return _bus_override + with _lock: + if _bus is None: + _bus = build_worker_bus() + return _bus + + +def reset_worker_state() -> None: + """Forget the cached engines, bus and loops (tests, or after reconfiguring sync_db).""" + global _bus, _bus_override + with _lock: + _bus = None + _bus_override = None + stop_worker_loop(_dispose_cached) + + +def load_policy() -> OutboxPolicy: + """Outbox limits from DB-hydrated settings, env/defaults if the DB can't say.""" + from background_tasks.worker.settings import settings_for + + try: + settings = settings_for(BackgroundTasksSettings, MODULE_NAME) + except Exception: + logger.warning("outbox: settings unreadable from the DB; using env/defaults") + settings = BackgroundTasksSettings() + return OutboxPolicy( + max_attempts=settings.outbox_max_attempts, + lease_seconds=settings.outbox_lease_seconds, + handler_timeout_seconds=settings.outbox_handler_timeout_seconds, + ) + + +def run_dispatch(ids: list[str] | None) -> dict[str, int]: + """Body of the ``dispatch_outbox`` (``ids``) and ``sweep_outbox`` (``None``) tasks.""" + policy = load_policy() + factory = worker_session_factory() + bus = worker_bus() + result = run_on_worker_loop(lambda: dispatch(factory, bus, policy, ids=ids)) + summary = result.as_dict() + if any(summary.values()): + logger.info("outbox.dispatch %s", summary) + return summary diff --git a/modules/background_tasks/background_tasks/outbox_transport.py b/modules/background_tasks/background_tasks/outbox_transport.py index c8392c1a3..17fe1dbd7 100644 --- a/modules/background_tasks/background_tasks/outbox_transport.py +++ b/modules/background_tasks/background_tasks/outbox_transport.py @@ -1,198 +1,15 @@ -"""The ``DurableTransport`` background_tasks installs on an ``EventBus``. - -``enqueue`` writes one ``OutboxMessage`` per durable handler in the publisher's -own session, so the event exists exactly when the publisher's transaction -commits. After the commit it asks a worker to deliver those rows right away -(``dispatch_outbox``). That hand-off is best-effort: if the process dies between -the commit and the publish, or the broker is down, the rows are still -``pending`` and the ``sweep_outbox`` beat task delivers them within one sweep -interval. - -Two installations: - -- :class:`OutboxTransport` on the web app's bus. In eager mode - (``task_always_eager``, tests and single-process dev) there is no worker: - rows are delivered by running the dispatcher on the app's own engine once the - publisher commits — from ``on_commit`` for a request's session, and from the - session's ``after_commit`` event for any other (a CLI script, a startup hook, - a durable handler's ``ctx.db``). ``send_task`` would be no use there: Celery - ignores ``task_always_eager`` for it, so the row would wait forever. -- :class:`WorkerOutboxTransport` on the bus a Celery worker builds, so a - durable handler (or any task) can publish durably through ``ctx.db``. Its - rows are handed to ``dispatch_outbox`` after that session commits — for a - handler, after the transaction that also marks its own delivery done. -""" +"""Moved to :mod:`background_tasks.outbox.transport`; this alias goes in 0.0.46 (module layout).""" from __future__ import annotations -import asyncio -import logging -import uuid -from collections.abc import Sequence -from typing import TYPE_CHECKING, Any - -from simple_module_core.durable_events import event_key, serialize_event -from simple_module_db import RequestSession, current_tenant_id -from simple_module_db.tenancy import is_all_tenants -from simple_module_db.transaction import SESSION_START_KEY -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import Session - -from background_tasks.after_commit import after_commit, current_app -from background_tasks.constants import INTERNAL_TASK_DISPATCH_OUTBOX, OutboxMode, OutboxScope -from background_tasks.models import OutboxMessage -from background_tasks.outbox_claim import OutboxPolicy -from background_tasks.outbox_clock import db_now -from background_tasks.outbox_dispatch import dispatch - -if TYPE_CHECKING: - from simple_module_core.events import EventBus - from simple_module_db import DatabaseState - - from background_tasks.services import BackgroundTasksServices - -logger = logging.getLogger(__name__) - - -class _OutboxWriter: - """Writes the rows; subclasses decide how they reach a dispatcher.""" - - async def enqueue( - self, event: Any, handler_names: Sequence[str], db: AsyncSession | Session - ) -> None: - payload = serialize_event(event) - key = event_key(type(event)) - # The database's clock, the one the claim compares against: a - # publisher whose clock ran ahead would make its rows look not yet - # due to the worker that dispatches them right after the commit. - now = await db_now(db) - # The publisher's scope, so the handler runs in it: its tenant, the - # platform (inside ``all_tenants()``), or none bound — which must not - # be mistaken for the platform, or a public route's event would be - # handled with access to every tenant. - tenant_id = current_tenant_id.get() - if tenant_id is not None: - scope = OutboxScope.TENANT - elif is_all_tenants(): - scope = OutboxScope.PLATFORM - else: - scope = OutboxScope.UNSCOPED - rows = [ - OutboxMessage( - event_key=key, - payload=payload, - handler_name=name, - tenant_id=tenant_id, - scope=scope, - next_attempt_at=now, - created_at=now, - ) - for name in handler_names - ] - if isinstance(db, AsyncSession): - db.add_all(rows) - await db.flush() - else: - db.add_all(rows) - db.flush() - self.schedule(db, [row.id for row in rows]) - - def schedule(self, db: AsyncSession | Session, ids: Sequence[uuid.UUID]) -> None: - raise NotImplementedError - - -class WorkerOutboxTransport(_OutboxWriter): - """Durable publishing from inside a Celery worker (GH #459).""" - - def schedule(self, db: AsyncSession | Session, ids: Sequence[uuid.UUID]) -> None: - """Hand *ids* to ``dispatch_outbox`` once *db* commits; nothing if it rolls back.""" - if not ids: - return - id_strs = [str(i) for i in ids] - after_commit( - db, [lambda: current_app.send_task(INTERNAL_TASK_DISPATCH_OUTBOX, args=[id_strs])] - ) - - -class OutboxTransport(_OutboxWriter): - """Outbox-backed durable delivery for the web app (GH #459).""" - - def __init__( - self, - *, - services: BackgroundTasksServices, - bus: EventBus, - db_state: DatabaseState, - mode: OutboxMode = OutboxMode.CELERY, - ) -> None: - self._mode = mode - self._services = services - self._bus = bus - self._db_state = db_state - # Eager in-process deliveries still running (kept so they aren't - # garbage-collected mid-flight, and so ``drain`` can await them). - self._tasks: set[asyncio.Task[Any]] = set() - # Chained eager publishes must not share a StaticPool transaction (#655). - self._dispatch_lock = asyncio.Lock() - - def policy(self) -> OutboxPolicy: - s = self._services.settings - return OutboxPolicy( - max_attempts=s.outbox_max_attempts, - lease_seconds=s.outbox_lease_seconds, - handler_timeout_seconds=s.outbox_handler_timeout_seconds, - ) - - def schedule(self, db: AsyncSession | Session, ids: Sequence[uuid.UUID]) -> None: - """Deliver *ids* once *db* commits; nothing if it rolls back.""" - if not ids: - return - id_list = list(ids) - if self._mode == OutboxMode.INPROCESS or self._services.settings.task_always_eager: - if isinstance(db, RequestSession) and SESSION_START_KEY in db.info: - db.on_commit(lambda: self._dispatch(id_list)) - else: - after_commit(db, [lambda: self._dispatch_soon(id_list)]) - return - id_strs = [str(i) for i in id_list] - after_commit(db, [lambda: self._send(id_strs)]) - - async def _dispatch(self, ids: list[uuid.UUID]) -> Any: - async with self._dispatch_lock: - return await dispatch(self._db_state.session_factory, self._bus, self.policy(), ids=ids) - - def _dispatch_soon(self, ids: list[uuid.UUID]) -> None: - """Eager delivery from a session's ``after_commit`` event. - - That event fires inside the commit, so the delivery is started as a - task on the running loop rather than awaited. With no loop (a sync - session in a plain script) it runs to completion right here. - """ - try: - loop = asyncio.get_running_loop() - except RuntimeError: - asyncio.run(self._dispatch(ids)) - return - task = loop.create_task(self._dispatch(ids)) - self._tasks.add(task) - task.add_done_callback(self._finished) - - def _finished(self, task: asyncio.Task[Any]) -> None: - self._tasks.discard(task) - if not task.cancelled() and task.exception() is not None: - logger.error("outbox: eager delivery failed", exc_info=task.exception()) - - async def drain(self) -> None: - """Wait for eager deliveries started from ``after_commit`` (tests, shutdown).""" - while self._tasks: - await asyncio.gather(*list(self._tasks), return_exceptions=True) - - def _send(self, ids: list[str]) -> None: - celery = self._services.celery - if celery is None: - logger.warning("outbox: Celery not started; %d row(s) wait for the sweep", len(ids)) - return - celery.send_task(INTERNAL_TASK_DISPATCH_OUTBOX, args=[ids]) +import importlib +import sys +from simple_module_core.deprecation import deprecated_module -__all__ = ["OutboxTransport", "WorkerOutboxTransport"] +deprecated_module( + "background_tasks.outbox_transport", + remove_in="0.0.46", + use="background_tasks.outbox.transport", +) +sys.modules[__name__] = importlib.import_module("background_tasks.outbox.transport") diff --git a/modules/background_tasks/background_tasks/outbox_worker.py b/modules/background_tasks/background_tasks/outbox_worker.py index 49598160f..9c8c47271 100644 --- a/modules/background_tasks/background_tasks/outbox_worker.py +++ b/modules/background_tasks/background_tasks/outbox_worker.py @@ -1,188 +1,15 @@ -"""What the outbox Celery tasks need in a worker process: an async DB and a bus. - -**Database.** Durable handlers take an ``AsyncSession`` (``ctx.db``), the same -kind request code uses, so module service code runs unchanged. Celery tasks are -synchronous, so each dispatch runs on the calling thread's long-lived outbox -event loop (:mod:`.outbox_loop`), and the engine — built once per loop from the -URL :mod:`.sync_db` was pinned to — keeps a small connection pool on that loop. -Under the ``threads`` pool every Celery thread therefore has its own loop and -engine (a pool of one, plus two overflow). -Its session class gets the framework's entity listeners and the host's tenancy -policy, like the sync engine's. - -**Bus.** A worker never builds the app, so it builds its own ``EventBus`` once -per process by calling every installed module's -``register_event_handlers(bus, None)``. That is where durable subscriptions -come from. It carries a :class:`~.outbox_transport.WorkerOutboxTransport`, so -a durable handler can itself ``publish_durable`` through ``ctx.db``. In the web -process ``on_startup`` hands over the app's own bus instead, which already has -both. -""" +"""Moved to :mod:`background_tasks.outbox.worker`; this alias goes in 0.0.46 (module layout).""" from __future__ import annotations -import inspect -import logging -import threading -from typing import Any - -from simple_module_core.events import EventBus -from simple_module_db import init_db -from simple_module_db.listeners import attach_session_listeners -from simple_module_db.query_filter import bind_engine_policy -from simple_module_db.tenant_cleanup import register_tenant_cleanup -from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker - -from background_tasks import sync_db -from background_tasks.constants import MODULE_NAME -from background_tasks.outbox_claim import OutboxPolicy -from background_tasks.outbox_dispatch import dispatch -from background_tasks.outbox_loop import run_on_worker_loop, stop_worker_loop, worker_loop_cache -from background_tasks.outbox_transport import WorkerOutboxTransport -from background_tasks.settings import BackgroundTasksSettings - -logger = logging.getLogger(__name__) - -_lock = threading.Lock() -_ENGINE_KEY = "outbox_worker_engine" -_bus: EventBus | None = None -_bus_override: EventBus | None = None - - -def worker_session_factory() -> async_sessionmaker[Any]: - """The calling thread's async session factory on the worker's database. - - Cached on the thread's outbox loop: its pooled connections can only be - used there, and :func:`run_dispatch` runs on that same loop. - """ - url = sync_db.async_database_url() - policy = sync_db.engine_tenancy() - options = sync_db.database_options() - key = (url, policy.tenant_strict, policy.default_tenant_id, tuple(sorted(options.items()))) - cache = worker_loop_cache() - cached: tuple[Any, async_sessionmaker[Any], AsyncEngine] | None = cache.get(_ENGINE_KEY) - if cached is not None and cached[0] == key: - return cached[1] - if cached is not None: - _dispose_soon(cached[2]) - # Small pool of its own: one thread delivers one row at a time. Timeouts and - # recycle are shared with the sync workers. - state = init_db( - url, - pool_size=1, - max_overflow=2, - pool_recycle=options.get("pool_recycle", 1800), - pool_timeout=options.get("pool_timeout", 30), - connect_timeout=options.get("connect_timeout", 10), - statement_timeout_ms=options.get("statement_timeout_ms", 0), - idle_in_tx_timeout_ms=options.get("idle_in_tx_timeout_ms", 0), - ) - attach_session_listeners(state.sync_session_class) - bind_engine_policy(state.engine.sync_engine, policy) - cache[_ENGINE_KEY] = (key, state.session_factory, state.engine) - return state.session_factory - - -def _dispose_soon(engine: AsyncEngine) -> None: - try: - run_on_worker_loop(engine.dispose) - except Exception: - logger.warning("outbox: disposing a reconfigured worker engine failed", exc_info=True) - - -async def _dispose_cached(cache: dict[str, Any]) -> None: - cached = cache.pop(_ENGINE_KEY, None) - if cached is not None: - await cached[2].dispose() - - -def use_bus(bus: EventBus | None) -> None: - """Dispatch with *bus* (the web app's own) instead of building one; ``None`` clears.""" - global _bus_override - _bus_override = bus - - -def _register(module: Any, bus: EventBus) -> None: - hook = module.register_event_handlers - params = inspect.signature(hook).parameters - accepts_app = "app" in params or any( - p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values() - ) - if accepts_app: - hook(bus, app=None) - else: - hook(bus) - - -def build_worker_bus() -> EventBus: - """An ``EventBus`` carrying every installed module's subscriptions.""" - from simple_module_core.discovery import discover_modules, topological_sort - - bus = EventBus() - register_tenant_cleanup(bus) - bus.durable_transport = WorkerOutboxTransport() - # Lenient on purpose: the worker delivers whatever is installed, and a - # module set the web process refuses is reported there, not here. - for module in topological_sort(discover_modules()): - try: - _register(module, bus) - except Exception: - # One module's handler wiring needing the app must not stop every - # other module's durable events from being delivered. Logged once - # per worker process (the bus is built once). - logger.exception( - "outbox.worker_bus: %s.register_event_handlers failed without an app; " - "durable handlers it subscribes after that point are unknown to this " - "worker, and their events will retry until they go dead. Subscribe " - "durable handlers before anything that needs the app.", - module.meta.name, - ) - logger.info("outbox.worker_bus: %d durable subscription(s)", len(bus.durable)) - return bus - - -def worker_bus() -> EventBus: - global _bus - if _bus_override is not None: - return _bus_override - with _lock: - if _bus is None: - _bus = build_worker_bus() - return _bus - - -def reset_worker_state() -> None: - """Forget the cached engines, bus and loops (tests, or after reconfiguring sync_db).""" - global _bus, _bus_override - with _lock: - _bus = None - _bus_override = None - stop_worker_loop(_dispose_cached) - - -def load_policy() -> OutboxPolicy: - """Outbox limits from DB-hydrated settings, env/defaults if the DB can't say.""" - from background_tasks.worker_settings import settings_for - - try: - settings = settings_for(BackgroundTasksSettings, MODULE_NAME) - except Exception: - logger.warning("outbox: settings unreadable from the DB; using env/defaults") - settings = BackgroundTasksSettings() - return OutboxPolicy( - max_attempts=settings.outbox_max_attempts, - lease_seconds=settings.outbox_lease_seconds, - handler_timeout_seconds=settings.outbox_handler_timeout_seconds, - ) +import importlib +import sys +from simple_module_core.deprecation import deprecated_module -def run_dispatch(ids: list[str] | None) -> dict[str, int]: - """Body of the ``dispatch_outbox`` (``ids``) and ``sweep_outbox`` (``None``) tasks.""" - policy = load_policy() - factory = worker_session_factory() - bus = worker_bus() - result = run_on_worker_loop(lambda: dispatch(factory, bus, policy, ids=ids)) - summary = result.as_dict() - if any(summary.values()): - logger.info("outbox.dispatch %s", summary) - return summary +deprecated_module( + "background_tasks.outbox_worker", + remove_in="0.0.46", + use="background_tasks.outbox.worker", +) +sys.modules[__name__] = importlib.import_module("background_tasks.outbox.worker") diff --git a/modules/background_tasks/background_tasks/pages/Detail.tsx b/modules/background_tasks/background_tasks/pages/Detail.tsx index 3cb009d4a..0ed97299c 100644 --- a/modules/background_tasks/background_tasks/pages/Detail.tsx +++ b/modules/background_tasks/background_tasks/pages/Detail.tsx @@ -7,15 +7,15 @@ import { usePermissions } from '@simple-module-py/ui/hooks/use-permissions'; import { AdminLayout } from '@simple-module-py/ui/layouts/AdminLayout'; import { ArrowLeft, RefreshCcw } from 'lucide-react'; import { useState } from 'react'; -import { DetailFacts } from './components/DetailFacts'; -import { LogsPanel } from './components/LogsPanel'; -import { PayloadCards } from './components/PayloadCards'; -import { ProgressCard } from './components/ProgressCard'; -import { RetryConfirmDialog } from './components/RetryConfirmDialog'; -import { StatusPill } from './components/StatusPill'; -import { TracebackCard } from './components/TracebackCard'; -import { RETRYABLE_STATUSES, type TaskDetail, VIEW_BASE, VIEW_INDEX } from './constants'; -import { retryExecution } from './retry'; +import { DetailFacts } from '../ui/components/DetailFacts'; +import { LogsPanel } from '../ui/components/LogsPanel'; +import { PayloadCards } from '../ui/components/PayloadCards'; +import { ProgressCard } from '../ui/components/ProgressCard'; +import { RetryConfirmDialog } from '../ui/components/RetryConfirmDialog'; +import { StatusPill } from '../ui/components/StatusPill'; +import { TracebackCard } from '../ui/components/TracebackCard'; +import { RETRYABLE_STATUSES, type TaskDetail, VIEW_BASE, VIEW_INDEX } from '../ui/utils/constants'; +import { retryExecution } from '../ui/utils/retry'; interface Props { execution: TaskDetail; diff --git a/modules/background_tasks/background_tasks/pages/Index.tsx b/modules/background_tasks/background_tasks/pages/Index.tsx index c4a8a7780..4105f4c5b 100644 --- a/modules/background_tasks/background_tasks/pages/Index.tsx +++ b/modules/background_tasks/background_tasks/pages/Index.tsx @@ -5,15 +5,22 @@ import { Button } from '@simple-module-py/ui/components/ui/button'; import { usePermissions } from '@simple-module-py/ui/hooks/use-permissions'; import { AdminLayout } from '@simple-module-py/ui/layouts/AdminLayout'; import { useEffect, useRef, useState } from 'react'; -import { COLUMN_COUNT, ExecutionsTable } from './components/ExecutionsTable'; -import { RetryAllDialog } from './components/RetryAllDialog'; -import { RetryConfirmDialog } from './components/RetryConfirmDialog'; -import { type StatusCounts, StatusStrip } from './components/StatusStrip'; -import { TaskFilters } from './components/TaskFilters'; -import { TasksEmptyRow, type WorkerPresence } from './components/TasksEmpty'; -import { WorkerHealthBanner } from './components/WorkerHealthBanner'; -import { QUEUE_ALL, STATUS_ALL, TASK_STATUS, TENANT_ALL, VIEW_BASE, VIEW_INDEX } from './constants'; -import { type Execution, retryAllFailed, retryExecution } from './retry'; +import { COLUMN_COUNT, ExecutionsTable } from '../ui/components/ExecutionsTable'; +import { RetryAllDialog } from '../ui/components/RetryAllDialog'; +import { RetryConfirmDialog } from '../ui/components/RetryConfirmDialog'; +import { type StatusCounts, StatusStrip } from '../ui/components/StatusStrip'; +import { TaskFilters } from '../ui/components/TaskFilters'; +import { TasksEmptyRow, type WorkerPresence } from '../ui/components/TasksEmpty'; +import { WorkerHealthBanner } from '../ui/components/WorkerHealthBanner'; +import { + QUEUE_ALL, + STATUS_ALL, + TASK_STATUS, + TENANT_ALL, + VIEW_BASE, + VIEW_INDEX, +} from '../ui/utils/constants'; +import { type Execution, retryAllFailed, retryExecution } from '../ui/utils/retry'; interface Pagination { page: number; diff --git a/modules/background_tasks/background_tasks/pages/Logs.tsx b/modules/background_tasks/background_tasks/pages/Logs.tsx index 028ee73f6..1c27e20c8 100644 --- a/modules/background_tasks/background_tasks/pages/Logs.tsx +++ b/modules/background_tasks/background_tasks/pages/Logs.tsx @@ -6,10 +6,10 @@ import { Input } from '@simple-module-py/ui/components/ui/input'; import { AdminLayout } from '@simple-module-py/ui/layouts/AdminLayout'; import { ArrowLeft } from 'lucide-react'; import { type FormEvent, useState } from 'react'; -import { LogLevelSelect } from './components/LogLevelSelect'; -import { LogTable } from './components/LogTable'; -import { VIEW_BASE, VIEW_INDEX } from './constants'; -import type { LogItem } from './logs'; +import { LogLevelSelect } from '../ui/components/LogLevelSelect'; +import { LogTable } from '../ui/components/LogTable'; +import { VIEW_BASE, VIEW_INDEX } from '../ui/utils/constants'; +import type { LogItem } from '../ui/utils/logs'; interface Props { logs: LogItem[]; diff --git a/modules/background_tasks/background_tasks/pages/Outbox.tsx b/modules/background_tasks/background_tasks/pages/Outbox.tsx index d78728580..8baf42db0 100644 --- a/modules/background_tasks/background_tasks/pages/Outbox.tsx +++ b/modules/background_tasks/background_tasks/pages/Outbox.tsx @@ -8,15 +8,15 @@ import { AdminLayout } from '@simple-module-py/ui/layouts/AdminLayout'; import { cn } from '@simple-module-py/ui/lib/utils'; import { ArrowLeft, Trash2 } from 'lucide-react'; import { useState } from 'react'; -import { OutboxTable } from './components/OutboxTable'; -import { VIEW_BASE, VIEW_INDEX } from './constants'; +import { OutboxTable } from '../ui/components/OutboxTable'; +import { VIEW_BASE, VIEW_INDEX } from '../ui/utils/constants'; import { discardOutboxMessage, OPEN_OUTBOX_STATUSES, OUTBOX_STATUS_LABEL_KEY, type OutboxMessage, retryOutboxMessage, -} from './outbox'; +} from '../ui/utils/outbox'; interface Props { messages: OutboxMessage[]; diff --git a/modules/background_tasks/background_tasks/pages/Workers.tsx b/modules/background_tasks/background_tasks/pages/Workers.tsx index 5e34dc560..c021ddba6 100644 --- a/modules/background_tasks/background_tasks/pages/Workers.tsx +++ b/modules/background_tasks/background_tasks/pages/Workers.tsx @@ -8,11 +8,16 @@ import { ageOf, isStale } from '@simple-module-py/ui/lib/relative-time'; import { ArrowLeft, RefreshCw } from 'lucide-react'; import { useEffect, useState } from 'react'; import { toast } from 'sonner'; +import { RoutesTable } from '../ui/components/RoutesTable'; +import { WorkerCard, WorkersSkeleton } from '../ui/components/WorkerCard'; +import { BrokerUnreachable, NoWorkers } from '../ui/components/WorkerEmptyStates'; +import { + formatClock, + type TaskRoute, + VIEW_INDEX, + type WorkerSnapshot, +} from '../ui/utils/constants'; import { api } from './api'; -import { RoutesTable } from './components/RoutesTable'; -import { WorkerCard, WorkersSkeleton } from './components/WorkerCard'; -import { BrokerUnreachable, NoWorkers } from './components/WorkerEmptyStates'; -import { formatClock, type TaskRoute, VIEW_INDEX, type WorkerSnapshot } from './constants'; interface Props { /** Deferred: undefined until the broker poll (up to ~1s) answers. */ diff --git a/modules/background_tasks/background_tasks/runner.py b/modules/background_tasks/background_tasks/runner.py index a4213a9d7..fd47e4256 100644 --- a/modules/background_tasks/background_tasks/runner.py +++ b/modules/background_tasks/background_tasks/runner.py @@ -1,85 +1,13 @@ -"""Build the Celery app for a worker or beat process. - -The repo's ``scripts/run_worker.py`` and the one ``smpy new`` scaffolds both -call :func:`build_runner_celery`. They used to carry their own copies of this -bootstrap, and the scaffolded copy fell behind: it built Celery from the -environment alone, so a generated app's worker ignored settings saved in the -admin UI and could read a different ``.env`` from the web process (GH #490). -""" +"""Moved to :mod:`background_tasks.worker.runner`; this alias goes in 0.0.46 (module layout).""" from __future__ import annotations -import os -from pathlib import Path -from typing import TYPE_CHECKING - -from simple_module_core.dotenv import find_env_file, load_dotenv_into_environ - -if TYPE_CHECKING: - from celery import Celery - -_PROJECT_ROOT = "SM_PROJECT_ROOT" - - -def _load_env(fallback_env_file: Path | None) -> None: - """Load ``.env`` the way the web process resolves it. - - ``find_env_file`` (SM_PROJECT_ROOT, then a bounded walk-up) keeps the worker - and the web process on the same file. ``fallback_env_file`` applies only - when that resolves to no file at all — celery launched from an unrelated - cwd, possibly inside some other checkout, with SM_PROJECT_ROOT unset. It is - adopted by setting SM_PROJECT_ROOT rather than by loading the file directly, - so every later reader (``BootstrapSettings``' sqlite anchor, the host - settings' ``.env`` scan) resolves the same project instead of the cwd. - """ - env_file = find_env_file() - if ( - fallback_env_file is not None - and _PROJECT_ROOT not in os.environ - and not env_file.is_file() - and fallback_env_file.is_file() - ): - os.environ[_PROJECT_ROOT] = str(fallback_env_file.parent) - env_file = find_env_file() - load_dotenv_into_environ(env_file) - - -def build_runner_celery(fallback_env_file: Path | None = None) -> Celery: - """``.env`` → host settings → DB-hydrated ``BackgroundTasksSettings`` → Celery.""" - _load_env(fallback_env_file) - - # Imported after ``.env`` is in the environment, so nothing below can - # resolve a setting before it is. - from simple_module_hosting import merge_host_settings - from simple_module_hosting.leases import configure_process_leases +import importlib +import sys - from background_tasks.celery_app import build_celery - from background_tasks.db_settings import worker_database_options - from background_tasks.log_sink import install_log_sink - from background_tasks.settings import BackgroundTasksSettings - from background_tasks.sync_db import configure_database, set_database_url - from background_tasks.tenant_context import set_default_tenant - from background_tasks.worker_settings import load_runner_settings +from simple_module_core.deprecation import deprecated_module - # The worker never builds the app, so resolve the host settings the same - # way host/main.py does (env → DB → default): task bodies must hit the same - # database with the same fail-closed tenant rules as request code. - host = merge_host_settings() - default_tenant = None if host.multi_tenant else host.default_tenant or None - set_database_url( - host.database_url, tenant_strict=host.multi_tenant, default_tenant=default_tenant - ) - configure_database(worker_database_options(host, BackgroundTasksSettings())) - set_default_tenant(default_tenant) - # Task bodies share the web process's leases (GH #539): same Redis or table. - configure_process_leases(host.redis_url, database_url=host.database_url) - # Hydrated from the DB like the web process, so an admin-edited setting - # reaches the worker at boot, not only the web app. - settings = load_runner_settings() - # Re-apply the engine options now the worker's DB-stored overrides are known. - configure_database(worker_database_options(host, settings)) - celery = build_celery(settings) - # Task logs: the worker never runs the web lifespan that installs the sink. - # Prefork children rebuild it on ``worker_process_init``. - install_log_sink(settings) - return celery +deprecated_module( + "background_tasks.runner", remove_in="0.0.46", use="background_tasks.worker.runner" +) +sys.modules[__name__] = importlib.import_module("background_tasks.worker.runner") diff --git a/modules/background_tasks/background_tasks/settings.py b/modules/background_tasks/background_tasks/settings.py index e9a06aea9..6ddef63e6 100644 --- a/modules/background_tasks/background_tasks/settings.py +++ b/modules/background_tasks/background_tasks/settings.py @@ -1,175 +1,15 @@ -"""BackgroundTasks module settings (DB-backed). - -Values come from defaults at boot, then get hydrated from the DB by the -hosting lifespan before module ``on_startup`` runs. Runtime changes go -through ``settings.module_settings.reload.apply_changes_and_reload``. - -The broker and result-backend URLs are deployment plumbing rather than module -config, so they stay env-readable through ``SM_REDIS_URL`` — one variable that -seeds both. They have to work *before* any DB row exists: the production -validator below rejects the localhost defaults, so without them a containerised -app can't boot far enough to hydrate settings, and a worker process (which -never sees ``app.state``) has no other source at all. Like every env var it -beats a stored value (env → DB → default): ``env_seeded_fields`` reports both -fields to the hydrator, since ``SM_REDIS_URL`` is read by their default -factories where pydantic's env source cannot see it. With it unset, the two -fields keep separate DB overrides for anyone who wants the broker and the -backend on different Redis databases. - -``SM_BG_TASKS_BROKER_URL`` and ``SM_BG_TASKS_RESULT_BACKEND`` still work and -take precedence over ``SM_REDIS_URL`` — a deployment that set them meant them — -but log a deprecation warning. Being real env reads, they also beat a stored -value (env → DB → default, GH #486). They are answered by pydantic's own env source -now that ``env_prefix`` is set (GH #283), which is also what stops every other -field being readable from its bare, unprefixed name. - -The other env read is ``SM_ENVIRONMENT``, consulted by the -``@model_validator`` to refuse a localhost broker in production — that's a -host-level setting, not a background_tasks-module field. - -Process wiring, including ``outbox_mode`` and sweep cadence, requires a restart; -``json_schema_extra`` exposes that requirement in the settings UI. -""" +"""Moved to :mod:`background_tasks.core.settings`; this alias goes in 0.0.46 (module layout).""" from __future__ import annotations -import logging -from typing import Literal +import importlib +import sys -from pydantic import Field, ValidationInfo, field_validator, model_validator -from pydantic_core import PydanticCustomError -from simple_module_core.settings_hydrate import RELATED_FIELDS_CTX +from simple_module_core.deprecation import deprecated_module -from background_tasks.constants import ( - DEFAULT_OUTBOX_LEASE_SECONDS, - DEFAULT_OUTBOX_MAX_ATTEMPTS, - DEFAULT_OUTBOX_SWEEP_INTERVAL_SECONDS, - DEFAULT_RETENTION_DAYS, - ENV_PREFIX, - max_outbox_handler_timeout, +deprecated_module( + "background_tasks.settings", + remove_in="0.0.46", + use="background_tasks.core.settings", ) -from background_tasks.settings_celery import _CELERY_RESTART, CelerySettingsFields -from background_tasks.settings_env_checks import ( - forbid_localhost_broker_in_production, - warn_on_legacy_redis_vars, -) -from background_tasks.tracking import TrackingMode - -__all__ = ["DEFAULT_RETENTION_DAYS", "ENV_PREFIX", "BackgroundTasksSettings"] - -logger = logging.getLogger(__name__) - - -class BackgroundTasksSettings(CelerySettingsFields): - """Configuration for the Celery + Redis task runner.""" - - execution_tracking: TrackingMode = Field( - default=TrackingMode.FULL, json_schema_extra=_CELERY_RESTART - ) - track_publish: bool = Field(default=True, json_schema_extra=_CELERY_RESTART) - - @classmethod - def settings_customise_sources( - cls, settings_cls, init_settings, env_settings, dotenv_settings, file_secret_settings - ): - # Tracking controls are DB-backed; preserve env wiring for older fields. - def db_only(source): - def read(): - return { - k: v - for k, v in source().items() - if k not in {"execution_tracking", "track_publish"} - } - - return read - - return ( - init_settings, - db_only(env_settings), - db_only(dotenv_settings), - db_only(file_secret_settings), - ) - - # ── Durable event outbox (GH #459) ────────────────────── - outbox_mode: Literal["celery", "inprocess"] = Field( - default="celery", json_schema_extra=_CELERY_RESTART - ) - outbox_max_attempts: int = Field( - default=DEFAULT_OUTBOX_MAX_ATTEMPTS, - ge=1, - le=100, - description="Delivery attempts before a durable event is marked dead (1-100).", - ) - outbox_sweep_interval_seconds: int = Field( - default=DEFAULT_OUTBOX_SWEEP_INTERVAL_SECONDS, - ge=1, - json_schema_extra=_CELERY_RESTART, - ) - outbox_lease_seconds: int = Field( - default=DEFAULT_OUTBOX_LEASE_SECONDS, - ge=10, - le=86400, - description="Seconds a claimed delivery may run before another worker reclaims it.", - ) - # Empty: the lease minus a safety margin (10 %, at least 5 s), the most allowed. - outbox_handler_timeout_seconds: int | None = Field( - default=None, ge=1, description="Seconds a durable handler may run per attempt." - ) - - @field_validator("task_time_limit") - @classmethod - def _soft_limit_below_hard(cls, hard: int | None, info: ValidationInfo) -> int | None: - """A soft limit at or above the hard one can never fire (#548). - - Attached to the later-declared hard limit so the error carries that - field's ``loc`` and the settings form shows it under the field. It names - the soft limit as related so boot hydration can drop a stored soft limit - when the hard one is pinned by env. - """ - soft = info.data.get("task_soft_time_limit") - if hard is not None and soft is not None and soft >= hard: - raise PydanticCustomError( - "time_limit_order", - "task_time_limit ({hard}) must be greater than task_soft_time_limit ({soft}); " - "otherwise the soft limit can never fire", - {"hard": hard, "soft": soft, RELATED_FIELDS_CTX: ["task_soft_time_limit"]}, - ) - return hard - - @field_validator("outbox_handler_timeout_seconds") - @classmethod - def _handler_timeout_fits_the_lease( - cls, timeout: int | None, info: ValidationInfo - ) -> int | None: - """A handler outliving its lease would be reclaimed and run twice at once. - - A field validator (not a model one) so the error carries the field's - ``loc`` and the settings form shows it under that field. The lease is - declared first, so it is already in ``info.data`` unless it was invalid - itself, in which case that error is the one to fix first. - """ - lease = info.data.get("outbox_lease_seconds") - if timeout is None or lease is None: - return timeout - ceiling = max_outbox_handler_timeout(lease) - if timeout > ceiling: - raise PydanticCustomError( - "handler_timeout_exceeds_lease", - "outbox_handler_timeout_seconds ({timeout}) must leave a safety margin " - "inside outbox_lease_seconds ({lease}): at most " - "{ceiling}. Raise the lease or lower the timeout.", - { - "timeout": timeout, - "lease": lease, - "ceiling": f"{ceiling:g}", - RELATED_FIELDS_CTX: ["outbox_lease_seconds"], - }, - ) - return timeout - - @model_validator(mode="after") - def _check_broker_env(self) -> BackgroundTasksSettings: - """Deprecation warnings and the production localhost-broker refusal.""" - warn_on_legacy_redis_vars(self.broker_url, self.result_backend) - forbid_localhost_broker_in_production(self.broker_url, self.result_backend) - return self +sys.modules[__name__] = importlib.import_module("background_tasks.core.settings") diff --git a/modules/background_tasks/background_tasks/sync_db.py b/modules/background_tasks/background_tasks/sync_db.py index a3e8b3fd9..28f577bf9 100644 --- a/modules/background_tasks/background_tasks/sync_db.py +++ b/modules/background_tasks/background_tasks/sync_db.py @@ -1,246 +1,15 @@ -"""Sync SQLAlchemy session factory for Celery signal handlers. - -Celery signals are called synchronously from inside the worker / web-process -hot path. Building an event-loop just to await a query is both ugly and -deadlock-prone (web-process signals can fire from inside an already-running -loop). We instead maintain a second, sync engine pointed at the same DB URL -and borrow a short-lived session from it whenever a signal fires. - -The engine is lazily built on first use and process-global. It's cheap — -SQLAlchemy's connection pool is per-process so the signal path reuses -connections after the first signal. -""" +"""Moved to :mod:`background_tasks.core.sync_db`; this alias goes in 0.0.46 (module layout).""" from __future__ import annotations -import logging -import os -from collections.abc import Generator -from contextlib import contextmanager - -from celery.signals import worker_process_init -from simple_module_db import DEFAULT_TENANT_ID -from simple_module_db.audit_sinks import discover_audit_callback -from simple_module_db.engine_options import connect_args -from simple_module_db.listeners import attach_session_listeners -from simple_module_db.provider import DatabaseProvider, detect_provider -from simple_module_db.query_filter import EngineTenancy, bind_engine_policy -from sqlalchemy import create_engine -from sqlalchemy.engine import Engine -from sqlalchemy.orm import Session, sessionmaker -from sqlmodel import SQLModel - -logger = logging.getLogger(__name__) - -_engine: Engine | None = None -# PID that built ``_engine``. A prefork Celery child inherits the parent's -# engine (and its sockets) across fork(); using them would interleave two -# processes on one Postgres connection. -_engine_pid: int | None = None -_session_factory: sessionmaker[Session] | None = None -_url_override: str | None = None -_database_options: dict[str, int] = {} -_tenant_strict: bool = False -_default_tenant: str | None = None - - -class WorkerTenancy(SQLModel, frozen=True): - """Snapshot of the host tenancy configured for this worker (#685).""" - - multi_tenant: bool - default_tenant: str | None - - -def worker_tenancy() -> WorkerTenancy | None: - """Return configured tenancy, or ``None`` when no database URL is pinned.""" - if _url_override is None: - return None - return WorkerTenancy(multi_tenant=_tenant_strict, default_tenant=_default_tenant) - - -class WorkerSession(Session): - """This engine's own session class, so the entity listeners attached to it - never double up with a ``DatabaseState``'s (each state has its own - ``Session`` subclass; listeners on the base class would fire for both).""" - - -def _sync_url(async_url: str) -> str: - """Convert an async SQLAlchemy URL to its sync driver equivalent. - - Mirrors ``host/migrations/env.py`` so signals use the same URL shape - Alembic does. - """ - return async_url.replace("+aiosqlite", "").replace("+asyncpg", "+psycopg2") - - -def set_database_url( - url: str | None, *, tenant_strict: bool = False, default_tenant: str | None = None -) -> None: - """Pin the URL used to build the sync engine. - - The web process loads ``.env`` via pydantic-settings, but those values - never land in ``os.environ`` — so reading ``SM_DATABASE_URL`` directly - from the env can silently drop us back to the SQLite default while the - rest of the app uses Postgres. ``BackgroundTasksModule.on_startup`` - calls this with the resolved ``settings.database_url`` so signals use - the same DB the app is on. Pass ``None`` to clear the override (used in - tests + on shutdown). - - ``tenant_strict`` mirrors the host's ``multi_tenant``: task bodies get the - same fail-closed tenant rules as request code. - - ``default_tenant`` mirrors the host's ``default_tenant`` (ignored when - strict): an insert with no tenant bound — an ``all_tenants()`` block in a - task body — is stamped with it, as ``DatabaseState.default_tenant_id`` does - in the web process, instead of landing in ``DEFAULT_TENANT_ID`` where the - install's own scoped reads never see it. - """ - global _url_override, _tenant_strict, _default_tenant - default_tenant = None if tenant_strict else (default_tenant or None) - if (_url_override, _tenant_strict, _default_tenant) == (url, tenant_strict, default_tenant): - return - _url_override = url - _tenant_strict = tenant_strict - _default_tenant = default_tenant - _drop_engine() - - -def _drop_engine() -> None: - """Dispose the cached engine, never touching sockets inherited via fork.""" - global _engine, _session_factory, _engine_pid - if _engine is not None: - # In a forked child the pooled sockets belong to the parent: close=False - # forgets them without sending a protocol-level close down the shared fd. - _engine.dispose(close=_engine_pid == os.getpid()) - _engine = None - _session_factory = None - _engine_pid = None - - -def _resolve_url() -> str: - if _url_override is not None: - return _url_override - return os.environ.get("SM_DATABASE_URL", "sqlite:///./app.db") - - -def async_database_url() -> str: - """The same database as :func:`sync_session`, spelled for an async driver. - - For the outbox dispatcher, whose durable handlers take an ``AsyncSession``. - """ - url = _resolve_url() - if "+aiosqlite" in url or "+asyncpg" in url: - return url - if url.startswith("sqlite://"): - return url.replace("sqlite://", "sqlite+aiosqlite://", 1) - for prefix in ("postgresql+psycopg2://", "postgresql+psycopg://", "postgresql://"): - if url.startswith(prefix): - return url.replace(prefix, "postgresql+asyncpg://", 1) - return url - - -def engine_tenancy() -> EngineTenancy: - """The policy worker engines run with (mirrors the host's). - - Includes the installed audit sink, so a task's writes are audited exactly - as a request's are when ``audit_log`` is installed (#498). It comes from - the ``simple_module.audit_sinks`` entry point, not an import: this module - does not depend on ``audit_log``. - """ - return EngineTenancy( - tenant_strict=_tenant_strict, - default_tenant_id=_default_tenant or DEFAULT_TENANT_ID, - audit_callback=discover_audit_callback(), - ) - - -def configure_database(options: dict[str, int]) -> None: - """Pin resolved settings before building either worker engine.""" - global _database_options - if options != _database_options: - _database_options = dict(options) - _drop_engine() - - -def database_options() -> dict[str, int]: - """A copy for the outbox engine and its cache key.""" - return dict(_database_options) - - -def _build_engine() -> Engine: - sync_url = _sync_url(_resolve_url()) - # Small pool — signals fire sequentially per worker process. - options = database_options() - args = connect_args( - sync_url, - connect_timeout=options.pop("connect_timeout", 10), - statement_timeout_ms=options.pop("statement_timeout_ms", 0), - idle_in_tx_timeout_ms=options.pop("idle_in_tx_timeout_ms", 0), - ) - if detect_provider(sync_url) == DatabaseProvider.SQLITE: - options = {} - else: - options = {"pool_size": 2, "max_overflow": 3, "pool_recycle": 1800} | options - engine = create_engine(sync_url, pool_pre_ping=True, connect_args=args, **options) - # A worker never runs create_app, so without this its sessions would have - # no tenant filter at all: the tenant restored around a task body would - # scope nothing (#371). - attach_session_listeners(WorkerSession) - bind_engine_policy(engine, engine_tenancy()) - return engine - - -def get_sync_session_factory() -> sessionmaker[Session]: - """Return the process-global sync session factory, building it once.""" - global _engine, _session_factory, _engine_pid - if _session_factory is not None and _engine_pid != os.getpid(): - _drop_engine() - if _session_factory is None: - _engine = _build_engine() - _engine_pid = os.getpid() - _session_factory = sessionmaker(bind=_engine, class_=WorkerSession, expire_on_commit=False) - return _session_factory - - -def dispose_sync_engine() -> None: - """Release pooled connections and drop the cached engine. - - Called from :meth:`BackgroundTasksModule.on_shutdown` so lifespan - restarts within one process (test runners, uvicorn dev reload) don't - accumulate engines against the old DB URL. - """ - global _url_override, _tenant_strict, _default_tenant - _drop_engine() - _url_override = None - configure_database({}) - _tenant_strict = False - _default_tenant = None - - -@worker_process_init.connect -def _dispose_inherited_engine(**_: object) -> None: - """Forget the parent's engine in a fresh prefork child (rebuilt lazily).""" - _drop_engine() - - -@worker_process_init.connect -def _init_sentry(**_: object) -> None: - """Start Sentry in each prefork child (no-op without ``SM_SENTRY_DSN``).""" - from simple_module_hosting.sentry import PROCESS_WORKER, init_sentry - - init_sentry(PROCESS_WORKER) +import importlib +import sys +from simple_module_core.deprecation import deprecated_module -@contextmanager -def sync_session() -> Generator[Session]: - """Open a short-lived sync session; commit on success, rollback on error.""" - factory = get_sync_session_factory() - session = factory() - try: - yield session - session.commit() - except Exception: - session.rollback() - raise - finally: - session.close() +deprecated_module( + "background_tasks.sync_db", + remove_in="0.0.46", + use="background_tasks.core.sync_db", +) +sys.modules[__name__] = importlib.import_module("background_tasks.core.sync_db") diff --git a/modules/background_tasks/background_tasks/task_routing.py b/modules/background_tasks/background_tasks/task_routing.py index 4d2ddfed3..4c8ef0a20 100644 --- a/modules/background_tasks/background_tasks/task_routing.py +++ b/modules/background_tasks/background_tasks/task_routing.py @@ -1,158 +1,17 @@ -"""Per-module task routing, queue membership and package selection. - -A module may ship a light ``/task_config.py`` exporting any of: - -- ``TASK_ROUTES``: Celery ``task_routes`` entries (glob patterns allowed; a - value is a queue name or a full route dict). -- ``QUEUES``: the queues the module's tasks run on. Defaults to the install's - ``task_default_queue``. -- ``BEAT_SCHEDULE``: periodic entries, merged over ``tasks.BEAT_SCHEDULE``. - -It is read *without* importing ``tasks.py``, so a worker started with a queue -subset can decide which modules to load without importing the heavy ones. -Keep it free of heavy imports (and of celery). +"""Moved to :mod:`background_tasks.worker.task_routing`; +this alias goes in 0.0.46 (module layout). """ from __future__ import annotations import importlib -import logging -from dataclasses import dataclass, field -from typing import Any - -from background_tasks.config_guards import ( - WorkerScope, - sanitize_task_routes, - worker_scope_from_env, -) -from background_tasks.settings import BackgroundTasksSettings - -logger = logging.getLogger(__name__) - -OVERRIDE_SOURCE = "override" - - -@dataclass(frozen=True) -class ModuleTaskConfig: - package: str - routes: dict[str, Any] = field(default_factory=dict) - queues: frozenset[str] | None = None - beat_schedule: dict[str, Any] = field(default_factory=dict) - - -def load_task_config(package: str) -> ModuleTaskConfig | None: - """Import ``.task_config``; ``None`` when the module ships none.""" - try: - mod = importlib.import_module(f"{package}.task_config") - except ModuleNotFoundError as exc: - # Only "no task_config.py" is absence; a task_config whose own import - # fails must surface. - if exc.name in (package, f"{package}.task_config"): - return None - raise - routes = getattr(mod, "TASK_ROUTES", None) - queues = getattr(mod, "QUEUES", None) - beat = getattr(mod, "BEAT_SCHEDULE", None) - return ModuleTaskConfig( - package=package, - routes=dict(routes) if isinstance(routes, dict) else {}, - queues=frozenset(queues) if isinstance(queues, (set, frozenset, list, tuple)) else None, - beat_schedule=dict(beat) if isinstance(beat, dict) else {}, - ) +import sys +from simple_module_core.deprecation import deprecated_module -def load_task_configs(packages: list[str]) -> dict[str, ModuleTaskConfig]: - configs: dict[str, ModuleTaskConfig] = {} - for pkg in packages: - cfg = load_task_config(pkg) - if cfg is not None: - configs[pkg] = cfg - return configs - - -def route_queue(value: Any) -> str: - """The queue a ``task_routes`` value names ('' when it names none).""" - if isinstance(value, str): - return value - if isinstance(value, dict): - return str(value.get("queue") or "") - return "" - - -def merge_routes( - configs: dict[str, ModuleTaskConfig], override: dict[str, Any] -) -> tuple[Any, list[dict[str, str]]]: - """Merge module routes (sorted package order, first wins) under the override. - - Returns the ``task_routes`` value and a flat ``[{pattern, queue, module}]`` - list describing where each effective entry came from (override first). - - Celery's ``MapRoute`` tries exact names first and then ``*`` patterns in - insertion order, first match wins, so folding both into one dict would let - a module's broader glob placed earlier beat an override glob, and a module's - exact name beat an override glob. With an override the value is therefore a - router *list* ``[override, modules]``, which Celery consults in order. - """ - # Defence for values stored before the settings validator rejected bad - # routes: drop (and log) unsafe entries instead of handing them to Celery. - override = sanitize_task_routes(override) - routes: dict[str, Any] = {} - owner: dict[str, str] = {} - for pkg in sorted(configs): - for pattern, value in configs[pkg].routes.items(): - if pattern in routes: - logger.warning( - "Task route %r from %s clashes with %s; keeping %s's entry", - pattern, - pkg, - owner[pattern], - owner[pattern], - ) - continue - routes[pattern] = value - owner[pattern] = pkg - module_info = [ - {"pattern": p, "queue": route_queue(v), "module": owner[p]} - for p, v in routes.items() - if p not in override - ] - if not override: - return routes, module_info - override_info = [ - {"pattern": p, "queue": route_queue(v), "module": OVERRIDE_SOURCE} - for p, v in override.items() - ] - return [dict(override), routes], override_info + module_info - - -def select_packages( - packages: list[str], - configs: dict[str, ModuleTaskConfig], - settings: BackgroundTasksSettings, - scope: WorkerScope | None = None, -) -> list[str]: - """Packages this process imports ``tasks.py`` for. - - Order: ``exclude_modules`` always loses; then ``include_modules`` always - wins; then, when ``worker_queues`` is set, a package needs a queue overlap; - otherwise every package is loaded. ``scope`` defaults to the process - environment (:func:`worker_scope_from_env`); it is never stored settings. - """ - scope = scope or worker_scope_from_env() - exclude = scope.exclude_modules - include = scope.include_modules - wanted = scope.queues - selected: list[str] = [] - for pkg in packages: - if pkg in exclude: - continue - if pkg in include or not wanted: - selected.append(pkg) - continue - cfg = configs.get(pkg) - queues = cfg.queues if cfg is not None else None - if queues is None: - queues = frozenset({settings.task_default_queue}) - if queues & wanted: - selected.append(pkg) - return selected +deprecated_module( + "background_tasks.task_routing", + remove_in="0.0.46", + use="background_tasks.worker.task_routing", +) +sys.modules[__name__] = importlib.import_module("background_tasks.worker.task_routing") diff --git a/modules/background_tasks/background_tasks/tasks.py b/modules/background_tasks/background_tasks/tasks.py index a8d14a290..25519844a 100644 --- a/modules/background_tasks/background_tasks/tasks.py +++ b/modules/background_tasks/background_tasks/tasks.py @@ -24,7 +24,7 @@ from sqlalchemy.orm import aliased from sqlmodel import col -from background_tasks.constants import ( +from background_tasks.core.constants import ( DEMO_ECHO_TASK, INTERNAL_TASK_DISPATCH_OUTBOX, INTERNAL_TASK_PURGE_OLD, @@ -35,12 +35,12 @@ OutboxStatus, TaskStatus, ) -from background_tasks.log_purge import purge_log_records +from background_tasks.core.settings import BackgroundTasksSettings +from background_tasks.core.sync_db import sync_session +from background_tasks.core.tracking import TrackingMode +from background_tasks.logs.purge import purge_log_records from background_tasks.models import OutboxMessage, TaskExecution -from background_tasks.settings import BackgroundTasksSettings -from background_tasks.sync_db import sync_session -from background_tasks.tracking import TrackingMode -from background_tasks.worker_settings import load_runner_settings +from background_tasks.worker.settings import load_runner_settings logger = logging.getLogger(__name__) @@ -186,7 +186,7 @@ def dispatch_outbox(ids: list[str]) -> dict[str, int]: """Deliver the outbox rows a just-committed transaction wrote.""" # Imported here: the dispatcher pulls in the async engine stack, which a # worker only needs once it actually delivers an event. - from background_tasks.outbox_worker import run_dispatch + from background_tasks.outbox.worker import run_dispatch return run_dispatch(list(ids)) @@ -194,6 +194,6 @@ def dispatch_outbox(ids: list[str]) -> dict[str, int]: @shared_task(name=INTERNAL_TASK_SWEEP_OUTBOX, sm_tracking=TrackingMode.FAILURES) def sweep_outbox() -> dict[str, int]: """Deliver every due outbox row and reclaim expired leases (beat, every 30 s).""" - from background_tasks.outbox_worker import run_dispatch + from background_tasks.outbox.worker import run_dispatch return run_dispatch(None) diff --git a/modules/background_tasks/background_tasks/tenant_context.py b/modules/background_tasks/background_tasks/tenant_context.py index 7c06f8e0f..42fa8feb7 100644 --- a/modules/background_tasks/background_tasks/tenant_context.py +++ b/modules/background_tasks/background_tasks/tenant_context.py @@ -1,220 +1,17 @@ -"""Carry the enqueuing request's tenant into the Celery task that runs later. - -A task body runs in a worker with no request, so ``current_tenant_id`` is -unset there — under strict isolation any tenant-scoped query would raise, and -without it the query would read every tenant. The tenant is stamped onto the -message headers at publish time and restored around the task body. - -Platform tasks that legitimately span tenants (sweeps, purges) wrap their -work in ``simple_module_db.all_tenants()`` instead. +"""Moved to :mod:`background_tasks.executions.tenant_context`; +this alias goes in 0.0.46 (module layout). """ from __future__ import annotations -import logging -from collections.abc import Generator -from contextlib import contextmanager -from contextvars import ContextVar, Token -from typing import Any - -from simple_module_db import TenantIsolationError, current_tenant_id, is_valid_tenant_id -from simple_module_db.listeners import current_user_id +import importlib +import sys -from background_tasks.constants import OWNER_ID_MAX_LENGTH +from simple_module_core.deprecation import deprecated_module -TENANT_HEADER = "sm_tenant_id" -OWNER_HEADER = "sm_owner_id" - -_log = logging.getLogger(__name__) -_tokens: ContextVar[tuple[tuple[str, Token[str | None], Token[str | None]], ...]] = ContextVar( - "sm_task_binding_tokens", default=() +deprecated_module( + "background_tasks.tenant_context", + remove_in="0.0.46", + use="background_tasks.executions.tenant_context", ) -# The owner of the task body running on this context, so tasks it enqueues -# carry the same owner. Kept apart from ``current_user_id`` on purpose: that -# one also stamps ``created_by`` on audited rows. -_task_owner: ContextVar[str | None] = ContextVar("sm_task_owner", default=None) -# Single-tenant hosts (``default_tenant``): tasks with no tenant on the message -# — beat jobs, tasks enqueued outside a request — run as that tenant. -_default_tenant: str | None = None - - -def set_default_tenant(tenant_id: str | None) -> None: - global _default_tenant - _default_tenant = tenant_id or None - - -def default_tenant() -> str | None: - """The tenant platform work runs as on a single-tenant host, else ``None``.""" - return _default_tenant - - -def stamp_tenant(headers: dict[str, Any] | None) -> None: - """Record the current tenant on an outgoing message (publish side). - - An explicit ``headers={"sm_tenant_id": ...}`` passed to ``send_task`` is - kept only when no tenant is bound (platform code enqueueing work for a - tenant) or it names the bound tenant: request code must not be able to - schedule work as another tenant. - """ - if headers is None: - return - tenant_id = current_tenant_id.get() - explicit = headers.get(TENANT_HEADER) - if explicit: - if not is_valid_tenant_id(explicit): - raise TenantIsolationError(f"Invalid tenant id on task headers: {explicit!r}") - if tenant_id is not None and explicit != tenant_id: - raise TenantIsolationError( - f"Cannot enqueue a task for tenant '{explicit}' in context of tenant '{tenant_id}'" - ) - return - if tenant_id is not None: - headers[TENANT_HEADER] = tenant_id - - -def stamp_owner(headers: dict[str, Any] | None) -> None: - """Record the user whose request enqueues this message (publish side). - - A task that enqueues a task has no ``current_user_id`` in its worker, so - :func:`restore_tenant` binds the incoming message's owner for the body and - this stamps it onward; :func:`publish_as_owner` does the same for a retry, - which must keep the original owner rather than adopt the operator. Failing - both, the bound user is the owner. A header already on the message is - ignored when a user is bound (request code must not name another owner). - """ - if headers is None: - return - user_id = _task_owner.get() - if user_id is None: - user_id = current_user_id.get() - if user_id: - headers[OWNER_HEADER] = str(user_id)[:OWNER_ID_MAX_LENGTH] - - -@contextmanager -def publish_as_owner(owner_user_id: str | None) -> Generator[None]: - """Publish tasks as *owner_user_id* (``None``: no owner), whoever is bound.""" - token = _task_owner.set(owner_user_id or "") - try: - yield - finally: - _task_owner.reset(token) - - -def published_owner(headers: dict[str, Any] | None) -> str | None: - """The owner an outgoing message carries (call after :func:`stamp_owner`).""" - value = (headers or {}).get(OWNER_HEADER) - return str(value)[:OWNER_ID_MAX_LENGTH] if value else None - - -def published_tenant(headers: dict[str, Any] | None) -> str | None: - """The tenant an outgoing message is for: its header, else the publisher's. - - Call after :func:`stamp_tenant`. The header may also have been set by - platform code naming a tenant while none is bound. - """ - return (headers or {}).get(TENANT_HEADER) or current_tenant_id.get() - - -def _tenant_of(task: Any) -> str | None: - return _tenant_of_request(getattr(task, "request", None)) - - -def header_of_request(request: Any, name: str) -> Any: - """A message header off a running task's request (``Request`` or publish-side object).""" - value = getattr(request, name, None) - # A worker ``Request`` (revoked signal) keeps headers in a dict instead. - for attr in ("headers", "request_dict"): - if value is None and isinstance(getattr(request, attr, None), dict): - value = getattr(request, attr).get(name) - return value - - -def message_owner(*, task: Any = None, request: Any = None) -> str | None: - """The owner named on a running task's message header, if any.""" - request = request if request is not None else getattr(task, "request", None) - if request is None: - return None - value = header_of_request(request, OWNER_HEADER) - return str(value)[:OWNER_ID_MAX_LENGTH] if value else None - - -def _tenant_of_request(request: Any) -> str | None: - if request is None: - return None - value = header_of_request(request, TENANT_HEADER) - if not value: - return None - if not is_valid_tenant_id(value): - # Never run a task body as a malformed tenant; it runs unscoped and, - # under strict mode, fails closed on its first tenant-scoped query. - _log.warning("Ignoring invalid tenant id on task message: %r", str(value)[:80]) - return None - return str(value) - - -def message_tenant(*, task: Any = None, request: Any = None) -> str | None: - """The tenant named on a running task's message header, if any. - - Signal handlers stamp it onto the ``TaskExecution`` row: the publish signal - may never have written the row (it fires in the publisher's process, which - can have a different DB, or none bound), so the worker-side upsert must - carry the tenant itself. The configured default tenant is deliberately not - returned — a platform message stays a platform row. - """ - return _tenant_of_request(request) if request is not None else _tenant_of(task) - - -def restore_tenant(*, task_id: str | None, task: Any) -> None: - """Enter the message's tenant for the task body (prerun).""" - if not task_id: - return - # Always set both, so a headerless worker task never inherits the previous - # binding. An in-process run (``apply()``, eager mode) is its caller's own - # context, so without headers it keeps the caller's tenant and owner. - eager = bool(getattr(getattr(task, "request", None), "is_eager", False)) - tenant = _tenant_of(task) or (current_tenant_id.get() if eager else _default_tenant) - owner = message_owner(task=task) or (_task_owner.get() if eager else "") # "": no owner - owner_token = _task_owner.set(owner) - tenant_token = current_tenant_id.set(tenant) - _tokens.set((*_tokens.get(), (task_id, tenant_token, owner_token))) - - -def release_tenant(*, task_id: str | None) -> None: - """Leave the binding :func:`restore_tenant` entered for *task_id* (postrun).""" - stack = _tokens.get() - ids = [entry[0] for entry in stack] - if task_id not in ids: - return - depth = len(ids) - 1 - ids[::-1].index(task_id) - if depth != len(stack) - 1: - # A prerun without its postrun: unwind it too rather than strand it. - _log.warning("Unwinding %d unfinished task binding(s)", len(stack) - 1 - depth) - _, tenant_token, owner_token = stack[depth] - _tokens.set(stack[:depth]) - try: - _task_owner.reset(owner_token) - current_tenant_id.reset(tenant_token) - except ValueError: - # Token from another context (exotic pool patching): clear instead, so - # the next task on this worker never inherits a tenant or owner. - _task_owner.set(None) - current_tenant_id.set(None) - - -__all__ = [ - "OWNER_HEADER", - "TENANT_HEADER", - "default_tenant", - "header_of_request", - "message_owner", - "message_tenant", - "publish_as_owner", - "published_owner", - "published_tenant", - "release_tenant", - "restore_tenant", - "set_default_tenant", - "stamp_owner", - "stamp_tenant", -] +sys.modules[__name__] = importlib.import_module("background_tasks.executions.tenant_context") diff --git a/modules/background_tasks/background_tasks/pages/components/DetailFacts.tsx b/modules/background_tasks/background_tasks/ui/components/DetailFacts.tsx similarity index 99% rename from modules/background_tasks/background_tasks/pages/components/DetailFacts.tsx rename to modules/background_tasks/background_tasks/ui/components/DetailFacts.tsx index 9667e91f8..5833a6104 100644 --- a/modules/background_tasks/background_tasks/pages/components/DetailFacts.tsx +++ b/modules/background_tasks/background_tasks/ui/components/DetailFacts.tsx @@ -10,7 +10,7 @@ import { shortenId, type TaskDetail, VIEW_BASE, -} from '../constants'; +} from '../utils/constants'; interface Props { execution: TaskDetail; diff --git a/modules/background_tasks/background_tasks/pages/components/ExecutionRow.tsx b/modules/background_tasks/background_tasks/ui/components/ExecutionRow.tsx similarity index 97% rename from modules/background_tasks/background_tasks/pages/components/ExecutionRow.tsx rename to modules/background_tasks/background_tasks/ui/components/ExecutionRow.tsx index 6e6e98a65..a0f5d1a01 100644 --- a/modules/background_tasks/background_tasks/pages/components/ExecutionRow.tsx +++ b/modules/background_tasks/background_tasks/ui/components/ExecutionRow.tsx @@ -2,8 +2,8 @@ import { Link, router } from '@inertiajs/react'; import { keys, useT } from '@simple-module-py/i18n'; import { TableCell, TableRow } from '@simple-module-py/ui/components/ui/table'; import { useRelativeTime } from '@simple-module-py/ui/hooks/use-relative-time'; -import { formatDuration, RETRYABLE_STATUSES, VIEW_BASE } from '../constants'; -import type { Execution } from '../retry'; +import { formatDuration, RETRYABLE_STATUSES, VIEW_BASE } from '../utils/constants'; +import type { Execution } from '../utils/retry'; import { StatusPill } from './StatusPill'; interface Props { diff --git a/modules/background_tasks/background_tasks/pages/components/ExecutionsTable.tsx b/modules/background_tasks/background_tasks/ui/components/ExecutionsTable.tsx similarity index 98% rename from modules/background_tasks/background_tasks/pages/components/ExecutionsTable.tsx rename to modules/background_tasks/background_tasks/ui/components/ExecutionsTable.tsx index 06346de96..988f903c8 100644 --- a/modules/background_tasks/background_tasks/pages/components/ExecutionsTable.tsx +++ b/modules/background_tasks/background_tasks/ui/components/ExecutionsTable.tsx @@ -9,7 +9,7 @@ import { TableRow, } from '@simple-module-py/ui/components/ui/table'; import type { ReactNode } from 'react'; -import type { Execution } from '../retry'; +import type { Execution } from '../utils/retry'; import { ExecutionRow } from './ExecutionRow'; /** Task, Status, Queue, Tenant, Queued, Duration, Actions. */ diff --git a/modules/background_tasks/background_tasks/pages/components/LogLevelSelect.tsx b/modules/background_tasks/background_tasks/ui/components/LogLevelSelect.tsx similarity index 93% rename from modules/background_tasks/background_tasks/pages/components/LogLevelSelect.tsx rename to modules/background_tasks/background_tasks/ui/components/LogLevelSelect.tsx index 4b4d79de2..63ceb75f8 100644 --- a/modules/background_tasks/background_tasks/pages/components/LogLevelSelect.tsx +++ b/modules/background_tasks/background_tasks/ui/components/LogLevelSelect.tsx @@ -1,6 +1,6 @@ import { keys, useT } from '@simple-module-py/i18n'; import { NativeSelect, NativeSelectOption } from '@simple-module-py/ui/components/ui/native-select'; -import { LOG_LEVEL_ALL, LOG_LEVELS } from '../logs'; +import { LOG_LEVEL_ALL, LOG_LEVELS } from '../utils/logs'; interface Props { value: string; diff --git a/modules/background_tasks/background_tasks/pages/components/LogTable.tsx b/modules/background_tasks/background_tasks/ui/components/LogTable.tsx similarity index 96% rename from modules/background_tasks/background_tasks/pages/components/LogTable.tsx rename to modules/background_tasks/background_tasks/ui/components/LogTable.tsx index f4321bfe5..c8033e666 100644 --- a/modules/background_tasks/background_tasks/pages/components/LogTable.tsx +++ b/modules/background_tasks/background_tasks/ui/components/LogTable.tsx @@ -10,8 +10,8 @@ import { TableRow, } from '@simple-module-py/ui/components/ui/table'; import { cn } from '@simple-module-py/ui/lib/utils'; -import { EM_DASH, formatTs } from '../constants'; -import { type LogItem, levelClass } from '../logs'; +import { EM_DASH, formatTs } from '../utils/constants'; +import { type LogItem, levelClass } from '../utils/logs'; interface Props { logs: LogItem[]; diff --git a/modules/background_tasks/background_tasks/pages/components/LogsPanel.tsx b/modules/background_tasks/background_tasks/ui/components/LogsPanel.tsx similarity index 98% rename from modules/background_tasks/background_tasks/pages/components/LogsPanel.tsx rename to modules/background_tasks/background_tasks/ui/components/LogsPanel.tsx index 586bc457a..3f9409bcf 100644 --- a/modules/background_tasks/background_tasks/pages/components/LogsPanel.tsx +++ b/modules/background_tasks/background_tasks/ui/components/LogsPanel.tsx @@ -1,7 +1,7 @@ import { keys, t } from '@simple-module-py/i18n'; import { useEffect, useState } from 'react'; import { toast } from 'sonner'; -import { fetchExecutionLogs, LOG_LEVEL_ALL, type LogPage } from '../logs'; +import { fetchExecutionLogs, LOG_LEVEL_ALL, type LogPage } from '../utils/logs'; import { LogLevelSelect } from './LogLevelSelect'; import { LogTable } from './LogTable'; diff --git a/modules/background_tasks/background_tasks/pages/components/OutboxTable.tsx b/modules/background_tasks/background_tasks/ui/components/OutboxTable.tsx similarity index 98% rename from modules/background_tasks/background_tasks/pages/components/OutboxTable.tsx rename to modules/background_tasks/background_tasks/ui/components/OutboxTable.tsx index 84c24ea77..e0a4540a7 100644 --- a/modules/background_tasks/background_tasks/pages/components/OutboxTable.tsx +++ b/modules/background_tasks/background_tasks/ui/components/OutboxTable.tsx @@ -11,14 +11,14 @@ import { TableRow, } from '@simple-module-py/ui/components/ui/table'; import { cn } from '@simple-module-py/ui/lib/utils'; -import { EM_DASH, formatTs } from '../constants'; +import { EM_DASH, formatTs } from '../utils/constants'; import { OUTBOX_PILL_CLASS, OUTBOX_STATUS, OUTBOX_STATUS_LABEL_KEY, type OutboxMessage, shortName, -} from '../outbox'; +} from '../utils/outbox'; const TH = 'text-[11px] font-semibold uppercase tracking-[0.08em] text-muted-foreground'; const COLUMN_COUNT = 7; diff --git a/modules/background_tasks/background_tasks/pages/components/PayloadCards.tsx b/modules/background_tasks/background_tasks/ui/components/PayloadCards.tsx similarity index 96% rename from modules/background_tasks/background_tasks/pages/components/PayloadCards.tsx rename to modules/background_tasks/background_tasks/ui/components/PayloadCards.tsx index 2e9cefc05..14620d491 100644 --- a/modules/background_tasks/background_tasks/pages/components/PayloadCards.tsx +++ b/modules/background_tasks/background_tasks/ui/components/PayloadCards.tsx @@ -1,7 +1,7 @@ import { keys, useT } from '@simple-module-py/i18n'; import { Card } from '@simple-module-py/ui/components/ui/card'; import type { ReactNode } from 'react'; -import { formatPayload, type TaskDetail } from '../constants'; +import { formatPayload, type TaskDetail } from '../utils/constants'; interface Props { execution: TaskDetail; diff --git a/modules/background_tasks/background_tasks/pages/components/ProgressCard.tsx b/modules/background_tasks/background_tasks/ui/components/ProgressCard.tsx similarity index 96% rename from modules/background_tasks/background_tasks/pages/components/ProgressCard.tsx rename to modules/background_tasks/background_tasks/ui/components/ProgressCard.tsx index f525bac00..3b38cb62b 100644 --- a/modules/background_tasks/background_tasks/pages/components/ProgressCard.tsx +++ b/modules/background_tasks/background_tasks/ui/components/ProgressCard.tsx @@ -1,7 +1,7 @@ import { keys, useT } from '@simple-module-py/i18n'; import { Card } from '@simple-module-py/ui/components/ui/card'; import { Progress } from '@simple-module-py/ui/components/ui/progress'; -import type { TaskDetail } from '../constants'; +import type { TaskDetail } from '../utils/constants'; interface Props { execution: Pick< diff --git a/modules/background_tasks/background_tasks/pages/components/RetryAllDialog.tsx b/modules/background_tasks/background_tasks/ui/components/RetryAllDialog.tsx similarity index 97% rename from modules/background_tasks/background_tasks/pages/components/RetryAllDialog.tsx rename to modules/background_tasks/background_tasks/ui/components/RetryAllDialog.tsx index 476faf7e1..063e4eca7 100644 --- a/modules/background_tasks/background_tasks/pages/components/RetryAllDialog.tsx +++ b/modules/background_tasks/background_tasks/ui/components/RetryAllDialog.tsx @@ -1,7 +1,7 @@ import { keys, useT } from '@simple-module-py/i18n'; import { ConfirmActionDialog } from '@simple-module-py/ui/components/ConfirmActionDialog'; import { RefreshCcw } from 'lucide-react'; -import { QUEUE_ALL, SEGMENT_LABEL_KEY, STATUS_ALL } from '../constants'; +import { QUEUE_ALL, SEGMENT_LABEL_KEY, STATUS_ALL } from '../utils/constants'; interface Props { open: boolean; diff --git a/modules/background_tasks/background_tasks/pages/components/RetryConfirmDialog.tsx b/modules/background_tasks/background_tasks/ui/components/RetryConfirmDialog.tsx similarity index 98% rename from modules/background_tasks/background_tasks/pages/components/RetryConfirmDialog.tsx rename to modules/background_tasks/background_tasks/ui/components/RetryConfirmDialog.tsx index 4234311e6..c1362fa53 100644 --- a/modules/background_tasks/background_tasks/pages/components/RetryConfirmDialog.tsx +++ b/modules/background_tasks/background_tasks/ui/components/RetryConfirmDialog.tsx @@ -2,7 +2,7 @@ import { keys, useT } from '@simple-module-py/i18n'; import { ConfirmActionDialog } from '@simple-module-py/ui/components/ConfirmActionDialog'; import { RefreshCcw } from 'lucide-react'; import { useMemo } from 'react'; -import { formatCompactPayload } from '../constants'; +import { formatCompactPayload } from '../utils/constants'; /** Everything the dialog needs about the row it is about to re-enqueue. */ export interface RetryTarget { diff --git a/modules/background_tasks/background_tasks/pages/components/RoutesTable.tsx b/modules/background_tasks/background_tasks/ui/components/RoutesTable.tsx similarity index 96% rename from modules/background_tasks/background_tasks/pages/components/RoutesTable.tsx rename to modules/background_tasks/background_tasks/ui/components/RoutesTable.tsx index 257fbd2ce..5b0c01d0f 100644 --- a/modules/background_tasks/background_tasks/pages/components/RoutesTable.tsx +++ b/modules/background_tasks/background_tasks/ui/components/RoutesTable.tsx @@ -7,7 +7,7 @@ import { TableHeader, TableRow, } from '@simple-module-py/ui/components/ui/table'; -import { ROUTE_OVERRIDE_SOURCE, type TaskRoute } from '../constants'; +import { ROUTE_OVERRIDE_SOURCE, type TaskRoute } from '../utils/constants'; /** * The effective task routes: which queue each pattern is sent to and who set diff --git a/modules/background_tasks/background_tasks/pages/components/StatusPill.tsx b/modules/background_tasks/background_tasks/ui/components/StatusPill.tsx similarity index 96% rename from modules/background_tasks/background_tasks/pages/components/StatusPill.tsx rename to modules/background_tasks/background_tasks/ui/components/StatusPill.tsx index e0c3555f5..8d6b6519c 100644 --- a/modules/background_tasks/background_tasks/pages/components/StatusPill.tsx +++ b/modules/background_tasks/background_tasks/ui/components/StatusPill.tsx @@ -1,6 +1,6 @@ import { useT } from '@simple-module-py/i18n'; import { cn } from '@simple-module-py/ui/lib/utils'; -import { STATUS_LABEL_KEY, STATUS_PILL_CLASS, type TaskStatus } from '../constants'; +import { STATUS_LABEL_KEY, STATUS_PILL_CLASS, type TaskStatus } from '../utils/constants'; interface Props { status: TaskStatus; diff --git a/modules/background_tasks/background_tasks/pages/components/StatusStrip.tsx b/modules/background_tasks/background_tasks/ui/components/StatusStrip.tsx similarity index 98% rename from modules/background_tasks/background_tasks/pages/components/StatusStrip.tsx rename to modules/background_tasks/background_tasks/ui/components/StatusStrip.tsx index e3d711bb5..8efcaacee 100644 --- a/modules/background_tasks/background_tasks/pages/components/StatusStrip.tsx +++ b/modules/background_tasks/background_tasks/ui/components/StatusStrip.tsx @@ -1,7 +1,7 @@ import { keys, useT } from '@simple-module-py/i18n'; import { StatCard } from '@simple-module-py/ui/components/StatCard'; import { cn } from '@simple-module-py/ui/lib/utils'; -import { TASK_STATUS, type TaskStatus } from '../constants'; +import { TASK_STATUS, type TaskStatus } from '../utils/constants'; /** Per-status totals, plus the one windowed count the strip shows. */ export type StatusCounts = Partial> & { success_24h?: number }; diff --git a/modules/background_tasks/background_tasks/pages/components/TaskFilters.tsx b/modules/background_tasks/background_tasks/ui/components/TaskFilters.tsx similarity index 99% rename from modules/background_tasks/background_tasks/pages/components/TaskFilters.tsx rename to modules/background_tasks/background_tasks/ui/components/TaskFilters.tsx index 0dc251c79..417c40411 100644 --- a/modules/background_tasks/background_tasks/pages/components/TaskFilters.tsx +++ b/modules/background_tasks/background_tasks/ui/components/TaskFilters.tsx @@ -15,7 +15,7 @@ import { SEGMENT_STATUSES, STATUS_ALL, TENANT_ALL, -} from '../constants'; +} from '../utils/constants'; interface Props { search: string; diff --git a/modules/background_tasks/background_tasks/pages/components/TasksEmpty.tsx b/modules/background_tasks/background_tasks/ui/components/TasksEmpty.tsx similarity index 97% rename from modules/background_tasks/background_tasks/pages/components/TasksEmpty.tsx rename to modules/background_tasks/background_tasks/ui/components/TasksEmpty.tsx index 4154d3a94..029451cc6 100644 --- a/modules/background_tasks/background_tasks/pages/components/TasksEmpty.tsx +++ b/modules/background_tasks/background_tasks/ui/components/TasksEmpty.tsx @@ -5,8 +5,8 @@ import { TableEmptyRow } from '@simple-module-py/ui/components/TableEmptyRow'; import { Button } from '@simple-module-py/ui/components/ui/button'; import type { TFunction } from 'i18next'; import { Activity, type LucideIcon, PlugZap, ServerCog, ServerOff } from 'lucide-react'; -import { VIEW_BASE } from '../constants'; -import { diagnoseWorkerHealth } from '../worker-health'; +import { VIEW_BASE } from '../utils/constants'; +import { diagnoseWorkerHealth } from '../utils/worker-health'; export interface WorkerPresence { broker_reachable: boolean; diff --git a/modules/background_tasks/background_tasks/pages/components/TracebackCard.tsx b/modules/background_tasks/background_tasks/ui/components/TracebackCard.tsx similarity index 100% rename from modules/background_tasks/background_tasks/pages/components/TracebackCard.tsx rename to modules/background_tasks/background_tasks/ui/components/TracebackCard.tsx diff --git a/modules/background_tasks/background_tasks/pages/components/WorkerCard.tsx b/modules/background_tasks/background_tasks/ui/components/WorkerCard.tsx similarity index 99% rename from modules/background_tasks/background_tasks/pages/components/WorkerCard.tsx rename to modules/background_tasks/background_tasks/ui/components/WorkerCard.tsx index d650f6834..5c8b4434d 100644 --- a/modules/background_tasks/background_tasks/pages/components/WorkerCard.tsx +++ b/modules/background_tasks/background_tasks/ui/components/WorkerCard.tsx @@ -3,7 +3,7 @@ import { Card } from '@simple-module-py/ui/components/ui/card'; import { Skeleton } from '@simple-module-py/ui/components/ui/skeleton'; import { cn } from '@simple-module-py/ui/lib/utils'; import type { ReactNode } from 'react'; -import { EM_DASH, formatSoftware, formatUptime, type WorkerInfo } from '../constants'; +import { EM_DASH, formatSoftware, formatUptime, type WorkerInfo } from '../utils/constants'; function Stat({ label, children }: { label: string; children: ReactNode }) { return ( diff --git a/modules/background_tasks/background_tasks/pages/components/WorkerEmptyStates.tsx b/modules/background_tasks/background_tasks/ui/components/WorkerEmptyStates.tsx similarity index 100% rename from modules/background_tasks/background_tasks/pages/components/WorkerEmptyStates.tsx rename to modules/background_tasks/background_tasks/ui/components/WorkerEmptyStates.tsx diff --git a/modules/background_tasks/background_tasks/pages/components/WorkerHealthBanner.tsx b/modules/background_tasks/background_tasks/ui/components/WorkerHealthBanner.tsx similarity index 96% rename from modules/background_tasks/background_tasks/pages/components/WorkerHealthBanner.tsx rename to modules/background_tasks/background_tasks/ui/components/WorkerHealthBanner.tsx index 859d18f31..4be9acaeb 100644 --- a/modules/background_tasks/background_tasks/pages/components/WorkerHealthBanner.tsx +++ b/modules/background_tasks/background_tasks/ui/components/WorkerHealthBanner.tsx @@ -5,9 +5,9 @@ import { Button } from '@simple-module-py/ui/components/ui/button'; import { apiResult } from '@simple-module-py/ui/lib/api'; import { ServerCrash, ServerOff } from 'lucide-react'; import { useEffect, useState } from 'react'; -import { api } from '../api'; -import { VIEW_BASE, type WorkerSnapshot } from '../constants'; -import { diagnoseWorkerHealth, type WorkerHealthState } from '../worker-health'; +import { api } from '../../pages/api'; +import { VIEW_BASE, type WorkerSnapshot } from '../utils/constants'; +import { diagnoseWorkerHealth, type WorkerHealthState } from '../utils/worker-health'; interface Props { /** Tasks queued or already running — the backlog whose fate is in question. */ diff --git a/modules/background_tasks/background_tasks/pages/constants.ts b/modules/background_tasks/background_tasks/ui/utils/constants.ts similarity index 100% rename from modules/background_tasks/background_tasks/pages/constants.ts rename to modules/background_tasks/background_tasks/ui/utils/constants.ts diff --git a/modules/background_tasks/background_tasks/pages/logs.ts b/modules/background_tasks/background_tasks/ui/utils/logs.ts similarity index 97% rename from modules/background_tasks/background_tasks/pages/logs.ts rename to modules/background_tasks/background_tasks/ui/utils/logs.ts index 804e47431..e6e7aba81 100644 --- a/modules/background_tasks/background_tasks/pages/logs.ts +++ b/modules/background_tasks/background_tasks/ui/utils/logs.ts @@ -1,7 +1,7 @@ // Frontend mirror of the retained-log DTOs (background_tasks/contracts/logs.py). import { apiResult } from '@simple-module-py/ui/lib/api'; -import { api } from './api'; +import { api } from '../../pages/api'; export interface LogItem { id: string; diff --git a/modules/background_tasks/background_tasks/pages/outbox.ts b/modules/background_tasks/background_tasks/ui/utils/outbox.ts similarity index 98% rename from modules/background_tasks/background_tasks/pages/outbox.ts rename to modules/background_tasks/background_tasks/ui/utils/outbox.ts index b7e138e3f..1bc6a5db0 100644 --- a/modules/background_tasks/background_tasks/pages/outbox.ts +++ b/modules/background_tasks/background_tasks/ui/utils/outbox.ts @@ -3,7 +3,7 @@ import { keys, t } from '@simple-module-py/i18n'; import { apiResult } from '@simple-module-py/ui/lib/api'; import { toast } from 'sonner'; -import { api } from './api'; +import { api } from '../../pages/api'; export const OUTBOX_STATUS = { PENDING: 'pending', diff --git a/modules/background_tasks/background_tasks/pages/retry.ts b/modules/background_tasks/background_tasks/ui/utils/retry.ts similarity index 99% rename from modules/background_tasks/background_tasks/pages/retry.ts rename to modules/background_tasks/background_tasks/ui/utils/retry.ts index 779f0636a..671b08a07 100644 --- a/modules/background_tasks/background_tasks/pages/retry.ts +++ b/modules/background_tasks/background_tasks/ui/utils/retry.ts @@ -5,7 +5,7 @@ import { keys, t } from '@simple-module-py/i18n'; import { apiResult } from '@simple-module-py/ui/lib/api'; import { toast } from 'sonner'; -import { api } from './api'; +import { api } from '../../pages/api'; import { QUEUE_ALL, STATUS_ALL, type TaskStatus, TENANT_ALL } from './constants'; export interface Execution { diff --git a/modules/background_tasks/background_tasks/pages/worker-health.ts b/modules/background_tasks/background_tasks/ui/utils/worker-health.ts similarity index 100% rename from modules/background_tasks/background_tasks/pages/worker-health.ts rename to modules/background_tasks/background_tasks/ui/utils/worker-health.ts diff --git a/modules/background_tasks/background_tasks/web/__init__.py b/modules/background_tasks/background_tasks/web/__init__.py new file mode 100644 index 000000000..097de3aa9 --- /dev/null +++ b/modules/background_tasks/background_tasks/web/__init__.py @@ -0,0 +1 @@ +"""The HTTP side: request deps and the admin, status, log and outbox endpoints and views.""" diff --git a/modules/background_tasks/background_tasks/deps.py b/modules/background_tasks/background_tasks/web/deps.py similarity index 89% rename from modules/background_tasks/background_tasks/deps.py rename to modules/background_tasks/background_tasks/web/deps.py index d1abcacdb..3eb1ab447 100644 --- a/modules/background_tasks/background_tasks/deps.py +++ b/modules/background_tasks/background_tasks/web/deps.py @@ -8,10 +8,10 @@ from simple_module_db import RequestSession, current_tenant_id from simple_module_db.deps import get_db -from background_tasks.log_service import LogScope, LogService -from background_tasks.outbox_service import OutboxScope, OutboxService -from background_tasks.outbox_transport import OutboxTransport -from background_tasks.service import BackgroundTaskService +from background_tasks.executions.service import BackgroundTaskService +from background_tasks.logs.service import LogScope, LogService +from background_tasks.outbox.service import OutboxScope, OutboxService +from background_tasks.outbox.transport import OutboxTransport async def get_background_task_service( diff --git a/modules/background_tasks/background_tasks/endpoints/__init__.py b/modules/background_tasks/background_tasks/web/endpoints/__init__.py similarity index 100% rename from modules/background_tasks/background_tasks/endpoints/__init__.py rename to modules/background_tasks/background_tasks/web/endpoints/__init__.py diff --git a/modules/background_tasks/background_tasks/endpoints/api_admin.py b/modules/background_tasks/background_tasks/web/endpoints/api_admin.py similarity index 94% rename from modules/background_tasks/background_tasks/endpoints/api_admin.py rename to modules/background_tasks/background_tasks/web/endpoints/api_admin.py index 77b036a75..789722fe3 100644 --- a/modules/background_tasks/background_tasks/endpoints/api_admin.py +++ b/modules/background_tasks/background_tasks/web/endpoints/api_admin.py @@ -9,22 +9,22 @@ from simple_module_hosting.pagination import page_params from simple_module_hosting.permissions import RequiresPermission -from background_tasks.constants import ( - ADMIN_ROUTER_PREFIX, - MODULE_NAME, - PERM_MANAGE, - PERM_VIEW, - TaskStatus, -) from background_tasks.contracts.schemas import ( RetryFailedResult, TaskExecutionDetail, TaskExecutionListResponse, WorkerSnapshot, ) -from background_tasks.deps import get_background_task_service -from background_tasks.service import BackgroundTaskService -from background_tasks.workers_state import poll_workers +from background_tasks.core.constants import ( + ADMIN_ROUTER_PREFIX, + MODULE_NAME, + PERM_MANAGE, + PERM_VIEW, + TaskStatus, +) +from background_tasks.executions.service import BackgroundTaskService +from background_tasks.web.deps import get_background_task_service +from background_tasks.worker.fleet_state import poll_workers router = APIRouter( prefix=ADMIN_ROUTER_PREFIX, diff --git a/modules/background_tasks/background_tasks/endpoints/api_status.py b/modules/background_tasks/background_tasks/web/endpoints/api_status.py similarity index 97% rename from modules/background_tasks/background_tasks/endpoints/api_status.py rename to modules/background_tasks/background_tasks/web/endpoints/api_status.py index 0afec8c9d..f67ee90f0 100644 --- a/modules/background_tasks/background_tasks/endpoints/api_status.py +++ b/modules/background_tasks/background_tasks/web/endpoints/api_status.py @@ -23,8 +23,8 @@ from sqlalchemy import select from sqlmodel import col -from background_tasks.constants import MODULE_NAME, PERM_VIEW, STATUS_ROUTER_PREFIX from background_tasks.contracts.schemas import TaskStatusDTO +from background_tasks.core.constants import MODULE_NAME, PERM_VIEW, STATUS_ROUTER_PREFIX from background_tasks.models import TaskExecution, dto_columns, newest_first router = APIRouter(prefix=STATUS_ROUTER_PREFIX, tags=[f"{MODULE_NAME}-status"]) diff --git a/modules/background_tasks/background_tasks/endpoints/logs_api.py b/modules/background_tasks/background_tasks/web/endpoints/logs_api.py similarity index 86% rename from modules/background_tasks/background_tasks/endpoints/logs_api.py rename to modules/background_tasks/background_tasks/web/endpoints/logs_api.py index 993fe4153..3c153cf6b 100644 --- a/modules/background_tasks/background_tasks/endpoints/logs_api.py +++ b/modules/background_tasks/background_tasks/web/endpoints/logs_api.py @@ -9,11 +9,16 @@ from simple_module_hosting.pagination import page_params from simple_module_hosting.permissions import RequiresPermission -from background_tasks.constants import ADMIN_ROUTER_PREFIX, LOG_PER_PAGE, MODULE_NAME, PERM_VIEW from background_tasks.contracts.logs import LogListResponse -from background_tasks.deps import get_background_task_service, get_log_service -from background_tasks.log_service import LogService -from background_tasks.service import BackgroundTaskService +from background_tasks.core.constants import ( + ADMIN_ROUTER_PREFIX, + LOG_PER_PAGE, + MODULE_NAME, + PERM_VIEW, +) +from background_tasks.executions.service import BackgroundTaskService +from background_tasks.logs.service import LogService +from background_tasks.web.deps import get_background_task_service, get_log_service router = APIRouter( prefix=ADMIN_ROUTER_PREFIX, diff --git a/modules/background_tasks/background_tasks/endpoints/logs_views.py b/modules/background_tasks/background_tasks/web/endpoints/logs_views.py similarity index 88% rename from modules/background_tasks/background_tasks/endpoints/logs_views.py rename to modules/background_tasks/background_tasks/web/endpoints/logs_views.py index 2ce35c552..3f1b3620d 100644 --- a/modules/background_tasks/background_tasks/endpoints/logs_views.py +++ b/modules/background_tasks/background_tasks/web/endpoints/logs_views.py @@ -9,9 +9,9 @@ from simple_module_hosting.permissions import RequiresPermission from simple_module_inertia import InertiaResponse -from background_tasks.constants import LOG_LEVEL_NAMES, LOG_PER_PAGE, PERM_VIEW -from background_tasks.deps import get_log_service -from background_tasks.log_service import LogService +from background_tasks.core.constants import LOG_LEVEL_NAMES, LOG_PER_PAGE, PERM_VIEW +from background_tasks.logs.service import LogService +from background_tasks.web.deps import get_log_service router = APIRouter(dependencies=[Depends(RequiresPermission(PERM_VIEW))]) diff --git a/modules/background_tasks/background_tasks/endpoints/outbox_api.py b/modules/background_tasks/background_tasks/web/endpoints/outbox_api.py similarity index 92% rename from modules/background_tasks/background_tasks/endpoints/outbox_api.py rename to modules/background_tasks/background_tasks/web/endpoints/outbox_api.py index 5b95f9b71..030e7f5a5 100644 --- a/modules/background_tasks/background_tasks/endpoints/outbox_api.py +++ b/modules/background_tasks/background_tasks/web/endpoints/outbox_api.py @@ -9,16 +9,16 @@ from simple_module_hosting.pagination import page_params from simple_module_hosting.permissions import RequiresPermission -from background_tasks.constants import ( +from background_tasks.contracts.outbox import OutboxItem, OutboxListResponse +from background_tasks.core.constants import ( ADMIN_ROUTER_PREFIX, MODULE_NAME, PERM_MANAGE, PERM_VIEW, OutboxStatus, ) -from background_tasks.contracts.outbox import OutboxItem, OutboxListResponse -from background_tasks.deps import get_outbox_service -from background_tasks.outbox_service import OutboxService +from background_tasks.outbox.service import OutboxService +from background_tasks.web.deps import get_outbox_service router = APIRouter( prefix=f"{ADMIN_ROUTER_PREFIX}/outbox", diff --git a/modules/background_tasks/background_tasks/endpoints/outbox_views.py b/modules/background_tasks/background_tasks/web/endpoints/outbox_views.py similarity index 89% rename from modules/background_tasks/background_tasks/endpoints/outbox_views.py rename to modules/background_tasks/background_tasks/web/endpoints/outbox_views.py index 360d4a478..dbf4fa1cd 100644 --- a/modules/background_tasks/background_tasks/endpoints/outbox_views.py +++ b/modules/background_tasks/background_tasks/web/endpoints/outbox_views.py @@ -9,9 +9,9 @@ from simple_module_hosting.permissions import RequiresPermission from simple_module_inertia import InertiaResponse -from background_tasks.constants import PERM_VIEW, OutboxStatus -from background_tasks.deps import get_outbox_service -from background_tasks.outbox_service import OutboxService +from background_tasks.core.constants import PERM_VIEW, OutboxStatus +from background_tasks.outbox.service import OutboxService +from background_tasks.web.deps import get_outbox_service router = APIRouter(dependencies=[Depends(RequiresPermission(PERM_VIEW))]) diff --git a/modules/background_tasks/background_tasks/endpoints/views.py b/modules/background_tasks/background_tasks/web/endpoints/views.py similarity index 95% rename from modules/background_tasks/background_tasks/endpoints/views.py rename to modules/background_tasks/background_tasks/web/endpoints/views.py index 6dc7a079c..7ef65f7d9 100644 --- a/modules/background_tasks/background_tasks/endpoints/views.py +++ b/modules/background_tasks/background_tasks/web/endpoints/views.py @@ -11,16 +11,16 @@ from simple_module_hosting.permissions import RequiresPermission from simple_module_inertia import InertiaResponse, defer -from background_tasks.celery_app import describe_routes -from background_tasks.constants import ( +from background_tasks.core.constants import ( PERM_VIEW, PLATFORM_TENANT_FILTER, TaskStatus, ) -from background_tasks.deps import get_background_task_service -from background_tasks.service import BackgroundTaskService -from background_tasks.worker_inspector import redact_broker_url -from background_tasks.workers_state import poll_workers +from background_tasks.executions.service import BackgroundTaskService +from background_tasks.web.deps import get_background_task_service +from background_tasks.worker.celery_app import describe_routes +from background_tasks.worker.fleet_state import poll_workers +from background_tasks.worker.inspector import redact_broker_url router = APIRouter(dependencies=[Depends(RequiresPermission(PERM_VIEW))]) diff --git a/modules/background_tasks/background_tasks/worker/__init__.py b/modules/background_tasks/background_tasks/worker/__init__.py new file mode 100644 index 000000000..cd770c538 --- /dev/null +++ b/modules/background_tasks/background_tasks/worker/__init__.py @@ -0,0 +1,6 @@ +"""The Celery side. + +The app factory and runner, tenant-stamping task bases, routing, heartbeats, +resource references, the signal handlers that keep executions in sync, hydrated +worker settings, log context and fleet inspection. +""" diff --git a/modules/background_tasks/background_tasks/_signal_bus.py b/modules/background_tasks/background_tasks/worker/_signal_bus.py similarity index 100% rename from modules/background_tasks/background_tasks/_signal_bus.py rename to modules/background_tasks/background_tasks/worker/_signal_bus.py diff --git a/modules/background_tasks/background_tasks/_signal_support.py b/modules/background_tasks/background_tasks/worker/_signal_support.py similarity index 95% rename from modules/background_tasks/background_tasks/_signal_support.py rename to modules/background_tasks/background_tasks/worker/_signal_support.py index e7cc70611..0629e49f0 100644 --- a/modules/background_tasks/background_tasks/_signal_support.py +++ b/modules/background_tasks/background_tasks/worker/_signal_support.py @@ -1,4 +1,4 @@ -"""Signal-handler helpers shared across :mod:`background_tasks.signals`. +"""Signal-handler helpers shared across :mod:`background_tasks.worker.signals`. Kept in a separate module so :mod:`.signals` stays under the 300-line cap. These helpers are intentionally side-effect-free except for the DB write in @@ -20,11 +20,11 @@ from sqlalchemy.orm import Session from sqlmodel import col +from background_tasks.core.sync_db import sync_session +from background_tasks.core.tracking import records_signal, tracking_mode +from background_tasks.executions.tenant_context import message_owner from background_tasks.models import TaskExecution -from background_tasks.sync_db import sync_session -from background_tasks.task_resource import message_resource -from background_tasks.tenant_context import message_owner -from background_tasks.tracking import records_signal, tracking_mode +from background_tasks.worker.task_resource import message_resource logger = logging.getLogger(__name__) diff --git a/modules/background_tasks/background_tasks/worker/celery_app.py b/modules/background_tasks/background_tasks/worker/celery_app.py new file mode 100644 index 000000000..af03e1901 --- /dev/null +++ b/modules/background_tasks/background_tasks/worker/celery_app.py @@ -0,0 +1,224 @@ +"""Build and configure the Celery application. + +Invoked from two places: + +- :meth:`BackgroundTasksModule.on_startup` in the web process, so the API + can enqueue tasks via ``app.state.background_tasks.celery.send_task(...)``. +- ``scripts/run_worker.py`` at worker boot, so the worker process uses an + identical config (broker URL, signal handlers, autodiscovered tasks). + +Task discovery uses :py:meth:`celery.Celery.autodiscover_tasks` with the +top-level package name of every installed module (enumerated via the +``simple_module`` entry-point group). Any module that ships a ``tasks.py`` +is picked up automatically — no framework hook, no per-module registration. +""" + +from __future__ import annotations + +import importlib +import logging +from importlib.metadata import entry_points + +from celery import Celery +from celery.schedules import schedule +from simple_module_core.discovery import ENTRY_POINT_GROUP + +from background_tasks.core.config_guards import celery_overrides_from_env +from background_tasks.core.constants import ( + BEAT_PURGE_OLD, + BEAT_SWEEP_OUTBOX, + BEAT_SWEEP_STUCK, + INTERNAL_TASK_PURGE_OLD, + INTERNAL_TASK_SWEEP_OUTBOX, + INTERNAL_TASK_SWEEP_STUCK, + MODULE_NAME, +) +from background_tasks.core.settings import BackgroundTasksSettings +from background_tasks.worker.task_routing import load_task_configs, merge_routes, select_packages +from background_tasks.worker.tenant_celery import TenantCelery + +logger = logging.getLogger(__name__) + + +def _discover_task_packages() -> list[str]: + """Return the top-level package name of every installed simple_module. + + Celery's ``autodiscover_tasks`` imports ``.tasks`` for each name + returned here, so any module that ships a ``tasks.py`` registers its + tasks without a per-module hook. + """ + packages: set[str] = set() + for ep in entry_points(group=ENTRY_POINT_GROUP): + module_path = ep.value.split(":", 1)[0] + top_level = module_path.split(".", 1)[0] + packages.add(top_level) + return sorted(packages) + + +def _collect_module_beat_schedules(packages: list[str]) -> dict: + """Merge every installed module's optional ``tasks.BEAT_SCHEDULE`` mapping. + + A module registers periodic work the same way it registers tasks — by + shipping a ``tasks.py`` — and additionally exporting a module-level + ``BEAT_SCHEDULE`` dict of ``{entry_name: celery beat entry}``. Because + ``build_celery`` runs identically in the web and worker processes, this is + worker-safe and deterministic, unlike runtime registration via + ``add_periodic_task`` / the ``on_after_finalize`` signal (and unlike the + commonly-cited ``on_after_configure``, which never fires here — see GH #199). + + Importing ``.tasks`` is idempotent (autodiscover imports the same + modules), so doing it eagerly here has no extra side effects. + """ + merged: dict = {} + for pkg in packages: + try: + tasks_mod = importlib.import_module(f"{pkg}.tasks") + except ModuleNotFoundError as exc: + # The module simply ships no tasks.py — skip. But a tasks.py that + # *exists* and fails to import (its own dependency is missing) must + # surface loudly, not be mistaken for "no tasks.py" and silently + # drop the module's periodic work. + if exc.name in (pkg, f"{pkg}.tasks"): + continue + raise + schedule_dict = getattr(tasks_mod, "BEAT_SCHEDULE", None) + if not isinstance(schedule_dict, dict): + continue + for name, entry in schedule_dict.items(): + if name in merged: + logger.warning( + "Beat entry %r from %s.tasks overrides an earlier module's entry", name, pkg + ) + merged[name] = entry + return merged + + +# setting name -> Celery conf key (they differ only for the memory cap, which +# Celery names without the unit although it is KiB). +_TUNING_KEYS = { + "worker_max_tasks_per_child": "worker_max_tasks_per_child", + "worker_max_memory_per_child_kb": "worker_max_memory_per_child", + "task_reject_on_worker_lost": "task_reject_on_worker_lost", + "task_soft_time_limit": "task_soft_time_limit", + "task_time_limit": "task_time_limit", + "worker_concurrency": "worker_concurrency", +} + + +def _tuning_conf(settings: BackgroundTasksSettings) -> dict: + """Celery conf for the optional tuning fields; ``None`` keeps Celery's default.""" + conf = { + key: getattr(settings, name) + for name, key in _TUNING_KEYS.items() + if getattr(settings, name) is not None + } + if settings.visibility_timeout_seconds is not None: + conf["broker_transport_options"] = { + "visibility_timeout": settings.visibility_timeout_seconds + } + return conf + + +def describe_routes(settings: BackgroundTasksSettings) -> list[dict[str, str]]: + """Effective ``[{pattern, queue, module}]`` routes, for the Workers admin page.""" + configs = load_task_configs(_discover_task_packages()) + return merge_routes(configs, settings.task_routes_override)[1] + + +def build_celery(settings: BackgroundTasksSettings) -> Celery: + """Construct a Celery app wired to the project's Redis broker.""" + celery = TenantCelery(MODULE_NAME) + + all_packages = _discover_task_packages() + # Routes and beat entries come from the light ``task_config`` of *every* + # module; only the heavy ``tasks.py`` imports are narrowed to this + # process's subset (GH #450). + configs = load_task_configs(all_packages) + packages = select_packages(all_packages, configs, settings) + task_routes, _ = merge_routes(configs, settings.task_routes_override) + + # The internal entries always ship; modules contribute more via their + # own ``tasks.BEAT_SCHEDULE``. ``setdefault`` keeps the built-ins authoritative + # if a module reuses one of their entry names. See GH #199. + beat_schedule = { + BEAT_SWEEP_STUCK: { + "task": INTERNAL_TASK_SWEEP_STUCK, + "schedule": schedule(settings.stuck_sweep_interval_seconds), + }, + BEAT_PURGE_OLD: { + "task": INTERNAL_TASK_PURGE_OLD, + "schedule": schedule(settings.purge_interval_seconds), + }, + # Delivers durable events whose post-commit dispatch never ran (a crash + # between commit and publish) and retries failed ones (GH #459). + BEAT_SWEEP_OUTBOX: { + "task": INTERNAL_TASK_SWEEP_OUTBOX, + "schedule": schedule(settings.outbox_sweep_interval_seconds), + }, + } + builtin_beat = frozenset(beat_schedule) + for name, entry in _collect_module_beat_schedules(packages).items(): + if name in beat_schedule: + logger.warning( + "Module beat entry %r clashes with a built-in entry; the built-in wins", name + ) + continue + beat_schedule[name] = entry + # ``task_config.BEAT_SCHEDULE`` wins over the same entry in ``tasks.py``. + for pkg in sorted(configs): + for name, entry in configs[pkg].beat_schedule.items(): + if name in builtin_beat: + logger.warning( + "task_config beat entry %r clashes with a built-in; built-in wins", name + ) + continue + beat_schedule[name] = entry + + celery.conf.update( + sm_execution_tracking=settings.execution_tracking, + sm_track_publish=settings.track_publish, + broker_url=settings.broker_url, + result_backend=settings.result_backend, + task_default_queue=settings.task_default_queue, + # Run tasks synchronously inside the calling process. Tests + # toggle this on (via ``SM_BG_TASKS_TASK_ALWAYS_EAGER=true``) so + # ``task.delay()`` doesn't reach for a real broker. + task_always_eager=settings.task_always_eager, + task_eager_propagates=settings.task_eager_propagates, + # ``task_track_started`` gives us the ``STARTED`` state so + # ``task_prerun`` can flip our row to ``running``. + task_track_started=True, + # ``task_acks_late`` + ``worker_prefetch_multiplier=1`` gives us + # at-least-once semantics, which matches how the admin UI asks users + # to think about retries — duplicates beat lost work. + task_acks_late=True, + worker_prefetch_multiplier=1, + task_serializer="json", + result_serializer="json", + accept_content=["json"], + timezone="UTC", + enable_utc=True, + broker_connection_retry_on_startup=True, + beat_schedule=beat_schedule, + beat_scheduler="celery.beat:PersistentScheduler", + task_routes=task_routes, + ) + celery.conf.update(_tuning_conf(settings)) + overrides = celery_overrides_from_env() + if overrides: + logger.info("Applying celery_overrides: %s", sorted(overrides)) + celery.conf.update(overrides) + + # Autodiscover across every installed simple_module — a module just ships + # a `tasks.py` and the worker picks it up. + logger.info("Celery autodiscover_tasks across: %s", packages) + celery.autodiscover_tasks(packages, related_name="tasks", force=True) + + # Side-effect import: connects signal handlers to this Celery instance's + # ``celery.signals.*`` globals. Safe to import repeatedly. + from background_tasks.worker import signals # noqa: F401 + from background_tasks.worker.log_context import install_log_filter + + install_log_filter() + + return celery diff --git a/modules/background_tasks/background_tasks/workers_state.py b/modules/background_tasks/background_tasks/worker/fleet_state.py similarity index 94% rename from modules/background_tasks/background_tasks/workers_state.py rename to modules/background_tasks/background_tasks/worker/fleet_state.py index 5ab94cadd..f95bd4d5e 100644 --- a/modules/background_tasks/background_tasks/workers_state.py +++ b/modules/background_tasks/background_tasks/worker/fleet_state.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING from background_tasks.contracts.schemas import WorkerSnapshot -from background_tasks.worker_inspector import WorkerInspector +from background_tasks.worker.inspector import WorkerInspector if TYPE_CHECKING: from fastapi import Request diff --git a/modules/background_tasks/background_tasks/worker_inspector.py b/modules/background_tasks/background_tasks/worker/inspector.py similarity index 100% rename from modules/background_tasks/background_tasks/worker_inspector.py rename to modules/background_tasks/background_tasks/worker/inspector.py diff --git a/modules/background_tasks/background_tasks/log_context.py b/modules/background_tasks/background_tasks/worker/log_context.py similarity index 100% rename from modules/background_tasks/background_tasks/log_context.py rename to modules/background_tasks/background_tasks/worker/log_context.py diff --git a/modules/background_tasks/background_tasks/worker/runner.py b/modules/background_tasks/background_tasks/worker/runner.py new file mode 100644 index 000000000..25788cef5 --- /dev/null +++ b/modules/background_tasks/background_tasks/worker/runner.py @@ -0,0 +1,85 @@ +"""Build the Celery app for a worker or beat process. + +The repo's ``scripts/run_worker.py`` and the one ``smpy new`` scaffolds both +call :func:`build_runner_celery`. They used to carry their own copies of this +bootstrap, and the scaffolded copy fell behind: it built Celery from the +environment alone, so a generated app's worker ignored settings saved in the +admin UI and could read a different ``.env`` from the web process (GH #490). +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import TYPE_CHECKING + +from simple_module_core.dotenv import find_env_file, load_dotenv_into_environ + +if TYPE_CHECKING: + from celery import Celery + +_PROJECT_ROOT = "SM_PROJECT_ROOT" + + +def _load_env(fallback_env_file: Path | None) -> None: + """Load ``.env`` the way the web process resolves it. + + ``find_env_file`` (SM_PROJECT_ROOT, then a bounded walk-up) keeps the worker + and the web process on the same file. ``fallback_env_file`` applies only + when that resolves to no file at all — celery launched from an unrelated + cwd, possibly inside some other checkout, with SM_PROJECT_ROOT unset. It is + adopted by setting SM_PROJECT_ROOT rather than by loading the file directly, + so every later reader (``BootstrapSettings``' sqlite anchor, the host + settings' ``.env`` scan) resolves the same project instead of the cwd. + """ + env_file = find_env_file() + if ( + fallback_env_file is not None + and _PROJECT_ROOT not in os.environ + and not env_file.is_file() + and fallback_env_file.is_file() + ): + os.environ[_PROJECT_ROOT] = str(fallback_env_file.parent) + env_file = find_env_file() + load_dotenv_into_environ(env_file) + + +def build_runner_celery(fallback_env_file: Path | None = None) -> Celery: + """``.env`` → host settings → DB-hydrated ``BackgroundTasksSettings`` → Celery.""" + _load_env(fallback_env_file) + + # Imported after ``.env`` is in the environment, so nothing below can + # resolve a setting before it is. + from simple_module_hosting import merge_host_settings + from simple_module_hosting.leases import configure_process_leases + + from background_tasks.core.db_settings import worker_database_options + from background_tasks.core.settings import BackgroundTasksSettings + from background_tasks.core.sync_db import configure_database, set_database_url + from background_tasks.executions.tenant_context import set_default_tenant + from background_tasks.logs.sink import install_log_sink + from background_tasks.worker.celery_app import build_celery + from background_tasks.worker.settings import load_runner_settings + + # The worker never builds the app, so resolve the host settings the same + # way host/main.py does (env → DB → default): task bodies must hit the same + # database with the same fail-closed tenant rules as request code. + host = merge_host_settings() + default_tenant = None if host.multi_tenant else host.default_tenant or None + set_database_url( + host.database_url, tenant_strict=host.multi_tenant, default_tenant=default_tenant + ) + configure_database(worker_database_options(host, BackgroundTasksSettings())) + set_default_tenant(default_tenant) + # Task bodies share the web process's leases (GH #539): same Redis or table. + configure_process_leases(host.redis_url, database_url=host.database_url) + # Hydrated from the DB like the web process, so an admin-edited setting + # reaches the worker at boot, not only the web app. + settings = load_runner_settings() + # Re-apply the engine options now the worker's DB-stored overrides are known. + configure_database(worker_database_options(host, settings)) + celery = build_celery(settings) + # Task logs: the worker never runs the web lifespan that installs the sink. + # Prefork children rebuild it on ``worker_process_init``. + install_log_sink(settings) + return celery diff --git a/modules/background_tasks/background_tasks/worker/settings.py b/modules/background_tasks/background_tasks/worker/settings.py new file mode 100644 index 000000000..5f1afe1f3 --- /dev/null +++ b/modules/background_tasks/background_tasks/worker/settings.py @@ -0,0 +1,84 @@ +"""Hydrated DB-backed module settings for task code (GH #378). + +A Celery worker never runs the hosting lifespan that hydrates +``app.state..settings``, and ``DbBackedSettings`` ignores the +environment, so a bare ``FileStorageSettings()`` in a task returns pydantic +defaults without complaint. Use :func:`settings_for` instead:: + + cfg = settings_for(FileStorageSettings, "file_storage") + +It reads the SYSTEM-scope overrides through the worker's sync session and +caches the result per process for ``DEFAULT_TTL_SECONDS`` so a hot task does not +query on every call. Admin edits therefore reach a worker within the TTL. +""" + +from __future__ import annotations + +import logging +import threading +import time +from typing import cast + +from pydantic_settings import BaseSettings +from simple_module_db.stored_settings import hydrate_settings_sync + +from background_tasks.core.constants import MODULE_NAME +from background_tasks.core.settings import BackgroundTasksSettings +from background_tasks.core.sync_db import sync_session + +logger = logging.getLogger(__name__) + +DEFAULT_TTL_SECONDS = 30.0 + +_lock = threading.Lock() +_cache: dict[tuple[type, str], tuple[float, BaseSettings]] = {} + + +def settings_for[T: BaseSettings]( + cls: type[T], package: str, *, ttl: float = DEFAULT_TTL_SECONDS +) -> T: + """Return ``cls`` hydrated from DB overrides, cached per process for ``ttl`` seconds. + + ``ttl=0`` always re-reads. + """ + key = (cls, package) + now = time.monotonic() + with _lock: + hit = _cache.get(key) + if hit is not None and ttl > 0 and now - hit[0] < ttl: + # A copy: BaseSettings is mutable, so one task editing its settings + # must not change what the next task in this process sees. + return cast(T, hit[1]).model_copy(deep=True) + with sync_session() as session: + value = hydrate_settings_sync(cls, session, package, lenient=True) + with _lock: + _cache[key] = (now, value) + return value.model_copy(deep=True) + + +def clear_settings_cache() -> None: + """Drop every cached settings object (tests, or after a known write).""" + with _lock: + _cache.clear() + + +def load_runner_settings() -> BackgroundTasksSettings: + """``BackgroundTasksSettings`` for the worker/beat process, DB overrides included. + + ``scripts/run_worker.py`` used to build these from the environment alone, so + an admin's DB-backed edit (a queue, a time limit) + reached the web process but never the worker, and the two disagreed on + config. Hydration is the same call the web lifespan makes (``cls(**db_overrides)``), + so precedence matches the web process: env → DB → default (GH #486). If the DB is + unreachable or not yet migrated the worker still boots on env/defaults, + loudly, rather than refusing to start. + """ + try: + return settings_for(BackgroundTasksSettings, MODULE_NAME, ttl=0) + except Exception: + logger.warning( + "Could not read background_tasks settings from the database; " + "starting from environment/defaults only", + exc_info=True, + ) + return BackgroundTasksSettings() diff --git a/modules/background_tasks/background_tasks/signals.py b/modules/background_tasks/background_tasks/worker/signals.py similarity index 94% rename from modules/background_tasks/background_tasks/signals.py rename to modules/background_tasks/background_tasks/worker/signals.py index b3c637b46..25b9d993c 100644 --- a/modules/background_tasks/background_tasks/signals.py +++ b/modules/background_tasks/background_tasks/worker/signals.py @@ -24,25 +24,12 @@ from celery import signals from simple_module_db import current_tenant_id -from background_tasks._signal_bus import bind_event_bus, publish_from_signal, unbind_event_bus -from background_tasks._signal_support import ( - apply_signal, - coerce_args_kwargs, - jsonable_result, - now_utc, - render_traceback, - task_id_from, - task_name_of, -) -from background_tasks.constants import ( +from background_tasks.contracts.events import TaskFailed +from background_tasks.core.constants import ( DEFAULT_QUEUE, TaskStatus, ) -from background_tasks.contracts.events import TaskFailed -from background_tasks.log_context import signal_task_finished, signal_task_started -from background_tasks.models import TaskExecution -from background_tasks.task_heartbeat import forget_heartbeat -from background_tasks.tenant_context import ( +from background_tasks.executions.tenant_context import ( TENANT_HEADER, message_tenant, published_owner, @@ -51,6 +38,23 @@ restore_tenant, stamp_owner, ) +from background_tasks.models import TaskExecution +from background_tasks.worker._signal_bus import ( + bind_event_bus, + publish_from_signal, + unbind_event_bus, +) +from background_tasks.worker._signal_support import ( + apply_signal, + coerce_args_kwargs, + jsonable_result, + now_utc, + render_traceback, + task_id_from, + task_name_of, +) +from background_tasks.worker.log_context import signal_task_finished, signal_task_started +from background_tasks.worker.task_heartbeat import forget_heartbeat # ── Enqueue ───────────────────────────────────────────────────── diff --git a/modules/background_tasks/background_tasks/task_heartbeat.py b/modules/background_tasks/background_tasks/worker/task_heartbeat.py similarity index 94% rename from modules/background_tasks/background_tasks/task_heartbeat.py rename to modules/background_tasks/background_tasks/worker/task_heartbeat.py index eef08893a..4baedf19c 100644 --- a/modules/background_tasks/background_tasks/task_heartbeat.py +++ b/modules/background_tasks/background_tasks/worker/task_heartbeat.py @@ -26,16 +26,16 @@ from sqlalchemy import update from sqlmodel import col -from background_tasks._signal_support import now_utc -from background_tasks.constants import ( +from background_tasks.core.constants import ( HEARTBEAT_MIN_INTERVAL_SECONDS, PROGRESS_MESSAGE_MAX_LENGTH, PROGRESS_PHASE_MAX_LENGTH, TaskStatus, ) +from background_tasks.core.sync_db import sync_session +from background_tasks.core.tracking import TrackingMode, tracking_mode from background_tasks.models import TaskExecution -from background_tasks.sync_db import sync_session -from background_tasks.tracking import TrackingMode, tracking_mode +from background_tasks.worker._signal_support import now_utc logger = logging.getLogger(__name__) @@ -86,7 +86,7 @@ def heartbeat( """Tell the stuck sweep this task is still alive, optionally with progress. Refreshes ``heartbeat_at`` on the current task's execution row, at most - once per :data:`~background_tasks.constants.HEARTBEAT_MIN_INTERVAL_SECONDS` + once per :data:`~background_tasks.core.constants.HEARTBEAT_MIN_INTERVAL_SECONDS` per task unless *force* is set. Call it more often than the task's stuck threshold — every loop iteration is fine. diff --git a/modules/background_tasks/background_tasks/task_resource.py b/modules/background_tasks/background_tasks/worker/task_resource.py similarity index 94% rename from modules/background_tasks/background_tasks/task_resource.py rename to modules/background_tasks/background_tasks/worker/task_resource.py index 7ebad6ec8..f940a3fb4 100644 --- a/modules/background_tasks/background_tasks/task_resource.py +++ b/modules/background_tasks/background_tasks/worker/task_resource.py @@ -5,12 +5,12 @@ import re from typing import TYPE_CHECKING, Any -from background_tasks.constants import ( +from background_tasks.core.constants import ( RESOURCE_REF_MAX_LENGTH, SM_RESOURCE_ID_HEADER, SM_RESOURCE_TYPE_HEADER, ) -from background_tasks.tenant_context import header_of_request +from background_tasks.executions.tenant_context import header_of_request if TYPE_CHECKING: from celery.canvas import Signature diff --git a/modules/background_tasks/background_tasks/worker/task_routing.py b/modules/background_tasks/background_tasks/worker/task_routing.py new file mode 100644 index 000000000..da3250c4d --- /dev/null +++ b/modules/background_tasks/background_tasks/worker/task_routing.py @@ -0,0 +1,158 @@ +"""Per-module task routing, queue membership and package selection. + +A module may ship a light ``/task_config.py`` exporting any of: + +- ``TASK_ROUTES``: Celery ``task_routes`` entries (glob patterns allowed; a + value is a queue name or a full route dict). +- ``QUEUES``: the queues the module's tasks run on. Defaults to the install's + ``task_default_queue``. +- ``BEAT_SCHEDULE``: periodic entries, merged over ``tasks.BEAT_SCHEDULE``. + +It is read *without* importing ``tasks.py``, so a worker started with a queue +subset can decide which modules to load without importing the heavy ones. +Keep it free of heavy imports (and of celery). +""" + +from __future__ import annotations + +import importlib +import logging +from dataclasses import dataclass, field +from typing import Any + +from background_tasks.core.config_guards import ( + WorkerScope, + sanitize_task_routes, + worker_scope_from_env, +) +from background_tasks.core.settings import BackgroundTasksSettings + +logger = logging.getLogger(__name__) + +OVERRIDE_SOURCE = "override" + + +@dataclass(frozen=True) +class ModuleTaskConfig: + package: str + routes: dict[str, Any] = field(default_factory=dict) + queues: frozenset[str] | None = None + beat_schedule: dict[str, Any] = field(default_factory=dict) + + +def load_task_config(package: str) -> ModuleTaskConfig | None: + """Import ``.task_config``; ``None`` when the module ships none.""" + try: + mod = importlib.import_module(f"{package}.task_config") + except ModuleNotFoundError as exc: + # Only "no task_config.py" is absence; a task_config whose own import + # fails must surface. + if exc.name in (package, f"{package}.task_config"): + return None + raise + routes = getattr(mod, "TASK_ROUTES", None) + queues = getattr(mod, "QUEUES", None) + beat = getattr(mod, "BEAT_SCHEDULE", None) + return ModuleTaskConfig( + package=package, + routes=dict(routes) if isinstance(routes, dict) else {}, + queues=frozenset(queues) if isinstance(queues, (set, frozenset, list, tuple)) else None, + beat_schedule=dict(beat) if isinstance(beat, dict) else {}, + ) + + +def load_task_configs(packages: list[str]) -> dict[str, ModuleTaskConfig]: + configs: dict[str, ModuleTaskConfig] = {} + for pkg in packages: + cfg = load_task_config(pkg) + if cfg is not None: + configs[pkg] = cfg + return configs + + +def route_queue(value: Any) -> str: + """The queue a ``task_routes`` value names ('' when it names none).""" + if isinstance(value, str): + return value + if isinstance(value, dict): + return str(value.get("queue") or "") + return "" + + +def merge_routes( + configs: dict[str, ModuleTaskConfig], override: dict[str, Any] +) -> tuple[Any, list[dict[str, str]]]: + """Merge module routes (sorted package order, first wins) under the override. + + Returns the ``task_routes`` value and a flat ``[{pattern, queue, module}]`` + list describing where each effective entry came from (override first). + + Celery's ``MapRoute`` tries exact names first and then ``*`` patterns in + insertion order, first match wins, so folding both into one dict would let + a module's broader glob placed earlier beat an override glob, and a module's + exact name beat an override glob. With an override the value is therefore a + router *list* ``[override, modules]``, which Celery consults in order. + """ + # Defence for values stored before the settings validator rejected bad + # routes: drop (and log) unsafe entries instead of handing them to Celery. + override = sanitize_task_routes(override) + routes: dict[str, Any] = {} + owner: dict[str, str] = {} + for pkg in sorted(configs): + for pattern, value in configs[pkg].routes.items(): + if pattern in routes: + logger.warning( + "Task route %r from %s clashes with %s; keeping %s's entry", + pattern, + pkg, + owner[pattern], + owner[pattern], + ) + continue + routes[pattern] = value + owner[pattern] = pkg + module_info = [ + {"pattern": p, "queue": route_queue(v), "module": owner[p]} + for p, v in routes.items() + if p not in override + ] + if not override: + return routes, module_info + override_info = [ + {"pattern": p, "queue": route_queue(v), "module": OVERRIDE_SOURCE} + for p, v in override.items() + ] + return [dict(override), routes], override_info + module_info + + +def select_packages( + packages: list[str], + configs: dict[str, ModuleTaskConfig], + settings: BackgroundTasksSettings, + scope: WorkerScope | None = None, +) -> list[str]: + """Packages this process imports ``tasks.py`` for. + + Order: ``exclude_modules`` always loses; then ``include_modules`` always + wins; then, when ``worker_queues`` is set, a package needs a queue overlap; + otherwise every package is loaded. ``scope`` defaults to the process + environment (:func:`worker_scope_from_env`); it is never stored settings. + """ + scope = scope or worker_scope_from_env() + exclude = scope.exclude_modules + include = scope.include_modules + wanted = scope.queues + selected: list[str] = [] + for pkg in packages: + if pkg in exclude: + continue + if pkg in include or not wanted: + selected.append(pkg) + continue + cfg = configs.get(pkg) + queues = cfg.queues if cfg is not None else None + if queues is None: + queues = frozenset({settings.task_default_queue}) + if queues & wanted: + selected.append(pkg) + return selected diff --git a/modules/background_tasks/background_tasks/tenant_celery.py b/modules/background_tasks/background_tasks/worker/tenant_celery.py similarity index 97% rename from modules/background_tasks/background_tasks/tenant_celery.py rename to modules/background_tasks/background_tasks/worker/tenant_celery.py index d825c8bb3..06883328a 100644 --- a/modules/background_tasks/background_tasks/tenant_celery.py +++ b/modules/background_tasks/background_tasks/worker/tenant_celery.py @@ -6,7 +6,7 @@ from celery import Celery, Task -from background_tasks.tenant_context import stamp_owner, stamp_tenant +from background_tasks.executions.tenant_context import stamp_owner, stamp_tenant class TenantPublishMixin: diff --git a/modules/background_tasks/background_tasks/worker_settings.py b/modules/background_tasks/background_tasks/worker_settings.py index edff112fe..7eb01ed1b 100644 --- a/modules/background_tasks/background_tasks/worker_settings.py +++ b/modules/background_tasks/background_tasks/worker_settings.py @@ -1,84 +1,15 @@ -"""Hydrated DB-backed module settings for task code (GH #378). - -A Celery worker never runs the hosting lifespan that hydrates -``app.state..settings``, and ``DbBackedSettings`` ignores the -environment, so a bare ``FileStorageSettings()`` in a task returns pydantic -defaults without complaint. Use :func:`settings_for` instead:: - - cfg = settings_for(FileStorageSettings, "file_storage") - -It reads the SYSTEM-scope overrides through the worker's sync session and -caches the result per process for ``DEFAULT_TTL_SECONDS`` so a hot task does not -query on every call. Admin edits therefore reach a worker within the TTL. -""" +"""Moved to :mod:`background_tasks.worker.settings`; this alias goes in 0.0.46 (module layout).""" from __future__ import annotations -import logging -import threading -import time -from typing import cast - -from pydantic_settings import BaseSettings -from simple_module_db.stored_settings import hydrate_settings_sync - -from background_tasks.constants import MODULE_NAME -from background_tasks.settings import BackgroundTasksSettings -from background_tasks.sync_db import sync_session - -logger = logging.getLogger(__name__) - -DEFAULT_TTL_SECONDS = 30.0 - -_lock = threading.Lock() -_cache: dict[tuple[type, str], tuple[float, BaseSettings]] = {} - - -def settings_for[T: BaseSettings]( - cls: type[T], package: str, *, ttl: float = DEFAULT_TTL_SECONDS -) -> T: - """Return ``cls`` hydrated from DB overrides, cached per process for ``ttl`` seconds. - - ``ttl=0`` always re-reads. - """ - key = (cls, package) - now = time.monotonic() - with _lock: - hit = _cache.get(key) - if hit is not None and ttl > 0 and now - hit[0] < ttl: - # A copy: BaseSettings is mutable, so one task editing its settings - # must not change what the next task in this process sees. - return cast(T, hit[1]).model_copy(deep=True) - with sync_session() as session: - value = hydrate_settings_sync(cls, session, package, lenient=True) - with _lock: - _cache[key] = (now, value) - return value.model_copy(deep=True) - - -def clear_settings_cache() -> None: - """Drop every cached settings object (tests, or after a known write).""" - with _lock: - _cache.clear() - +import importlib +import sys -def load_runner_settings() -> BackgroundTasksSettings: - """``BackgroundTasksSettings`` for the worker/beat process, DB overrides included. +from simple_module_core.deprecation import deprecated_module - ``scripts/run_worker.py`` used to build these from the environment alone, so - an admin's DB-backed edit (a queue, a time limit) - reached the web process but never the worker, and the two disagreed on - config. Hydration is the same call the web lifespan makes (``cls(**db_overrides)``), - so precedence matches the web process: env → DB → default (GH #486). If the DB is - unreachable or not yet migrated the worker still boots on env/defaults, - loudly, rather than refusing to start. - """ - try: - return settings_for(BackgroundTasksSettings, MODULE_NAME, ttl=0) - except Exception: - logger.warning( - "Could not read background_tasks settings from the database; " - "starting from environment/defaults only", - exc_info=True, - ) - return BackgroundTasksSettings() +deprecated_module( + "background_tasks.worker_settings", + remove_in="0.0.46", + use="background_tasks.worker.settings", +) +sys.modules[__name__] = importlib.import_module("background_tasks.worker.settings") diff --git a/modules/background_tasks/tests-js/background_tasks_DetailPage.test.tsx b/modules/background_tasks/tests-js/background_tasks_DetailPage.test.tsx index efb6f5e84..e594e8194 100644 --- a/modules/background_tasks/tests-js/background_tasks_DetailPage.test.tsx +++ b/modules/background_tasks/tests-js/background_tasks_DetailPage.test.tsx @@ -20,8 +20,8 @@ vi.mock('@inertiajs/react', () => ({ vi.mock('sonner', () => ({ toast: { success: vi.fn(), error: vi.fn() } })); import { toast } from 'sonner'; -import type { TaskDetail } from '../background_tasks/pages/constants'; import Detail from '../background_tasks/pages/Detail'; +import type { TaskDetail } from '../background_tasks/ui/utils/constants'; import { requestsOf, stubFetch } from './stub-fetch'; const rd = keys.background_tasks.retry_dialog; diff --git a/modules/background_tasks/tests-js/background_tasks_IndexPage.test.tsx b/modules/background_tasks/tests-js/background_tasks_IndexPage.test.tsx index fe3e97fce..a03c04aa0 100644 --- a/modules/background_tasks/tests-js/background_tasks_IndexPage.test.tsx +++ b/modules/background_tasks/tests-js/background_tasks_IndexPage.test.tsx @@ -21,7 +21,7 @@ vi.mock('sonner', () => ({ toast: { success: vi.fn(), error: vi.fn() } })); import { toast } from 'sonner'; import Index from '../background_tasks/pages/Index'; -import type { Execution } from '../background_tasks/pages/retry'; +import type { Execution } from '../background_tasks/ui/utils/retry'; import { requestsOf, stubFetch } from './stub-fetch'; const rd = keys.background_tasks.retry_dialog; diff --git a/modules/background_tasks/tests-js/background_tasks_LogsTab.test.tsx b/modules/background_tasks/tests-js/background_tasks_LogsTab.test.tsx index 7ab5c4a57..edbaf10f6 100644 --- a/modules/background_tasks/tests-js/background_tasks_LogsTab.test.tsx +++ b/modules/background_tasks/tests-js/background_tasks_LogsTab.test.tsx @@ -7,7 +7,7 @@ configureI18n({ locale: 'en', messages: {} }); vi.mock('sonner', () => ({ toast: { success: vi.fn(), error: vi.fn() } })); -import { LogsPanel } from '../background_tasks/pages/components/LogsPanel'; +import { LogsPanel } from '../background_tasks/ui/components/LogsPanel'; import { requestsOf, stubFetch } from './stub-fetch'; const line = (message: string, level_name = 'INFO', level = 20) => ({ diff --git a/modules/background_tasks/tests-js/background_tasks_OutboxPage.test.tsx b/modules/background_tasks/tests-js/background_tasks_OutboxPage.test.tsx index a3f80ca92..e969a73f4 100644 --- a/modules/background_tasks/tests-js/background_tasks_OutboxPage.test.tsx +++ b/modules/background_tasks/tests-js/background_tasks_OutboxPage.test.tsx @@ -22,7 +22,7 @@ vi.mock('sonner', () => ({ toast: { success: vi.fn(), error: vi.fn() } })); import { toast } from 'sonner'; import Outbox from '../background_tasks/pages/Outbox'; -import type { OutboxMessage } from '../background_tasks/pages/outbox'; +import type { OutboxMessage } from '../background_tasks/ui/utils/outbox'; import { requestsOf, stubFetch } from './stub-fetch'; const ob = keys.background_tasks.outbox; diff --git a/modules/background_tasks/tests-js/background_tasks_WorkerHealthBanner.test.tsx b/modules/background_tasks/tests-js/background_tasks_WorkerHealthBanner.test.tsx index ee2798f4a..231408043 100644 --- a/modules/background_tasks/tests-js/background_tasks_WorkerHealthBanner.test.tsx +++ b/modules/background_tasks/tests-js/background_tasks_WorkerHealthBanner.test.tsx @@ -9,7 +9,7 @@ vi.mock('@inertiajs/react', () => ({ Link: ({ children }: { children: unknown }) => {children as string}, })); -import { WorkerHealthBanner } from '../background_tasks/pages/components/WorkerHealthBanner'; +import { WorkerHealthBanner } from '../background_tasks/ui/components/WorkerHealthBanner'; import { requestsOf, stubFetch } from './stub-fetch'; const snapshot = (reachable: boolean) => ({ diff --git a/modules/background_tasks/tests-js/background_tasks_WorkersPage.test.tsx b/modules/background_tasks/tests-js/background_tasks_WorkersPage.test.tsx index e0c562062..69c5d775c 100644 --- a/modules/background_tasks/tests-js/background_tasks_WorkersPage.test.tsx +++ b/modules/background_tasks/tests-js/background_tasks_WorkersPage.test.tsx @@ -14,8 +14,8 @@ vi.mock('@inertiajs/react', () => ({ })); vi.mock('sonner', () => ({ toast: { success: vi.fn(), error: vi.fn() } })); -import type { WorkerSnapshot } from '../background_tasks/pages/constants'; import Workers from '../background_tasks/pages/Workers'; +import type { WorkerSnapshot } from '../background_tasks/ui/utils/constants'; const w = keys.background_tasks.workers; const base = { broker_url_redacted: 'redis://localhost:6379/0', queues: ['default'], routes: [] }; diff --git a/modules/background_tasks/tests-js/outbox.test.ts b/modules/background_tasks/tests-js/outbox.test.ts index 1dc0c915f..bef53d7c2 100644 --- a/modules/background_tasks/tests-js/outbox.test.ts +++ b/modules/background_tasks/tests-js/outbox.test.ts @@ -7,7 +7,7 @@ import { discardOutboxMessage, retryOutboxMessage, shortName, -} from '../background_tasks/pages/outbox'; +} from '../background_tasks/ui/utils/outbox'; import { requestsOf, stubFetch } from './stub-fetch'; describe('outbox actions', () => { diff --git a/modules/background_tasks/tests-js/retry.test.ts b/modules/background_tasks/tests-js/retry.test.ts index a421bab34..71430262c 100644 --- a/modules/background_tasks/tests-js/retry.test.ts +++ b/modules/background_tasks/tests-js/retry.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; vi.mock('sonner', () => ({ toast: { success: vi.fn(), error: vi.fn() } })); -import { retryExecution } from '../background_tasks/pages/retry'; +import { retryExecution } from '../background_tasks/ui/utils/retry'; const row = { id: 'abc', task_name: 'demo.task' }; diff --git a/modules/background_tasks/tests-js/task-formatters.test.ts b/modules/background_tasks/tests-js/task-formatters.test.ts index a1f2f89e2..15a561be6 100644 --- a/modules/background_tasks/tests-js/task-formatters.test.ts +++ b/modules/background_tasks/tests-js/task-formatters.test.ts @@ -8,7 +8,7 @@ import { formatTs, formatUptime, shortenId, -} from '../background_tasks/pages/constants'; +} from '../background_tasks/ui/utils/constants'; describe('formatDuration', () => { test('is a dash until the run has finished', () => { diff --git a/modules/background_tasks/tests/conftest.py b/modules/background_tasks/tests/conftest.py index 640aaa81f..dd4d046c7 100644 --- a/modules/background_tasks/tests/conftest.py +++ b/modules/background_tasks/tests/conftest.py @@ -9,8 +9,8 @@ from unittest.mock import MagicMock import pytest -from background_tasks import sync_db -from background_tasks.constants import PERM_VIEW, TaskStatus +from background_tasks.core import sync_db +from background_tasks.core.constants import PERM_VIEW, TaskStatus from background_tasks.models import TaskExecution from sqlalchemy import select diff --git a/modules/background_tasks/tests/test_bg_redis_health.py b/modules/background_tasks/tests/core/health/test_background_tasks_redis_health.py similarity index 98% rename from modules/background_tasks/tests/test_bg_redis_health.py rename to modules/background_tasks/tests/core/health/test_background_tasks_redis_health.py index 35d7325ca..c7b0d2581 100644 --- a/modules/background_tasks/tests/test_bg_redis_health.py +++ b/modules/background_tasks/tests/core/health/test_background_tasks_redis_health.py @@ -11,7 +11,7 @@ from types import SimpleNamespace import pytest -from background_tasks.health import CHECK_REDIS, build_redis_check +from background_tasks.core.health import CHECK_REDIS, build_redis_check from simple_module_core.health import HealthStatus diff --git a/modules/background_tasks/tests/test_bg_invalidation.py b/modules/background_tasks/tests/core/invalidation/test_background_tasks_invalidation.py similarity index 97% rename from modules/background_tasks/tests/test_bg_invalidation.py rename to modules/background_tasks/tests/core/invalidation/test_background_tasks_invalidation.py index 8ef48c3a6..defff9a5b 100644 --- a/modules/background_tasks/tests/test_bg_invalidation.py +++ b/modules/background_tasks/tests/core/invalidation/test_background_tasks_invalidation.py @@ -12,9 +12,9 @@ import asyncio import pytest -from background_tasks import invalidation as bg_invalidation -from background_tasks.invalidation import RedisInvalidationTransport -from background_tasks.settings import BackgroundTasksSettings +from background_tasks.core import invalidation as bg_invalidation +from background_tasks.core.invalidation import RedisInvalidationTransport +from background_tasks.core.settings import BackgroundTasksSettings from simple_module_core.invalidation import Invalidation, InvalidationBus @@ -280,7 +280,7 @@ async def test_the_transport_is_installed_when_broadcast_is_on(self, patched_red class _StubApp: def __init__(self, bus: InvalidationBus, settings: BackgroundTasksSettings) -> None: - from background_tasks.services import BackgroundTasksServices + from background_tasks.core.services import BackgroundTasksServices class _Sm: invalidation = bus diff --git a/modules/background_tasks/tests/test_bg_invalidation_degraded.py b/modules/background_tasks/tests/core/invalidation/test_background_tasks_invalidation_degraded.py similarity index 96% rename from modules/background_tasks/tests/test_bg_invalidation_degraded.py rename to modules/background_tasks/tests/core/invalidation/test_background_tasks_invalidation_degraded.py index c882906a3..fec781ff4 100644 --- a/modules/background_tasks/tests/test_bg_invalidation_degraded.py +++ b/modules/background_tasks/tests/core/invalidation/test_background_tasks_invalidation_degraded.py @@ -18,7 +18,7 @@ import contextlib import socket -from background_tasks.invalidation import RedisInvalidationTransport +from background_tasks.core.invalidation import RedisInvalidationTransport from simple_module_core.invalidation import Invalidation, InvalidationBus from simple_module_test import wait_for @@ -53,7 +53,7 @@ class TestMuteBrokerListener: async def test_a_mute_broker_makes_the_listener_complain(self, monkeypatch, caplog): import logging - import background_tasks.invalidation as bg_invalidation + import background_tasks.core.invalidation as bg_invalidation writers: list[asyncio.StreamWriter] = [] @@ -98,7 +98,7 @@ async def test_stop_still_returns_promptly_while_wedged(self, monkeypatch): ``on_shutdown`` awaits ``stop()``, so a listener stuck on a read would hang the whole application shutdown. """ - import background_tasks.invalidation as bg_invalidation + import background_tasks.core.invalidation as bg_invalidation writers: list[asyncio.StreamWriter] = [] @@ -165,7 +165,7 @@ async def test_a_mute_broker_is_bounded_by_the_publish_timeout(self, monkeypatch not an option: the same client serves the subscription, and a quiet channel must not be torn down for being quiet. """ - import background_tasks.invalidation as bg_invalidation + import background_tasks.core.invalidation as bg_invalidation # The handler must *hold* the connection open and answer nothing — that # is what "mute" means. It must also be closable from here: a socket the diff --git a/modules/background_tasks/tests/test_bg_invalidation_redis.py b/modules/background_tasks/tests/core/invalidation/test_background_tasks_invalidation_redis.py similarity index 99% rename from modules/background_tasks/tests/test_bg_invalidation_redis.py rename to modules/background_tasks/tests/core/invalidation/test_background_tasks_invalidation_redis.py index 67afccb18..09c66ac57 100644 --- a/modules/background_tasks/tests/test_bg_invalidation_redis.py +++ b/modules/background_tasks/tests/core/invalidation/test_background_tasks_invalidation_redis.py @@ -20,7 +20,7 @@ import os import pytest -from background_tasks.invalidation import RedisInvalidationTransport +from background_tasks.core.invalidation import RedisInvalidationTransport from simple_module_core.invalidation import Invalidation, InvalidationBus from simple_module_test import wait_for diff --git a/modules/background_tasks/tests/test_beat_schedule.py b/modules/background_tasks/tests/core/settings/test_background_tasks_beat_schedule.py similarity index 96% rename from modules/background_tasks/tests/test_beat_schedule.py rename to modules/background_tasks/tests/core/settings/test_background_tasks_beat_schedule.py index bb3d29921..de89efeb3 100644 --- a/modules/background_tasks/tests/test_beat_schedule.py +++ b/modules/background_tasks/tests/core/settings/test_background_tasks_beat_schedule.py @@ -12,9 +12,9 @@ import types import pytest -from background_tasks import celery_app -from background_tasks.constants import INTERNAL_TASK_SWEEP_STUCK -from background_tasks.settings import BackgroundTasksSettings +from background_tasks.core.constants import INTERNAL_TASK_SWEEP_STUCK +from background_tasks.core.settings import BackgroundTasksSettings +from background_tasks.worker import celery_app def _install_fake_module(monkeypatch, pkg: str, beat_schedule: dict | None) -> None: diff --git a/modules/background_tasks/tests/test_bg_config_guards.py b/modules/background_tasks/tests/core/settings/test_background_tasks_config_guards.py similarity index 94% rename from modules/background_tasks/tests/test_bg_config_guards.py rename to modules/background_tasks/tests/core/settings/test_background_tasks_config_guards.py index 869a71e5e..cb04b2efd 100644 --- a/modules/background_tasks/tests/test_bg_config_guards.py +++ b/modules/background_tasks/tests/core/settings/test_background_tasks_config_guards.py @@ -5,14 +5,14 @@ import logging import pytest -from background_tasks import celery_app -from background_tasks.config_guards import ( +from background_tasks.core.config_guards import ( celery_overrides_from_env, filter_celery_overrides, is_denied_key, sanitize_task_routes, ) -from background_tasks.settings import BackgroundTasksSettings +from background_tasks.core.settings import BackgroundTasksSettings +from background_tasks.worker import celery_app @pytest.mark.parametrize( @@ -99,7 +99,7 @@ def test_task_routes_override_is_reduced_to_safe_routes(caplog) -> None: "ok.dict": {"queue": "q", "priority": 3}, } assert sanitize_task_routes("nope") == {} - from background_tasks.task_routing import merge_routes + from background_tasks.worker.task_routing import merge_routes # Values stored before the settings validator rejected them are dropped at build time. assert {r["pattern"] for r in merge_routes({}, raw)[1]} == {"ok.*", "ok.dict"} @@ -117,7 +117,7 @@ def test_task_routes_override_accepts_json_string_and_rejects_bad_input() -> Non def test_env_helpers_log_bad_input(monkeypatch, caplog) -> None: - from background_tasks.config_guards import env_csv, env_json_object + from background_tasks.core.config_guards import env_csv, env_json_object monkeypatch.setenv("X_JSON", "[1]") monkeypatch.setenv("X_BAD", "{nope") diff --git a/modules/background_tasks/tests/test_bg_db_timeouts.py b/modules/background_tasks/tests/core/settings/test_background_tasks_db_timeouts.py similarity index 92% rename from modules/background_tasks/tests/test_bg_db_timeouts.py rename to modules/background_tasks/tests/core/settings/test_background_tasks_db_timeouts.py index ad5bf5919..cae33198a 100644 --- a/modules/background_tasks/tests/test_bg_db_timeouts.py +++ b/modules/background_tasks/tests/core/settings/test_background_tasks_db_timeouts.py @@ -3,9 +3,10 @@ from types import SimpleNamespace import pytest -from background_tasks import outbox_worker, sync_db -from background_tasks.db_settings import worker_database_options -from background_tasks.settings import BackgroundTasksSettings +from background_tasks.core import sync_db +from background_tasks.core.db_settings import worker_database_options +from background_tasks.core.settings import BackgroundTasksSettings +from background_tasks.outbox import worker as outbox_worker from simple_module_test.database import database_url_for_tests from sqlalchemy import event, text from sqlalchemy.exc import DBAPIError diff --git a/modules/background_tasks/tests/test_bg_redis_url.py b/modules/background_tasks/tests/core/settings/test_background_tasks_redis_url.py similarity index 97% rename from modules/background_tasks/tests/test_bg_redis_url.py rename to modules/background_tasks/tests/core/settings/test_background_tasks_redis_url.py index 96ea6e89e..48f838507 100644 --- a/modules/background_tasks/tests/test_bg_redis_url.py +++ b/modules/background_tasks/tests/core/settings/test_background_tasks_redis_url.py @@ -17,8 +17,8 @@ import logging import pytest -from background_tasks.constants import DEFAULT_BROKER_URL, DEFAULT_RESULT_BACKEND -from background_tasks.settings import BackgroundTasksSettings +from background_tasks.core.constants import DEFAULT_BROKER_URL, DEFAULT_RESULT_BACKEND +from background_tasks.core.settings import BackgroundTasksSettings @pytest.fixture(autouse=True) diff --git a/modules/background_tasks/tests/test_background_tasks_settings.py b/modules/background_tasks/tests/core/settings/test_background_tasks_settings.py similarity index 98% rename from modules/background_tasks/tests/test_background_tasks_settings.py rename to modules/background_tasks/tests/core/settings/test_background_tasks_settings.py index 401fcc208..047eb313b 100644 --- a/modules/background_tasks/tests/test_background_tasks_settings.py +++ b/modules/background_tasks/tests/core/settings/test_background_tasks_settings.py @@ -3,7 +3,7 @@ import logging import pytest -from background_tasks.settings import BackgroundTasksSettings +from background_tasks.core.settings import BackgroundTasksSettings from pydantic import ValidationError diff --git a/modules/background_tasks/tests/test_bg_settings_env.py b/modules/background_tasks/tests/core/settings/test_background_tasks_settings_env.py similarity index 97% rename from modules/background_tasks/tests/test_bg_settings_env.py rename to modules/background_tasks/tests/core/settings/test_background_tasks_settings_env.py index 309041f7d..c8232b609 100644 --- a/modules/background_tasks/tests/test_bg_settings_env.py +++ b/modules/background_tasks/tests/core/settings/test_background_tasks_settings_env.py @@ -10,8 +10,8 @@ from __future__ import annotations import pytest -from background_tasks.constants import DEFAULT_BROKER_URL, DEFAULT_RESULT_BACKEND -from background_tasks.settings import BackgroundTasksSettings +from background_tasks.core.constants import DEFAULT_BROKER_URL, DEFAULT_RESULT_BACKEND +from background_tasks.core.settings import BackgroundTasksSettings def test_defaults_when_env_unset(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/modules/background_tasks/tests/test_bg_startup_log_masking.py b/modules/background_tasks/tests/core/settings/test_background_tasks_startup_log_masking.py similarity index 100% rename from modules/background_tasks/tests/test_bg_startup_log_masking.py rename to modules/background_tasks/tests/core/settings/test_background_tasks_startup_log_masking.py diff --git a/modules/background_tasks/tests/test_sync_db.py b/modules/background_tasks/tests/core/sync_db/test_background_tasks_sync_db.py similarity index 97% rename from modules/background_tasks/tests/test_sync_db.py rename to modules/background_tasks/tests/core/sync_db/test_background_tasks_sync_db.py index f70b57636..4ea09cd26 100644 --- a/modules/background_tasks/tests/test_sync_db.py +++ b/modules/background_tasks/tests/core/sync_db/test_background_tasks_sync_db.py @@ -12,7 +12,7 @@ from collections.abc import Iterator import pytest -from background_tasks import sync_db +from background_tasks.core import sync_db @pytest.fixture(autouse=True) diff --git a/modules/background_tasks/tests/test_sync_db_fork.py b/modules/background_tasks/tests/core/sync_db/test_background_tasks_sync_db_fork.py similarity index 98% rename from modules/background_tasks/tests/test_sync_db_fork.py rename to modules/background_tasks/tests/core/sync_db/test_background_tasks_sync_db_fork.py index 0cbf48fdf..a6d9cf5ef 100644 --- a/modules/background_tasks/tests/test_sync_db_fork.py +++ b/modules/background_tasks/tests/core/sync_db/test_background_tasks_sync_db_fork.py @@ -10,7 +10,7 @@ from collections.abc import Iterator import pytest -from background_tasks import sync_db +from background_tasks.core import sync_db from sqlalchemy import text diff --git a/modules/background_tasks/tests/test_retry_failed_bulk.py b/modules/background_tasks/tests/executions/retry/test_background_tasks_retry_failed_bulk.py similarity index 99% rename from modules/background_tasks/tests/test_retry_failed_bulk.py rename to modules/background_tasks/tests/executions/retry/test_background_tasks_retry_failed_bulk.py index ace1bb1e1..2fddecc49 100644 --- a/modules/background_tasks/tests/test_retry_failed_bulk.py +++ b/modules/background_tasks/tests/executions/retry/test_background_tasks_retry_failed_bulk.py @@ -14,7 +14,7 @@ import httpx import pytest -from background_tasks.constants import TaskStatus +from background_tasks.core.constants import TaskStatus ADMIN_BASE = "/api/background_tasks/admin" RETRY_FAILED = f"{ADMIN_BASE}/executions/retry-failed" diff --git a/modules/background_tasks/tests/test_retry_failed_guards.py b/modules/background_tasks/tests/executions/retry/test_background_tasks_retry_failed_guards.py similarity index 98% rename from modules/background_tasks/tests/test_retry_failed_guards.py rename to modules/background_tasks/tests/executions/retry/test_background_tasks_retry_failed_guards.py index 7db3f1d39..e9a6f2e19 100644 --- a/modules/background_tasks/tests/test_retry_failed_guards.py +++ b/modules/background_tasks/tests/executions/retry/test_background_tasks_retry_failed_guards.py @@ -12,8 +12,8 @@ import httpx import pytest -from background_tasks import retry_service -from background_tasks.constants import TaskStatus +from background_tasks.core.constants import TaskStatus +from background_tasks.executions import retry_service from background_tasks.models import TaskExecution ADMIN_BASE = "/api/background_tasks/admin" diff --git a/modules/background_tasks/tests/test_retry_sweep_ordering.py b/modules/background_tasks/tests/executions/retry/test_background_tasks_retry_sweep_ordering.py similarity index 97% rename from modules/background_tasks/tests/test_retry_sweep_ordering.py rename to modules/background_tasks/tests/executions/retry/test_background_tasks_retry_sweep_ordering.py index 95276b526..5063aff58 100644 --- a/modules/background_tasks/tests/test_retry_sweep_ordering.py +++ b/modules/background_tasks/tests/executions/retry/test_background_tasks_retry_sweep_ordering.py @@ -24,10 +24,10 @@ import httpx import pytest -from background_tasks.constants import RETRY_ALL_BATCH, TABLE_TASK_EXECUTION, TaskStatus from background_tasks.contracts.events import TaskRetried -from background_tasks.filters import bulk_retry_conditions -from background_tasks.retry_service import claim_query +from background_tasks.core.constants import RETRY_ALL_BATCH, TABLE_TASK_EXECUTION, TaskStatus +from background_tasks.executions.filters import bulk_retry_conditions +from background_tasks.executions.retry_service import claim_query from sqlalchemy import event from sqlalchemy.dialects import postgresql diff --git a/modules/background_tasks/tests/test_background_tasks_after_commit.py b/modules/background_tasks/tests/executions/service/test_background_tasks_after_commit.py similarity index 96% rename from modules/background_tasks/tests/test_background_tasks_after_commit.py rename to modules/background_tasks/tests/executions/service/test_background_tasks_after_commit.py index 73628d344..ecee48d0c 100644 --- a/modules/background_tasks/tests/test_background_tasks_after_commit.py +++ b/modules/background_tasks/tests/executions/service/test_background_tasks_after_commit.py @@ -16,10 +16,11 @@ import httpx import pytest -from background_tasks import enqueue_after_commit, sync_db -from background_tasks.constants import TaskStatus +from background_tasks import enqueue_after_commit +from background_tasks.core import sync_db +from background_tasks.core.constants import TaskStatus +from background_tasks.executions.service import BackgroundTaskService from background_tasks.models import TaskExecution -from background_tasks.service import BackgroundTaskService from simple_module_core.events import EventBus from simple_module_db import RequestSession, finalize_session from simple_module_db.transaction import SESSION_START_KEY, rollback_session @@ -252,7 +253,7 @@ async def test_a_task_name_is_sent_through_the_current_app( self, session: RequestSession, monkeypatch: pytest.MonkeyPatch ) -> None: app = MagicMock(name="Celery") - monkeypatch.setattr("background_tasks.after_commit.current_app", app) + monkeypatch.setattr("background_tasks.executions.after_commit.current_app", app) task_id = enqueue_after_commit(session, "orders.ship", 7) session.add(TaskExecution(task_name="orders.ship")) diff --git a/modules/background_tasks/tests/test_background_tasks_eager_bindings.py b/modules/background_tasks/tests/executions/service/test_background_tasks_eager_bindings.py similarity index 84% rename from modules/background_tasks/tests/test_background_tasks_eager_bindings.py rename to modules/background_tasks/tests/executions/service/test_background_tasks_eager_bindings.py index 0609f5132..577741d01 100644 --- a/modules/background_tasks/tests/test_background_tasks_eager_bindings.py +++ b/modules/background_tasks/tests/executions/service/test_background_tasks_eager_bindings.py @@ -1,11 +1,11 @@ """Eager tasks keep the caller's authority and validate headers (#751, #752).""" import pytest -from background_tasks import celery_app -from background_tasks import signals as _signals # noqa: F401 - connects the handlers -from background_tasks import tenant_context as task_context -from background_tasks.settings import BackgroundTasksSettings -from background_tasks.tenant_context import OWNER_HEADER, TENANT_HEADER, stamp_owner +from background_tasks.core.settings import BackgroundTasksSettings +from background_tasks.executions import tenant_context as task_context +from background_tasks.executions.tenant_context import OWNER_HEADER, TENANT_HEADER, stamp_owner +from background_tasks.worker import celery_app +from background_tasks.worker import signals as _signals # noqa: F401 - connects the handlers from celery import current_app from simple_module_db import TenantIsolationError, current_tenant_id, tenant_context from simple_module_db.listeners import current_user_id diff --git a/modules/background_tasks/tests/test_bg_index_has_executions.py b/modules/background_tasks/tests/executions/service/test_background_tasks_index_has_executions.py similarity index 97% rename from modules/background_tasks/tests/test_bg_index_has_executions.py rename to modules/background_tasks/tests/executions/service/test_background_tasks_index_has_executions.py index 095f2d1db..3607af2d7 100644 --- a/modules/background_tasks/tests/test_bg_index_has_executions.py +++ b/modules/background_tasks/tests/executions/service/test_background_tasks_index_has_executions.py @@ -13,7 +13,7 @@ import httpx import pytest -from background_tasks.constants import TaskStatus +from background_tasks.core.constants import TaskStatus from background_tasks.models import TaskExecution VIEW_BASE = "/admin/background-tasks/" diff --git a/modules/background_tasks/tests/test_bg_list_item_payload.py b/modules/background_tasks/tests/executions/service/test_background_tasks_list_item_payload.py similarity index 97% rename from modules/background_tasks/tests/test_bg_list_item_payload.py rename to modules/background_tasks/tests/executions/service/test_background_tasks_list_item_payload.py index 5585bb9f8..377388100 100644 --- a/modules/background_tasks/tests/test_bg_list_item_payload.py +++ b/modules/background_tasks/tests/executions/service/test_background_tasks_list_item_payload.py @@ -13,8 +13,8 @@ import httpx import pytest -from background_tasks.constants import TaskStatus from background_tasks.contracts.schemas import TaskExecutionListItem +from background_tasks.core.constants import TaskStatus from background_tasks.models import TaskExecution VIEW_BASE = "/admin/background-tasks/" diff --git a/modules/background_tasks/tests/test_bg_newest_first_status.py b/modules/background_tasks/tests/executions/service/test_background_tasks_newest_first_status.py similarity index 93% rename from modules/background_tasks/tests/test_bg_newest_first_status.py rename to modules/background_tasks/tests/executions/service/test_background_tasks_newest_first_status.py index 466d4d48c..3499ad6c9 100644 --- a/modules/background_tasks/tests/test_bg_newest_first_status.py +++ b/modules/background_tasks/tests/executions/service/test_background_tasks_newest_first_status.py @@ -4,9 +4,9 @@ from datetime import UTC, datetime -from background_tasks.constants import TaskStatus +from background_tasks.core.constants import TaskStatus +from background_tasks.executions.service import BackgroundTaskService from background_tasks.models import TaskExecution -from background_tasks.service import BackgroundTaskService from simple_module_core.tenancy import TenantRole URL = "/api/background_tasks/tasks/{}/status" diff --git a/modules/background_tasks/tests/test_background_tasks_order_index.py b/modules/background_tasks/tests/executions/service/test_background_tasks_order_index.py similarity index 98% rename from modules/background_tasks/tests/test_background_tasks_order_index.py rename to modules/background_tasks/tests/executions/service/test_background_tasks_order_index.py index f1d85af49..e748a1147 100644 --- a/modules/background_tasks/tests/test_background_tasks_order_index.py +++ b/modules/background_tasks/tests/executions/service/test_background_tasks_order_index.py @@ -17,7 +17,7 @@ def _script() -> ScriptDirectory: - root = Path(__file__).resolve().parents[3] + root = Path(__file__).resolve().parents[5] return ScriptDirectory.from_config(Config(str(root / "host/alembic.ini"))) diff --git a/modules/background_tasks/tests/test_background_tasks_publish_validation.py b/modules/background_tasks/tests/executions/service/test_background_tasks_publish_validation.py similarity index 93% rename from modules/background_tasks/tests/test_background_tasks_publish_validation.py rename to modules/background_tasks/tests/executions/service/test_background_tasks_publish_validation.py index f2f738d63..58c5ac280 100644 --- a/modules/background_tasks/tests/test_background_tasks_publish_validation.py +++ b/modules/background_tasks/tests/executions/service/test_background_tasks_publish_validation.py @@ -4,9 +4,9 @@ from uuid import uuid4 import pytest -from background_tasks import celery_app -from background_tasks.settings import BackgroundTasksSettings -from background_tasks.tenant_context import TENANT_HEADER +from background_tasks.core.settings import BackgroundTasksSettings +from background_tasks.executions.tenant_context import TENANT_HEADER +from background_tasks.worker import celery_app from celery import current_app from simple_module_db import TenantIsolationError, tenant_context diff --git a/modules/background_tasks/tests/test_background_tasks_read_projection.py b/modules/background_tasks/tests/executions/service/test_background_tasks_read_projection.py similarity index 96% rename from modules/background_tasks/tests/test_background_tasks_read_projection.py rename to modules/background_tasks/tests/executions/service/test_background_tasks_read_projection.py index dc201073e..aa907b7c3 100644 --- a/modules/background_tasks/tests/test_background_tasks_read_projection.py +++ b/modules/background_tasks/tests/executions/service/test_background_tasks_read_projection.py @@ -5,9 +5,9 @@ from unittest.mock import MagicMock import pytest -from background_tasks.constants import TaskStatus +from background_tasks.core.constants import TaskStatus +from background_tasks.executions.service import BackgroundTaskService from background_tasks.models import TaskExecution -from background_tasks.service import BackgroundTaskService from simple_module_core.events import EventBus from sqlalchemy import event diff --git a/modules/background_tasks/tests/test_background_tasks_retention_retries.py b/modules/background_tasks/tests/executions/service/test_background_tasks_retention_retries.py similarity index 96% rename from modules/background_tasks/tests/test_background_tasks_retention_retries.py rename to modules/background_tasks/tests/executions/service/test_background_tasks_retention_retries.py index b40c93b32..39877cc03 100644 --- a/modules/background_tasks/tests/test_background_tasks_retention_retries.py +++ b/modules/background_tasks/tests/executions/service/test_background_tasks_retention_retries.py @@ -3,8 +3,9 @@ from datetime import UTC, datetime, timedelta from types import SimpleNamespace -from background_tasks import sync_db, tasks -from background_tasks.constants import OutboxStatus, TaskStatus +from background_tasks import tasks +from background_tasks.core import sync_db +from background_tasks.core.constants import OutboxStatus, TaskStatus from background_tasks.models import OutboxMessage, RetainedLog, TaskExecution from sqlalchemy import select, text diff --git a/modules/background_tasks/tests/test_background_tasks_savepoint_publish.py b/modules/background_tasks/tests/executions/service/test_background_tasks_savepoint_publish.py similarity index 95% rename from modules/background_tasks/tests/test_background_tasks_savepoint_publish.py rename to modules/background_tasks/tests/executions/service/test_background_tasks_savepoint_publish.py index cc99c69dd..24f144ff3 100644 --- a/modules/background_tasks/tests/test_background_tasks_savepoint_publish.py +++ b/modules/background_tasks/tests/executions/service/test_background_tasks_savepoint_publish.py @@ -3,7 +3,8 @@ from unittest.mock import MagicMock import pytest -from background_tasks import enqueue_after_commit, sync_db +from background_tasks import enqueue_after_commit +from background_tasks.core import sync_db from sqlalchemy import text diff --git a/modules/background_tasks/tests/test_bg_service.py b/modules/background_tasks/tests/executions/service/test_background_tasks_service.py similarity index 98% rename from modules/background_tasks/tests/test_bg_service.py rename to modules/background_tasks/tests/executions/service/test_background_tasks_service.py index 1b5228b7f..1d1a18f0f 100644 --- a/modules/background_tasks/tests/test_bg_service.py +++ b/modules/background_tasks/tests/executions/service/test_background_tasks_service.py @@ -8,10 +8,10 @@ from unittest.mock import MagicMock import pytest -from background_tasks.constants import TaskStatus from background_tasks.contracts.events import TaskRetried +from background_tasks.core.constants import TaskStatus +from background_tasks.executions.service import BackgroundTaskService from background_tasks.models import TaskExecution -from background_tasks.service import BackgroundTaskService from fastapi import HTTPException from simple_module_core.events import EventBus from simple_module_db import RequestSession, finalize_session diff --git a/modules/background_tasks/tests/test_background_tasks_tz_columns.py b/modules/background_tasks/tests/executions/service/test_background_tasks_tz_columns.py similarity index 93% rename from modules/background_tasks/tests/test_background_tasks_tz_columns.py rename to modules/background_tasks/tests/executions/service/test_background_tasks_tz_columns.py index 5f0e02ee0..d076b9192 100644 --- a/modules/background_tasks/tests/test_background_tasks_tz_columns.py +++ b/modules/background_tasks/tests/executions/service/test_background_tasks_tz_columns.py @@ -7,9 +7,9 @@ from unittest.mock import MagicMock import pytest -from background_tasks.constants import TaskStatus +from background_tasks.core.constants import TaskStatus +from background_tasks.executions.service import BackgroundTaskService from background_tasks.models import TaskExecution -from background_tasks.service import BackgroundTaskService from simple_module_core.events import EventBus from simple_module_db import RequestSession from users.models.refresh_token import RefreshToken diff --git a/modules/background_tasks/tests/test_bg_log_purge.py b/modules/background_tasks/tests/logs/purge/test_background_tasks_log_purge.py similarity index 86% rename from modules/background_tasks/tests/test_bg_log_purge.py rename to modules/background_tasks/tests/logs/purge/test_background_tasks_log_purge.py index 25a3a4eae..69bfa26e1 100644 --- a/modules/background_tasks/tests/test_bg_log_purge.py +++ b/modules/background_tasks/tests/logs/purge/test_background_tasks_log_purge.py @@ -5,9 +5,11 @@ from datetime import UTC, datetime, timedelta import pytest -from background_tasks import log_purge, sync_db, tasks +from background_tasks import tasks +from background_tasks.core import sync_db +from background_tasks.core.settings import BackgroundTasksSettings +from background_tasks.logs import purge as log_purge from background_tasks.models import RetainedLog -from background_tasks.settings import BackgroundTasksSettings from sqlalchemy import select pytestmark = pytest.mark.usefixtures("sync_sqlite") diff --git a/modules/background_tasks/tests/test_log_context.py b/modules/background_tasks/tests/logs/sink/test_background_tasks_log_context.py similarity index 97% rename from modules/background_tasks/tests/test_log_context.py rename to modules/background_tasks/tests/logs/sink/test_background_tasks_log_context.py index 9c6a119b4..61cc9000b 100644 --- a/modules/background_tasks/tests/test_log_context.py +++ b/modules/background_tasks/tests/logs/sink/test_background_tasks_log_context.py @@ -8,14 +8,14 @@ from types import SimpleNamespace import pytest -from background_tasks.log_context import ( +from background_tasks.worker.log_context import ( LogContextFilter, _signal_tokens, bind_task_context, get_log_context, install_log_filter, ) -from background_tasks.signals import on_task_postrun, on_task_prerun +from background_tasks.worker.signals import on_task_postrun, on_task_prerun def _make_record() -> logging.LogRecord: diff --git a/modules/background_tasks/tests/test_bg_log_sink.py b/modules/background_tasks/tests/logs/sink/test_background_tasks_log_sink.py similarity index 93% rename from modules/background_tasks/tests/test_bg_log_sink.py rename to modules/background_tasks/tests/logs/sink/test_background_tasks_log_sink.py index 3ec17bf52..0f2908f34 100644 --- a/modules/background_tasks/tests/test_bg_log_sink.py +++ b/modules/background_tasks/tests/logs/sink/test_background_tasks_log_sink.py @@ -6,10 +6,11 @@ from collections.abc import Iterator import pytest -from background_tasks import log_sink, sync_db -from background_tasks.log_context import bind_task_context +from background_tasks.core import sync_db +from background_tasks.core.settings import BackgroundTasksSettings +from background_tasks.logs import sink as log_sink from background_tasks.models import RetainedLog -from background_tasks.settings import BackgroundTasksSettings +from background_tasks.worker.log_context import bind_task_context from simple_module_db import tenant_context from sqlalchemy import select diff --git a/modules/background_tasks/tests/test_bg_log_sink_loop.py b/modules/background_tasks/tests/logs/sink/test_background_tasks_log_sink_loop.py similarity index 92% rename from modules/background_tasks/tests/test_bg_log_sink_loop.py rename to modules/background_tasks/tests/logs/sink/test_background_tasks_log_sink_loop.py index 3f09a85de..e0244ce29 100644 --- a/modules/background_tasks/tests/test_bg_log_sink_loop.py +++ b/modules/background_tasks/tests/logs/sink/test_background_tasks_log_sink_loop.py @@ -8,10 +8,11 @@ from collections.abc import Iterator import pytest -from background_tasks import log_sink, sync_db from background_tasks.contracts.schemas import TaskExecutionDetail, TaskExecutionListItem +from background_tasks.core import sync_db +from background_tasks.core.settings import BackgroundTasksSettings +from background_tasks.logs import sink as log_sink from background_tasks.models import RetainedLog -from background_tasks.settings import BackgroundTasksSettings from sqlalchemy import func, select pytestmark = pytest.mark.usefixtures("sync_sqlite") diff --git a/modules/background_tasks/tests/test_bg_log_sink_nul.py b/modules/background_tasks/tests/logs/sink/test_background_tasks_log_sink_nul.py similarity index 93% rename from modules/background_tasks/tests/test_bg_log_sink_nul.py rename to modules/background_tasks/tests/logs/sink/test_background_tasks_log_sink_nul.py index 6960b96f9..f9cc08669 100644 --- a/modules/background_tasks/tests/test_bg_log_sink_nul.py +++ b/modules/background_tasks/tests/logs/sink/test_background_tasks_log_sink_nul.py @@ -4,7 +4,7 @@ import logging -from background_tasks import log_sink +from background_tasks.logs import sink as log_sink def test_to_row_strips_nul_from_message_ids_and_extra(): diff --git a/modules/background_tasks/tests/test_outbox_admin.py b/modules/background_tasks/tests/outbox/admin/test_background_tasks_outbox_admin.py similarity index 96% rename from modules/background_tasks/tests/test_outbox_admin.py rename to modules/background_tasks/tests/outbox/admin/test_background_tasks_outbox_admin.py index 9a759eac5..dca362477 100644 --- a/modules/background_tasks/tests/test_outbox_admin.py +++ b/modules/background_tasks/tests/outbox/admin/test_background_tasks_outbox_admin.py @@ -8,10 +8,10 @@ import httpx import pytest -from background_tasks.constants import INTERNAL_TASK_DISPATCH_OUTBOX, OutboxStatus +from background_tasks.core.constants import INTERNAL_TASK_DISPATCH_OUTBOX, OutboxStatus from background_tasks.models import OutboxMessage -from background_tasks.outbox_claim import OutboxPolicy -from background_tasks.outbox_dispatch import dispatch +from background_tasks.outbox.claim import OutboxPolicy +from background_tasks.outbox.dispatch import dispatch from simple_module_core import DurableContext from simple_module_core.durable_events import event_key, serialize_event from simple_module_core.events import Event diff --git a/modules/background_tasks/tests/test_outbox_tenancy.py b/modules/background_tasks/tests/outbox/admin/test_background_tasks_outbox_tenancy.py similarity index 97% rename from modules/background_tasks/tests/test_outbox_tenancy.py rename to modules/background_tasks/tests/outbox/admin/test_background_tasks_outbox_tenancy.py index 626aa9b07..227dd747e 100644 --- a/modules/background_tasks/tests/test_outbox_tenancy.py +++ b/modules/background_tasks/tests/outbox/admin/test_background_tasks_outbox_tenancy.py @@ -11,7 +11,7 @@ from datetime import UTC, datetime import pytest -from background_tasks.constants import PERM_MANAGE, PERM_VIEW, OutboxStatus +from background_tasks.core.constants import PERM_MANAGE, PERM_VIEW, OutboxStatus from background_tasks.models import OutboxMessage from simple_module_core.tenancy import TenantRole, tenant_role diff --git a/modules/background_tasks/tests/test_background_tasks_delivery_context.py b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_delivery_context.py similarity index 95% rename from modules/background_tasks/tests/test_background_tasks_delivery_context.py rename to modules/background_tasks/tests/outbox/delivery/test_background_tasks_delivery_context.py index 42b7f871b..f7a7b195b 100644 --- a/modules/background_tasks/tests/test_background_tasks_delivery_context.py +++ b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_delivery_context.py @@ -4,8 +4,8 @@ from dataclasses import dataclass import pytest -from background_tasks.outbox_claim import OutboxPolicy -from background_tasks.outbox_dispatch import dispatch +from background_tasks.outbox.claim import OutboxPolicy +from background_tasks.outbox.dispatch import dispatch from simple_module_core import durable_events from simple_module_core.events import Event diff --git a/modules/background_tasks/tests/test_bg_inprocess_sweep.py b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_inprocess_sweep.py similarity index 94% rename from modules/background_tasks/tests/test_bg_inprocess_sweep.py rename to modules/background_tasks/tests/outbox/delivery/test_background_tasks_inprocess_sweep.py index 47b91f755..21d40f984 100644 --- a/modules/background_tasks/tests/test_bg_inprocess_sweep.py +++ b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_inprocess_sweep.py @@ -7,9 +7,9 @@ from datetime import UTC, datetime, timedelta import pytest -from background_tasks.constants import OutboxStatus, TaskStatus +from background_tasks.core.constants import OutboxStatus, TaskStatus from background_tasks.models import OutboxMessage, TaskExecution -from background_tasks.outbox_sweep import start_sweep, stop_sweep +from background_tasks.outbox.sweep import start_sweep, stop_sweep from background_tasks.tasks import purge_old_executions from simple_module_core.durable_events import event_key, serialize_event from simple_module_core.events import Event @@ -122,7 +122,7 @@ def test_transport_import_does_not_load_celery(): [ sys.executable, "-c", - "import sys; import background_tasks.outbox_transport; " + "import sys; import background_tasks.outbox.transport; " "assert 'celery' not in sys.modules", ], check=True, @@ -130,7 +130,7 @@ def test_transport_import_does_not_load_celery(): def test_purge_uses_db_retention(sync_sqlite, monkeypatch): - from background_tasks.sync_db import get_sync_session_factory, sync_session + from background_tasks.core.sync_db import get_sync_session_factory, sync_session Setting.metadata.create_all(get_sync_session_factory().kw["bind"]) @@ -160,7 +160,7 @@ def test_purge_uses_db_retention(sync_sqlite, monkeypatch): async def test_two_web_loops_deliver_once(app): if app.state.sm.db.engine.dialect.name != "postgresql": pytest.skip("requires PostgreSQL") - from background_tasks.outbox_sweep import sweep_loop + from background_tasks.outbox.sweep import sweep_loop seen = [] diff --git a/modules/background_tasks/tests/test_outbox_concurrency.py b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_concurrency.py similarity index 97% rename from modules/background_tasks/tests/test_outbox_concurrency.py rename to modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_concurrency.py index a19b308c2..58f7de48e 100644 --- a/modules/background_tasks/tests/test_outbox_concurrency.py +++ b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_concurrency.py @@ -16,10 +16,10 @@ from pathlib import Path import pytest -from background_tasks.constants import OutboxStatus, TaskStatus +from background_tasks.core.constants import OutboxStatus, TaskStatus from background_tasks.models import OutboxMessage, TaskExecution -from background_tasks.outbox_claim import OutboxPolicy, claim -from background_tasks.outbox_dispatch import deliver_one, dispatch +from background_tasks.outbox.claim import OutboxPolicy, claim +from background_tasks.outbox.dispatch import deliver_one, dispatch from simple_module_core import DurableContext from simple_module_core.durable_events import event_key, serialize_event from simple_module_core.events import Event, EventBus diff --git a/modules/background_tasks/tests/test_outbox_delivery.py b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_delivery.py similarity index 97% rename from modules/background_tasks/tests/test_outbox_delivery.py rename to modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_delivery.py index 3c3b05bf1..b1d457868 100644 --- a/modules/background_tasks/tests/test_outbox_delivery.py +++ b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_delivery.py @@ -8,10 +8,10 @@ from datetime import UTC, datetime, timedelta import pytest -from background_tasks.constants import INTERNAL_TASK_DISPATCH_OUTBOX, OutboxStatus, TaskStatus +from background_tasks.core.constants import INTERNAL_TASK_DISPATCH_OUTBOX, OutboxStatus, TaskStatus from background_tasks.models import OutboxMessage, TaskExecution -from background_tasks.outbox_claim import OutboxPolicy -from background_tasks.outbox_dispatch import dispatch +from background_tasks.outbox.claim import OutboxPolicy +from background_tasks.outbox.dispatch import dispatch from simple_module_core import DurableContext from simple_module_core.events import Event from simple_module_db import current_tenant_id, finalize_session, tenant_context diff --git a/modules/background_tasks/tests/test_bg_outbox_eager_platform.py b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_eager_platform.py similarity index 95% rename from modules/background_tasks/tests/test_bg_outbox_eager_platform.py rename to modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_eager_platform.py index aa6525884..f9dcb3d97 100644 --- a/modules/background_tasks/tests/test_bg_outbox_eager_platform.py +++ b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_eager_platform.py @@ -14,9 +14,9 @@ from dataclasses import dataclass import pytest -from background_tasks.constants import OutboxStatus -from background_tasks.outbox_claim import OutboxPolicy -from background_tasks.outbox_dispatch import dispatch +from background_tasks.core.constants import OutboxStatus +from background_tasks.outbox.claim import OutboxPolicy +from background_tasks.outbox.dispatch import dispatch from simple_module_core import DurableContext from simple_module_core.events import Event from simple_module_db import all_tenants, current_tenant_id, finalize_session, tenant_context diff --git a/modules/background_tasks/tests/test_bg_outbox_eager_serialization.py b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_eager_serialization.py similarity index 97% rename from modules/background_tasks/tests/test_bg_outbox_eager_serialization.py rename to modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_eager_serialization.py index b94dd6c2c..9f5807150 100644 --- a/modules/background_tasks/tests/test_bg_outbox_eager_serialization.py +++ b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_eager_serialization.py @@ -3,7 +3,7 @@ import asyncio from dataclasses import dataclass -from background_tasks.constants import OutboxStatus +from background_tasks.core.constants import OutboxStatus from background_tasks.models import TaskExecution from simple_module_core import DurableContext from simple_module_core.events import Event diff --git a/modules/background_tasks/tests/test_bg_outbox_fence_guard.py b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_fence_guard.py similarity index 97% rename from modules/background_tasks/tests/test_bg_outbox_fence_guard.py rename to modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_fence_guard.py index 52d7f5b3f..3ee31cf1f 100644 --- a/modules/background_tasks/tests/test_bg_outbox_fence_guard.py +++ b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_fence_guard.py @@ -17,10 +17,10 @@ from pathlib import Path import pytest -from background_tasks.constants import OutboxStatus, TaskStatus +from background_tasks.core.constants import OutboxStatus, TaskStatus from background_tasks.models import OutboxMessage, TaskExecution -from background_tasks.outbox_claim import OutboxPolicy -from background_tasks.outbox_dispatch import dispatch +from background_tasks.outbox.claim import OutboxPolicy +from background_tasks.outbox.dispatch import dispatch from simple_module_core import DurableContext from simple_module_core.durable_events import event_key, serialize_event from simple_module_core.events import Event, EventBus diff --git a/modules/background_tasks/tests/test_bg_outbox_guard_savepoint.py b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_guard_savepoint.py similarity index 96% rename from modules/background_tasks/tests/test_bg_outbox_guard_savepoint.py rename to modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_guard_savepoint.py index 40b7d0531..682e6f19c 100644 --- a/modules/background_tasks/tests/test_bg_outbox_guard_savepoint.py +++ b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_guard_savepoint.py @@ -18,10 +18,10 @@ from datetime import UTC, datetime, timedelta import pytest -from background_tasks.constants import OutboxStatus, TaskStatus +from background_tasks.core.constants import OutboxStatus, TaskStatus from background_tasks.models import TaskExecution -from background_tasks.outbox_claim import OutboxPolicy -from background_tasks.outbox_dispatch import dispatch +from background_tasks.outbox.claim import OutboxPolicy +from background_tasks.outbox.dispatch import dispatch from simple_module_core import DurableContext from simple_module_core.events import Event from simple_module_db import finalize_session diff --git a/modules/background_tasks/tests/test_bg_outbox_publish_scope.py b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_publish_scope.py similarity index 96% rename from modules/background_tasks/tests/test_bg_outbox_publish_scope.py rename to modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_publish_scope.py index abe4be5c2..074baa8dd 100644 --- a/modules/background_tasks/tests/test_bg_outbox_publish_scope.py +++ b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_publish_scope.py @@ -15,9 +15,9 @@ from dataclasses import dataclass import pytest -from background_tasks.constants import OutboxScope, OutboxStatus -from background_tasks.outbox_claim import OutboxPolicy -from background_tasks.outbox_dispatch import dispatch +from background_tasks.core.constants import OutboxScope, OutboxStatus +from background_tasks.outbox.claim import OutboxPolicy +from background_tasks.outbox.dispatch import dispatch from file_storage.models import StoredFile from simple_module_core import DurableContext from simple_module_core.events import Event diff --git a/modules/background_tasks/tests/test_bg_outbox_retry_policy.py b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_retry_policy.py similarity index 95% rename from modules/background_tasks/tests/test_bg_outbox_retry_policy.py rename to modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_retry_policy.py index e29d07e18..990720f66 100644 --- a/modules/background_tasks/tests/test_bg_outbox_retry_policy.py +++ b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_retry_policy.py @@ -15,12 +15,12 @@ from datetime import UTC, datetime, timedelta import pytest -from background_tasks import outbox_claim -from background_tasks.constants import OutboxStatus, TaskStatus +from background_tasks.core.constants import OutboxStatus, TaskStatus +from background_tasks.core.settings import BackgroundTasksSettings from background_tasks.models import OutboxMessage, TaskExecution -from background_tasks.outbox_claim import OutboxPolicy, claim, reclaim_expired -from background_tasks.outbox_dispatch import deliver_one, dispatch -from background_tasks.settings import BackgroundTasksSettings +from background_tasks.outbox import claim as outbox_claim +from background_tasks.outbox.claim import OutboxPolicy, claim, reclaim_expired +from background_tasks.outbox.dispatch import deliver_one, dispatch from simple_module_core import DurableContext from simple_module_core.events import Event from simple_module_db import finalize_session diff --git a/modules/background_tasks/tests/test_bg_outbox_swallowed_cancel.py b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_swallowed_cancel.py similarity index 91% rename from modules/background_tasks/tests/test_bg_outbox_swallowed_cancel.py rename to modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_swallowed_cancel.py index 42fb51896..aa68c101c 100644 --- a/modules/background_tasks/tests/test_bg_outbox_swallowed_cancel.py +++ b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_swallowed_cancel.py @@ -14,10 +14,10 @@ from dataclasses import dataclass import pytest -from background_tasks.constants import OutboxStatus, TaskStatus +from background_tasks.core.constants import OutboxStatus, TaskStatus from background_tasks.models import TaskExecution -from background_tasks.outbox_claim import OutboxPolicy -from background_tasks.outbox_dispatch import dispatch +from background_tasks.outbox.claim import OutboxPolicy +from background_tasks.outbox.dispatch import dispatch from simple_module_core import DurableContext from simple_module_core.events import Event from simple_module_db import finalize_session diff --git a/modules/background_tasks/tests/test_bg_outbox_worker_chain.py b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_worker_chain.py similarity index 94% rename from modules/background_tasks/tests/test_bg_outbox_worker_chain.py rename to modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_worker_chain.py index 6e2a9ea09..c85985fc7 100644 --- a/modules/background_tasks/tests/test_bg_outbox_worker_chain.py +++ b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_worker_chain.py @@ -18,11 +18,13 @@ from unittest.mock import MagicMock import pytest -from background_tasks import outbox_transport, outbox_worker, sync_db -from background_tasks.constants import INTERNAL_TASK_DISPATCH_OUTBOX, OutboxStatus +from background_tasks.core import sync_db +from background_tasks.core.constants import INTERNAL_TASK_DISPATCH_OUTBOX, OutboxStatus from background_tasks.models import OutboxMessage -from background_tasks.outbox_claim import OutboxPolicy -from background_tasks.outbox_dispatch import dispatch +from background_tasks.outbox import transport as outbox_transport +from background_tasks.outbox import worker as outbox_worker +from background_tasks.outbox.claim import OutboxPolicy +from background_tasks.outbox.dispatch import dispatch from background_tasks.tasks import dispatch_outbox from simple_module_core import DurableContext, discovery from simple_module_core.durable_events import event_key, serialize_event @@ -139,7 +141,7 @@ def test_a_module_needing_the_app_is_logged_once_per_worker(monkeypatch, caplog) monkeypatch.setattr(discovery, "topological_sort", lambda modules: modules) outbox_worker.reset_worker_state() try: - with caplog.at_level(logging.WARNING, logger="background_tasks.outbox_worker"): + with caplog.at_level(logging.WARNING, logger="background_tasks.outbox.worker"): outbox_worker.worker_bus() outbox_worker.worker_bus() finally: diff --git a/modules/background_tasks/tests/test_outbox_worker_task.py b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_worker_task.py similarity index 94% rename from modules/background_tasks/tests/test_outbox_worker_task.py rename to modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_worker_task.py index 334db4ee6..760458514 100644 --- a/modules/background_tasks/tests/test_outbox_worker_task.py +++ b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_worker_task.py @@ -7,16 +7,17 @@ from datetime import UTC, datetime, timedelta import pytest -from background_tasks import outbox_worker, sync_db -from background_tasks.celery_app import build_celery -from background_tasks.constants import ( +from background_tasks.core import sync_db +from background_tasks.core.constants import ( BEAT_SWEEP_OUTBOX, INTERNAL_TASK_SWEEP_OUTBOX, OutboxStatus, ) +from background_tasks.core.settings import BackgroundTasksSettings from background_tasks.models import OutboxMessage -from background_tasks.settings import BackgroundTasksSettings +from background_tasks.outbox import worker as outbox_worker from background_tasks.tasks import dispatch_outbox, purge_old_executions, sweep_outbox +from background_tasks.worker.celery_app import build_celery from simple_module_core import DurableContext from simple_module_core.durable_events import event_key, serialize_event from simple_module_core.events import Event, EventBus diff --git a/modules/background_tasks/tests/test_bg_outbox_loop_interrupt.py b/modules/background_tasks/tests/outbox/loop/test_background_tasks_outbox_loop_interrupt.py similarity index 94% rename from modules/background_tasks/tests/test_bg_outbox_loop_interrupt.py rename to modules/background_tasks/tests/outbox/loop/test_background_tasks_outbox_loop_interrupt.py index 9bc4e9a85..ec6ce41e9 100644 --- a/modules/background_tasks/tests/test_bg_outbox_loop_interrupt.py +++ b/modules/background_tasks/tests/outbox/loop/test_background_tasks_outbox_loop_interrupt.py @@ -7,7 +7,7 @@ import threading import pytest -from background_tasks.outbox_loop import run_on_worker_loop, stop_worker_loop +from background_tasks.outbox.loop import run_on_worker_loop, stop_worker_loop class _SoftLimitError(Exception): diff --git a/modules/background_tasks/tests/test_bg_outbox_thread_loops.py b/modules/background_tasks/tests/outbox/loop/test_background_tasks_outbox_thread_loops.py similarity index 94% rename from modules/background_tasks/tests/test_bg_outbox_thread_loops.py rename to modules/background_tasks/tests/outbox/loop/test_background_tasks_outbox_thread_loops.py index f9cc65f4b..62014e3df 100644 --- a/modules/background_tasks/tests/test_bg_outbox_thread_loops.py +++ b/modules/background_tasks/tests/outbox/loop/test_background_tasks_outbox_thread_loops.py @@ -12,8 +12,9 @@ import threading import time -from background_tasks import outbox_worker, sync_db -from background_tasks.outbox_loop import run_on_worker_loop, stop_worker_loop +from background_tasks.core import sync_db +from background_tasks.outbox import worker as outbox_worker +from background_tasks.outbox.loop import run_on_worker_loop, stop_worker_loop def _in_threads(n: int, body) -> tuple[float, list]: diff --git a/modules/background_tasks/tests/test_admin_api.py b/modules/background_tasks/tests/web/api/test_background_tasks_admin_api.py similarity index 99% rename from modules/background_tasks/tests/test_admin_api.py rename to modules/background_tasks/tests/web/api/test_background_tasks_admin_api.py index c5baa6488..213ab8381 100644 --- a/modules/background_tasks/tests/test_admin_api.py +++ b/modules/background_tasks/tests/web/api/test_background_tasks_admin_api.py @@ -7,7 +7,7 @@ import httpx import pytest -from background_tasks.constants import TaskStatus +from background_tasks.core.constants import TaskStatus from background_tasks.models import TaskExecution ADMIN_BASE = "/api/background_tasks/admin" diff --git a/modules/background_tasks/tests/test_detail_props.py b/modules/background_tasks/tests/web/api/test_background_tasks_detail_props.py similarity index 97% rename from modules/background_tasks/tests/test_detail_props.py rename to modules/background_tasks/tests/web/api/test_background_tasks_detail_props.py index f45d001c2..db8a60a37 100644 --- a/modules/background_tasks/tests/test_detail_props.py +++ b/modules/background_tasks/tests/web/api/test_background_tasks_detail_props.py @@ -13,7 +13,7 @@ import httpx import pytest -from background_tasks.constants import TaskStatus +from background_tasks.core.constants import TaskStatus from background_tasks.models import TaskExecution VIEW_BASE = "/admin/background-tasks" diff --git a/modules/background_tasks/tests/test_index_filters.py b/modules/background_tasks/tests/web/api/test_background_tasks_index_filters.py similarity index 99% rename from modules/background_tasks/tests/test_index_filters.py rename to modules/background_tasks/tests/web/api/test_background_tasks_index_filters.py index 9f4903315..55bc341b6 100644 --- a/modules/background_tasks/tests/test_index_filters.py +++ b/modules/background_tasks/tests/web/api/test_background_tasks_index_filters.py @@ -13,7 +13,7 @@ import httpx import pytest -from background_tasks.constants import TaskStatus +from background_tasks.core.constants import TaskStatus from background_tasks.models import TaskExecution VIEW_BASE = "/admin/background-tasks/" diff --git a/modules/background_tasks/tests/test_bg_log_api.py b/modules/background_tasks/tests/web/api/test_background_tasks_log_api.py similarity index 100% rename from modules/background_tasks/tests/test_bg_log_api.py rename to modules/background_tasks/tests/web/api/test_background_tasks_log_api.py diff --git a/modules/background_tasks/tests/test_bg_page_clamp.py b/modules/background_tasks/tests/web/api/test_background_tasks_page_clamp.py similarity index 97% rename from modules/background_tasks/tests/test_bg_page_clamp.py rename to modules/background_tasks/tests/web/api/test_background_tasks_page_clamp.py index 8a94d2934..32fe94213 100644 --- a/modules/background_tasks/tests/test_bg_page_clamp.py +++ b/modules/background_tasks/tests/web/api/test_background_tasks_page_clamp.py @@ -13,7 +13,7 @@ import httpx import pytest -from background_tasks.constants import TaskStatus +from background_tasks.core.constants import TaskStatus from background_tasks.models import TaskExecution VIEW_BASE = "/admin/background-tasks/" diff --git a/modules/background_tasks/tests/test_bg_pagination_bounds.py b/modules/background_tasks/tests/web/api/test_background_tasks_pagination_bounds.py similarity index 100% rename from modules/background_tasks/tests/test_bg_pagination_bounds.py rename to modules/background_tasks/tests/web/api/test_background_tasks_pagination_bounds.py diff --git a/modules/background_tasks/tests/test_bg_task_status_api.py b/modules/background_tasks/tests/web/api/test_background_tasks_task_status_api.py similarity index 98% rename from modules/background_tasks/tests/test_bg_task_status_api.py rename to modules/background_tasks/tests/web/api/test_background_tasks_task_status_api.py index b0ddfc95d..6d691e5f0 100644 --- a/modules/background_tasks/tests/test_bg_task_status_api.py +++ b/modules/background_tasks/tests/web/api/test_background_tasks_task_status_api.py @@ -4,7 +4,7 @@ import httpx import pytest -from background_tasks.constants import TaskStatus +from background_tasks.core.constants import TaskStatus from simple_module_core.tenancy import TenantRole URL = "/api/background_tasks/tasks/{}/status" diff --git a/modules/background_tasks/tests/test_bg_users_ordering.py b/modules/background_tasks/tests/web/api/test_background_tasks_users_ordering.py similarity index 100% rename from modules/background_tasks/tests/test_bg_users_ordering.py rename to modules/background_tasks/tests/web/api/test_background_tasks_users_ordering.py diff --git a/modules/background_tasks/tests/test_bg_index_worker_presence.py b/modules/background_tasks/tests/worker/inspector/test_background_tasks_index_worker_presence.py similarity index 99% rename from modules/background_tasks/tests/test_bg_index_worker_presence.py rename to modules/background_tasks/tests/worker/inspector/test_background_tasks_index_worker_presence.py index 800327b6e..7ab46d249 100644 --- a/modules/background_tasks/tests/test_bg_index_worker_presence.py +++ b/modules/background_tasks/tests/worker/inspector/test_background_tasks_index_worker_presence.py @@ -14,8 +14,8 @@ import httpx import pytest -from background_tasks import worker_inspector as wi from background_tasks.contracts.schemas import WorkerInfo, WorkerSnapshot +from background_tasks.worker import inspector as wi from simple_module_test import InertiaPage # Trailing slash: the index is registered at "/" under the prefix, and the diff --git a/modules/background_tasks/tests/test_worker_inspector.py b/modules/background_tasks/tests/worker/inspector/test_background_tasks_worker_inspector.py similarity index 93% rename from modules/background_tasks/tests/test_worker_inspector.py rename to modules/background_tasks/tests/worker/inspector/test_background_tasks_worker_inspector.py index a2744926d..b6c744fe3 100644 --- a/modules/background_tasks/tests/test_worker_inspector.py +++ b/modules/background_tasks/tests/worker/inspector/test_background_tasks_worker_inspector.py @@ -23,7 +23,7 @@ def _make_celery_with_dead_broker() -> Celery: def test_dead_broker_returns_unreachable_snapshot(): - from background_tasks.worker_inspector import WorkerInspector + from background_tasks.worker.inspector import WorkerInspector inspector = WorkerInspector(_make_celery_with_dead_broker(), timeout=0.2) snapshot = inspector.snapshot() @@ -35,7 +35,7 @@ def test_dead_broker_returns_unreachable_snapshot(): def test_inspect_payloads_are_merged_into_worker_info(): - from background_tasks.worker_inspector import WorkerInspector + from background_tasks.worker.inspector import WorkerInspector celery = MagicMock(spec=Celery) # Probe call: ensure_connection succeeds (broker reachable). @@ -75,7 +75,7 @@ def test_inspect_payloads_are_merged_into_worker_info(): def test_worker_in_stats_but_not_ping_is_offline(): - from background_tasks.worker_inspector import WorkerInspector + from background_tasks.worker.inspector import WorkerInspector celery = MagicMock(spec=Celery) celery.connection.return_value.ensure_connection.return_value = None @@ -95,7 +95,7 @@ def test_worker_in_stats_but_not_ping_is_offline(): def test_redis_error_during_probe_returns_unreachable_snapshot(): """A redis-py exception (e.g. auth failure) is converted, not propagated.""" - from background_tasks.worker_inspector import WorkerInspector + from background_tasks.worker.inspector import WorkerInspector from redis.exceptions import AuthenticationError celery = MagicMock(spec=Celery) @@ -136,7 +136,7 @@ def test_probes_run_concurrently_not_sequentially(): the assertion allows generous slack so this cannot flake on a loaded box while still failing outright if the calls go back to being serial. """ - from background_tasks.worker_inspector import WorkerInspector + from background_tasks.worker.inspector import WorkerInspector def _slow_probe(*_: object, **__: object) -> None: time.sleep(PROBE_DELAY) # intentional-sleep: simulated slow probe @@ -161,7 +161,7 @@ def test_all_four_probes_are_still_issued_when_nobody_replies(): """Concurrency must not drop probes — a worker that answers only stats (degraded, not answering ping) must still be reported. See test_worker_in_stats_but_not_ping_is_offline.""" - from background_tasks.worker_inspector import WorkerInspector + from background_tasks.worker.inspector import WorkerInspector celery, inspect = _celery_with_live_broker_no_workers() WorkerInspector(celery, timeout=0.05).snapshot() @@ -174,7 +174,7 @@ def test_all_four_probes_are_still_issued_when_nobody_replies(): def test_workers_present_still_collects_every_detail(): """The short-circuit must not skip the calls when workers do reply.""" - from background_tasks.worker_inspector import WorkerInspector + from background_tasks.worker.inspector import WorkerInspector celery = MagicMock(spec=Celery) celery.connection.return_value.ensure_connection.return_value = None diff --git a/modules/background_tasks/tests/test_workers_endpoints.py b/modules/background_tasks/tests/worker/inspector/test_background_tasks_workers_endpoints.py similarity index 96% rename from modules/background_tasks/tests/test_workers_endpoints.py rename to modules/background_tasks/tests/worker/inspector/test_background_tasks_workers_endpoints.py index 4bc177461..42b1c3d31 100644 --- a/modules/background_tasks/tests/test_workers_endpoints.py +++ b/modules/background_tasks/tests/worker/inspector/test_background_tasks_workers_endpoints.py @@ -6,8 +6,8 @@ import httpx import pytest -from background_tasks import worker_inspector as wi from background_tasks.contracts.schemas import WorkerInfo, WorkerSnapshot +from background_tasks.worker import inspector as wi from simple_module_test import InertiaPage JSON_BASE = "/api/background_tasks/admin" @@ -108,7 +108,7 @@ async def test_routes_prop_lists_effective_routes( ): monkeypatch.setattr(wi.WorkerInspector, "snapshot", lambda self: fake_snapshot) monkeypatch.setattr( - "background_tasks.endpoints.views.describe_routes", + "background_tasks.web.endpoints.views.describe_routes", lambda settings: [{"pattern": "x.*", "queue": "q", "module": "x"}], ) resp = await authenticated_client.get( @@ -127,7 +127,7 @@ async def test_route_queues_are_listed_after_the_default( ): monkeypatch.setattr(wi.WorkerInspector, "snapshot", lambda self: fake_snapshot) monkeypatch.setattr( - "background_tasks.endpoints.views.describe_routes", + "background_tasks.web.endpoints.views.describe_routes", lambda _s: [ {"pattern": "a.*", "queue": "heavy", "module": "a"}, {"pattern": "b.*", "queue": "", "module": "b"}, diff --git a/modules/background_tasks/tests/test_workers_uptime.py b/modules/background_tasks/tests/worker/inspector/test_background_tasks_workers_uptime.py similarity index 95% rename from modules/background_tasks/tests/test_workers_uptime.py rename to modules/background_tasks/tests/worker/inspector/test_background_tasks_workers_uptime.py index 01ba57dc2..8a4803454 100644 --- a/modules/background_tasks/tests/test_workers_uptime.py +++ b/modules/background_tasks/tests/worker/inspector/test_background_tasks_workers_uptime.py @@ -15,9 +15,9 @@ import httpx import pytest -from background_tasks import worker_inspector as wi from background_tasks.contracts.schemas import WorkerInfo, WorkerSnapshot -from background_tasks.worker_inspector import redact_broker_url +from background_tasks.worker import inspector as wi +from background_tasks.worker.inspector import redact_broker_url from simple_module_test import InertiaPage VIEW_WORKERS = "/admin/background-tasks/workers" @@ -197,7 +197,7 @@ class TestSnapshotErrorRedaction: """ def test_a_password_inside_the_message_is_stripped(self) -> None: - from background_tasks.worker_inspector import redact_urls + from background_tasks.worker.inspector import redact_urls message = "Cannot connect to redis://bob:hunter2@redis.internal:6379/4: timed out" @@ -206,7 +206,7 @@ def test_a_password_inside_the_message_is_stripped(self) -> None: ) def test_every_url_in_the_message_is_covered(self) -> None: - from background_tasks.worker_inspector import redact_urls + from background_tasks.worker.inspector import redact_urls redacted = redact_urls("broker redis://:a@h:1/0 and redis://:b@h:1/1 both refused") @@ -214,19 +214,19 @@ def test_every_url_in_the_message_is_covered(self) -> None: assert redacted.count("***") == 2 def test_a_message_with_no_url_is_untouched(self) -> None: - from background_tasks.worker_inspector import redact_urls + from background_tasks.worker.inspector import redact_urls assert redact_urls("[Errno 111] Connection refused") == "[Errno 111] Connection refused" def test_none_stays_none(self) -> None: - from background_tasks.worker_inspector import redact_urls + from background_tasks.worker.inspector import redact_urls assert redact_urls(None) is None def test_an_unreachable_broker_reports_a_redacted_error(self, monkeypatch) -> None: """The inspector redacts at the source, so nothing downstream can forget to — the view, the JSON snapshot and Doctor all read this.""" - from background_tasks.worker_inspector import WorkerInspector + from background_tasks.worker.inspector import WorkerInspector from kombu.exceptions import OperationalError class _Celery: diff --git a/modules/background_tasks/tests/test_bg_routes_override_api.py b/modules/background_tasks/tests/worker/runner/test_background_tasks_routes_override_api.py similarity index 96% rename from modules/background_tasks/tests/test_bg_routes_override_api.py rename to modules/background_tasks/tests/worker/runner/test_background_tasks_routes_override_api.py index 42977eb0d..45b266b1d 100644 --- a/modules/background_tasks/tests/test_bg_routes_override_api.py +++ b/modules/background_tasks/tests/worker/runner/test_background_tasks_routes_override_api.py @@ -3,7 +3,7 @@ from __future__ import annotations import pytest -from background_tasks.celery_app import describe_routes +from background_tasks.worker.celery_app import describe_routes URL = "/api/settings/modules/background_tasks" ROUTES = '{"background_tasks.demo_echo": "heavy"}' diff --git a/modules/background_tasks/tests/test_bg_runner_settings.py b/modules/background_tasks/tests/worker/runner/test_background_tasks_runner_settings.py similarity index 93% rename from modules/background_tasks/tests/test_bg_runner_settings.py rename to modules/background_tasks/tests/worker/runner/test_background_tasks_runner_settings.py index ab5602d89..1095fb90f 100644 --- a/modules/background_tasks/tests/test_bg_runner_settings.py +++ b/modules/background_tasks/tests/worker/runner/test_background_tasks_runner_settings.py @@ -6,9 +6,9 @@ from collections.abc import Iterator import pytest -from background_tasks import sync_db -from background_tasks.settings import BackgroundTasksSettings -from background_tasks.worker_settings import clear_settings_cache, load_runner_settings +from background_tasks.core import sync_db +from background_tasks.core.settings import BackgroundTasksSettings +from background_tasks.worker.settings import clear_settings_cache, load_runner_settings from simple_module_db.stored_settings import Setting from sqlalchemy import create_engine from sqlalchemy.orm import Session @@ -88,7 +88,7 @@ def test_unreachable_db_falls_back_to_env_settings(monkeypatch, caplog) -> None: def boom(*a, **k): raise RuntimeError("no database") - monkeypatch.setattr("background_tasks.worker_settings.settings_for", boom) + monkeypatch.setattr("background_tasks.worker.settings.settings_for", boom) monkeypatch.setenv("SM_BG_TASKS_TASK_TIME_LIMIT", "12") cfg = load_runner_settings() assert isinstance(cfg, BackgroundTasksSettings) @@ -108,8 +108,8 @@ def test_stored_invalid_task_routes_do_not_stop_the_worker(db_url: str, caplog) def test_shared_runner_builds_celery_from_db_settings(db_url, tmp_path, monkeypatch) -> None: """GH #490: the entry point scaffolded apps use must hydrate from the DB.""" - from background_tasks import tenant_context - from background_tasks.runner import build_runner_celery + from background_tasks.executions import tenant_context + from background_tasks.worker.runner import build_runner_celery from simple_module_core import leases from simple_module_db.leases import DbLeaseBackend @@ -142,7 +142,7 @@ def test_fallback_env_file_is_adopted_as_the_project_root(tmp_path, monkeypatch) """Launched from inside some other checkout, the walk-up stops at *its* missing ``.env``; the script-relative file must still win, and the sqlite anchor must follow it rather than the cwd (GH #490 review).""" - from background_tasks.runner import _load_env + from background_tasks.worker.runner import _load_env from simple_module_hosting.bootstrap_settings import BootstrapSettings app = tmp_path / "app" @@ -163,7 +163,7 @@ def test_fallback_env_file_is_adopted_as_the_project_root(tmp_path, monkeypatch) def test_explicit_project_root_is_never_overridden(tmp_path, monkeypatch) -> None: - from background_tasks.runner import _load_env + from background_tasks.worker.runner import _load_env app = tmp_path / "app" app.mkdir() diff --git a/modules/background_tasks/tests/test_bg_task_routing.py b/modules/background_tasks/tests/worker/runner/test_background_tasks_task_routing.py similarity index 97% rename from modules/background_tasks/tests/test_bg_task_routing.py rename to modules/background_tasks/tests/worker/runner/test_background_tasks_task_routing.py index 281fc9f36..e3ade88d7 100644 --- a/modules/background_tasks/tests/test_bg_task_routing.py +++ b/modules/background_tasks/tests/worker/runner/test_background_tasks_task_routing.py @@ -7,10 +7,10 @@ import types import pytest -from background_tasks import celery_app -from background_tasks.config_guards import WorkerScope -from background_tasks.settings import BackgroundTasksSettings -from background_tasks.task_routing import ( +from background_tasks.core.config_guards import WorkerScope +from background_tasks.core.settings import BackgroundTasksSettings +from background_tasks.worker import celery_app +from background_tasks.worker.task_routing import ( ModuleTaskConfig, load_task_config, merge_routes, @@ -135,7 +135,7 @@ def test_include_beats_queue_filter_and_exclude_beats_include(self): class TestSettings: def test_scope_is_read_from_env_only(self, monkeypatch): - from background_tasks.config_guards import worker_scope_from_env + from background_tasks.core.config_guards import worker_scope_from_env monkeypatch.setenv("SM_BG_TASKS_WORKER_QUEUES", "gpu, default") monkeypatch.setenv("SM_BG_TASKS_EXCLUDE_MODULES", "a,b") diff --git a/modules/background_tasks/tests/test_bg_worker_settings.py b/modules/background_tasks/tests/worker/runner/test_background_tasks_worker_settings.py similarity index 94% rename from modules/background_tasks/tests/test_bg_worker_settings.py rename to modules/background_tasks/tests/worker/runner/test_background_tasks_worker_settings.py index 2d30411cd..11a9aa431 100644 --- a/modules/background_tasks/tests/test_bg_worker_settings.py +++ b/modules/background_tasks/tests/worker/runner/test_background_tasks_worker_settings.py @@ -5,8 +5,8 @@ from collections.abc import Iterator import pytest -from background_tasks import sync_db -from background_tasks.worker_settings import clear_settings_cache, settings_for +from background_tasks.core import sync_db +from background_tasks.worker.settings import clear_settings_cache, settings_for from pydantic import Field from simple_module_core.settings_base import DbBackedSettings from simple_module_db.stored_settings import Setting diff --git a/modules/background_tasks/tests/test_background_tasks_argument_json.py b/modules/background_tasks/tests/worker/signals/test_background_tasks_argument_json.py similarity index 94% rename from modules/background_tasks/tests/test_background_tasks_argument_json.py rename to modules/background_tasks/tests/worker/signals/test_background_tasks_argument_json.py index 0438b4fb3..e34c76d9d 100644 --- a/modules/background_tasks/tests/test_background_tasks_argument_json.py +++ b/modules/background_tasks/tests/worker/signals/test_background_tasks_argument_json.py @@ -5,11 +5,11 @@ from uuid import UUID import pytest -from background_tasks import sync_db -from background_tasks.constants import TaskStatus +from background_tasks.core import sync_db +from background_tasks.core.constants import TaskStatus +from background_tasks.executions.retry_service import RetryCoordinator from background_tasks.models import TaskExecution -from background_tasks.retry_service import RetryCoordinator -from background_tasks.signals import on_task_publish, on_task_success +from background_tasks.worker.signals import on_task_publish, on_task_success from celery import Celery from simple_module_core.events import EventBus from sqlalchemy import select diff --git a/modules/background_tasks/tests/test_background_tasks_execution_bindings.py b/modules/background_tasks/tests/worker/signals/test_background_tasks_execution_bindings.py similarity index 92% rename from modules/background_tasks/tests/test_background_tasks_execution_bindings.py rename to modules/background_tasks/tests/worker/signals/test_background_tasks_execution_bindings.py index d7c8872ee..657a56f8e 100644 --- a/modules/background_tasks/tests/test_background_tasks_execution_bindings.py +++ b/modules/background_tasks/tests/worker/signals/test_background_tasks_execution_bindings.py @@ -4,10 +4,10 @@ from types import SimpleNamespace import pytest -from background_tasks import tenant_context as task_context -from background_tasks.log_context import get_log_context -from background_tasks.signals import on_task_postrun, on_task_prerun -from background_tasks.tenant_context import OWNER_HEADER, TENANT_HEADER, stamp_owner +from background_tasks.executions import tenant_context as task_context +from background_tasks.executions.tenant_context import OWNER_HEADER, TENANT_HEADER, stamp_owner +from background_tasks.worker.log_context import get_log_context +from background_tasks.worker.signals import on_task_postrun, on_task_prerun from celery import Celery from simple_module_db import current_tenant_id, tenant_context from simple_module_db.listeners import current_user_id diff --git a/modules/background_tasks/tests/test_background_tasks_heartbeat.py b/modules/background_tasks/tests/worker/signals/test_background_tasks_heartbeat.py similarity index 96% rename from modules/background_tasks/tests/test_background_tasks_heartbeat.py rename to modules/background_tasks/tests/worker/signals/test_background_tasks_heartbeat.py index f479120a1..bd83aff83 100644 --- a/modules/background_tasks/tests/test_background_tasks_heartbeat.py +++ b/modules/background_tasks/tests/worker/signals/test_background_tasks_heartbeat.py @@ -14,11 +14,12 @@ from datetime import UTC, datetime, timedelta import pytest -from background_tasks import heartbeat, sync_db -from background_tasks import task_heartbeat as heartbeat_mod -from background_tasks.constants import TASK_STUCK_AFTER_ATTR, TaskStatus +from background_tasks import heartbeat +from background_tasks.core import sync_db +from background_tasks.core.constants import TASK_STUCK_AFTER_ATTR, TaskStatus from background_tasks.models import TaskExecution from background_tasks.tasks import stuck_thresholds, sweep_stuck_tasks +from background_tasks.worker import task_heartbeat as heartbeat_mod from celery import Celery, shared_task from sqlalchemy import select diff --git a/modules/background_tasks/tests/test_bg_quiet_sweep_unrecorded.py b/modules/background_tasks/tests/worker/signals/test_background_tasks_quiet_sweep_unrecorded.py similarity index 85% rename from modules/background_tasks/tests/test_bg_quiet_sweep_unrecorded.py rename to modules/background_tasks/tests/worker/signals/test_background_tasks_quiet_sweep_unrecorded.py index afdb123ae..3dea25881 100644 --- a/modules/background_tasks/tests/test_bg_quiet_sweep_unrecorded.py +++ b/modules/background_tasks/tests/worker/signals/test_background_tasks_quiet_sweep_unrecorded.py @@ -5,12 +5,12 @@ import uuid from pathlib import Path -from background_tasks import sync_db -from background_tasks.constants import INTERNAL_TASK_SWEEP_OUTBOX +from background_tasks.core import sync_db +from background_tasks.core.constants import INTERNAL_TASK_SWEEP_OUTBOX from background_tasks.models import TaskExecution -from background_tasks.signals import on_task_failure, on_task_publish, on_task_success +from background_tasks.worker.signals import on_task_failure, on_task_publish, on_task_success from sqlalchemy import select -from test_signals import sync_sqlite # noqa: F401 +from test_background_tasks_signals import sync_sqlite # noqa: F401 def _rows() -> list[TaskExecution]: @@ -54,7 +54,7 @@ def test_other_tasks_still_recorded(sync_sqlite: Path): # noqa: F811 def test_timeout_over_lease_error_is_located_on_the_field(): import pytest - from background_tasks.settings import BackgroundTasksSettings + from background_tasks.core.settings import BackgroundTasksSettings from pydantic import ValidationError with pytest.raises(ValidationError) as exc: diff --git a/modules/background_tasks/tests/test_background_tasks_resource_status.py b/modules/background_tasks/tests/worker/signals/test_background_tasks_resource_status.py similarity index 99% rename from modules/background_tasks/tests/test_background_tasks_resource_status.py rename to modules/background_tasks/tests/worker/signals/test_background_tasks_resource_status.py index 54e630218..8262a63b0 100644 --- a/modules/background_tasks/tests/test_background_tasks_resource_status.py +++ b/modules/background_tasks/tests/worker/signals/test_background_tasks_resource_status.py @@ -1,7 +1,7 @@ """Task resource grants use filtered policies and stay tenant scoped.""" import pytest -from background_tasks.constants import TaskStatus +from background_tasks.core.constants import TaskStatus from background_tasks.models import TaskExecution from simple_module_core.resources import ResourceType from simple_module_core.tenancy import TenantRole diff --git a/modules/background_tasks/tests/test_background_tasks_resource_transport.py b/modules/background_tasks/tests/worker/signals/test_background_tasks_resource_transport.py similarity index 91% rename from modules/background_tasks/tests/test_background_tasks_resource_transport.py rename to modules/background_tasks/tests/worker/signals/test_background_tasks_resource_transport.py index bb6b8c23e..075ed344f 100644 --- a/modules/background_tasks/tests/test_background_tasks_resource_transport.py +++ b/modules/background_tasks/tests/worker/signals/test_background_tasks_resource_transport.py @@ -3,10 +3,10 @@ from types import SimpleNamespace import pytest -from background_tasks import sync_db -from background_tasks.constants import SM_TRACKING_HEADER +from background_tasks.core import sync_db +from background_tasks.core.constants import SM_TRACKING_HEADER from background_tasks.models import TaskExecution -from background_tasks.signals import on_task_failure, on_task_publish +from background_tasks.worker.signals import on_task_failure, on_task_publish from sqlalchemy import select @@ -87,9 +87,9 @@ def test_resource_pair_is_required_by_database(sync_sqlite): def test_off_tracking_keeps_owner_tenant_but_never_inherits_resource(monkeypatch): - from background_tasks import _signal_support - from background_tasks.signals import on_task_postrun, on_task_prerun - from background_tasks.tenant_context import stamp_owner, stamp_tenant + from background_tasks.executions.tenant_context import stamp_owner, stamp_tenant + from background_tasks.worker import _signal_support + from background_tasks.worker.signals import on_task_postrun, on_task_prerun from simple_module_db import current_tenant_id def forbidden(): diff --git a/modules/background_tasks/tests/test_background_tasks_result_json.py b/modules/background_tasks/tests/worker/signals/test_background_tasks_result_json.py similarity index 90% rename from modules/background_tasks/tests/test_background_tasks_result_json.py rename to modules/background_tasks/tests/worker/signals/test_background_tasks_result_json.py index f66632b57..b56e486d6 100644 --- a/modules/background_tasks/tests/test_background_tasks_result_json.py +++ b/modules/background_tasks/tests/worker/signals/test_background_tasks_result_json.py @@ -4,10 +4,11 @@ from types import SimpleNamespace import pytest -from background_tasks import sync_db, tasks -from background_tasks.constants import TaskStatus +from background_tasks import tasks +from background_tasks.core import sync_db +from background_tasks.core.constants import TaskStatus from background_tasks.models import TaskExecution -from background_tasks.signals import on_task_success +from background_tasks.worker.signals import on_task_success from celery import Celery from sqlalchemy import select @@ -71,7 +72,7 @@ def job(): def test_result_whose_repr_raises_is_still_storable(): - from background_tasks._signal_support import jsonable_result + from background_tasks.worker._signal_support import jsonable_result class Hostile: def __repr__(self): diff --git a/modules/background_tasks/tests/test_signal_tenant_stamp.py b/modules/background_tasks/tests/worker/signals/test_background_tasks_signal_tenant_stamp.py similarity index 95% rename from modules/background_tasks/tests/test_signal_tenant_stamp.py rename to modules/background_tasks/tests/worker/signals/test_background_tasks_signal_tenant_stamp.py index 757e32560..fee3babf1 100644 --- a/modules/background_tasks/tests/test_signal_tenant_stamp.py +++ b/modules/background_tasks/tests/worker/signals/test_background_tasks_signal_tenant_stamp.py @@ -12,9 +12,10 @@ from pathlib import Path from types import SimpleNamespace -from background_tasks import sync_db +from background_tasks.core import sync_db +from background_tasks.executions.tenant_context import TENANT_HEADER from background_tasks.models import TaskExecution -from background_tasks.signals import ( +from background_tasks.worker.signals import ( on_task_failure, on_task_postrun, on_task_prerun, @@ -23,7 +24,6 @@ on_task_revoked, on_task_success, ) -from background_tasks.tenant_context import TENANT_HEADER from simple_module_db import tenant_context from sqlalchemy import select diff --git a/modules/background_tasks/tests/test_signals.py b/modules/background_tasks/tests/worker/signals/test_background_tasks_signals.py similarity index 93% rename from modules/background_tasks/tests/test_signals.py rename to modules/background_tasks/tests/worker/signals/test_background_tasks_signals.py index 88dd92e3d..5fc0ec525 100644 --- a/modules/background_tasks/tests/test_signals.py +++ b/modules/background_tasks/tests/worker/signals/test_background_tasks_signals.py @@ -1,6 +1,6 @@ """Tests for Celery signal handlers: upsert path and terminal transitions. -Signals are sync, so we point :mod:`background_tasks.sync_db` at a temporary +Signals are sync, so we point :mod:`background_tasks.core.sync_db` at a temporary SQLite file via ``SM_DATABASE_URL`` for the duration of each test. The module-level engine cache is also reset so each test gets a fresh engine. """ @@ -13,13 +13,14 @@ from pathlib import Path import pytest -from background_tasks import _signal_bus as bg_signals_bus -from background_tasks import sync_db -from background_tasks._signal_support import upsert_by_celery_id -from background_tasks.constants import TaskStatus from background_tasks.contracts.events import TaskFailed +from background_tasks.core import sync_db +from background_tasks.core.constants import TaskStatus +from background_tasks.executions.tenant_context import release_tenant from background_tasks.models import TaskExecution -from background_tasks.signals import ( +from background_tasks.worker import _signal_bus as bg_signals_bus +from background_tasks.worker._signal_support import upsert_by_celery_id +from background_tasks.worker.signals import ( bind_event_bus, on_task_failure, on_task_prerun, @@ -94,6 +95,9 @@ class _Task: name = "demo.echo" on_task_prerun(sender=_Task, task_id=task_id, task=_Task(), args=[], kwargs={}) + # No postrun here: unbind the tenant/owner prerun entered so it cannot leak into + # tests that run after this one (the nested layout reorders files). + release_tenant(task_id=task_id) factory = sync_db.get_sync_session_factory() row = _first_row(factory) assert row is not None diff --git a/modules/background_tasks/tests/test_sweep_stuck.py b/modules/background_tasks/tests/worker/signals/test_background_tasks_sweep_stuck.py similarity index 97% rename from modules/background_tasks/tests/test_sweep_stuck.py rename to modules/background_tasks/tests/worker/signals/test_background_tasks_sweep_stuck.py index df6226478..ea25c0671 100644 --- a/modules/background_tasks/tests/test_sweep_stuck.py +++ b/modules/background_tasks/tests/worker/signals/test_background_tasks_sweep_stuck.py @@ -13,8 +13,8 @@ from datetime import UTC, datetime, timedelta import pytest -from background_tasks import sync_db -from background_tasks.constants import TaskStatus +from background_tasks.core import sync_db +from background_tasks.core.constants import TaskStatus from background_tasks.models import TaskExecution from background_tasks.tasks import sweep_stuck_tasks from sqlalchemy import select diff --git a/modules/background_tasks/tests/test_bg_task_progress.py b/modules/background_tasks/tests/worker/signals/test_background_tasks_task_progress.py similarity index 92% rename from modules/background_tasks/tests/test_bg_task_progress.py rename to modules/background_tasks/tests/worker/signals/test_background_tasks_task_progress.py index a7118d5af..89c13ee7e 100644 --- a/modules/background_tasks/tests/test_bg_task_progress.py +++ b/modules/background_tasks/tests/worker/signals/test_background_tasks_task_progress.py @@ -7,12 +7,13 @@ from types import SimpleNamespace import pytest -from background_tasks import heartbeat, sync_db -from background_tasks import task_heartbeat as heartbeat_mod -from background_tasks.constants import TaskStatus +from background_tasks import heartbeat +from background_tasks.core import sync_db +from background_tasks.core.constants import TaskStatus +from background_tasks.executions.tenant_context import OWNER_HEADER, publish_as_owner from background_tasks.models import TaskExecution -from background_tasks.signals import on_task_postrun, on_task_prerun, on_task_publish -from background_tasks.tenant_context import OWNER_HEADER, publish_as_owner +from background_tasks.worker import task_heartbeat as heartbeat_mod +from background_tasks.worker.signals import on_task_postrun, on_task_prerun, on_task_publish from simple_module_db.listeners import current_user_id from sqlalchemy import select diff --git a/modules/background_tasks/tests/test_bg_task_progress_audit.py b/modules/background_tasks/tests/worker/signals/test_background_tasks_task_progress_audit.py similarity index 89% rename from modules/background_tasks/tests/test_bg_task_progress_audit.py rename to modules/background_tasks/tests/worker/signals/test_background_tasks_task_progress_audit.py index 841c60fc7..fd7e087b9 100644 --- a/modules/background_tasks/tests/test_bg_task_progress_audit.py +++ b/modules/background_tasks/tests/worker/signals/test_background_tasks_task_progress_audit.py @@ -6,10 +6,11 @@ from collections.abc import Iterator import pytest -from background_tasks import heartbeat, sync_db -from background_tasks import task_heartbeat as heartbeat_mod -from background_tasks.constants import TaskStatus +from background_tasks import heartbeat +from background_tasks.core import sync_db +from background_tasks.core.constants import TaskStatus from background_tasks.models import TaskExecution +from background_tasks.worker import task_heartbeat as heartbeat_mod from simple_module_db.audit_sinks import discover_audit_callback from sqlalchemy import create_engine, func, select diff --git a/modules/background_tasks/tests/test_background_tasks_tracking_modes.py b/modules/background_tasks/tests/worker/signals/test_background_tasks_tracking_modes.py similarity index 92% rename from modules/background_tasks/tests/test_background_tasks_tracking_modes.py rename to modules/background_tasks/tests/worker/signals/test_background_tasks_tracking_modes.py index 8e751a251..95c1c6a9e 100644 --- a/modules/background_tasks/tests/test_background_tasks_tracking_modes.py +++ b/modules/background_tasks/tests/worker/signals/test_background_tasks_tracking_modes.py @@ -4,9 +4,9 @@ from unittest.mock import Mock import pytest -from background_tasks import sync_db +from background_tasks.core import sync_db from background_tasks.models import TaskExecution -from background_tasks.signals import ( +from background_tasks.worker.signals import ( on_task_failure, on_task_postrun, on_task_prerun, @@ -32,7 +32,7 @@ ], ) def test_lifecycle_writes(mode, outcome, writes, sync_sqlite, monkeypatch): - from background_tasks import _signal_support + from background_tasks.worker import _signal_support from simple_module_db import current_tenant_id opened = Mock(wraps=_signal_support.sync_session) @@ -77,7 +77,7 @@ def test_lifecycle_writes(mode, outcome, writes, sync_sqlite, monkeypatch): @pytest.mark.parametrize("mode", ["terminal", "failures", "off"]) def test_retries_and_heartbeat_do_not_open_sessions(mode, monkeypatch): - from background_tasks import _signal_support, task_heartbeat + from background_tasks.worker import _signal_support, task_heartbeat request = SimpleNamespace(id="retry-test", called_directly=False, headers={"sm_tracking": mode}) task = SimpleNamespace(name="demo.retry", request=request) @@ -93,7 +93,7 @@ def forbidden(): def test_tracking_settings_are_db_only(monkeypatch): - from background_tasks.settings import BackgroundTasksSettings + from background_tasks.core.settings import BackgroundTasksSettings monkeypatch.setenv("SM_BG_TASKS_EXECUTION_TRACKING", "off") monkeypatch.setenv("SM_BG_TASKS_TRACK_PUBLISH", "false") @@ -104,7 +104,7 @@ def test_tracking_settings_are_db_only(monkeypatch): @pytest.mark.parametrize("track_publish", [True, False]) def test_worker_recovers_metadata_without_publish_tracking(track_publish, sync_sqlite, monkeypatch): - from background_tasks import _signal_support + from background_tasks.worker import _signal_support from celery import current_app monkeypatch.setitem(current_app.conf, "sm_track_publish", track_publish) @@ -139,7 +139,7 @@ def test_worker_recovers_metadata_without_publish_tracking(track_publish, sync_s def test_resolution_precedence_and_invalid_override(monkeypatch): from background_tasks import SM_TRACKING_HEADER, TrackingMode - from background_tasks.tracking import tracking_mode + from background_tasks.core.tracking import tracking_mode from celery import current_app monkeypatch.setitem(current_app.conf, "sm_execution_tracking", "terminal") @@ -174,8 +174,8 @@ def work(): def test_db_hydration_and_frozen_publish_settings(monkeypatch): - from background_tasks import _signal_support, celery_app - from background_tasks.settings import BackgroundTasksSettings + from background_tasks.core.settings import BackgroundTasksSettings + from background_tasks.worker import _signal_support, celery_app from celery import current_app from simple_module_core.settings_hydrate import build_settings, env_supplied_fields @@ -217,8 +217,8 @@ def task(): def test_postrun_always_cleans_context_on_tracking_error(monkeypatch): - from background_tasks import signals, task_heartbeat - from background_tasks.tenant_context import release_tenant + from background_tasks.executions.tenant_context import release_tenant + from background_tasks.worker import signals, task_heartbeat from simple_module_db import current_tenant_id task = SimpleNamespace( diff --git a/modules/background_tasks/tests/test_bg_worker_audit.py b/modules/background_tasks/tests/worker/signals/test_background_tasks_worker_audit.py similarity index 96% rename from modules/background_tasks/tests/test_bg_worker_audit.py rename to modules/background_tasks/tests/worker/signals/test_background_tasks_worker_audit.py index 755376d2f..fe4b09a61 100644 --- a/modules/background_tasks/tests/test_bg_worker_audit.py +++ b/modules/background_tasks/tests/worker/signals/test_background_tasks_worker_audit.py @@ -11,7 +11,7 @@ from collections.abc import Iterator import pytest -from background_tasks import sync_db +from background_tasks.core import sync_db from celery import Celery from simple_module_db import create_module_base from simple_module_db.audit_sinks import discover_audit_callback @@ -111,8 +111,8 @@ def test_without_a_sink_the_worker_writes_no_audit_rows(worker_db, monkeypatch) def test_a_durable_handlers_write_on_the_outbox_engine_is_audited(worker_db) -> None: """The outbox worker builds its own async engine; it carries the sink too.""" - from background_tasks.outbox_loop import run_on_worker_loop, stop_worker_loop - from background_tasks.outbox_worker import worker_session_factory + from background_tasks.outbox.loop import run_on_worker_loop, stop_worker_loop + from background_tasks.outbox.worker import worker_session_factory factory = worker_session_factory() diff --git a/modules/background_tasks/tests/test_bg_worker_audit_bookkeeping.py b/modules/background_tasks/tests/worker/signals/test_background_tasks_worker_audit_bookkeeping.py similarity index 95% rename from modules/background_tasks/tests/test_bg_worker_audit_bookkeeping.py rename to modules/background_tasks/tests/worker/signals/test_background_tasks_worker_audit_bookkeeping.py index 63b80437e..c710aef45 100644 --- a/modules/background_tasks/tests/test_bg_worker_audit_bookkeeping.py +++ b/modules/background_tasks/tests/worker/signals/test_background_tasks_worker_audit_bookkeeping.py @@ -11,10 +11,10 @@ from collections.abc import Iterator import pytest -from background_tasks import sync_db -from background_tasks._signal_support import now_utc -from background_tasks.constants import TaskStatus +from background_tasks.core import sync_db +from background_tasks.core.constants import TaskStatus from background_tasks.models import Base, OutboxMessage, TaskExecution +from background_tasks.worker._signal_support import now_utc from simple_module_db.audit_sinks import discover_audit_callback from sqlalchemy import create_engine, select diff --git a/modules/background_tasks/tests/test_background_tasks_custom_bases.py b/modules/background_tasks/tests/worker/tenancy/test_background_tasks_custom_bases.py similarity index 93% rename from modules/background_tasks/tests/test_background_tasks_custom_bases.py rename to modules/background_tasks/tests/worker/tenancy/test_background_tasks_custom_bases.py index 678f1aee4..857fb8832 100644 --- a/modules/background_tasks/tests/test_background_tasks_custom_bases.py +++ b/modules/background_tasks/tests/worker/tenancy/test_background_tasks_custom_bases.py @@ -3,9 +3,9 @@ from uuid import uuid4 import pytest -from background_tasks import signals as _signals # noqa: F401 - connects the handlers -from background_tasks.tenant_celery import TenantCelery, TenantTask -from background_tasks.tenant_context import OWNER_HEADER, TENANT_HEADER +from background_tasks.executions.tenant_context import OWNER_HEADER, TENANT_HEADER +from background_tasks.worker import signals as _signals # noqa: F401 - connects the handlers +from background_tasks.worker.tenant_celery import TenantCelery, TenantTask from celery import Task, _state, current_app, shared_task from simple_module_db import TenantIsolationError, current_tenant_id, tenant_context from simple_module_db.listeners import current_user_id diff --git a/modules/background_tasks/tests/test_execution_tenancy.py b/modules/background_tasks/tests/worker/tenancy/test_background_tasks_execution_tenancy.py similarity index 95% rename from modules/background_tasks/tests/test_execution_tenancy.py rename to modules/background_tasks/tests/worker/tenancy/test_background_tasks_execution_tenancy.py index 6338baec0..c89fde3a7 100644 --- a/modules/background_tasks/tests/test_execution_tenancy.py +++ b/modules/background_tasks/tests/worker/tenancy/test_background_tasks_execution_tenancy.py @@ -13,12 +13,12 @@ import httpx import pytest -from background_tasks import sync_db -from background_tasks.constants import PLATFORM_TENANT_FILTER, TaskStatus +from background_tasks.core import sync_db +from background_tasks.core.constants import PLATFORM_TENANT_FILTER, TaskStatus +from background_tasks.executions.retry_service import RetryCoordinator +from background_tasks.executions.tenant_context import TENANT_HEADER from background_tasks.models import TaskExecution -from background_tasks.retry_service import RetryCoordinator -from background_tasks.signals import on_task_publish -from background_tasks.tenant_context import TENANT_HEADER +from background_tasks.worker.signals import on_task_publish from simple_module_core.tenancy import TenantRole from simple_module_db import current_tenant_id, tenant_context from sqlalchemy import select diff --git a/modules/background_tasks/tests/test_tenant_context.py b/modules/background_tasks/tests/worker/tenancy/test_background_tasks_tenant_context.py similarity index 97% rename from modules/background_tasks/tests/test_tenant_context.py rename to modules/background_tasks/tests/worker/tenancy/test_background_tasks_tenant_context.py index 84da65394..c4ff18c41 100644 --- a/modules/background_tasks/tests/test_tenant_context.py +++ b/modules/background_tasks/tests/worker/tenancy/test_background_tasks_tenant_context.py @@ -5,7 +5,7 @@ from types import SimpleNamespace import pytest -from background_tasks.tenant_context import ( +from background_tasks.executions.tenant_context import ( TENANT_HEADER, release_tenant, restore_tenant, diff --git a/modules/background_tasks/tests/test_background_tasks_worker_tenancy.py b/modules/background_tasks/tests/worker/tenancy/test_background_tasks_worker_tenancy.py similarity index 96% rename from modules/background_tasks/tests/test_background_tasks_worker_tenancy.py rename to modules/background_tasks/tests/worker/tenancy/test_background_tasks_worker_tenancy.py index 4a9a4eeda..9ef66fbfd 100644 --- a/modules/background_tasks/tests/test_background_tasks_worker_tenancy.py +++ b/modules/background_tasks/tests/worker/tenancy/test_background_tasks_worker_tenancy.py @@ -1,7 +1,7 @@ """Worker tenancy is public and distinguishes unconfigured from single tenant (#685).""" import pytest -from background_tasks import sync_db +from background_tasks.core import sync_db @pytest.fixture(autouse=True) diff --git a/modules/background_tasks/tests/test_worker_tenancy.py b/modules/background_tasks/tests/worker/tenancy/test_background_tasks_worker_tenancy_context.py similarity index 96% rename from modules/background_tasks/tests/test_worker_tenancy.py rename to modules/background_tasks/tests/worker/tenancy/test_background_tasks_worker_tenancy_context.py index 9749a81ee..b3b05b0ff 100644 --- a/modules/background_tasks/tests/test_worker_tenancy.py +++ b/modules/background_tasks/tests/worker/tenancy/test_background_tasks_worker_tenancy_context.py @@ -5,8 +5,8 @@ from types import SimpleNamespace import pytest -from background_tasks import sync_db -from background_tasks.tenant_context import TENANT_HEADER, release_tenant, restore_tenant +from background_tasks.core import sync_db +from background_tasks.executions.tenant_context import TENANT_HEADER, release_tenant, restore_tenant from simple_module_db import ( DEFAULT_TENANT_ID, MultiTenantMixin, diff --git a/modules/file_storage/file_storage/core/worker.py b/modules/file_storage/file_storage/core/worker.py index 9d37d99d0..248dd0bc5 100644 --- a/modules/file_storage/file_storage/core/worker.py +++ b/modules/file_storage/file_storage/core/worker.py @@ -16,8 +16,8 @@ from typing import BinaryIO from uuid import UUID -from background_tasks.outbox_loop import run_on_worker_loop -from background_tasks.outbox_worker import worker_bus, worker_session_factory +from background_tasks.outbox.loop import run_on_worker_loop +from background_tasks.outbox.worker import worker_bus, worker_session_factory from fastapi import UploadFile from simple_module_db import independent_session from simple_module_db.callbacks import register_on_commit diff --git a/modules/file_storage/file_storage/core/worker_support.py b/modules/file_storage/file_storage/core/worker_support.py index f382ab57e..9b2970778 100644 --- a/modules/file_storage/file_storage/core/worker_support.py +++ b/modules/file_storage/file_storage/core/worker_support.py @@ -7,7 +7,7 @@ def backend_and_settings() -> tuple[StorageBackend, FileStorageSettings]: - from background_tasks.worker_settings import settings_for + from background_tasks.worker.settings import settings_for settings = settings_for(FileStorageSettings, constants.MODULE_NAME) return build_backend(settings), settings diff --git a/modules/file_storage/file_storage/tasks.py b/modules/file_storage/file_storage/tasks.py index f32090f05..d828b83b4 100644 --- a/modules/file_storage/file_storage/tasks.py +++ b/modules/file_storage/file_storage/tasks.py @@ -52,7 +52,7 @@ def _backend_and_settings(): @shared_task(name=constants.Task.REAP_STALE_UPLOADS) def reap_stale_uploads() -> dict[str, int]: """Abort expired sessions, purge old finished rows, then orphaned S3 uploads (> 2 x TTL).""" - from background_tasks.sync_db import sync_session + from background_tasks.core.sync_db import sync_session from simple_module_db import all_tenants from sqlalchemy import delete, select @@ -147,7 +147,7 @@ async def _abort_orphans(backend, older_than: timedelta, tracked: set[str]) -> i def compute_checksum(file_id: str, tenant_id: str) -> str | None: """Fill in ``StoredFile.checksum_sha256`` for a multipart upload.""" from background_tasks import heartbeat - from background_tasks.sync_db import sync_session + from background_tasks.core.sync_db import sync_session from simple_module_db import tenant_context from file_storage.backends.guard import require_backend @@ -182,7 +182,7 @@ def compute_checksum(file_id: str, tenant_id: str) -> str | None: @shared_task(name=constants.Task.REAP_OBJECTS) def reap_objects() -> int: """Reconcile completed objects, with twice the upload TTL as a safety window.""" - from background_tasks.sync_db import sync_session + from background_tasks.core.sync_db import sync_session from file_storage.core.object_reaper import reap_objects as reconcile @@ -204,7 +204,7 @@ def reconcile_checksums() -> int: after publish failures or worker exits; duplicate checksum jobs are safe. """ from background_tasks import enqueue_after_commit - from background_tasks.sync_db import sync_session + from background_tasks.core.sync_db import sync_session from simple_module_db import all_tenants, tenant_context from sqlalchemy import func from sqlmodel import col, select diff --git a/modules/file_storage/tests/_file_storage_worker_support.py b/modules/file_storage/tests/_file_storage_worker_support.py index 9d5361ce8..5c1c613f9 100644 --- a/modules/file_storage/tests/_file_storage_worker_support.py +++ b/modules/file_storage/tests/_file_storage_worker_support.py @@ -1,8 +1,9 @@ """A real worker database and filesystem, without a Celery broker.""" import pytest -from background_tasks import outbox_worker, sync_db -from background_tasks.worker_settings import clear_settings_cache +from background_tasks.core import sync_db +from background_tasks.outbox import worker as outbox_worker +from background_tasks.worker.settings import clear_settings_cache from file_storage.models import StoredFile from simple_module_core.events import EventBus from simple_module_db.stored_settings import Setting diff --git a/modules/file_storage/tests/core/tenant_deletion/test_file_storage_tenant_deletion.py b/modules/file_storage/tests/core/tenant_deletion/test_file_storage_tenant_deletion.py index 8cdb5ac5d..2ed546791 100644 --- a/modules/file_storage/tests/core/tenant_deletion/test_file_storage_tenant_deletion.py +++ b/modules/file_storage/tests/core/tenant_deletion/test_file_storage_tenant_deletion.py @@ -5,11 +5,11 @@ from unittest.mock import AsyncMock import pytest -from background_tasks.constants import OutboxStatus +from background_tasks.core.constants import OutboxStatus from background_tasks.models import OutboxMessage -from background_tasks.outbox_claim import OutboxPolicy -from background_tasks.outbox_dispatch import dispatch -from background_tasks.outbox_transport import WorkerOutboxTransport +from background_tasks.outbox.claim import OutboxPolicy +from background_tasks.outbox.dispatch import dispatch +from background_tasks.outbox.transport import WorkerOutboxTransport from file_storage.backends.filesystem import FilesystemBackend from file_storage.core import constants from file_storage.core.tenant_deletion import register_tenant_deletion diff --git a/modules/file_storage/tests/core/worker/test_file_storage_object_reaper.py b/modules/file_storage/tests/core/worker/test_file_storage_object_reaper.py index 21817b6d6..d543687fe 100644 --- a/modules/file_storage/tests/core/worker/test_file_storage_object_reaper.py +++ b/modules/file_storage/tests/core/worker/test_file_storage_object_reaper.py @@ -5,7 +5,7 @@ import uuid from datetime import timedelta -from background_tasks import sync_db +from background_tasks.core import sync_db from file_storage import tasks from file_storage.core.object_reaper import reap_objects from file_storage.models import StoredFile diff --git a/modules/file_storage/tests/core/worker/test_file_storage_upload_tasks.py b/modules/file_storage/tests/core/worker/test_file_storage_upload_tasks.py index b9c97bb2b..19d8e3c8b 100644 --- a/modules/file_storage/tests/core/worker/test_file_storage_upload_tasks.py +++ b/modules/file_storage/tests/core/worker/test_file_storage_upload_tasks.py @@ -1,7 +1,7 @@ """The janitor and checksum tasks (``file_storage.tasks``) (#454). Tasks run in a worker with its own sync engine, so these point -``background_tasks.sync_db`` at a throwaway SQLite file and swap in a +``background_tasks.core.sync_db`` at a throwaway SQLite file and swap in a filesystem backend, which is all a worker needs to exercise. """ @@ -13,7 +13,7 @@ from datetime import UTC, datetime, timedelta import pytest -from background_tasks import sync_db +from background_tasks.core import sync_db from file_storage import tasks from file_storage.backends.filesystem import FilesystemBackend from file_storage.core import constants diff --git a/modules/file_storage/tests/core/worker/test_file_storage_worker.py b/modules/file_storage/tests/core/worker/test_file_storage_worker.py index 4c205b867..ad820ce93 100644 --- a/modules/file_storage/tests/core/worker/test_file_storage_worker.py +++ b/modules/file_storage/tests/core/worker/test_file_storage_worker.py @@ -7,7 +7,7 @@ import pytest from _file_storage_worker_support import set_setting -from background_tasks import outbox_worker +from background_tasks.outbox import worker as outbox_worker from file_storage import contracts from file_storage.backends.guard import BackendUnavailableError from file_storage.contracts import FileUploaded, NotSupportedError, StoredFileOut diff --git a/modules/file_storage/tests/core/worker/test_file_storage_worker_failures.py b/modules/file_storage/tests/core/worker/test_file_storage_worker_failures.py index a63297665..878044920 100644 --- a/modules/file_storage/tests/core/worker/test_file_storage_worker_failures.py +++ b/modules/file_storage/tests/core/worker/test_file_storage_worker_failures.py @@ -6,7 +6,7 @@ import pytest from _file_storage_worker_support import set_setting -from background_tasks import outbox_worker +from background_tasks.outbox import worker as outbox_worker from file_storage import contracts from file_storage.backends.filesystem import FilesystemBackend from file_storage.models import StoredFile diff --git a/modules/file_storage/tests/multipart/guards/test_file_storage_checksum_reconciliation.py b/modules/file_storage/tests/multipart/guards/test_file_storage_checksum_reconciliation.py index 17c55abde..1a54b9b1e 100644 --- a/modules/file_storage/tests/multipart/guards/test_file_storage_checksum_reconciliation.py +++ b/modules/file_storage/tests/multipart/guards/test_file_storage_checksum_reconciliation.py @@ -4,7 +4,7 @@ from unittest.mock import Mock import pytest -from background_tasks import sync_db +from background_tasks.core import sync_db from file_storage import tasks from file_storage.core import constants from file_storage.models import Base, StoredFile diff --git a/modules/settings/tests/core/service/test_settings_default_clears_override.py b/modules/settings/tests/core/service/test_settings_default_clears_override.py index da5a7854e..5651600b2 100644 --- a/modules/settings/tests/core/service/test_settings_default_clears_override.py +++ b/modules/settings/tests/core/service/test_settings_default_clears_override.py @@ -3,7 +3,7 @@ from __future__ import annotations import pytest -from background_tasks.settings import DEFAULT_RETENTION_DAYS +from background_tasks.core.settings import DEFAULT_RETENTION_DAYS from settings.core.service import SettingService from settings.module_settings.store import SettingsStore diff --git a/modules/settings/tests/module_settings/fields/test_settings_lenient_related_fields_validators.py b/modules/settings/tests/module_settings/fields/test_settings_lenient_related_fields_validators.py index 6341b5719..b31f991c5 100644 --- a/modules/settings/tests/module_settings/fields/test_settings_lenient_related_fields_validators.py +++ b/modules/settings/tests/module_settings/fields/test_settings_lenient_related_fields_validators.py @@ -7,7 +7,7 @@ from __future__ import annotations import pytest -from background_tasks.settings import ENV_PREFIX, BackgroundTasksSettings +from background_tasks.core.settings import ENV_PREFIX, BackgroundTasksSettings from file_storage.core.settings import FileStorageSettings from keycloak.core.settings import KeycloakSettings from pydantic import ValidationError diff --git a/modules/settings/tests/module_settings/hydrate/test_settings_hydrate.py b/modules/settings/tests/module_settings/hydrate/test_settings_hydrate.py index 31368bda3..8a5b5880e 100644 --- a/modules/settings/tests/module_settings/hydrate/test_settings_hydrate.py +++ b/modules/settings/tests/module_settings/hydrate/test_settings_hydrate.py @@ -219,7 +219,7 @@ async def test_env_wins_even_where_env_alone_fails_validation(db_session, monkey @pytest.mark.asyncio async def test_background_tasks_env_var_beats_stored_override(db_session, monkeypatch) -> None: """The bundled class the bug was real for: ``SM_BG_TASKS_*`` must win.""" - from background_tasks.settings import BackgroundTasksSettings + from background_tasks.core.settings import BackgroundTasksSettings store = SettingsStore(SettingService(db_session)) await store.set_override("background_tasks", "task_default_queue", "from-db", "string") diff --git a/modules/settings/tests/web/api/test_settings_module_api_json_fields.py b/modules/settings/tests/web/api/test_settings_module_api_json_fields.py index 7628d25ee..8b3dca30a 100644 --- a/modules/settings/tests/web/api/test_settings_module_api_json_fields.py +++ b/modules/settings/tests/web/api/test_settings_module_api_json_fields.py @@ -7,7 +7,7 @@ from __future__ import annotations -from background_tasks.settings import BackgroundTasksSettings +from background_tasks.core.settings import BackgroundTasksSettings from settings.core.service import SettingService from settings.module_settings.store import SettingsStore from simple_module_core.settings_hydrate import hydrate_settings, input_type_for_field diff --git a/modules/tenants/tests/_tenants_deletion_support.py b/modules/tenants/tests/_tenants_deletion_support.py index c2264fe64..6d4f33768 100644 --- a/modules/tenants/tests/_tenants_deletion_support.py +++ b/modules/tenants/tests/_tenants_deletion_support.py @@ -1,6 +1,6 @@ """Outbox writer with dispatch deliberately paused for deletion tests.""" -from background_tasks.outbox_transport import _OutboxWriter +from background_tasks.outbox.transport import _OutboxWriter from simple_module_core.events import EventBus diff --git a/modules/tenants/tests/core/service/test_tenants_teardown_integration.py b/modules/tenants/tests/core/service/test_tenants_teardown_integration.py index 5578f4509..f3a684358 100644 --- a/modules/tenants/tests/core/service/test_tenants_teardown_integration.py +++ b/modules/tenants/tests/core/service/test_tenants_teardown_integration.py @@ -6,8 +6,8 @@ import pytest from api_tokens.models import ApiToken from audit_log.models import AuditEntry -from background_tasks.outbox_claim import OutboxPolicy -from background_tasks.outbox_dispatch import dispatch +from background_tasks.outbox.claim import OutboxPolicy +from background_tasks.outbox.dispatch import dispatch from feature_flags.models import FeatureFlagOverride from file_storage.backends.filesystem import FilesystemBackend from file_storage.models import StoredFile diff --git a/modules/users/tests/test_users_deletion_events.py b/modules/users/tests/test_users_deletion_events.py index a5731c5d3..f0563a624 100644 --- a/modules/users/tests/test_users_deletion_events.py +++ b/modules/users/tests/test_users_deletion_events.py @@ -4,7 +4,7 @@ import pytest from background_tasks.models import OutboxMessage -from background_tasks.outbox_transport import _OutboxWriter +from background_tasks.outbox.transport import _OutboxWriter from permissions.models import UserPermission from permissions.module import PermissionsModule from sharing.core.constants import SUBJECT_USER diff --git a/scripts/run_worker.py b/scripts/run_worker.py index 7b57d8f1a..cdde7f8e9 100644 --- a/scripts/run_worker.py +++ b/scripts/run_worker.py @@ -6,17 +6,17 @@ uv run celery -A scripts.run_worker:celery beat -l info Both the web process and the worker go through the same -:func:`background_tasks.celery_app.build_celery` factory, so the broker +:func:`background_tasks.worker.celery_app.build_celery` factory, so the broker config, autodiscovered tasks, and signal handlers stay in lockstep. The bootstrap around it (``.env``, host settings, DB-hydrated module settings) -lives in :mod:`background_tasks.runner`, shared with scaffolded apps. +lives in :mod:`background_tasks.worker.runner`, shared with scaffolded apps. """ from __future__ import annotations from pathlib import Path -from background_tasks.runner import build_runner_celery +from background_tasks.worker.runner import build_runner_celery # Module-level name ``celery`` is what ``celery -A scripts.run_worker:celery`` # looks for. Keep it stable. The fallback is the repo-root ``.env`` (next to From 591b945ce5fbebcd0d4f908c818193bfa5abe18f Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Fri, 6 Nov 2026 03:51:45 +0100 Subject: [PATCH 2/2] docs(background_tasks): fix stale paths after layout move Claude-Session: https://claude.ai/code/session_01FXHA3ZicrtZ4C5QPEgxfgq --- docs/modules/audit_log.md | 2 +- docs/modules/background_tasks.md | 4 ++-- framework/db/simple_module_db/query_filter.py | 2 +- .../background_tasks/background_tasks/ui/utils/constants.ts | 2 +- .../test_background_tasks_invalidation_degraded.py | 2 +- .../invalidation/test_background_tasks_invalidation_redis.py | 4 ++-- .../outbox/delivery/test_background_tasks_outbox_delivery.py | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/modules/audit_log.md b/docs/modules/audit_log.md index ce69ea59e..0eb707ad6 100644 --- a/docs/modules/audit_log.md +++ b/docs/modules/audit_log.md @@ -31,7 +31,7 @@ A worker never runs `create_app`, so it has no `app.state.sm.db` for `on_startup audit_log = "audit_log.core.capture:audit_callback" ``` -`background_tasks` binds whatever `simple_module_db.audit_sinks.discover_audit_callback()` finds to both of its worker engines — the sync engine task bodies and signal handlers use (`sync_db`) and the async engine durable outbox handlers run on (`outbox_worker`). A task's ORM writes, including `TaskExecution` lifecycle rows, are therefore audited exactly like a request's whenever `audit_log` is installed; neither package imports the other. The worker's audit rows carry no `user_id` (no request principal). +`background_tasks` binds whatever `simple_module_db.audit_sinks.discover_audit_callback()` finds to both of its worker engines — the sync engine task bodies and signal handlers use (`core/sync_db`) and the async engine durable outbox handlers run on (`outbox/worker`). A task's ORM writes, including `TaskExecution` lifecycle rows, are therefore audited exactly like a request's whenever `audit_log` is installed; neither package imports the other. The worker's audit rows carry no `user_id` (no request principal). ### Statement DML: bulk, Core and raw writes diff --git a/docs/modules/background_tasks.md b/docs/modules/background_tasks.md index 7aeee49e6..1910ba253 100644 --- a/docs/modules/background_tasks.md +++ b/docs/modules/background_tasks.md @@ -633,7 +633,7 @@ Published (and consumable from any other module): ## Worker introspection -`WorkerInspector` (`background_tasks/worker_inspector.py`) wraps `celery.control.inspect()`: +`WorkerInspector` (`background_tasks/worker/inspector.py`) wraps `celery.control.inspect()`: | Method | Returns | |---|---| @@ -645,7 +645,7 @@ It's synchronous; the view route calls it via `asyncio.to_thread`. The table below describes full tracking with `track_publish=True`; reduced modes gate these writes as described under [Execution tracking](#execution-tracking). -Synchronous handlers in `signals.py`, all bound through Celery's `signals..connect`, persisting through a separate **sync** SQLAlchemy engine (`sync_db.py`) so they don't have to spin up an event loop: +Synchronous handlers in `worker/signals.py`, all bound through Celery's `signals..connect`, persisting through a separate **sync** SQLAlchemy engine (`core/sync_db.py`) so they don't have to spin up an event loop: | Signal | What it writes | |---|---| diff --git a/framework/db/simple_module_db/query_filter.py b/framework/db/simple_module_db/query_filter.py index d025bd3ac..ba30dfbb4 100644 --- a/framework/db/simple_module_db/query_filter.py +++ b/framework/db/simple_module_db/query_filter.py @@ -46,7 +46,7 @@ ) # ``_engine_policy`` (imported above) is re-exported: the per-engine policy map lives in -# ``_engine_policies``; modules/background_tasks/tests/test_worker_tenancy.py reads it here. +# ``_engine_policies``; the background_tasks worker tenancy test reads it here. class EngineTenancy: diff --git a/modules/background_tasks/background_tasks/ui/utils/constants.ts b/modules/background_tasks/background_tasks/ui/utils/constants.ts index 4864d1548..3a9049897 100644 --- a/modules/background_tasks/background_tasks/ui/utils/constants.ts +++ b/modules/background_tasks/background_tasks/ui/utils/constants.ts @@ -1,4 +1,4 @@ -// Frontend mirror of background_tasks/constants.py. +// Frontend mirror of background_tasks/core/constants.py. // Kept small and hand-maintained — the backend file is the source of truth. import { keys } from '@simple-module-py/i18n'; diff --git a/modules/background_tasks/tests/core/invalidation/test_background_tasks_invalidation_degraded.py b/modules/background_tasks/tests/core/invalidation/test_background_tasks_invalidation_degraded.py index fec781ff4..72eca3f29 100644 --- a/modules/background_tasks/tests/core/invalidation/test_background_tasks_invalidation_degraded.py +++ b/modules/background_tasks/tests/core/invalidation/test_background_tasks_invalidation_degraded.py @@ -1,6 +1,6 @@ """The transport when the broker is down, or up and mute. -Split from ``test_bg_invalidation_redis.py`` — that file asks whether a working +Split from ``test_background_tasks_invalidation_redis.py`` — that file asks whether a working Redis round trip works, these ask what happens when it does not, which needs no Redis at all and a different set of fixtures. Keeping them together pushed the file past the 300-line cap. diff --git a/modules/background_tasks/tests/core/invalidation/test_background_tasks_invalidation_redis.py b/modules/background_tasks/tests/core/invalidation/test_background_tasks_invalidation_redis.py index 09c66ac57..4bc51db59 100644 --- a/modules/background_tasks/tests/core/invalidation/test_background_tasks_invalidation_redis.py +++ b/modules/background_tasks/tests/core/invalidation/test_background_tasks_invalidation_redis.py @@ -1,6 +1,6 @@ """The transport against a real ``redis-server``, not the fake. -``test_bg_invalidation.py`` proves this repo's own logic — retry, timeout, +``test_background_tasks_invalidation.py`` proves this repo's own logic — retry, timeout, teardown — against a fake client. It cannot prove that redis-py does what the fake pretends: that ``pubsub().listen()`` yields frames shaped ``{"type": "message", "data": ...}``, that ``data`` arrives as ``bytes``, that @@ -11,7 +11,7 @@ These tests are the answer to that. They need no ``docker-up``: the ``redis_server`` fixture starts its own ephemeral server. The cases where the -broker is *not* working live in ``test_bg_invalidation_degraded.py``. +broker is *not* working live in ``test_background_tasks_invalidation_degraded.py``. """ from __future__ import annotations diff --git a/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_delivery.py b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_delivery.py index b1d457868..dd8033e2f 100644 --- a/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_delivery.py +++ b/modules/background_tasks/tests/outbox/delivery/test_background_tasks_outbox_delivery.py @@ -182,7 +182,7 @@ async def always(event: InvoiceDue, ctx: DurableContext) -> None: assert (await dispatch(factory, bus, policy, now=now + timedelta(days=1))).dead == 0 async def test_unknown_handler_is_retried(self, app, outbox_rows): - """Often a rolling deploy: a later worker may know it (see test_bg_outbox_retry_policy).""" + """Often a rolling deploy: a later worker may know it (see the retry-policy tests).""" now = datetime.now(UTC) async with app.state.sm.db.session_factory() as session: session.add(