From 0099a270548f3993ae3759c6360ef7179e20bd67 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 4 Sep 2026 13:46:58 -0700 Subject: [PATCH] Emit OpenTelemetry spans over OTLP Adds tracing to each gunicorn worker via a post_fork hook, exporting spans over OTLP/HTTP to whatever OTEL_EXPORTER_OTLP_ENDPOINT names (Grafana Tempo, in the compose stack this runs in). Requests arriving with a W3C traceparent become children of the caller's span, so a request shows up as one trace across nginx and simone rather than two unrelated halves. post_fork rather than import time, or the opentelemetry-instrument wrapper: BatchSpanProcessor owns a background export thread, and threads don't survive fork(), so a provider built in the master would leave the worker exporting nothing. That agrees with the existing reason this app doesn't run with --preload. DJANGO_SETTINGS_MODULE moves up to module scope in the same file. DjangoInstrumentor().instrument() reads Django settings itself, and gunicorn runs post_fork before the worker imports simone.wsgi -- where the setdefault used to live. Without this, Django's lazy settings would configure themselves from global_settings first, wsgi.py's later get_wsgi_application() would never load simone.settings at all, and every request would 500 on ROOT_URLCONF. The whole hook is a no-op when OTEL_EXPORTER_OTLP_ENDPOINT is unset, so running outside the compose stack needs no collector listening. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013LW5eUDxvLcyY9JKbfqMux --- gunicorn.conf.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 5 ++++ requirements.txt | 14 ++++++++++ 3 files changed, 91 insertions(+) diff --git a/gunicorn.conf.py b/gunicorn.conf.py index d3bb7ad..09680b3 100644 --- a/gunicorn.conf.py +++ b/gunicorn.conf.py @@ -14,6 +14,18 @@ # so gunicorn defaults to one. If that's ever increased, Cron would start # once per worker and tick (and message Slack) that many times over. +import os + +# Set here, not left to simone/wsgi.py's own setdefault: post_fork below calls +# DjangoInstrumentor().instrument(), which reads Django settings itself (to find +# MIDDLEWARE etc.), and gunicorn runs post_fork *before* the worker imports +# simone.wsgi. If DJANGO_SETTINGS_MODULE isn't set by then, Django's lazy +# settings object configures itself from global_settings -- and once that's +# happened, wsgi.py's later get_wsgi_application() never loads simone.settings +# at all (LazySettings only consults the env var the first time it's forced to +# configure). Every request then 500s with AttributeError on ROOT_URLCONF. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'simone.settings') + def worker_abort(worker): ''' @@ -25,3 +37,63 @@ def worker_abort(worker): import sys faulthandler.dump_traceback(file=sys.stderr) + + +def post_fork(server, worker): + ''' + Wire OpenTelemetry tracing into each worker. + + Done here rather than at import time (or via the `opentelemetry-instrument` + CLI wrapper) for the same reason the module comment above gives for not + using --preload, plus one of its own: BatchSpanProcessor owns a background + export thread, and a thread does not survive fork() -- only the forking + thread's state does. A TracerProvider built in the master would leave every + worker holding a processor whose export thread only ever existed in the + parent, silently exporting nothing. post_fork runs inside each freshly + forked worker, before it imports simone.wsgi, so DjangoInstrumentor is in + place before Django's own URL resolution and middleware load. + + Spans go straight to Tempo's OTLP/HTTP receiver (OTEL_EXPORTER_OTLP_ENDPOINT + in the compose environment), not through logit -- there is nothing logit + would add to spans an SDK already produced. Each one is a child of the span + logit lifts from nginx's access log line for the same request: nginx sets a + traceparent header and the default W3C propagator picks it up here with no + code of our own. + + A no-op when OTEL_EXPORTER_OTLP_ENDPOINT is unset, so running outside the + compose stack (script/run directly, the dev docker-compose.yml) doesn't + need a collector listening or spend every request's teardown waiting on a + connection refused. + ''' + if not os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT'): + return + + from opentelemetry import trace + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, + ) + from opentelemetry.instrumentation.django import DjangoInstrumentor + from opentelemetry.instrumentation.logging import LoggingInstrumentor + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + + # service.name is what lets Tempo resolve a root service for these spans, + # matching the `set` component logit stamps onto nginx's own. + resource = Resource.create( + { + 'service.name': os.environ.get('OTEL_SERVICE_NAME', 'simone'), + 'service.namespace': 'xormedia', + } + ) + provider = TracerProvider(resource=resource) + # The exporter reads OTEL_EXPORTER_OTLP_ENDPOINT itself and appends + # /v1/traces per the OTLP spec. This package only speaks protobuf, which + # Tempo's receiver accepts natively. + provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) + trace.set_tracer_provider(provider) + + DjangoInstrumentor().instrument() + # Puts the active trace/span id into every log record, so a log line can be + # matched back to the trace it happened in. + LoggingInstrumentor().instrument(set_logging_format=True) diff --git a/pyproject.toml b/pyproject.toml index c947253..df16674 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,11 @@ dependencies = [ "holidays>=0.11.3.1", "mysql-connector-python>=9.1.0", "nltk>=3.6.6", + "opentelemetry-api", + "opentelemetry-exporter-otlp-proto-http", + "opentelemetry-instrumentation-django", + "opentelemetry-instrumentation-logging", + "opentelemetry-sdk", "pylev>=1.4.0", "requests>=2.26.0", "slack-bolt>=1.10.0", diff --git a/requirements.txt b/requirements.txt index 1f04979..2b08c99 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ defusedxml==0.8.0rc2 django==5.2.17 django-debug-toolbar==7.1.0 exceptiongroup==1.3.1; python_version=='3.10' +googleapis-common-protos==1.75.3 gunicorn==26.0.0 h11==0.16.0 hishel==1.3.0 @@ -28,9 +29,21 @@ msgpack==1.2.1 mypy-extensions==1.1.0 mysql-connector-python==26.7.0 nltk==3.10.3 +opentelemetry-api==1.44.0 +opentelemetry-exporter-otlp-proto-common==1.44.0 +opentelemetry-exporter-otlp-proto-http==1.44.0 +opentelemetry-instrumentation==0.65b0 +opentelemetry-instrumentation-django==0.65b0 +opentelemetry-instrumentation-logging==0.65b0 +opentelemetry-instrumentation-wsgi==0.65b0 +opentelemetry-proto==1.44.0 +opentelemetry-sdk==1.44.0 +opentelemetry-semantic-conventions==0.65b0 +opentelemetry-util-http==0.65b0 packaging==26.3 pathspec==1.1.1 platformdirs==4.11.2 +protobuf==7.36.1 proviso==0.3.0 pycodestyle==2.14.0 pycparser==3.0 @@ -54,4 +67,5 @@ typing-extensions==4.16.0 unearth==0.18.2 urllib3==2.7.0 word2number==1.1 +wrapt==2.4.1rc1 zipp==4.1.0; python_version=='3.10'