Emit OpenTelemetry spans over OTLP

gunicorn.conf.py

@@ -14,6 +14,18 @@
 # so gunicorn defaults to one. If that's ever increased, Cron would start
 # once per worker and tick (and message Slack) that many times over.
 
+import os
+
+# Set here, not left to simone/wsgi.py's own setdefault: post_fork below calls
+# DjangoInstrumentor().instrument(), which reads Django settings itself (to find
+# MIDDLEWARE etc.), and gunicorn runs post_fork *before* the worker imports
+# simone.wsgi. If DJANGO_SETTINGS_MODULE isn't set by then, Django's lazy
+# settings object configures itself from global_settings -- and once that's
+# happened, wsgi.py's later get_wsgi_application() never loads simone.settings
+# at all (LazySettings only consults the env var the first time it's forced to
+# configure). Every request then 500s with AttributeError on ROOT_URLCONF.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'simone.settings')
+
 
 def worker_abort(worker):
     '''
@@ -25,3 +37,63 @@ def worker_abort(worker):
     import sys
 
     faulthandler.dump_traceback(file=sys.stderr)
+
+
+def post_fork(server, worker):
+    '''
+    Wire OpenTelemetry tracing into each worker.
+
+    Done here rather than at import time (or via the `opentelemetry-instrument`
+    CLI wrapper) for the same reason the module comment above gives for not
+    using --preload, plus one of its own: BatchSpanProcessor owns a background
+    export thread, and a thread does not survive fork() -- only the forking
+    thread's state does. A TracerProvider built in the master would leave every
+    worker holding a processor whose export thread only ever existed in the
+    parent, silently exporting nothing. post_fork runs inside each freshly
+    forked worker, before it imports simone.wsgi, so DjangoInstrumentor is in
+    place before Django's own URL resolution and middleware load.
+
+    Spans go straight to Tempo's OTLP/HTTP receiver (OTEL_EXPORTER_OTLP_ENDPOINT
+    in the compose environment), not through logit -- there is nothing logit
+    would add to spans an SDK already produced. Each one is a child of the span
+    logit lifts from nginx's access log line for the same request: nginx sets a
+    traceparent header and the default W3C propagator picks it up here with no
+    code of our own.
+
+    A no-op when OTEL_EXPORTER_OTLP_ENDPOINT is unset, so running outside the
+    compose stack (script/run directly, the dev docker-compose.yml) doesn't
+    need a collector listening or spend every request's teardown waiting on a
+    connection refused.
+    '''
+    if not os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT'):
+        return
+
+    from opentelemetry import trace
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+        OTLPSpanExporter,
+    )
+    from opentelemetry.instrumentation.django import DjangoInstrumentor
+    from opentelemetry.instrumentation.logging import LoggingInstrumentor
+    from opentelemetry.sdk.resources import Resource
+    from opentelemetry.sdk.trace import TracerProvider
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+    # service.name is what lets Tempo resolve a root service for these spans,
+    # matching the `set` component logit stamps onto nginx's own.
+    resource = Resource.create(
+        {
+            'service.name': os.environ.get('OTEL_SERVICE_NAME', 'simone'),
+            'service.namespace': 'xormedia',
+        }
+    )
+    provider = TracerProvider(resource=resource)
+    # The exporter reads OTEL_EXPORTER_OTLP_ENDPOINT itself and appends
+    # /v1/traces per the OTLP spec. This package only speaks protobuf, which
+    # Tempo's receiver accepts natively.
+    provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
+    trace.set_tracer_provider(provider)
+
+    DjangoInstrumentor().instrument()
+    # Puts the active trace/span id into every log record, so a log line can be
+    # matched back to the trace it happened in.
+    LoggingInstrumentor().instrument(set_logging_format=True)

pyproject.toml

@@ -16,6 +16,11 @@ dependencies = [
     "holidays>=0.11.3.1",
     "mysql-connector-python>=9.1.0",
     "nltk>=3.6.6",
+    "opentelemetry-api",
+    "opentelemetry-exporter-otlp-proto-http",
+    "opentelemetry-instrumentation-django",
+    "opentelemetry-instrumentation-logging",
+    "opentelemetry-sdk",
     "pylev>=1.4.0",
     "requests>=2.26.0",
     "slack-bolt>=1.10.0",

requirements.txt

@@ -14,6 +14,7 @@ defusedxml==0.8.0rc2
 django==5.2.17
 django-debug-toolbar==7.1.0
 exceptiongroup==1.3.1; python_version=='3.10'
+googleapis-common-protos==1.75.3
 gunicorn==26.0.0
 h11==0.16.0
 hishel==1.3.0
@@ -28,9 +29,21 @@ msgpack==1.2.1
 mypy-extensions==1.1.0
 mysql-connector-python==26.7.0
 nltk==3.10.3
+opentelemetry-api==1.44.0
+opentelemetry-exporter-otlp-proto-common==1.44.0
+opentelemetry-exporter-otlp-proto-http==1.44.0
+opentelemetry-instrumentation==0.65b0
+opentelemetry-instrumentation-django==0.65b0
+opentelemetry-instrumentation-logging==0.65b0
+opentelemetry-instrumentation-wsgi==0.65b0
+opentelemetry-proto==1.44.0
+opentelemetry-sdk==1.44.0
+opentelemetry-semantic-conventions==0.65b0
+opentelemetry-util-http==0.65b0
 packaging==26.3
 pathspec==1.1.1
 platformdirs==4.11.2
+protobuf==7.36.1
 proviso==0.3.0
 pycodestyle==2.14.0
 pycparser==3.0
@@ -54,4 +67,5 @@ typing-extensions==4.16.0
 unearth==0.18.2
 urllib3==2.7.0
 word2number==1.1
+wrapt==2.4.1rc1
 zipp==4.1.0; python_version=='3.10'