From eea8860df5bc4c1ec23d5c7f249e1cc0af019353 Mon Sep 17 00:00:00 2001 From: Cagri Yonca Date: Tue, 29 Sep 2026 14:36:28 +0200 Subject: [PATCH] fix: defer boot_agent until after eventlet monkey_patch to prevent recursion error Signed-off-by: Cagri Yonca --- src/instana/__init__.py | 150 ++++++++++++--- tests/clients/test_sqlalchemy.py | 75 ++++---- tests/frameworks/test_eventlet_autotrace.py | 195 ++++++++++++++++++++ 3 files changed, 358 insertions(+), 62 deletions(-) create mode 100644 tests/frameworks/test_eventlet_autotrace.py diff --git a/src/instana/__init__.py b/src/instana/__init__.py index 3df5cdb1..02c9b7bc 100644 --- a/src/instana/__init__.py +++ b/src/instana/__init__.py @@ -14,8 +14,6 @@ import os import sys from importlib import util as importlib_util -from typing import Tuple - from instana.collector.helpers.runtime import ( is_autowrapt_instrumented, is_webhook_instrumented, @@ -59,10 +57,7 @@ def load(_: object) -> None: - """ - Method used to activate the Instana sensor via AUTOWRAPT_BOOTSTRAP - environment variable. - """ + """Activate the Instana Tracer via the AUTOWRAPT_BOOTSTRAP environment variable.""" # Work around https://bugs.python.org/issue32573 if not hasattr(sys, "argv"): sys.argv = [""] @@ -100,16 +95,91 @@ def key_to_bool(k: str) -> bool: monkey.patch_all() -def get_aws_lambda_handler() -> Tuple[str, str]: +# Guards against boot_agent() being called more than once when monkey_patch() +# is invoked multiple times in the same process (e.g. application code also +# calls it after Instana has already wrapped it). +_eventlet_booted = False + + +def _defer_boot_until_eventlet_patch() -> None: + """Defer boot_agent() until after eventlet.monkey_patch() has been called. + + Strategy + -------- + When gunicorn uses the eventlet worker class, ``monkey_patch()`` is called + inside ``EventletWorker.patch()``, which runs only in the forked worker + process (``init_process → self.patch()``). We register a post-import hook + via ``wrapt.when_imported`` so that the wrap only happens if + ``gunicorn.workers.geventlet`` is actually loaded (i.e. ``-k eventlet`` is + in use). This means: + + * Gunicorn ≥26 (which removed the eventlet worker) never imports the module + and the hook never fires — no ``ModuleNotFoundError`` at startup. + * Gunicorn <26 with a non-eventlet worker class also never imports it. + * Docker ``CMD ["gunicorn", ...]`` (PID 1 arbiter) works correctly because + ``EventletWorker.patch()`` is called only post-fork in the worker. + + If gunicorn is not present the fallback wraps ``eventlet.monkey_patch`` + directly and boots on the first call. """ - For instrumenting AWS Lambda, users specify their original lambda handler - in the LAMBDA_HANDLER environment variable. This function searches for and - parses that environment variable or returns the defaults. + import wrapt + + def _boot_once() -> None: + global _eventlet_booted + if not _eventlet_booted: + _eventlet_booted = True + if is_truthy(os.environ.get("INSTANA_AUTOPROFILE", None)): + _start_profiler() + boot_agent() + + if importlib_util.find_spec("gunicorn"): + # Preferred path: use a post-import hook so we only wrap when gunicorn + # actually loads the eventlet worker module (gunicorn -k eventlet). + # Gunicorn ≥26 dropped geventlet entirely; when_imported is a no-op if + # the module never gets imported, so there is no ModuleNotFoundError. + @wrapt.when_imported("gunicorn.workers.geventlet") + def _on_geventlet_imported(module: object) -> None: + def _after_worker_patch( + wrapped: object, + instance: object, + args: tuple[object, ...], + kwargs: dict[str, object], + ) -> None: + wrapped(*args, **kwargs) + _boot_once() + + wrapt.wrap_function_wrapper( + module, + "EventletWorker.patch", + _after_worker_patch, + ) + else: + # Fallback: no gunicorn — application calls monkey_patch() directly. + # Boot immediately after the first invocation. + def _after_monkey_patch( + wrapped: object, + instance: object, + args: tuple[object, ...], + kwargs: dict[str, object], + ) -> None: + result = wrapped(*args, **kwargs) + _boot_once() + return result + + wrapt.wrap_function_wrapper("eventlet", "monkey_patch", _after_monkey_patch) + + +def get_aws_lambda_handler() -> tuple[str, str]: + """Return the AWS Lambda handler module and function name. + + Users specify their original lambda handler in the LAMBDA_HANDLER + environment variable. This function searches for and parses that + environment variable or returns the defaults. The default handler value for AWS Lambda is 'lambda_function.lambda_handler' - which equates to the function "lambda_handler in a file named - lambda_function.py" or in Python terms - "from lambda_function import lambda_handler" + which equates to the function ``lambda_handler`` in a file named + ``lambda_function.py``, or in Python terms + ``from lambda_function import lambda_handler``. """ handler_module = "lambda_function" handler_function = "lambda_handler" @@ -126,11 +196,10 @@ def get_aws_lambda_handler() -> Tuple[str, str]: def lambda_handler(event: str, context: str) -> None: - """ - Entry point for AWS Lambda monitoring. + """Entry point for AWS Lambda monitoring. - This function will trigger the initialization of Instana monitoring and then call - the original user specified lambda handler function. + Triggers the initialization of Instana monitoring and then calls + the original user-specified lambda handler function. """ module_name, function_name = get_aws_lambda_handler() @@ -153,7 +222,10 @@ def lambda_handler(event: str, context: str) -> None: def boot_agent() -> None: - """Initialize the Instana agent and conditionally load auto-instrumentation.""" + """Initialize the Instana agent and conditionally load auto-instrumentation. + + Imports all instrumentation modules unless INSTANA_DISABLE_AUTO_INSTR is set. + """ import instana.singletons # noqa: F401 @@ -169,6 +241,7 @@ def boot_agent() -> None: elasticsearch, # noqa: F401 fastapi, # noqa: F401 flask, # noqa: F401 + gevent, # noqa: F401 grpcio, # noqa: F401 httpx, # noqa: F401 logging, # noqa: F401 @@ -187,7 +260,6 @@ def boot_agent() -> None: starlette, # noqa: F401 urllib3, # noqa: F401 werkzeug, # noqa: F401 - gevent, # noqa: F401 ) from instana.instrumentation.aiohttp import ( client as aiohttp_client, # noqa: F401 @@ -223,7 +295,10 @@ def boot_agent() -> None: def _start_profiler() -> None: - """Start the Instana Auto Profile.""" + """Start the Instana Auto Profile. + + Retrieves the profiler singleton and starts it if available. + """ from instana.singletons import get_profiler if profiler := get_profiler(): @@ -251,17 +326,36 @@ def _start_profiler() -> None: f"Instana: No use in monitoring this process type ({os.path.basename(sys.argv[0])}). Will go sit in a corner quietly." ) else: - # Automatic gevent monkey patching - # unless auto instrumentation is off, then the customer should do manual gevent monkey patching if ( (is_autowrapt_instrumented() or is_webhook_instrumented()) and "INSTANA_DISABLE_AUTO_INSTR" not in os.environ - and importlib_util.find_spec("gevent") ): - apply_gevent_monkey_patch() + # Automatic gevent monkey patching + # unless auto instrumentation is off, then the customer should do manual gevent monkey patching + if importlib_util.find_spec("gevent"): + apply_gevent_monkey_patch() + + # Eventlet deferred boot: opt-in via INSTANA_EVENTLET_DEFERRED_BOOT=true. + # When set, boot_agent() is deferred until after eventlet.monkey_patch() to + # prevent the ssl.SSLContext RecursionError with gunicorn eventlet workers. + # Without the opt-in we boot immediately, avoiding silent tracing gaps when + # eventlet is installed as a transitive dependency but not actually in use + # (e.g. sync/gthread workers, Celery, plain scripts). + if importlib_util.find_spec("eventlet") and is_truthy( + os.environ.get("INSTANA_EVENTLET_DEFERRED_BOOT", None) + ): + # boot_agent() will be called by the wrapper after monkey_patch; + # do not call it here to avoid a double boot. + _defer_boot_until_eventlet_patch() + return_early = True + else: + return_early = False + else: + return_early = False - # AutoProfile - if "INSTANA_AUTOPROFILE" in os.environ: - _start_profiler() + if not return_early: + # AutoProfile + if is_truthy(os.environ.get("INSTANA_AUTOPROFILE", None)): + _start_profiler() - boot_agent() + boot_agent() diff --git a/tests/clients/test_sqlalchemy.py b/tests/clients/test_sqlalchemy.py index eb931a72..4ec48977 100644 --- a/tests/clients/test_sqlalchemy.py +++ b/tests/clients/test_sqlalchemy.py @@ -14,11 +14,9 @@ from instana.span.span import get_current_span from tests.helpers import testenv -engine = create_engine( - f"postgresql://{testenv['postgresql_user']}:{testenv['postgresql_pw']}@{testenv['postgresql_host']}:{testenv['postgresql_port']}/{testenv['postgresql_db']}" -) +_DB_URL = f"postgresql+psycopg2://{testenv['postgresql_user']}:{testenv['postgresql_pw']}@{testenv['postgresql_host']}:{testenv['postgresql_port']}/{testenv['postgresql_db']}" +_DB_URL_DISPLAY = f"postgresql+psycopg2://{testenv['postgresql_host']}:{testenv['postgresql_port']}/{testenv['postgresql_db']}" -Session = sessionmaker(bind=engine) Base = declarative_base() @@ -35,37 +33,49 @@ def __repr__(self) -> str: return f"" -@pytest.fixture(scope="class") -def db_setup() -> None: +@pytest.fixture(scope="module") +def engine(): + return create_engine(_DB_URL) + + +@pytest.fixture(scope="module") +def db_setup(engine) -> None: tracer = get_tracer() with tracer.start_as_current_span("metadata") as span: Base.metadata.create_all(engine) span.end() -stan_user = StanUser( - name="IAmStan", - fullname="Stan Robot", - password="3X}vP66ADoCFT2g?HPvoem2eJh,zWXgd36Rb/{aRq/>7EYy6@EEH4BP(oeXac@mR", -) -stan_user2 = StanUser( - name="IAmStanToo", - fullname="Stan Robot 2", - password="3X}vP66ADoCFT2g?HPvoem2eJh,zWXgd36Rb/{aRq/>7EYy6@EEH4BP(oeXac@mR", -) +@pytest.fixture +def stan_user(): + return StanUser( + name="IAmStan", + fullname="Stan Robot", + password="3X}vP66ADoCFT2g?HPvoem2eJh,zWXgd36Rb/{aRq/>7EYy6@EEH4BP(oeXac@mR", + ) + -sqlalchemy_url = f"postgresql://{testenv['postgresql_host']}:{testenv['postgresql_port']}/{testenv['postgresql_db']}" +@pytest.fixture +def stan_user2(): + return StanUser( + name="IAmStanToo", + fullname="Stan Robot 2", + password="3X}vP66ADoCFT2g?HPvoem2eJh,zWXgd36Rb/{aRq/>7EYy6@EEH4BP(oeXac@mR", + ) @pytest.mark.usefixtures("db_setup") class TestSQLAlchemy: @pytest.fixture(autouse=True) - def _resource(self) -> Generator[None, None, None]: + def _resource(self, engine, stan_user, stan_user2) -> Generator[None, None, None]: """Clear all spans before a test run""" + self.engine = engine + self.stan_user = stan_user + self.stan_user2 = stan_user2 self.tracer = get_tracer() self.recorder = self.tracer.span_processor self.recorder.clear_spans() - self.session = Session() + self.session = sessionmaker(bind=engine)() yield """Ensure that allow_exit_as_root has the default value""" self.session.close() @@ -73,7 +83,7 @@ def _resource(self) -> Generator[None, None, None]: def test_session_add(self) -> None: with self.tracer.start_as_current_span("test"): - self.session.add(stan_user) + self.session.add(self.stan_user) self.session.commit() spans = self.recorder.queued_spans() @@ -100,7 +110,7 @@ def test_session_add(self) -> None: assert "sqlalchemy" in sql_span.data assert sql_span.data["sqlalchemy"]["eng"] == "postgresql" - assert sqlalchemy_url == sql_span.data["sqlalchemy"]["url"] + assert _DB_URL_DISPLAY == sql_span.data["sqlalchemy"]["url"] assert ( sql_span.data["sqlalchemy"]["sql"] == "INSERT INTO churchofstan (name, fullname, password) VALUES (%(name)s, %(fullname)s, %(password)s) RETURNING churchofstan.id" @@ -113,7 +123,7 @@ def test_session_add(self) -> None: def test_session_add_as_root_exit_span(self) -> None: agent.options.allow_exit_as_root = True - self.session.add(stan_user2) + self.session.add(self.stan_user2) self.session.commit() spans = self.recorder.queued_spans() @@ -136,7 +146,7 @@ def test_session_add_as_root_exit_span(self) -> None: assert "sqlalchemy" in sql_span.data assert sql_span.data["sqlalchemy"]["eng"] == "postgresql" - assert sqlalchemy_url == sql_span.data["sqlalchemy"]["url"] + assert _DB_URL_DISPLAY == sql_span.data["sqlalchemy"]["url"] assert ( sql_span.data["sqlalchemy"]["sql"] == "INSERT INTO churchofstan (name, fullname, password) VALUES (%(name)s, %(fullname)s, %(password)s) RETURNING churchofstan.id" @@ -149,7 +159,7 @@ def test_session_add_as_root_exit_span(self) -> None: def test_transaction(self) -> None: with self.tracer.start_as_current_span("test"): # noqa: SIM117 - with engine.begin() as connection: + with self.engine.begin() as connection: connection.execute(text("select 1")) connection.execute( text( @@ -186,7 +196,7 @@ def test_transaction(self) -> None: assert "sqlalchemy" in sql_span0.data assert sql_span0.data["sqlalchemy"]["eng"] == "postgresql" - assert sqlalchemy_url == sql_span0.data["sqlalchemy"]["url"] + assert _DB_URL_DISPLAY == sql_span0.data["sqlalchemy"]["url"] assert sql_span0.data["sqlalchemy"]["sql"] == "select 1" assert not sql_span0.data["sqlalchemy"]["err"] @@ -200,7 +210,7 @@ def test_transaction(self) -> None: assert "sqlalchemy" in sql_span1.data assert sql_span1.data["sqlalchemy"]["eng"] == "postgresql" - assert sqlalchemy_url == sql_span1.data["sqlalchemy"]["url"] + assert _DB_URL_DISPLAY == sql_span1.data["sqlalchemy"]["url"] assert ( sql_span1.data["sqlalchemy"]["sql"] == "select (name, fullname, password) from churchofstan where name='doesntexist'" @@ -243,7 +253,7 @@ def test_error_logging(self) -> None: assert "sqlalchemy" in sql_span.data assert sql_span.data["sqlalchemy"]["eng"] == "postgresql" - assert sqlalchemy_url == sql_span.data["sqlalchemy"]["url"] + assert _DB_URL_DISPLAY == sql_span.data["sqlalchemy"]["url"] assert ( sql_span.data["sqlalchemy"]["sql"] == "htVwGrCwVThisIsInvalidSQLaw4ijXd88" ) @@ -271,11 +281,8 @@ def test_error_before_tracing(self) -> None: current_span = get_current_span() assert not current_span.is_recording() - invalid_connection_url = "postgresql://user1:pwd1@localhost:9999/mydb1" - with pytest.raises( - OperationalError, - match=r"^(\(psycopg2\.OperationalError\)).*", - ) as context_manager: + invalid_connection_url = "postgresql+psycopg2://user1:pwd1@localhost:9999/mydb1" + with pytest.raises(OperationalError) as context_manager: engine = create_engine(invalid_connection_url) with engine.connect() as connection: (version,) = connection.execute(text("select version()")).fetchone() @@ -284,7 +291,7 @@ def test_error_before_tracing(self) -> None: assert not the_exception.connection_invalidated def test_if_not_tracing(self) -> None: - with engine.begin() as connection: + with self.engine.begin() as connection: connection.execute(text("select 1")) connection.execute( text( @@ -306,7 +313,7 @@ def test_context_restored_after_query(self) -> None: after_cursor_execute used get_current_span() instead of conn.span, which could corrupt the context stack and cause child spans to be dropped.""" with self.tracer.start_as_current_span("test") as parent_span: - with engine.begin() as connection: + with self.engine.begin() as connection: connection.execute(text("select 1")) # After the sqlalchemy span ends, the active span must be back to diff --git a/tests/frameworks/test_eventlet_autotrace.py b/tests/frameworks/test_eventlet_autotrace.py new file mode 100644 index 00000000..224cb578 --- /dev/null +++ b/tests/frameworks/test_eventlet_autotrace.py @@ -0,0 +1,195 @@ +# (c) Copyright IBM Corp. 2026 +from __future__ import annotations + +import os +import types +from collections.abc import Generator +from unittest.mock import MagicMock, patch + +import pytest + +import instana +from instana import _defer_boot_until_eventlet_patch + + +def _make_fake_geventlet_module() -> types.ModuleType: + """Return a minimal fake gunicorn.workers.geventlet module with EventletWorker.patch.""" + + class EventletWorker: + def patch(self) -> None: + pass + + mod = types.ModuleType("gunicorn.workers.geventlet") + mod.EventletWorker = EventletWorker # type: ignore[attr-defined] + return mod + + +class TestEventletDeferredBootWithGunicorn: + """Tests for the gunicorn path: when_imported hook on gunicorn.workers.geventlet.""" + + @pytest.fixture(autouse=True) + def reset_booted_flag(self) -> Generator[None, None, None]: + instana._eventlet_booted = False + yield + instana._eventlet_booted = False + os.environ.pop("INSTANA_AUTOPROFILE", None) + + def _call_geventlet_hook(self) -> tuple[MagicMock, MagicMock]: + """Register the deferred-boot hook (gunicorn path) and simulate the module import. + + Returns (mock_wrap, fake_module) so callers can extract the wrapper and + invoke it directly. + """ + fake_mod = _make_fake_geventlet_module() + captured_when_imported_cb = [] + + def fake_when_imported(name: str): + def decorator(fn): + captured_when_imported_cb.append((name, fn)) + return fn + return decorator + + with patch("wrapt.when_imported", side_effect=fake_when_imported), \ + patch("instana.importlib_util.find_spec", return_value=True): + _defer_boot_until_eventlet_patch() + + assert len(captured_when_imported_cb) == 1 + _, cb = captured_when_imported_cb[0] + + # Now simulate the module being imported: call the callback with the fake module. + with patch("wrapt.wrap_function_wrapper") as mock_wrap: + cb(fake_mod) + + return mock_wrap, fake_mod + + def test_when_imported_registered_for_geventlet(self) -> None: + """When gunicorn is present, wrapt.when_imported is called for gunicorn.workers.geventlet.""" + registered = [] + + def fake_when_imported(name: str): + def decorator(fn): + registered.append(name) + return fn + return decorator + + with patch("wrapt.when_imported", side_effect=fake_when_imported), \ + patch("instana.importlib_util.find_spec", return_value=True): + _defer_boot_until_eventlet_patch() + + assert registered == ["gunicorn.workers.geventlet"] + + def test_gunicorn26_no_crash_when_geventlet_never_imported(self) -> None: + """Gunicorn ≥26 removed geventlet. when_imported never fires → no ModuleNotFoundError.""" + # when_imported registers a lazy hook; if the module is never imported the + # callback is never called. Simulate by registering but never invoking the cb. + with patch("wrapt.when_imported", return_value=lambda fn: fn), \ + patch("instana.importlib_util.find_spec", return_value=True): + # Must not raise + _defer_boot_until_eventlet_patch() + + def test_worker_patch_hook_boots_agent(self) -> None: + """After geventlet is imported, EventletWorker.patch() wrapper boots agent.""" + mock_wrap, fake_mod = self._call_geventlet_hook() + args, _ = mock_wrap.call_args + wrapper = args[2] + + mock_wrapped = MagicMock(return_value=None) + with patch("instana.boot_agent") as mock_boot: + wrapper(mock_wrapped, None, (), {}) + mock_wrapped.assert_called_once_with() + mock_boot.assert_called_once() + + def test_worker_patch_wraps_correct_target(self) -> None: + """wrap_function_wrapper is called on the module object with 'EventletWorker.patch'.""" + mock_wrap, fake_mod = self._call_geventlet_hook() + args, _ = mock_wrap.call_args + assert args[0] is fake_mod + assert args[1] == "EventletWorker.patch" + + def test_worker_patch_hook_double_boot_protection(self) -> None: + """EventletWorker.patch() called twice boots agent only once.""" + mock_wrap, _ = self._call_geventlet_hook() + wrapper = mock_wrap.call_args[0][2] + + mock_wrapped = MagicMock(return_value=None) + with patch("instana.boot_agent") as mock_boot: + wrapper(mock_wrapped, None, (), {}) + wrapper(mock_wrapped, None, (), {}) + mock_boot.assert_called_once() + + def test_worker_patch_hook_with_autoprofile(self) -> None: + """INSTANA_AUTOPROFILE=true: profiler starts before boot_agent in worker.""" + os.environ["INSTANA_AUTOPROFILE"] = "true" + mock_wrap, _ = self._call_geventlet_hook() + wrapper = mock_wrap.call_args[0][2] + + mock_wrapped = MagicMock(return_value=None) + with patch("instana._start_profiler") as mock_profiler, \ + patch("instana.boot_agent") as mock_boot: + wrapper(mock_wrapped, None, (), {}) + mock_profiler.assert_called_once() + mock_boot.assert_called_once() + + +class TestEventletDeferredBootWithoutGunicorn: + """Tests for the fallback path: application calls monkey_patch() directly.""" + + @pytest.fixture(autouse=True) + def reset_booted_flag(self) -> Generator[None, None, None]: + instana._eventlet_booted = False + yield + instana._eventlet_booted = False + os.environ.pop("INSTANA_AUTOPROFILE", None) + + def test_registers_monkey_patch_wrapper_when_no_gunicorn(self) -> None: + """Without gunicorn, wrap eventlet.monkey_patch directly.""" + with patch("wrapt.wrap_function_wrapper") as mock_wrap, \ + patch("instana.importlib_util.find_spec", return_value=None): + _defer_boot_until_eventlet_patch() + mock_wrap.assert_called_once() + args, _ = mock_wrap.call_args + assert args[0] == "eventlet" + assert args[1] == "monkey_patch" + + def test_monkey_patch_wrapper_boots_agent(self) -> None: + """Fallback wrapper boots agent after the first monkey_patch() call.""" + with patch("wrapt.wrap_function_wrapper") as mock_wrap, \ + patch("instana.importlib_util.find_spec", return_value=None): + _defer_boot_until_eventlet_patch() + wrapper = mock_wrap.call_args[0][2] + + mock_wrapped = MagicMock(return_value="patched") + with patch("instana.boot_agent") as mock_boot: + result = wrapper(mock_wrapped, None, (), {}) + assert result == "patched" + mock_wrapped.assert_called_once_with() + mock_boot.assert_called_once() + + def test_monkey_patch_wrapper_double_boot_protection(self) -> None: + """Fallback wrapper calls boot_agent only once even if monkey_patch() is called twice.""" + with patch("wrapt.wrap_function_wrapper") as mock_wrap, \ + patch("instana.importlib_util.find_spec", return_value=None): + _defer_boot_until_eventlet_patch() + wrapper = mock_wrap.call_args[0][2] + + mock_wrapped = MagicMock(return_value=None) + with patch("instana.boot_agent") as mock_boot: + wrapper(mock_wrapped, None, (), {}) + wrapper(mock_wrapped, None, (), {}) + mock_boot.assert_called_once() + + def test_monkey_patch_wrapper_with_autoprofile(self) -> None: + """INSTANA_AUTOPROFILE=true in fallback path: profiler starts before boot_agent.""" + os.environ["INSTANA_AUTOPROFILE"] = "true" + + with patch("wrapt.wrap_function_wrapper") as mock_wrap, \ + patch("instana.importlib_util.find_spec", return_value=None): + _defer_boot_until_eventlet_patch() + wrapper = mock_wrap.call_args[0][2] + + mock_wrapped = MagicMock(return_value=None) + with patch("instana._start_profiler") as mock_profiler, \ + patch("instana.boot_agent") as mock_boot: + wrapper(mock_wrapped, None, (), {}) + mock_profiler.assert_called_once() + mock_boot.assert_called_once()