diff --git a/pyproject.toml b/pyproject.toml index 0c1b34b..330aa83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ build-backend = "hatchling.build" [dependency-groups] dev = [ "mypy>=2.4.1", + "opentelemetry-test-utils>=0.66b2", "pytest>=9.2", "ruff>=0.17.3", ] @@ -48,7 +49,7 @@ enable_error_code = [ disallow_any_expr = true disallow_any_explicit = true # Calls into these packages are untyped at their end, not ours. -untyped_calls_exclude = ["robot"] +untyped_calls_exclude = ["robot", "opentelemetry"] [[tool.mypy.overrides]] module = "tests.*" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..dc5973c --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,14 @@ +"""Isolation shared by the unit and integration tests.""" + +from collections.abc import Iterator + +import pytest +from opentelemetry.test.globals_test import reset_logging_globals, reset_trace_globals + + +@pytest.fixture(autouse=True) +def _reset_opentelemetry_globals() -> Iterator[None]: + # The listener registers its providers as process globals, which the SDK accepts once. + yield + reset_trace_globals() + reset_logging_globals() diff --git a/tests/integration/suites/__init__.py b/tests/integration/suites/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/suites/control.robot b/tests/integration/suites/control.robot new file mode 100644 index 0000000..52d52ad --- /dev/null +++ b/tests/integration/suites/control.robot @@ -0,0 +1,15 @@ +*** Settings *** +Documentation Control structures, setup and teardown for the listener tests. +Suite Setup Log suite setup +Test Teardown Log test teardown + +*** Test Cases *** +Control Structures + FOR ${i} IN RANGE 1 + Log loop + END + IF True + Log if branch + ELSE + Log else branch + END diff --git a/tests/integration/suites/current_span_library.py b/tests/integration/suites/current_span_library.py new file mode 100644 index 0000000..d9fe438 --- /dev/null +++ b/tests/integration/suites/current_span_library.py @@ -0,0 +1,15 @@ +"""Robot Framework library whose keyword creates an OpenTelemetry span. + +The keyword uses only the global tracer, like a library instrumented +independently of the listener would. The test asserts that this span +nests below the span of the running keyword. +""" + +from opentelemetry import trace + + +def open_a_span() -> None: + with trace.get_tracer("other-instrumentation").start_as_current_span( + "http.request" + ): + pass diff --git a/tests/integration/suites/demo.robot b/tests/integration/suites/demo.robot new file mode 100644 index 0000000..be2aa60 --- /dev/null +++ b/tests/integration/suites/demo.robot @@ -0,0 +1,19 @@ +*** Settings *** +Documentation Suite used by the listener tests. + +*** Keywords *** +Greet + [Arguments] ${name} + Log Hello ${name} + Log bold html=True + +*** Test Cases *** +Passes + Greet world + Log careful WARN + +Fails + Should Be Equal 500 200 + +Skips + Skip not today diff --git a/tests/integration/suites/library.robot b/tests/integration/suites/library.robot new file mode 100644 index 0000000..2e0c141 --- /dev/null +++ b/tests/integration/suites/library.robot @@ -0,0 +1,8 @@ +*** Settings *** +Documentation Calls a keyword that creates an OpenTelemetry span of its own. +... The test asserts that the span nests below the keyword span. +Library ${CURDIR}/current_span_library.py + +*** Test Cases *** +Uses Another Instrumentation + Open A Span diff --git a/tests/integration/test_robot_runs.py b/tests/integration/test_robot_runs.py new file mode 100644 index 0000000..bd76e4d --- /dev/null +++ b/tests/integration/test_robot_runs.py @@ -0,0 +1,210 @@ +"""The listener inside real Robot Framework runs.""" + +import io +from collections.abc import Sequence +from pathlib import Path + +from opentelemetry.sdk._logs.export import InMemoryLogRecordExporter +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from robot import run as run_robot + +from RobotmkLibrary.instrumentation.listener import RobotmkListener + +SUITES = Path(__file__).parent / "suites" + + +class _InMemoryListener(RobotmkListener): + """The listener with in-memory exporters, readable after the run.""" + + def __init__(self, *, export_control_structures: bool = False) -> None: + self.spans = InMemorySpanExporter() + self.logs = InMemoryLogRecordExporter() + super().__init__( + span_exporter=self.spans, + log_exporter=self.logs, + export_control_structures=export_control_structures, + ) + + +def _run(listener: RobotmkListener, target: str, **robot_options: object) -> int: + """Run ``target`` from ``SUITES`` under ``listener`` without output files.""" + return_code: int = run_robot( + str(SUITES / target), + listener=listener, + output=None, + log=None, + report=None, + stdout=io.StringIO(), + **robot_options, + ) + return return_code + + +def _span_tree( + spans: Sequence[ReadableSpan], +) -> list[tuple[str, str | None, str, str, str]]: + """(name, parent name, rf.type, rf.status, status code) per span, parents first.""" + names = {span.context.span_id: span.name for span in spans} + return [ + ( + span.name, + None if span.parent is None else names[span.parent.span_id], + str((span.attributes or {})["rf.type"]), + str((span.attributes or {})["rf.status"]), + span.status.status_code.name, + ) + for span in sorted( + spans, key=lambda span: (span.start_time or 0, -(span.end_time or 0)) + ) + ] + + +def test_a_run_exports_the_suite_tree() -> None: + """A robot run exports spans nested like the suite. + + demo.robot with a passing, a failing and a skipped test runs under the + listener. Outcome: robot returns 1, and every suite, test and keyword is a + span below its parent with the Robot status and the mapped span status. + """ + listener = _InMemoryListener() + + assert _run(listener, "demo.robot") == 1 + + assert _span_tree(listener.spans.get_finished_spans()) == [ + ("Demo", None, "suite", "FAIL", "ERROR"), + ("Passes", "Demo", "test", "PASS", "OK"), + ("Greet", "Passes", "keyword", "PASS", "OK"), + ("BuiltIn.Log", "Greet", "keyword", "PASS", "OK"), + ("BuiltIn.Log", "Greet", "keyword", "PASS", "OK"), + ("BuiltIn.Log", "Passes", "keyword", "PASS", "OK"), + ("Fails", "Demo", "test", "FAIL", "ERROR"), + ("BuiltIn.Should Be Equal", "Fails", "keyword", "FAIL", "ERROR"), + ("Skips", "Demo", "test", "SKIP", "UNSET"), + ("BuiltIn.Skip", "Skips", "keyword", "SKIP", "UNSET"), + ] + + +def test_setup_and_teardown_keywords_carry_their_type() -> None: + """Setup and teardown keywords carry their type and nest where they ran. + + control.robot with a suite setup and a test teardown runs under the + listener. Outcome: exactly one span each with rf.keyword.type SETUP and + TEARDOWN, below the suite span and the test span respectively. + """ + listener = _InMemoryListener() + _run(listener, "control.robot") + + spans = listener.spans.get_finished_spans() + spans_by_name = {span.name: span for span in spans} + spans_by_keyword_type = { + (span.attributes or {})["rf.keyword.type"]: span + for span in spans + if (span.attributes or {})["rf.type"] == "keyword" + and (span.attributes or {})["rf.keyword.type"] != "KEYWORD" + } + assert set(spans_by_keyword_type) == {"SETUP", "TEARDOWN"} + + setup_parent = spans_by_keyword_type["SETUP"].parent + assert setup_parent is not None + assert setup_parent.span_id == spans_by_name["Control"].context.span_id + + teardown_parent = spans_by_keyword_type["TEARDOWN"].parent + assert teardown_parent is not None + assert ( + teardown_parent.span_id == spans_by_name["Control Structures"].context.span_id + ) + + +def test_log_records_belong_to_the_innermost_span() -> None: + """Log messages become records of the keyword that logged them. + + demo.robot runs under the listener. Outcome: in log order, every message is + a record of the BuiltIn keyword span that logged it, including the failure + message and the skip reason. + """ + listener = _InMemoryListener() + _run(listener, "demo.robot") + + spans_by_id = { + span.context.span_id: span for span in listener.spans.get_finished_spans() + } + assert [ + (spans_by_id[item.log_record.span_id].name, item.log_record.body) + for item in listener.logs.get_finished_logs() + ] == [ + ("BuiltIn.Log", "Hello world"), + ("BuiltIn.Log", "bold"), + ("BuiltIn.Log", "careful"), + ("BuiltIn.Should Be Equal", "500 != 200"), + ("BuiltIn.Skip", "not today"), + ] + + +def test_control_structures_become_spans_on_request() -> None: + """With export_control_structures, FOR and IF/ELSE become spans in the tree. + + control.robot runs with the option on. Outcome: FOR, ITERATION, IF/ELSE + ROOT, IF and ELSE are control spans between the test and the keywords, and + the ELSE branch that did not run carries NOT RUN with status UNSET. + """ + listener = _InMemoryListener(export_control_structures=True) + _run(listener, "control.robot") + + assert _span_tree(listener.spans.get_finished_spans()) == [ + ("Control", None, "suite", "PASS", "OK"), + ("BuiltIn.Log", "Control", "keyword", "PASS", "OK"), + ("Control Structures", "Control", "test", "PASS", "OK"), + ("FOR", "Control Structures", "control", "PASS", "OK"), + ("ITERATION", "FOR", "control", "PASS", "OK"), + ("BuiltIn.Log", "ITERATION", "keyword", "PASS", "OK"), + ("IF/ELSE ROOT", "Control Structures", "control", "PASS", "OK"), + ("IF", "IF/ELSE ROOT", "control", "PASS", "OK"), + ("BuiltIn.Log", "IF", "keyword", "PASS", "OK"), + ("ELSE", "IF/ELSE ROOT", "control", "NOT RUN", "UNSET"), + ("BuiltIn.Log", "ELSE", "keyword", "NOT RUN", "UNSET"), + ("BuiltIn.Log", "Control Structures", "keyword", "PASS", "OK"), + ] + + +def test_other_instrumentations_nest_below_the_running_keyword() -> None: + """Spans from other instrumentations nest below the running keyword. + + library.robot calls a keyword that opens a span through the global tracer. + Outcome: that span's parent is the keyword span and it keeps its own + instrumentation scope. + """ + listener = _InMemoryListener() + _run(listener, "library.robot") + + spans_by_name = {span.name: span for span in listener.spans.get_finished_spans()} + other = spans_by_name["http.request"] + + other_parent = other.parent + assert other_parent is not None + assert ( + other_parent.span_id + == spans_by_name["current_span_library.Open A Span"].context.span_id + ) + assert other.instrumentation_scope is not None + assert other.instrumentation_scope.name == "other-instrumentation" + + +def test_an_unreachable_collector_does_not_fail_the_run() -> None: + """An unreachable collector neither fails nor stalls the run. + + demo.robot runs with the real OTLP exporters pointed at a closed port and a + one-second timeout. Outcome: robot returns 0. + """ + return_code: int = run_robot( + str(SUITES / "demo.robot"), + test="Passes", + listener="RobotmkLibrary.instrumentation.listener.RobotmkListener;endpoint=http://127.0.0.1:9;timeout=1", + output=None, + log=None, + report=None, + stdout=io.StringIO(), + stderr=io.StringIO(), + ) + + assert return_code == 0 diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_keyword_monitoring.py b/tests/unit/test_keyword_monitoring.py new file mode 100644 index 0000000..92c3c13 --- /dev/null +++ b/tests/unit/test_keyword_monitoring.py @@ -0,0 +1,39 @@ +"""The keyword monitoring keyword called directly, without Robot Framework.""" + +import logging + +import pytest + +from RobotmkLibrary.keyword_monitoring import monitor_subsequent_keyword_runtime + + +@pytest.mark.parametrize( + ("discover_as", "message"), + [ + pytest.param( + None, + "The subsequent keyword will be discovered in Checkmk using its own name", + id="own name", + ), + pytest.param( + "Login", + 'The subsequent keyword will be discovered in Checkmk as: "Login"', + id="given name", + ), + ], +) +def test_the_keyword_logs_the_checkmk_marker( + caplog: pytest.LogCaptureFixture, discover_as: str | None, message: str +) -> None: + """The keyword logs the marker Checkmk matches to discover the next keyword. + + The keyword is called with and without discover_as while the logger is + captured. Outcome: one INFO record with the marker text, naming the given + name when there is one. + """ + caplog.set_level(logging.INFO, logger="RobotmkLibrary.keyword_monitoring") + + monitor_subsequent_keyword_runtime(discover_as=discover_as) + + assert caplog.records[0].levelname == "INFO" + assert caplog.records[0].getMessage() == message diff --git a/tests/unit/test_listener.py b/tests/unit/test_listener.py new file mode 100644 index 0000000..1dcce52 --- /dev/null +++ b/tests/unit/test_listener.py @@ -0,0 +1,772 @@ +"""The listener hooks called directly with stand-ins for the Robot Framework objects.""" + +from collections.abc import Callable, Generator, Mapping, Sequence +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import TypedDict, Unpack + +import pytest +from opentelemetry import trace +from opentelemetry.sdk._logs.export import InMemoryLogRecordExporter +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.util.types import AttributeValue + +from RobotmkLibrary import __version__ +from RobotmkLibrary.instrumentation._exceptions import ( + GlobalLoggerProviderAlreadySetError, + GlobalTracerProviderAlreadySetError, +) +from RobotmkLibrary.instrumentation._logs import ( + LoggerProvider, + SeverityNumber, + set_logger_provider, +) +from RobotmkLibrary.instrumentation._protocols import StatusResult +from RobotmkLibrary.instrumentation.listener import RobotmkListener + +PNG = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk" + "YAAAAAYAAjCB0C8AAAAASUVORK5CYII=" +) +SELENIUM_SCREENSHOT = ( + '
'
+PNG_LOG_ATTRIBUTES = {
+ "rf.log.level": "INFO",
+ "rf.log.html": "false",
+ "rf.log.image": "image/png",
+}
+
+
+class ListenerOptions(TypedDict, total=False):
+ """The tunable arguments of RobotmkListener, as the test helpers forward them."""
+
+ export_control_structures: bool
+ export_keyword_arguments: bool
+ max_log_length: int
+ max_image_size: int
+ max_image_size_total: int
+
+
+def _datetime_to_ns(datetime_: datetime) -> int:
+ return int(datetime_.timestamp()) * 1_000_000_000 + datetime_.microsecond * 1_000
+
+
+@dataclass
+class Result:
+ """Stand-in for the suite, test, keyword and body item results of Robot Framework."""
+
+ name: str = "Demo"
+ full_name: str = ""
+ status: str = "PASS"
+ message: str = ""
+ start_time: datetime | None = None
+ end_time: datetime | None = None
+ type: str = "KEYWORD"
+ args: Sequence[object] = ()
+
+ def __post_init__(self) -> None:
+ self.full_name = self.full_name or self.name
+
+
+@dataclass
+class Message:
+ """Stand-in for a Robot Framework log message."""
+
+ message: str
+ level: str = "INFO"
+ html: bool = False
+ timestamp: datetime | None = None
+
+
+class _InMemoryListener(RobotmkListener):
+ """The listener with in-memory exporters, readable after close()."""
+
+ def __init__(self, **options: Unpack[ListenerOptions]) -> None:
+ self.spans = InMemorySpanExporter()
+ self.logs = InMemoryLogRecordExporter()
+ super().__init__(span_exporter=self.spans, log_exporter=self.logs, **options)
+
+
+@contextmanager
+def _make_listener(**options: Unpack[ListenerOptions]) -> Generator[_InMemoryListener]:
+ listener = _InMemoryListener(**options)
+ try:
+ yield listener
+ finally:
+ listener.close()
+
+
+@pytest.mark.parametrize(
+ ("start", "end", "result", "name", "expected"),
+ [
+ pytest.param(
+ RobotmkListener.start_suite,
+ RobotmkListener.end_suite,
+ Result(name="Demo", status="FAIL"),
+ "Demo",
+ {"rf.type": "suite", "rf.suite.name": "Demo", "rf.status": "FAIL"},
+ id="suite",
+ ),
+ pytest.param(
+ RobotmkListener.start_test,
+ RobotmkListener.end_test,
+ Result(name="Passes", full_name="Demo.Passes"),
+ "Passes",
+ {
+ "rf.type": "test",
+ "rf.test.name": "Passes",
+ "rf.test.full_name": "Demo.Passes",
+ "rf.status": "PASS",
+ },
+ id="test",
+ ),
+ pytest.param(
+ RobotmkListener.start_keyword,
+ RobotmkListener.end_keyword,
+ Result(name="Log", full_name="BuiltIn.Log", type="SETUP"),
+ "BuiltIn.Log",
+ {
+ "rf.type": "keyword",
+ "rf.keyword.name": "BuiltIn.Log",
+ "rf.keyword.type": "SETUP",
+ "rf.keyword.args": "",
+ "rf.status": "PASS",
+ },
+ id="keyword",
+ ),
+ ],
+)
+def test_each_result_becomes_one_span(
+ start: Callable[[RobotmkListener, object, Result], None],
+ end: Callable[[RobotmkListener, object, Result], None],
+ result: Result,
+ name: str,
+ expected: dict[str, str],
+) -> None:
+ """Each result becomes one span.
+
+ The start and end hooks are called with a result stand-in, without Robot
+ Framework. Outcome: one span named after the result, carrying the rf.*
+ attributes of its kind.
+ """
+ with _make_listener() as listener:
+ start(listener, None, result)
+ end(listener, None, result)
+
+ (span,) = listener.spans.get_finished_spans()
+ assert span.name == name
+ assert dict(span.attributes or {}) == expected
+
+
+def test_span_takes_times_from_robotframework_result() -> None:
+ """The span starts and ends at the times of the result.
+
+ A suite is started and ended with a result holding start and end times.
+ Outcome: the span times are those times in nanoseconds since the epoch.
+ """
+ start_time = datetime(2026, 11, 17, 10, 0, 0, 250_000, tzinfo=UTC)
+ end_time = datetime(2026, 11, 17, 10, 0, 1, 500_000, tzinfo=UTC)
+ suite_result = Result(start_time=start_time, end_time=end_time)
+
+ with _make_listener() as listener:
+ listener.start_suite(None, suite_result)
+ listener.end_suite(None, suite_result)
+
+ (span,) = listener.spans.get_finished_spans()
+ assert span.start_time == _datetime_to_ns(start_time)
+ assert span.end_time == _datetime_to_ns(end_time)
+
+
+@pytest.mark.parametrize(
+ ("robotframework_status", "span_status_code", "description"),
+ [
+ pytest.param("PASS", "OK", None, id="pass"),
+ pytest.param("FAIL", "ERROR", "500 != 200", id="fail"),
+ pytest.param("SKIP", "UNSET", None, id="skip"),
+ pytest.param("NOT RUN", "UNSET", None, id="not run"),
+ ],
+)
+def test_the_robotframework_status_maps_to_the_span_status(
+ robotframework_status: str, span_status_code: str, description: str | None
+) -> None:
+ """The Robot Framework status becomes the span status.
+
+ A test with the given status is started and ended. Outcome: the span
+ attribute rf.status holds the Robot status verbatim; the span status code
+ is OK for PASS, ERROR with the failure message as description for FAIL,
+ and UNSET for SKIP and NOT RUN.
+ """
+ test_result = Result(status=robotframework_status, message="500 != 200")
+
+ with _make_listener() as listener:
+ listener.start_test(None, test_result)
+ listener.end_test(None, test_result)
+
+ (span,) = listener.spans.get_finished_spans()
+ assert (span.attributes or {})["rf.status"] == robotframework_status
+ assert span.status.status_code.name == span_status_code
+ assert span.status.description == description
+
+
+def test_spans_nest_in_hook_order() -> None:
+ """Nested hooks produce nested spans.
+
+ A suite, a test and a keyword are started and ended in nesting order,
+ followed by a second keyword in the same test. Outcome: the test span's
+ parent is the suite span, the first keyword span's parent is the test
+ span, and the second keyword's parent is the test span as well, not the
+ ended first keyword.
+ """
+ suite_result = Result(name="Demo")
+ test_result = Result(name="Passes")
+ keyword_result = Result(name="Log")
+ next_keyword_result = Result(name="Sleep")
+
+ with _make_listener() as listener:
+ listener.start_suite(None, suite_result)
+ listener.start_test(None, test_result)
+ listener.start_keyword(None, keyword_result)
+ listener.end_keyword(None, keyword_result)
+ listener.start_keyword(None, next_keyword_result)
+ listener.end_keyword(None, next_keyword_result)
+ listener.end_test(None, test_result)
+ listener.end_suite(None, suite_result)
+
+ spans_by_name = {span.name: span for span in listener.spans.get_finished_spans()}
+ assert spans_by_name["Demo"].parent is None
+
+ passes_parent = spans_by_name["Passes"].parent
+ assert passes_parent is not None
+ assert passes_parent.span_id == spans_by_name["Demo"].context.span_id
+
+ log_parent = spans_by_name["Log"].parent
+ assert log_parent is not None
+ assert log_parent.span_id == spans_by_name["Passes"].context.span_id
+
+ sleep_parent = spans_by_name["Sleep"].parent
+ assert sleep_parent is not None
+ assert sleep_parent.span_id == spans_by_name["Passes"].context.span_id
+
+
+def test_the_running_span_is_the_current_span_of_the_process() -> None:
+ """The span of the running item is the OpenTelemetry current span.
+
+ A test and a keyword are started and ended while the global current span
+ is read after each hook. Outcome: the current span is the keyword span
+ while the keyword runs, the test span again after the keyword ended, and
+ no span after the test ended.
+ """
+ test_result = Result(name="Test")
+ keyword_result = Result(name="Log")
+
+ with _make_listener() as listener:
+ listener.start_test(None, test_result)
+ listener.start_keyword(None, keyword_result)
+ current_during_keyword = trace.get_current_span()
+ listener.end_keyword(None, keyword_result)
+ current_after_keyword = trace.get_current_span()
+ listener.end_test(None, test_result)
+ current_after_test = trace.get_current_span()
+
+ spans_by_name = {span.name: span for span in listener.spans.get_finished_spans()}
+ assert current_during_keyword.get_span_context() == spans_by_name["Log"].context
+ assert current_after_keyword.get_span_context() == spans_by_name["Test"].context
+ assert current_after_test is trace.INVALID_SPAN
+
+
+def test_spans_and_records_carry_the_library_scope() -> None:
+ """Spans and log records carry the library as instrumentation scope.
+
+ A suite with one log message runs through the hooks. Outcome: span and
+ record name the scope RobotmkLibrary with the package version.
+ """
+ suite_result = Result()
+
+ with _make_listener() as listener:
+ listener.start_suite(None, suite_result)
+ listener.log_message(Message("hello"))
+ listener.end_suite(None, suite_result)
+
+ (span,) = listener.spans.get_finished_spans()
+ (log,) = listener.logs.get_finished_logs()
+ assert span.instrumentation_scope is not None
+ assert span.instrumentation_scope.name == "RobotmkLibrary"
+ assert span.instrumentation_scope.version == __version__
+
+ assert log.instrumentation_scope is not None
+ assert log.instrumentation_scope.name == "RobotmkLibrary"
+ assert log.instrumentation_scope.version == __version__
+
+
+def test_control_structures_are_skipped_by_default_and_keep_the_nesting() -> None:
+ """Control structures get no span by default and do not break the nesting.
+
+ A FOR body item between a test and a keyword runs through the hooks with the
+ default options. Outcome: only the test and keyword spans exist, and the
+ keyword span's parent is the test span.
+ """
+ test_result = Result(name="Test")
+ loop_result = Result(type="FOR")
+ keyword_result = Result(name="Log")
+
+ with _make_listener() as listener:
+ listener.start_test(None, test_result)
+ listener.start_body_item(None, loop_result)
+ listener.start_keyword(None, keyword_result)
+ listener.end_keyword(None, keyword_result)
+ listener.end_body_item(None, loop_result)
+ listener.end_test(None, test_result)
+
+ spans_by_name = {span.name: span for span in listener.spans.get_finished_spans()}
+ assert set(spans_by_name) == {"Test", "Log"}
+ assert "FOR" not in spans_by_name
+
+ log_parent = spans_by_name["Log"].parent
+ assert log_parent is not None
+ assert log_parent.span_id == spans_by_name["Test"].context.span_id
+
+
+def test_a_log_message_inside_a_skipped_control_structure_belongs_to_the_enclosing_span() -> (
+ None
+):
+ """A message logged inside a control structure without a span goes to the span around it.
+
+ A message is logged inside a FOR body item with the default options.
+ Outcome: the record references the test span.
+ """
+ test_result = Result(name="Test")
+ loop_result = Result(type="FOR")
+
+ with _make_listener() as listener:
+ listener.start_test(None, test_result)
+ listener.start_body_item(None, loop_result)
+ listener.log_message(Message("inside the loop"))
+ listener.end_body_item(None, loop_result)
+ listener.end_test(None, test_result)
+
+ (span,) = listener.spans.get_finished_spans()
+ (item,) = listener.logs.get_finished_logs()
+ assert item.log_record.span_id == span.context.span_id
+
+
+def test_control_structures_become_spans_on_request() -> None:
+ """Control structures become spans with export_control_structures.
+
+ A FOR body item between a test and a keyword runs through the hooks with
+ export_control_structures on. Outcome: the FOR span exists between the test
+ and the keyword span, carrying rf.type control and rf.control.type FOR.
+ """
+ test_result = Result(name="Test")
+ loop_result = Result(type="FOR")
+ keyword_result = Result(name="Log")
+
+ with _make_listener(export_control_structures=True) as listener:
+ listener.start_test(None, test_result)
+ listener.start_body_item(None, loop_result)
+ listener.start_keyword(None, keyword_result)
+ listener.end_keyword(None, keyword_result)
+ listener.end_body_item(None, loop_result)
+ listener.end_test(None, test_result)
+
+ spans_by_name = {span.name: span for span in listener.spans.get_finished_spans()}
+ assert set(spans_by_name) == {"Test", "FOR", "Log"}
+ assert "FOR" in spans_by_name
+ assert dict(spans_by_name["FOR"].attributes or {}) == {
+ "rf.type": "control",
+ "rf.control.type": "FOR",
+ "rf.status": "PASS",
+ }
+
+ for_parent = spans_by_name["FOR"].parent
+ assert for_parent is not None
+ assert for_parent.span_id == spans_by_name["Test"].context.span_id
+
+ log_parent = spans_by_name["Log"].parent
+ assert log_parent is not None
+ assert log_parent.span_id == spans_by_name["FOR"].context.span_id
+
+
+def test_a_failing_start_hook_keeps_the_stack_intact() -> None:
+ """An exception in a start hook does not desynchronise the span stack.
+
+ A listener subclass fails while opening the span of the keyword Greet; the
+ hooks continue as Robot Framework would, with Greet's end hook and a
+ following keyword. Outcome: the exception propagates, Greet has no span,
+ and the following keyword's parent is still the test span.
+ """
+
+ class _FailingOnGreet(_InMemoryListener):
+ def _start_span(
+ self,
+ name: str,
+ attributes: Mapping[str, AttributeValue],
+ result: StatusResult,
+ ) -> None:
+ if name == "Greet":
+ raise ValueError(name)
+ super()._start_span(name, attributes, result)
+
+ test_result = Result(name="Test")
+ greet_result = Result(name="Greet")
+ log_result = Result(name="Log")
+ listener = _FailingOnGreet()
+
+ listener.start_test(None, test_result)
+ with pytest.raises(ValueError, match="Greet"):
+ # Robotframework catches and logs exceptions in listeners in production
+ listener.start_keyword(None, greet_result)
+ listener.end_keyword(None, greet_result)
+ listener.start_keyword(None, log_result)
+ listener.end_keyword(None, log_result)
+ listener.end_test(None, test_result)
+
+ listener.close()
+
+ spans_by_name = {span.name: span for span in listener.spans.get_finished_spans()}
+ assert set(spans_by_name) == {"Test", "Log"}
+
+ log_parent = spans_by_name["Log"].parent
+ assert log_parent is not None
+ assert log_parent.span_id == spans_by_name["Test"].context.span_id
+
+
+@pytest.mark.parametrize(
+ ("args", "options", "expected"),
+ [
+ pytest.param(("500", "200"), {}, "500 200", id="strings"),
+ pytest.param(({"a": 1}, 42, None), {}, "{'a': 1} 42 None", id="objects"),
+ pytest.param((), {}, "", id="no arguments"),
+ pytest.param(("x",), {"export_keyword_arguments": False}, None, id="off"),
+ ],
+)
+def test_robotframework_keyword_arguments_are_exported_as_rf_keyword_args(
+ args: Sequence[object], options: ListenerOptions, expected: str | None
+) -> None:
+ """Keyword arguments become the rf.keyword.args attribute.
+
+ A keyword with the given arguments runs through the hooks. Outcome: strings
+ and objects are joined by four spaces, no arguments give an empty attribute,
+ and export_keyword_arguments=False leaves the attribute out.
+ """
+ keyword_result = Result(args=args)
+
+ with _make_listener(**options) as listener:
+ listener.start_keyword(None, keyword_result)
+ listener.end_keyword(None, keyword_result)
+
+ (span,) = listener.spans.get_finished_spans()
+ assert (span.attributes or {}).get("rf.keyword.args") == expected
+
+
+def test_a_log_message_becomes_a_record_of_the_open_span() -> None:
+ """A log message becomes a log record of the span that is open.
+
+ An HTML message is logged inside a keyword. Outcome: the record keeps the
+ body, carries rf.log.level and rf.log.html and references the keyword span.
+ """
+ keyword_result = Result()
+
+ with _make_listener() as listener:
+ listener.start_keyword(None, keyword_result)
+ listener.log_message(Message("x", html=True))
+ listener.end_keyword(None, keyword_result)
+
+ (span,) = listener.spans.get_finished_spans()
+ (item,) = listener.logs.get_finished_logs()
+ record = item.log_record
+ record_attributes = record.attributes or {}
+
+ assert record.body == "x"
+ assert record_attributes["rf.log.level"] == "INFO"
+ assert record_attributes["rf.log.html"] == "true"
+
+ assert record.trace_id == span.context.trace_id
+ assert record.span_id == span.context.span_id
+
+
+def test_log_record_takes_timestamp_from_robotframework_message() -> None:
+ """The log record is stamped with the time of the message.
+
+ A message with a timestamp is logged inside a suite. Outcome: the record's
+ timestamp is that time in nanoseconds since the epoch.
+ """
+ start_time = datetime(2026, 11, 17, 10, 0, 0, 250_000, tzinfo=UTC)
+ with _make_listener() as listener:
+ listener.start_suite(None, Result())
+ listener.log_message(Message("x", timestamp=start_time))
+
+ (item,) = listener.logs.get_finished_logs()
+ assert item.log_record.timestamp == _datetime_to_ns(start_time)
+
+
+@pytest.mark.parametrize(
+ ("level", "severity"),
+ [
+ pytest.param("WARN", SeverityNumber.WARN, id="warn stays warn"),
+ pytest.param("FAIL", SeverityNumber.ERROR, id="fail is error"),
+ pytest.param("SKIP", SeverityNumber.INFO, id="skip is info"),
+ pytest.param("HURZ", SeverityNumber.UNSPECIFIED, id="unknown is unspecified"),
+ ],
+)
+def test_log_levels_map_to_severities_and_preserves_level_in_attribute(
+ level: str, severity: SeverityNumber
+) -> None:
+ """The Robot Framework log level becomes the record severity.
+
+ A message of the given level is logged inside a suite. Outcome: the
+ severity text and rf.log.level are the level as given; the severity
+ number is the OpenTelemetry level of the same name, ERROR for FAIL, INFO
+ for SKIP and UNSPECIFIED for a level OpenTelemetry does not know.
+ """
+ with _make_listener() as listener:
+ listener.start_suite(None, Result())
+ listener.log_message(Message("x", level=level))
+
+ (item,) = listener.logs.get_finished_logs()
+ record = item.log_record
+ assert record.severity_text == level
+ assert record.severity_number == severity
+ assert (record.attributes or {})["rf.log.level"] == level
+
+
+def test_a_log_message_outside_any_span_is_dropped() -> None:
+ """A log message with no open span is dropped.
+
+ A message is logged before any start hook. Outcome: no log record.
+ """
+ with _make_listener() as listener:
+ listener.log_message(Message("orphan"))
+
+ assert not listener.logs.get_finished_logs()
+
+
+def test_log_bodies_are_cut_at_max_log_length() -> None:
+ """Log bodies are cut at max_log_length.
+
+ "Hello world" is logged with max_log_length=5. Outcome: the record body is
+ "Hello".
+ """
+ with _make_listener(max_log_length=5) as listener:
+ listener.start_suite(None, Result())
+ listener.log_message(Message("Hello world"))
+
+ (item,) = listener.logs.get_finished_logs()
+ assert item.log_record.body == "Hello"
+
+
+@pytest.mark.parametrize(
+ ("message", "records"),
+ [
+ pytest.param(SELENIUM_SCREENSHOT, [(PNG, PNG_LOG_ATTRIBUTES)], id="selenium"),
+ pytest.param(BROWSER_SCREENSHOT, [(PNG, PNG_LOG_ATTRIBUTES)], id="browser"),
+ pytest.param(
+ FILE_SCREENSHOT,
+ [(FILE_SCREENSHOT, {"rf.log.level": "INFO", "rf.log.html": "true"})],
+ id="file based stays html",
+ ),
+ pytest.param(
+ f'Before