From c076ab6e751b39d418f9e9b9ad8f88aec2640f80 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Mon, 24 Aug 2026 20:52:58 +0000 Subject: [PATCH 1/6] Define configuration --- .../configurations/artifacts/artifact.py | 2 ++ .../configurations/artifacts/timeline.py | 24 +++++++++++++++++++ .../interuss/scd/single_s2_cell.jsonnet | 23 ++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 monitoring/benchmarker/configurations/artifacts/timeline.py diff --git a/monitoring/benchmarker/configurations/artifacts/artifact.py b/monitoring/benchmarker/configurations/artifacts/artifact.py index dbad71223d..45676261e7 100644 --- a/monitoring/benchmarker/configurations/artifacts/artifact.py +++ b/monitoring/benchmarker/configurations/artifacts/artifact.py @@ -8,8 +8,10 @@ from monitoring.benchmarker.configurations.artifacts.raw_report import ( RawReportSpecification, ) +from monitoring.benchmarker.configurations.artifacts.timeline import TimelineSpecification class ArtifactSpecification(ImplicitDict): raw_report: Optional[RawReportSpecification] matplotlib_figure: Optional[MatplotlibFigureSpecification] + timeline: Optional[TimelineSpecification] diff --git a/monitoring/benchmarker/configurations/artifacts/timeline.py b/monitoring/benchmarker/configurations/artifacts/timeline.py new file mode 100644 index 0000000000..e934c28e46 --- /dev/null +++ b/monitoring/benchmarker/configurations/artifacts/timeline.py @@ -0,0 +1,24 @@ +from typing import Optional + +from implicitdict import ImplicitDict + +from monitoring.benchmarker.configurations.loads import OperationType + + +class TimelineOperation(ImplicitDict): + type: OperationType + + color: Optional[str] + """CSS color (e.g., #ff1122) for this operation. Picked automatically if not specified.""" + + success_indicator_width: Optional[float] + """Width of the termination line indicating success or failure of each of these operations.""" + + +class TimelineSpecification(ImplicitDict): + name: str + """Machine-level name for this report. Used as the output file name.""" + + operations: list[TimelineOperation] + """Operations to display on the timeline.""" + \ No newline at end of file diff --git a/monitoring/benchmarker/configurations/interuss/scd/single_s2_cell.jsonnet b/monitoring/benchmarker/configurations/interuss/scd/single_s2_cell.jsonnet index d4811552d0..16b03b10ef 100644 --- a/monitoring/benchmarker/configurations/interuss/scd/single_s2_cell.jsonnet +++ b/monitoring/benchmarker/configurations/interuss/scd/single_s2_cell.jsonnet @@ -209,6 +209,29 @@ local shape = { name: 'report', }, }, + { + timeline: { + name: 'timeline', + operations: [ + { + type: "workflow.flight_planner.flight", + color: "#32aced", + }, + { + type: "query.astm.f3548.v21.dss.createOperationalIntentReference", + color: "#c7c46b", + }, + { + type: "query.astm.f3548.v21.dss.updateOperationalIntentReference", + color: "#70c76b", + }, + { + type: "query.astm.f3548.v21.dss.deleteOperationalIntentReference", + color: "#c77f6b", + }, + ], + } + }, { matplotlib_figure: { name: 'scalability_curve', From 17a7fcbcd6c73992229542d42c2391393f0864cf Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Mon, 24 Aug 2026 22:56:52 +0000 Subject: [PATCH 2/6] Add Gemini implementation --- .../benchmarker/artifacts/generation.py | 5 + .../artifacts/timeline/__init__.py | 14 + .../artifacts/timeline/templates/index.html | 288 ++++++ .../timeline/templates/scenario.html | 908 ++++++++++++++++++ .../artifacts/timeline/timeline.py | 371 +++++++ 5 files changed, 1586 insertions(+) create mode 100644 monitoring/benchmarker/artifacts/timeline/__init__.py create mode 100644 monitoring/benchmarker/artifacts/timeline/templates/index.html create mode 100644 monitoring/benchmarker/artifacts/timeline/templates/scenario.html create mode 100644 monitoring/benchmarker/artifacts/timeline/timeline.py diff --git a/monitoring/benchmarker/artifacts/generation.py b/monitoring/benchmarker/artifacts/generation.py index cbb4bce2b3..3d662f07ee 100644 --- a/monitoring/benchmarker/artifacts/generation.py +++ b/monitoring/benchmarker/artifacts/generation.py @@ -4,6 +4,7 @@ generate_matplotlib_figure, ) from monitoring.benchmarker.artifacts.raw_report import generate_raw_report +from monitoring.benchmarker.artifacts.timeline.timeline import generate_timeline from monitoring.benchmarker.configurations.artifacts.artifact import ( ArtifactSpecification, ) @@ -52,3 +53,7 @@ def generate_artifacts( if "matplotlib_figure" in spec and spec.matplotlib_figure is not None: generate_matplotlib_figure(report, spec.matplotlib_figure, output_dir) + + if "timeline" in spec and spec.timeline is not None: + generate_timeline(report, spec.timeline, output_dir) + diff --git a/monitoring/benchmarker/artifacts/timeline/__init__.py b/monitoring/benchmarker/artifacts/timeline/__init__.py new file mode 100644 index 0000000000..3aecc097f6 --- /dev/null +++ b/monitoring/benchmarker/artifacts/timeline/__init__.py @@ -0,0 +1,14 @@ +import os + +from jinja2 import Environment, FileSystemLoader + +jinja_env = Environment( + loader=FileSystemLoader( + [ + os.path.abspath(os.path.join(os.path.dirname(__file__), relpath)) + for relpath in ("templates", "../../../monitorlib/html/templates") + ] + ), + trim_blocks=True, + lstrip_blocks=True, +) diff --git a/monitoring/benchmarker/artifacts/timeline/templates/index.html b/monitoring/benchmarker/artifacts/timeline/templates/index.html new file mode 100644 index 0000000000..67afad9e39 --- /dev/null +++ b/monitoring/benchmarker/artifacts/timeline/templates/index.html @@ -0,0 +1,288 @@ + + + + + + Benchmarker Timeline - {{ spec.name }} + + + +
+
+
+

Timeline Artifact: {{ spec.name }}

+
+ Benchmark Performance Timelines +
+
+
+ {% if report.codebase_version %} + Version: {{ report.codebase_version }} + {% endif %} + {% if report.commit_hash %} + Commit: {{ report.commit_hash[:8] }} + {% endif %} +
+
+
+ +
+
+

Configured Operations of Interest

+
+ {% for op in spec.operations %} +
+
+
+
{{ op.type.split('.')[-1] }}
+ {{ op.type }} +
+
+ {% endfor %} +
+
+ +
+
+

Benchmark Scenarios

+
+ + + + + + + + + + + + + {% for s in scenarios_summary %} + + + + + + + + + {% endfor %} + +
ScenarioDurationSteps / ProgressionOriginsOperationsAction
+ + Scenario {{ s.index }} + + {{ "%.1f"|format(s.duration) }}s +
+ {% for step in s.steps %} + + LF {{ step.load_factor }} {{ step.termination_symbol }} + + {% endfor %} +
+ 		{{ s.origins_count }} +
{{ s.operations_count }} total
+
+ {{ s.successful_operations }} ✔ + {% if s.unsuccessful_operations > 0 %} +  | {{ s.unsuccessful_operations }} ✖ + {% endif %} +
+ 		+ + View Timeline → + +
+
+
+ + diff --git a/monitoring/benchmarker/artifacts/timeline/templates/scenario.html b/monitoring/benchmarker/artifacts/timeline/templates/scenario.html new file mode 100644 index 0000000000..78d5a8de4b --- /dev/null +++ b/monitoring/benchmarker/artifacts/timeline/templates/scenario.html @@ -0,0 +1,908 @@ + + + + + + Scenario {{ scenario_index }} Timeline - {{ spec.name }} + + + +
+
+
+ ← Overview +

Scenario {{ scenario_index }} Timeline

+
+ {% if prev_scenario %} + ← Prev + {% endif %} + {% if next_scenario %} + Next → + {% endif %} +
+
+ +
+
+ Operations: + {% for op in timeline_data.operation_types %} +
+ + {{ op.name }} +
+ {% endfor %} +
+ + Success + + Fail +
+
+ +
+ + + +
+ + Duration: {{ "%.1f"|format(timeline_data.scenario_duration) }}s + {{ timeline_data.stats.total_steps }} Steps + {{ timeline_data.stats.total_origins }} Origins + {{ timeline_data.stats.total_operations }} Ops +
+
+
+ +
+ +
+
+ Controls: Left-drag or Mouse wheel to scroll • Shift + Left-drag or Shift + Wheel to zoom +
+
+ + + + + + diff --git a/monitoring/benchmarker/artifacts/timeline/timeline.py b/monitoring/benchmarker/artifacts/timeline/timeline.py new file mode 100644 index 0000000000..3dee17b19c --- /dev/null +++ b/monitoring/benchmarker/artifacts/timeline/timeline.py @@ -0,0 +1,371 @@ +from __future__ import annotations + +import json +import os +import re +from typing import Any + +from loguru import logger + +from monitoring.benchmarker.artifacts.timeline import jinja_env +from monitoring.benchmarker.configurations.artifacts.timeline import ( + TimelineSpecification, +) +from monitoring.benchmarker.reports.report import ( + BenchmarkRunReport, + BenchmarkScenarioReport, + StepTerminationReason, +) +from monitoring.monitorlib.fetch import Query + +DEFAULT_PALETTE = [ + "#32aced", + "#c7c46b", + "#70c76b", + "#c77f6b", + "#9b59b6", + "#e67e22", + "#1abc9c", + "#e74c3c", + "#34495e", + "#16a085", + "#27ae60", + "#2980b9", + "#8e44ad", + "#2c3e50", + "#f39c12", + "#d35400", +] + + +def natural_sort_key(s: str) -> list[int | str]: + return [ + int(text) if text.isdigit() else text.lower() + for text in re.split(r"(\d+)", s) + ] + + +def _extract_query_summary(query: Query | None) -> dict[str, Any] | None: + if query is None: + return None + summary: dict[str, Any] = {} + if "query_type" in query and query.query_type: + summary["query_type"] = query.query_type + if "participant_id" in query and query.participant_id: + summary["participant_id"] = query.participant_id + if "request" in query and query.request: + req = query.request + if "method" in req and req.method: + summary["method"] = req.method + if "url" in req and req.url: + summary["url"] = req.url + if "response" in query and query.response: + resp = query.response + if "status_code" in resp and resp.status_code is not None: + summary["status_code"] = resp.status_code + if "elapsed_s" in resp and resp.elapsed_s is not None: + summary["elapsed_s"] = resp.elapsed_s + return summary if summary else None + + +def _assign_lanes_for_origin(operations: list[dict[str, Any]]) -> list[list[dict[str, Any]]]: + """Assign operations for a single origin into the minimum non-overlapping swim lanes, + preferentially placing longer-running operations in lanes further to the left. + """ + if not operations: + return [[]] + + # Sort by duration descending, tie-break by start time ascending + operations.sort(key=lambda op: (-op["duration"], op["t0"])) + + lanes: list[list[dict[str, Any]]] = [] + for op in operations: + placed = False + for lane in lanes: + # Check overlap with all operations currently in this lane + overlap = any( + max(op["t0"], placed_op["t0"]) < min(op["t1"], placed_op["t1"]) + for placed_op in lane + ) + if not overlap: + lane.append(op) + placed = True + break + if not placed: + lanes.append([op]) + + return lanes if lanes else [[]] + + +def compute_scenario_timeline_data( + scenario_index: int, + scenario: BenchmarkScenarioReport, + spec: TimelineSpecification, +) -> dict[str, Any]: + # Operation specifications & colors + spec_ops_map = {op_spec.type: op_spec for op_spec in spec.operations} + color_map: dict[str, str] = {} + indicator_width_map: dict[str, float] = {} + for idx, op_spec in enumerate(spec.operations): + color = ( + op_spec.color + if "color" in op_spec and op_spec.color + else DEFAULT_PALETTE[idx % len(DEFAULT_PALETTE)] + ) + color_map[op_spec.type] = color + indicator_width_map[op_spec.type] = ( + float(op_spec.success_indicator_width) + if "success_indicator_width" in op_spec + and op_spec.success_indicator_width is not None + else 2.0 + ) + + # Extract steps + steps_data = [] + earliest_step_time = None + latest_step_time = None + for step_idx, step in enumerate(scenario.steps): + t_start = step.start_time.datetime.timestamp() + t_end = step.end_time.datetime.timestamp() + t_stab = ( + step.throughput_stability_time.datetime.timestamp() + if "throughput_stability_time" in step + and step.throughput_stability_time is not None + else None + ) + + if earliest_step_time is None or t_start < earliest_step_time: + earliest_step_time = t_start + if latest_step_time is None or t_end > latest_step_time: + latest_step_time = t_end + + term_reason = str(step.termination_reason) + # Identify symbol + if term_reason == StepTerminationReason.Completed: + term_symbol = "✔" + elif term_reason == StepTerminationReason.StabilityNotAchieved: + term_symbol = "👎" + elif term_reason == StepTerminationReason.Unstable: + term_symbol = "🛑" + else: + term_symbol = "✔" if "complete" in term_reason.lower() else "❓" + + steps_data.append( + { + "step_index": step_idx, + "load_factor": step.load_factor, + "start_time": t_start, + "throughput_stability_time": t_stab, + "end_time": t_end, + "termination_reason": term_reason, + "termination_symbol": term_symbol, + "start_time_iso": str(step.start_time), + "stability_time_iso": str(step.throughput_stability_time) + if "throughput_stability_time" in step + and step.throughput_stability_time is not None + else None, + "end_time_iso": str(step.end_time), + } + ) + + # Collect operations of interest per origin + origin_ops: dict[str, list[dict[str, Any]]] = {} + earliest_op_time = None + latest_op_time = None + total_ops_count = 0 + successful_ops_count = 0 + unsuccessful_ops_count = 0 + + for op_group in scenario.operations: + op_type = op_group.type + if op_type not in spec_ops_map: + continue + + color = color_map[op_type] + indicator_width = indicator_width_map[op_type] + + for orig_group in op_group.origins: + origin = orig_group.origin + if origin not in origin_ops: + origin_ops[origin] = [] + + if "outcomes" in orig_group and orig_group.outcomes: + for outcome in orig_group.outcomes: + if "successful" in outcome and outcome.successful: + for op in outcome.successful: + t0 = op.t0.datetime.timestamp() + t1 = op.t1.datetime.timestamp() + if earliest_op_time is None or t0 < earliest_op_time: + earliest_op_time = t0 + if latest_op_time is None or t1 > latest_op_time: + latest_op_time = t1 + + origin_ops[origin].append( + { + "type": op_type, + "t0": t0, + "t1": t1, + "duration": max(0.0, t1 - t0), + "t0_iso": str(op.t0), + "t1_iso": str(op.t1), + "success": True, + "color": color, + "indicator_width": indicator_width, + "query": _extract_query_summary(op.query) + if "query" in op and op.query is not None + else None, + } + ) + total_ops_count += 1 + successful_ops_count += 1 + + if "unsuccessful" in outcome and outcome.unsuccessful: + for op in outcome.unsuccessful: + t0 = op.t0.datetime.timestamp() + t1 = op.t1.datetime.timestamp() + if earliest_op_time is None or t0 < earliest_op_time: + earliest_op_time = t0 + if latest_op_time is None or t1 > latest_op_time: + latest_op_time = t1 + + origin_ops[origin].append( + { + "type": op_type, + "t0": t0, + "t1": t1, + "duration": max(0.0, t1 - t0), + "t0_iso": str(op.t0), + "t1_iso": str(op.t1), + "success": False, + "color": color, + "indicator_width": indicator_width, + "query": _extract_query_summary(op.query) + if "query" in op and op.query is not None + else None, + } + ) + total_ops_count += 1 + unsuccessful_ops_count += 1 + + # Determine scenario start and end boundaries + all_starts = [t for t in (earliest_step_time, earliest_op_time) if t is not None] + all_ends = [t for t in (latest_step_time, latest_op_time) if t is not None] + + scenario_start = min(all_starts) if all_starts else 0.0 + scenario_end = max(all_ends) if all_ends else scenario_start + 60.0 + if scenario_end <= scenario_start: + scenario_end = scenario_start + 1.0 + + # Sort origins and assign lanes + sorted_origin_names = sorted(origin_ops.keys(), key=natural_sort_key) + origins_data = [] + total_operation_lanes = 0 + + for orig_name in sorted_origin_names: + lanes = _assign_lanes_for_origin(origin_ops[orig_name]) + num_lanes = len(lanes) + total_operation_lanes += num_lanes + origins_data.append( + { + "origin": orig_name, + "num_lanes": num_lanes, + "lanes": lanes, + } + ) + + # Operation types list for legend and UI + operation_types_data = [ + { + "type": op_spec.type, + "name": op_spec.type.split(".")[-1], + "color": color_map[op_spec.type], + "indicator_width": indicator_width_map[op_spec.type], + } + for op_spec in spec.operations + ] + + return { + "scenario_index": scenario_index, + "scenario_start": scenario_start, + "scenario_end": scenario_end, + "scenario_duration": scenario_end - scenario_start, + "steps": steps_data, + "origins": origins_data, + "operation_types": operation_types_data, + "total_operation_lanes": total_operation_lanes, + "stats": { + "total_operations": total_ops_count, + "successful_operations": successful_ops_count, + "unsuccessful_operations": unsuccessful_ops_count, + "total_steps": len(steps_data), + "total_origins": len(origins_data), + }, + } + + +def generate_timeline( + report: BenchmarkRunReport, + spec: TimelineSpecification, + output_dir: str, +) -> None: + timeline_dir = os.path.join(output_dir, spec.name) + os.makedirs(timeline_dir, exist_ok=True) + logger.info(f"Generating timeline artifact in {timeline_dir}") + + scenarios = report.report.scenarios + scenarios_summary = [] + scenarios_timeline_data = [] + + for idx, scenario in enumerate(scenarios): + timeline_data = compute_scenario_timeline_data(idx, scenario, spec) + scenarios_timeline_data.append(timeline_data) + scenarios_summary.append( + { + "index": idx, + "filename": f"s{idx}.html", + "duration": timeline_data["scenario_duration"], + "steps_count": len(timeline_data["steps"]), + "origins_count": len(timeline_data["origins"]), + "operations_count": timeline_data["stats"]["total_operations"], + "successful_operations": timeline_data["stats"]["successful_operations"], + "unsuccessful_operations": timeline_data["stats"]["unsuccessful_operations"], + "steps": timeline_data["steps"], + } + ) + + # Render scenario pages + scenario_template = jinja_env.get_template("scenario.html") + for idx, timeline_data in enumerate(scenarios_timeline_data): + scenario_file = os.path.join(timeline_dir, f"s{idx}.html") + prev_scenario = f"s{idx - 1}.html" if idx > 0 else None + next_scenario = f"s{idx + 1}.html" if idx < len(scenarios) - 1 else None + + with open(scenario_file, "w") as f: + f.write( + scenario_template.render( + scenario_index=idx, + scenario_data_json=json.dumps(timeline_data), + timeline_data=timeline_data, + spec=spec, + report=report, + prev_scenario=prev_scenario, + next_scenario=next_scenario, + total_scenarios=len(scenarios), + ) + ) + + # Render index overview page + index_template = jinja_env.get_template("index.html") + index_file = os.path.join(timeline_dir, "index.html") + with open(index_file, "w") as f: + f.write( + index_template.render( + report=report, + spec=spec, + scenarios_summary=scenarios_summary, + ) + ) + + logger.info( + f"Timeline artifact successfully generated: {index_file} ({len(scenarios)} scenarios)" + ) From 6d5d83dc024f40ba21d0255d9d0e8aecb0227340 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Mon, 24 Aug 2026 23:26:02 +0000 Subject: [PATCH 3/6] Make Gemini adjustments --- .../artifacts/timeline/templates/index.html | 44 +++++------ .../timeline/templates/scenario.html | 33 ++++++--- .../artifacts/timeline/timeline.py | 33 ++++++++- monitoring/monitorlib/formatting.py | 74 +++++++++++++++++++ 4 files changed, 149 insertions(+), 35 deletions(-) diff --git a/monitoring/benchmarker/artifacts/timeline/templates/index.html b/monitoring/benchmarker/artifacts/timeline/templates/index.html index 67afad9e39..665dec30bd 100644 --- a/monitoring/benchmarker/artifacts/timeline/templates/index.html +++ b/monitoring/benchmarker/artifacts/timeline/templates/index.html @@ -155,6 +155,7 @@ font-weight: 500; font-size: 0.875rem; transition: background-color 0.15s ease-in-out; + white-space: nowrap; } .btn:hover { @@ -169,6 +170,7 @@ font-weight: 600; margin-right: 0.25rem; margin-bottom: 0.25rem; + white-space: nowrap; } .status-completed { @@ -204,10 +206,10 @@

Timeline Artifact: {{ spec.name }}

- {% if report.codebase_version %} + {% if "codebase_version" in report and report.codebase_version %} Version: {{ report.codebase_version }} {% endif %} - {% if report.commit_hash %} + {% if "commit_hash" in report and report.commit_hash %} Commit: {{ report.commit_hash[:8] }} {% endif %}
@@ -237,45 +239,45 @@

Benchmark Scenarios

- - + + - - - + + + {% for s in scenarios_summary %} - - + - - + - diff --git a/monitoring/benchmarker/artifacts/timeline/templates/scenario.html b/monitoring/benchmarker/artifacts/timeline/templates/scenario.html index 78d5a8de4b..edb68f69ad 100644 --- a/monitoring/benchmarker/artifacts/timeline/templates/scenario.html +++ b/monitoring/benchmarker/artifacts/timeline/templates/scenario.html @@ -3,7 +3,7 @@ - Scenario {{ scenario_index }} Timeline - {{ spec.name }} + Scenario {{ scenario_index }} ({{ scenario_name }}) Timeline - {{ spec.name }}
ScenarioDurationScenarioDuration Steps / ProgressionOriginsOperationsActionOriginsOperationsAction
+ - Scenario {{ s.index }} + Scenario {{ s.index }} +
+ {{ s.name }} +
{{ "%.1f"|format(s.duration) }}s{{ s.duration_shorthand }}
{% for step in s.steps %} - LF {{ step.load_factor }} {{ step.termination_symbol }} + LF {{ step.load_factor }} {{ step.termination_symbol }} {% endfor %}
{{ s.origins_count }} -
{{ s.operations_count }} total
-
- {{ s.successful_operations }} ✔ - {% if s.unsuccessful_operations > 0 %} -  | {{ s.unsuccessful_operations }} ✖ - {% endif %} +
{{ s.origins_count }} +
{{ s.operations_count }} total
+
+ {{ s.successful_operations }} ✔{% if s.unsuccessful_operations > 0 %} | {{ s.unsuccessful_operations }} ✖{% endif %}
+ - View Timeline → + View Timeline →