diff --git a/monitoring/benchmarker/artifacts/generation.py b/monitoring/benchmarker/artifacts/generation.py index cbb4bce2b3..f395763556 100644 --- a/monitoring/benchmarker/artifacts/generation.py +++ b/monitoring/benchmarker/artifacts/generation.py @@ -4,6 +4,7 @@ generate_matplotlib_figure, ) from monitoring.benchmarker.artifacts.raw_report import generate_raw_report +from monitoring.benchmarker.artifacts.timeline.timeline import generate_timeline from monitoring.benchmarker.configurations.artifacts.artifact import ( ArtifactSpecification, ) @@ -52,3 +53,6 @@ def generate_artifacts( if "matplotlib_figure" in spec and spec.matplotlib_figure is not None: generate_matplotlib_figure(report, spec.matplotlib_figure, output_dir) + + if "timeline" in spec and spec.timeline is not None: + generate_timeline(report, spec.timeline, output_dir) diff --git a/monitoring/benchmarker/artifacts/timeline/__init__.py b/monitoring/benchmarker/artifacts/timeline/__init__.py new file mode 100644 index 0000000000..3aecc097f6 --- /dev/null +++ b/monitoring/benchmarker/artifacts/timeline/__init__.py @@ -0,0 +1,14 @@ +import os + +from jinja2 import Environment, FileSystemLoader + +jinja_env = Environment( + loader=FileSystemLoader( + [ + os.path.abspath(os.path.join(os.path.dirname(__file__), relpath)) + for relpath in ("templates", "../../../monitorlib/html/templates") + ] + ), + trim_blocks=True, + lstrip_blocks=True, +) diff --git a/monitoring/benchmarker/artifacts/timeline/templates/index.html b/monitoring/benchmarker/artifacts/timeline/templates/index.html new file mode 100644 index 0000000000..665dec30bd --- /dev/null +++ b/monitoring/benchmarker/artifacts/timeline/templates/index.html @@ -0,0 +1,290 @@ + + + + + + Benchmarker Timeline - {{ spec.name }} + + + +
+
+
+

Timeline Artifact: {{ spec.name }}

+
+ Benchmark Performance Timelines +
+
+
+ {% if "codebase_version" in report and report.codebase_version %} + Version: {{ report.codebase_version }} + {% endif %} + {% if "commit_hash" in report and report.commit_hash %} + Commit: {{ report.commit_hash[:8] }} + {% endif %} +
+
+
+ +
+
+

Configured Operations of Interest

+
+ {% for op in spec.operations %} +
+
+
+
{{ op.type.split('.')[-1] }}
+ {{ op.type }} +
+
+ {% endfor %} +
+
+ +
+
+

Benchmark Scenarios

+
+ + + + + + + + + + + + + {% for s in scenarios_summary %} + + + + + + + + + {% endfor %} + +
ScenarioDurationSteps / ProgressionOriginsOperationsAction
+ + Scenario {{ s.index }} + +
+ {{ s.name }} +
+ 		{{ s.duration_shorthand }} +
+ {% for step in s.steps %} + + LF {{ step.load_factor }} {{ step.termination_symbol }} + + {% endfor %} +
+ 		{{ s.origins_count }} +
{{ s.operations_count }} total
+
+ {{ s.successful_operations }} ✔{% if s.unsuccessful_operations > 0 %} | {{ s.unsuccessful_operations }} ✖{% endif %} +
+ 		+ + View Timeline → + +
+
+
+ + diff --git a/monitoring/benchmarker/artifacts/timeline/templates/scenario.html b/monitoring/benchmarker/artifacts/timeline/templates/scenario.html new file mode 100644 index 0000000000..3791926c06 --- /dev/null +++ b/monitoring/benchmarker/artifacts/timeline/templates/scenario.html @@ -0,0 +1,994 @@ + + + + + + Scenario {{ scenario_index }} ({{ scenario_name }}) Timeline - {{ spec.name }} + + + +
+
+
+ ← Overview +

Scenario {{ scenario_index }} ({{ scenario_name }}) Timeline

+
+ {% if prev_scenario %} + ← Prev + {% endif %} + {% if next_scenario %} + Next → + {% endif %} +
+
+ +
+
+ Operations: + {% for op in timeline_data.operation_types %} +
+ + {{ op.name }} +
+ {% endfor %} +
+ + Success + + Fail +
+
+ +
+ + + +
+ + Duration: {{ timeline_data.duration_shorthand }} + {{ timeline_data.stats.total_steps }} Steps + {{ timeline_data.stats.total_origins }} Origins + {{ timeline_data.stats.total_operations }} Ops +
+
+
+ +
+ +
+
+ Controls: Left-drag or Mouse wheel to scroll • Shift + Left-drag or Shift + Wheel to zoom • Hover Time column to inspect active operations +
+
+ + + + + + diff --git a/monitoring/benchmarker/artifacts/timeline/timeline.py b/monitoring/benchmarker/artifacts/timeline/timeline.py new file mode 100644 index 0000000000..7f4d338abe --- /dev/null +++ b/monitoring/benchmarker/artifacts/timeline/timeline.py @@ -0,0 +1,404 @@ +from __future__ import annotations + +import json +import os +import re +from typing import Any + +from loguru import logger + +from monitoring.benchmarker.artifacts.timeline import jinja_env +from monitoring.benchmarker.configurations.artifacts.timeline import ( + TimelineSpecification, +) +from monitoring.benchmarker.reports.report import ( + BenchmarkRunReport, + BenchmarkScenarioReport, + StepTerminationReason, +) +from monitoring.monitorlib.fetch import Query +from monitoring.monitorlib.formatting import format_duration_shorthand + +DEFAULT_PALETTE = [ + "#32aced", + "#c7c46b", + "#70c76b", + "#c77f6b", + "#9b59b6", + "#e67e22", + "#1abc9c", + "#e74c3c", + "#34495e", + "#16a085", + "#27ae60", + "#2980b9", + "#8e44ad", + "#2c3e50", + "#f39c12", + "#d35400", +] + + +def natural_sort_key(s: str) -> list[int | str]: + return [ + int(text) if text.isdigit() else text.lower() for text in re.split(r"(\d+)", s) + ] + + +def _extract_query_summary(query: Query | None) -> dict[str, Any] | None: + if query is None: + return None + summary: dict[str, Any] = {} + if "query_type" in query and query.query_type: + summary["query_type"] = query.query_type + if "participant_id" in query and query.participant_id: + summary["participant_id"] = query.participant_id + if "request" in query and query.request: + req = query.request + if "method" in req and req.method: + summary["method"] = req.method + if "url" in req and req.url: + summary["url"] = req.url + if "response" in query and query.response: + resp = query.response + if "status_code" in resp and resp.status_code is not None: + summary["status_code"] = resp.status_code + if "elapsed_s" in resp and resp.elapsed_s is not None: + summary["elapsed_s"] = resp.elapsed_s + return summary if summary else None + + +def _assign_lanes_for_origin( + operations: list[dict[str, Any]], +) -> list[list[dict[str, Any]]]: + """Assign operations for a single origin into the minimum non-overlapping swim lanes, + preferentially placing longer-running operations in lanes further to the left. + """ + if not operations: + return [[]] + + # Sort by duration descending, tie-break by start time ascending + operations.sort(key=lambda op: (-op["duration"], op["t0"])) + + lanes: list[list[dict[str, Any]]] = [] + for op in operations: + placed = False + for lane in lanes: + # Check overlap with all operations currently in this lane + overlap = any( + max(op["t0"], placed_op["t0"]) < min(op["t1"], placed_op["t1"]) + for placed_op in lane + ) + if not overlap: + lane.append(op) + placed = True + break + if not placed: + lanes.append([op]) + + return lanes if lanes else [[]] + + +def compute_scenario_timeline_data( + scenario_index: int, + scenario: BenchmarkScenarioReport, + spec: TimelineSpecification, + scenario_name: str | None = None, +) -> dict[str, Any]: + # Operation specifications & colors + spec_ops_map = {op_spec.type: op_spec for op_spec in spec.operations} + color_map: dict[str, str] = {} + indicator_width_map: dict[str, float] = {} + for idx, op_spec in enumerate(spec.operations): + color = ( + op_spec.color + if "color" in op_spec and op_spec.color + else DEFAULT_PALETTE[idx % len(DEFAULT_PALETTE)] + ) + color_map[op_spec.type] = color + indicator_width_map[op_spec.type] = ( + float(op_spec.success_indicator_width) + if "success_indicator_width" in op_spec + and op_spec.success_indicator_width is not None + else 2.0 + ) + + # Extract steps + steps_data = [] + earliest_step_time = None + latest_step_time = None + for step_idx, step in enumerate(scenario.steps): + t_start = step.start_time.datetime.timestamp() + t_end = step.end_time.datetime.timestamp() + t_stab = ( + step.throughput_stability_time.datetime.timestamp() + if "throughput_stability_time" in step + and step.throughput_stability_time is not None + else None + ) + + if earliest_step_time is None or t_start < earliest_step_time: + earliest_step_time = t_start + if latest_step_time is None or t_end > latest_step_time: + latest_step_time = t_end + + term_reason = str(step.termination_reason) + # Identify symbol + if term_reason == StepTerminationReason.Completed: + term_symbol = "✔" + elif term_reason == StepTerminationReason.StabilityNotAchieved: + term_symbol = "👎" + elif term_reason == StepTerminationReason.Unstable: + term_symbol = "🛑" + else: + term_symbol = "✔" if "complete" in term_reason.lower() else "❓" + + steps_data.append( + { + "step_index": step_idx, + "load_factor": step.load_factor, + "start_time": t_start, + "throughput_stability_time": t_stab, + "end_time": t_end, + "termination_reason": term_reason, + "termination_symbol": term_symbol, + "start_time_iso": str(step.start_time), + "stability_time_iso": str(step.throughput_stability_time) + if "throughput_stability_time" in step + and step.throughput_stability_time is not None + else None, + "end_time_iso": str(step.end_time), + } + ) + + # Collect operations of interest per origin + origin_ops: dict[str, list[dict[str, Any]]] = {} + earliest_op_time = None + latest_op_time = None + total_ops_count = 0 + successful_ops_count = 0 + unsuccessful_ops_count = 0 + + for op_group in scenario.operations: + op_type = op_group.type + if op_type not in spec_ops_map: + continue + + color = color_map[op_type] + indicator_width = indicator_width_map[op_type] + + for orig_group in op_group.origins: + origin = orig_group.origin + if origin not in origin_ops: + origin_ops[origin] = [] + + if "outcomes" in orig_group and orig_group.outcomes: + for outcome in orig_group.outcomes: + if "successful" in outcome and outcome.successful: + for op in outcome.successful: + t0 = op.t0.datetime.timestamp() + t1 = op.t1.datetime.timestamp() + if earliest_op_time is None or t0 < earliest_op_time: + earliest_op_time = t0 + if latest_op_time is None or t1 > latest_op_time: + latest_op_time = t1 + + origin_ops[origin].append( + { + "type": op_type, + "t0": t0, + "t1": t1, + "duration": max(0.0, t1 - t0), + "t0_iso": str(op.t0), + "t1_iso": str(op.t1), + "success": True, + "color": color, + "indicator_width": indicator_width, + "query": _extract_query_summary(op.query) + if "query" in op and op.query is not None + else None, + } + ) + total_ops_count += 1 + successful_ops_count += 1 + + if "unsuccessful" in outcome and outcome.unsuccessful: + for op in outcome.unsuccessful: + t0 = op.t0.datetime.timestamp() + t1 = op.t1.datetime.timestamp() + if earliest_op_time is None or t0 < earliest_op_time: + earliest_op_time = t0 + if latest_op_time is None or t1 > latest_op_time: + latest_op_time = t1 + + origin_ops[origin].append( + { + "type": op_type, + "t0": t0, + "t1": t1, + "duration": max(0.0, t1 - t0), + "t0_iso": str(op.t0), + "t1_iso": str(op.t1), + "success": False, + "color": color, + "indicator_width": indicator_width, + "query": _extract_query_summary(op.query) + if "query" in op and op.query is not None + else None, + } + ) + total_ops_count += 1 + unsuccessful_ops_count += 1 + + # Determine scenario start and end boundaries + all_starts = [t for t in (earliest_step_time, earliest_op_time) if t is not None] + all_ends = [t for t in (latest_step_time, latest_op_time) if t is not None] + + scenario_start = min(all_starts) if all_starts else 0.0 + scenario_end = max(all_ends) if all_ends else scenario_start + 60.0 + if scenario_end <= scenario_start: + scenario_end = scenario_start + 1.0 + + duration = scenario_end - scenario_start + duration_shorthand = format_duration_shorthand(duration) + + # Sort origins and assign lanes + sorted_origin_names = sorted(origin_ops.keys(), key=natural_sort_key) + origins_data = [] + total_operation_lanes = 0 + + for orig_name in sorted_origin_names: + lanes = _assign_lanes_for_origin(origin_ops[orig_name]) + num_lanes = len(lanes) + total_operation_lanes += num_lanes + origins_data.append( + { + "origin": orig_name, + "num_lanes": num_lanes, + "lanes": lanes, + } + ) + + # Operation types list for legend and UI + operation_types_data = [ + { + "type": op_spec.type, + "name": op_spec.type.split(".")[-1], + "color": color_map[op_spec.type], + "indicator_width": indicator_width_map[op_spec.type], + } + for op_spec in spec.operations + ] + + return { + "scenario_index": scenario_index, + "scenario_name": scenario_name or f"Scenario {scenario_index}", + "scenario_start": scenario_start, + "scenario_end": scenario_end, + "scenario_duration": duration, + "duration_shorthand": duration_shorthand, + "steps": steps_data, + "origins": origins_data, + "operation_types": operation_types_data, + "total_operation_lanes": total_operation_lanes, + "stats": { + "total_operations": total_ops_count, + "successful_operations": successful_ops_count, + "unsuccessful_operations": unsuccessful_ops_count, + "total_steps": len(steps_data), + "total_origins": len(origins_data), + }, + } + + +def generate_timeline( + report: BenchmarkRunReport, + spec: TimelineSpecification, + output_dir: str, +) -> None: + timeline_dir = os.path.join(output_dir, spec.name) + os.makedirs(timeline_dir, exist_ok=True) + logger.info(f"Generating timeline artifact in {timeline_dir}") + + scenarios = report.report.scenarios + config_scenarios = ( + report.configuration.scenarios + if "configuration" in report + and report.configuration + and "scenarios" in report.configuration + and report.configuration.scenarios + else [] + ) + + scenarios_summary = [] + scenarios_timeline_data = [] + + for idx, scenario in enumerate(scenarios): + scenario_name = ( + config_scenarios[idx].name + if idx < len(config_scenarios) + and "name" in config_scenarios[idx] + and config_scenarios[idx].name + else f"Scenario {idx}" + ) + timeline_data = compute_scenario_timeline_data( + idx, scenario, spec, scenario_name + ) + scenarios_timeline_data.append(timeline_data) + scenarios_summary.append( + { + "index": idx, + "name": scenario_name, + "filename": f"s{idx}.html", + "duration": timeline_data["scenario_duration"], + "duration_shorthand": timeline_data["duration_shorthand"], + "steps_count": len(timeline_data["steps"]), + "origins_count": len(timeline_data["origins"]), + "operations_count": timeline_data["stats"]["total_operations"], + "successful_operations": timeline_data["stats"][ + "successful_operations" + ], + "unsuccessful_operations": timeline_data["stats"][ + "unsuccessful_operations" + ], + "steps": timeline_data["steps"], + } + ) + + # Render scenario pages + scenario_template = jinja_env.get_template("scenario.html") + for idx, timeline_data in enumerate(scenarios_timeline_data): + scenario_file = os.path.join(timeline_dir, f"s{idx}.html") + prev_scenario = f"s{idx - 1}.html" if idx > 0 else None + next_scenario = f"s{idx + 1}.html" if idx < len(scenarios) - 1 else None + + with open(scenario_file, "w") as f: + f.write( + scenario_template.render( + scenario_index=idx, + scenario_name=timeline_data["scenario_name"], + scenario_data_json=json.dumps(timeline_data), + timeline_data=timeline_data, + spec=spec, + report=report, + prev_scenario=prev_scenario, + next_scenario=next_scenario, + total_scenarios=len(scenarios), + ) + ) + + # Render index overview page + index_template = jinja_env.get_template("index.html") + index_file = os.path.join(timeline_dir, "index.html") + with open(index_file, "w") as f: + f.write( + index_template.render( + report=report, + spec=spec, + scenarios_summary=scenarios_summary, + ) + ) + + logger.info( + f"Timeline artifact successfully generated: {index_file} ({len(scenarios)} scenarios)" + ) diff --git a/monitoring/benchmarker/configurations/artifacts/artifact.py b/monitoring/benchmarker/configurations/artifacts/artifact.py index dbad71223d..6897765fe2 100644 --- a/monitoring/benchmarker/configurations/artifacts/artifact.py +++ b/monitoring/benchmarker/configurations/artifacts/artifact.py @@ -8,8 +8,12 @@ from monitoring.benchmarker.configurations.artifacts.raw_report import ( RawReportSpecification, ) +from monitoring.benchmarker.configurations.artifacts.timeline import ( + TimelineSpecification, +) class ArtifactSpecification(ImplicitDict): raw_report: Optional[RawReportSpecification] matplotlib_figure: Optional[MatplotlibFigureSpecification] + timeline: Optional[TimelineSpecification] diff --git a/monitoring/benchmarker/configurations/artifacts/timeline.py b/monitoring/benchmarker/configurations/artifacts/timeline.py new file mode 100644 index 0000000000..47fd6e12b0 --- /dev/null +++ b/monitoring/benchmarker/configurations/artifacts/timeline.py @@ -0,0 +1,23 @@ +from typing import Optional + +from implicitdict import ImplicitDict + +from monitoring.benchmarker.configurations.loads import OperationType + + +class TimelineOperation(ImplicitDict): + type: OperationType + + color: Optional[str] + """CSS color (e.g., #ff1122) for this operation. Picked automatically if not specified.""" + + success_indicator_width: Optional[float] + """Width of the termination line indicating success or failure of each of these operations.""" + + +class TimelineSpecification(ImplicitDict): + name: str + """Machine-level name for this report. Used as the output file name.""" + + operations: list[TimelineOperation] + """Operations to display on the timeline.""" diff --git a/monitoring/benchmarker/configurations/interuss/scd/single_s2_cell.jsonnet b/monitoring/benchmarker/configurations/interuss/scd/single_s2_cell.jsonnet index d4811552d0..227b1a2975 100644 --- a/monitoring/benchmarker/configurations/interuss/scd/single_s2_cell.jsonnet +++ b/monitoring/benchmarker/configurations/interuss/scd/single_s2_cell.jsonnet @@ -81,8 +81,8 @@ local shape = { flight_generation: { independent_time_location_shape: { time: { - fixed_spacing: '29s', - uniform_random_spacing: '2s', + fixed_spacing: '36s', + uniform_random_spacing: '7.2s', }, location: { fixed_location: location, @@ -93,7 +93,7 @@ local shape = { }, }, flight_execution: { - end_flight_after_start: '10s', + end_flight_after_start: '5s', }, scd_behavior: { dss_pool: ['uss%d_dss_pool' % uss], @@ -209,6 +209,30 @@ local shape = { name: 'report', }, }, + { + timeline: { + name: 'timeline', + operations: [ + { + type: "workflow.flight_planner.flight", + color: "#32aced", + success_indicator_width: 5, + }, + { + type: "query.astm.f3548.v21.dss.createOperationalIntentReference", + color: "#c7c46b", + }, + { + type: "query.astm.f3548.v21.dss.updateOperationalIntentReference", + color: "#70c76b", + }, + { + type: "query.astm.f3548.v21.dss.deleteOperationalIntentReference", + color: "#c2c2c2", + }, + ], + } + }, { matplotlib_figure: { name: 'scalability_curve', diff --git a/monitoring/monitorlib/formatting.py b/monitoring/monitorlib/formatting.py index 3ab1d1ad7e..01fae847be 100644 --- a/monitoring/monitorlib/formatting.py +++ b/monitoring/monitorlib/formatting.py @@ -145,3 +145,76 @@ def make_datetime(t) -> datetime.datetime: def limit_resolution(value: float, resolution: float) -> float: """Change resolution of a value""" return round(value / resolution) * resolution + + +def format_duration_shorthand(duration: float | datetime.timedelta) -> str: + """Produce a smart shorthand string describing a duration. + + Has a maximum of two significant units, and tenths of a second only for values <10s. + Examples: + - 3.8s (<10s) + - 48s (>=10s and <60s) + - 1m3s (60s <= d < 3600s) + - 12m42s + - 1h5m (3600s <= d < 86400s) + - 4h (if minutes == 0) + - 6m (if seconds == 0) + - 1d6h + - 3w4d + """ + if isinstance(duration, datetime.timedelta): + seconds = duration.total_seconds() + else: + seconds = float(duration) + + if seconds < 0: + return f"-{format_duration_shorthand(-seconds)}" + + if seconds < 10.0: + rounded = round(seconds, 1) + if rounded >= 10.0: + return "10s" + return f"{rounded:.1f}s" + + if seconds < 60.0: + rounded = round(seconds) + if rounded >= 60: + return "1m" + return f"{rounded}s" + + if seconds < 3600.0: + total_sec = round(seconds) + mins = total_sec // 60 + secs = total_sec % 60 + if mins >= 60: + return "1h" + if secs == 0: + return f"{mins}m" + return f"{mins}m{secs}s" + + if seconds < 86400.0: + total_min = round(seconds / 60.0) + hours = total_min // 60 + mins = total_min % 60 + if hours >= 24: + return "1d" + if mins == 0: + return f"{hours}h" + return f"{hours}h{mins}m" + + if seconds < 604800.0: + total_hours = round(seconds / 3600.0) + days = total_hours // 24 + hours = total_hours % 24 + if days >= 7: + return "1w" + if hours == 0: + return f"{days}d" + return f"{days}d{hours}h" + + total_days = round(seconds / 86400.0) + weeks = total_days // 7 + days = total_days % 7 + if days == 0: + return f"{weeks}w" + return f"{weeks}w{days}d" diff --git a/schemas/monitoring/benchmarker/configurations/artifacts/artifact/ArtifactSpecification.json b/schemas/monitoring/benchmarker/configurations/artifacts/artifact/ArtifactSpecification.json index 4bc42b1ea3..d821b0c062 100644 --- a/schemas/monitoring/benchmarker/configurations/artifacts/artifact/ArtifactSpecification.json +++ b/schemas/monitoring/benchmarker/configurations/artifacts/artifact/ArtifactSpecification.json @@ -26,6 +26,16 @@ "$ref": "../raw_report/RawReportSpecification.json" } ] + }, + "timeline": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "../timeline/TimelineSpecification.json" + } + ] } }, "type": "object" diff --git a/schemas/monitoring/benchmarker/configurations/artifacts/timeline/TimelineOperation.json b/schemas/monitoring/benchmarker/configurations/artifacts/timeline/TimelineOperation.json new file mode 100644 index 0000000000..3aecfa21d4 --- /dev/null +++ b/schemas/monitoring/benchmarker/configurations/artifacts/timeline/TimelineOperation.json @@ -0,0 +1,32 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/benchmarker/configurations/artifacts/timeline/TimelineOperation.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "monitoring.benchmarker.configurations.artifacts.timeline.TimelineOperation, as defined in monitoring/benchmarker/configurations/artifacts/timeline.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "color": { + "description": "CSS color (e.g., #ff1122) for this operation. Picked automatically if not specified.", + "type": [ + "string", + "null" + ] + }, + "success_indicator_width": { + "description": "Width of the termination line indicating success or failure of each of these operations.", + "type": [ + "number", + "null" + ] + }, + "type": { + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" +} \ No newline at end of file diff --git a/schemas/monitoring/benchmarker/configurations/artifacts/timeline/TimelineSpecification.json b/schemas/monitoring/benchmarker/configurations/artifacts/timeline/TimelineSpecification.json new file mode 100644 index 0000000000..6114a86cc2 --- /dev/null +++ b/schemas/monitoring/benchmarker/configurations/artifacts/timeline/TimelineSpecification.json @@ -0,0 +1,27 @@ +{ + "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/benchmarker/configurations/artifacts/timeline/TimelineSpecification.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "monitoring.benchmarker.configurations.artifacts.timeline.TimelineSpecification, as defined in monitoring/benchmarker/configurations/artifacts/timeline.py", + "properties": { + "$ref": { + "description": "Path to content that replaces the $ref", + "type": "string" + }, + "name": { + "description": "Machine-level name for this report. Used as the output file name.", + "type": "string" + }, + "operations": { + "description": "Operations to display on the timeline.", + "items": { + "$ref": "TimelineOperation.json" + }, + "type": "array" + } + }, + "required": [ + "name", + "operations" + ], + "type": "object" +} \ No newline at end of file