diff --git a/.github/workflows/viewer.yml b/.github/workflows/viewer.yml new file mode 100644 index 0000000..dfd363e --- /dev/null +++ b/.github/workflows/viewer.yml @@ -0,0 +1,31 @@ +name: viewer +on: + pull_request: + paths: ["viewer/**", "schema/**", ".github/workflows/viewer.yml"] + push: + branches: [main, master] + paths: ["viewer/**", "schema/**", ".github/workflows/viewer.yml"] +jobs: + build-and-render: + runs-on: ubuntu-latest + defaults: + run: + working-directory: viewer + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: npm + cache-dependency-path: viewer/package-lock.json + - run: npm ci + - run: npm run check + - run: npm run build + - run: npx playwright install --with-deps chromium + - run: npm run render:ci + - uses: actions/upload-artifact@v4 + with: + name: eval-viewer-and-chart + path: | + viewer/dist + viewer/chart-artifacts diff --git a/.gitignore b/.gitignore index 72a9547..3afb48b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ /target +/viewer/node_modules +/viewer/dist +/viewer/chart-artifacts *.swp .DS_Store diff --git a/crates/eval/src/bundle.rs b/crates/eval/src/bundle.rs new file mode 100644 index 0000000..819f870 --- /dev/null +++ b/crates/eval/src/bundle.rs @@ -0,0 +1,298 @@ +//! Portable, attributed benchmark results for files, APIs, viewers, and CI. + +use std::collections::HashSet; +use std::path::Path; + +use serde::{Deserialize, Serialize}; + +use crate::ScoreReport; + +pub const RESULT_BUNDLE_SCHEMA: &str = "eval.result-bundle.v1"; +pub const RESULT_CATALOG_SCHEMA: &str = "eval.result-catalog.v1"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResultBundle { + pub schema: String, + pub id: String, + pub generated_at: String, + pub benchmark: BenchmarkIdentity, + pub subject: SubjectIdentity, + pub provenance: Provenance, + pub metrics: Vec, + pub observations: Vec, + pub runs: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BenchmarkIdentity { + pub id: String, + pub version: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub title: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SubjectIdentity { + pub kind: String, + pub provider: String, + pub name: String, + pub revision: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Provenance { + pub harness_commit: String, + pub task_set_commit: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scorer: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_url: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MetricDefinition { + pub id: String, + pub label: String, + pub unit: String, + pub goal: MetricGoal, + pub aggregation: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MetricGoal { + Minimize, + Maximize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MetricObservation { + pub metric_id: String, + pub value: f64, + pub sample_count: usize, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub lower_bound: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub upper_bound: Option, + pub run_ids: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub task_ids: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RunRecord { + pub id: String, + pub task_id: String, + pub trial: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cost_usd: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub score: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub artifacts: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ArtifactRef { + pub name: String, + pub media_type: String, + pub url: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sha256: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResultCatalog { + pub schema: String, + pub bundles: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CatalogEntry { + pub id: String, + pub url: String, + pub benchmark_id: String, + pub subject: SubjectIdentity, + pub generated_at: String, +} + +#[derive(Debug, thiserror::Error)] +pub enum BundleError { + #[error("result bundle must use schema {RESULT_BUNDLE_SCHEMA}")] + WrongSchema, + #[error("duplicate {kind} id: {id}")] + DuplicateId { kind: &'static str, id: String }, + #[error("observation references unknown metric: {0}")] + UnknownMetric(String), + #[error("observation references unknown run: {0}")] + UnknownRun(String), + #[error("metric observation must reference at least one sample and run")] + EmptyObservation, + #[error("confidence interval is invalid for metric {0}")] + InvalidBounds(String), + #[error("reading result bundle {path}")] + Read { + path: String, + #[source] + source: std::io::Error, + }, + #[error("parsing result bundle")] + Parse(#[from] serde_json::Error), + #[error("writing result bundle {path}")] + Write { + path: String, + #[source] + source: std::io::Error, + }, +} + +impl ResultBundle { + pub fn validate(&self) -> Result<(), BundleError> { + if self.schema != RESULT_BUNDLE_SCHEMA { + return Err(BundleError::WrongSchema); + } + let mut metric_ids = HashSet::new(); + for metric in &self.metrics { + if !metric_ids.insert(metric.id.as_str()) { + return Err(BundleError::DuplicateId { + kind: "metric", + id: metric.id.clone(), + }); + } + } + let mut run_ids = HashSet::new(); + for run in &self.runs { + if !run_ids.insert(run.id.as_str()) { + return Err(BundleError::DuplicateId { + kind: "run", + id: run.id.clone(), + }); + } + } + for observation in &self.observations { + if !metric_ids.contains(observation.metric_id.as_str()) { + return Err(BundleError::UnknownMetric(observation.metric_id.clone())); + } + if observation.sample_count == 0 || observation.run_ids.is_empty() { + return Err(BundleError::EmptyObservation); + } + if observation + .run_ids + .iter() + .any(|id| !run_ids.contains(id.as_str())) + { + let id = observation + .run_ids + .iter() + .find(|id| !run_ids.contains(id.as_str())) + .unwrap(); + return Err(BundleError::UnknownRun(id.clone())); + } + if matches!((observation.lower_bound, observation.upper_bound), (Some(low), Some(high)) if low > observation.value || high < observation.value || low > high) + { + return Err(BundleError::InvalidBounds(observation.metric_id.clone())); + } + } + Ok(()) + } + + pub fn from_json_file(path: &Path) -> Result { + let bytes = std::fs::read(path).map_err(|source| BundleError::Read { + path: path.display().to_string(), + source, + })?; + let bundle: Self = serde_json::from_slice(&bytes)?; + bundle.validate()?; + Ok(bundle) + } + + pub fn write_json_file(&self, path: &Path) -> Result<(), BundleError> { + self.validate()?; + let mut bytes = serde_json::to_vec_pretty(self)?; + bytes.push(b'\n'); + std::fs::write(path, bytes).map_err(|source| BundleError::Write { + path: path.display().to_string(), + source, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn bundle() -> ResultBundle { + ResultBundle { + schema: RESULT_BUNDLE_SCHEMA.into(), + id: "run-1".into(), + generated_at: "2026-09-24T00:00:00Z".into(), + benchmark: BenchmarkIdentity { + id: "cadbench".into(), + version: "1".into(), + title: None, + }, + subject: SubjectIdentity { + kind: "model".into(), + provider: "example".into(), + name: "model-a".into(), + revision: "2026-09".into(), + }, + provenance: Provenance { + harness_commit: "abc".into(), + task_set_commit: "def".into(), + scorer: Some("objective-v1".into()), + source_url: None, + }, + metrics: vec![MetricDefinition { + id: "pass_rate".into(), + label: "Pass rate".into(), + unit: "percent".into(), + goal: MetricGoal::Maximize, + aggregation: "mean".into(), + }], + observations: vec![MetricObservation { + metric_id: "pass_rate".into(), + value: 0.75, + sample_count: 1, + lower_bound: None, + upper_bound: None, + run_ids: vec!["r1".into()], + task_ids: vec!["task-1".into()], + }], + runs: vec![RunRecord { + id: "r1".into(), + task_id: "task-1".into(), + trial: 1, + started_at: None, + duration_ms: Some(100), + cost_usd: None, + score: None, + artifacts: vec![], + }], + } + } + + #[test] + fn validates_attributed_observations() { + bundle().validate().unwrap(); + } + + #[test] + fn rejects_unknown_run_attribution() { + let mut value = bundle(); + value.observations[0].run_ids[0] = "missing".into(); + assert!(matches!(value.validate(), Err(BundleError::UnknownRun(_)))); + } + + #[test] + fn round_trips_json() { + let value = bundle(); + let encoded = serde_json::to_string(&value).unwrap(); + let decoded: ResultBundle = serde_json::from_str(&encoded).unwrap(); + decoded.validate().unwrap(); + } +} diff --git a/crates/eval/src/lib.rs b/crates/eval/src/lib.rs index dfe55c5..e7a37f8 100644 --- a/crates/eval/src/lib.rs +++ b/crates/eval/src/lib.rs @@ -62,10 +62,16 @@ //! ``` mod backend; +mod bundle; mod score; mod task; pub mod testing; pub use backend::Backend; +pub use bundle::{ + ArtifactRef, BenchmarkIdentity, BundleError, CatalogEntry, MetricDefinition, MetricGoal, + MetricObservation, Provenance, RESULT_BUNDLE_SCHEMA, RESULT_CATALOG_SCHEMA, ResultBundle, + ResultCatalog, RunRecord, SubjectIdentity, +}; pub use score::{CriterionResult, ScoreReport, Verdict}; pub use task::{Criterion, Task}; diff --git a/crates/eval/src/task.rs b/crates/eval/src/task.rs index aaf4433..5d0c1e4 100644 --- a/crates/eval/src/task.rs +++ b/crates/eval/src/task.rs @@ -21,6 +21,15 @@ pub struct Task { /// layer as context. Never parsed for control flow here or in the /// backend — the typed decisions, not the prose, choose the design. pub brief: String, + /// What the backend is given besides the brief -- input files, stated + /// parameters -- as the task file's `[input]` table. + /// + /// Untyped here on purpose: what a task can hand a backend is the domain + /// harness's business (an SVG and a height for a CAD part, a netlist for a + /// board), so each harness parses this into its own type when it loads + /// the task and refuses one it cannot read. Absent for a brief-only task. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input: Option, pub rubric: Vec>, } @@ -44,3 +53,43 @@ impl Task { self.rubric.iter().map(|c| c.id.as_str()).collect() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Debug, Deserialize)] + #[serde(tag = "kind", rename_all = "snake_case")] + enum Check { + Anything, + } + + #[test] + fn a_tasks_input_table_is_kept_and_a_brief_only_task_has_none() { + let with: Task = toml::from_str( + r#" + id = "t" + family = "f" + brief = "b" + [input] + svg = "part.svg" + height_mm = 12.0 + [[rubric]] + id = "r" + description = "d" + kind = "anything" + "#, + ) + .unwrap(); + let input = with.input.expect("the [input] table"); + assert_eq!(input["svg"].as_str(), Some("part.svg")); + assert_eq!(input["height_mm"].as_float(), Some(12.0)); + assert!(matches!(with.rubric[0].check, Check::Anything)); + + let without: Task = toml::from_str( + "id = \"t\"\nfamily = \"f\"\nbrief = \"b\"\nrubric = []\n", + ) + .unwrap(); + assert!(without.input.is_none()); + } +} diff --git a/schema/eval.result-bundle.v1.schema.json b/schema/eval.result-bundle.v1.schema.json new file mode 100644 index 0000000..bded683 --- /dev/null +++ b/schema/eval.result-bundle.v1.schema.json @@ -0,0 +1,45 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://fpl.dev/schemas/eval.result-bundle.v1.schema.json", + "title": "FPL eval result bundle v1", + "type": "object", + "required": ["schema", "id", "generated_at", "benchmark", "subject", "provenance", "metrics", "observations", "runs"], + "properties": { + "schema": { "const": "eval.result-bundle.v1" }, + "id": { "type": "string", "minLength": 1 }, + "generated_at": { "type": "string", "format": "date-time" }, + "benchmark": { + "type": "object", "required": ["id", "version"], + "properties": { "id": { "type": "string" }, "version": { "type": "string" }, "title": { "type": "string" } }, + "additionalProperties": false + }, + "subject": { "$ref": "#/$defs/subject" }, + "provenance": { + "type": "object", "required": ["harness_commit", "task_set_commit"], + "properties": { "harness_commit": { "type": "string" }, "task_set_commit": { "type": "string" }, "scorer": { "type": "string" }, "source_url": { "type": "string", "format": "uri-reference" } }, + "additionalProperties": false + }, + "metrics": { "type": "array", "items": { "$ref": "#/$defs/metric" } }, + "observations": { "type": "array", "items": { "$ref": "#/$defs/observation" } }, + "runs": { "type": "array", "items": { "$ref": "#/$defs/run" } } + }, + "additionalProperties": false, + "$defs": { + "subject": { + "type": "object", "required": ["kind", "provider", "name", "revision"], + "properties": { "kind": { "type": "string" }, "provider": { "type": "string" }, "name": { "type": "string" }, "revision": { "type": "string" } }, "additionalProperties": false + }, + "metric": { + "type": "object", "required": ["id", "label", "unit", "goal", "aggregation"], + "properties": { "id": { "type": "string" }, "label": { "type": "string" }, "unit": { "type": "string" }, "goal": { "enum": ["minimize", "maximize"] }, "aggregation": { "type": "string" } }, "additionalProperties": false + }, + "observation": { + "type": "object", "required": ["metric_id", "value", "sample_count", "run_ids"], + "properties": { "metric_id": { "type": "string" }, "value": { "type": "number" }, "sample_count": { "type": "integer", "minimum": 1 }, "lower_bound": { "type": "number" }, "upper_bound": { "type": "number" }, "run_ids": { "type": "array", "minItems": 1, "items": { "type": "string" } }, "task_ids": { "type": "array", "items": { "type": "string" } } }, "additionalProperties": false + }, + "run": { + "type": "object", "required": ["id", "task_id", "trial"], + "properties": { "id": { "type": "string" }, "task_id": { "type": "string" }, "trial": { "type": "integer", "minimum": 0 }, "started_at": { "type": "string", "format": "date-time" }, "duration_ms": { "type": "integer", "minimum": 0 }, "cost_usd": { "type": "number", "minimum": 0 }, "score": { "type": "object" }, "artifacts": { "type": "array", "items": { "type": "object", "required": ["name", "media_type", "url"], "properties": { "name": { "type": "string" }, "media_type": { "type": "string" }, "url": { "type": "string", "format": "uri-reference" }, "sha256": { "type": "string" } }, "additionalProperties": false } }, "additionalProperties": false + } + } +} diff --git a/viewer/README.md b/viewer/README.md new file mode 100644 index 0000000..81a7843 --- /dev/null +++ b/viewer/README.md @@ -0,0 +1,11 @@ +# Eval result viewer + +The viewer consumes `eval.result-bundle.v1` without containing benchmark-specific scoring logic. + +- Default: loads `public/results/catalog.json`. +- API/catalog: `/?catalog=https://example.test/api/eval/catalog.json` +- Direct bundles: repeat `?bundle=https://example.test/results/run.json`. +- Offline/Canvas: open or import the same JSON bundle file. +- CI: `npm run render:ci` writes attributed `pareto.svg` and `pareto.png` artifacts. + +Remote APIs must return the same JSON shapes and allow the viewer origin through CORS. A catalog is only an index of bundle URLs; immutable bundles remain the source of truth. diff --git a/viewer/index.html b/viewer/index.html new file mode 100644 index 0000000..85072ef --- /dev/null +++ b/viewer/index.html @@ -0,0 +1,13 @@ + + + + + + + Eval evidence viewer + + +
+ + + diff --git a/viewer/package-lock.json b/viewer/package-lock.json new file mode 100644 index 0000000..42ff9a6 --- /dev/null +++ b/viewer/package-lock.json @@ -0,0 +1,1257 @@ +{ + "name": "@fpl/eval-viewer", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@fpl/eval-viewer", + "version": "0.1.0", + "devDependencies": { + "@playwright/test": "1.63.0", + "@sveltejs/vite-plugin-svelte": "7.3.1", + "svelte": "5.57.1", + "svelte-check": "4.7.6", + "typescript": "5.9.3", + "vite": "8.3.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.151.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.151.0.tgz", + "integrity": "sha512-J1yXrIlNDZVzE3ada310xeAw7nH8yCAyLPuUIsjKatFPmfn5bS1oW+cM+QsGOtVWd5nhSpbwZWx/rue+r5Z+PA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/oxc-project" + } + }, + "node_modules/@playwright/test": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.63.0.tgz", + "integrity": "sha512-oxMK4vllB9RK5NQ2l1pq1IfOf2AvnEuj/vYGDj0H2nMtmtZpKtCwt/l00GEO6xjGfpBNAvjovvYdCm50dRQkpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@rolldown/binding-android-arm-eabi": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.10.tgz", + "integrity": "sha512-bp9svZb+QurZeh+8H4BhrZkifEB0YBNvTVzNSJnJQkj4NrRwmQoDUCGP0vSN7PbvLeM7l1tK6GXL8mrTiH2myg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.10.tgz", + "integrity": "sha512-wm6Dld3RXUAZ/gRWKyUy+4W1B5CB5UeFaOzsSWJWEdxZXHH8rCYiZ5dGe6oJmhsunAPWzL7FZV+VtvmN5Ye2eA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.10.tgz", + "integrity": "sha512-UbEfXq/AqGNgRTV3ik+X/iR6mUxu2QdYAadwRxJWquUGnW6gDqdP1FtLtFXRow7RJx0ssRwi80XAPr4r+4DtsA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.10.tgz", + "integrity": "sha512-7f5h17q5KZVx/ji1vb8OTq31ch1O2I7K8NPIr44GkyWTApXMIsmhWqZfgpOH10xeauqghDAvGlZktasCkcF6Eg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.10.tgz", + "integrity": "sha512-ynOk/eEYhC6ZB2xCGvKrEOwE58oBy9LnrAqtkrDF9Fz1VTaNdGZTsV0VarJdhPwb+sOJTGjCLwcuyRJZ1dnMcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.10.tgz", + "integrity": "sha512-ERrAs185meZZhGan7a4l3RiiJK1ArSDlHdST++uvSxe+FDbR4TwUPahT/cbZJvaG6fIpDpF78surN+tX708Y4Q==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.10.tgz", + "integrity": "sha512-KN7OHKD0J3jy1UzBwZWPxpwhODf9IARUIJcrH+yLYKOcmegZ8luEUM38lDP1bDVj40yP6PsSzCqOJF76vljFnQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.10.tgz", + "integrity": "sha512-8l9wP8O+wa8zD6iw6egSfzVtu7oZVfH3hlUsMM4MwbLMhxleqeoXbZzjddyK3YyNlwLhqznq3tF7PkNJ8T/V2w==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.10.tgz", + "integrity": "sha512-SeXNKeQzA5kLhz/J0CH6ZP0/HJ3v1xm/0YbiYpE0kK7emfRC2OIGGIaE14xzkISEGv2aYuUSpiLiU5Gbq+OI0A==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.10.tgz", + "integrity": "sha512-mtht0nR+y8/hart4175Ll15w7lY8dg7CtQ+j2FDNTsDRspOWTK/2V3l0aj9sIj7XmvqxT8Yli/wq22e7feTTWg==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.10.tgz", + "integrity": "sha512-FSM94nGd55NYo48usCyM/nHfUKRnqc9+b0vJNuKV0oCCpIp/OGims7rO1Nv/DkFkt0S/s2rxsJ2kkS8J3HcpeA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.10.tgz", + "integrity": "sha512-C3YxNB16myRLs7o+B+6PnQ6jBsdIS4+AE4Ah8glVGhDpEv9AOvxhZ/1duAb4B0UGczEK/lBbccksd8VI+p6zfw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.10.tgz", + "integrity": "sha512-571TlE/F1eeTjjdjYAMMMPs1Mfv3MtX6s3+ZKVU6HiUjZ5Njc6c/qzNy/8K3zALTZnaw3JQVYrHxvNfjm43KAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.10.tgz", + "integrity": "sha512-QXW+ZWaiqs2c7Fi++D/SsW07LTPcUrncxcskJGfGNBoaLik1IU6fJymz4HsqwEO0u5Iq11yTO0B/mc4cPk7jrQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.10.tgz", + "integrity": "sha512-5FQFGgah17YeMtG1Yd5a+rMxQpTksyNXxRtKz06FVTaQw3RKYUJQbUoKk0/5jrXBpDo+7makNP7UHA2LQyH64A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@sveltejs/acorn-typescript": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/@sveltejs/acorn-typescript/-/acorn-typescript-1.0.13.tgz", + "integrity": "sha512-wgKggnhZVL9Bfx1OaKKTrYY9BFRk6C8UAkQNUcIv1+llzYrIqy+RZm5HPKzn0NpEBvTVhTqB4kQyllZywsRBRQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^8.9.0" + } + }, + "node_modules/@sveltejs/load-config": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@sveltejs/load-config/-/load-config-0.2.3.tgz", + "integrity": "sha512-VT3qmUb8pRV2QrZjd8iAmtg8lf4W0TIjZbvXtz5MKei/q96teWZgGJyyidJzOjzZzvdq616eSRVeMYIQChUTAQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 18.0.0" + } + }, + "node_modules/@sveltejs/vite-plugin-svelte": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-7.3.1.tgz", + "integrity": "sha512-ZPsLN8B1e/En+Ak5s4V7srFDT532oS0qieLsQwu63NGKsS+iAjoO2Js1BochlHlglcU+Pt7WAO3C5Ee+4f6gVA==", + "dev": true, + "license": "MIT", + "dependencies": { + "deepmerge": "^4.3.1", + "magic-string": "^1.0.0", + "obug": "^2.1.0", + "vitefu": "^1.1.2" + }, + "engines": { + "node": "^20.19 || ^22.12 || >=24" + }, + "peerDependencies": { + "svelte": "^5.46.4", + "vite": "^8.0.0-beta.7 || ^8.0.0" + } + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/acorn": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", + "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/aria-query": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.1.tgz", + "integrity": "sha512-Z/ZeOgVl7bcSYZ/u/rh0fOpvEpq//LZmdbkXyc7syVzjPAhfOa9ebsdTSjEBDU4vs5nC98Kfduj1uFo0qyET3g==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/axobject-query": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", + "integrity": "sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/chokidar": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", + "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/devalue": { + "version": "5.9.4", + "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.9.4.tgz", + "integrity": "sha512-sPAT4pztbu6586/hrhOnMKS17IJrvg12mXiSPSS3W5qDeN2RGgvZ0diZCm31dBbnevfVmujNO3IM2wrS4Y2Rhg==", + "dev": true, + "license": "MIT" + }, + "node_modules/esm-env": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.2.2.tgz", + "integrity": "sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/esrap": { + "version": "2.3.11", + "resolved": "https://registry.npmjs.org/esrap/-/esrap-2.3.11.tgz", + "integrity": "sha512-1kmQFiDHcf1T69K9qKtn2MasiKlaq6xNw2ZEgHD8jNyyoNEfx9eCOiJpYDmc0Yk0FKT0bzx5dyBocMDAitWpaQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.4.15" + }, + "peerDependencies": { + "@typescript-eslint/types": "^8.2.0" + }, + "peerDependenciesMeta": { + "@typescript-eslint/types": { + "optional": true + } + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/is-reference": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-3.0.3.tgz", + "integrity": "sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.6" + } + }, + "node_modules/lightningcss": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", + "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.33.0", + "lightningcss-darwin-arm64": "1.33.0", + "lightningcss-darwin-x64": "1.33.0", + "lightningcss-freebsd-x64": "1.33.0", + "lightningcss-linux-arm-gnueabihf": "1.33.0", + "lightningcss-linux-arm64-gnu": "1.33.0", + "lightningcss-linux-arm64-musl": "1.33.0", + "lightningcss-linux-x64-gnu": "1.33.0", + "lightningcss-linux-x64-musl": "1.33.0", + "lightningcss-win32-arm64-msvc": "1.33.0", + "lightningcss-win32-x64-msvc": "1.33.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", + "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", + "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", + "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", + "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", + "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", + "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", + "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", + "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", + "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", + "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", + "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/locate-character": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-character/-/locate-character-3.0.0.tgz", + "integrity": "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA==", + "dev": true, + "license": "MIT" + }, + "node_modules/magic-string": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-1.4.2.tgz", + "integrity": "sha512-vG+rjFRj1PqdIBozIxAGMjPlOhaVe+GXpbttY/iSK7rGcJRMlwNJO7dcUwmUqkymsFLJiNGI06t4D7Fr7yRC9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.6.0" + } + }, + "node_modules/mri": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", + "integrity": "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/nanoid": { + "version": "3.3.19", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", + "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/obug": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.2.1.tgz", + "integrity": "sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/playwright": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", + "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", + "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/postcss": { + "version": "8.5.28", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", + "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.18", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/readdirp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", + "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/rolldown": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.10.tgz", + "integrity": "sha512-OxkA08pSryMK7B3XiFA09B4OJ1xJMPgIYCBMY2xchzpqgBGsV1o0DetPAE+Sl3N3L4oCPiEzmHVSOj7iR04Zog==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.151.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm-eabi": "1.2.10", + "@rolldown/binding-android-arm64": "1.2.10", + "@rolldown/binding-darwin-arm64": "1.2.10", + "@rolldown/binding-darwin-x64": "1.2.10", + "@rolldown/binding-freebsd-x64": "1.2.10", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.10", + "@rolldown/binding-linux-arm64-gnu": "1.2.10", + "@rolldown/binding-linux-arm64-musl": "1.2.10", + "@rolldown/binding-linux-ppc64-gnu": "1.2.10", + "@rolldown/binding-linux-s390x-gnu": "1.2.10", + "@rolldown/binding-linux-x64-gnu": "1.2.10", + "@rolldown/binding-linux-x64-musl": "1.2.10", + "@rolldown/binding-openharmony-arm64": "1.2.10", + "@rolldown/binding-win32-arm64-msvc": "1.2.10", + "@rolldown/binding-win32-x64-msvc": "1.2.10" + } + }, + "node_modules/sade": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", + "integrity": "sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==", + "dev": true, + "license": "MIT", + "dependencies": { + "mri": "^1.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/svelte": { + "version": "5.57.1", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.57.1.tgz", + "integrity": "sha512-Uqj49lWKB+iSSnneuwiYYJ7MZgkB+eXr0LXBhv4uDuAkXqnWmq65Sxflfvp0Lc6MdKjMUxGaeOKWJqz5SNiVIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/remapping": "^2.3.5", + "@jridgewell/sourcemap-codec": "^1.6.0", + "@sveltejs/acorn-typescript": "^1.0.13", + "@types/estree": "^1.0.9", + "acorn": "^8.18.0", + "aria-query": "5.3.1", + "axobject-query": "^4.1.0", + "clsx": "^2.1.1", + "devalue": "^5.9.2", + "esm-env": "^1.2.1", + "esrap": "^2.3.6", + "is-reference": "^3.0.3", + "locate-character": "^3.0.0", + "magic-string": "^0.30.11", + "zimmerframe": "^1.1.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/svelte-check": { + "version": "4.7.6", + "resolved": "https://registry.npmjs.org/svelte-check/-/svelte-check-4.7.6.tgz", + "integrity": "sha512-t2scM//ZuVbSY/T2w6FSBw1v9s2NEmh/g+sy1lqtosW5ylBV5AF4wFb1Ts9Kf3MbfPDUDJDZ9L436YT0SPTdvw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.25", + "@sveltejs/load-config": "^0.2.3", + "chokidar": "^4.0.1", + "fdir": "^6.2.0", + "picocolors": "^1.0.0", + "sade": "^1.7.4" + }, + "bin": { + "svelte-check": "bin/svelte-check" + }, + "engines": { + "node": ">= 18.0.0" + }, + "peerDependencies": { + "svelte": "^4.0.0 || ^5.0.0-next.0", + "typescript": "^5.0.0 || ^6.0.0" + } + }, + "node_modules/svelte/node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/vite": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.3.0.tgz", + "integrity": "sha512-lhZBVvEHefgE+HQZC9O7EBJgCU/nVzFNl7vkS4RE0APtWLP02/8QVIkQtzBxPquh7lq5/78NHipTj7ODQ6XuyQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "lightningcss": "^1.33.0", + "picomatch": "^4.0.7", + "postcss": "^8.5.28", + "rolldown": "~1.2.6", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.7.1", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitefu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/vitefu/-/vitefu-1.1.3.tgz", + "integrity": "sha512-ub4okH7Z5KLjb6hDyjqrGXqWtWvoYdU3IGm/NorpgHncKoLTCfRIbvlhBm7r0YstIaQRYlp4yEbFqDcKSzXSSg==", + "dev": true, + "license": "MIT", + "workspaces": [ + "tests/deps/*", + "tests/projects/*", + "tests/projects/workspace/packages/*" + ], + "peerDependencies": { + "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "vite": { + "optional": true + } + } + }, + "node_modules/zimmerframe": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/zimmerframe/-/zimmerframe-1.1.5.tgz", + "integrity": "sha512-msJxIvYDYcoNL+PJsu+7qmpDWsYmAxTY+2TNYXXF0hzBzBk0BMecOqDOG/EckUoKCuKwObfbugIl8QpqHDXeFA==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/viewer/package.json b/viewer/package.json new file mode 100644 index 0000000..8f1f144 --- /dev/null +++ b/viewer/package.json @@ -0,0 +1,21 @@ +{ + "name": "@fpl/eval-viewer", + "private": true, + "version": "0.1.0", + "type": "module", + "scripts": { + "dev": "vite", + "check": "svelte-check --tsconfig ./tsconfig.json", + "build": "vite build", + "preview": "vite preview", + "render:ci": "node scripts/render-chart.mjs" + }, + "devDependencies": { + "@sveltejs/vite-plugin-svelte": "7.3.1", + "@playwright/test": "1.63.0", + "svelte": "5.57.1", + "svelte-check": "4.7.6", + "typescript": "5.9.3", + "vite": "8.3.0" + } +} diff --git a/viewer/public/results/catalog.json b/viewer/public/results/catalog.json new file mode 100644 index 0000000..652fc39 --- /dev/null +++ b/viewer/public/results/catalog.json @@ -0,0 +1,7 @@ +{ + "schema": "eval.result-catalog.v1", + "bundles": [ + { "id": "cadbench-model-a", "url": "model-a.json" }, + { "id": "cadbench-model-b", "url": "model-b.json" } + ] +} diff --git a/viewer/public/results/model-a.json b/viewer/public/results/model-a.json new file mode 100644 index 0000000..e825324 --- /dev/null +++ b/viewer/public/results/model-a.json @@ -0,0 +1,22 @@ +{ + "schema": "eval.result-bundle.v1", + "id": "cadbench-model-a-2026-09-24", + "generated_at": "2026-09-24T06:00:00Z", + "benchmark": { "id": "cadbench", "version": "2026.09", "title": "CADBench" }, + "subject": { "kind": "model", "provider": "FPL", "name": "Model A", "revision": "2026-09-20" }, + "provenance": { "harness_commit": "4b77600eaf1", "task_set_commit": "9c01f34ac77", "scorer": "cadbench-objective-v2", "source_url": "./results/model-a.json" }, + "metrics": [ + { "id": "pass_rate", "label": "Automated pass rate", "unit": "ratio", "goal": "maximize", "aggregation": "mean" }, + { "id": "latency", "label": "Median latency", "unit": "seconds", "goal": "minimize", "aggregation": "median" }, + { "id": "cost", "label": "Mean run cost", "unit": "USD", "goal": "minimize", "aggregation": "mean" } + ], + "observations": [ + { "metric_id": "pass_rate", "value": 0.86, "sample_count": 24, "lower_bound": 0.78, "upper_bound": 0.92, "run_ids": ["a-01", "a-02"], "task_ids": ["bracket", "enclosure"] }, + { "metric_id": "latency", "value": 48.2, "sample_count": 24, "run_ids": ["a-01", "a-02"], "task_ids": ["bracket", "enclosure"] }, + { "metric_id": "cost", "value": 0.19, "sample_count": 24, "run_ids": ["a-01", "a-02"] } + ], + "runs": [ + { "id": "a-01", "task_id": "bracket", "trial": 1, "duration_ms": 45200, "cost_usd": 0.18 }, + { "id": "a-02", "task_id": "enclosure", "trial": 1, "duration_ms": 51100, "cost_usd": 0.20 } + ] +} diff --git a/viewer/public/results/model-b.json b/viewer/public/results/model-b.json new file mode 100644 index 0000000..7399500 --- /dev/null +++ b/viewer/public/results/model-b.json @@ -0,0 +1,22 @@ +{ + "schema": "eval.result-bundle.v1", + "id": "cadbench-model-b-2026-09-24", + "generated_at": "2026-09-24T06:30:00Z", + "benchmark": { "id": "cadbench", "version": "2026.09", "title": "CADBench" }, + "subject": { "kind": "model", "provider": "FPL", "name": "Model B", "revision": "2026-09-23" }, + "provenance": { "harness_commit": "4b77600eaf1", "task_set_commit": "9c01f34ac77", "scorer": "cadbench-objective-v2", "source_url": "./results/model-b.json" }, + "metrics": [ + { "id": "pass_rate", "label": "Automated pass rate", "unit": "ratio", "goal": "maximize", "aggregation": "mean" }, + { "id": "latency", "label": "Median latency", "unit": "seconds", "goal": "minimize", "aggregation": "median" }, + { "id": "cost", "label": "Mean run cost", "unit": "USD", "goal": "minimize", "aggregation": "mean" } + ], + "observations": [ + { "metric_id": "pass_rate", "value": 0.79, "sample_count": 24, "lower_bound": 0.69, "upper_bound": 0.87, "run_ids": ["b-01", "b-02"], "task_ids": ["bracket", "enclosure"] }, + { "metric_id": "latency", "value": 31.4, "sample_count": 24, "run_ids": ["b-01", "b-02"], "task_ids": ["bracket", "enclosure"] }, + { "metric_id": "cost", "value": 0.11, "sample_count": 24, "run_ids": ["b-01", "b-02"] } + ], + "runs": [ + { "id": "b-01", "task_id": "bracket", "trial": 1, "duration_ms": 29800, "cost_usd": 0.10 }, + { "id": "b-02", "task_id": "enclosure", "trial": 1, "duration_ms": 33000, "cost_usd": 0.12 } + ] +} diff --git a/viewer/scripts/render-chart.mjs b/viewer/scripts/render-chart.mjs new file mode 100644 index 0000000..347a435 --- /dev/null +++ b/viewer/scripts/render-chart.mjs @@ -0,0 +1,28 @@ +import { spawn } from 'node:child_process'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { chromium } from '@playwright/test'; + +const output = process.env.EVAL_CHART_DIR ?? 'chart-artifacts'; +const port = Number(process.env.EVAL_VIEWER_PORT ?? 4173); +const server = spawn('npm', ['run', 'preview', '--', '--host', '127.0.0.1', '--port', String(port)], { + stdio: 'ignore' +}); + +try { + const browser = await chromium.launch({ headless: true }); + const page = await browser.newPage({ viewport: { width: 1600, height: 1000 }, deviceScaleFactor: 2 }); + for (let attempt = 0; attempt < 40; attempt += 1) { + try { const response = await page.goto(`http://127.0.0.1:${port}/`); if (response?.ok()) break; } + catch { if (attempt === 39) throw new Error('viewer did not start'); } + await new Promise((resolve) => setTimeout(resolve, 250)); + } + await page.getByText('comparable results').waitFor(); + await mkdir(output, { recursive: true }); + const plot = page.locator('#pareto-plot'); + await writeFile(`${output}/pareto.svg`, `${await plot.evaluate((node) => node.outerHTML)}\n`); + await plot.screenshot({ path: `${output}/pareto.png` }); + await writeFile(`${output}/source-url.txt`, `${page.url()}\n`); + await browser.close(); +} finally { + server.kill('SIGTERM'); +} diff --git a/viewer/src/App.svelte b/viewer/src/App.svelte new file mode 100644 index 0000000..d160ba9 --- /dev/null +++ b/viewer/src/App.svelte @@ -0,0 +1,178 @@ + + + + +
+ eval/field +
Loaded from{source || 'local data'}
+ +
+ +
+ + +
+
+

Pareto field

{points.length} comparable result{points.length === 1 ? '' : 's'}

+
+ + + +
+
+ +
+ + Pareto comparison of {xDefinition?.label} and {yDefinition?.label} + {points.length} attributed benchmark results. Orange rings indicate the Pareto frontier. + + {#each [0, .25, .5, .75, 1] as tick} + + + {display(tickValue(yDomain, tick))} + {display(xDomain[0] + tick*(xDomain[1]-xDomain[0]))} + {/each} + {#if frontier.length > 1} `${sx(point.x.value)},${sy(point.y.value)}`).join(' ')} class="frontier" />{/if} + {#each points as point} + selectedPoint = point.bundle} onkeydown={(event) => event.key === 'Enter' && (selectedPoint = point.bundle)}> + {#if point.x.lower_bound !== undefined && point.x.upper_bound !== undefined}{/if} + {#if point.y.lower_bound !== undefined && point.y.upper_bound !== undefined}{/if} + + {point.bundle.subject.name} + + {/each} + {xDefinition?.label} ({xDefinition?.unit}) · {xDefinition?.goal} + {yDefinition?.label} ({yDefinition?.unit}) · {yDefinition?.goal} + +
+ +
+

Source ledger

+
+ {#each visible as bundle} selectedPoint = bundle}> + + + {/each}
ModelBenchmarkHarnessTasksRunsGenerated
{bundle.subject.name}{bundle.subject.provider} / {bundle.subject.revision}{bundle.benchmark.id} v{bundle.benchmark.version}{short(bundle.provenance.harness_commit)}{short(bundle.provenance.task_set_commit)}{bundle.runs.length}{bundle.generated_at.slice(0, 10)}
+
+
+
+ +{#if selectedPoint} + +{/if} diff --git a/viewer/src/main.ts b/viewer/src/main.ts new file mode 100644 index 0000000..d018782 --- /dev/null +++ b/viewer/src/main.ts @@ -0,0 +1,5 @@ +import { mount } from 'svelte'; +import App from './App.svelte'; +import './style.css'; + +mount(App, { target: document.getElementById('app')! }); diff --git a/viewer/src/style.css b/viewer/src/style.css new file mode 100644 index 0000000..9dc52fb --- /dev/null +++ b/viewer/src/style.css @@ -0,0 +1,17 @@ +@import url('https://fonts.googleapis.com/css2?family=Figtree:wght@400;520;650&family=IBM+Plex+Mono:wght@400;500&display=swap'); +:root { font-family: Figtree, sans-serif; color: #182126; background: #f5f7f8; font-synthesis: none; } +* { box-sizing: border-box; } body { margin: 0; min-width: 320px; } button, select, input { font: inherit; } +header { height: 66px; display: flex; align-items: center; gap: 28px; padding: 0 28px; border-bottom: 1px solid #c8d2d7; } +.wordmark { color: #182126; font-weight: 650; font-size: 20px; text-decoration: none; letter-spacing: -.04em; } +.source { display: flex; min-width: 0; gap: 9px; font-size: 12px; color: #607078; }.source strong { font-family: 'IBM Plex Mono', monospace; color: #33434a; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.file-button, button { cursor: pointer; border: 1px solid #98a8af; background: #fff; padding: 8px 12px; color: #182126; }.file-button { margin-left: auto; font-size: 13px; }.file-button input { display: none; } +button:focus-visible, select:focus-visible, label:has(input:focus-visible), .point:focus-visible { outline: 3px solid #2257d6; outline-offset: 2px; } +main { display: grid; grid-template-columns: 260px minmax(0, 1fr); min-height: calc(100vh - 66px); } +main > aside { padding: 34px 22px; border-right: 1px solid #c8d2d7; } h1 { font-size: 36px; line-height: .98; margin: 0 0 16px; letter-spacing: -.045em; } .intro { color: #607078; line-height: 1.45; font-size: 14px; } +.bundle-list { margin-top: 34px; display: grid; gap: 8px; }.bundle-list label { display: flex; gap: 10px; align-items: start; padding: 11px 9px; border-left: 3px solid transparent; cursor: pointer; }.bundle-list label.active { border-color: #2257d6; background: #e9eef8; }.bundle-list span { display: grid; }.bundle-list small, td small { color: #607078; margin-top: 3px; }.empty, .error { font-size: 13px; line-height: 1.45; }.error { color: #a63b18; } +.workbench { padding: 30px clamp(18px, 4vw, 56px) 60px; overflow: hidden; }.plot-header { display: flex; justify-content: space-between; gap: 24px; align-items: end; }.plot-header h2, .ledger h2 { margin: 0; font-size: 21px; }.plot-header p { margin: 4px 0 0; color: #607078; }.axes { display: flex; gap: 10px; align-items: end; }.axes label { font-size: 11px; color: #607078; display: grid; gap: 4px; }.axes select { min-width: 150px; background: #fff; border: 1px solid #98a8af; padding: 7px; } +.plot-wrap { margin-top: 20px; border-top: 1px solid #98a8af; border-bottom: 1px solid #98a8af; } svg { display: block; width: 100%; max-height: 62vh; }.grid { stroke: #dce3e6; stroke-width: 1; }.frontier { fill: none; stroke: #e66a2c; stroke-width: 2; stroke-dasharray: 5 5; }.point { cursor: pointer; }.point circle { fill: #2257d6; stroke: #f5f7f8; stroke-width: 3; }.point.on-frontier circle { stroke: #e66a2c; stroke-width: 4; }.point text { font: 12px Figtree, sans-serif; fill: #33434a; }.interval { stroke: #2257d6; stroke-width: 2; }.tick { font: 9px 'IBM Plex Mono', monospace; fill: #607078; }.axis-label { font: 11px 'IBM Plex Mono', monospace; fill: #607078; } +.ledger { margin-top: 32px; }.table-wrap { overflow-x: auto; } table { margin-top: 12px; width: 100%; border-collapse: collapse; font-size: 13px; } th { text-align: left; color: #607078; font-weight: 520; border-bottom: 1px solid #98a8af; } th, td { padding: 10px 12px 10px 0; } tbody tr { border-bottom: 1px solid #dce3e6; cursor: pointer; } tbody tr:hover, tbody tr.selected { background: #e9eef8; } td strong, td small { display: block; } +.drawer { position: fixed; right: 0; top: 0; bottom: 0; width: min(440px, 94vw); background: #fff; border-left: 1px solid #98a8af; padding: 36px; overflow-y: auto; box-shadow: -12px 0 28px #18212618; }.close { position: absolute; right: 18px; top: 18px; border: 0; font-size: 28px; }.kicker { color: #2257d6; margin: 0 0 5px; }.drawer h2 { font-size: 30px; margin: 0 0 28px; }.drawer dl { display: grid; grid-template-columns: 120px 1fr; font-size: 13px; }.drawer dt { color: #607078; }.drawer dd { margin: 0 0 12px; font-family: 'IBM Plex Mono', monospace; overflow-wrap: anywhere; }.drawer h3 { margin-top: 30px; }.metric { padding: 12px 0; border-top: 1px solid #dce3e6; display: grid; grid-template-columns: 1fr auto; }.metric small { grid-column: 1/-1; color: #607078; margin-top: 5px; }.source-link { display: inline-block; margin-top: 24px; color: #2257d6; } +@media (max-width: 800px) { header .source { display: none; } main { grid-template-columns: 1fr; } main > aside { border-right: 0; border-bottom: 1px solid #c8d2d7; } .bundle-list { grid-template-columns: repeat(auto-fit,minmax(180px,1fr)); margin-top: 18px; } .plot-header { align-items: start; flex-direction: column; }.axes { flex-wrap: wrap; } } +@media (prefers-reduced-motion: reduce) { * { scroll-behavior: auto !important; } } diff --git a/viewer/src/types.ts b/viewer/src/types.ts new file mode 100644 index 0000000..cb8c8d6 --- /dev/null +++ b/viewer/src/types.ts @@ -0,0 +1,24 @@ +export type Goal = 'minimize' | 'maximize'; + +export interface MetricDefinition { + id: string; label: string; unit: string; goal: Goal; aggregation: string; +} +export interface MetricObservation { + metric_id: string; value: number; sample_count: number; + lower_bound?: number; upper_bound?: number; run_ids: string[]; task_ids?: string[]; +} +export interface RunRecord { + id: string; task_id: string; trial: number; duration_ms?: number; cost_usd?: number; + artifacts?: { name: string; media_type: string; url: string; sha256?: string }[]; +} +export interface ResultBundle { + schema: 'eval.result-bundle.v1'; id: string; generated_at: string; + benchmark: { id: string; version: string; title?: string }; + subject: { kind: string; provider: string; name: string; revision: string }; + provenance: { harness_commit: string; task_set_commit: string; scorer?: string; source_url?: string }; + metrics: MetricDefinition[]; observations: MetricObservation[]; runs: RunRecord[]; +} +export interface ResultCatalog { + schema: 'eval.result-catalog.v1'; + bundles: { id: string; url: string }[]; +} diff --git a/viewer/tsconfig.json b/viewer/tsconfig.json new file mode 100644 index 0000000..f21c4a6 --- /dev/null +++ b/viewer/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "allowJs": true, + "checkJs": true, + "isolatedModules": true, + "moduleResolution": "bundler", + "skipLibCheck": true, + "strict": true + }, + "include": ["src/**/*.ts", "src/**/*.svelte", "vite.config.ts"] +} diff --git a/viewer/vite.config.ts b/viewer/vite.config.ts new file mode 100644 index 0000000..ea0856b --- /dev/null +++ b/viewer/vite.config.ts @@ -0,0 +1,7 @@ +import { svelte } from '@sveltejs/vite-plugin-svelte'; +import { defineConfig } from 'vite'; + +export default defineConfig({ + plugins: [svelte()], + base: './' +});