diff --git a/agents/anyplot/briefs.py b/agents/anyplot/briefs.py index 2b4460e243e..d9fbf607fe8 100644 --- a/agents/anyplot/briefs.py +++ b/agents/anyplot/briefs.py @@ -10,7 +10,8 @@ from collections.abc import Iterator, Sequence from typing import Any -from .data.roles import DataRole +from core.spec_roles import DataRole + from .policy import fence from .schemas import MAX_TOP_VALUES, Binding, DatasetProfile from .session_state import SessionView diff --git a/agents/anyplot/code/readiness.py b/agents/anyplot/code/readiness.py index b6e6aaac3f4..eec862d52b6 100644 --- a/agents/anyplot/code/readiness.py +++ b/agents/anyplot/code/readiness.py @@ -74,7 +74,9 @@ from dataclasses import dataclass, field from typing import Literal +from core.adaptability import CRITICAL_VALUES from core.palette import IMPRINT +from core.spec_roles import MAP_SPECS from ..schemas import MAX_NOTE_CHARS from .normalise import SUPPORTED_LIBRARIES, measure_canvas @@ -84,25 +86,7 @@ ReadinessStatus = Literal["blocked", "coupled", "clean"] -MAP_SPECS = frozenset( - { - "bubble-map-geographic", - "cartogram-area-distortion", - "choropleth-basic", - "contour-map-geographic", - "flowmap-origin-destination", - "heatmap-geographic", - "map-connection-lines", - "map-marker-clustered", - "map-projections", - "map-route-path", - "map-tile-background", - "map-tilegrid", - "scatter-map-geographic", - } -) MIN_PALETTE_ENTRIES = 6 -CRITICAL_VALUES = frozenset({1.645, 1.96, 2.326, 2.576, 3.291}) MAX_SECURITY_REASONS = 5 MAX_SNIPPET_CHARS = 48 MAX_LINE_SNIPPET_CHARS = 2 * MAX_SNIPPET_CHARS diff --git a/agents/anyplot/data/bindings.py b/agents/anyplot/data/bindings.py index 7964df0b3f6..bc72615093e 100644 --- a/agents/anyplot/data/bindings.py +++ b/agents/anyplot/data/bindings.py @@ -41,10 +41,11 @@ from dataclasses import dataclass, field from difflib import SequenceMatcher +from core.spec_roles import DataRole, RoleKind + from ..schemas import Binding, ColumnDtype, ColumnProfile, DatasetProfile from .computed import match_score, window_line, window_rank from .parse import value_like -from .roles import DataRole, RoleKind from .shape import long_suggestion, takes_labels from .signs import has_negative, sign_line from .temporal import CAMEL as _CAMEL diff --git a/agents/anyplot/data/computed.py b/agents/anyplot/data/computed.py index 186f914ac92..d522cb24275 100644 --- a/agents/anyplot/data/computed.py +++ b/agents/anyplot/data/computed.py @@ -29,7 +29,7 @@ from collections.abc import Iterable from dataclasses import replace -from .roles import DataRole +from core.spec_roles import DataRole COMPUTED_ROLES: dict[str, frozenset[str]] = { diff --git a/agents/anyplot/data/shape.py b/agents/anyplot/data/shape.py index 2b8e5b6c139..4b642f71057 100644 --- a/agents/anyplot/data/shape.py +++ b/agents/anyplot/data/shape.py @@ -71,12 +71,13 @@ from dataclasses import dataclass from typing import Any +from core.spec_roles import DataRole + from ..language import DEFAULT_LANGUAGE, canonical from ..schemas import MAX_ROWS, Binding, ColumnProfile, DatasetProfile from .months import fold, named_months from .parse import ID_WORDS from .prepare import MAX_LONGER_COLUMNS -from .roles import DataRole from .temporal import WORDS, time_keys diff --git a/agents/anyplot/data/signs.py b/agents/anyplot/data/signs.py index d5fcc76a298..fa9b10b5d49 100644 --- a/agents/anyplot/data/signs.py +++ b/agents/anyplot/data/signs.py @@ -25,8 +25,9 @@ from collections.abc import Iterable from dataclasses import replace +from core.spec_roles import DataRole + from ..schemas import ColumnProfile -from .roles import DataRole NON_NEGATIVE_ROLES: frozenset[str] = frozenset( diff --git a/agents/anyplot/schemas.py b/agents/anyplot/schemas.py index 3aab0c0c1a8..547a5635012 100644 --- a/agents/anyplot/schemas.py +++ b/agents/anyplot/schemas.py @@ -43,6 +43,8 @@ model_validator, ) +from core.spec_roles import ROLE_PATTERN + from .language import DEFAULT_LANGUAGE, LANGUAGE_PATTERN @@ -158,10 +160,7 @@ def artifact_names(themes: Iterable[str]) -> list[ArtifactName]: return names + code_and_data -# A role is matched against the spec's `## Data` bullets later; the pattern only keeps -# arbitrary text out. Digits and capitals are allowed because 21 of 325 specs name -# roles such as `log2_fold_change`, `lower_95` or `temperature_K`. -ROLE_PATTERN = r"^[A-Za-z_][A-Za-z0-9_]{0,31}$" +# ROLE_PATTERN (imported above) lives with the role parser in core/spec_roles.py. RENDER_ID_PATTERN = r"^[A-Za-z0-9_-]{1,64}$" _WHITESPACE = re.compile(r"\s+") diff --git a/agents/anyplot/session_state.py b/agents/anyplot/session_state.py index b5b17fd1c2d..db43dd4db7e 100644 --- a/agents/anyplot/session_state.py +++ b/agents/anyplot/session_state.py @@ -20,9 +20,10 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator +from core.spec_roles import DataRole, parse_roles + from .data.bindings import BindingCheck, check_bindings, resolve_role from .data.computed import mark_computed -from .data.roles import DataRole, parse_roles from .data.signs import mark_non_negative from .language import DEFAULT_LANGUAGE, SOURCES, LanguageSource, is_valid from .schemas import MAX_CODE_CHARS, Binding, DatasetProfile diff --git a/agents/main.py b/agents/main.py index 7c186733dc0..709ebb4c553 100644 --- a/agents/main.py +++ b/agents/main.py @@ -135,7 +135,6 @@ from agents.anyplot import agent as agent_module from agents.anyplot import telemetry -from agents.anyplot.code.readiness import MAP_SPECS from agents.anyplot.data.bindings import check_bindings, label_siblings from agents.anyplot.data.classify import classify_paste from agents.anyplot.data.parse import ( @@ -149,7 +148,6 @@ without_context, ) from agents.anyplot.data.prepare import Operation, PrepareError, apply_operations, with_date_notes -from agents.anyplot.data.roles import DataRole from agents.anyplot.data.shape import long_suggestion from agents.anyplot.data.store import StoreFull from agents.anyplot.data.transcribe import ( @@ -238,6 +236,7 @@ ) from agents.stream import MAX_DONE_ROLES, Translator from core.constants import SUPPORTED_LIBRARIES +from core.spec_roles import MAP_SPECS, DataRole logger = logging.getLogger("anyplot.agents") diff --git a/automation/scripts/adaptability_check.py b/automation/scripts/adaptability_check.py new file mode 100644 index 00000000000..d843782f8d5 --- /dev/null +++ b/automation/scripts/adaptability_check.py @@ -0,0 +1,2787 @@ +#!/usr/bin/env python3 +"""The adaptability checker as one standard-library file, for the workflow runner. + +GENERATED by `uv run python -m core.adaptability_mirror` from core/spec_roles.py and +core/adaptability.py; edit those modules, never this file. The module docstring of +core/adaptability.py describes the rules and the record; this copy runs where `core` +is not importable: + + python3 adaptability_check.py check plots//implementations/python/.py + python3 adaptability_check.py report plots/ + +tests/unit/core/test_adaptability_mirror.py keeps it identical to the build. +""" + +from __future__ import annotations + +import argparse +import ast +import builtins +import io +import json +import random +import re +import sys +import tokenize +from collections import Counter +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal + + +# --- core/spec_roles.py ---------------------------------------------------------------------------------- + +# A role is matched against the spec's `## Data` bullets later; the pattern only keeps +# arbitrary text out. Digits and capitals are allowed because 21 of 325 specs name +# roles such as `log2_fold_change`, `lower_95` or `temperature_K`. +ROLE_PATTERN = r"^[A-Za-z_][A-Za-z0-9_]{0,31}$" + +RoleKind = Literal["numeric", "categorical", "datetime", "text", "boolean"] + +MAX_ROLE_NAME_CHARS = 32 +MAX_VARIADIC_NAME_CHARS = 30 + +TYPE_WORDS: dict[str, RoleKind] = { + "numeric": "numeric", + "numerical": "numeric", + "number": "numeric", + "numbers": "numeric", + "float": "numeric", + "floats": "numeric", + "double": "numeric", + "int": "numeric", + "ints": "numeric", + "integer": "numeric", + "integers": "numeric", + "continuous": "numeric", + "count": "numeric", + "counts": "numeric", + "categorical": "categorical", + "category": "categorical", + "categories": "categorical", + "ordinal": "categorical", + "nominal": "categorical", + "datetime": "datetime", + "date": "datetime", + "dates": "datetime", + "time": "datetime", + "timestamp": "datetime", + "timestamps": "datetime", + "string": "text", + "strings": "text", + "str": "text", + "text": "text", + "label": "text", + "labels": "text", + "boolean": "boolean", + "bool": "boolean", + "binary": "boolean", +} + +QUALIFIER_WORDS: frozenset[str] = frozenset( + { + "optional", + "required", + "array", + "arrays", + "list", + "lists", + "of", + "or", + "and", + "d", # the letter left from "1D", "2D" + "matrix", + "pair", + "pairs", + "column", + "columns", + "metadata", + "ordered", + } +) + +# Possessive quantifiers (`*+`, `++`) and the run-start lookbehinds keep every +# pattern linear in the bullet length: nothing after a whitespace or digit run +# can match inside that run, so there is never anything to backtrack into. +_HEAD_TOKEN = re.compile( + r"\s*+(?:`(?P[^`]++)`|(?P\.\.\.|…)|(?Pthrough|to)\b|(?Por)\b|(?P,|/|and\b))", + re.IGNORECASE, +) +_TAIL = re.compile(r"\s*+(?:\((?P[^)]*+)\))?\s*+(?:[-–—:]++\s*+)?(?P.*)", re.DOTALL) +_BOLD = re.compile(r"\*\*(?P