diff --git a/src/openjd/model/v2023_09/_model.py b/src/openjd/model/v2023_09/_model.py index 373d2e0..cc90d8a 100644 --- a/src/openjd/model/v2023_09/_model.py +++ b/src/openjd/model/v2023_09/_model.py @@ -1829,6 +1829,13 @@ class StepParameterSpaceDefinition(OpenJDModel_v2023_09): reshape_field_to_dict={"taskParameterDefinitions": "name"}, ) + # §2 makes task parameter type names case-insensitive under EXPR, the same as + # job parameter type names. Must run before discriminated-union resolution. + @field_validator("taskParameterDefinitions", mode="before") + @classmethod + def _normalize_parameter_type_case(cls, v: Any, info: ValidationInfo) -> Any: + return _normalize_parameter_type_case(v, info) + @field_validator("taskParameterDefinitions") @classmethod def _validate_parameters(cls, v: TaskParameterList) -> TaskParameterList: @@ -3948,11 +3955,18 @@ def _expr_param_gate(value: JobParameterType, info: ValidationInfo) -> JobParame return value +# Parameter type names are ASCII (§2). str.upper() is Unicode-aware and folds +# U+0131 to 'I', which would make 'ıNT' a spelling of 'INT'. +_ASCII_UPPERCASE = str.maketrans("abcdefghijklmnopqrstuvwxyz", "ABCDEFGHIJKLMNOPQRSTUVWXYZ") + + def _normalize_parameter_type_case(value: Any, info: ValidationInfo) -> Any: """Uppercase the ``type`` discriminator of each parameter definition when the EXPR extension is enabled (RFC 0007 makes parameter type names case-insensitive, e.g. ``int`` == ``INT``, ``list[int]`` == ``LIST[INT]``). Runs before discriminated-union resolution. + + Applies to job parameter and task parameter type names alike, per §2. """ context = cast(Optional[ModelParsingContext], info.context) if not (context and "EXPR" in context.extensions): @@ -3962,7 +3976,7 @@ def _normalize_parameter_type_case(value: Any, info: ValidationInfo) -> Any: normalized: list[Any] = [] for item in value: if isinstance(item, dict) and isinstance(item.get("type"), str): - item = {**item, "type": item["type"].upper()} + item = {**item, "type": item["type"].translate(_ASCII_UPPERCASE)} normalized.append(item) return normalized diff --git a/test/openjd/model_v0/v2023_09/test_environment_template.py b/test/openjd/model_v0/v2023_09/test_environment_template.py index ed0ca37..a8f5acf 100644 --- a/test/openjd/model_v0/v2023_09/test_environment_template.py +++ b/test/openjd/model_v0/v2023_09/test_environment_template.py @@ -5,6 +5,7 @@ import pytest from pydantic import ValidationError +from openjd.model import DecodeValidationError, decode_environment_template from openjd.model._parse import _parse_model from openjd.model.v2023_09 import EnvironmentTemplate @@ -256,3 +257,72 @@ def test_parse_fails(self, data: dict[str, Any], expected_num_errors: int) -> No # THEN assert len(excinfo.value.errors()) == expected_num_errors + + +class TestEnvironmentTemplateParameterTypeNameCase: + """Template Schemas §2: job parameter type names are case-sensitive in base + 2023-09 and case-insensitive when the EXPR extension is enabled. An + environment template's ``parameterDefinitions`` carries the same + ``JobParameterDefinition`` union as a job template's, so the same rule applies. + + ``EnvironmentTemplate`` is the third registration site of the shared fold, and + it is the one an audit found unpinned: neutering it left the whole model_v0 + suite green. The job template counterpart is + ``test_list_parameters.py::TestJobParameterTypeNameCase`` and the task + parameter one is ``test_parameter_space.py::TestTaskParameterTypeNameCase``. + """ + + @staticmethod + def _tmpl(type_name: str, *, extensions: tuple[str, ...] = ("EXPR",)) -> dict[str, Any]: + template: dict[str, Any] = { + "specificationVersion": "environment-2023-09", + "parameterDefinitions": [{"name": "P", "type": type_name}], + "environment": ENVIRONMENT, + } + if extensions: + template["extensions"] = list(extensions) + return template + + @staticmethod + def _decode(template: dict[str, Any]) -> None: + decode_environment_template(template=template, supported_extensions=["EXPR"]) + + # (canonical spelling, a mis-cased spelling) + TYPES: tuple = ( + pytest.param("STRING", "string", id="string"), + pytest.param("INT", "iNt", id="int"), + pytest.param("PATH", "pAtH", id="path"), + ) + + @pytest.mark.parametrize("canonical, miscased", TYPES) + def test_no_expr_canonical_case_accepted(self, canonical: str, miscased: str) -> None: + # Case 1 of 4. + self._decode(self._tmpl(canonical, extensions=())) + + @pytest.mark.parametrize("canonical, miscased", TYPES) + def test_no_expr_miscased_rejected(self, canonical: str, miscased: str) -> None: + # Case 2 of 4. Fails if the fold is registered without its EXPR gate. + with pytest.raises(DecodeValidationError) as excinfo: + self._decode(self._tmpl(miscased, extensions=())) + message = str(excinfo.value) + assert "parameterDefinitions[0]" in message, message + assert f"'{miscased}'" in message, message + + @pytest.mark.parametrize("canonical, miscased", TYPES) + def test_with_expr_canonical_case_accepted(self, canonical: str, miscased: str) -> None: + # Case 3 of 4. + self._decode(self._tmpl(canonical)) + + @pytest.mark.parametrize("canonical, miscased", TYPES) + def test_with_expr_miscased_accepted(self, canonical: str, miscased: str) -> None: + # Case 4 of 4. Fails if the fold is not registered on this model at all, + # which is the state an audit found untested. + self._decode(self._tmpl(miscased)) + + @pytest.mark.parametrize("type_name", ("\u0131NT", "\u017fTRING", "\ufb02OAT")) + def test_non_ascii_lookalike_rejected_with_expr(self, type_name: str) -> None: + # str.upper() folds U+0131 to 'I', U+017F to 'S' and U+FB02 (fl) to 'FL', + # so a Unicode-aware fold would read these as INT, STRING and FLOAT. + with pytest.raises(DecodeValidationError) as excinfo: + self._decode(self._tmpl(type_name)) + assert f"'{type_name}'" in str(excinfo.value), str(excinfo.value) diff --git a/test/openjd/model_v0/v2023_09/test_list_parameters.py b/test/openjd/model_v0/v2023_09/test_list_parameters.py index 89e66e6..ff12a07 100644 --- a/test/openjd/model_v0/v2023_09/test_list_parameters.py +++ b/test/openjd/model_v0/v2023_09/test_list_parameters.py @@ -109,3 +109,82 @@ def test_requires_expr(self): def test_rejects(self, param): with pytest.raises(DecodeValidationError): _decode(_tmpl(param)) + + +class TestJobParameterTypeNameCase: + """Template Schemas §2: job parameter type names are case-sensitive in base + 2023-09 and case-insensitive when the EXPR extension is enabled. + + ``TestListValid.test_case_insensitive_type`` covers one of the four + extension-by-spelling combinations. This covers all four, on both an + EXPR-only type and a base type, and pins that the shared fold is ASCII. + + The task parameter counterpart is + ``test_parameter_space.py::TestTaskParameterTypeNameCase``. + """ + + # (canonical spelling, a mis-cased spelling, a valid default) + TYPES: tuple = ( + ("STRING", "string", "a"), # base type, available without EXPR + ("INT", "iNt", 1), # base type + ("LIST[INT]", "list[int]", [1, 2]), # EXPR-only type + ("LIST[LIST[INT]]", "List[List[Int]]", [[1], [2]]), # EXPR-only, nested brackets + ) + + @staticmethod + def _param(type_name, default): + return {"name": "P", "type": type_name, "default": default} + + @pytest.mark.parametrize("canonical, miscased, default", TYPES) + def test_no_expr_canonical_case(self, canonical, miscased, default): + # Case 1 of 4. Without EXPR the spec spelling is the only one accepted. + # A base type is accepted; an EXPR-only type is rejected for needing EXPR, + # which is a different rejection from the casing one in case 2. + template = _tmpl(self._param(canonical, default), extensions=()) + if canonical.startswith("LIST["): + with pytest.raises(DecodeValidationError, match="requires the EXPR extension"): + _decode(template) + else: + _decode(template) + + @pytest.mark.parametrize("canonical, miscased, default", TYPES) + def test_no_expr_miscased_rejected(self, canonical, miscased, default): + # Case 2 of 4. Without EXPR, type names are case-sensitive. + with pytest.raises(DecodeValidationError) as excinfo: + _decode(_tmpl(self._param(miscased, default), extensions=())) + message = str(excinfo.value) + assert "parameterDefinitions[0]" in message, message + assert f"'{miscased}'" in message, message + # Rejected for the casing, not for the extension. An EXPR-only type spelled + # correctly would say "requires the EXPR extension" instead. + assert "requires the EXPR extension" not in message, message + + @pytest.mark.parametrize("canonical, miscased, default", TYPES) + def test_with_expr_canonical_case_accepted(self, canonical, miscased, default): + # Case 3 of 4. Enabling EXPR must not break the spec spelling. + _decode(_tmpl(self._param(canonical, default))) + + @pytest.mark.parametrize("canonical, miscased, default", TYPES) + def test_with_expr_miscased_accepted(self, canonical, miscased, default): + # Case 4 of 4. + _decode(_tmpl(self._param(miscased, default))) + + # ── The fold is ASCII ── + + # str.upper() folds each of these wholly into the type-name alphabet: U+0131 + # dotless i to 'I', U+017F long s to 'S', U+FB02 ligature fl to 'FL', U+FB06 + # ligature st to 'ST'. A Unicode-aware fold reads them as real type names. + LOOKALIKES = ("\u0131NT", "\u017fTRING", "\ufb02OAT", "\ufb06RING") + + @pytest.mark.parametrize("type_name", LOOKALIKES) + def test_non_ascii_lookalike_rejected_with_expr(self, type_name): + with pytest.raises(DecodeValidationError) as excinfo: + _decode(_tmpl(self._param(type_name, None))) + assert f"'{type_name}'" in str(excinfo.value), str(excinfo.value) + + @pytest.mark.parametrize("type_name", LOOKALIKES) + def test_non_ascii_lookalike_rejected_without_expr(self, type_name): + # Inert against this change by design: without EXPR no fold runs. Present + # so the pair covers both extension states. + with pytest.raises(DecodeValidationError): + _decode(_tmpl(self._param(type_name, None), extensions=())) diff --git a/test/openjd/model_v0/v2023_09/test_parameter_space.py b/test/openjd/model_v0/v2023_09/test_parameter_space.py index 442ba36..aa354b7 100644 --- a/test/openjd/model_v0/v2023_09/test_parameter_space.py +++ b/test/openjd/model_v0/v2023_09/test_parameter_space.py @@ -6,6 +6,7 @@ import pytest from pydantic import ValidationError +from openjd.model import DecodeValidationError, decode_job_template, parse_model from openjd.model._parse import _parse_model from openjd.model.v2023_09 import ( FloatTaskParameterDefinition, @@ -773,3 +774,300 @@ def test_parse_fails(self, data: dict[str, Any], expected_num_errors: int) -> No # THEN assert len(excinfo.value.errors()) == expected_num_errors, str(excinfo.value) + + +class TestTaskParameterTypeNameCase: + """Template Schemas §2: task parameter type names are case-sensitive in base + 2023-09 and case-insensitive when the EXPR extension is enabled. + + The conformance fixture is + ``EXPR/job_templates/proposed/3.4.1--task-param-type-case-insensitive.yaml`` + (openjd-specifications#166). + + These go through ``decode_job_template`` rather than the module's usual + ``_parse_model`` because the rule is gated on the extension set. The public + ``parse_model`` also supplies one for a bare model, which + ``test_bare_model_via_public_parse_model_honours_expr`` covers. + """ + + # Every task parameter type, with a deliberately mis-cased spelling for each. + # The mis-cased spellings vary in shape on purpose: all-lower, leading-cap, + # alternating, and a bracketed name. + TYPES: tuple = ( + pytest.param("INT", "int", "1-3", None, id="int"), + pytest.param("FLOAT", "Float", ["1.0", "2.0"], None, id="float"), + pytest.param("STRING", "sTrInG", ["fg", "bg"], None, id="string"), + pytest.param("PATH", "pAtH", ["/tmp/a", "/tmp/b"], None, id="path"), + pytest.param( + "CHUNK[INT]", + "chunk[int]", + "1-3", + {"defaultTaskCount": 1, "rangeConstraint": "CONTIGUOUS"}, + id="chunk-int", + ), + ) + + @staticmethod + def _tmpl(type_name: str, range_value: Any, chunks: Any, extensions: tuple[str, ...]) -> dict: + param: dict[str, Any] = {"name": "F", "type": type_name, "range": range_value} + if chunks is not None: + param["chunks"] = chunks + template: dict[str, Any] = { + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "steps": [ + { + "name": "S", + "parameterSpace": {"taskParameterDefinitions": [param]}, + "script": {"actions": {"onRun": {"command": "echo", "args": ["hi"]}}}, + } + ], + } + if extensions: + template["extensions"] = list(extensions) + return template + + @staticmethod + def _decode(template: dict) -> None: + # The caller allowlists both extensions in every case. What varies is whether + # the template declares them, because the effective set is the intersection. + decode_job_template(template=template, supported_extensions=["EXPR", "TASK_CHUNKING"]) + + @staticmethod + def _extensions_for(canonical: str, expr: bool) -> tuple[str, ...]: + exts = ("TASK_CHUNKING",) if canonical == "CHUNK[INT]" else () + return (*exts, "EXPR") if expr else exts + + # ── The four cases: EXPR on/off x spelling canonical/mis-cased ── + + @pytest.mark.parametrize("canonical, miscased, range_value, chunks", TYPES) + def test_no_expr_canonical_case_accepted( + self, canonical: str, miscased: str, range_value: Any, chunks: Any + ) -> None: + # Case 1 of 4. Without EXPR, the spec spelling is the only accepted one, + # and it is accepted. Negative control: proves a rejection in case 2 is + # about the casing and not about the type being unavailable. + self._decode( + self._tmpl(canonical, range_value, chunks, self._extensions_for(canonical, expr=False)) + ) + + @pytest.mark.parametrize("canonical, miscased, range_value, chunks", TYPES) + def test_no_expr_miscased_rejected( + self, canonical: str, miscased: str, range_value: Any, chunks: Any + ) -> None: + # Case 2 of 4. Without EXPR, type names are case-sensitive, so a mis-cased + # spelling must be rejected. This is the case that fails if the normalizer + # is registered without its EXPR gate. + with pytest.raises(DecodeValidationError) as excinfo: + self._decode( + self._tmpl( + miscased, range_value, chunks, self._extensions_for(canonical, expr=False) + ) + ) + message = str(excinfo.value) + assert "steps[0] -> parameterSpace -> taskParameterDefinitions[0]" in message, message + # The author's spelling appears in the diagnostic, not the canonical one. + assert f"'{miscased}'" in message, message + # Rejected for the casing, not for a missing extension. CHUNK[INT] is the + # case that could otherwise fail for the wrong reason. + assert "requires the TASK_CHUNKING extension" not in message, message + + @pytest.mark.parametrize("canonical, miscased, range_value, chunks", TYPES) + def test_with_expr_canonical_case_accepted( + self, canonical: str, miscased: str, range_value: Any, chunks: Any + ) -> None: + # Case 3 of 4. Enabling EXPR must not break the spec spelling. Negative + # control against a fold that rewrites the name into something unmatchable. + self._decode( + self._tmpl(canonical, range_value, chunks, self._extensions_for(canonical, expr=True)) + ) + + @pytest.mark.parametrize("canonical, miscased, range_value, chunks", TYPES) + def test_with_expr_miscased_accepted( + self, canonical: str, miscased: str, range_value: Any, chunks: Any + ) -> None: + # Case 4 of 4. With EXPR, a mis-cased spelling is equivalent to the + # canonical one. This is the conformance fixture's assertion, and the case + # that fails if the normalizer is not registered at all. + self._decode( + self._tmpl(miscased, range_value, chunks, self._extensions_for(canonical, expr=True)) + ) + + # ── The gate is on EXPR, not on the extension that supplies the type ── + + def test_chunk_int_miscased_needs_expr_not_only_task_chunking(self) -> None: + # TASK_CHUNKING makes CHUNK[INT] available; EXPR is what makes its name + # case-insensitive. Declaring only TASK_CHUNKING must still reject + # 'chunk[int]', which pins that the fold reads EXPR specifically. + with pytest.raises(DecodeValidationError): + self._decode( + self._tmpl( + "chunk[int]", + "1-3", + {"defaultTaskCount": 1, "rangeConstraint": "CONTIGUOUS"}, + ("TASK_CHUNKING",), + ) + ) + + def test_expr_declared_but_not_allowlisted_is_an_extension_error(self) -> None: + # Pins the intersection semantics, not the fold: the rejection comes from + # the extension allowlist before the type name is reached. + with pytest.raises(DecodeValidationError) as excinfo: + decode_job_template( + template=self._tmpl("int", "1-3", None, ("EXPR",)), + supported_extensions=[], + ) + assert "Unsupported extension names: EXPR" in str(excinfo.value), str(excinfo.value) + + def test_bare_model_via_public_parse_model_honours_expr(self) -> None: + # public parse_model builds a context from supported_extensions, so the fold + # reaches a bare StepParameterSpaceDefinition with no enclosing template. + parse_model( + model=StepParameterSpaceDefinition, + obj={"taskParameterDefinitions": [{"name": "F", "type": "int", "range": "1-3"}]}, + supported_extensions=["EXPR"], + ) + with pytest.raises(DecodeValidationError): + parse_model( + model=StepParameterSpaceDefinition, + obj={"taskParameterDefinitions": [{"name": "F", "type": "int", "range": "1-3"}]}, + supported_extensions=[], + ) + + # ── The fold is ASCII, so a non-ASCII character is not a spelling variant ── + + # str.upper() folds each of these wholly into the type-name alphabet: U+0131 + # LATIN SMALL LETTER DOTLESS I to 'I', U+017F LONG S to 'S', U+FB02 ligature fl + # to 'FL', U+FB06 ligature st to 'ST'. A Unicode-aware fold reads them as INT, + # STRING, FLOAT and STRING. + NON_ASCII_LOOKALIKES: tuple = ( + pytest.param("\u0131NT", id="dotless-i-int"), + pytest.param("\u017fTRING", id="long-s-string"), + pytest.param("\ufb02OAT", id="fl-ligature-float"), + pytest.param("\ufb06RING", id="st-ligature-string"), + ) + + @pytest.mark.parametrize("type_name", NON_ASCII_LOOKALIKES) + def test_non_ascii_lookalike_rejected_with_expr(self, type_name: str) -> None: + with pytest.raises(DecodeValidationError) as excinfo: + self._decode(self._tmpl(type_name, "1-3", None, ("EXPR",))) + assert f"'{type_name}'" in str(excinfo.value), str(excinfo.value) + + @pytest.mark.parametrize("type_name", NON_ASCII_LOOKALIKES) + def test_non_ascii_lookalike_rejected_without_expr(self, type_name: str) -> None: + # Inert against this change by design: without EXPR no fold runs, so this + # cannot fail. Present so the pair covers both extension states. + with pytest.raises(DecodeValidationError): + self._decode(self._tmpl(type_name, "1-3", None, ())) + + # ── Malformed input reaches the fold now that it runs on this field ── + + @pytest.mark.parametrize( + "parameter_space", + ( + pytest.param({"taskParameterDefinitions": None}, id="null-list"), + pytest.param({"taskParameterDefinitions": {}}, id="mapping-not-list"), + pytest.param({"taskParameterDefinitions": "int"}, id="string-not-list"), + pytest.param({"taskParameterDefinitions": ["int"]}, id="list-of-non-dicts"), + pytest.param( + {"taskParameterDefinitions": [{"name": "F", "type": 3, "range": "1-3"}]}, + id="type-is-int", + ), + pytest.param( + {"taskParameterDefinitions": [{"name": "F", "type": True, "range": "1-3"}]}, + id="type-is-bool", + ), + pytest.param( + {"taskParameterDefinitions": [{"name": "F", "type": ["int"], "range": "1-3"}]}, + id="type-is-list", + ), + pytest.param( + {"taskParameterDefinitions": [{"name": "F", "type": None, "range": "1-3"}]}, + id="type-is-null", + ), + pytest.param( + {"taskParameterDefinitions": [{"name": "F", "range": "1-3"}]}, id="type-missing" + ), + ), + ) + def test_malformed_input_is_a_validation_error_not_a_crash(self, parameter_space: Any) -> None: + # Registering the fold on this field routed these through it for the first + # time, so its isinstance guards became load-bearing here. Without them + # these raise TypeError, AttributeError or KeyError out of + # decode_job_template instead of DecodeValidationError. pytest.raises + # fails on any other exception type, which is the pin. + template: dict[str, Any] = { + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["EXPR"], + "name": "T", + "steps": [ + { + "name": "S", + "parameterSpace": parameter_space, + "script": {"actions": {"onRun": {"command": "echo", "args": ["hi"]}}}, + } + ], + } + with pytest.raises(DecodeValidationError): + self._decode(template) + + # ── A mis-cased name now reaches the validators that run after the fold ── + + def test_miscased_duplicate_names_report_the_duplicate_not_the_type(self) -> None: + # Before the fold ran on this field, 'int' failed at discriminator + # resolution and never reached the unique-name rule. Now it does. + with pytest.raises(DecodeValidationError, match="Duplicate values for name"): + self._decode( + { + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["EXPR"], + "name": "T", + "steps": [ + { + "name": "S", + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "F", "type": "int", "range": "1-3"}, + {"name": "F", "type": "INT", "range": "1-3"}, + ] + }, + "script": {"actions": {"onRun": {"command": "echo", "args": ["hi"]}}}, + } + ], + } + ) + + def test_two_miscased_chunk_int_report_the_one_chunk_rule(self) -> None: + # Same reason: the one-CHUNK[INT]-per-step rule is only reachable once the + # mis-cased spellings resolve to the CHUNK[INT] variant. + chunks = {"defaultTaskCount": 1, "rangeConstraint": "CONTIGUOUS"} + with pytest.raises(DecodeValidationError, match="Only one CHUNK\\[INT\\]"): + self._decode( + { + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["EXPR", "TASK_CHUNKING"], + "name": "T", + "steps": [ + { + "name": "S", + "parameterSpace": { + "taskParameterDefinitions": [ + { + "name": "A", + "type": "chunk[int]", + "range": "1-3", + "chunks": chunks, + }, + { + "name": "B", + "type": "Chunk[Int]", + "range": "1-3", + "chunks": chunks, + }, + ] + }, + "script": {"actions": {"onRun": {"command": "echo", "args": ["hi"]}}}, + } + ], + } + )