Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/tangle-cli/src/tangle_cli/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,6 @@
try:
__version__ = metadata_version("tangle-cli")
except PackageNotFoundError:
__version__ = "0.1.36"
__version__ = "0.1.37"

__all__ = ["TangleDynamicDiscoveryClient", "__version__"]
33 changes: 31 additions & 2 deletions packages/tangle-cli/src/tangle_cli/pipeline_compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@
from .python_pipeline.ref import CallableRef
from .python_pipeline.registered import _REGISTERED_URL_PLACEHOLDER
from .python_pipeline.subpipeline import _SUBPIPELINE_URL_PLACEHOLDER, SubpipelineRef
from .python_pipeline.trace import _strip_optional_none, trace_pipeline
from .python_pipeline.trace import _own_task_annotations, _strip_optional_none, trace_pipeline
from .python_pipeline.types import In
from .schema_validation import (
CALLER_ANNOTATION_POLICY,
Expand Down Expand Up @@ -1144,6 +1144,7 @@ def _process_subpipeline_children(
ambient_context=ambient_passthrough or None,
config_identity=child_identity,
task_annotations=child_task_annotations,
own_task_annotations=_own_task_annotations_for_key(child_fn, child_doc, effective),
)

# Cycle detection (covers self-reference). active_stack holds the
Expand Down Expand Up @@ -1446,6 +1447,7 @@ def _compile_key_for(
ambient_context: Mapping[str, Any] | None = None,
config_identity: str | None = None,
task_annotations: Mapping[str, Any] | None = None,
own_task_annotations: Mapping[str, Any] | None = None,
) -> PipelineCompileKey:
"""Build the canonical :class:`PipelineCompileKey` for ``pipeline_fn``.

Expand Down Expand Up @@ -1483,13 +1485,18 @@ def _compile_key_for(
* **``task_annotations``** (defaults pushed in by ancestor
``.with_task_annotations`` call sites) -> the same envelope plus
``"\x00task_annotations"``.
* **``own_task_annotations``** (the child's own resolved
``@pipeline(task_annotations=...)``, see
:func:`_own_task_annotations_for_key`) -> the same envelope plus
``"\x00own_task_annotations"``, so two wrappers over one function that
differ only there never share a sidecar.
"""
src = _pipeline_source_path(pipeline_fn)
if src is not None:
source_canon = canonical_repo_path(src)
else:
source_canon = f"<dynamic:{pipeline_fn.fn.__qualname__}>"
if ambient_context or config_identity is not None or task_annotations:
if ambient_context or config_identity is not None or task_annotations or own_task_annotations:
# Reserved keys (NUL-prefixed) can never be real config keys, so an
# envelope never collides with an effective-only fingerprint over the
# same keys (e.g. effective={x:1},ambient={} vs effective={x:1},ambient={y:2}).
Expand All @@ -1500,6 +1507,8 @@ def _compile_key_for(
envelope["\x00config"] = config_identity
if task_annotations:
envelope["\x00task_annotations"] = dict(task_annotations)
if own_task_annotations:
envelope["\x00own_task_annotations"] = dict(own_task_annotations)
fingerprint_input: Mapping[str, Any] = envelope
else:
# Byte-identical to today's key: the default-isolation guarantee.
Expand All @@ -1513,6 +1522,26 @@ def _compile_key_for(
)


def _own_task_annotations_for_key(
pipeline_fn: PipelineFn,
doc: CfgDocument | None,
overrides: Mapping[str, Any],
) -> dict[str, Any]:
"""A child's own ``@pipeline(task_annotations=...)`` as its trace applies
them, for its compile key.

A function is resolved against the cfg :func:`_resolve_cfg_for_pipeline`
builds for the child (its config plus ``overrides``). Without a config
file it contributes nothing: the child's own compile then fails on the
missing file.
"""
if not callable(pipeline_fn.task_annotations):
return _own_task_annotations(pipeline_fn, None)
if doc is None:
return {}
return _own_task_annotations(pipeline_fn, build_cfg(doc.data, overrides=dict(overrides), coerce=False))


def _validate_artifact(artifact: SubgraphArtifact, ctx: CompileContext) -> None:
"""Validate one planned artifact in memory (no writes).

Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "tangle-cli"
version = "0.1.36"
version = "0.1.37"
description = "CLI for Tangle, the open-source ML pipeline orchestration platform"
readme = "README.md"
authors = [
Expand Down
2 changes: 1 addition & 1 deletion tests/test_packaging.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ def test_tangle_cli_wheel_supports_expert_no_deps_import_path_without_tangle_api
requires_dist = [line for line in metadata.splitlines() if line.startswith("Requires-Dist: ")]
assert not any(name.startswith("tangle_api/") for name in names)
assert "tangle_cli/openapi/openapi.json" not in names
assert "Version: 0.1.36" in metadata
assert "Version: 0.1.37" in metadata
assert "Requires-Dist: tangle-api==0.1.1" in requires_dist
assert not any("extra == 'native'" in line for line in requires_dist)
assert "Provides-Extra: native" in metadata
Expand Down
59 changes: 59 additions & 0 deletions tests/test_task_annotation_defaults.py
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,65 @@ def root():
assert _annotations(grandchild_b, "GC Task") == {"env": "b"}


@pytest.mark.parametrize(
("left", "right"),
[
('{"team": "left"}', '{"team": "right"}'),
('lambda cfg: {"team": cfg.left}', 'lambda cfg: {"team": cfg.right}'),
],
ids=["mapping", "function"],
)
def test_wrappers_differing_only_in_their_own_defaults_get_separate_sidecars(tmp_path, left, right):
out = _compile(
tmp_path,
f"""
def child():
op.named("Child Task")()

left = pipeline("Child", task_annotations={left})(child)
right = pipeline("Child", task_annotations={right})(child)

@pipeline("Root")
def root():
subpipeline(left).named("Left")()
subpipeline(right).named("Right")()
""",
config="left: left\nright: right\n",
)

assert _child(out, "Left") != _child(out, "Right")
assert _annotations(_child(out, "Left"), "Child Task") == {"team": "left"}
assert _annotations(_child(out, "Right"), "Child Task") == {"team": "right"}


@pytest.mark.parametrize(
"declared",
['{"team": "left"}', 'lambda cfg: {"team": cfg.left}'],
ids=["mapping", "function"],
)
def test_wrappers_with_the_same_own_defaults_share_a_sidecar(tmp_path, declared):
out = _compile(
tmp_path,
f"""
def child():
op.named("Child Task")()

first = pipeline("Child", task_annotations={declared})(child)
second = pipeline("Child", task_annotations={declared})(child)

@pipeline("Root")
def root():
subpipeline(first).named("First")()
subpipeline(second).named("Second")()
subpipeline(first).named("First Again")()
""",
config="left: left\n",
)

assert _child(out, "First") == _child(out, "Second") == _child(out, "First Again")
assert _annotations(_child(out, "First"), "Child Task") == {"team": "left"}


@pytest.mark.parametrize("declared", ['"davies"', 'lambda cfg: "davies"'])
def test_pipeline_defaults_that_are_not_a_mapping_fail_the_compile(tmp_path, declared):
with pytest.raises(CompileError, match=r"'Root'.*must be a mapping or a function of cfg returning one, got str"):
Expand Down
2 changes: 1 addition & 1 deletion uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading