Redo documentation with diataxis

README.md

@@ -33,7 +33,14 @@ We have taken these ideas a step further by introducing the concept of the `Mode
 
 We aim to provide additional (and optional) tools for workflow orchestration on top of the configuration framework.
 
-[More information is available in our wiki](https://github.com/Point72/ccflow/wiki)
+## Documentation
+
+Our [wiki](https://github.com/Point72/ccflow/wiki) is organized along the four kinds of documentation:
+
+- **[Tutorials](https://github.com/Point72/ccflow/wiki/Tutorials)** — hands-on lessons, from [First Steps](https://github.com/Point72/ccflow/wiki/First-Steps) through building ETL applications to a [configurable calculator driven from the CLI](https://github.com/Point72/ccflow/wiki/Building-a-Configurable-Calculator).
+- **[How-to Guides](https://github.com/Point72/ccflow/wiki/How-to-Guides)** — recipes for specific tasks like configuring complex values, caching, and retries.
+- **[Reference](https://github.com/Point72/ccflow/wiki/Reference)** — the core types, built-in models, and options.
+- **[Explanation](https://github.com/Point72/ccflow/wiki/Explanation)** — the [design goals](https://github.com/Point72/ccflow/wiki/Design-Goals), [core concepts](https://github.com/Point72/ccflow/wiki/Core-Concepts), and [why ccflow leans on Hydra](https://github.com/Point72/ccflow/wiki/Configuration-and-Hydra).
 
 ## Installation
 

ccflow/examples/calculator/__init__.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+from typing import List, Optional
+
+from ccflow import RootModelRegistry, load_config as load_config_base
+
+__all__ = ("load_config",)
+
+
+def load_config(
+    config_dir: str = "",
+    config_name: str = "",
+    overrides: Optional[List[str]] = None,
+    *,
+    overwrite: bool = True,
+    basepath: str = "",
+) -> RootModelRegistry:
+    return load_config_base(
+        root_config_dir=str(Path(__file__).resolve().parent / "config"),
+        root_config_name="base",
+        config_dir=config_dir,
+        config_name=config_name,
+        overrides=overrides,
+        overwrite=overwrite,
+        basepath=basepath,
+    )

ccflow/examples/calculator/__main__.py

@@ -0,0 +1,33 @@
+import hydra
+
+from ccflow.utils.hydra import cfg_run
+
+__all__ = ("main",)
+
+
+@hydra.main(config_path="config", config_name="base", version_base=None)
+def main(cfg):
+    cfg_run(cfg)
+
+
+# Run the default calculator (add) on some input:
+# python -m ccflow.examples.calculator +context.values=[1,2,3]
+#
+# Swap the calculation (function is in the defaults list, so no `+`):
+# python -m ccflow.examples.calculator function=scale +context.values=[1,2,3]
+#
+# Configure the selected function's field:
+# python -m ccflow.examples.calculator function=scale function.factor=10 +context.values=[1,2,3]
+# python -m ccflow.examples.calculator function=power function.exponent=3 +context.values=[1,2,3]
+#
+# A composed calculation (round the result of power):
+# python -m ccflow.examples.calculator function=rounded function.digits=1 +context.values=[1.5,2.5]
+#
+# A diamond that reuses a shared `mean` node, deduped by the graph + cache evaluator:
+# python -m ccflow.examples.calculator function=tail_ratio +context.values=[1,2,3,10]
+#
+# Inspect the composed configuration without running it:
+# python -m ccflow.examples.calculator.explain function=power
+
+if __name__ == "__main__":
+    main()

ccflow/examples/calculator/config/base.yaml

@@ -0,0 +1,37 @@
+# @package _global_
+
+# Base configuration for the functional calculator example.
+#
+# The `function` config group selects which calculator runs, and the `output`
+# config group selects how the result is emitted (print, log, or write to file).
+#
+# Run the default calculator, printing the result:
+#   python -m ccflow.examples.calculator +context.values=[1,2,3]
+# Swap the calculator (it is in the defaults list, so no `+`):
+#   python -m ccflow.examples.calculator function=power +context.values=[1,2,3]
+# Configure the selected calculator's field:
+#   python -m ccflow.examples.calculator function=power function.exponent=3 +context.values=[1,2,3]
+# Change how the result is emitted:
+#   python -m ccflow.examples.calculator output=log +context.values=[1,2,3]
+#   python -m ccflow.examples.calculator output=write +context.values=[1,2,3]
+# A diamond that reuses a shared `mean` node (deduped by the graph + cache evaluator below):
+#   python -m ccflow.examples.calculator function=tail_ratio +context.values=[1,2,3,10]
+
+defaults:
+  - _self_
+  - function: add
+  - output: print
+
+callable: /output
+
+# Run every workflow under a graph + cache evaluator, so shared nodes in a
+# composed graph (e.g. the `mean` inside `tail_ratio`) are evaluated once.
+cli:
+  model:
+    _target_: ccflow.FlowOptions
+    evaluator:
+      _target_: ccflow.evaluators.MultiEvaluator
+      evaluators:
+        - _target_: ccflow.evaluators.GraphEvaluator
+        - _target_: ccflow.evaluators.MemoryCacheEvaluator
+    cacheable: true

ccflow/examples/calculator/config/function/add.yaml

@@ -0,0 +1,2 @@
+_target_: ccflow.examples.calculator.functions.add
+offset: 0.0

ccflow/examples/calculator/config/function/power.yaml

@@ -0,0 +1,2 @@
+_target_: ccflow.examples.calculator.functions.power
+exponent: 2.0

ccflow/examples/calculator/config/function/rounded.yaml

@@ -0,0 +1,7 @@
+# Composes two functions: round the result of `power`.
+# `value` is a regular parameter fed by an upstream calculator model.
+_target_: ccflow.examples.calculator.functions.rounded
+digits: 2
+value:
+  _target_: ccflow.examples.calculator.functions.power
+  exponent: 2.0

ccflow/examples/calculator/config/function/scale.yaml

@@ -0,0 +1,2 @@
+_target_: ccflow.examples.calculator.functions.scale
+factor: 1.0

ccflow/examples/calculator/config/function/tail_ratio.yaml

@@ -0,0 +1,12 @@
+# A diamond: tail_ratio needs upper_gap and lower_gap, and both need mean.
+# Under the graph + cache evaluator (see base.yaml), the shared mean is
+# computed once instead of once per branch.
+_target_: ccflow.examples.calculator.functions.tail_ratio
+upper:
+  _target_: ccflow.examples.calculator.functions.upper_gap
+  center:
+    _target_: ccflow.examples.calculator.functions.mean
+lower:
+  _target_: ccflow.examples.calculator.functions.lower_gap
+  center:
+    _target_: ccflow.examples.calculator.functions.mean

ccflow/examples/calculator/config/output/log.yaml

@@ -0,0 +1,6 @@
+_target_: ccflow.PublisherModel
+model: /function
+publisher:
+  _target_: ccflow.publishers.LogPublisher
+  level: info
+field: value

ccflow/examples/calculator/config/output/print.yaml

@@ -0,0 +1,5 @@
+_target_: ccflow.PublisherModel
+model: /function
+publisher:
+  _target_: ccflow.publishers.PrintPublisher
+field: value

ccflow/examples/calculator/config/output/write.yaml

@@ -0,0 +1,7 @@
+_target_: ccflow.PublisherModel
+model: /function
+publisher:
+  _target_: ccflow.publishers.GenericFilePublisher
+  name: result
+  suffix: .txt
+field: value

ccflow/examples/calculator/explain.py

@@ -0,0 +1,13 @@
+from ccflow.utils.hydra import cfg_explain_cli
+
+from .__main__ import main
+
+__all__ = ("explain",)
+
+
+def explain():
+    cfg_explain_cli(config_path="config", config_name="base", hydra_main=main)
+
+
+if __name__ == "__main__":
+    explain()

ccflow/examples/calculator/functions.py

@@ -0,0 +1,74 @@
+"""Functional calculator stages built with the ``@Flow.model`` API.
+
+Each calculator reads its input operands from the runtime context and exposes
+its own configuration as ordinary bound parameters. This is the split that makes
+the functions interchangeable from configuration: the context carries *what to
+compute on*, while the bound fields carry *how to compute*.
+"""
+
+from ccflow import ContextBase, Flow, FromContext
+
+__all__ = ("Numbers", "add", "scale", "power", "rounded", "mean", "upper_gap", "lower_gap", "tail_ratio")
+
+
+class Numbers(ContextBase):
+    """Runtime input for the calculators: the operands to compute on."""
+
+    values: list[float] = []
+
+
+@Flow.model(context_type=Numbers)
+def add(values: FromContext[list[float]], offset: float = 0.0) -> float:
+    """Sum the input values, then add a configurable offset."""
+    return sum(values) + offset
+
+
+@Flow.model(context_type=Numbers)
+def scale(values: FromContext[list[float]], factor: float = 1.0) -> float:
+    """Sum the input values, then multiply by a configurable factor."""
+    return sum(values) * factor
+
+
+@Flow.model(context_type=Numbers)
+def power(values: FromContext[list[float]], exponent: float = 2.0) -> float:
+    """Raise each input value to a configurable exponent and sum the results."""
+    return sum(value**exponent for value in values)
+
+
+@Flow.model
+def rounded(value: float, digits: int = 2) -> float:
+    """Round an upstream calculator's result to a configurable precision.
+
+    ``value`` is a regular parameter, so it can be bound to a literal or fed by
+    another calculator model wired in through configuration.
+    """
+    return round(value, digits)
+
+
+@Flow.model(context_type=Numbers)
+def mean(values: FromContext[list[float]]) -> float:
+    """Average of the input values."""
+    return sum(values) / len(values)
+
+
+@Flow.model(context_type=Numbers)
+def upper_gap(values: FromContext[list[float]], center: float) -> float:
+    """How far the largest value sits above ``center`` (typically the mean)."""
+    return max(values) - center
+
+
+@Flow.model(context_type=Numbers)
+def lower_gap(values: FromContext[list[float]], center: float) -> float:
+    """How far the smallest value sits below ``center`` (typically the mean)."""
+    return center - min(values)
+
+
+@Flow.model
+def tail_ratio(upper: float, lower: float) -> float:
+    """Ratio of the upper gap to the lower gap.
+
+    ``upper`` and ``lower`` are regular parameters fed by ``upper_gap`` and
+    ``lower_gap``, which both depend on ``mean`` — a diamond whose shared node
+    the graph evaluator computes once.
+    """
+    return upper / lower

ccflow/tests/examples/test_calculator.py

@@ -0,0 +1,128 @@
+import sys
+from unittest.mock import patch
+
+import pytest
+
+from ccflow import Flow, FlowOptionsOverride, FromContext, ModelRegistry
+from ccflow.evaluators import GraphEvaluator, MemoryCacheEvaluator, MultiEvaluator
+from ccflow.examples.calculator import load_config
+from ccflow.examples.calculator.__main__ import main
+from ccflow.examples.calculator.explain import explain
+from ccflow.examples.calculator.functions import Numbers, add, lower_gap, power, rounded, scale, tail_ratio, upper_gap
+
+
+class TestCalculatorFunctions:
+    def test_add(self):
+        model = add(offset=1.0)
+        assert model(Numbers(values=[1, 2, 3])).value == 7.0
+
+    def test_scale(self):
+        model = scale(factor=10.0)
+        assert model(Numbers(values=[1, 2, 3])).value == 60.0
+
+    def test_power(self):
+        model = power(exponent=3.0)
+        assert model(Numbers(values=[1, 2, 3])).value == 36.0
+
+    def test_context_from_dict(self):
+        # cfg_run passes the context as a plain dict; it validates to Numbers.
+        model = add()
+        assert model({"values": [1, 2, 3]}).value == 6.0
+
+    def test_composition_derives_deps(self):
+        # @Flow.model generates __deps__ from the wiring; the graph evaluator runs it.
+        model = rounded(value=power(exponent=2.0))
+        assert hasattr(model, "__deps__")
+        assert model.flow.inspect().bound_inputs["value"] is not None
+        evaluator = MultiEvaluator(evaluators=[GraphEvaluator(), MemoryCacheEvaluator()])
+        with FlowOptionsOverride(options={"cacheable": True, "evaluator": evaluator}):
+            assert model({"values": [1.5, 2.5]}).value == 8.5
+
+    def test_diamond_dedup(self):
+        # tail_ratio needs upper_gap and lower_gap, which both need the mean:
+        # a diamond whose shared node the graph evaluator computes once.
+        calls = {"mean": 0}
+
+        @Flow.model(context_type=Numbers)
+        def counting_mean(values: FromContext[list[float]]) -> float:
+            calls["mean"] += 1
+            return sum(values) / len(values)
+
+        center = counting_mean()
+        model = tail_ratio(upper=upper_gap(center=center), lower=lower_gap(center=center))
+
+        model({"values": [1, 2, 3, 10]})
+        assert calls["mean"] == 2  # once per branch under plain evaluation
+
+        calls["mean"] = 0
+        evaluator = MultiEvaluator(evaluators=[GraphEvaluator(), MemoryCacheEvaluator()])
+        with FlowOptionsOverride(options={"cacheable": True, "evaluator": evaluator}):
+            assert model({"values": [1, 2, 3, 10]}).value == 2.0
+        assert calls["mean"] == 1  # shared node evaluated once
+
+
+class TestCalculatorConfig:
+    def _run(self, overrides, values):
+        try:
+            registry = load_config(overrides=overrides)
+            return registry["/function"]({"values": values}).value
+        finally:
+            ModelRegistry.root().clear()
+
+    def test_default_function(self):
+        assert self._run([], [1, 2, 3]) == 6.0
+
+    def test_swap_function(self):
+        assert self._run(["function=scale", "function.factor=10"], [1, 2, 3]) == 60.0
+
+    def test_configure_function(self):
+        assert self._run(["function=power", "function.exponent=3"], [1, 2, 3]) == 36.0
+
+    def test_composed_function(self):
+        # `rounded` wraps `power` (exponent 2) as an upstream model input.
+        assert self._run(["function=rounded", "function.digits=1"], [1.5, 2.5]) == 8.5
+
+    def test_diamond_function(self):
+        # tail_ratio composes a shared-mean diamond entirely in YAML.
+        assert self._run(["function=tail_ratio"], [1, 2, 3, 10]) == 2.0
+
+
+class TestCalculatorOutput:
+    def _run_output(self, overrides, values):
+        try:
+            registry = load_config(overrides=overrides)
+            return registry["/output"]({"values": values}).value
+        finally:
+            ModelRegistry.root().clear()
+
+    def test_output_print(self, capsys):
+        result = self._run_output(["output=print", "function=power", "function.exponent=3"], [1, 2, 3])
+        assert result == 36.0
+        assert "36.0" in capsys.readouterr().out
+
+    def test_output_log(self):
+        result = self._run_output(["output=log"], [1, 2, 3])
+        assert result == 6.0
+
+    def test_output_write(self, tmp_path, monkeypatch):
+        monkeypatch.chdir(tmp_path)
+        try:
+            registry = load_config(overrides=["output=write", "function=scale", "function.factor=10"])
+            registry["/output"]({"values": [1, 2, 3]})
+        finally:
+            ModelRegistry.root().clear()
+        assert (tmp_path / "result.txt").read_text() == "60.0"
+
+
+class TestCalculatorCli:
+    @pytest.mark.skipif(sys.version_info >= (3, 14), reason="Hydra shell completion help string incompatible with Python 3.14 argparse")
+    def test_cli(self):
+        with patch("ccflow.examples.calculator.__main__.cfg_run") as mock_cfg_run:
+            with patch("sys.argv", ["calculator", "function=power", "+context.values=[1,2,3]"]):
+                main()
+                mock_cfg_run.assert_called_once()
+
+    def test_explain(self):
+        with patch("ccflow.examples.calculator.explain.cfg_explain_cli") as mock_cfg_explain:
+            explain()
+            mock_cfg_explain.assert_called_once()