From 3db43c8908340d5d22f3f745731ee606cf84bad4 Mon Sep 17 00:00:00 2001 From: Anton Krivoborodov <63401640+antonkri@users.noreply.github.com> Date: Wed, 24 Jun 2026 10:43:29 +0000 Subject: [PATCH 1/3] Add sphinx-needs filtering, Markdown and TRLC export rules (WIP playground) Add Bazel macros and Python tools to post-process needs.json: - filter_needs_json / component_requirements / feature_requirements / assumptions_of_use: extract a subset of sphinx-needs elements - sphinx_needs_to_md: render needs as a Markdown document - sphinx_needs_to_trlc: convert S-CORE requirements to TRLC WIP / demonstration only. --- docs.bzl | 223 +++++++++++++++++++++++ scripts_bazel/BUILD | 24 +++ scripts_bazel/README_needs_rules.md | 83 +++++++++ scripts_bazel/filter_needs_json.py | 166 ++++++++++++++++++ scripts_bazel/sphinx_needs_to_md.py | 173 ++++++++++++++++++ scripts_bazel/sphinx_needs_to_trlc.py | 244 ++++++++++++++++++++++++++ 6 files changed, 913 insertions(+) create mode 100644 scripts_bazel/README_needs_rules.md create mode 100644 scripts_bazel/filter_needs_json.py create mode 100644 scripts_bazel/sphinx_needs_to_md.py create mode 100644 scripts_bazel/sphinx_needs_to_trlc.py diff --git a/docs.bzl b/docs.bzl index 7f5ad0ced..e5fbc4f15 100644 --- a/docs.bzl +++ b/docs.bzl @@ -96,6 +96,229 @@ def _merge_sourcelinks(name, sourcelinks, known_good = None): tools = [merge_sourcelinks_tool], ) +def filtered_needs_json( + name, + src, + types = [], + components = [], + component_attr = "component", + visibility = None): + """Extract a subset of sphinx-needs elements from a needs.json file. + + Produces a `.json` file containing only the needs that match all of + the given filters. This is useful to hand a downstream consumer just the + elements (e.g. `feat_req`) of one or more particular components. + + Args: + name: Name of the generated target. The output file is `.json`. + src: Label of a `needs_json` build output (a directory containing + `needs.json`), e.g. `":needs_json"` or `"@score_process//:needs_json"`. + types: Optional list of sphinx-needs element types to keep + (e.g. `["feat_req", "comp_req"]`). If empty, all types are kept. + components: Optional list of component names to keep. If empty, all + components are kept. + component_attr: Need attribute matched against `components`. + Defaults to `"component"`. + visibility: Standard Bazel visibility for the generated target. + """ + filter_tool = Label("//scripts_bazel:filter_needs_json") + + type_args = " ".join(["--type '%s'" % t for t in types]) + component_args = " ".join(["--component '%s'" % c for c in components]) + + native.genrule( + name = name, + srcs = [src], + outs = [name + ".json"], + cmd = """ + $(location {filter_tool}) \ + --output $@ \ + --component-attr '{component_attr}' \ + {type_args} \ + {component_args} \ + $(location {src})/needs.json + """.format( + filter_tool = filter_tool, + component_attr = component_attr, + type_args = type_args, + component_args = component_args, + src = src, + ), + tools = [filter_tool], + visibility = visibility, + ) + +def component_requirements( + name, + src = "//:needs_json", + component = None, + visibility = None): + """Extract the component requirements (`comp_req`) from a needs.json file. + + Convenience wrapper around `filtered_needs_json`. Produces a `.json` + file containing only `comp_req` elements. + + Args: + name: Name of the generated target. The output file is `.json`. + src: Label of a `needs_json` build output. Defaults to the calling + package's `//:needs_json`. + component: Optional component name. If given, only component requirements + tagged with that component are kept; if omitted, all component + requirements are kept. + visibility: Standard Bazel visibility for the generated target. + """ + filtered_needs_json( + name = name, + src = src, + types = ["comp_req"], + components = [component] if component else [], + component_attr = "tags", + visibility = visibility, + ) + +def feature_requirements( + name, + src = "//:needs_json", + feature = None, + visibility = None): + """Extract the feature requirements (`feat_req`) from a needs.json file. + + Convenience wrapper around `filtered_needs_json`. Produces a `.json` + file containing only `feat_req` elements. + + Args: + name: Name of the generated target. The output file is `.json`. + src: Label of a `needs_json` build output. Defaults to the calling + package's `//:needs_json`. + feature: Optional feature name. If given, only feature requirements + tagged with that feature are kept; if omitted, all feature + requirements are kept. + visibility: Standard Bazel visibility for the generated target. + """ + filtered_needs_json( + name = name, + src = src, + types = ["feat_req"], + components = [feature] if feature else [], + component_attr = "tags", + visibility = visibility, + ) + +def assumptions_of_use( + name, + src = "//:needs_json", + component = None, + visibility = None): + """Extract the assumptions of use (`aou_req`) from a needs.json file. + + Convenience wrapper around `filtered_needs_json`. Produces a `.json` + file containing only `aou_req` elements. + + Args: + name: Name of the generated target. The output file is `.json`. + src: Label of a `needs_json` build output. Defaults to the calling + package's `//:needs_json`. + component: Optional component name. If given, only assumptions of use + tagged with that component are kept; if omitted, all assumptions of + use are kept. + visibility: Standard Bazel visibility for the generated target. + """ + filtered_needs_json( + name = name, + src = src, + types = ["aou_req"], + components = [component] if component else [], + component_attr = "tags", + visibility = visibility, + ) + +def sphinx_needs_to_md( + name, + src, + title = "Sphinx-needs elements", + visibility = None): + """Render the sphinx-needs elements of a needs.json file as a Markdown document. + + Produces a `.md` file containing a human readable description of every + sphinx-needs element found in `src`. Typically `src` is the output of a + `filtered_needs_json` target, but any `needs.json`-style file works. + + Args: + name: Name of the generated target. The output file is `.md`. + src: Label of a needs.json file, e.g. a `filtered_needs_json` target + (`":my_feat_reqs"`) or a `needs_json` directory output. + title: Title rendered at the top of the generated document. + visibility: Standard Bazel visibility for the generated target. + """ + sphinx_needs_to_md_tool = Label("//scripts_bazel:sphinx_needs_to_md") + + native.genrule( + name = name, + srcs = [src], + outs = [name + ".md"], + cmd = """ + $(location {sphinx_needs_to_md_tool}) \ + --output $@ \ + --title '{title}' \ + $(location {src}) + """.format( + sphinx_needs_to_md_tool = sphinx_needs_to_md_tool, + title = title, + src = src, + ), + tools = [sphinx_needs_to_md_tool], + visibility = visibility, + ) + +def sphinx_needs_to_trlc( + name, + src, + package = "Needs", + visibility = None): + """Convert the requirement sphinx-needs elements of a needs.json file into TRLC. + + TRLC ("Treat Requirements Like Code", + https://github.com/bmw-software-engineering/trlc) is requirements-only tooling. + Only the S-CORE requirement element types are converted; everything else is + ignored: + + * `feat_req` -> `ScoreReq.FeatReq` (feature requirement) + * `comp_req` -> `ScoreReq.CompReq` (component requirement) + * `aou_req` -> `ScoreReq.AoU` (assumption of use) + + Produces a `.trlc` data file in package `package` targeting the S-CORE + requirements metamodel (package `ScoreReq`) from + https://github.com/eclipse-score/tooling/tree/main/bazel/rules/rules_score/trlc. + Validate the output together with that metamodel (e.g. via `trlc_requirements` + using `score_requirements_model` as `spec`). + + Args: + name: Name of the generated target. The output file is `.trlc`. + src: Label of a needs.json file, e.g. a `filtered_needs_json` target + (`":my_feat_reqs"`) or a `needs_json` directory output. + package: TRLC package name used for the generated requirements. + visibility: Standard Bazel visibility for the generated target. + """ + sphinx_needs_to_trlc_tool = Label("//scripts_bazel:sphinx_needs_to_trlc") + + native.genrule( + name = name, + srcs = [src], + outs = [name + ".trlc"], + cmd = """ + $(location {sphinx_needs_to_trlc_tool}) \ + --output $@ \ + --package '{package}' \ + $(location {src}) + """.format( + sphinx_needs_to_trlc_tool = sphinx_needs_to_trlc_tool, + package = package, + src = src, + ), + tools = [sphinx_needs_to_trlc_tool], + visibility = visibility, + ) + def _missing_requirements(deps): """Add Python hub dependencies if they are missing.""" found = [] diff --git a/scripts_bazel/BUILD b/scripts_bazel/BUILD index e2d0402d2..50eac7eb3 100644 --- a/scripts_bazel/BUILD +++ b/scripts_bazel/BUILD @@ -45,3 +45,27 @@ py_binary( visibility = ["//visibility:public"], deps = [], ) + +py_binary( + name = "filter_needs_json", + srcs = ["filter_needs_json.py"], + main = "filter_needs_json.py", + visibility = ["//visibility:public"], + deps = [], +) + +py_binary( + name = "sphinx_needs_to_md", + srcs = ["sphinx_needs_to_md.py"], + main = "sphinx_needs_to_md.py", + visibility = ["//visibility:public"], + deps = [], +) + +py_binary( + name = "sphinx_needs_to_trlc", + srcs = ["sphinx_needs_to_trlc.py"], + main = "sphinx_needs_to_trlc.py", + visibility = ["//visibility:public"], + deps = [], +) diff --git a/scripts_bazel/README_needs_rules.md b/scripts_bazel/README_needs_rules.md new file mode 100644 index 000000000..b3f6ae5c8 --- /dev/null +++ b/scripts_bazel/README_needs_rules.md @@ -0,0 +1,83 @@ +# Sphinx-needs processing rules (playground) + +> **WIP / demonstration only.** This branch (`ankr_rules_score_playground`) is a +> proof of concept to explore how `rules_score` could post-process sphinx-needs +> data. It is **not** production ready and is shared for demonstration purposes. + +## Why these changes + +The documentation build already produces a `needs.json` file that contains every +sphinx-needs element (requirements, assumptions of use, ...). Downstream tooling +often needs only a *subset* of that data, or the data in a *different format*. + +This change adds a small set of Bazel macros (in [docs.bzl](../docs.bzl)) plus +the backing Python tools (in this folder) to: + +1. **Filter** a `needs.json` down to selected element types and/or components. +2. **Render** the selected elements as a human readable Markdown document. +3. **Convert** S-CORE requirement elements into [TRLC](https://github.com/bmw-software-engineering/trlc) + data targeting the S-CORE requirements metamodel. + +The goal is to show how requirements managed as sphinx-needs can be bridged to +other consumers (review docs, TRLC-based tooling) without manual copying. + +## What was added + +### Bazel macros (`docs.bzl`) + +| Macro | Output | Purpose | +| --- | --- | --- | +| `filtered_needs_json` | `.json` | Keep only needs matching the given `types` / `components`. | +| `component_requirements` | `.json` | Convenience wrapper for `comp_req` elements. | +| `feature_requirements` | `.json` | Convenience wrapper for `feat_req` elements. | +| `assumptions_of_use` | `.json` | Convenience wrapper for `aou_req` elements. | +| `sphinx_needs_to_md` | `.md` | Render needs as a Markdown document. | +| `sphinx_needs_to_trlc` | `.trlc` | Convert S-CORE requirements to TRLC. | + +### Python tools (`scripts_bazel/`) + +- [filter_needs_json.py](filter_needs_json.py) — extract a subset of needs. +- [sphinx_needs_to_md.py](sphinx_needs_to_md.py) — render needs as Markdown. +- [sphinx_needs_to_trlc.py](sphinx_needs_to_trlc.py) — convert needs to TRLC. + +The matching `py_binary` targets are declared in [BUILD](BUILD). + +## How to use + +In a `BUILD` file that already has a `needs_json` target, load the macros and +chain them: + +```starlark +load("@docs-as-code//:docs.bzl", "feature_requirements", "sphinx_needs_to_md", "sphinx_needs_to_trlc") + +# 1. Filter: keep only the feature requirements of one feature. +feature_requirements( + name = "my_feature_reqs", + src = "//:needs_json", + feature = "my_feature", +) + +# 2. Render the filtered set as Markdown. +sphinx_needs_to_md( + name = "my_feature_reqs_md", + src = ":my_feature_reqs", + title = "My feature requirements", +) + +# 3. Convert the filtered set to TRLC. +sphinx_needs_to_trlc( + name = "my_feature_reqs_trlc", + src = ":my_feature_reqs", + package = "MyFeature", +) +``` + +Build any of the targets to produce the corresponding output file: + +```bash +bazel build //path/to:my_feature_reqs_md +bazel build //path/to:my_feature_reqs_trlc +``` + +You can also call `filtered_needs_json` directly for full control over the +`types`, `components`, and `component_attr` filters. diff --git a/scripts_bazel/filter_needs_json.py b/scripts_bazel/filter_needs_json.py new file mode 100644 index 000000000..c078afafd --- /dev/null +++ b/scripts_bazel/filter_needs_json.py @@ -0,0 +1,166 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +""" +Extract a subset of sphinx-needs elements from a needs.json file. + +A need is kept when it matches *all* of the active filters: + +* ``--type``: the value of the need's ``type`` attribute is in the requested + list of element types (e.g. ``feat_req``). If no ``--type`` is given, needs + of any type are kept. +* ``--component``: the value of the need's component attribute (configurable + via ``--component-attr``, default ``component``) matches one of the requested + component names. The attribute may hold a single string or a list of strings; + a need is kept when any of its values matches. If no ``--component`` is given, + needs of any component are kept. + +The top-level structure of the needs.json file is preserved; only the per-need +entries are filtered. +""" + +import argparse +import json +import logging +import sys +from pathlib import Path +from typing import Any + +logging.basicConfig(level=logging.INFO, format="%(message)s") +logger = logging.getLogger(__name__) + + +def _attribute_values(need: dict[str, Any], attr: str) -> list[str]: + """Return the values of ``attr`` on a need as a list of strings.""" + value = need.get(attr) + if value is None: + return [] + if isinstance(value, list): + return [str(v) for v in value] # pyright: ignore[reportUnknownVariableType] + return [str(value)] + + +def _keep_need( + need: dict[str, Any], + types: set[str], + components: set[str], + component_attr: str, +) -> bool: + if types and need.get("type") not in types: + return False + if components: + values = set(_attribute_values(need, component_attr)) + if values.isdisjoint(components): + return False + return True + + +def filter_needs( + data: dict[str, Any], + types: set[str], + components: set[str], + component_attr: str, +) -> dict[str, Any]: + """Return a copy of ``data`` keeping only the needs that match the filters.""" + for version in data.get("versions", {}).values(): + needs = version.get("needs", {}) + version["needs"] = { + need_id: need + for need_id, need in needs.items() + if _keep_need(need, types, components, component_attr) + } + return data + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Extract a subset of sphinx-needs elements from a needs.json file." + ) + ) + _ = parser.add_argument( + "--output", + required=True, + type=Path, + help="Path of the filtered needs.json file to write.", + ) + _ = parser.add_argument( + "--type", + dest="types", + action="append", + default=[], + metavar="ELEMENT_TYPE", + help=( + "Sphinx-needs element type to keep (e.g. 'feat_req'). " + "May be given multiple times. If omitted, all types are kept." + ), + ) + _ = parser.add_argument( + "--component", + dest="components", + action="append", + default=[], + metavar="COMPONENT", + help=( + "Component name to keep. May be given multiple times. " + "If omitted, all components are kept." + ), + ) + _ = parser.add_argument( + "--component-attr", + default="component", + help=( + "Need attribute matched against the values given via --component. " + "Defaults to 'component'." + ), + ) + _ = parser.add_argument( + "input", + type=Path, + help="Input needs.json file to filter.", + ) + + args = parser.parse_args() + + with open(args.input) as f: + data = json.load(f) + + filtered = filter_needs( + data, + types=set(args.types), + components=set(args.components), + component_attr=args.component_attr, + ) + + kept = sum( + len(version.get("needs", {})) + for version in filtered.get("versions", {}).values() + ) + logger.info( + "Filtered '%s' -> '%s' (%d needs kept, types=%s, components=%s)", + args.input, + args.output, + kept, + sorted(args.types) or "ALL", + sorted(args.components) or "ALL", + ) + + args.output.parent.mkdir(parents=True, exist_ok=True) + with open(args.output, "w") as f: + json.dump(filtered, f, indent=2, sort_keys=True) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts_bazel/sphinx_needs_to_md.py b/scripts_bazel/sphinx_needs_to_md.py new file mode 100644 index 000000000..e71a1c55c --- /dev/null +++ b/scripts_bazel/sphinx_needs_to_md.py @@ -0,0 +1,173 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +""" +Render the sphinx-needs elements of a needs.json file as a Markdown document. + +The input is a needs.json file (typically the output of ``filtered_needs_json``). +Each need is rendered as a Markdown section. Needs are grouped by their ``type`` +and sorted by ``id`` so the output is stable and diff friendly. +""" + +import argparse +import json +import logging +import sys +from pathlib import Path +from typing import Any + +logging.basicConfig(level=logging.INFO, format="%(message)s") +logger = logging.getLogger(__name__) + +# Attributes rendered (in this order) as a table for every need. +# Only attributes that are present and non-empty on a need are shown. +_HEADER_ATTRS: list[tuple[str, str]] = [ + ("title", "Title"), + ("type_name", "Type name"), + ("status", "Status"), + ("safety", "Safety"), + ("security", "Security"), + ("reqtype", "Requirement type"), + ("tags", "Tags"), + ("docname", "Document"), +] + + +def _format_value(value: object) -> str: + """Render an attribute value as a single line Markdown-safe string.""" + text = ( + ", ".join(str(v) for v in value) # pyright: ignore[reportUnknownVariableType] + if isinstance(value, list) + else str(value) + ) + # Escape the Markdown table cell separator. + return text.replace("|", "\\|") + + +def _collect_needs(data: dict[str, Any]) -> list[dict[str, Any]]: + """Return all needs across every version, sorted by id.""" + needs: list[dict[str, Any]] = [] + for version in data.get("versions", {}).values(): + needs.extend(version.get("needs", {}).values()) + return sorted(needs, key=lambda need: str(need.get("id", ""))) + + +def _render_need(need: dict[str, Any]) -> str: + """Render a single need as a Markdown block.""" + lines: list[str] = [] + need_id = str(need.get("id", "")) + lines.append(f"### `{need_id}`") + lines.append("") + + rows = [ + (label, _format_value(value)) + for attr, label in _HEADER_ATTRS + if (value := need.get(attr)) not in (None, "", []) + ] + if rows: + lines.append("| Attribute | Value |") + lines.append("| --- | --- |") + for label, value in rows: + lines.append(f"| {label} | {value} |") + lines.append("") + + content = str(need.get("content", "")).strip() + if content: + lines.append("**Content:**") + lines.append("") + lines.append("```") + lines.extend(content.splitlines()) + lines.append("```") + return "\n".join(lines).rstrip() + + +def render_document(data: dict[str, Any], title: str) -> str: + """Render the whole needs document as Markdown.""" + needs = _collect_needs(data) + types: dict[str, int] = {} + for need in needs: + type_ = str(need.get("type", "")) + types[type_] = types.get(type_, 0) + 1 + + blocks: list[str] = [] + blocks.append(f"# {title}") + blocks.append(f"Total needs: **{len(needs)}**") + + if types: + summary_lines = ["| Type | Count |", "| --- | --- |"] + summary_lines.extend( + f"| `{type_}` | {count} |" for type_, count in sorted(types.items()) + ) + blocks.append("\n".join(summary_lines)) + + current_type = None + for need in sorted( + needs, key=lambda n: (str(n.get("type", "")), str(n.get("id", ""))) + ): + type_ = str(need.get("type", "")) + if type_ != current_type: + current_type = type_ + blocks.append(f"## `{type_}`") + blocks.append(_render_need(need)) + + return "\n\n".join(blocks) + "\n" + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Render the sphinx-needs elements of a needs.json file as Markdown." + ) + ) + _ = parser.add_argument( + "--output", + required=True, + type=Path, + help="Path of the Markdown file to write.", + ) + _ = parser.add_argument( + "--title", + default="Sphinx-needs elements", + help="Title rendered at the top of the document.", + ) + _ = parser.add_argument( + "input", + type=Path, + help="Input needs.json file to document.", + ) + + args = parser.parse_args() + + with open(args.input) as f: + data = json.load(f) + + document = render_document(data, title=args.title) + + args.output.parent.mkdir(parents=True, exist_ok=True) + with open(args.output, "w") as f: + _ = f.write(document) + + need_count = sum( + len(version.get("needs", {})) for version in data.get("versions", {}).values() + ) + logger.info( + "Documented '%s' -> '%s' (%d needs)", + args.input, + args.output, + need_count, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts_bazel/sphinx_needs_to_trlc.py b/scripts_bazel/sphinx_needs_to_trlc.py new file mode 100644 index 000000000..5eda3f652 --- /dev/null +++ b/scripts_bazel/sphinx_needs_to_trlc.py @@ -0,0 +1,244 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +""" +Convert the requirement sphinx-needs elements of a needs.json file into TRLC. + +TRLC ("Treat Requirements Like Code", https://github.com/bmw-software-engineering/trlc) +is a requirements-only tooling. Therefore only the S-CORE requirement element +types are converted: + +* ``feat_req`` -> ``ScoreReq.FeatReq`` (feature requirement) +* ``comp_req`` -> ``ScoreReq.CompReq`` (component requirement) +* ``aou_req`` -> ``ScoreReq.AoU`` (assumption of use) + +All other sphinx-needs elements are ignored. + +The generated ``.trlc`` data file targets the S-CORE requirements metamodel +(package ``ScoreReq``) defined in +https://github.com/eclipse-score/tooling/tree/main/bazel/rules/rules_score/trlc +and must be validated together with that metamodel. +""" + +import argparse +import json +import logging +import re +import sys +from pathlib import Path +from typing import Any + +logging.basicConfig(level=logging.INFO, format="%(message)s") +logger = logging.getLogger(__name__) + +# sphinx-needs type -> ScoreReq metamodel type. +_TYPE_MAP: dict[str, str] = { + "feat_req": "FeatReq", + "comp_req": "CompReq", + "aou_req": "AoU", +} + +# ScoreReq type -> the metamodel type its ``derived_from`` references, if any. +# Only references that resolve to an emitted object of this type are rendered. +_DERIVED_FROM_TARGET: dict[str, str] = { + "FeatReq": "AssumedSystemReq", + "CompReq": "FeatReq", +} + + +def _string_literal(value: str) -> str: + """Return a TRLC string literal for ``value``.""" + text = str(value) + if "\n" in text: + if '"""' not in text: + return f'"""\n{text}\n"""' + if "'''" not in text: + return f"'''\n{text}\n'''" + return '"""\n{text}\n"""'.format(text=text.replace('"""', '\\"\\"\\"')) + return '"{text}"'.format(text=text.replace('"', '\\"')) + + +def _asil(value: object) -> str: + """Map a sphinx-needs safety value to a ScoreReq.Asil enum literal.""" + text = str(value or "").upper().replace("ASIL", "").strip(" _-") + if text == "B": + return "ScoreReq.Asil.B" + if text == "D": + return "ScoreReq.Asil.D" + return "ScoreReq.Asil.QM" + + +def _version(need: dict[str, Any]) -> int: + """Return the integer version of a need, defaulting to 1.""" + try: + return int(need.get("version", 1)) + except (TypeError, ValueError): + return 1 + + +def _identifier(raw: str, used: set[str]) -> str: + """Turn an arbitrary need id into a unique, valid TRLC identifier.""" + candidate = re.sub(r"[^A-Za-z0-9_]", "_", str(raw)) + if not candidate or not (candidate[0].isalpha() or candidate[0] == "_"): + candidate = "n_" + candidate + unique = candidate + suffix = 2 + while unique in used: + unique = f"{candidate}_{suffix}" + suffix += 1 + used.add(unique) + return unique + + +def _collect_requirement_needs(data: dict[str, Any]) -> list[dict[str, Any]]: + """Return all needs whose type maps to a ScoreReq requirement, sorted.""" + needs: list[dict[str, Any]] = [] + for version in data.get("versions", {}).values(): + for need in version.get("needs", {}).values(): + if need.get("type") in _TYPE_MAP: + needs.append(need) + needs.sort(key=lambda n: (str(n.get("type", "")), str(n.get("id", "")))) + return needs + + +def _build_id_map(needs: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + """Map original need id -> {ident, score_type, version} for every need.""" + id_map: dict[str, dict[str, Any]] = {} + used: set[str] = set() + for need in needs: + original = str(need.get("id", "need")) + id_map[original] = { + "ident": _identifier(original, used), + "score_type": _TYPE_MAP[need["type"]], + "version": _version(need), + } + return id_map + + +def _render_object(need: dict[str, Any], id_map: dict[str, dict[str, Any]]) -> str: + """Render a single requirement need as a ScoreReq object.""" + info = id_map[str(need.get("id"))] + score_type = info["score_type"] + description = str(need.get("content") or need.get("title") or info["ident"]) + + lines = [ + "ScoreReq.{score_type} {ident} {{".format( + score_type=score_type, ident=info["ident"] + ), + f" description = {_string_literal(description)}", + f" version = {_version(need)}", + " safety = {value}".format(value=_asil(need.get("safety"))), + ] + + target_type = _DERIVED_FROM_TARGET.get(score_type) + if target_type: + refs: list[str] = [] + for ref in need.get("derived_from") or []: # pyright: ignore[reportUnknownVariableType] + ref_info = id_map.get(str(ref)) + if ref_info and ref_info["score_type"] == target_type: + refs.append( + "{ident} @ {version}".format( + ident=ref_info["ident"], version=ref_info["version"] + ) + ) + if refs: + lines.append(" derived_from = [{refs}]".format(refs=", ".join(refs))) + + lines.append("}") + return "\n".join(lines) + + +def render_trlc(data: dict[str, Any], package: str) -> str: + """Render the requirements data file (``.trlc``).""" + needs = _collect_requirement_needs(data) + id_map = _build_id_map(needs) + + blocks = [ + f"package {package}", + "", + "import ScoreReq", + ] + + body: list[str] = [] + current_type = None + section_open = False + for need in needs: + type_ = str(need.get("type")) + if type_ != current_type: + if section_open: + body.append("}") + current_type = type_ + body.append("") + body.append(f"section {_string_literal(type_)} {{") + section_open = True + obj = _render_object(need, id_map) + body.append("\n".join(" " + line for line in obj.splitlines())) + if section_open: + body.append("}") + + blocks.append("\n".join(body)) + return "\n".join(blocks).rstrip() + "\n" + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Convert the requirement sphinx-needs elements of a needs.json file " + "into TRLC targeting the S-CORE requirements metamodel (ScoreReq)." + ), + ) + _ = parser.add_argument( + "--output", + required=True, + type=Path, + help="Path of the TRLC data file (.trlc) to write.", + ) + _ = parser.add_argument( + "--package", + default="Needs", + help="TRLC package name used for the generated requirements.", + ) + _ = parser.add_argument( + "input", + type=Path, + help="Input needs.json file to convert.", + ) + + args = parser.parse_args() + + package = re.sub(r"[^A-Za-z0-9_]", "_", args.package) + if not package or not (package[0].isalpha() or package[0] == "_"): + package = "Needs" + + with open(args.input) as f: + data = json.load(f) + + objects = render_trlc(data, package=package) + + args.output.parent.mkdir(parents=True, exist_ok=True) + with open(args.output, "w") as f: + _ = f.write(objects) + + converted = len(_collect_requirement_needs(data)) + logger.info( + "Converted '%s' -> '%s' (%d requirements, package '%s')", + args.input, + args.output, + converted, + package, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 52fb989f2cd7b9cae53cac0b958389e55f469d2d Mon Sep 17 00:00:00 2001 From: Anton Krivoborodov <63401640+antonkri@users.noreply.github.com> Date: Wed, 24 Jun 2026 11:52:38 +0000 Subject: [PATCH 2/3] Add requirement checklist need type and validation rule (WIP playground) Adds a 'req_chklst' sphinx-needs type that pins the reviewed state of build outputs via a SHA256 hash, plus a 'requirements_checklist' Bazel rule and 'validate_checklist.py' tool that recompute the hash over the validated target outputs and fail the build when it drifts. --- docs.bzl | 85 ++++++++++- scripts_bazel/BUILD | 10 +- scripts_bazel/README_needs_rules.md | 64 ++++++++ scripts_bazel/validate_checklist.py | 139 ++++++++++++++++++ src/extensions/score_metamodel/metamodel.yaml | 29 ++++ 5 files changed, 320 insertions(+), 7 deletions(-) create mode 100644 scripts_bazel/validate_checklist.py diff --git a/docs.bzl b/docs.bzl index e5fbc4f15..0388f7748 100644 --- a/docs.bzl +++ b/docs.bzl @@ -63,8 +63,10 @@ def _rewrite_needs_json_to_sourcelinks(labels): s = str(x) if s.endswith("//:needs_json"): out.append(s.replace("//:needs_json", "//:sourcelinks_json")) + #Items which do not end up with '//:needs_json' shall not be appended to 'out'. #They are treated separately and are not related to source code linking. + return out def _merge_sourcelinks(name, sourcelinks, known_good = None): @@ -319,19 +321,90 @@ def sphinx_needs_to_trlc( visibility = visibility, ) +def requirements_checklist( + name, + checklist_id, + deps, + src = "//:needs_json", + visibility = None): + """Validate a requirement checklist (`req_chklst`) against its build output. + + Building this target recomputes the SHA256 over the concatenated outputs of + `deps` and compares it to the `sha256` attribute of the `req_chklst` need + `checklist_id` (looked up in `src`'s `needs.json`). The build **fails** when + the hashes differ, i.e. when a validated target output has changed since the + checklist was last reviewed. + + Typical usage validates the extracted requirements of a component against the + checklist that reviewed them: + + component_requirements( + name = "bitmanipulation_comp_reqs", + component = "bitmanipulation", + ) + + requirements_checklist( + name = "bitmanipulation_req_checklist", + checklist_id = "req_chklst__bitmanipulation__comp_req", + deps = [":bitmanipulation_comp_reqs"], + ) + + Run with `bazel build //:bitmanipulation_req_checklist`. On the first run (or + after the requirements change) the build fails and prints the actual SHA256; + copy it into the `sha256` attribute of the checklist need once the checklist + has been (re-)reviewed. + + Args: + name: Name of the generated target. The output file is `.sha256`. + checklist_id: Id of the `req_chklst` need to validate + (e.g. `"req_chklst__bitmanipulation__comp_req"`). + deps: List of labels whose outputs are hashed and validated. Usually a + single `component_requirements`/`filtered_needs_json` target. + src: Label of a `needs_json` build output containing the checklist need. + Defaults to the calling package's `//:needs_json`. + visibility: Standard Bazel visibility for the generated target. + """ + validate_tool = Label("//scripts_bazel:validate_checklist") + + dep_args = " ".join(["$(locations %s)" % d for d in deps]) + + native.genrule( + name = name, + srcs = [src] + deps, + outs = [name + ".sha256"], + cmd = """ + $(location {validate_tool}) \ + --needs-json $(location {src})/needs.json \ + --checklist-id '{checklist_id}' \ + --output $@ \ + {dep_args} + """.format( + validate_tool = validate_tool, + checklist_id = checklist_id, + src = src, + dep_args = dep_args, + ), + tools = [validate_tool], + visibility = visibility, + ) + def _missing_requirements(deps): """Add Python hub dependencies if they are missing.""" found = [] missing = [] + def _target_to_packagename(target): return str(target).split("/")[-1].split(":")[0] + all_packages = [_target_to_packagename(pkg) for pkg in all_requirements] + def _find(pkg): for dep in deps: dep_pkg = _target_to_packagename(dep) if dep_pkg == pkg: return True return False + for pkg in all_packages: if _find(pkg): found.append(pkg) @@ -453,7 +526,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good = srcs = [incremental_src], data = docs_data, deps = deps, - env = docs_env + env = docs_env, ) docs_sources_env["ACTION"] = "incremental" @@ -463,7 +536,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good = srcs = [incremental_src], data = combo_data, deps = deps, - env = docs_sources_env + env = docs_sources_env, ) native.alias( @@ -479,7 +552,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good = srcs = [incremental_src], data = docs_data, deps = deps, - env = docs_env + env = docs_env, ) docs_env["ACTION"] = "check" @@ -489,7 +562,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good = srcs = [incremental_src], data = docs_data, deps = deps, - env = docs_env + env = docs_env, ) docs_env["ACTION"] = "live_preview" @@ -499,7 +572,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good = srcs = [incremental_src], data = docs_data, deps = deps, - env = docs_env + env = docs_env, ) docs_sources_env["ACTION"] = "live_preview" @@ -509,7 +582,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good = srcs = [incremental_src], data = combo_data, deps = deps, - env = docs_sources_env + env = docs_sources_env, ) py_venv( diff --git a/scripts_bazel/BUILD b/scripts_bazel/BUILD index 50eac7eb3..2ecb3199f 100644 --- a/scripts_bazel/BUILD +++ b/scripts_bazel/BUILD @@ -33,9 +33,9 @@ py_binary( py_binary( name = "merge_sourcelinks", srcs = ["merge_sourcelinks.py"], - deps= [ "//src/extensions/score_source_code_linker"], main = "merge_sourcelinks.py", visibility = ["//visibility:public"], + deps = ["//src/extensions/score_source_code_linker"], ) py_binary( @@ -69,3 +69,11 @@ py_binary( visibility = ["//visibility:public"], deps = [], ) + +py_binary( + name = "validate_checklist", + srcs = ["validate_checklist.py"], + main = "validate_checklist.py", + visibility = ["//visibility:public"], + deps = [], +) diff --git a/scripts_bazel/README_needs_rules.md b/scripts_bazel/README_needs_rules.md index b3f6ae5c8..9abaf9a1d 100644 --- a/scripts_bazel/README_needs_rules.md +++ b/scripts_bazel/README_needs_rules.md @@ -17,6 +17,9 @@ the backing Python tools (in this folder) to: 2. **Render** the selected elements as a human readable Markdown document. 3. **Convert** S-CORE requirement elements into [TRLC](https://github.com/bmw-software-engineering/trlc) data targeting the S-CORE requirements metamodel. +4. **Validate** a reviewable *requirement checklist* against the build output it + was reviewed against, by pinning a SHA256 hash in a `req_chklst` sphinx-needs + element and failing the build when the output drifts. The goal is to show how requirements managed as sphinx-needs can be bridged to other consumers (review docs, TRLC-based tooling) without manual copying. @@ -33,15 +36,25 @@ other consumers (review docs, TRLC-based tooling) without manual copying. | `assumptions_of_use` | `.json` | Convenience wrapper for `aou_req` elements. | | `sphinx_needs_to_md` | `.md` | Render needs as a Markdown document. | | `sphinx_needs_to_trlc` | `.trlc` | Convert S-CORE requirements to TRLC. | +| `requirements_checklist` | `.sha256` | Validate a `req_chklst` need against its target output via SHA256. | ### Python tools (`scripts_bazel/`) - [filter_needs_json.py](filter_needs_json.py) — extract a subset of needs. - [sphinx_needs_to_md.py](sphinx_needs_to_md.py) — render needs as Markdown. - [sphinx_needs_to_trlc.py](sphinx_needs_to_trlc.py) — convert needs to TRLC. +- [validate_checklist.py](validate_checklist.py) — validate a checklist hash. The matching `py_binary` targets are declared in [BUILD](BUILD). +### Metamodel + +A new `req_chklst` need type is added in +[metamodel.yaml](../src/extensions/score_metamodel/metamodel.yaml). It carries a +mandatory `sha256` attribute, an optional `targets` attribute (the Bazel labels +it validates), and an optional `checklist` link to the rendered checklist +document. + ## How to use In a `BUILD` file that already has a `needs_json` target, load the macros and @@ -81,3 +94,54 @@ bazel build //path/to:my_feature_reqs_trlc You can also call `filtered_needs_json` directly for full control over the `types`, `components`, and `component_attr` filters. + +## Requirement checklists + +A *requirement checklist* couples a human review (a checklist `.rst` document) +with the exact build output that was reviewed. The state of that output is +pinned via a SHA256 hash stored on a `req_chklst` sphinx-needs element. When the +output later changes, the checklist is considered stale and the build fails +until the checklist is re-reviewed and the hash updated. + +### 1. Declare the checklist need + +Add a `req_chklst` element (e.g. next to the checklist `.rst`). It references the +checklist document, the validated Bazel target(s), and the expected hash: + +```rst +.. req_chklst:: Bitmanipulation Component Requirements Checklist + :id: req_chklst__bitmanipulation__comp_req + :status: valid + :checklist: doc__bitmanipulation_req_inspection + :targets: //:bitmanipulation_comp_reqs + :sha256: 0000000000000000000000000000000000000000000000000000000000000000 +``` + +### 2. Declare the validation target + +```starlark +load("@docs-as-code//:docs.bzl", "component_requirements", "requirements_checklist") + +component_requirements( + name = "bitmanipulation_comp_reqs", + component = "bitmanipulation", +) + +requirements_checklist( + name = "bitmanipulation_req_checklist", + checklist_id = "req_chklst__bitmanipulation__comp_req", + deps = [":bitmanipulation_comp_reqs"], +) +``` + +### 3. Validate + +```bash +bazel build //:bitmanipulation_req_checklist +``` + +The build hashes the `deps` output and compares it to the `sha256` on the +checklist need. On the first run (placeholder hash) the build **fails** and +prints the actual hash — review the checklist, then paste that hash into the +`sha256` attribute. From then on the build passes until the validated +requirements change again, at which point it fails and asks for a re-review. diff --git a/scripts_bazel/validate_checklist.py b/scripts_bazel/validate_checklist.py new file mode 100644 index 000000000..e6f8d21f7 --- /dev/null +++ b/scripts_bazel/validate_checklist.py @@ -0,0 +1,139 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +""" +Validate a requirement checklist against the build output it was reviewed against. + +A ``req_chklst`` sphinx-needs element pins the state of one or more build +outputs (e.g. the extracted component requirements) via a ``sha256`` attribute. +This script: + +1. Reads ``needs.json`` and looks up the checklist need by its id. +2. Computes the SHA256 over the concatenated input files (sorted by path, so the + result is independent of the order in which Bazel passes them). +3. Compares the computed hash with the ``sha256`` attribute of the checklist need. + +On match it writes the verified hash to ``--output`` and exits ``0``. On mismatch +(or when the need / attribute is missing) it logs the expected and actual hashes +and exits ``1``, which fails the Bazel build. +""" + +import argparse +import hashlib +import json +import logging +import sys +from pathlib import Path +from typing import Any + +logging.basicConfig(level=logging.INFO, format="%(message)s") +logger = logging.getLogger(__name__) + + +def find_need(data: dict[str, Any], need_id: str) -> dict[str, Any] | None: + """Return the need with id ``need_id`` from a needs.json structure.""" + for version in data.get("versions", {}).values(): + needs = version.get("needs", {}) + if need_id in needs: + return needs[need_id] + return None + + +def compute_sha256(paths: list[Path]) -> str: + """Return the SHA256 over the concatenated contents of ``paths`` (sorted).""" + digest = hashlib.sha256() + for path in sorted(paths, key=lambda p: p.name): + digest.update(path.read_bytes()) + return digest.hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Validate a requirement checklist (req_chklst) against the SHA256 of " + "the build output it was reviewed against." + ) + ) + _ = parser.add_argument( + "--needs-json", + required=True, + type=Path, + help="Path of the needs.json file containing the checklist need.", + ) + _ = parser.add_argument( + "--checklist-id", + required=True, + help="Id of the req_chklst need to validate (e.g. 'req_chklst__foo').", + ) + _ = parser.add_argument( + "--output", + required=True, + type=Path, + help="Path of the stamp file to write with the verified hash on success.", + ) + _ = parser.add_argument( + "inputs", + nargs="+", + type=Path, + help="Build output files whose combined SHA256 is validated.", + ) + + args = parser.parse_args() + + with open(args.needs_json) as f: + data = json.load(f) + + need = find_need(data, args.checklist_id) + if need is None: + logger.error( + "Checklist need '%s' not found in '%s'.", + args.checklist_id, + args.needs_json, + ) + return 1 + + expected = need.get("sha256") + if not expected: + logger.error( + "Checklist need '%s' has no 'sha256' attribute.", + args.checklist_id, + ) + return 1 + + actual = compute_sha256(args.inputs) + + if expected != actual: + logger.error( + "Checklist '%s' is OUT OF DATE.\n" + " expected (sha256 in need): %s\n" + " actual (build output): %s\n" + "The validated target output has changed since the checklist was " + "last reviewed. Re-review the checklist and update its 'sha256' " + "attribute to '%s'.", + args.checklist_id, + expected, + actual, + actual, + ) + return 1 + + logger.info("Checklist '%s' is up to date (sha256=%s).", args.checklist_id, actual) + + args.output.parent.mkdir(parents=True, exist_ok=True) + _ = args.output.write_text(actual + "\n") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/extensions/score_metamodel/metamodel.yaml b/src/extensions/score_metamodel/metamodel.yaml index 42d2c0a73..431cff832 100644 --- a/src/extensions/score_metamodel/metamodel.yaml +++ b/src/extensions/score_metamodel/metamodel.yaml @@ -402,6 +402,30 @@ needs_types: - requirement_excl_process parts: 3 + # Requirement Checklist + # A reviewable checklist element that pins the state of a set of build outputs + # (e.g. the extracted component requirements) via a SHA256 hash. It references + # the checklist document (the rendered `.rst`) and the Bazel target(s) whose + # output is validated against `sha256`. See the `requirements_checklist` + # Bazel rule in `docs.bzl`. + req_chklst: + title: Requirement Checklist + prefix: req_chklst__ + mandatory_options: + # req-Id: tool_req__docs_common_attr_status + status: ^(valid|draft|invalid)$ + # SHA256 (64 lowercase hex chars) of the concatenated target outputs that + # this checklist was reviewed against. + sha256: ^[0-9a-f]{64}$ + optional_options: + # Bazel target label(s) whose output is validated against `sha256`. + # Multiple labels may be separated by spaces or commas. + targets: ^.*$ + optional_links: + # Link to the checklist document (the rendered `.rst` checklist file). + checklist: document + parts: 3 + # - Architecture - # Architecture Element @@ -993,6 +1017,11 @@ needs_extra_links: incoming: covered by outgoing: covers + # Requirement Checklist -> checklist document (rendered .rst) + checklist: + incoming: is checklist for + outgoing: checklist document + # Architecture consists_of: incoming: forms part of From 7c107040628971988ecb6d08cbd98f0c16c18bb2 Mon Sep 17 00:00:00 2001 From: Anton Krivoborodov <63401640+antonkri@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:13:38 +0000 Subject: [PATCH 3/3] Add architecture extraction & checklist rules, empty-SHA reporting - Add feature_architecture / component_architecture macros (no static/dynamic split) - Add architecture_checklist macro and arch_chklst need type - Report computed SHA256 on empty sha256 attribute in validate_checklist - Make req_chklst/arch_chklst sha256 optional so empty values build - Document new macros and need type --- docs.bzl | 128 +++++++++++++ scripts_bazel/README_needs_rules.md | 177 ++++++++++++++++++ scripts_bazel/validate_checklist.py | 12 +- src/extensions/score_metamodel/metamodel.yaml | 32 +++- 4 files changed, 344 insertions(+), 5 deletions(-) diff --git a/docs.bzl b/docs.bzl index 0388f7748..d5a1e40c9 100644 --- a/docs.bzl +++ b/docs.bzl @@ -234,6 +234,66 @@ def assumptions_of_use( visibility = visibility, ) +def feature_architecture( + name, + src = "//:needs_json", + feature = None, + visibility = None): + """Extract the feature architecture from a needs.json file. + + Convenience wrapper around `filtered_needs_json`. Produces a `.json` + file containing the feature architecture elements. Static (`feat_arc_sta`) + and dynamic (`feat_arc_dyn`) architecture are not differentiated; both are + kept. + + Args: + name: Name of the generated target. The output file is `.json`. + src: Label of a `needs_json` build output. Defaults to the calling + package's `//:needs_json`. + feature: Optional feature name. If given, only feature architecture + elements tagged with that feature are kept; if omitted, all feature + architecture elements are kept. + visibility: Standard Bazel visibility for the generated target. + """ + filtered_needs_json( + name = name, + src = src, + types = ["feat_arc_sta", "feat_arc_dyn"], + components = [feature] if feature else [], + component_attr = "tags", + visibility = visibility, + ) + +def component_architecture( + name, + src = "//:needs_json", + component = None, + visibility = None): + """Extract the component architecture from a needs.json file. + + Convenience wrapper around `filtered_needs_json`. Produces a `.json` + file containing the component architecture elements. Static (`comp_arc_sta`) + and dynamic (`comp_arc_dyn`) architecture are not differentiated; both are + kept. + + Args: + name: Name of the generated target. The output file is `.json`. + src: Label of a `needs_json` build output. Defaults to the calling + package's `//:needs_json`. + component: Optional component name. If given, only component architecture + elements tagged with that component are kept; if omitted, all + component architecture elements are kept. + visibility: Standard Bazel visibility for the generated target. + """ + filtered_needs_json( + name = name, + src = src, + types = ["comp_arc_sta", "comp_arc_dyn"], + components = [component] if component else [], + component_attr = "tags", + visibility = visibility, + ) + def sphinx_needs_to_md( name, src, @@ -388,6 +448,74 @@ def requirements_checklist( visibility = visibility, ) +def architecture_checklist( + name, + checklist_id, + deps, + src = "//:needs_json", + visibility = None): + """Validate an architecture checklist (`arch_chklst`) against its build output. + + Building this target recomputes the SHA256 over the concatenated outputs of + `deps` and compares it to the `sha256` attribute of the `arch_chklst` need + `checklist_id` (looked up in `src`'s `needs.json`). The build **fails** when + the hashes differ, i.e. when a validated target output has changed since the + checklist was last reviewed. + + Typical usage validates the extracted architecture of a component against the + checklist that reviewed it: + + component_architecture( + name = "bitmanipulation_comp_arch", + component = "bitmanipulation", + ) + + architecture_checklist( + name = "bitmanipulation_arch_checklist", + checklist_id = "arch_chklst__bitmanipulation__comp_arc", + deps = [":bitmanipulation_comp_arch"], + ) + + Run with `bazel build //:bitmanipulation_arch_checklist`. On the first run (or + after the architecture changes) the build fails and prints the actual SHA256; + copy it into the `sha256` attribute of the checklist need once the checklist + has been (re-)reviewed. + + Args: + name: Name of the generated target. The output file is `.sha256`. + checklist_id: Id of the `arch_chklst` need to validate + (e.g. `"arch_chklst__bitmanipulation__comp_arc"`). + deps: List of labels whose outputs are hashed and validated. Usually a + single `feature_architecture`/`component_architecture`/ + `filtered_needs_json` target. + src: Label of a `needs_json` build output containing the checklist need. + Defaults to the calling package's `//:needs_json`. + visibility: Standard Bazel visibility for the generated target. + """ + validate_tool = Label("//scripts_bazel:validate_checklist") + + dep_args = " ".join(["$(locations %s)" % d for d in deps]) + + native.genrule( + name = name, + srcs = [src] + deps, + outs = [name + ".sha256"], + cmd = """ + $(location {validate_tool}) \ + --needs-json $(location {src})/needs.json \ + --checklist-id '{checklist_id}' \ + --output $@ \ + {dep_args} + """.format( + validate_tool = validate_tool, + checklist_id = checklist_id, + src = src, + dep_args = dep_args, + ), + tools = [validate_tool], + visibility = visibility, + ) + def _missing_requirements(deps): """Add Python hub dependencies if they are missing.""" found = [] diff --git a/scripts_bazel/README_needs_rules.md b/scripts_bazel/README_needs_rules.md index 9abaf9a1d..2e55c86c8 100644 --- a/scripts_bazel/README_needs_rules.md +++ b/scripts_bazel/README_needs_rules.md @@ -34,9 +34,12 @@ other consumers (review docs, TRLC-based tooling) without manual copying. | `component_requirements` | `.json` | Convenience wrapper for `comp_req` elements. | | `feature_requirements` | `.json` | Convenience wrapper for `feat_req` elements. | | `assumptions_of_use` | `.json` | Convenience wrapper for `aou_req` elements. | +| `feature_architecture` | `.json` | Convenience wrapper for `feat_arc_sta` / `feat_arc_dyn` elements. | +| `component_architecture` | `.json` | Convenience wrapper for `comp_arc_sta` / `comp_arc_dyn` elements. | | `sphinx_needs_to_md` | `.md` | Render needs as a Markdown document. | | `sphinx_needs_to_trlc` | `.trlc` | Convert S-CORE requirements to TRLC. | | `requirements_checklist` | `.sha256` | Validate a `req_chklst` need against its target output via SHA256. | +| `architecture_checklist` | `.sha256` | Validate an `arch_chklst` need against its target output via SHA256. | ### Python tools (`scripts_bazel/`) @@ -55,6 +58,10 @@ mandatory `sha256` attribute, an optional `targets` attribute (the Bazel labels it validates), and an optional `checklist` link to the rendered checklist document. +The analogous `arch_chklst` need type (validated by `architecture_checklist`) +is defined in the same file and works the same way for feature/component +architecture outputs. + ## How to use In a `BUILD` file that already has a `needs_json` target, load the macros and @@ -95,6 +102,29 @@ bazel build //path/to:my_feature_reqs_trlc You can also call `filtered_needs_json` directly for full control over the `types`, `components`, and `component_attr` filters. +## Incremental builds and caching + +Because every step is a separate Bazel target, Bazel only re-runs the work that +is actually affected by a change. Edit any `.rst` file and the documentation +build (the `needs_json` target) is re-executed, because its inputs changed. + +However, the filtering, rendering, conversion and checklist targets sit +*downstream* of `needs_json` and Bazel compares their inputs before re-running +them. If your edit does not touch a given subset — for example you change an +unrelated feature and the `component_requirements` output for a component stays +byte-for-byte identical — then that `component_requirements` target produces the +same output as before, and **every target that depends on it +(`sphinx_needs_to_md`, `sphinx_needs_to_trlc`, `requirements_checklist`, ...) is +not re-executed**. Bazel serves their previous results from cache instead. + +In practice this means: + +- Changing an `.rst` file only re-runs the doc build plus the filtered targets + whose content actually changed. +- A `requirements_checklist` (or `architecture_checklist`) only re-validates — + and can only fail — when the requirements/architecture it pins really change. + Unrelated edits elsewhere in the documentation leave it untouched. + ## Requirement checklists A *requirement checklist* couples a human review (a checklist `.rst` document) @@ -145,3 +175,150 @@ checklist need. On the first run (placeholder hash) the build **fails** and prints the actual hash — review the checklist, then paste that hash into the `sha256` attribute. From then on the build passes until the validated requirements change again, at which point it fails and asks for a re-review. + +## Architecture checklists + +An *architecture checklist* works exactly like a requirement checklist, but +pins the reviewed state of an *architecture* output (a feature or component +architecture) instead of requirements. The review (a checklist `.rst` +document) is coupled to the extracted architecture via a SHA256 hash stored on +an `arch_chklst` sphinx-needs element. When the architecture later changes, the +checklist is considered stale and the build fails until it is re-reviewed and +the hash updated. + +### 1. Declare the checklist need + +Add an `arch_chklst` element (e.g. next to the architecture `.rst`). It +references the checklist document, the validated Bazel target(s), and the +expected hash: + +```rst +.. arch_chklst:: Baselibs Feature Architecture Checklist + :id: arch_chklst__baselibs__feat_arc + :status: valid + :checklist: doc__baselibs_architecture + :targets: //:baselibs_feature_arch + :sha256: 0000000000000000000000000000000000000000000000000000000000000000 +``` + +### 2. Declare the validation target + +```starlark +load("@docs-as-code//:docs.bzl", "feature_architecture", "architecture_checklist") + +feature_architecture( + name = "baselibs_feature_arch", + feature = "baselibs", +) + +architecture_checklist( + name = "baselibs_feat_arch_checklist", + checklist_id = "arch_chklst__baselibs__feat_arc", + deps = [":baselibs_feature_arch"], +) +``` + +Use `component_architecture` instead of `feature_architecture` to validate a +single component's architecture. + +### 3. Validate + +```bash +bazel build //:baselibs_feat_arch_checklist +``` + +The build hashes the `deps` output and compares it to the `sha256` on the +checklist need. On the first run (placeholder hash) the build **fails** and +prints the actual hash — review the checklist, then paste that hash into the +`sha256` attribute. From then on the build passes until the validated +architecture changes again, at which point it fails and asks for a re-review. + +## Worked example: bitmanipulation (baselibs) + +The [baselibs](https://github.com/eclipse-score/baselibs) repository wires both +checklist kinds for its `bitmanipulation` component. The flow below is a +complete, real example that you can copy. + +### Requirements checklist + +`BUILD`: + +```starlark +component_requirements( + name = "bitmanipulation_comp_reqs", + component = "bitmanipulation", +) + +requirements_checklist( + name = "bitmanipulation_req_checklist", + checklist_id = "req_chklst__bitmanipulation__comp_req", + deps = [":bitmanipulation_comp_reqs"], +) +``` + +Checklist need (next to the requirements inspection `.rst`): + +```rst +.. req_chklst:: Bitmanipulation Component Requirements Checklist + :id: req_chklst__bitmanipulation__comp_req + :status: valid + :checklist: doc__bitmanipulation_req_inspection + :targets: //:bitmanipulation_comp_reqs + :sha256: +``` + +### Architecture checklist + +`BUILD`: + +```starlark +component_architecture( + name = "bitmanipulation_comp_arch", + component = "bitmanipulation", +) + +architecture_checklist( + name = "bitmanipulation_arch_checklist", + checklist_id = "arch_chklst__bitmanipulation__comp_arc", + deps = [":bitmanipulation_comp_arch"], +) +``` + +Checklist need (next to the architecture inspection `.rst`): + +```rst +.. arch_chklst:: Bitmanipulation Component Architecture Checklist + :id: arch_chklst__bitmanipulation__comp_arc + :status: valid + :checklist: doc__bitmanipulation_arc_inspection + :targets: //:bitmanipulation_comp_arch + :sha256: +``` + +### Gotcha: the component filter matches on `tags` + +`component_requirements` / `component_architecture` keep only needs whose +`tags` contain the requested component name (see +[filtered_needs_json](filter_needs_json.py), `--component-attr tags`). In +baselibs that tag is *not* set on each element directly — it is injected for a +whole document via `needextend`, e.g. for the requirements: + +```rst +.. needextend:: "__bitmanipulation__" in id + :+tags: baselibs, bitmanipulation +``` + +The architecture view ids do **not** contain `__bitmanipulation__` (they read +`comp_arc_sta__baselibs__bit_manipulation`), so that `needextend` does not tag +them and `component_architecture(component = "bitmanipulation")` would extract +*zero* needs. Add a matching `needextend` in the architecture document so the +views get the component tag: + +```rst +.. needextend:: docname is not None and "bitmanipulation" in docname and "architecture" in docname and type in ["comp_arc_sta", "comp_arc_dyn"] + :+tags: baselibs, bitmanipulation +``` + +After that, `bazel build //:bitmanipulation_comp_arch` keeps the architecture +view(s) and the checklist validates as expected. If a `component_*` target +unexpectedly extracts `0 needs`, check the `tags` of the elements first. diff --git a/scripts_bazel/validate_checklist.py b/scripts_bazel/validate_checklist.py index e6f8d21f7..75d7b8b80 100644 --- a/scripts_bazel/validate_checklist.py +++ b/scripts_bazel/validate_checklist.py @@ -103,15 +103,21 @@ def main() -> int: return 1 expected = need.get("sha256") + + actual = compute_sha256(args.inputs) + if not expected: logger.error( - "Checklist need '%s' has no 'sha256' attribute.", + "Checklist '%s' has an EMPTY 'sha256' attribute.\n" + "Review the target output and, if correct, pin it by setting the " + "checklist's 'sha256' attribute to:\n" + "\n" + " %s\n", args.checklist_id, + actual, ) return 1 - actual = compute_sha256(args.inputs) - if expected != actual: logger.error( "Checklist '%s' is OUT OF DATE.\n" diff --git a/src/extensions/score_metamodel/metamodel.yaml b/src/extensions/score_metamodel/metamodel.yaml index 431cff832..28e741247 100644 --- a/src/extensions/score_metamodel/metamodel.yaml +++ b/src/extensions/score_metamodel/metamodel.yaml @@ -414,10 +414,38 @@ needs_types: mandatory_options: # req-Id: tool_req__docs_common_attr_status status: ^(valid|draft|invalid)$ + optional_options: # SHA256 (64 lowercase hex chars) of the concatenated target outputs that - # this checklist was reviewed against. - sha256: ^[0-9a-f]{64}$ + # this checklist was reviewed against. May be left empty to let the + # `requirements_checklist` build fail and report the computed SHA256 to + # pin (see `validate_checklist.py`). + sha256: ^([0-9a-f]{64})?$ + # Bazel target label(s) whose output is validated against `sha256`. + # Multiple labels may be separated by spaces or commas. + targets: ^.*$ + optional_links: + # Link to the checklist document (the rendered `.rst` checklist file). + checklist: document + parts: 3 + + # Architecture Checklist + # A reviewable checklist element that pins the state of a set of build outputs + # (e.g. the extracted feature/component architecture) via a SHA256 hash. It + # references the checklist document (the rendered `.rst`) and the Bazel + # target(s) whose output is validated against `sha256`. See the + # `architecture_checklist` Bazel rule in `docs.bzl`. + arch_chklst: + title: Architecture Checklist + prefix: arch_chklst__ + mandatory_options: + # req-Id: tool_req__docs_common_attr_status + status: ^(valid|draft|invalid)$ optional_options: + # SHA256 (64 lowercase hex chars) of the concatenated target outputs that + # this checklist was reviewed against. May be left empty to let the + # `architecture_checklist` build fail and report the computed SHA256 to + # pin (see `validate_checklist.py`). + sha256: ^([0-9a-f]{64})?$ # Bazel target label(s) whose output is validated against `sha256`. # Multiple labels may be separated by spaces or commas. targets: ^.*$