From 063e357cdc83c783eccc2e6aa6c6dcfbccd7511a Mon Sep 17 00:00:00 2001 From: Frank Scholter Peres Date: Tue, 23 Jun 2026 10:00:18 +0200 Subject: [PATCH 1/5] first draft --- src/extensions/score_metamodel/metamodel.yaml | 68 +++++++++++++++++++ .../tests/test_metamodel_load.py | 35 ++++++++++ 2 files changed, 103 insertions(+) diff --git a/src/extensions/score_metamodel/metamodel.yaml b/src/extensions/score_metamodel/metamodel.yaml index 42d2c0a73..920824a03 100644 --- a/src/extensions/score_metamodel/metamodel.yaml +++ b/src/extensions/score_metamodel/metamodel.yaml @@ -919,6 +919,66 @@ needs_types: fully_verifies: ANY partially_verifies: ANY + mod_ver_report: + title: Module Verification Report + prefix: mod_vrep__ + mandatory_options: + safety: ^(QM|ASIL_B)$ + security: ^(YES|NO)$ + status: ^(valid|invalid)$ + verification_method: ^.*$ + optional_options: + requirements_coverage_percent: ^(100|[1-9]?[0-9])$ + structural_coverage_percent: ^(100|[1-9]?[0-9])$ + branch_coverage_percent: ^(100|[1-9]?[0-9])$ + verdict: ^(pass|fail|open)$ + report_version: ^.*$ + release_baseline: ^.*$ + mandatory_links: + belongs_to: mod + optional_links: + contains: ANY + evidence: ANY + covers: ANY + realizes: workproduct + tags: + - verification_report + parts: 3 + + # Formal inspection evidence modeled as a first-class artifact. + mod_insp: + title: Module Inspection Record + prefix: mod_insp__ + mandatory_options: + safety: ^(QM|ASIL_B)$ + security: ^(YES|NO)$ + status: ^(valid|invalid)$ + inspection_type: ^(requirements|architecture|implementation|traceability|safety_analysis|security_analysis|other)$ + inspection_state: ^(planned|in_review|rework_required|approved)$ + checklist_ref: ^.*$ + reviewers: ^.*$ + optional_options: + checklist_type: ^(req|arc|impl|safety|security|custom)$ + moderator: ^.*$ + approver: ^.*$ + findings_total: ^[0-9]+$ + findings_open: ^[0-9]+$ + pr_link: ^https://github\.com/[^/]+/[^/]+/pull/\d+$ + correction_issue: ^https://github\.com/[^/]+/[^/]+/issues/\d+$ + inspection_date: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + mandatory_links: + belongs_to: mod + inspects: ANY + optional_links: + contains: ANY + evidence: ANY + approved_by: role + supported_by: role + tags: + - inspection + - verification_evidence + parts: 3 + # https://eclipse-score.github.io/process_description/main/permalink.html?id=gd_temp__change_decision_record dec_rec: title: Decision Record @@ -1052,6 +1112,14 @@ needs_extra_links: partially_verifies: incoming: partially_verified_by outgoing: partially_verifies + + evidence: + incoming: evidence_for + outgoing: evidence + + inspects: + incoming: inspected_by + outgoing: inspects ############################################################## # Graph Checks # The graph checks focus on the relation of the needs and their attributes. diff --git a/src/extensions/score_metamodel/tests/test_metamodel_load.py b/src/extensions/score_metamodel/tests/test_metamodel_load.py index e8aa0daa0..c23ce8509 100644 --- a/src/extensions/score_metamodel/tests/test_metamodel_load.py +++ b/src/extensions/score_metamodel/tests/test_metamodel_load.py @@ -10,6 +10,7 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* +import json from pathlib import Path from unittest.mock import mock_open, patch @@ -92,3 +93,37 @@ def test_load_metamodel_data(): assert defined_graph_check["check"] == { "link1": "opt1 == test", } + + +def test_default_metamodel_contains_generic_verification_and_inspection_types(): + """Default metamodel contains generic module verification and inspection types.""" + result = load_metamodel_data() + + needs_types = {need_type["directive"]: need_type for need_type in result.needs_types} + + assert "mod_ver_report" in needs_types + assert "mod_insp" in needs_types + + mod_ver_report = needs_types["mod_ver_report"] + assert mod_ver_report["mandatory_links_str"]["belongs_to"] == "mod" + assert mod_ver_report["optional_links_str"]["contains"] == "ANY" + assert mod_ver_report["optional_links_str"]["evidence"] == "ANY" + assert mod_ver_report["optional_links_str"]["covers"] == "ANY" + + mod_insp = needs_types["mod_insp"] + assert mod_insp["mandatory_links_str"]["inspects"] == "ANY" + assert mod_insp["optional_links_str"]["contains"] == "ANY" + assert mod_insp["optional_links_str"]["evidence"] == "ANY" + + assert "evidence" in result.needs_links + assert "inspects" in result.needs_links + + +def test_metamodel_schema_json_is_valid(): + """The metamodel JSON schema file must be syntactically valid JSON.""" + schema_path = Path(__file__).resolve().parent.parent / "metamodel-schema.json" + with open(schema_path, encoding="utf-8") as schema_file: + parsed = json.load(schema_file) + + assert isinstance(parsed, dict) + assert "$schema" in parsed From e9c48ce8364dc33a8412f858b0a693ba8155c032 Mon Sep 17 00:00:00 2001 From: Frank Scholter Peres Date: Wed, 24 Jun 2026 08:57:28 +0000 Subject: [PATCH 2/5] Addded needs to requirements and added test --- docs/internals/requirements/requirements.rst | 42 +++++++ src/extensions/score_metamodel/metamodel.yaml | 14 +++ .../test_options_verification_evidence.rst | 113 ++++++++++++++++++ 3 files changed, 169 insertions(+) create mode 100644 src/extensions/score_metamodel/tests/rst/options/test_options_verification_evidence.rst diff --git a/docs/internals/requirements/requirements.rst b/docs/internals/requirements/requirements.rst index edfa719a4..bb47d4550 100644 --- a/docs/internals/requirements/requirements.rst +++ b/docs/internals/requirements/requirements.rst @@ -46,6 +46,7 @@ This section provides an overview of current process requirements and their clar Req, 'tool_req__docs' in id and implemented == "YES" and "Requirements" in tags and status == "valid", 'tool_req__docs' in id and implemented == "PARTIAL" and "Requirements" in tags and status == "valid", 'tool_req__docs' in id and implemented == "NO" and "Requirements" in tags and status == "valid", 'tool_req__docs' in id and "Requirements" in tags and status != "valid" Arch, 'tool_req__docs' in id and implemented == "YES" and "Architecture" in tags and status == "valid", 'tool_req__docs' in id and implemented == "PARTIAL" and "Architecture" in tags and status == "valid", 'tool_req__docs' in id and implemented == "NO" and "Architecture" in tags and status == "valid", 'tool_req__docs' in id and "Architecture" in tags and status != "valid" DDesign, 'tool_req__docs' in id and implemented == "YES" and "Detailed Design & Code" in tags and status == "valid", 'tool_req__docs' in id and implemented == "PARTIAL" and "Detailed Design & Code" in tags and status == "valid", 'tool_req__docs' in id and implemented == "NO" and "Detailed Design & Code" in tags and status == "valid", 'tool_req__docs' in id and "Detailed Design & Code" in tags and status != "valid" + Verif, 'tool_req__docs' in id and implemented == "YES" and "Verification Evidence" in tags and status == "valid", 'tool_req__docs' in id and implemented == "PARTIAL" and "Verification Evidence" in tags and status == "valid", 'tool_req__docs' in id and implemented == "NO" and "Verification Evidence" in tags and status == "valid", 'tool_req__docs' in id and "Verification Evidence" in tags and status != "valid" TVR, 'tool_req__docs' in id and implemented == "YES" and "Tool Verification Reports" in tags and status == "valid", 'tool_req__docs' in id and implemented == "PARTIAL" and "Tool Verification Reports" in tags and status == "valid", 'tool_req__docs' in id and implemented == "NO" and "Tool Verification Reports" in tags and status == "valid", 'tool_req__docs' in id and "Tool Verification Reports" in tags and status != "valid" Other, 'tool_req__docs' in id and implemented == "YES" and "Process / Other" in tags and status == "valid", 'tool_req__docs' in id and implemented == "PARTIAL" and "Process / Other" in tags and status == "valid", 'tool_req__docs' in id and implemented == "NO" and "Process / Other" in tags and status == "valid", 'tool_req__docs' in id and "Process / Other" in tags and status != "valid" SftyAn, 'tool_req__docs' in id and implemented == "YES" and "Safety Analysis" in tags and status == "valid", 'tool_req__docs' in id and implemented == "PARTIAL" and "Safety Analysis" in tags and status == "valid", 'tool_req__docs' in id and implemented == "NO" and "Safety Analysis" in tags and status == "valid", 'tool_req__docs' in id and "Safety Analysis" in tags and status != "valid" @@ -864,6 +865,47 @@ Testing Docs-AS-Code shall provide a way to gather statistics on linkages to implementation(source_code_links) & tests(testlink) for all needs. It shall also be possible to filter these by type and use the provided statistics in the documentation (via diagrams drawn from it etc.) +๐Ÿ”Ž Verification Evidence +######################## + +.. tool_req:: Support machine-readable module verification reports + :id: tool_req__docs_verification_report_need + :tags: Verification Evidence + :implemented: YES + :version: 1 + :satisfies: gd_req__verification_reporting + :parent_covered: NO: process wording is broader than the currently modeled report artifact. + + Docs-as-Code shall support a machine-readable module verification report need type. + + The need type shall: + + * use ``mod_ver_report`` as directive type + * classify the report by ``safety``, ``security``, ``status`` and ``verification_method`` + * link the report to the verified module via ``belongs_to`` + * allow links to contained verification evidence via ``contains`` + * allow links to covered artifacts via ``covers`` + * allow links to backing documents or work products via ``evidence`` and ``realizes`` + +.. tool_req:: Support machine-readable inspection records + :id: tool_req__docs_inspection_record_need + :tags: Verification Evidence + :implemented: YES + :version: 1 + :satisfies: gd_req__verification_checks + :parent_covered: NO: process wording defines verification checks, while the tool models a first-class inspection record artifact. + + Docs-as-Code shall support a machine-readable inspection record need type. + + The need type shall: + + * use ``mod_insp`` as directive type + * classify the inspection by ``inspection_type`` and ``inspection_state`` + * record the checklist reference and reviewer list via ``checklist_ref`` and ``reviewers`` + * link the inspection to the verified module via ``belongs_to`` + * link the inspected artifacts via ``inspects`` + * allow links to backing evidence via ``evidence`` + ๐Ÿงช Tool Verification Reports ############################ diff --git a/src/extensions/score_metamodel/metamodel.yaml b/src/extensions/score_metamodel/metamodel.yaml index 920824a03..427ed52b4 100644 --- a/src/extensions/score_metamodel/metamodel.yaml +++ b/src/extensions/score_metamodel/metamodel.yaml @@ -919,13 +919,18 @@ needs_types: fully_verifies: ANY partially_verifies: ANY + # req-Id: tool_req__docs_verification_report_need mod_ver_report: title: Module Verification Report prefix: mod_vrep__ mandatory_options: + # req-Id: tool_req__docs_common_attr_safety safety: ^(QM|ASIL_B)$ + # req-Id: tool_req__docs_common_attr_security security: ^(YES|NO)$ + # req-Id: tool_req__docs_common_attr_status status: ^(valid|invalid)$ + # req-Id: tool_req__docs_verification_report_need verification_method: ^.*$ optional_options: requirements_coverage_percent: ^(100|[1-9]?[0-9])$ @@ -935,8 +940,10 @@ needs_types: report_version: ^.*$ release_baseline: ^.*$ mandatory_links: + # req-Id: tool_req__docs_verification_report_need belongs_to: mod optional_links: + # req-Id: tool_req__docs_verification_report_need contains: ANY evidence: ANY covers: ANY @@ -946,13 +953,18 @@ needs_types: parts: 3 # Formal inspection evidence modeled as a first-class artifact. + # req-Id: tool_req__docs_inspection_record_need mod_insp: title: Module Inspection Record prefix: mod_insp__ mandatory_options: + # req-Id: tool_req__docs_common_attr_safety safety: ^(QM|ASIL_B)$ + # req-Id: tool_req__docs_common_attr_security security: ^(YES|NO)$ + # req-Id: tool_req__docs_common_attr_status status: ^(valid|invalid)$ + # req-Id: tool_req__docs_inspection_record_need inspection_type: ^(requirements|architecture|implementation|traceability|safety_analysis|security_analysis|other)$ inspection_state: ^(planned|in_review|rework_required|approved)$ checklist_ref: ^.*$ @@ -967,9 +979,11 @@ needs_types: correction_issue: ^https://github\.com/[^/]+/[^/]+/issues/\d+$ inspection_date: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ mandatory_links: + # req-Id: tool_req__docs_inspection_record_need belongs_to: mod inspects: ANY optional_links: + # req-Id: tool_req__docs_inspection_record_need contains: ANY evidence: ANY approved_by: role diff --git a/src/extensions/score_metamodel/tests/rst/options/test_options_verification_evidence.rst b/src/extensions/score_metamodel/tests/rst/options/test_options_verification_evidence.rst new file mode 100644 index 000000000..14b069078 --- /dev/null +++ b/src/extensions/score_metamodel/tests/rst/options/test_options_verification_evidence.rst @@ -0,0 +1,113 @@ +.. + # ******************************************************************************* + # Copyright (c) 2026 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* +#CHECK: check_options + + +.. Base architecture and requirement objects used by verification evidence tests + +.. feat:: Verification Feature + :id: feat__verification_feature + :security: YES + :safety: ASIL_B + :status: valid + +.. comp:: Verification Component + :id: comp__verification_component + :security: YES + :safety: ASIL_B + :status: valid + :belongs_to: feat__verification_feature + +.. mod:: Verification Module + :id: mod__verification_module + :security: YES + :safety: ASIL_B + :status: valid + :includes: comp__verification_component + +.. comp_req:: Verification Requirement + :id: comp_req__verification__sample + :reqtype: Functional + :security: YES + :safety: ASIL_B + :status: valid + :content: Requirement text for verification evidence tests. + + +.. Valid machine-readable verification report need +#EXPECT-NOT[+2]: does not follow pattern + +.. mod_ver_report:: Verification Report Valid + :id: mod_vrep__verification__valid + :safety: ASIL_B + :security: YES + :status: valid + :verification_method: test_and_inspection + :requirements_coverage_percent: 95 + :structural_coverage_percent: 90 + :branch_coverage_percent: 85 + :verdict: pass + :report_version: 1.0.0 + :release_baseline: main + :belongs_to: mod__verification_module + :covers: comp_req__verification__sample + + +.. Invalid verdict value in module verification report +#EXPECT[+2]: mod_vrep__verification__bad_verdict.verdict (pending): does not follow pattern + +.. mod_ver_report:: Verification Report Invalid Verdict + :id: mod_vrep__verification__bad_verdict + :safety: ASIL_B + :security: YES + :status: invalid + :verification_method: inspection + :verdict: pending + :belongs_to: mod__verification_module + + +.. Valid machine-readable inspection record need +#EXPECT-NOT[+2]: does not follow pattern + +.. mod_insp:: Inspection Record Valid + :id: mod_insp__verification__valid + :safety: ASIL_B + :security: YES + :status: valid + :inspection_type: requirements + :inspection_state: approved + :checklist_ref: gd_chklst__req_inspection + :reviewers: reviewer_a,reviewer_b + :checklist_type: req + :findings_total: 1 + :findings_open: 0 + :inspection_date: 2026-06-24 + :belongs_to: mod__verification_module + :inspects: comp_req__verification__sample + + +.. Invalid inspection_state value in module inspection record +#EXPECT[+2]: mod_insp__verification__bad_state.inspection_state (approved_late): does not follow pattern + +.. mod_insp:: Inspection Record Invalid State + :id: mod_insp__verification__bad_state + :safety: ASIL_B + :security: YES + :status: invalid + :inspection_type: architecture + :inspection_state: approved_late + :checklist_ref: gd_chklst__arch_inspection_checklist + :reviewers: reviewer_a + :belongs_to: mod__verification_module + :inspects: comp_req__verification__sample From 9155fd68d557f811246c60eeea593e2f1c7fa9d2 Mon Sep 17 00:00:00 2001 From: Frank Scholter Peres Date: Wed, 24 Jun 2026 09:02:34 +0000 Subject: [PATCH 3/5] fixed lint --- src/extensions/score_metamodel/tests/test_metamodel_load.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/extensions/score_metamodel/tests/test_metamodel_load.py b/src/extensions/score_metamodel/tests/test_metamodel_load.py index c23ce8509..77243eba1 100644 --- a/src/extensions/score_metamodel/tests/test_metamodel_load.py +++ b/src/extensions/score_metamodel/tests/test_metamodel_load.py @@ -99,7 +99,9 @@ def test_default_metamodel_contains_generic_verification_and_inspection_types(): """Default metamodel contains generic module verification and inspection types.""" result = load_metamodel_data() - needs_types = {need_type["directive"]: need_type for need_type in result.needs_types} + needs_types = { + need_type["directive"]: need_type for need_type in result.needs_types + } assert "mod_ver_report" in needs_types assert "mod_insp" in needs_types From 60b3493c7b3a5be5ecab37b1750d67e975d68989 Mon Sep 17 00:00:00 2001 From: Frank Scholter Peres Date: Thu, 25 Jun 2026 07:03:48 +0000 Subject: [PATCH 4/5] removed overlapping test --- .../tests/test_metamodel_load.py | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/src/extensions/score_metamodel/tests/test_metamodel_load.py b/src/extensions/score_metamodel/tests/test_metamodel_load.py index 77243eba1..b5c7bfeb8 100644 --- a/src/extensions/score_metamodel/tests/test_metamodel_load.py +++ b/src/extensions/score_metamodel/tests/test_metamodel_load.py @@ -94,33 +94,6 @@ def test_load_metamodel_data(): "link1": "opt1 == test", } - -def test_default_metamodel_contains_generic_verification_and_inspection_types(): - """Default metamodel contains generic module verification and inspection types.""" - result = load_metamodel_data() - - needs_types = { - need_type["directive"]: need_type for need_type in result.needs_types - } - - assert "mod_ver_report" in needs_types - assert "mod_insp" in needs_types - - mod_ver_report = needs_types["mod_ver_report"] - assert mod_ver_report["mandatory_links_str"]["belongs_to"] == "mod" - assert mod_ver_report["optional_links_str"]["contains"] == "ANY" - assert mod_ver_report["optional_links_str"]["evidence"] == "ANY" - assert mod_ver_report["optional_links_str"]["covers"] == "ANY" - - mod_insp = needs_types["mod_insp"] - assert mod_insp["mandatory_links_str"]["inspects"] == "ANY" - assert mod_insp["optional_links_str"]["contains"] == "ANY" - assert mod_insp["optional_links_str"]["evidence"] == "ANY" - - assert "evidence" in result.needs_links - assert "inspects" in result.needs_links - - def test_metamodel_schema_json_is_valid(): """The metamodel JSON schema file must be syntactically valid JSON.""" schema_path = Path(__file__).resolve().parent.parent / "metamodel-schema.json" From 72bdbf9717c5460155024c0a07ce9c0f0d8e50fc Mon Sep 17 00:00:00 2001 From: Frank Scholter Peres Date: Thu, 25 Jun 2026 07:09:00 +0000 Subject: [PATCH 5/5] fix formatting --- src/extensions/score_metamodel/tests/test_metamodel_load.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/extensions/score_metamodel/tests/test_metamodel_load.py b/src/extensions/score_metamodel/tests/test_metamodel_load.py index b5c7bfeb8..f70bf055d 100644 --- a/src/extensions/score_metamodel/tests/test_metamodel_load.py +++ b/src/extensions/score_metamodel/tests/test_metamodel_load.py @@ -94,6 +94,7 @@ def test_load_metamodel_data(): "link1": "opt1 == test", } + def test_metamodel_schema_json_is_valid(): """The metamodel JSON schema file must be syntactically valid JSON.""" schema_path = Path(__file__).resolve().parent.parent / "metamodel-schema.json"