diff --git a/docs/internals/requirements/requirements.rst b/docs/internals/requirements/requirements.rst index edfa719a4..bb47d4550 100644 --- a/docs/internals/requirements/requirements.rst +++ b/docs/internals/requirements/requirements.rst @@ -46,6 +46,7 @@ This section provides an overview of current process requirements and their clar Req, 'tool_req__docs' in id and implemented == "YES" and "Requirements" in tags and status == "valid", 'tool_req__docs' in id and implemented == "PARTIAL" and "Requirements" in tags and status == "valid", 'tool_req__docs' in id and implemented == "NO" and "Requirements" in tags and status == "valid", 'tool_req__docs' in id and "Requirements" in tags and status != "valid" Arch, 'tool_req__docs' in id and implemented == "YES" and "Architecture" in tags and status == "valid", 'tool_req__docs' in id and implemented == "PARTIAL" and "Architecture" in tags and status == "valid", 'tool_req__docs' in id and implemented == "NO" and "Architecture" in tags and status == "valid", 'tool_req__docs' in id and "Architecture" in tags and status != "valid" DDesign, 'tool_req__docs' in id and implemented == "YES" and "Detailed Design & Code" in tags and status == "valid", 'tool_req__docs' in id and implemented == "PARTIAL" and "Detailed Design & Code" in tags and status == "valid", 'tool_req__docs' in id and implemented == "NO" and "Detailed Design & Code" in tags and status == "valid", 'tool_req__docs' in id and "Detailed Design & Code" in tags and status != "valid" + Verif, 'tool_req__docs' in id and implemented == "YES" and "Verification Evidence" in tags and status == "valid", 'tool_req__docs' in id and implemented == "PARTIAL" and "Verification Evidence" in tags and status == "valid", 'tool_req__docs' in id and implemented == "NO" and "Verification Evidence" in tags and status == "valid", 'tool_req__docs' in id and "Verification Evidence" in tags and status != "valid" TVR, 'tool_req__docs' in id and implemented == "YES" and "Tool Verification Reports" in tags and status == "valid", 'tool_req__docs' in id and implemented == "PARTIAL" and "Tool Verification Reports" in tags and status == "valid", 'tool_req__docs' in id and implemented == "NO" and "Tool Verification Reports" in tags and status == "valid", 'tool_req__docs' in id and "Tool Verification Reports" in tags and status != "valid" Other, 'tool_req__docs' in id and implemented == "YES" and "Process / Other" in tags and status == "valid", 'tool_req__docs' in id and implemented == "PARTIAL" and "Process / Other" in tags and status == "valid", 'tool_req__docs' in id and implemented == "NO" and "Process / Other" in tags and status == "valid", 'tool_req__docs' in id and "Process / Other" in tags and status != "valid" SftyAn, 'tool_req__docs' in id and implemented == "YES" and "Safety Analysis" in tags and status == "valid", 'tool_req__docs' in id and implemented == "PARTIAL" and "Safety Analysis" in tags and status == "valid", 'tool_req__docs' in id and implemented == "NO" and "Safety Analysis" in tags and status == "valid", 'tool_req__docs' in id and "Safety Analysis" in tags and status != "valid" @@ -864,6 +865,47 @@ Testing Docs-AS-Code shall provide a way to gather statistics on linkages to implementation(source_code_links) & tests(testlink) for all needs. It shall also be possible to filter these by type and use the provided statistics in the documentation (via diagrams drawn from it etc.) +๐Ÿ”Ž Verification Evidence +######################## + +.. tool_req:: Support machine-readable module verification reports + :id: tool_req__docs_verification_report_need + :tags: Verification Evidence + :implemented: YES + :version: 1 + :satisfies: gd_req__verification_reporting + :parent_covered: NO: process wording is broader than the currently modeled report artifact. + + Docs-as-Code shall support a machine-readable module verification report need type. + + The need type shall: + + * use ``mod_ver_report`` as directive type + * classify the report by ``safety``, ``security``, ``status`` and ``verification_method`` + * link the report to the verified module via ``belongs_to`` + * allow links to contained verification evidence via ``contains`` + * allow links to covered artifacts via ``covers`` + * allow links to backing documents or work products via ``evidence`` and ``realizes`` + +.. tool_req:: Support machine-readable inspection records + :id: tool_req__docs_inspection_record_need + :tags: Verification Evidence + :implemented: YES + :version: 1 + :satisfies: gd_req__verification_checks + :parent_covered: NO: process wording defines verification checks, while the tool models a first-class inspection record artifact. + + Docs-as-Code shall support a machine-readable inspection record need type. + + The need type shall: + + * use ``mod_insp`` as directive type + * classify the inspection by ``inspection_type`` and ``inspection_state`` + * record the checklist reference and reviewer list via ``checklist_ref`` and ``reviewers`` + * link the inspection to the verified module via ``belongs_to`` + * link the inspected artifacts via ``inspects`` + * allow links to backing evidence via ``evidence`` + ๐Ÿงช Tool Verification Reports ############################ diff --git a/src/extensions/score_metamodel/metamodel.yaml b/src/extensions/score_metamodel/metamodel.yaml index 42d2c0a73..427ed52b4 100644 --- a/src/extensions/score_metamodel/metamodel.yaml +++ b/src/extensions/score_metamodel/metamodel.yaml @@ -919,6 +919,80 @@ needs_types: fully_verifies: ANY partially_verifies: ANY + # req-Id: tool_req__docs_verification_report_need + mod_ver_report: + title: Module Verification Report + prefix: mod_vrep__ + mandatory_options: + # req-Id: tool_req__docs_common_attr_safety + safety: ^(QM|ASIL_B)$ + # req-Id: tool_req__docs_common_attr_security + security: ^(YES|NO)$ + # req-Id: tool_req__docs_common_attr_status + status: ^(valid|invalid)$ + # req-Id: tool_req__docs_verification_report_need + verification_method: ^.*$ + optional_options: + requirements_coverage_percent: ^(100|[1-9]?[0-9])$ + structural_coverage_percent: ^(100|[1-9]?[0-9])$ + branch_coverage_percent: ^(100|[1-9]?[0-9])$ + verdict: ^(pass|fail|open)$ + report_version: ^.*$ + release_baseline: ^.*$ + mandatory_links: + # req-Id: tool_req__docs_verification_report_need + belongs_to: mod + optional_links: + # req-Id: tool_req__docs_verification_report_need + contains: ANY + evidence: ANY + covers: ANY + realizes: workproduct + tags: + - verification_report + parts: 3 + + # Formal inspection evidence modeled as a first-class artifact. + # req-Id: tool_req__docs_inspection_record_need + mod_insp: + title: Module Inspection Record + prefix: mod_insp__ + mandatory_options: + # req-Id: tool_req__docs_common_attr_safety + safety: ^(QM|ASIL_B)$ + # req-Id: tool_req__docs_common_attr_security + security: ^(YES|NO)$ + # req-Id: tool_req__docs_common_attr_status + status: ^(valid|invalid)$ + # req-Id: tool_req__docs_inspection_record_need + inspection_type: ^(requirements|architecture|implementation|traceability|safety_analysis|security_analysis|other)$ + inspection_state: ^(planned|in_review|rework_required|approved)$ + checklist_ref: ^.*$ + reviewers: ^.*$ + optional_options: + checklist_type: ^(req|arc|impl|safety|security|custom)$ + moderator: ^.*$ + approver: ^.*$ + findings_total: ^[0-9]+$ + findings_open: ^[0-9]+$ + pr_link: ^https://github\.com/[^/]+/[^/]+/pull/\d+$ + correction_issue: ^https://github\.com/[^/]+/[^/]+/issues/\d+$ + inspection_date: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + mandatory_links: + # req-Id: tool_req__docs_inspection_record_need + belongs_to: mod + inspects: ANY + optional_links: + # req-Id: tool_req__docs_inspection_record_need + contains: ANY + evidence: ANY + approved_by: role + supported_by: role + tags: + - inspection + - verification_evidence + parts: 3 + # https://eclipse-score.github.io/process_description/main/permalink.html?id=gd_temp__change_decision_record dec_rec: title: Decision Record @@ -1052,6 +1126,14 @@ needs_extra_links: partially_verifies: incoming: partially_verified_by outgoing: partially_verifies + + evidence: + incoming: evidence_for + outgoing: evidence + + inspects: + incoming: inspected_by + outgoing: inspects ############################################################## # Graph Checks # The graph checks focus on the relation of the needs and their attributes. diff --git a/src/extensions/score_metamodel/tests/rst/options/test_options_verification_evidence.rst b/src/extensions/score_metamodel/tests/rst/options/test_options_verification_evidence.rst new file mode 100644 index 000000000..14b069078 --- /dev/null +++ b/src/extensions/score_metamodel/tests/rst/options/test_options_verification_evidence.rst @@ -0,0 +1,113 @@ +.. + # ******************************************************************************* + # Copyright (c) 2026 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* +#CHECK: check_options + + +.. Base architecture and requirement objects used by verification evidence tests + +.. feat:: Verification Feature + :id: feat__verification_feature + :security: YES + :safety: ASIL_B + :status: valid + +.. comp:: Verification Component + :id: comp__verification_component + :security: YES + :safety: ASIL_B + :status: valid + :belongs_to: feat__verification_feature + +.. mod:: Verification Module + :id: mod__verification_module + :security: YES + :safety: ASIL_B + :status: valid + :includes: comp__verification_component + +.. comp_req:: Verification Requirement + :id: comp_req__verification__sample + :reqtype: Functional + :security: YES + :safety: ASIL_B + :status: valid + :content: Requirement text for verification evidence tests. + + +.. Valid machine-readable verification report need +#EXPECT-NOT[+2]: does not follow pattern + +.. mod_ver_report:: Verification Report Valid + :id: mod_vrep__verification__valid + :safety: ASIL_B + :security: YES + :status: valid + :verification_method: test_and_inspection + :requirements_coverage_percent: 95 + :structural_coverage_percent: 90 + :branch_coverage_percent: 85 + :verdict: pass + :report_version: 1.0.0 + :release_baseline: main + :belongs_to: mod__verification_module + :covers: comp_req__verification__sample + + +.. Invalid verdict value in module verification report +#EXPECT[+2]: mod_vrep__verification__bad_verdict.verdict (pending): does not follow pattern + +.. mod_ver_report:: Verification Report Invalid Verdict + :id: mod_vrep__verification__bad_verdict + :safety: ASIL_B + :security: YES + :status: invalid + :verification_method: inspection + :verdict: pending + :belongs_to: mod__verification_module + + +.. Valid machine-readable inspection record need +#EXPECT-NOT[+2]: does not follow pattern + +.. mod_insp:: Inspection Record Valid + :id: mod_insp__verification__valid + :safety: ASIL_B + :security: YES + :status: valid + :inspection_type: requirements + :inspection_state: approved + :checklist_ref: gd_chklst__req_inspection + :reviewers: reviewer_a,reviewer_b + :checklist_type: req + :findings_total: 1 + :findings_open: 0 + :inspection_date: 2026-06-24 + :belongs_to: mod__verification_module + :inspects: comp_req__verification__sample + + +.. Invalid inspection_state value in module inspection record +#EXPECT[+2]: mod_insp__verification__bad_state.inspection_state (approved_late): does not follow pattern + +.. mod_insp:: Inspection Record Invalid State + :id: mod_insp__verification__bad_state + :safety: ASIL_B + :security: YES + :status: invalid + :inspection_type: architecture + :inspection_state: approved_late + :checklist_ref: gd_chklst__arch_inspection_checklist + :reviewers: reviewer_a + :belongs_to: mod__verification_module + :inspects: comp_req__verification__sample diff --git a/src/extensions/score_metamodel/tests/test_metamodel_load.py b/src/extensions/score_metamodel/tests/test_metamodel_load.py index e8aa0daa0..f70bf055d 100644 --- a/src/extensions/score_metamodel/tests/test_metamodel_load.py +++ b/src/extensions/score_metamodel/tests/test_metamodel_load.py @@ -10,6 +10,7 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* +import json from pathlib import Path from unittest.mock import mock_open, patch @@ -92,3 +93,13 @@ def test_load_metamodel_data(): assert defined_graph_check["check"] == { "link1": "opt1 == test", } + + +def test_metamodel_schema_json_is_valid(): + """The metamodel JSON schema file must be syntactically valid JSON.""" + schema_path = Path(__file__).resolve().parent.parent / "metamodel-schema.json" + with open(schema_path, encoding="utf-8") as schema_file: + parsed = json.load(schema_file) + + assert isinstance(parsed, dict) + assert "$schema" in parsed