From e9ab576adb82f0d0cea7405ea28fef534f1f279e Mon Sep 17 00:00:00 2001 From: Ron <45816308+rjaegers@users.noreply.github.com> Date: Thu, 26 Feb 2026 20:35:52 +0000 Subject: [PATCH 1/6] docs: generate additional docs --- .github/workflows/wc-document-generation.yml | 44 +++- docs/support/bats_sbdl_converter.py | 215 ++++++++++++++++++ docs/support/generate-sbdl.py | 190 ++++++++++++++++ docs/support/gherkin-to-sbdl.py | 5 +- docs/support/gherkin_mapping_config.py | 21 ++ docs/support/gherkin_sbdl_converter.py | 26 ++- .../requirements-traceability-matrix.md.j2 | 131 +++++++++++ .../software-requirements-specification.md.j2 | 14 +- .../software-test-specification.md.j2 | 119 ++++++++++ test/base/integration-tests.bats | 1 + test/cpp/integration-tests.bats | 28 ++- test/rust/integration-tests.bats | 9 + 12 files changed, 789 insertions(+), 14 deletions(-) create mode 100644 docs/support/bats_sbdl_converter.py create mode 100644 docs/support/generate-sbdl.py create mode 100644 docs/templates/requirements-traceability-matrix.md.j2 create mode 100644 docs/templates/software-test-specification.md.j2 diff --git a/.github/workflows/wc-document-generation.yml b/.github/workflows/wc-document-generation.yml index 581e94a2..bd3cad75 100644 --- a/.github/workflows/wc-document-generation.yml +++ b/.github/workflows/wc-document-generation.yml @@ -29,9 +29,51 @@ jobs: set -Eeuo pipefail python docs/support/gherkin-to-sbdl.py test/cpp/features/*.feature sbdl -m template-fill --template docs/templates/software-requirements-specification.md.j2 output.sbdl > software-requirements-specification.md + - name: Generate C++ test specification and traceability documents + run: | + set -Eeuo pipefail + python docs/support/generate-sbdl.py --config test-specification \ + --gherkin test/cpp/features/*.feature \ + --bats test/cpp/integration-tests.bats test/base/integration-tests.bats \ + --flavor "C++ (cpp)" \ + -o cpp-test-specification.sbdl + sbdl -m template-fill --template docs/templates/software-test-specification.md.j2 cpp-test-specification.sbdl > cpp-software-test-specification.md + sbdl -m template-fill --template docs/templates/requirements-traceability-matrix.md.j2 cpp-test-specification.sbdl > cpp-requirements-traceability-matrix.md + - name: Generate Rust test specification and traceability documents + run: | + set -Eeuo pipefail + python docs/support/generate-sbdl.py --config test-specification \ + --gherkin test/cpp/features/*.feature \ + --bats test/rust/integration-tests.bats test/base/integration-tests.bats \ + --flavor Rust \ + -o rust-test-specification.sbdl + sbdl -m template-fill --template docs/templates/software-test-specification.md.j2 rust-test-specification.sbdl > rust-software-test-specification.md + sbdl -m template-fill --template docs/templates/requirements-traceability-matrix.md.j2 rust-test-specification.sbdl > rust-requirements-traceability-matrix.md + - uses: docker://pandoc/extra:3.7.0@sha256:a703d335fa237f8fc3303329d87e2555dca5187930da38bfa9010fa4e690933a + with: + args: >- + --template eisvogel --listings --number-sections + --output software-requirements-specification.pdf software-requirements-specification.md + - uses: docker://pandoc/extra:3.7.0@sha256:a703d335fa237f8fc3303329d87e2555dca5187930da38bfa9010fa4e690933a + with: + args: >- + --template eisvogel --listings --number-sections + --output cpp-software-test-specification.pdf cpp-software-test-specification.md + - uses: docker://pandoc/extra:3.7.0@sha256:a703d335fa237f8fc3303329d87e2555dca5187930da38bfa9010fa4e690933a + with: + args: >- + --template eisvogel --listings --number-sections + --output cpp-requirements-traceability-matrix.pdf cpp-requirements-traceability-matrix.md + - uses: docker://pandoc/extra:3.7.0@sha256:a703d335fa237f8fc3303329d87e2555dca5187930da38bfa9010fa4e690933a + with: + args: >- + --template eisvogel --listings --number-sections + --output rust-software-test-specification.pdf rust-software-test-specification.md - uses: docker://pandoc/extra:3.7.0@sha256:a703d335fa237f8fc3303329d87e2555dca5187930da38bfa9010fa4e690933a with: - args: --template eisvogel --listings --number-sections --output software-requirements-specification.pdf software-requirements-specification.md + args: >- + --template eisvogel --listings --number-sections + --output rust-requirements-traceability-matrix.pdf rust-requirements-traceability-matrix.md - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: documents diff --git a/docs/support/bats_sbdl_converter.py b/docs/support/bats_sbdl_converter.py new file mode 100644 index 00000000..1cc1f1d4 --- /dev/null +++ b/docs/support/bats_sbdl_converter.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""BATS test file to SBDL converter. + +Parses BATS test files and extracts @test definitions along with their +optional `# bats test_tags=` annotations, producing SBDL `test` elements +with traceability to requirements via tag-based mapping. +""" + +import re +from dataclasses import dataclass, field +from typing import Dict, List, Optional + +try: + import sbdl + + def sanitize_identifier(name: str) -> str: + """Convert a name to a hyphenated slug identifier.""" + slug = re.sub(r'[^a-z0-9]+', '-', name.lower()) + return slug.strip('-') + + def sanitize_description(text: str) -> str: + return sbdl.SBDL_Parser.sanitize(text) + +except ImportError: + def sanitize_identifier(name: str) -> str: + """Convert a name to a hyphenated slug identifier.""" + slug = re.sub(r'[^a-z0-9]+', '-', name.lower()) + return slug.strip('-') + + def sanitize_description(text: str) -> str: + return text.replace('"', '\\"') + + +@dataclass +class BatsTest: + """Represents a single BATS test case.""" + + name: str + identifier: str + tags: List[str] = field(default_factory=list) + file_path: str = "" + line_number: int = 0 + + def __post_init__(self): + if not self.identifier: + self.identifier = sanitize_identifier(self.name) + + +class BatsConverter: + """Converts BATS test files to SBDL test elements.""" + + # Regex patterns for BATS file parsing + _TAG_PATTERN = re.compile(r"^#\s*bats\s+test_tags\s*=\s*(.+)$", re.IGNORECASE) + _TEST_PATTERN = re.compile(r'^@test\s+"(.+?)"\s*\{', re.MULTILINE) + + def __init__(self, requirement_tag_map: Optional[Dict[str, str]] = None): + """Initialize the converter. + + Args: + requirement_tag_map: Optional mapping from tag names to requirement + identifiers (SBDL element IDs). When provided, tests with + matching tags will get a `requirement` relation in SBDL output. + """ + self.requirement_tag_map = requirement_tag_map or {} + + @staticmethod + def _extract_flavor(file_path: str) -> str: + """Extract the flavor prefix from a BATS file path. + + Derives a prefix from the parent directory name of the BATS file, + e.g. 'test/cpp/integration-tests.bats' -> 'cpp'. + + Args: + file_path: Path to the BATS file. + + Returns: + Flavor string derived from the parent directory. + """ + import os + parent = os.path.basename(os.path.dirname(os.path.abspath(file_path))) + return sanitize_identifier(parent) if parent else "" + + def extract_from_bats_file(self, file_path: str) -> List[BatsTest]: + """Extract all test definitions from a BATS file. + + Parses `@test "..."` blocks and any preceding `# bats test_tags=...` + comment lines. + + Args: + file_path: Path to the .bats file. + + Returns: + List of BatsTest objects. + """ + try: + with open(file_path, "r", encoding="utf-8") as f: + lines = f.readlines() + except (IOError, OSError) as e: + print(f"Error reading {file_path}: {e}") + return [] + + tests: List[BatsTest] = [] + pending_tags: List[str] = [] + + for line_number, line in enumerate(lines, start=1): + stripped = line.strip() + + # Check for tag annotation + tag_match = self._TAG_PATTERN.match(stripped) + if tag_match: + tags_str = tag_match.group(1) + pending_tags = [t.strip() for t in tags_str.split(",") if t.strip()] + continue + + # Check for test definition + test_match = self._TEST_PATTERN.match(stripped) + if test_match: + test_name = test_match.group(1) + flavor = self._extract_flavor(file_path) + base_id = sanitize_identifier(test_name) + identifier = f"{flavor}-{base_id}" if flavor else base_id + test = BatsTest( + name=test_name, + identifier=identifier, + tags=list(pending_tags), + file_path=file_path, + line_number=line_number, + ) + tests.append(test) + pending_tags = [] + print(f" Extracted BATS test: {test.identifier}") + continue + + # Reset pending tags if we hit a non-comment, non-empty line + # that isn't a test (so tags don't bleed across unrelated lines) + if stripped and not stripped.startswith("#"): + pending_tags = [] + + return tests + + def write_sbdl_output(self, tests: List[BatsTest], output_file: str): + """Write extracted BATS tests as SBDL test elements. + + Args: + tests: List of BatsTest objects. + output_file: Path to write the SBDL output. + """ + with open(output_file, "w", encoding="utf-8") as f: + f.write("#!sbdl\n") + + for test in tests: + escaped_desc = sanitize_description(test.name) + identifier = test.identifier + + f.write(f'{identifier} is test {{ description is "{escaped_desc}" custom:title is "{escaped_desc}"') + + # Add tag property if tags exist + if test.tags: + tag_str = ",".join(test.tags) + f.write(f" tag is {tag_str}") + + # Add requirement relation via tag mapping + req_ids = self._resolve_requirement_relations(test) + if req_ids: + f.write(f" requirement is {','.join(req_ids)}") + + f.write(" }\n") + + def _resolve_requirement_relations(self, test: BatsTest) -> List[str]: + """Resolve requirement identifiers from test tags using the tag map. + + Args: + test: A BatsTest with tags. + + Returns: + List of requirement SBDL identifiers. + """ + req_ids = [] + for tag in test.tags: + tag_lower = tag.lower() + if tag_lower in self.requirement_tag_map: + entry = self.requirement_tag_map[tag_lower] + # Support both plain string and (identifier, type) tuple entries + req_id = entry[0] if isinstance(entry, tuple) else entry + if req_id not in req_ids: + req_ids.append(req_id) + return req_ids + + def _resolve_typed_relations(self, test: BatsTest) -> Dict[str, List[str]]: + """Resolve typed relations from test tags using the tag map. + + Groups resolved identifiers by their SBDL element type, so the + correct relation keyword (e.g. 'aspect', 'requirement') is used + in the SBDL output. + + Args: + test: A BatsTest with tags. + + Returns: + Dict mapping SBDL type names to lists of identifiers. + """ + relations: Dict[str, List[str]] = {} + for tag in test.tags: + tag_lower = tag.lower() + if tag_lower in self.requirement_tag_map: + entry = self.requirement_tag_map[tag_lower] + if isinstance(entry, tuple): + identifier, elem_type = entry + else: + identifier, elem_type = entry, "requirement" + if elem_type not in relations: + relations[elem_type] = [] + if identifier not in relations[elem_type]: + relations[elem_type].append(identifier) + return relations diff --git a/docs/support/generate-sbdl.py b/docs/support/generate-sbdl.py new file mode 100644 index 00000000..539d19cc --- /dev/null +++ b/docs/support/generate-sbdl.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +"""Unified converter for generating SBDL from Gherkin feature files and BATS test files. + +Supports multiple output configurations: + - requirements: Feature/Rule hierarchy as requirements (for SRS) + - test-specification: Feature→aspect, Rule→requirement, Scenario→test + BATS→test (for test spec & traceability) + +Usage examples: + # Generate requirements SBDL (existing behavior) + python generate-sbdl.py --config requirements --gherkin test/cpp/features/*.feature + + # Generate test specification SBDL with both Gherkin scenarios and BATS tests + python generate-sbdl.py --config test-specification \\ + --gherkin test/cpp/features/*.feature \\ + --bats test/cpp/integration-tests.bats test/base/integration-tests.bats +""" + +import argparse +import os +import re +import sys + +from gherkin_mapping_config import FEATURE_RULE_CONFIG, TEST_SPECIFICATION_CONFIG +from gherkin_sbdl_converter import GherkinConverter +from bats_sbdl_converter import BatsConverter + + +def main(): + configs = { + "requirements": FEATURE_RULE_CONFIG, + "test-specification": TEST_SPECIFICATION_CONFIG, + } + + parser = argparse.ArgumentParser( + description="Unified Gherkin + BATS to SBDL converter" + ) + parser.add_argument( + "--gherkin", + nargs="*", + default=[], + help="Paths to Gherkin feature files", + ) + parser.add_argument( + "--bats", + nargs="*", + default=[], + help="Paths to BATS test files", + ) + parser.add_argument( + "--output", + "-o", + default="output.sbdl", + help="Output SBDL file", + ) + parser.add_argument( + "--config", + choices=configs.keys(), + default="requirements", + help="Conversion configuration preset", + ) + parser.add_argument( + "--flavor", + default="", + help="Container flavor name for per-flavor document generation", + ) + + args = parser.parse_args() + + if not args.gherkin and not args.bats: + parser.error("At least one --gherkin or --bats file must be specified") + + config = configs[args.config] + gherkin_converter = GherkinConverter(config) + gherkin_elements = [] + + # Process Gherkin feature files + for feature_path in args.gherkin: + if os.path.isfile(feature_path): + print(f"Processing Gherkin: {feature_path}") + elements = gherkin_converter.extract_from_feature_file(feature_path) + gherkin_elements.extend(elements) + else: + print(f"File not found: {feature_path}", file=sys.stderr) + + # Build requirement tag map from Gherkin elements for BATS traceability. + # Maps lowercased sanitized identifiers and feature names to (identifier, element_type) + # tuples, enabling BATS tests tagged with e.g. "Compatibility" to trace to the + # corresponding Gherkin element with the correct SBDL relation type. + requirement_tag_map = {} + for elem in gherkin_elements: + if elem.metadata and elem.metadata.get("gherkin_type") in ("feature", "rule"): + entry = (elem.identifier, elem.element_type.value) + original = elem.metadata.get("original_name", "") + # Map the original feature/rule name (lowercased) to the SBDL identifier + type + if original: + requirement_tag_map[original.lower()] = entry + # Also map the slugified version for BATS tag matching + slug_original = re.sub(r'[^a-z0-9]+', '-', original.lower()).strip('-') + if slug_original: + requirement_tag_map[slug_original] = entry + # Also map the SBDL identifier itself + requirement_tag_map[elem.identifier.lower()] = entry + + # Process BATS test files + bats_converter = BatsConverter(requirement_tag_map=requirement_tag_map) + bats_tests = [] + + for bats_path in args.bats: + if os.path.isfile(bats_path): + print(f"Processing BATS: {bats_path}") + tests = bats_converter.extract_from_bats_file(bats_path) + bats_tests.extend(tests) + else: + print(f"File not found: {bats_path}", file=sys.stderr) + + # Write combined SBDL output + _write_combined_sbdl(gherkin_converter, gherkin_elements, bats_converter, bats_tests, args.output, args.flavor) + + total = len(gherkin_elements) + len(bats_tests) + print(f"Extracted {total} elements ({len(gherkin_elements)} from Gherkin, {len(bats_tests)} from BATS) to {args.output}") + + +def _write_combined_sbdl(gherkin_converter, gherkin_elements, bats_converter, bats_tests, output_file, flavor=""): + """Write a single combined SBDL file from both Gherkin and BATS sources.""" + import sbdl as sbdl_lib + + tokens = sbdl_lib.SBDL_Parser.Tokens + attrs = sbdl_lib.SBDL_Parser.Attributes + + # Build element type lookup for cross-type relation handling + element_types = {e.identifier: e.element_type for e in gherkin_elements} + + with open(output_file, "w", encoding="utf-8") as f: + f.write("#!sbdl\n\n") + + # Write flavor marker element for per-flavor document generation + if flavor: + escaped_flavor = sbdl_lib.SBDL_Parser.sanitize(flavor) + f.write(f'document-flavor is definition {{ description is "{escaped_flavor}" custom:title is "{escaped_flavor}" }}\n\n') + + f.write("# Elements extracted from Gherkin feature files\n") + + for element in gherkin_elements: + escaped_desc = sbdl_lib.SBDL_Parser.sanitize(element.description) + sbdl_type = element.element_type.value + f.write(f"{element.identifier} {tokens.declaration} {sbdl_type} ") + f.write(f"{tokens.declaration_group_delimeters[0]} ") + f.write(f"{attrs.description}{tokens.declaration_attribute_assign}") + f.write( + f"{tokens.declaration_attribute_delimeter}{escaped_desc}{tokens.declaration_attribute_delimeter} " + ) + + # Write custom:title with original name for display purposes + original_name = element.metadata.get('original_name', '') if element.metadata else '' + if original_name: + escaped_title = sbdl_lib.SBDL_Parser.sanitize(original_name) + f.write(f'custom:title{tokens.declaration_attribute_assign}') + f.write(f'{tokens.declaration_attribute_delimeter}{escaped_title}{tokens.declaration_attribute_delimeter} ') + + if element.parent: + parent_type = element_types.get(element.parent) + if parent_type and parent_type != element.element_type: + f.write(f"{parent_type.value}{tokens.declaration_attribute_assign}{element.parent} ") + else: + f.write(f"{attrs.parent}{tokens.declaration_attribute_assign}{element.parent} ") + + f.write(f"{tokens.declaration_group_delimeters[1]}\n") + + if bats_tests: + f.write("\n# Elements extracted from BATS integration test files\n") + + for test in bats_tests: + escaped_desc = sbdl_lib.SBDL_Parser.sanitize(test.name) + identifier = test.identifier + + f.write(f'{identifier} is test {{ description is "{escaped_desc}" custom:title is "{escaped_desc}"') + + if test.tags: + tag_str = ",".join(test.tags) + f.write(f" tag is {tag_str}") + + resolved = bats_converter._resolve_typed_relations(test) + for rel_type, rel_ids in resolved.items(): + f.write(f" {rel_type} is {','.join(rel_ids)}") + + f.write(" }\n") + + +if __name__ == "__main__": + main() diff --git a/docs/support/gherkin-to-sbdl.py b/docs/support/gherkin-to-sbdl.py index 8c6d505e..c68a4ae0 100755 --- a/docs/support/gherkin-to-sbdl.py +++ b/docs/support/gherkin-to-sbdl.py @@ -4,13 +4,14 @@ import os import sys -from gherkin_mapping_config import FEATURE_RULE_CONFIG, FEATURE_RULE_SCENARIO_CONFIG +from gherkin_mapping_config import FEATURE_RULE_CONFIG, FEATURE_RULE_SCENARIO_CONFIG, TEST_SPECIFICATION_CONFIG from gherkin_sbdl_converter import GherkinConverter def main(): configs = { 'feature-rule': FEATURE_RULE_CONFIG, - 'feature-rule-scenario': FEATURE_RULE_SCENARIO_CONFIG + 'feature-rule-scenario': FEATURE_RULE_SCENARIO_CONFIG, + 'test-specification': TEST_SPECIFICATION_CONFIG } parser = argparse.ArgumentParser(description='Configurable Gherkin to SBDL converter') diff --git a/docs/support/gherkin_mapping_config.py b/docs/support/gherkin_mapping_config.py index 113e5e59..b7517f5d 100644 --- a/docs/support/gherkin_mapping_config.py +++ b/docs/support/gherkin_mapping_config.py @@ -91,6 +91,27 @@ def get_mapping_for_type(self, gherkin_type: GherkinElementType) -> Optional[Hie ] ) +# Test Specification Configuration (Feature -> Rule -> Scenario with tests) +# Used for generating test specification and traceability documents. +# Features become aspects (system areas), Rules become requirements, +# and Scenarios become test elements traced to their parent requirements. +TEST_SPECIFICATION_CONFIG = ConversionConfig( + hierarchy_mappings=[ + HierarchyMapping( + gherkin_type=GherkinElementType.FEATURE, + sbdl_type=SBDLElementType.ASPECT + ), + HierarchyMapping( + gherkin_type=GherkinElementType.RULE, + sbdl_type=SBDLElementType.REQUIREMENT + ), + HierarchyMapping( + gherkin_type=GherkinElementType.SCENARIO, + sbdl_type=SBDLElementType.TEST + ) + ] +) + # Use Case Focused Configuration USECASE_CONFIG = ConversionConfig( hierarchy_mappings=[ diff --git a/docs/support/gherkin_sbdl_converter.py b/docs/support/gherkin_sbdl_converter.py index 888a8c6d..25e05a64 100644 --- a/docs/support/gherkin_sbdl_converter.py +++ b/docs/support/gherkin_sbdl_converter.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from typing import List, Dict, Optional from textwrap import dedent +import re import sys from gherkin.parser import Parser @@ -34,9 +35,9 @@ def __post_init__(self): self.metadata = {} def _make_sbdl_identifier(self, name: str) -> str: - return sbdl.SBDL_Parser.sanitize_identifier( - name.replace(" ", "_").replace("-", "_") - ).strip("_") + """Convert a name to a hyphenated slug identifier.""" + slug = re.sub(r'[^a-z0-9]+', '-', name.lower()) + return slug.strip('-') class GherkinConverter: """Converts Gherkin files to SBDL using configurable hierarchy mappings.""" @@ -72,6 +73,9 @@ def write_sbdl_output(self, elements: List[SBDLElement], output_file: str): tokens = sbdl.SBDL_Parser.Tokens attrs = sbdl.SBDL_Parser.Attributes + # Build a lookup from identifier to element type for cross-type relation handling + element_types = {e.identifier: e.element_type for e in elements} + with open(output_file, 'w', encoding='utf-8') as f: f.write("#!sbdl\n") @@ -85,8 +89,22 @@ def write_sbdl_output(self, elements: List[SBDLElement], output_file: str): f"{tokens.declaration_attribute_delimeter}{escaped_desc}{tokens.declaration_attribute_delimeter} " ) + # Write custom:title with original name for display purposes + original_name = element.metadata.get('original_name', '') if element.metadata else '' + if original_name: + escaped_title = sbdl.SBDL_Parser.sanitize(original_name) + f.write(f'custom:title{tokens.declaration_attribute_assign}') + f.write(f'{tokens.declaration_attribute_delimeter}{escaped_title}{tokens.declaration_attribute_delimeter} ') + if element.parent: - f.write(f"{attrs.parent}{tokens.declaration_attribute_assign}{element.parent} ") + parent_type = element_types.get(element.parent) + if parent_type and parent_type != element.element_type: + # Cross-type relation: use the parent's type name as relation + # e.g. test -> requirement becomes "requirement is " + # e.g. requirement -> aspect becomes "aspect is " + f.write(f"{parent_type.value}{tokens.declaration_attribute_assign}{element.parent} ") + else: + f.write(f"{attrs.parent}{tokens.declaration_attribute_assign}{element.parent} ") f.write(f"{tokens.declaration_group_delimeters[1]}\n") diff --git a/docs/templates/requirements-traceability-matrix.md.j2 b/docs/templates/requirements-traceability-matrix.md.j2 new file mode 100644 index 00000000..e219fa0f --- /dev/null +++ b/docs/templates/requirements-traceability-matrix.md.j2 @@ -0,0 +1,131 @@ +--- +title: "Requirements traceability matrix for amp-devcontainer + {%- if 'document-flavor' in sbdl %} ({{ sbdl['document-flavor']['custom:title'] }}){%- endif %}" +author: ["@rjaegers"] +colorlinks: true +date: "October-2025" +keywords: [Traceability, Requirements, RTM, amp-devcontainer] +lang: "en" +titlepage: true +titlepage-color: "0B5ED7" +titlepage-text-color: "FFFFFF" +titlepage-rule-color: "FFFFFF" +titlepage-rule-height: 2 +toc: true +toc-own-page: true +... + +# Introduction + +## Purpose + +This document provides a requirements traceability matrix (RTM) for amp-devcontainer. It maps each requirement to its associated verification tests, providing evidence of test coverage across the system. + +## Abstract + +amp-devcontainer is a set of [devcontainers](https://containers.dev/) tailored towards modern, embedded, software development. This traceability matrix traces requirements from the software requirements specification to their corresponding verification tests (Gherkin scenarios and BATS integration tests). + +{%- macro reencode(text) -%} +{{ text.encode('utf-8').decode('unicode_escape') }} +{%- endmacro -%} + +{%- macro sbdl_id_to_header(text) -%} +{{ text.replace('-', ' ') }} +{%- endmacro -%} + +{%- macro display_name(elem_id) -%} +{%- if elem_id in sbdl and 'custom:title' in sbdl[elem_id] -%} +{{ sbdl[elem_id]['custom:title'] }} +{%- else -%} +{{ elem_id | replace('-', ' ') }} +{%- endif -%} +{%- endmacro -%} + +{%- macro display_name_short(elem_id) -%} +{%- if elem_id in sbdl and 'custom:title' in sbdl[elem_id] -%} +{{ sbdl[elem_id]['custom:title'][:60] }} +{%- else -%} +{{ (elem_id | replace('-', ' '))[:60] }} +{%- endif -%} +{%- endmacro -%} + +# Traceability Matrix + +{%- set aspects = [] -%} +{%- for elem_id, elem in sbdl.items() -%} +{%- if elem.type == 'aspect' -%} +{%- set _ = aspects.append((elem_id, elem)) -%} +{%- endif -%} +{%- endfor -%} + +{%- for aspect_id, aspect in aspects %} + +## {{ reencode(display_name(aspect_id)) }} + +| Requirement | Test | Status | +|---|---|---| + +{%- for req_id, req in sbdl.items() -%} +{%- if req.type == 'requirement' and 'aspect' in req -%} +{%- for asp_ref in req.aspect if asp_ref.identifier == aspect_id -%} + +{%- set ns_tests = namespace(test_list=[]) -%} + +{%- for test_id, test_elem in sbdl.items() -%} +{%- if test_elem.type == 'test' -%} +{%- if 'requirement' in test_elem -%} +{%- for req_ref in test_elem.requirement if req_ref.identifier == req_id -%} +{%- set _ = ns_tests.test_list.append(test_id) -%} +{%- endfor -%} +{%- endif -%} +{%- endif -%} +{%- endfor -%} + +{%- if ns_tests.test_list -%} +{%- for test_id in ns_tests.test_list %} +| {{ reencode(display_name_short(req_id)) }} | {{ reencode(display_name_short(test_id)) }} | Traced | +{%- endfor -%} +{%- else %} +| {{ reencode(display_name_short(req_id)) }} | — | **Not covered** | +{%- endif -%} + +{%- endfor -%} +{%- endif -%} +{%- endfor -%} + +{%- endfor %} + +# Coverage Summary + +{%- set ns_summary = namespace(total_reqs=0, covered_reqs=0, total_tests=0) -%} + +{%- for elem_id, elem in sbdl.items() -%} +{%- if elem.type == 'requirement' -%} +{%- set ns_summary.total_reqs = ns_summary.total_reqs + 1 -%} + +{%- set ns_covered = namespace(is_covered=false) -%} +{%- for test_id, test_elem in sbdl.items() -%} +{%- if test_elem.type == 'test' and 'requirement' in test_elem -%} +{%- for req_ref in test_elem.requirement if req_ref.identifier == elem_id -%} +{%- set ns_covered.is_covered = true -%} +{%- endfor -%} +{%- endif -%} +{%- endfor -%} + +{%- if ns_covered.is_covered -%} +{%- set ns_summary.covered_reqs = ns_summary.covered_reqs + 1 -%} +{%- endif -%} + +{%- endif -%} + +{%- if elem.type == 'test' -%} +{%- set ns_summary.total_tests = ns_summary.total_tests + 1 -%} +{%- endif -%} +{%- endfor %} + +| Metric | Value | +|---|---| +| Total requirements | {{ ns_summary.total_reqs }} | +| Requirements with tests | {{ ns_summary.covered_reqs }} | +| Requirements without tests | {{ ns_summary.total_reqs - ns_summary.covered_reqs }} | +| Total tests | {{ ns_summary.total_tests }} | diff --git a/docs/templates/software-requirements-specification.md.j2 b/docs/templates/software-requirements-specification.md.j2 index dda4cbeb..2ba2c3dd 100644 --- a/docs/templates/software-requirements-specification.md.j2 +++ b/docs/templates/software-requirements-specification.md.j2 @@ -47,13 +47,21 @@ The containers may be used both for local development and continuous integration {%- endmacro -%} {%- macro sbdl_id_to_header(text) -%} -{{ text.replace('_', ' ') }} +{{ text.replace('-', ' ') }} +{%- endmacro -%} + +{%- macro display_name(elem_id) -%} +{%- if elem_id in sbdl and 'custom:title' in sbdl[elem_id] -%} +{{ sbdl[elem_id]['custom:title'] }} +{%- else -%} +{{ elem_id | replace('-', ' ') }} +{%- endif -%} {%- endmacro -%} {%- for req_id, requirement in sbdl.items() %} {%- if requirement.type == 'requirement' and 'parent' not in requirement %} -## {{ reencode(sbdl_id_to_header(req_id)) }} +## {{ reencode(display_name(req_id)) }} {{ reencode(requirement.description) }} @@ -61,7 +69,7 @@ The containers may be used both for local development and continuous integration {%- for child in requirement.child %} {%- set child_req = sbdl[child.identifier] %} -### {{ reencode(sbdl_id_to_header(child.identifier)) }} +### {{ reencode(display_name(child.identifier)) }} {{ reencode(child_req.description) }} diff --git a/docs/templates/software-test-specification.md.j2 b/docs/templates/software-test-specification.md.j2 new file mode 100644 index 00000000..27b68309 --- /dev/null +++ b/docs/templates/software-test-specification.md.j2 @@ -0,0 +1,119 @@ +--- +title: "Software test specification for amp-devcontainer + {%- if 'document-flavor' in sbdl %} ({{ sbdl['document-flavor']['custom:title'] }}){%- endif %}" +author: ["@rjaegers"] +colorlinks: true +date: "October-2025" +keywords: [Software, Test, Specification, STS, amp-devcontainer] +lang: "en" +titlepage: true +titlepage-color: "0B5ED7" +titlepage-text-color: "FFFFFF" +titlepage-rule-color: "FFFFFF" +titlepage-rule-height: 2 +toc: true +toc-own-page: true +... + +# Introduction + +## Purpose + +This document describes the software test specification for amp-devcontainer. It enumerates all tests, both from Gherkin scenarios (verification tests) and BATS integration tests, and traces them to the requirements they verify. + +## Definitions of key words + +The key words *MUST*, *MUST NOT*, *REQUIRED*, *SHALL*, *SHALL NOT*, *SHOULD*, *SHOULD NOT*, *RECOMMENDED*, *MAY*, and *OPTIONAL* in this document are to be interpreted as described in [RFC 2119](https://www.rfc-editor.org/rfc/rfc2119). + +## Abstract + +amp-devcontainer is a set of [devcontainers](https://containers.dev/) tailored towards modern, embedded, software development. This document specifies the tests that verify the requirements defined in the software requirements specification. + +# Tests + +{%- macro reencode(text) -%} +{{ text.encode('utf-8').decode('unicode_escape') }} +{%- endmacro -%} + +{%- macro sbdl_id_to_header(text) -%} +{{ text.replace('-', ' ') }} +{%- endmacro -%} + +{%- macro display_name(elem_id) -%} +{%- if elem_id in sbdl and 'custom:title' in sbdl[elem_id] -%} +{{ sbdl[elem_id]['custom:title'] }} +{%- else -%} +{{ elem_id | replace('-', ' ') }} +{%- endif -%} +{%- endmacro -%} + +{%- set aspects = [] -%} +{%- for elem_id, elem in sbdl.items() -%} +{%- if elem.type == 'aspect' -%} +{%- set _ = aspects.append((elem_id, elem)) -%} +{%- endif -%} +{%- endfor -%} + +{%- for aspect_id, aspect in aspects %} + +## {{ reencode(display_name(aspect_id)) }} + +{%- set ns = namespace(has_requirements=false) -%} + +{%- for req_id, req in sbdl.items() -%} +{%- if req.type == 'requirement' and 'aspect' in req -%} +{%- for asp_ref in req.aspect if asp_ref.identifier == aspect_id -%} +{%- set ns.has_requirements = true -%} + +### {{ reencode(display_name(req_id)) }} + +{{ reencode(req.description) }} + +{%- set ns_tests = namespace(has_tests=false) -%} + +#### Verification tests + +{%- for test_id, test_elem in sbdl.items() -%} +{%- if test_elem.type == 'test' -%} +{%- if 'requirement' in test_elem -%} +{%- for req_ref in test_elem.requirement if req_ref.identifier == req_id -%} +{%- set ns_tests.has_tests = true %} +- **{{ reencode(display_name(test_id)) }}**: {{ reencode(test_elem.description) }} +{%- endfor -%} +{%- endif -%} +{%- endif -%} +{%- endfor -%} + +{%- if not ns_tests.has_tests %} +- *No tests traced to this requirement.* +{%- endif -%} + +{%- endfor -%} +{%- endif -%} +{%- endfor -%} + +{%- if not ns.has_requirements %} + +*No requirements traced to this area.* +{%- endif -%} + +{%- endfor %} + +## Untraced tests + +The following tests are not traced to any requirement: + +{%- set ns_untraced = namespace(has_untraced=false) -%} + +{%- for test_id, test_elem in sbdl.items() -%} +{%- if test_elem.type == 'test' and 'requirement' not in test_elem -%} +{%- set ns_untraced.has_untraced = true %} +- **{{ reencode(display_name(test_id)) }}**: {{ reencode(test_elem.description) }} +{%- if 'tag' in test_elem %} *(tags: {{ test_elem.tag }})* {%- endif -%} +{%- endif -%} +{%- endfor -%} + +{%- if not ns_untraced.has_untraced %} + +*All tests are traced to requirements.* +{%- endif %} diff --git a/test/base/integration-tests.bats b/test/base/integration-tests.bats index 39b82d41..0d482420 100644 --- a/test/base/integration-tests.bats +++ b/test/base/integration-tests.bats @@ -7,6 +7,7 @@ setup() { load '/usr/local/bats-assert/load' } +# bats test_tags=security @test "cisco umbrella root certificate is included in system certificate store" { run openssl verify -CAfile /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/cisco-umbrella-root.pem assert_success diff --git a/test/cpp/integration-tests.bats b/test/cpp/integration-tests.bats index fd6cfd2f..7ecfcdde 100644 --- a/test/cpp/integration-tests.bats +++ b/test/cpp/integration-tests.bats @@ -28,7 +28,7 @@ teardown() { # that the tools are compatible with each other. E.g. that the host and embedded toolchains # are aligned in terms of major and minor versions. -# bats test_tags=Compatibility,Version,Clang +# bats test_tags=compatibility,version,clang @test "clang toolchain versions should be aligned with expected versions" { EXPECTED_VERSION=$(get_expected_semver_for clang) @@ -38,7 +38,7 @@ teardown() { done } -# bats test_tags=Compatibility,Version,GCC +# bats test_tags=compatibility,version,gcc @test "host gcc toolchain versions and alternatives should be aligned with expected versions" { EXPECTED_VERSION=$(get_expected_semver_for g++) @@ -48,14 +48,14 @@ teardown() { done } -# bats test_tags=Compatibility,Version,HostGCCArmGCC +# bats test_tags=compatibility,version,hostgccarmgcc @test "host and embedded gcc toolchain versions should be the same major and minor version" { EXPECTED_MAJOR_MINOR_VERSION=$(get_expected_semver_for g++ | cut -d. -f1,2) INSTALLED_MAJOR_MINOR_VERSION=$(arm-none-eabi-gcc -dumpfullversion | cut -d. -f1,2) assert_equal_print "$EXPECTED_MAJOR_MINOR_VERSION" "$INSTALLED_MAJOR_MINOR_VERSION" "Host and ARM GCC major and minor version" } -# bats test_tags=Compatibility,Version,Tools +# bats test_tags=compatibility,version,tools @test "supporting tool versions should be aligned with expected versions" { for TOOL in gdb gdb-multiarch git ninja; do EXPECTED_VERSION=$(get_expected_semver_for ${TOOL}) @@ -72,6 +72,7 @@ teardown() { done } +# bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system @test "valid code input should result in working executable using host compiler" { cmake --preset gcc cmake --build --preset gcc @@ -81,6 +82,7 @@ teardown() { assert_output "Hello World!" } +# bats test_tags=compilation,compile-for-arm-cortex-target-architecture @test "valid code input should result in elf executable using arm-none-eabi compiler" { cmake --preset gcc-arm-none-eabi cmake --build --preset gcc-arm-none-eabi @@ -90,6 +92,7 @@ teardown() { assert_output --partial "Machine: ARM" } +# bats test_tags=compilation,compile-for-microsoft-windows-operating-system @test "valid code input should result in Windows executable using clang-cl compiler" { install_win_sdk_when_ci_unset @@ -97,6 +100,7 @@ teardown() { cmake --build --preset clang-cl } +# bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system @test "compilation database should be generated on CMake configure" { cmake --preset gcc assert [ -e build/gcc/compile_commands.json ] @@ -105,21 +109,25 @@ teardown() { assert [ -e build/gcc-arm-none-eabi/compile_commands.json ] } +# bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system @test "invalid code input should result in failing build" { cmake --preset gcc run ! cmake --build --preset gcc-fail } +# bats test_tags=compilation,compilation-cache @test "using ccache as a compiler launcher should result in cached build using gcc compiler" { configure_and_build_with_ccache gcc } +# bats test_tags=compilation,compilation-cache @test "using ccache as a compiler launcher should result in cached build using clang-cl compiler" { install_win_sdk_when_ci_unset configure_and_build_with_ccache clang-cl } +# bats test_tags=static-and-dynamic-analysis,static-analysis @test "running clang-tidy as part of the build should result in warning diagnostics" { cmake --preset clang @@ -128,6 +136,7 @@ teardown() { assert_output --partial "warning: use a trailing return type for this function" } +# bats test_tags=static-and-dynamic-analysis,static-analysis @test "running include-what-you-use as part of the build should result in warning diagnostics" { cmake --preset clang @@ -136,12 +145,14 @@ teardown() { assert_output --partial "Warning: include-what-you-use reported diagnostics:" } +# bats test_tags=static-and-dynamic-analysis,code-formatting @test "running clang-format should result in re-formatted code" { run clang-format clang-tools/unformatted.cpp assert_success assert_output "int main() {}" } +# bats test_tags=static-and-dynamic-analysis,coverage-analysis @test "coverage information should be generated when running a testsuite" { cmake --preset coverage cmake --build --preset coverage @@ -155,6 +166,7 @@ teardown() { assert_output --partial "GCC Code Coverage Report" } +# bats test_tags=static-and-dynamic-analysis,fuzz-testing @test "crashes should be detected when fuzzing an executable" { cmake --preset clang cmake --build --preset fuzzing @@ -164,6 +176,7 @@ teardown() { assert_output --partial "SUMMARY: libFuzzer: deadly signal" } +# bats test_tags=static-and-dynamic-analysis,mutation-testing @test "a mutation score should be calculated when mutation testing a testsuite" { cmake --preset mutation cmake --build --preset mutation @@ -172,17 +185,20 @@ teardown() { assert_output --partial "[info] Mutation score:" } +# bats test_tags=static-and-dynamic-analysis,static-analysis @test "clangd should be able to analyze source files" { run clangd --check=gcc/main.cpp assert_success assert_output --partial "All checks completed, 0 errors" } +# bats test_tags=static-and-dynamic-analysis,static-analysis @test "clangd should start with a specified compile commands path" { run timeout 1s clangd --compile-commands-dir=/root/.amp refute_output --partial "Path specified by --compile-commands-dir does not exist. The argument will be ignored." } +# bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system @test "using lld as an alternative linker should result in working host executable" { cmake --preset gcc cmake --build --preset gcc-lld @@ -195,14 +211,17 @@ teardown() { assert_output "Hello World!" } +# bats test_tags=static-and-dynamic-analysis,static-analysis @test "sanitizers should detect undefined or suspicious behavior in code compiled with gcc" { build_and_run_with_sanitizers gcc } +# bats test_tags=static-and-dynamic-analysis,static-analysis @test "sanitizers should detect undefined or suspicious behavior in code compiled with clang" { build_and_run_with_sanitizers clang } +# bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system @test "using Conan as package manager should resolve external dependencies" { pushd package-managers/conan @@ -214,6 +233,7 @@ teardown() { popd } +# bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system @test "using CPM as package manager should resolve external dependencies" { cmake --preset cpm cmake --build --preset cpm diff --git a/test/rust/integration-tests.bats b/test/rust/integration-tests.bats index 0c803fcb..be5c59f8 100644 --- a/test/rust/integration-tests.bats +++ b/test/rust/integration-tests.bats @@ -15,6 +15,7 @@ teardown() { popd } +# bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system @test "valid code input should result in working executable targeting the host architecture" { rustc --out-dir build rust/hello.rs @@ -23,6 +24,7 @@ teardown() { assert_output "Hello, world!" } +# bats test_tags=compilation,compile-for-arm-cortex-target-architecture @test "valid code input should result in working elf executable targeting the cortex-m architecture" { pushd cortex-m @@ -36,6 +38,7 @@ teardown() { popd } +# bats test_tags=compilation,compile-for-arm-cortex-target-architecture @test "valid code input should result in working elf executable targeting the cortex-mf architecture" { pushd cortex-mf @@ -49,6 +52,7 @@ teardown() { popd } +# bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system @test "using cargo run should result in working executable" { pushd cargo @@ -59,12 +63,14 @@ teardown() { popd } +# bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system @test "invalid code input should result in failing build" { run rustc rust/fail.rs assert_failure assert_output --partial "error: " } +# bats test_tags=static-and-dynamic-analysis,static-analysis @test "running clippy should result in warning diagnostics" { pushd clippy @@ -75,6 +81,7 @@ teardown() { popd } +# bats test_tags=static-and-dynamic-analysis,code-formatting @test "running rustfmt should result in re-formatted code" { run rustfmt --color=never --check rust/unformatted.rs assert_failure @@ -88,6 +95,7 @@ teardown() { EOF } +# bats test_tags=static-and-dynamic-analysis,coverage-analysis @test "coverage information should be generated when running a testsuite" { pushd test @@ -104,6 +112,7 @@ EOF popd } +# bats test_tags=static-and-dynamic-analysis,mutation-testing @test "mutation testing a test executable should be supported" { pushd test From 2ac0e8dacf5dc212d9e38c7cccd0e34e305f1908 Mon Sep 17 00:00:00 2001 From: Ron <45816308+rjaegers@users.noreply.github.com> Date: Thu, 26 Feb 2026 20:47:18 +0000 Subject: [PATCH 2/6] refactor: clean up unused code --- docs/support/bats_sbdl_converter.py | 86 ++----------------- docs/support/generate-sbdl.py | 12 +-- docs/support/gherkin_sbdl_converter.py | 19 +++- .../requirements-traceability-matrix.md.j2 | 4 - .../software-requirements-specification.md.j2 | 4 - .../software-test-specification.md.j2 | 4 - 6 files changed, 30 insertions(+), 99 deletions(-) diff --git a/docs/support/bats_sbdl_converter.py b/docs/support/bats_sbdl_converter.py index 1cc1f1d4..448e1399 100644 --- a/docs/support/bats_sbdl_converter.py +++ b/docs/support/bats_sbdl_converter.py @@ -10,23 +10,15 @@ from dataclasses import dataclass, field from typing import Dict, List, Optional +from gherkin_sbdl_converter import to_slug + try: import sbdl - def sanitize_identifier(name: str) -> str: - """Convert a name to a hyphenated slug identifier.""" - slug = re.sub(r'[^a-z0-9]+', '-', name.lower()) - return slug.strip('-') - def sanitize_description(text: str) -> str: return sbdl.SBDL_Parser.sanitize(text) except ImportError: - def sanitize_identifier(name: str) -> str: - """Convert a name to a hyphenated slug identifier.""" - slug = re.sub(r'[^a-z0-9]+', '-', name.lower()) - return slug.strip('-') - def sanitize_description(text: str) -> str: return text.replace('"', '\\"') @@ -43,7 +35,7 @@ class BatsTest: def __post_init__(self): if not self.identifier: - self.identifier = sanitize_identifier(self.name) + self.identifier = to_slug(self.name) class BatsConverter: @@ -63,24 +55,7 @@ def __init__(self, requirement_tag_map: Optional[Dict[str, str]] = None): """ self.requirement_tag_map = requirement_tag_map or {} - @staticmethod - def _extract_flavor(file_path: str) -> str: - """Extract the flavor prefix from a BATS file path. - - Derives a prefix from the parent directory name of the BATS file, - e.g. 'test/cpp/integration-tests.bats' -> 'cpp'. - - Args: - file_path: Path to the BATS file. - - Returns: - Flavor string derived from the parent directory. - """ - import os - parent = os.path.basename(os.path.dirname(os.path.abspath(file_path))) - return sanitize_identifier(parent) if parent else "" - - def extract_from_bats_file(self, file_path: str) -> List[BatsTest]: + def extract_from_bats_file(self, file_path: str, flavor: str = "") -> List[BatsTest]: """Extract all test definitions from a BATS file. Parses `@test "..."` blocks and any preceding `# bats test_tags=...` @@ -88,6 +63,8 @@ def extract_from_bats_file(self, file_path: str) -> List[BatsTest]: Args: file_path: Path to the .bats file. + flavor: Optional identifier prefix for disambiguation + (e.g. 'cpp', 'rust', 'base'). Returns: List of BatsTest objects. @@ -116,8 +93,7 @@ def extract_from_bats_file(self, file_path: str) -> List[BatsTest]: test_match = self._TEST_PATTERN.match(stripped) if test_match: test_name = test_match.group(1) - flavor = self._extract_flavor(file_path) - base_id = sanitize_identifier(test_name) + base_id = to_slug(test_name) identifier = f"{flavor}-{base_id}" if flavor else base_id test = BatsTest( name=test_name, @@ -138,54 +114,6 @@ def extract_from_bats_file(self, file_path: str) -> List[BatsTest]: return tests - def write_sbdl_output(self, tests: List[BatsTest], output_file: str): - """Write extracted BATS tests as SBDL test elements. - - Args: - tests: List of BatsTest objects. - output_file: Path to write the SBDL output. - """ - with open(output_file, "w", encoding="utf-8") as f: - f.write("#!sbdl\n") - - for test in tests: - escaped_desc = sanitize_description(test.name) - identifier = test.identifier - - f.write(f'{identifier} is test {{ description is "{escaped_desc}" custom:title is "{escaped_desc}"') - - # Add tag property if tags exist - if test.tags: - tag_str = ",".join(test.tags) - f.write(f" tag is {tag_str}") - - # Add requirement relation via tag mapping - req_ids = self._resolve_requirement_relations(test) - if req_ids: - f.write(f" requirement is {','.join(req_ids)}") - - f.write(" }\n") - - def _resolve_requirement_relations(self, test: BatsTest) -> List[str]: - """Resolve requirement identifiers from test tags using the tag map. - - Args: - test: A BatsTest with tags. - - Returns: - List of requirement SBDL identifiers. - """ - req_ids = [] - for tag in test.tags: - tag_lower = tag.lower() - if tag_lower in self.requirement_tag_map: - entry = self.requirement_tag_map[tag_lower] - # Support both plain string and (identifier, type) tuple entries - req_id = entry[0] if isinstance(entry, tuple) else entry - if req_id not in req_ids: - req_ids.append(req_id) - return req_ids - def _resolve_typed_relations(self, test: BatsTest) -> Dict[str, List[str]]: """Resolve typed relations from test tags using the tag map. diff --git a/docs/support/generate-sbdl.py b/docs/support/generate-sbdl.py index 539d19cc..e1e4f358 100644 --- a/docs/support/generate-sbdl.py +++ b/docs/support/generate-sbdl.py @@ -17,11 +17,10 @@ import argparse import os -import re import sys from gherkin_mapping_config import FEATURE_RULE_CONFIG, TEST_SPECIFICATION_CONFIG -from gherkin_sbdl_converter import GherkinConverter +from gherkin_sbdl_converter import GherkinConverter, to_slug from bats_sbdl_converter import BatsConverter @@ -95,7 +94,7 @@ def main(): if original: requirement_tag_map[original.lower()] = entry # Also map the slugified version for BATS tag matching - slug_original = re.sub(r'[^a-z0-9]+', '-', original.lower()).strip('-') + slug_original = to_slug(original) if slug_original: requirement_tag_map[slug_original] = entry # Also map the SBDL identifier itself @@ -108,19 +107,20 @@ def main(): for bats_path in args.bats: if os.path.isfile(bats_path): print(f"Processing BATS: {bats_path}") - tests = bats_converter.extract_from_bats_file(bats_path) + flavor_prefix = to_slug(os.path.basename(os.path.dirname(os.path.abspath(bats_path)))) + tests = bats_converter.extract_from_bats_file(bats_path, flavor=flavor_prefix) bats_tests.extend(tests) else: print(f"File not found: {bats_path}", file=sys.stderr) # Write combined SBDL output - _write_combined_sbdl(gherkin_converter, gherkin_elements, bats_converter, bats_tests, args.output, args.flavor) + _write_combined_sbdl(gherkin_elements, bats_converter, bats_tests, args.output, args.flavor) total = len(gherkin_elements) + len(bats_tests) print(f"Extracted {total} elements ({len(gherkin_elements)} from Gherkin, {len(bats_tests)} from BATS) to {args.output}") -def _write_combined_sbdl(gherkin_converter, gherkin_elements, bats_converter, bats_tests, output_file, flavor=""): +def _write_combined_sbdl(gherkin_elements, bats_converter, bats_tests, output_file, flavor=""): """Write a single combined SBDL file from both Gherkin and BATS sources.""" import sbdl as sbdl_lib diff --git a/docs/support/gherkin_sbdl_converter.py b/docs/support/gherkin_sbdl_converter.py index 25e05a64..41792ca0 100644 --- a/docs/support/gherkin_sbdl_converter.py +++ b/docs/support/gherkin_sbdl_converter.py @@ -17,6 +17,22 @@ SBDLElementType, ) + +def to_slug(name: str) -> str: + """Convert a human-readable name to a hyphenated slug identifier. + + Lowercases the name and replaces any sequence of non-alphanumeric + characters with a single hyphen. + + Args: + name: The original name. + + Returns: + Hyphenated, lowercase slug suitable as an SBDL identifier. + """ + slug = re.sub(r'[^a-z0-9]+', '-', name.lower()) + return slug.strip('-') + @dataclass class SBDLElement: """A generalized SBDL element that can represent any type.""" @@ -36,8 +52,7 @@ def __post_init__(self): def _make_sbdl_identifier(self, name: str) -> str: """Convert a name to a hyphenated slug identifier.""" - slug = re.sub(r'[^a-z0-9]+', '-', name.lower()) - return slug.strip('-') + return to_slug(name) class GherkinConverter: """Converts Gherkin files to SBDL using configurable hierarchy mappings.""" diff --git a/docs/templates/requirements-traceability-matrix.md.j2 b/docs/templates/requirements-traceability-matrix.md.j2 index e219fa0f..a8ef9061 100644 --- a/docs/templates/requirements-traceability-matrix.md.j2 +++ b/docs/templates/requirements-traceability-matrix.md.j2 @@ -29,10 +29,6 @@ amp-devcontainer is a set of [devcontainers](https://containers.dev/) tailored t {{ text.encode('utf-8').decode('unicode_escape') }} {%- endmacro -%} -{%- macro sbdl_id_to_header(text) -%} -{{ text.replace('-', ' ') }} -{%- endmacro -%} - {%- macro display_name(elem_id) -%} {%- if elem_id in sbdl and 'custom:title' in sbdl[elem_id] -%} {{ sbdl[elem_id]['custom:title'] }} diff --git a/docs/templates/software-requirements-specification.md.j2 b/docs/templates/software-requirements-specification.md.j2 index 2ba2c3dd..5823b2b9 100644 --- a/docs/templates/software-requirements-specification.md.j2 +++ b/docs/templates/software-requirements-specification.md.j2 @@ -46,10 +46,6 @@ The containers may be used both for local development and continuous integration {{ text.encode('utf-8').decode('unicode_escape') }} {%- endmacro -%} -{%- macro sbdl_id_to_header(text) -%} -{{ text.replace('-', ' ') }} -{%- endmacro -%} - {%- macro display_name(elem_id) -%} {%- if elem_id in sbdl and 'custom:title' in sbdl[elem_id] -%} {{ sbdl[elem_id]['custom:title'] }} diff --git a/docs/templates/software-test-specification.md.j2 b/docs/templates/software-test-specification.md.j2 index 27b68309..db2fe5a9 100644 --- a/docs/templates/software-test-specification.md.j2 +++ b/docs/templates/software-test-specification.md.j2 @@ -35,10 +35,6 @@ amp-devcontainer is a set of [devcontainers](https://containers.dev/) tailored t {{ text.encode('utf-8').decode('unicode_escape') }} {%- endmacro -%} -{%- macro sbdl_id_to_header(text) -%} -{{ text.replace('-', ' ') }} -{%- endmacro -%} - {%- macro display_name(elem_id) -%} {%- if elem_id in sbdl and 'custom:title' in sbdl[elem_id] -%} {{ sbdl[elem_id]['custom:title'] }} From 9700fa106c902048a301611e0d265a51ebeeb364 Mon Sep 17 00:00:00 2001 From: Ron <45816308+rjaegers@users.noreply.github.com> Date: Thu, 26 Feb 2026 20:52:43 +0000 Subject: [PATCH 3/6] refactor: reduce duplication --- .github/workflows/wc-document-generation.yml | 2 +- docs/support/generate-sbdl.py | 35 +-------- docs/support/gherkin-to-sbdl.py | 40 ---------- docs/support/gherkin_mapping_config.py | 52 +----------- docs/support/gherkin_sbdl_converter.py | 83 +++++++++++--------- 5 files changed, 53 insertions(+), 159 deletions(-) delete mode 100755 docs/support/gherkin-to-sbdl.py diff --git a/.github/workflows/wc-document-generation.yml b/.github/workflows/wc-document-generation.yml index bd3cad75..f0956fbe 100644 --- a/.github/workflows/wc-document-generation.yml +++ b/.github/workflows/wc-document-generation.yml @@ -27,7 +27,7 @@ jobs: - name: Generate SRS document run: | set -Eeuo pipefail - python docs/support/gherkin-to-sbdl.py test/cpp/features/*.feature + python docs/support/generate-sbdl.py --config requirements --gherkin test/cpp/features/*.feature sbdl -m template-fill --template docs/templates/software-requirements-specification.md.j2 output.sbdl > software-requirements-specification.md - name: Generate C++ test specification and traceability documents run: | diff --git a/docs/support/generate-sbdl.py b/docs/support/generate-sbdl.py index e1e4f358..1b693b6f 100644 --- a/docs/support/generate-sbdl.py +++ b/docs/support/generate-sbdl.py @@ -20,7 +20,7 @@ import sys from gherkin_mapping_config import FEATURE_RULE_CONFIG, TEST_SPECIFICATION_CONFIG -from gherkin_sbdl_converter import GherkinConverter, to_slug +from gherkin_sbdl_converter import GherkinConverter, to_slug, write_gherkin_sbdl_elements from bats_sbdl_converter import BatsConverter @@ -124,12 +124,6 @@ def _write_combined_sbdl(gherkin_elements, bats_converter, bats_tests, output_fi """Write a single combined SBDL file from both Gherkin and BATS sources.""" import sbdl as sbdl_lib - tokens = sbdl_lib.SBDL_Parser.Tokens - attrs = sbdl_lib.SBDL_Parser.Attributes - - # Build element type lookup for cross-type relation handling - element_types = {e.identifier: e.element_type for e in gherkin_elements} - with open(output_file, "w", encoding="utf-8") as f: f.write("#!sbdl\n\n") @@ -139,32 +133,7 @@ def _write_combined_sbdl(gherkin_elements, bats_converter, bats_tests, output_fi f.write(f'document-flavor is definition {{ description is "{escaped_flavor}" custom:title is "{escaped_flavor}" }}\n\n') f.write("# Elements extracted from Gherkin feature files\n") - - for element in gherkin_elements: - escaped_desc = sbdl_lib.SBDL_Parser.sanitize(element.description) - sbdl_type = element.element_type.value - f.write(f"{element.identifier} {tokens.declaration} {sbdl_type} ") - f.write(f"{tokens.declaration_group_delimeters[0]} ") - f.write(f"{attrs.description}{tokens.declaration_attribute_assign}") - f.write( - f"{tokens.declaration_attribute_delimeter}{escaped_desc}{tokens.declaration_attribute_delimeter} " - ) - - # Write custom:title with original name for display purposes - original_name = element.metadata.get('original_name', '') if element.metadata else '' - if original_name: - escaped_title = sbdl_lib.SBDL_Parser.sanitize(original_name) - f.write(f'custom:title{tokens.declaration_attribute_assign}') - f.write(f'{tokens.declaration_attribute_delimeter}{escaped_title}{tokens.declaration_attribute_delimeter} ') - - if element.parent: - parent_type = element_types.get(element.parent) - if parent_type and parent_type != element.element_type: - f.write(f"{parent_type.value}{tokens.declaration_attribute_assign}{element.parent} ") - else: - f.write(f"{attrs.parent}{tokens.declaration_attribute_assign}{element.parent} ") - - f.write(f"{tokens.declaration_group_delimeters[1]}\n") + write_gherkin_sbdl_elements(f, gherkin_elements) if bats_tests: f.write("\n# Elements extracted from BATS integration test files\n") diff --git a/docs/support/gherkin-to-sbdl.py b/docs/support/gherkin-to-sbdl.py deleted file mode 100755 index c68a4ae0..00000000 --- a/docs/support/gherkin-to-sbdl.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import sys - -from gherkin_mapping_config import FEATURE_RULE_CONFIG, FEATURE_RULE_SCENARIO_CONFIG, TEST_SPECIFICATION_CONFIG -from gherkin_sbdl_converter import GherkinConverter - -def main(): - configs = { - 'feature-rule': FEATURE_RULE_CONFIG, - 'feature-rule-scenario': FEATURE_RULE_SCENARIO_CONFIG, - 'test-specification': TEST_SPECIFICATION_CONFIG - } - - parser = argparse.ArgumentParser(description='Configurable Gherkin to SBDL converter') - parser.add_argument('feature_files', nargs='+', help='Paths to feature files') - parser.add_argument('--output', '-o', default='output.sbdl', help='Output SBDL file') - parser.add_argument('--config', choices=configs.keys(), - default='feature-rule', help='Conversion configuration preset') - - args = parser.parse_args() - config = configs[args.config] - converter = GherkinConverter(config) - gherkin_elements = [] - - for feature_path in args.feature_files: - if os.path.isfile(feature_path): - print(f"Processing {feature_path}") - elements = converter.extract_from_feature_file(feature_path) - gherkin_elements.extend(elements) - else: - print(f"File not found: {feature_path}", file=sys.stderr) - - converter.write_sbdl_output(gherkin_elements, args.output) - print(f"Extracted {len(gherkin_elements)} elements to {args.output}") - -if __name__ == '__main__': - main() diff --git a/docs/support/gherkin_mapping_config.py b/docs/support/gherkin_mapping_config.py index b7517f5d..fde6ee1f 100644 --- a/docs/support/gherkin_mapping_config.py +++ b/docs/support/gherkin_mapping_config.py @@ -5,7 +5,7 @@ """ from dataclasses import dataclass -from typing import Dict, List, Optional, Union +from typing import List, Optional from enum import Enum class GherkinElementType(Enum): @@ -21,8 +21,6 @@ class SBDLElementType(Enum): """Supported SBDL element types for mapping.""" REQUIREMENT = "requirement" ASPECT = "aspect" - USECASE = "usecase" - DEFINITION = "definition" TEST = "test" @dataclass @@ -45,7 +43,7 @@ def get_mapping_for_type(self, gherkin_type: GherkinElementType) -> Optional[Hie # Predefined configurations for different use cases -# Current Configuration (Feature -> Rule mapping) +# Requirements Configuration (Feature -> Rule mapping as requirements) FEATURE_RULE_CONFIG = ConversionConfig( hierarchy_mappings=[ HierarchyMapping( @@ -59,38 +57,6 @@ def get_mapping_for_type(self, gherkin_type: GherkinElementType) -> Optional[Hie ] ) -# Extended Configuration (Feature -> Rule -> Scenario mapping) -FEATURE_RULE_SCENARIO_CONFIG = ConversionConfig( - hierarchy_mappings=[ - HierarchyMapping( - gherkin_type=GherkinElementType.FEATURE, - sbdl_type=SBDLElementType.ASPECT - ), - HierarchyMapping( - gherkin_type=GherkinElementType.RULE, - sbdl_type=SBDLElementType.REQUIREMENT - ), - HierarchyMapping( - gherkin_type=GherkinElementType.SCENARIO, - sbdl_type=SBDLElementType.TEST - ) - ] -) - -# Flat Configuration (All as requirements at same level) -FLAT_CONFIG = ConversionConfig( - hierarchy_mappings=[ - HierarchyMapping( - gherkin_type=GherkinElementType.FEATURE, - sbdl_type=SBDLElementType.REQUIREMENT - ), - HierarchyMapping( - gherkin_type=GherkinElementType.RULE, - sbdl_type=SBDLElementType.REQUIREMENT - ) - ] -) - # Test Specification Configuration (Feature -> Rule -> Scenario with tests) # Used for generating test specification and traceability documents. # Features become aspects (system areas), Rules become requirements, @@ -111,17 +77,3 @@ def get_mapping_for_type(self, gherkin_type: GherkinElementType) -> Optional[Hie ) ] ) - -# Use Case Focused Configuration -USECASE_CONFIG = ConversionConfig( - hierarchy_mappings=[ - HierarchyMapping( - gherkin_type=GherkinElementType.FEATURE, - sbdl_type=SBDLElementType.USECASE - ), - HierarchyMapping( - gherkin_type=GherkinElementType.SCENARIO, - sbdl_type=SBDLElementType.USECASE - ) - ] -) diff --git a/docs/support/gherkin_sbdl_converter.py b/docs/support/gherkin_sbdl_converter.py index 41792ca0..5787d1f7 100644 --- a/docs/support/gherkin_sbdl_converter.py +++ b/docs/support/gherkin_sbdl_converter.py @@ -54,6 +54,52 @@ def _make_sbdl_identifier(self, name: str) -> str: """Convert a name to a hyphenated slug identifier.""" return to_slug(name) +def write_gherkin_sbdl_elements(f, elements: List[SBDLElement]): + """Write a list of Gherkin-derived SBDL elements to an open file handle. + + Handles description, custom:title, and parent/cross-type relations + for each element. + + Args: + f: Writable file handle. + elements: List of SBDLElement objects to write. + """ + tokens = sbdl.SBDL_Parser.Tokens + attrs = sbdl.SBDL_Parser.Attributes + + # Build a lookup from identifier to element type for cross-type relation handling + element_types = {e.identifier: e.element_type for e in elements} + + for element in elements: + escaped_desc = sbdl.SBDL_Parser.sanitize(element.description) + sbdl_type = element.element_type.value + f.write(f"{element.identifier} {tokens.declaration} {sbdl_type} ") + f.write(f"{tokens.declaration_group_delimeters[0]} ") + f.write(f"{attrs.description}{tokens.declaration_attribute_assign}") + f.write( + f"{tokens.declaration_attribute_delimeter}{escaped_desc}{tokens.declaration_attribute_delimeter} " + ) + + # Write custom:title with original name for display purposes + original_name = element.metadata.get('original_name', '') if element.metadata else '' + if original_name: + escaped_title = sbdl.SBDL_Parser.sanitize(original_name) + f.write(f'custom:title{tokens.declaration_attribute_assign}') + f.write(f'{tokens.declaration_attribute_delimeter}{escaped_title}{tokens.declaration_attribute_delimeter} ') + + if element.parent: + parent_type = element_types.get(element.parent) + if parent_type and parent_type != element.element_type: + # Cross-type relation: use the parent's type name as relation + # e.g. test -> requirement becomes "requirement is " + # e.g. requirement -> aspect becomes "aspect is " + f.write(f"{parent_type.value}{tokens.declaration_attribute_assign}{element.parent} ") + else: + f.write(f"{attrs.parent}{tokens.declaration_attribute_assign}{element.parent} ") + + f.write(f"{tokens.declaration_group_delimeters[1]}\n") + + class GherkinConverter: """Converts Gherkin files to SBDL using configurable hierarchy mappings.""" @@ -85,43 +131,10 @@ def extract_from_feature_file(self, file_path: str) -> List[SBDLElement]: return [] def write_sbdl_output(self, elements: List[SBDLElement], output_file: str): - tokens = sbdl.SBDL_Parser.Tokens - attrs = sbdl.SBDL_Parser.Attributes - - # Build a lookup from identifier to element type for cross-type relation handling - element_types = {e.identifier: e.element_type for e in elements} - + """Write extracted Gherkin elements as a standalone SBDL file.""" with open(output_file, 'w', encoding='utf-8') as f: f.write("#!sbdl\n") - - for element in elements: - escaped_desc = sbdl.SBDL_Parser.sanitize(element.description) - sbdl_type = element.element_type.value - f.write(f"{element.identifier} {tokens.declaration} {sbdl_type} ") - f.write(f"{tokens.declaration_group_delimeters[0]} ") - f.write(f"{attrs.description}{tokens.declaration_attribute_assign}") - f.write( - f"{tokens.declaration_attribute_delimeter}{escaped_desc}{tokens.declaration_attribute_delimeter} " - ) - - # Write custom:title with original name for display purposes - original_name = element.metadata.get('original_name', '') if element.metadata else '' - if original_name: - escaped_title = sbdl.SBDL_Parser.sanitize(original_name) - f.write(f'custom:title{tokens.declaration_attribute_assign}') - f.write(f'{tokens.declaration_attribute_delimeter}{escaped_title}{tokens.declaration_attribute_delimeter} ') - - if element.parent: - parent_type = element_types.get(element.parent) - if parent_type and parent_type != element.element_type: - # Cross-type relation: use the parent's type name as relation - # e.g. test -> requirement becomes "requirement is " - # e.g. requirement -> aspect becomes "aspect is " - f.write(f"{parent_type.value}{tokens.declaration_attribute_assign}{element.parent} ") - else: - f.write(f"{attrs.parent}{tokens.declaration_attribute_assign}{element.parent} ") - - f.write(f"{tokens.declaration_group_delimeters[1]}\n") + write_gherkin_sbdl_elements(f, elements) def _extract_gherkin_element(self, element_data: Dict, element_type: GherkinElementType, parent_id: Optional[str]) -> Optional[SBDLElement]: mapping = self.config.get_mapping_for_type(element_type) From 5fade2320f3f18f3f21f542cbb3119c476931980 Mon Sep 17 00:00:00 2001 From: Ron <45816308+rjaegers@users.noreply.github.com> Date: Thu, 26 Feb 2026 21:08:43 +0000 Subject: [PATCH 4/6] docs: revise content of STS --- .../software-test-specification.md.j2 | 80 ++++++++++++++----- 1 file changed, 59 insertions(+), 21 deletions(-) diff --git a/docs/templates/software-test-specification.md.j2 b/docs/templates/software-test-specification.md.j2 index db2fe5a9..84414b29 100644 --- a/docs/templates/software-test-specification.md.j2 +++ b/docs/templates/software-test-specification.md.j2 @@ -14,35 +14,73 @@ titlepage-rule-height: 2 toc: true toc-own-page: true ... +{% macro reencode(text) -%} +{{ text.encode('utf-8').decode('unicode_escape') }} +{%- endmacro -%} + +{%- macro display_name(elem_id) -%} +{%- if elem_id in sbdl and 'custom:title' in sbdl[elem_id] -%} +{{ sbdl[elem_id]['custom:title'] }} +{%- else -%} +{{ elem_id | replace('-', ' ') }} +{%- endif -%} +{%- endmacro %} # Introduction ## Purpose -This document describes the software test specification for amp-devcontainer. It enumerates all tests, both from Gherkin scenarios (verification tests) and BATS integration tests, and traces them to the requirements they verify. +This document specifies the tests for amp-devcontainer +{%- if 'document-flavor' in sbdl %} ({{ sbdl['document-flavor']['custom:title'] }} flavor){%- endif %}. +It enumerates all tests, describes the test methodology, and traces each test to the requirement(s) it verifies. + +For the full requirement descriptions, refer to the *Software Requirements Specification* (SRS). +For the coverage overview, refer to the *Requirements Traceability Matrix* (RTM). + +## Scope + +This document covers the following types of tests: + +- **Gherkin verification tests**: Scenario-based tests defined in Gherkin feature files and executed with Playwright. These tests verify behavioral requirements at the system level. +- **BATS integration tests**: Shell-based integration tests defined in BATS (Bash Automated Testing System) files. These tests verify tool availability, version alignment, and end-to-end compilation and analysis workflows. ## Definitions of key words The key words *MUST*, *MUST NOT*, *REQUIRED*, *SHALL*, *SHALL NOT*, *SHOULD*, *SHOULD NOT*, *RECOMMENDED*, *MAY*, and *OPTIONAL* in this document are to be interpreted as described in [RFC 2119](https://www.rfc-editor.org/rfc/rfc2119). -## Abstract +## Referenced documents -amp-devcontainer is a set of [devcontainers](https://containers.dev/) tailored towards modern, embedded, software development. This document specifies the tests that verify the requirements defined in the software requirements specification. +| Document | Description | +|---|---| +| Software Requirements Specification (SRS) | Defines the requirements verified by the tests in this document | +| Requirements Traceability Matrix (RTM) | Maps requirements to tests and provides coverage metrics | -# Tests +# Test environment -{%- macro reencode(text) -%} -{{ text.encode('utf-8').decode('unicode_escape') }} -{%- endmacro -%} +Tests are executed in a container built from the amp-devcontainer image for the applicable flavor. The container provides all necessary compilers, tools, and dependencies. -{%- macro display_name(elem_id) -%} -{%- if elem_id in sbdl and 'custom:title' in sbdl[elem_id] -%} -{{ sbdl[elem_id]['custom:title'] }} -{%- else -%} -{{ elem_id | replace('-', ' ') }} -{%- endif -%} -{%- endmacro -%} +| Component | Description | +|---|---| +| Container runtime | OCI-compatible container engine (e.g. Docker, Podman) | +| Gherkin test runner | Playwright with Gherkin step definitions | +| Integration test runner | BATS (Bash Automated Testing System) | +| CI platform | GitHub Actions | + +# Test methodology + +## Gherkin verification tests + +Gherkin scenarios follow the Given-When-Then structure. Each scenario exercises a specific capability of the devcontainer by automating interactions with the development environment. Tests are executed via Playwright inside a running devcontainer instance (GitHub Codespace or local). A scenario passes when all of its steps complete without error. + +## BATS integration tests +BATS tests verify tool presence, version alignment, and end-to-end workflows (compilation, analysis, formatting) by executing shell commands inside the devcontainer. Each test function runs in isolation. A BATS test passes when the test function exits with code 0. + +## Expected results + +> **Note**: Structured expected-result data per test is not yet available in the current toolchain. This section will be extended when test result rendering is implemented. Currently, a test is considered passed when it completes without error as described above. + +# Tests {%- set aspects = [] -%} {%- for elem_id, elem in sbdl.items() -%} {%- if elem.type == 'aspect' -%} @@ -59,22 +97,18 @@ amp-devcontainer is a set of [devcontainers](https://containers.dev/) tailored t {%- for req_id, req in sbdl.items() -%} {%- if req.type == 'requirement' and 'aspect' in req -%} {%- for asp_ref in req.aspect if asp_ref.identifier == aspect_id -%} -{%- set ns.has_requirements = true -%} +{%- set ns.has_requirements = true %} ### {{ reencode(display_name(req_id)) }} -{{ reencode(req.description) }} - {%- set ns_tests = namespace(has_tests=false) -%} -#### Verification tests - {%- for test_id, test_elem in sbdl.items() -%} {%- if test_elem.type == 'test' -%} {%- if 'requirement' in test_elem -%} {%- for req_ref in test_elem.requirement if req_ref.identifier == req_id -%} {%- set ns_tests.has_tests = true %} -- **{{ reencode(display_name(test_id)) }}**: {{ reencode(test_elem.description) }} +- {{ reencode(display_name(test_id)) }} {%- endfor -%} {%- endif -%} {%- endif -%} @@ -104,7 +138,7 @@ The following tests are not traced to any requirement: {%- for test_id, test_elem in sbdl.items() -%} {%- if test_elem.type == 'test' and 'requirement' not in test_elem -%} {%- set ns_untraced.has_untraced = true %} -- **{{ reencode(display_name(test_id)) }}**: {{ reencode(test_elem.description) }} +- {{ reencode(display_name(test_id)) }} {%- if 'tag' in test_elem %} *(tags: {{ test_elem.tag }})* {%- endif -%} {%- endif -%} {%- endfor -%} @@ -113,3 +147,7 @@ The following tests are not traced to any requirement: *All tests are traced to requirements.* {%- endif %} + +# Anomaly handling + +Test failures are tracked as issues in the project's issue tracker. Each failure is triaged, assigned a severity, and linked to the affected requirement(s). Resolution is verified by re-running the affected test(s). From 3344cbb30f60e853a868fa566a085f9a46c18330 Mon Sep 17 00:00:00 2001 From: Ron <45816308+rjaegers@users.noreply.github.com> Date: Thu, 26 Feb 2026 21:18:49 +0000 Subject: [PATCH 5/6] docs: rename to software test plan --- .github/workflows/wc-document-generation.yml | 20 +++++++++---------- .../requirements-traceability-matrix.md.j2 | 7 +++---- ...ication.md.j2 => software-test-plan.md.j2} | 12 +++++------ 3 files changed, 19 insertions(+), 20 deletions(-) rename docs/templates/{software-test-specification.md.j2 => software-test-plan.md.j2} (93%) diff --git a/.github/workflows/wc-document-generation.yml b/.github/workflows/wc-document-generation.yml index f0956fbe..6088e47e 100644 --- a/.github/workflows/wc-document-generation.yml +++ b/.github/workflows/wc-document-generation.yml @@ -29,26 +29,26 @@ jobs: set -Eeuo pipefail python docs/support/generate-sbdl.py --config requirements --gherkin test/cpp/features/*.feature sbdl -m template-fill --template docs/templates/software-requirements-specification.md.j2 output.sbdl > software-requirements-specification.md - - name: Generate C++ test specification and traceability documents + - name: Generate C++ test plan and traceability documents run: | set -Eeuo pipefail python docs/support/generate-sbdl.py --config test-specification \ --gherkin test/cpp/features/*.feature \ --bats test/cpp/integration-tests.bats test/base/integration-tests.bats \ --flavor "C++ (cpp)" \ - -o cpp-test-specification.sbdl - sbdl -m template-fill --template docs/templates/software-test-specification.md.j2 cpp-test-specification.sbdl > cpp-software-test-specification.md - sbdl -m template-fill --template docs/templates/requirements-traceability-matrix.md.j2 cpp-test-specification.sbdl > cpp-requirements-traceability-matrix.md - - name: Generate Rust test specification and traceability documents + -o cpp-test-plan.sbdl + sbdl -m template-fill --template docs/templates/software-test-plan.md.j2 cpp-test-plan.sbdl > cpp-software-test-plan.md + sbdl -m template-fill --template docs/templates/requirements-traceability-matrix.md.j2 cpp-test-plan.sbdl > cpp-requirements-traceability-matrix.md + - name: Generate Rust test plan and traceability documents run: | set -Eeuo pipefail python docs/support/generate-sbdl.py --config test-specification \ --gherkin test/cpp/features/*.feature \ --bats test/rust/integration-tests.bats test/base/integration-tests.bats \ --flavor Rust \ - -o rust-test-specification.sbdl - sbdl -m template-fill --template docs/templates/software-test-specification.md.j2 rust-test-specification.sbdl > rust-software-test-specification.md - sbdl -m template-fill --template docs/templates/requirements-traceability-matrix.md.j2 rust-test-specification.sbdl > rust-requirements-traceability-matrix.md + -o rust-test-plan.sbdl + sbdl -m template-fill --template docs/templates/software-test-plan.md.j2 rust-test-plan.sbdl > rust-software-test-plan.md + sbdl -m template-fill --template docs/templates/requirements-traceability-matrix.md.j2 rust-test-plan.sbdl > rust-requirements-traceability-matrix.md - uses: docker://pandoc/extra:3.7.0@sha256:a703d335fa237f8fc3303329d87e2555dca5187930da38bfa9010fa4e690933a with: args: >- @@ -58,7 +58,7 @@ jobs: with: args: >- --template eisvogel --listings --number-sections - --output cpp-software-test-specification.pdf cpp-software-test-specification.md + --output cpp-software-test-plan.pdf cpp-software-test-plan.md - uses: docker://pandoc/extra:3.7.0@sha256:a703d335fa237f8fc3303329d87e2555dca5187930da38bfa9010fa4e690933a with: args: >- @@ -68,7 +68,7 @@ jobs: with: args: >- --template eisvogel --listings --number-sections - --output rust-software-test-specification.pdf rust-software-test-specification.md + --output rust-software-test-plan.pdf rust-software-test-plan.md - uses: docker://pandoc/extra:3.7.0@sha256:a703d335fa237f8fc3303329d87e2555dca5187930da38bfa9010fa4e690933a with: args: >- diff --git a/docs/templates/requirements-traceability-matrix.md.j2 b/docs/templates/requirements-traceability-matrix.md.j2 index a8ef9061..3f55a816 100644 --- a/docs/templates/requirements-traceability-matrix.md.j2 +++ b/docs/templates/requirements-traceability-matrix.md.j2 @@ -23,9 +23,8 @@ This document provides a requirements traceability matrix (RTM) for amp-devconta ## Abstract -amp-devcontainer is a set of [devcontainers](https://containers.dev/) tailored towards modern, embedded, software development. This traceability matrix traces requirements from the software requirements specification to their corresponding verification tests (Gherkin scenarios and BATS integration tests). - -{%- macro reencode(text) -%} +amp-devcontainer is a set of [devcontainers](https://containers.dev/) tailored towards modern, embedded, software development. This traceability matrix traces requirements from the *Software Requirements Specification (SRS)* to their corresponding verification tests (Gherkin scenarios and BATS integration tests) as enumerated in the *Software Test Plan (STP)*. +{% macro reencode(text) -%} {{ text.encode('utf-8').decode('unicode_escape') }} {%- endmacro -%} @@ -43,7 +42,7 @@ amp-devcontainer is a set of [devcontainers](https://containers.dev/) tailored t {%- else -%} {{ (elem_id | replace('-', ' '))[:60] }} {%- endif -%} -{%- endmacro -%} +{%- endmacro %} # Traceability Matrix diff --git a/docs/templates/software-test-specification.md.j2 b/docs/templates/software-test-plan.md.j2 similarity index 93% rename from docs/templates/software-test-specification.md.j2 rename to docs/templates/software-test-plan.md.j2 index 84414b29..b86371d6 100644 --- a/docs/templates/software-test-specification.md.j2 +++ b/docs/templates/software-test-plan.md.j2 @@ -1,10 +1,10 @@ --- -title: "Software test specification for amp-devcontainer +title: "Software test plan for amp-devcontainer {%- if 'document-flavor' in sbdl %} ({{ sbdl['document-flavor']['custom:title'] }}){%- endif %}" author: ["@rjaegers"] colorlinks: true date: "October-2025" -keywords: [Software, Test, Specification, STS, amp-devcontainer] +keywords: [Software, Test, Plan, STP, amp-devcontainer] lang: "en" titlepage: true titlepage-color: "0B5ED7" @@ -30,12 +30,12 @@ toc-own-page: true ## Purpose -This document specifies the tests for amp-devcontainer +This document describes the test plan for amp-devcontainer {%- if 'document-flavor' in sbdl %} ({{ sbdl['document-flavor']['custom:title'] }} flavor){%- endif %}. -It enumerates all tests, describes the test methodology, and traces each test to the requirement(s) it verifies. +It defines the test strategy, methodology, and environment, and enumerates the tests that verify each requirement. -For the full requirement descriptions, refer to the *Software Requirements Specification* (SRS). -For the coverage overview, refer to the *Requirements Traceability Matrix* (RTM). +For the full requirement descriptions, refer to the *Software Requirements Specification (SRS)*. +For the coverage overview, refer to the *Requirements Traceability Matrix (RTM)*. ## Scope From 768089ee8f4b08b4a422ed618b914df0d95fed1b Mon Sep 17 00:00:00 2001 From: Ron <45816308+rjaegers@users.noreply.github.com> Date: Fri, 27 Feb 2026 08:28:31 +0000 Subject: [PATCH 6/6] chore: switch to stable requirement ids --- docs/support/bats_sbdl_converter.py | 23 ++++++++--- docs/support/gherkin_sbdl_converter.py | 18 ++++++++- test/cpp/features/compatibility.feature | 4 ++ test/cpp/features/compilation.feature | 4 ++ test/cpp/features/debugging.feature | 2 + test/cpp/features/maintainability.feature | 5 +++ test/cpp/features/security.feature | 3 ++ .../features/static-dynamic-analysis.feature | 5 +++ test/cpp/integration-tests.bats | 40 +++++++++---------- test/rust/integration-tests.bats | 18 ++++----- 10 files changed, 86 insertions(+), 36 deletions(-) diff --git a/docs/support/bats_sbdl_converter.py b/docs/support/bats_sbdl_converter.py index 448e1399..cf61d63f 100644 --- a/docs/support/bats_sbdl_converter.py +++ b/docs/support/bats_sbdl_converter.py @@ -114,10 +114,21 @@ def extract_from_bats_file(self, file_path: str, flavor: str = "") -> List[BatsT return tests + _REQ_TAG_PATTERN = re.compile(r'^REQ-', re.IGNORECASE) + + def _add_relation(self, relations: Dict[str, List[str]], elem_type: str, identifier: str): + """Add a relation entry, creating the type list if needed.""" + if elem_type not in relations: + relations[elem_type] = [] + if identifier not in relations[elem_type]: + relations[elem_type].append(identifier) + def _resolve_typed_relations(self, test: BatsTest) -> Dict[str, List[str]]: """Resolve typed relations from test tags using the tag map. - Groups resolved identifiers by their SBDL element type, so the + Tags matching the REQ-* pattern are treated as direct requirement + references. Other tags are resolved through the tag map, which + groups resolved identifiers by their SBDL element type so the correct relation keyword (e.g. 'aspect', 'requirement') is used in the SBDL output. @@ -129,6 +140,11 @@ def _resolve_typed_relations(self, test: BatsTest) -> Dict[str, List[str]]: """ relations: Dict[str, List[str]] = {} for tag in test.tags: + # Direct REQ-* tag: use as requirement identifier directly + if self._REQ_TAG_PATTERN.match(tag): + self._add_relation(relations, "requirement", to_slug(tag)) + continue + tag_lower = tag.lower() if tag_lower in self.requirement_tag_map: entry = self.requirement_tag_map[tag_lower] @@ -136,8 +152,5 @@ def _resolve_typed_relations(self, test: BatsTest) -> Dict[str, List[str]]: identifier, elem_type = entry else: identifier, elem_type = entry, "requirement" - if elem_type not in relations: - relations[elem_type] = [] - if identifier not in relations[elem_type]: - relations[elem_type].append(identifier) + self._add_relation(relations, elem_type, identifier) return relations diff --git a/docs/support/gherkin_sbdl_converter.py b/docs/support/gherkin_sbdl_converter.py index 5787d1f7..a77aaf3e 100644 --- a/docs/support/gherkin_sbdl_converter.py +++ b/docs/support/gherkin_sbdl_converter.py @@ -81,9 +81,12 @@ def write_gherkin_sbdl_elements(f, elements: List[SBDLElement]): ) # Write custom:title with original name for display purposes + # When a REQ tag is present, format as "REQ-ID: Original Name" original_name = element.metadata.get('original_name', '') if element.metadata else '' + req_id = element.metadata.get('req_id', '') if element.metadata else '' if original_name: - escaped_title = sbdl.SBDL_Parser.sanitize(original_name) + display_title = f"{req_id}: {original_name}" if req_id else original_name + escaped_title = sbdl.SBDL_Parser.sanitize(display_title) f.write(f'custom:title{tokens.declaration_attribute_assign}') f.write(f'{tokens.declaration_attribute_delimeter}{escaped_title}{tokens.declaration_attribute_delimeter} ') @@ -147,14 +150,25 @@ def _extract_gherkin_element(self, element_data: Dict, element_type: GherkinElem if not name: return None + # Check for @REQ-* tag to use as stable identifier + req_id = None + for tag in element_data.get('tags', []): + tag_name = tag.get('name', '') + if tag_name.upper().startswith('@REQ-'): + req_id = tag_name[1:] # Remove @ prefix, preserve case + break + + identifier = req_id if req_id else name + return SBDLElement( - identifier=name, + identifier=identifier, element_type=mapping.sbdl_type, description=element_data.get('description', ''), parent=parent_id, metadata={ 'gherkin_type': element_type.value, 'original_name': name, + 'req_id': req_id, 'line': element_data.get('location', {}).get('line'), }, ) diff --git a/test/cpp/features/compatibility.feature b/test/cpp/features/compatibility.feature index 65cdebd8..e0a7f8a2 100644 --- a/test/cpp/features/compatibility.feature +++ b/test/cpp/features/compatibility.feature @@ -4,11 +4,13 @@ Feature: Compatibility to ensure that my development environment works well with a variety of tools and systems, I want my development environment to be compatible with commonly used tools and systems. + @REQ-COMPAT-0001 Rule: Open Container Initiative (OCI) Image Specification amp-devcontainer images *SHALL* be compatible with the [OCI image specification](https://github.com/opencontainers/image-spec/blob/main/spec.md) To guarantee compatibility with container runtimes and container- and image tooling; amp-devcontainer should be compatible with the OCI image specification. + @REQ-COMPAT-0002 Rule: Host architecture amp-devcontainer *SHALL* be compatible with both the x86-64 (AMD64) *and* AArch64 (ARM64) host architectures. @@ -17,6 +19,7 @@ Feature: Compatibility - Increasing useability on a wide range of host machines, from PC hardware using the x86-64 architecture to Apple Silicon using the AArch64 architecture - Unlocking the power efficiency of the AArch64 architecture, potentially reducing energy consumption and cost for metered ci-systems + @REQ-COMPAT-0003 Rule: Integrated Development Environment (IDE) amp-devcontainer *SHOULD* be compatible with [VS Code](https://code.visualstudio.com/) *and* [GitHub Codespaces](https://github.com/features/codespaces). @@ -24,6 +27,7 @@ Feature: Compatibility Where minimal effort means: with the least amount of additional set-up, user intervention or configuration for all functionality that is provided by amp-devcontainer. Features and functions should work "out-of-the-box" without being overly opinionated. + @REQ-COMPAT-0004 Rule: GitHub Actions amp-devcontainer *SHOULD* support seamless integration with [GitHub Actions](https://github.com/features/actions) without additional configuration. diff --git a/test/cpp/features/compilation.feature b/test/cpp/features/compilation.feature index de0e3e3c..2a3069fb 100644 --- a/test/cpp/features/compilation.feature +++ b/test/cpp/features/compilation.feature @@ -4,6 +4,7 @@ Feature: Compilation to generate a working product, when using compiled languages, source code needs to be compiled into working software. + @REQ-COMP-0001 Rule: Compile for container host architecture and operating system amp-devcontainer *SHALL* be able to compile valid source code into a working executable targeting the container host architecture and operating system. @@ -21,6 +22,7 @@ Feature: Compilation When the selected target is built Then the output should contain "Build finished with exit code 0" + @REQ-COMP-0002 Rule: Compile for ARM Cortex target architecture amp-devcontainer *SHOULD* be able to compile valid source-code into a working ELF executable targeting the ARM Cortex architecture. @@ -28,6 +30,7 @@ Feature: Compilation is a primary function for amp-devcontainer. It enables building firmware for micro-controllers based on the ARM Cortex architecture. + @REQ-COMP-0003 Rule: Compile for Microsoft® Windows operating system amp-devcontainer *SHOULD* be able to compile valid source-code into a working executable targeting the Microsoft® Windows operating system. @@ -36,6 +39,7 @@ Feature: Compilation - Cross platform code is written and needs to be compiled and deployed - Executables needs to be deployed outside of container context to a host running the Microsoft® Windows operating system + @REQ-COMP-0004 Rule: Compilation cache amp-devcontainer *MAY* be able to cache the results of a compilation to speed-up subsequent compilations. diff --git a/test/cpp/features/debugging.feature b/test/cpp/features/debugging.feature index 58a64287..7ecfbb57 100644 --- a/test/cpp/features/debugging.feature +++ b/test/cpp/features/debugging.feature @@ -4,6 +4,7 @@ Feature: Debugging to efficiently identify and resolve issues in my code, I want to be able to debug my source code within the development environment. + @REQ-DBG-0001 Rule: Debugging support amp-devcontainer *SHALL* provide debugging support for the primary programming language(s) used within the container. @@ -12,6 +13,7 @@ Feature: Debugging This capability is essential for diagnosing complex problems, understanding code flow, and ensuring the correctness of software. By having integrated debugging tools, developers can streamline their workflow and reduce the time spent on troubleshooting and fixing bugs. + @REQ-DBG-0002 Rule: Upload firmware to micro-controller amp-devcontainer *MAY* provide tools to upload compiled firmware to a connected micro-controller. diff --git a/test/cpp/features/maintainability.feature b/test/cpp/features/maintainability.feature index aa384582..6dd96b0e 100644 --- a/test/cpp/features/maintainability.feature +++ b/test/cpp/features/maintainability.feature @@ -4,6 +4,7 @@ Feature: Maintainability to ensure that I have access to a stable and reliable development environment, I want my development environment to be maintainable over time. + @REQ-MAINT-0001 Rule: Tool and dependency updates amp-devcontainer *SHOULD* contain up-to-date tools and dependencies. @@ -11,6 +12,7 @@ Feature: Maintainability It also helps prevent issues related to deprecated or unsupported software versions, reducing maintenance overhead and improving overall developer productivity. Regular updates can also introduce new features and improvements that enhance the development experience. + @REQ-MAINT-0002 Rule: Automatic updates amp-devcontainer *SHOULD* provide support for automatic updates when consumed as a dependency. @@ -18,12 +20,14 @@ Feature: Maintainability This reduces the maintenance burden on users, as they do not need to manually track and apply updates. Automatic updates can also help ensure compatibility with other dependencies and tools, improving the overall stability and reliability of the development environment. + @REQ-MAINT-0003 Rule: Re-usable build system amp-devcontainer *SHOULD* provide re-usable building blocks to enable building, publishing and testing derived containers. Providing re-usable building blocks for building, publishing and testing derived containers reduces duplication, and ensures consistent application of practices. Derived containers (i.e. containers using amp-devcontainer as a base for further extension) should be able to build, push and test in the same way that amp-devcontainer does, without the need to duplicate the build system. + @REQ-MAINT-0004 Rule: Architectural decisions amp-devcontainer *SHOULD* document its architectural decisions. @@ -31,6 +35,7 @@ Feature: Maintainability This information can be valuable for future maintainers, as it helps them understand the reasoning behind certain implementations and can guide them in making informed decisions when modifying or extending the environment. Clear documentation of architectural decisions can also facilitate collaboration among team members and improve overall maintainability. + @REQ-MAINT-0005 Rule: Container image size amp-devcontainer *SHOULD* aim to keep its container image size as small as possible without compromising functionality. diff --git a/test/cpp/features/security.feature b/test/cpp/features/security.feature index 061a0053..de322709 100644 --- a/test/cpp/features/security.feature +++ b/test/cpp/features/security.feature @@ -4,6 +4,7 @@ Feature: Security to have control over the security posture of my development environment, I want to have controls in place to identify and mitigate supply-chain vulnerabilities. + @REQ-SEC-0001 Rule: Build provenance amp-devcontainer *SHALL* include build provenance as specified in [SLSA v1.0 Build L3](https://slsa.dev/spec/v1.0/levels). @@ -12,6 +13,7 @@ Feature: Security Consumers have some way of knowing what the expected provenance should look like for a given container image and then compare each container image's actual provenance to those expectations. Doing so prevents several classes of [supply chain threats](https://slsa.dev/spec/v1.0/threats). + @REQ-SEC-0002 Rule: Signing amp-devcontainer *SHALL* cryptographically sign its released container images. @@ -19,6 +21,7 @@ Feature: Security It enables consumers to verify that the container image hasn't been tampered with and that it indeed originates from the expected publisher. This helps mitigate several classes of [supply chain threats](https://slsa.dev/spec/v1.0/threats). + @REQ-SEC-0003 Rule: Software Bill of Materials (SBOM) amp-devcontainer *SHOULD* provide a Software Bill of Materials (SBOM) for its released container images. diff --git a/test/cpp/features/static-dynamic-analysis.feature b/test/cpp/features/static-dynamic-analysis.feature index 3254790a..2f955b47 100644 --- a/test/cpp/features/static-dynamic-analysis.feature +++ b/test/cpp/features/static-dynamic-analysis.feature @@ -4,6 +4,7 @@ Feature: Static and dynamic analysis to maintain consistent, high-quality and bug-free code, I want my source code to be statically and dynamically analyzed. + @REQ-SDA-0001 Rule: Code formatting amp-devcontainer *MAY* provide code formatting tools for the primary programming language(s) used within the container. @@ -18,6 +19,7 @@ Feature: Static and dynamic analysis And the active document is saved Then the contents of "clang-tools/unformatted.cpp" should match the contents of "clang-tools/formatted.cpp" + @REQ-SDA-0002 Rule: Static analysis amp-devcontainer *MAY* provide static analysis tools for the primary programming language(s) used within the container. @@ -25,6 +27,7 @@ Feature: Static and dynamic analysis These tools can analyze the code for common pitfalls, coding standards violations, and potential bugs, providing developers with valuable feedback early in the development process. By integrating static analysis tools into the development environment, developers can catch issues before they become more significant problems, streamlining the development workflow and improving overall code quality. + @REQ-SDA-0003 Rule: Coverage analysis amp-devcontainer *SHOULD* provide code coverage analysis tools for the primary programming language(s) used within the container. @@ -32,6 +35,7 @@ Feature: Static and dynamic analysis This information can help identify gaps in test coverage, ensuring that critical parts of the code are adequately tested. By integrating code coverage analysis tools into the development environment, developers can improve their test suites, leading to higher code quality and increased confidence in the software's correctness. + @REQ-SDA-0004 Rule: Mutation testing amp-devcontainer *MAY* provide mutation testing tools for the primary programming language(s) used within the container. @@ -39,6 +43,7 @@ Feature: Static and dynamic analysis This process helps identify gaps in test coverage and ensures that the tests are robust enough to catch potential issues in the code. By integrating mutation testing tools into the development environment, developers can improve their test suites, leading to higher code quality and increased confidence in the software's correctness. + @REQ-SDA-0005 Rule: Fuzz testing amp-devcontainer *MAY* provide fuzz testing tools for the primary programming language(s) used within the container. diff --git a/test/cpp/integration-tests.bats b/test/cpp/integration-tests.bats index 7ecfcdde..7b185b94 100644 --- a/test/cpp/integration-tests.bats +++ b/test/cpp/integration-tests.bats @@ -72,7 +72,7 @@ teardown() { done } -# bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system +# bats test_tags=compilation,REQ-COMP-0001 @test "valid code input should result in working executable using host compiler" { cmake --preset gcc cmake --build --preset gcc @@ -82,7 +82,7 @@ teardown() { assert_output "Hello World!" } -# bats test_tags=compilation,compile-for-arm-cortex-target-architecture +# bats test_tags=compilation,REQ-COMP-0002 @test "valid code input should result in elf executable using arm-none-eabi compiler" { cmake --preset gcc-arm-none-eabi cmake --build --preset gcc-arm-none-eabi @@ -92,7 +92,7 @@ teardown() { assert_output --partial "Machine: ARM" } -# bats test_tags=compilation,compile-for-microsoft-windows-operating-system +# bats test_tags=compilation,REQ-COMP-0003 @test "valid code input should result in Windows executable using clang-cl compiler" { install_win_sdk_when_ci_unset @@ -100,7 +100,7 @@ teardown() { cmake --build --preset clang-cl } -# bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system +# bats test_tags=compilation,REQ-COMP-0001 @test "compilation database should be generated on CMake configure" { cmake --preset gcc assert [ -e build/gcc/compile_commands.json ] @@ -109,25 +109,25 @@ teardown() { assert [ -e build/gcc-arm-none-eabi/compile_commands.json ] } -# bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system +# bats test_tags=compilation,REQ-COMP-0001 @test "invalid code input should result in failing build" { cmake --preset gcc run ! cmake --build --preset gcc-fail } -# bats test_tags=compilation,compilation-cache +# bats test_tags=compilation,REQ-COMP-0004 @test "using ccache as a compiler launcher should result in cached build using gcc compiler" { configure_and_build_with_ccache gcc } -# bats test_tags=compilation,compilation-cache +# bats test_tags=compilation,REQ-COMP-0004 @test "using ccache as a compiler launcher should result in cached build using clang-cl compiler" { install_win_sdk_when_ci_unset configure_and_build_with_ccache clang-cl } -# bats test_tags=static-and-dynamic-analysis,static-analysis +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0002 @test "running clang-tidy as part of the build should result in warning diagnostics" { cmake --preset clang @@ -136,7 +136,7 @@ teardown() { assert_output --partial "warning: use a trailing return type for this function" } -# bats test_tags=static-and-dynamic-analysis,static-analysis +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0002 @test "running include-what-you-use as part of the build should result in warning diagnostics" { cmake --preset clang @@ -145,14 +145,14 @@ teardown() { assert_output --partial "Warning: include-what-you-use reported diagnostics:" } -# bats test_tags=static-and-dynamic-analysis,code-formatting +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0001 @test "running clang-format should result in re-formatted code" { run clang-format clang-tools/unformatted.cpp assert_success assert_output "int main() {}" } -# bats test_tags=static-and-dynamic-analysis,coverage-analysis +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0003 @test "coverage information should be generated when running a testsuite" { cmake --preset coverage cmake --build --preset coverage @@ -166,7 +166,7 @@ teardown() { assert_output --partial "GCC Code Coverage Report" } -# bats test_tags=static-and-dynamic-analysis,fuzz-testing +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0005 @test "crashes should be detected when fuzzing an executable" { cmake --preset clang cmake --build --preset fuzzing @@ -176,7 +176,7 @@ teardown() { assert_output --partial "SUMMARY: libFuzzer: deadly signal" } -# bats test_tags=static-and-dynamic-analysis,mutation-testing +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0004 @test "a mutation score should be calculated when mutation testing a testsuite" { cmake --preset mutation cmake --build --preset mutation @@ -185,20 +185,20 @@ teardown() { assert_output --partial "[info] Mutation score:" } -# bats test_tags=static-and-dynamic-analysis,static-analysis +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0002 @test "clangd should be able to analyze source files" { run clangd --check=gcc/main.cpp assert_success assert_output --partial "All checks completed, 0 errors" } -# bats test_tags=static-and-dynamic-analysis,static-analysis +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0002 @test "clangd should start with a specified compile commands path" { run timeout 1s clangd --compile-commands-dir=/root/.amp refute_output --partial "Path specified by --compile-commands-dir does not exist. The argument will be ignored." } -# bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system +# bats test_tags=compilation,REQ-COMP-0001 @test "using lld as an alternative linker should result in working host executable" { cmake --preset gcc cmake --build --preset gcc-lld @@ -211,17 +211,17 @@ teardown() { assert_output "Hello World!" } -# bats test_tags=static-and-dynamic-analysis,static-analysis +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0002 @test "sanitizers should detect undefined or suspicious behavior in code compiled with gcc" { build_and_run_with_sanitizers gcc } -# bats test_tags=static-and-dynamic-analysis,static-analysis +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0002 @test "sanitizers should detect undefined or suspicious behavior in code compiled with clang" { build_and_run_with_sanitizers clang } -# bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system +# bats test_tags=compilation,REQ-COMP-0001 @test "using Conan as package manager should resolve external dependencies" { pushd package-managers/conan @@ -233,7 +233,7 @@ teardown() { popd } -# bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system +# bats test_tags=compilation,REQ-COMP-0001 @test "using CPM as package manager should resolve external dependencies" { cmake --preset cpm cmake --build --preset cpm diff --git a/test/rust/integration-tests.bats b/test/rust/integration-tests.bats index be5c59f8..daa7c85a 100644 --- a/test/rust/integration-tests.bats +++ b/test/rust/integration-tests.bats @@ -15,7 +15,7 @@ teardown() { popd } -# bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system +# bats test_tags=compilation,REQ-COMP-0001 @test "valid code input should result in working executable targeting the host architecture" { rustc --out-dir build rust/hello.rs @@ -24,7 +24,7 @@ teardown() { assert_output "Hello, world!" } -# bats test_tags=compilation,compile-for-arm-cortex-target-architecture +# bats test_tags=compilation,REQ-COMP-0002 @test "valid code input should result in working elf executable targeting the cortex-m architecture" { pushd cortex-m @@ -38,7 +38,7 @@ teardown() { popd } -# bats test_tags=compilation,compile-for-arm-cortex-target-architecture +# bats test_tags=compilation,REQ-COMP-0002 @test "valid code input should result in working elf executable targeting the cortex-mf architecture" { pushd cortex-mf @@ -52,7 +52,7 @@ teardown() { popd } -# bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system +# bats test_tags=compilation,REQ-COMP-0001 @test "using cargo run should result in working executable" { pushd cargo @@ -63,14 +63,14 @@ teardown() { popd } -# bats test_tags=compilation,compile-for-container-host-architecture-and-operating-system +# bats test_tags=compilation,REQ-COMP-0001 @test "invalid code input should result in failing build" { run rustc rust/fail.rs assert_failure assert_output --partial "error: " } -# bats test_tags=static-and-dynamic-analysis,static-analysis +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0002 @test "running clippy should result in warning diagnostics" { pushd clippy @@ -81,7 +81,7 @@ teardown() { popd } -# bats test_tags=static-and-dynamic-analysis,code-formatting +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0001 @test "running rustfmt should result in re-formatted code" { run rustfmt --color=never --check rust/unformatted.rs assert_failure @@ -95,7 +95,7 @@ teardown() { EOF } -# bats test_tags=static-and-dynamic-analysis,coverage-analysis +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0003 @test "coverage information should be generated when running a testsuite" { pushd test @@ -112,7 +112,7 @@ EOF popd } -# bats test_tags=static-and-dynamic-analysis,mutation-testing +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0004 @test "mutation testing a test executable should be supported" { pushd test