diff --git a/.github/workflows/wc-document-generation.yml b/.github/workflows/wc-document-generation.yml index 581e94a2..6088e47e 100644 --- a/.github/workflows/wc-document-generation.yml +++ b/.github/workflows/wc-document-generation.yml @@ -27,11 +27,53 @@ jobs: - name: Generate SRS document run: | set -Eeuo pipefail - python docs/support/gherkin-to-sbdl.py test/cpp/features/*.feature + python docs/support/generate-sbdl.py --config requirements --gherkin test/cpp/features/*.feature sbdl -m template-fill --template docs/templates/software-requirements-specification.md.j2 output.sbdl > software-requirements-specification.md + - name: Generate C++ test plan and traceability documents + run: | + set -Eeuo pipefail + python docs/support/generate-sbdl.py --config test-specification \ + --gherkin test/cpp/features/*.feature \ + --bats test/cpp/integration-tests.bats test/base/integration-tests.bats \ + --flavor "C++ (cpp)" \ + -o cpp-test-plan.sbdl + sbdl -m template-fill --template docs/templates/software-test-plan.md.j2 cpp-test-plan.sbdl > cpp-software-test-plan.md + sbdl -m template-fill --template docs/templates/requirements-traceability-matrix.md.j2 cpp-test-plan.sbdl > cpp-requirements-traceability-matrix.md + - name: Generate Rust test plan and traceability documents + run: | + set -Eeuo pipefail + python docs/support/generate-sbdl.py --config test-specification \ + --gherkin test/cpp/features/*.feature \ + --bats test/rust/integration-tests.bats test/base/integration-tests.bats \ + --flavor Rust \ + -o rust-test-plan.sbdl + sbdl -m template-fill --template docs/templates/software-test-plan.md.j2 rust-test-plan.sbdl > rust-software-test-plan.md + sbdl -m template-fill --template docs/templates/requirements-traceability-matrix.md.j2 rust-test-plan.sbdl > rust-requirements-traceability-matrix.md + - uses: docker://pandoc/extra:3.7.0@sha256:a703d335fa237f8fc3303329d87e2555dca5187930da38bfa9010fa4e690933a + with: + args: >- + --template eisvogel --listings --number-sections + --output software-requirements-specification.pdf software-requirements-specification.md + - uses: docker://pandoc/extra:3.7.0@sha256:a703d335fa237f8fc3303329d87e2555dca5187930da38bfa9010fa4e690933a + with: + args: >- + --template eisvogel --listings --number-sections + --output cpp-software-test-plan.pdf cpp-software-test-plan.md + - uses: docker://pandoc/extra:3.7.0@sha256:a703d335fa237f8fc3303329d87e2555dca5187930da38bfa9010fa4e690933a + with: + args: >- + --template eisvogel --listings --number-sections + --output cpp-requirements-traceability-matrix.pdf cpp-requirements-traceability-matrix.md + - uses: docker://pandoc/extra:3.7.0@sha256:a703d335fa237f8fc3303329d87e2555dca5187930da38bfa9010fa4e690933a + with: + args: >- + --template eisvogel --listings --number-sections + --output rust-software-test-plan.pdf rust-software-test-plan.md - uses: docker://pandoc/extra:3.7.0@sha256:a703d335fa237f8fc3303329d87e2555dca5187930da38bfa9010fa4e690933a with: - args: --template eisvogel --listings --number-sections --output software-requirements-specification.pdf software-requirements-specification.md + args: >- + --template eisvogel --listings --number-sections + --output rust-requirements-traceability-matrix.pdf rust-requirements-traceability-matrix.md - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: documents diff --git a/docs/support/bats_sbdl_converter.py b/docs/support/bats_sbdl_converter.py new file mode 100644 index 00000000..cf61d63f --- /dev/null +++ b/docs/support/bats_sbdl_converter.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""BATS test file to SBDL converter. + +Parses BATS test files and extracts @test definitions along with their +optional `# bats test_tags=` annotations, producing SBDL `test` elements +with traceability to requirements via tag-based mapping. +""" + +import re +from dataclasses import dataclass, field +from typing import Dict, List, Optional + +from gherkin_sbdl_converter import to_slug + +try: + import sbdl + + def sanitize_description(text: str) -> str: + return sbdl.SBDL_Parser.sanitize(text) + +except ImportError: + def sanitize_description(text: str) -> str: + return text.replace('"', '\\"') + + +@dataclass +class BatsTest: + """Represents a single BATS test case.""" + + name: str + identifier: str + tags: List[str] = field(default_factory=list) + file_path: str = "" + line_number: int = 0 + + def __post_init__(self): + if not self.identifier: + self.identifier = to_slug(self.name) + + +class BatsConverter: + """Converts BATS test files to SBDL test elements.""" + + # Regex patterns for BATS file parsing + _TAG_PATTERN = re.compile(r"^#\s*bats\s+test_tags\s*=\s*(.+)$", re.IGNORECASE) + _TEST_PATTERN = re.compile(r'^@test\s+"(.+?)"\s*\{', re.MULTILINE) + + def __init__(self, requirement_tag_map: Optional[Dict[str, str]] = None): + """Initialize the converter. + + Args: + requirement_tag_map: Optional mapping from tag names to requirement + identifiers (SBDL element IDs). When provided, tests with + matching tags will get a `requirement` relation in SBDL output. + """ + self.requirement_tag_map = requirement_tag_map or {} + + def extract_from_bats_file(self, file_path: str, flavor: str = "") -> List[BatsTest]: + """Extract all test definitions from a BATS file. + + Parses `@test "..."` blocks and any preceding `# bats test_tags=...` + comment lines. + + Args: + file_path: Path to the .bats file. + flavor: Optional identifier prefix for disambiguation + (e.g. 'cpp', 'rust', 'base'). + + Returns: + List of BatsTest objects. + """ + try: + with open(file_path, "r", encoding="utf-8") as f: + lines = f.readlines() + except (IOError, OSError) as e: + print(f"Error reading {file_path}: {e}") + return [] + + tests: List[BatsTest] = [] + pending_tags: List[str] = [] + + for line_number, line in enumerate(lines, start=1): + stripped = line.strip() + + # Check for tag annotation + tag_match = self._TAG_PATTERN.match(stripped) + if tag_match: + tags_str = tag_match.group(1) + pending_tags = [t.strip() for t in tags_str.split(",") if t.strip()] + continue + + # Check for test definition + test_match = self._TEST_PATTERN.match(stripped) + if test_match: + test_name = test_match.group(1) + base_id = to_slug(test_name) + identifier = f"{flavor}-{base_id}" if flavor else base_id + test = BatsTest( + name=test_name, + identifier=identifier, + tags=list(pending_tags), + file_path=file_path, + line_number=line_number, + ) + tests.append(test) + pending_tags = [] + print(f" Extracted BATS test: {test.identifier}") + continue + + # Reset pending tags if we hit a non-comment, non-empty line + # that isn't a test (so tags don't bleed across unrelated lines) + if stripped and not stripped.startswith("#"): + pending_tags = [] + + return tests + + _REQ_TAG_PATTERN = re.compile(r'^REQ-', re.IGNORECASE) + + def _add_relation(self, relations: Dict[str, List[str]], elem_type: str, identifier: str): + """Add a relation entry, creating the type list if needed.""" + if elem_type not in relations: + relations[elem_type] = [] + if identifier not in relations[elem_type]: + relations[elem_type].append(identifier) + + def _resolve_typed_relations(self, test: BatsTest) -> Dict[str, List[str]]: + """Resolve typed relations from test tags using the tag map. + + Tags matching the REQ-* pattern are treated as direct requirement + references. Other tags are resolved through the tag map, which + groups resolved identifiers by their SBDL element type so the + correct relation keyword (e.g. 'aspect', 'requirement') is used + in the SBDL output. + + Args: + test: A BatsTest with tags. + + Returns: + Dict mapping SBDL type names to lists of identifiers. + """ + relations: Dict[str, List[str]] = {} + for tag in test.tags: + # Direct REQ-* tag: use as requirement identifier directly + if self._REQ_TAG_PATTERN.match(tag): + self._add_relation(relations, "requirement", to_slug(tag)) + continue + + tag_lower = tag.lower() + if tag_lower in self.requirement_tag_map: + entry = self.requirement_tag_map[tag_lower] + if isinstance(entry, tuple): + identifier, elem_type = entry + else: + identifier, elem_type = entry, "requirement" + self._add_relation(relations, elem_type, identifier) + return relations diff --git a/docs/support/generate-sbdl.py b/docs/support/generate-sbdl.py new file mode 100644 index 00000000..1b693b6f --- /dev/null +++ b/docs/support/generate-sbdl.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Unified converter for generating SBDL from Gherkin feature files and BATS test files. + +Supports multiple output configurations: + - requirements: Feature/Rule hierarchy as requirements (for SRS) + - test-specification: Feature→aspect, Rule→requirement, Scenario→test + BATS→test (for test spec & traceability) + +Usage examples: + # Generate requirements SBDL (existing behavior) + python generate-sbdl.py --config requirements --gherkin test/cpp/features/*.feature + + # Generate test specification SBDL with both Gherkin scenarios and BATS tests + python generate-sbdl.py --config test-specification \\ + --gherkin test/cpp/features/*.feature \\ + --bats test/cpp/integration-tests.bats test/base/integration-tests.bats +""" + +import argparse +import os +import sys + +from gherkin_mapping_config import FEATURE_RULE_CONFIG, TEST_SPECIFICATION_CONFIG +from gherkin_sbdl_converter import GherkinConverter, to_slug, write_gherkin_sbdl_elements +from bats_sbdl_converter import BatsConverter + + +def main(): + configs = { + "requirements": FEATURE_RULE_CONFIG, + "test-specification": TEST_SPECIFICATION_CONFIG, + } + + parser = argparse.ArgumentParser( + description="Unified Gherkin + BATS to SBDL converter" + ) + parser.add_argument( + "--gherkin", + nargs="*", + default=[], + help="Paths to Gherkin feature files", + ) + parser.add_argument( + "--bats", + nargs="*", + default=[], + help="Paths to BATS test files", + ) + parser.add_argument( + "--output", + "-o", + default="output.sbdl", + help="Output SBDL file", + ) + parser.add_argument( + "--config", + choices=configs.keys(), + default="requirements", + help="Conversion configuration preset", + ) + parser.add_argument( + "--flavor", + default="", + help="Container flavor name for per-flavor document generation", + ) + + args = parser.parse_args() + + if not args.gherkin and not args.bats: + parser.error("At least one --gherkin or --bats file must be specified") + + config = configs[args.config] + gherkin_converter = GherkinConverter(config) + gherkin_elements = [] + + # Process Gherkin feature files + for feature_path in args.gherkin: + if os.path.isfile(feature_path): + print(f"Processing Gherkin: {feature_path}") + elements = gherkin_converter.extract_from_feature_file(feature_path) + gherkin_elements.extend(elements) + else: + print(f"File not found: {feature_path}", file=sys.stderr) + + # Build requirement tag map from Gherkin elements for BATS traceability. + # Maps lowercased sanitized identifiers and feature names to (identifier, element_type) + # tuples, enabling BATS tests tagged with e.g. "Compatibility" to trace to the + # corresponding Gherkin element with the correct SBDL relation type. + requirement_tag_map = {} + for elem in gherkin_elements: + if elem.metadata and elem.metadata.get("gherkin_type") in ("feature", "rule"): + entry = (elem.identifier, elem.element_type.value) + original = elem.metadata.get("original_name", "") + # Map the original feature/rule name (lowercased) to the SBDL identifier + type + if original: + requirement_tag_map[original.lower()] = entry + # Also map the slugified version for BATS tag matching + slug_original = to_slug(original) + if slug_original: + requirement_tag_map[slug_original] = entry + # Also map the SBDL identifier itself + requirement_tag_map[elem.identifier.lower()] = entry + + # Process BATS test files + bats_converter = BatsConverter(requirement_tag_map=requirement_tag_map) + bats_tests = [] + + for bats_path in args.bats: + if os.path.isfile(bats_path): + print(f"Processing BATS: {bats_path}") + flavor_prefix = to_slug(os.path.basename(os.path.dirname(os.path.abspath(bats_path)))) + tests = bats_converter.extract_from_bats_file(bats_path, flavor=flavor_prefix) + bats_tests.extend(tests) + else: + print(f"File not found: {bats_path}", file=sys.stderr) + + # Write combined SBDL output + _write_combined_sbdl(gherkin_elements, bats_converter, bats_tests, args.output, args.flavor) + + total = len(gherkin_elements) + len(bats_tests) + print(f"Extracted {total} elements ({len(gherkin_elements)} from Gherkin, {len(bats_tests)} from BATS) to {args.output}") + + +def _write_combined_sbdl(gherkin_elements, bats_converter, bats_tests, output_file, flavor=""): + """Write a single combined SBDL file from both Gherkin and BATS sources.""" + import sbdl as sbdl_lib + + with open(output_file, "w", encoding="utf-8") as f: + f.write("#!sbdl\n\n") + + # Write flavor marker element for per-flavor document generation + if flavor: + escaped_flavor = sbdl_lib.SBDL_Parser.sanitize(flavor) + f.write(f'document-flavor is definition {{ description is "{escaped_flavor}" custom:title is "{escaped_flavor}" }}\n\n') + + f.write("# Elements extracted from Gherkin feature files\n") + write_gherkin_sbdl_elements(f, gherkin_elements) + + if bats_tests: + f.write("\n# Elements extracted from BATS integration test files\n") + + for test in bats_tests: + escaped_desc = sbdl_lib.SBDL_Parser.sanitize(test.name) + identifier = test.identifier + + f.write(f'{identifier} is test {{ description is "{escaped_desc}" custom:title is "{escaped_desc}"') + + if test.tags: + tag_str = ",".join(test.tags) + f.write(f" tag is {tag_str}") + + resolved = bats_converter._resolve_typed_relations(test) + for rel_type, rel_ids in resolved.items(): + f.write(f" {rel_type} is {','.join(rel_ids)}") + + f.write(" }\n") + + +if __name__ == "__main__": + main() diff --git a/docs/support/gherkin-to-sbdl.py b/docs/support/gherkin-to-sbdl.py deleted file mode 100755 index 8c6d505e..00000000 --- a/docs/support/gherkin-to-sbdl.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import sys - -from gherkin_mapping_config import FEATURE_RULE_CONFIG, FEATURE_RULE_SCENARIO_CONFIG -from gherkin_sbdl_converter import GherkinConverter - -def main(): - configs = { - 'feature-rule': FEATURE_RULE_CONFIG, - 'feature-rule-scenario': FEATURE_RULE_SCENARIO_CONFIG - } - - parser = argparse.ArgumentParser(description='Configurable Gherkin to SBDL converter') - parser.add_argument('feature_files', nargs='+', help='Paths to feature files') - parser.add_argument('--output', '-o', default='output.sbdl', help='Output SBDL file') - parser.add_argument('--config', choices=configs.keys(), - default='feature-rule', help='Conversion configuration preset') - - args = parser.parse_args() - config = configs[args.config] - converter = GherkinConverter(config) - gherkin_elements = [] - - for feature_path in args.feature_files: - if os.path.isfile(feature_path): - print(f"Processing {feature_path}") - elements = converter.extract_from_feature_file(feature_path) - gherkin_elements.extend(elements) - else: - print(f"File not found: {feature_path}", file=sys.stderr) - - converter.write_sbdl_output(gherkin_elements, args.output) - print(f"Extracted {len(gherkin_elements)} elements to {args.output}") - -if __name__ == '__main__': - main() diff --git a/docs/support/gherkin_mapping_config.py b/docs/support/gherkin_mapping_config.py index 113e5e59..fde6ee1f 100644 --- a/docs/support/gherkin_mapping_config.py +++ b/docs/support/gherkin_mapping_config.py @@ -5,7 +5,7 @@ """ from dataclasses import dataclass -from typing import Dict, List, Optional, Union +from typing import List, Optional from enum import Enum class GherkinElementType(Enum): @@ -21,8 +21,6 @@ class SBDLElementType(Enum): """Supported SBDL element types for mapping.""" REQUIREMENT = "requirement" ASPECT = "aspect" - USECASE = "usecase" - DEFINITION = "definition" TEST = "test" @dataclass @@ -45,7 +43,7 @@ def get_mapping_for_type(self, gherkin_type: GherkinElementType) -> Optional[Hie # Predefined configurations for different use cases -# Current Configuration (Feature -> Rule mapping) +# Requirements Configuration (Feature -> Rule mapping as requirements) FEATURE_RULE_CONFIG = ConversionConfig( hierarchy_mappings=[ HierarchyMapping( @@ -59,8 +57,11 @@ def get_mapping_for_type(self, gherkin_type: GherkinElementType) -> Optional[Hie ] ) -# Extended Configuration (Feature -> Rule -> Scenario mapping) -FEATURE_RULE_SCENARIO_CONFIG = ConversionConfig( +# Test Specification Configuration (Feature -> Rule -> Scenario with tests) +# Used for generating test specification and traceability documents. +# Features become aspects (system areas), Rules become requirements, +# and Scenarios become test elements traced to their parent requirements. +TEST_SPECIFICATION_CONFIG = ConversionConfig( hierarchy_mappings=[ HierarchyMapping( gherkin_type=GherkinElementType.FEATURE, @@ -76,31 +77,3 @@ def get_mapping_for_type(self, gherkin_type: GherkinElementType) -> Optional[Hie ) ] ) - -# Flat Configuration (All as requirements at same level) -FLAT_CONFIG = ConversionConfig( - hierarchy_mappings=[ - HierarchyMapping( - gherkin_type=GherkinElementType.FEATURE, - sbdl_type=SBDLElementType.REQUIREMENT - ), - HierarchyMapping( - gherkin_type=GherkinElementType.RULE, - sbdl_type=SBDLElementType.REQUIREMENT - ) - ] -) - -# Use Case Focused Configuration -USECASE_CONFIG = ConversionConfig( - hierarchy_mappings=[ - HierarchyMapping( - gherkin_type=GherkinElementType.FEATURE, - sbdl_type=SBDLElementType.USECASE - ), - HierarchyMapping( - gherkin_type=GherkinElementType.SCENARIO, - sbdl_type=SBDLElementType.USECASE - ) - ] -) diff --git a/docs/support/gherkin_sbdl_converter.py b/docs/support/gherkin_sbdl_converter.py index 888a8c6d..a77aaf3e 100644 --- a/docs/support/gherkin_sbdl_converter.py +++ b/docs/support/gherkin_sbdl_converter.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from typing import List, Dict, Optional from textwrap import dedent +import re import sys from gherkin.parser import Parser @@ -16,6 +17,22 @@ SBDLElementType, ) + +def to_slug(name: str) -> str: + """Convert a human-readable name to a hyphenated slug identifier. + + Lowercases the name and replaces any sequence of non-alphanumeric + characters with a single hyphen. + + Args: + name: The original name. + + Returns: + Hyphenated, lowercase slug suitable as an SBDL identifier. + """ + slug = re.sub(r'[^a-z0-9]+', '-', name.lower()) + return slug.strip('-') + @dataclass class SBDLElement: """A generalized SBDL element that can represent any type.""" @@ -34,9 +51,57 @@ def __post_init__(self): self.metadata = {} def _make_sbdl_identifier(self, name: str) -> str: - return sbdl.SBDL_Parser.sanitize_identifier( - name.replace(" ", "_").replace("-", "_") - ).strip("_") + """Convert a name to a hyphenated slug identifier.""" + return to_slug(name) + +def write_gherkin_sbdl_elements(f, elements: List[SBDLElement]): + """Write a list of Gherkin-derived SBDL elements to an open file handle. + + Handles description, custom:title, and parent/cross-type relations + for each element. + + Args: + f: Writable file handle. + elements: List of SBDLElement objects to write. + """ + tokens = sbdl.SBDL_Parser.Tokens + attrs = sbdl.SBDL_Parser.Attributes + + # Build a lookup from identifier to element type for cross-type relation handling + element_types = {e.identifier: e.element_type for e in elements} + + for element in elements: + escaped_desc = sbdl.SBDL_Parser.sanitize(element.description) + sbdl_type = element.element_type.value + f.write(f"{element.identifier} {tokens.declaration} {sbdl_type} ") + f.write(f"{tokens.declaration_group_delimeters[0]} ") + f.write(f"{attrs.description}{tokens.declaration_attribute_assign}") + f.write( + f"{tokens.declaration_attribute_delimeter}{escaped_desc}{tokens.declaration_attribute_delimeter} " + ) + + # Write custom:title with original name for display purposes + # When a REQ tag is present, format as "REQ-ID: Original Name" + original_name = element.metadata.get('original_name', '') if element.metadata else '' + req_id = element.metadata.get('req_id', '') if element.metadata else '' + if original_name: + display_title = f"{req_id}: {original_name}" if req_id else original_name + escaped_title = sbdl.SBDL_Parser.sanitize(display_title) + f.write(f'custom:title{tokens.declaration_attribute_assign}') + f.write(f'{tokens.declaration_attribute_delimeter}{escaped_title}{tokens.declaration_attribute_delimeter} ') + + if element.parent: + parent_type = element_types.get(element.parent) + if parent_type and parent_type != element.element_type: + # Cross-type relation: use the parent's type name as relation + # e.g. test -> requirement becomes "requirement is " + # e.g. requirement -> aspect becomes "aspect is " + f.write(f"{parent_type.value}{tokens.declaration_attribute_assign}{element.parent} ") + else: + f.write(f"{attrs.parent}{tokens.declaration_attribute_assign}{element.parent} ") + + f.write(f"{tokens.declaration_group_delimeters[1]}\n") + class GherkinConverter: """Converts Gherkin files to SBDL using configurable hierarchy mappings.""" @@ -69,26 +134,10 @@ def extract_from_feature_file(self, file_path: str) -> List[SBDLElement]: return [] def write_sbdl_output(self, elements: List[SBDLElement], output_file: str): - tokens = sbdl.SBDL_Parser.Tokens - attrs = sbdl.SBDL_Parser.Attributes - + """Write extracted Gherkin elements as a standalone SBDL file.""" with open(output_file, 'w', encoding='utf-8') as f: f.write("#!sbdl\n") - - for element in elements: - escaped_desc = sbdl.SBDL_Parser.sanitize(element.description) - sbdl_type = element.element_type.value - f.write(f"{element.identifier} {tokens.declaration} {sbdl_type} ") - f.write(f"{tokens.declaration_group_delimeters[0]} ") - f.write(f"{attrs.description}{tokens.declaration_attribute_assign}") - f.write( - f"{tokens.declaration_attribute_delimeter}{escaped_desc}{tokens.declaration_attribute_delimeter} " - ) - - if element.parent: - f.write(f"{attrs.parent}{tokens.declaration_attribute_assign}{element.parent} ") - - f.write(f"{tokens.declaration_group_delimeters[1]}\n") + write_gherkin_sbdl_elements(f, elements) def _extract_gherkin_element(self, element_data: Dict, element_type: GherkinElementType, parent_id: Optional[str]) -> Optional[SBDLElement]: mapping = self.config.get_mapping_for_type(element_type) @@ -101,14 +150,25 @@ def _extract_gherkin_element(self, element_data: Dict, element_type: GherkinElem if not name: return None + # Check for @REQ-* tag to use as stable identifier + req_id = None + for tag in element_data.get('tags', []): + tag_name = tag.get('name', '') + if tag_name.upper().startswith('@REQ-'): + req_id = tag_name[1:] # Remove @ prefix, preserve case + break + + identifier = req_id if req_id else name + return SBDLElement( - identifier=name, + identifier=identifier, element_type=mapping.sbdl_type, description=element_data.get('description', ''), parent=parent_id, metadata={ 'gherkin_type': element_type.value, 'original_name': name, + 'req_id': req_id, 'line': element_data.get('location', {}).get('line'), }, ) diff --git a/docs/templates/requirements-traceability-matrix.md.j2 b/docs/templates/requirements-traceability-matrix.md.j2 new file mode 100644 index 00000000..3f55a816 --- /dev/null +++ b/docs/templates/requirements-traceability-matrix.md.j2 @@ -0,0 +1,126 @@ +--- +title: "Requirements traceability matrix for amp-devcontainer + {%- if 'document-flavor' in sbdl %} ({{ sbdl['document-flavor']['custom:title'] }}){%- endif %}" +author: ["@rjaegers"] +colorlinks: true +date: "October-2025" +keywords: [Traceability, Requirements, RTM, amp-devcontainer] +lang: "en" +titlepage: true +titlepage-color: "0B5ED7" +titlepage-text-color: "FFFFFF" +titlepage-rule-color: "FFFFFF" +titlepage-rule-height: 2 +toc: true +toc-own-page: true +... + +# Introduction + +## Purpose + +This document provides a requirements traceability matrix (RTM) for amp-devcontainer. It maps each requirement to its associated verification tests, providing evidence of test coverage across the system. + +## Abstract + +amp-devcontainer is a set of [devcontainers](https://containers.dev/) tailored towards modern, embedded, software development. This traceability matrix traces requirements from the *Software Requirements Specification (SRS)* to their corresponding verification tests (Gherkin scenarios and BATS integration tests) as enumerated in the *Software Test Plan (STP)*. +{% macro reencode(text) -%} +{{ text.encode('utf-8').decode('unicode_escape') }} +{%- endmacro -%} + +{%- macro display_name(elem_id) -%} +{%- if elem_id in sbdl and 'custom:title' in sbdl[elem_id] -%} +{{ sbdl[elem_id]['custom:title'] }} +{%- else -%} +{{ elem_id | replace('-', ' ') }} +{%- endif -%} +{%- endmacro -%} + +{%- macro display_name_short(elem_id) -%} +{%- if elem_id in sbdl and 'custom:title' in sbdl[elem_id] -%} +{{ sbdl[elem_id]['custom:title'][:60] }} +{%- else -%} +{{ (elem_id | replace('-', ' '))[:60] }} +{%- endif -%} +{%- endmacro %} + +# Traceability Matrix + +{%- set aspects = [] -%} +{%- for elem_id, elem in sbdl.items() -%} +{%- if elem.type == 'aspect' -%} +{%- set _ = aspects.append((elem_id, elem)) -%} +{%- endif -%} +{%- endfor -%} + +{%- for aspect_id, aspect in aspects %} + +## {{ reencode(display_name(aspect_id)) }} + +| Requirement | Test | Status | +|---|---|---| + +{%- for req_id, req in sbdl.items() -%} +{%- if req.type == 'requirement' and 'aspect' in req -%} +{%- for asp_ref in req.aspect if asp_ref.identifier == aspect_id -%} + +{%- set ns_tests = namespace(test_list=[]) -%} + +{%- for test_id, test_elem in sbdl.items() -%} +{%- if test_elem.type == 'test' -%} +{%- if 'requirement' in test_elem -%} +{%- for req_ref in test_elem.requirement if req_ref.identifier == req_id -%} +{%- set _ = ns_tests.test_list.append(test_id) -%} +{%- endfor -%} +{%- endif -%} +{%- endif -%} +{%- endfor -%} + +{%- if ns_tests.test_list -%} +{%- for test_id in ns_tests.test_list %} +| {{ reencode(display_name_short(req_id)) }} | {{ reencode(display_name_short(test_id)) }} | Traced | +{%- endfor -%} +{%- else %} +| {{ reencode(display_name_short(req_id)) }} | — | **Not covered** | +{%- endif -%} + +{%- endfor -%} +{%- endif -%} +{%- endfor -%} + +{%- endfor %} + +# Coverage Summary + +{%- set ns_summary = namespace(total_reqs=0, covered_reqs=0, total_tests=0) -%} + +{%- for elem_id, elem in sbdl.items() -%} +{%- if elem.type == 'requirement' -%} +{%- set ns_summary.total_reqs = ns_summary.total_reqs + 1 -%} + +{%- set ns_covered = namespace(is_covered=false) -%} +{%- for test_id, test_elem in sbdl.items() -%} +{%- if test_elem.type == 'test' and 'requirement' in test_elem -%} +{%- for req_ref in test_elem.requirement if req_ref.identifier == elem_id -%} +{%- set ns_covered.is_covered = true -%} +{%- endfor -%} +{%- endif -%} +{%- endfor -%} + +{%- if ns_covered.is_covered -%} +{%- set ns_summary.covered_reqs = ns_summary.covered_reqs + 1 -%} +{%- endif -%} + +{%- endif -%} + +{%- if elem.type == 'test' -%} +{%- set ns_summary.total_tests = ns_summary.total_tests + 1 -%} +{%- endif -%} +{%- endfor %} + +| Metric | Value | +|---|---| +| Total requirements | {{ ns_summary.total_reqs }} | +| Requirements with tests | {{ ns_summary.covered_reqs }} | +| Requirements without tests | {{ ns_summary.total_reqs - ns_summary.covered_reqs }} | +| Total tests | {{ ns_summary.total_tests }} | diff --git a/docs/templates/software-requirements-specification.md.j2 b/docs/templates/software-requirements-specification.md.j2 index dda4cbeb..5823b2b9 100644 --- a/docs/templates/software-requirements-specification.md.j2 +++ b/docs/templates/software-requirements-specification.md.j2 @@ -46,14 +46,18 @@ The containers may be used both for local development and continuous integration {{ text.encode('utf-8').decode('unicode_escape') }} {%- endmacro -%} -{%- macro sbdl_id_to_header(text) -%} -{{ text.replace('_', ' ') }} +{%- macro display_name(elem_id) -%} +{%- if elem_id in sbdl and 'custom:title' in sbdl[elem_id] -%} +{{ sbdl[elem_id]['custom:title'] }} +{%- else -%} +{{ elem_id | replace('-', ' ') }} +{%- endif -%} {%- endmacro -%} {%- for req_id, requirement in sbdl.items() %} {%- if requirement.type == 'requirement' and 'parent' not in requirement %} -## {{ reencode(sbdl_id_to_header(req_id)) }} +## {{ reencode(display_name(req_id)) }} {{ reencode(requirement.description) }} @@ -61,7 +65,7 @@ The containers may be used both for local development and continuous integration {%- for child in requirement.child %} {%- set child_req = sbdl[child.identifier] %} -### {{ reencode(sbdl_id_to_header(child.identifier)) }} +### {{ reencode(display_name(child.identifier)) }} {{ reencode(child_req.description) }} diff --git a/docs/templates/software-test-plan.md.j2 b/docs/templates/software-test-plan.md.j2 new file mode 100644 index 00000000..b86371d6 --- /dev/null +++ b/docs/templates/software-test-plan.md.j2 @@ -0,0 +1,153 @@ +--- +title: "Software test plan for amp-devcontainer + {%- if 'document-flavor' in sbdl %} ({{ sbdl['document-flavor']['custom:title'] }}){%- endif %}" +author: ["@rjaegers"] +colorlinks: true +date: "October-2025" +keywords: [Software, Test, Plan, STP, amp-devcontainer] +lang: "en" +titlepage: true +titlepage-color: "0B5ED7" +titlepage-text-color: "FFFFFF" +titlepage-rule-color: "FFFFFF" +titlepage-rule-height: 2 +toc: true +toc-own-page: true +... +{% macro reencode(text) -%} +{{ text.encode('utf-8').decode('unicode_escape') }} +{%- endmacro -%} + +{%- macro display_name(elem_id) -%} +{%- if elem_id in sbdl and 'custom:title' in sbdl[elem_id] -%} +{{ sbdl[elem_id]['custom:title'] }} +{%- else -%} +{{ elem_id | replace('-', ' ') }} +{%- endif -%} +{%- endmacro %} + +# Introduction + +## Purpose + +This document describes the test plan for amp-devcontainer +{%- if 'document-flavor' in sbdl %} ({{ sbdl['document-flavor']['custom:title'] }} flavor){%- endif %}. +It defines the test strategy, methodology, and environment, and enumerates the tests that verify each requirement. + +For the full requirement descriptions, refer to the *Software Requirements Specification (SRS)*. +For the coverage overview, refer to the *Requirements Traceability Matrix (RTM)*. + +## Scope + +This document covers the following types of tests: + +- **Gherkin verification tests**: Scenario-based tests defined in Gherkin feature files and executed with Playwright. These tests verify behavioral requirements at the system level. +- **BATS integration tests**: Shell-based integration tests defined in BATS (Bash Automated Testing System) files. These tests verify tool availability, version alignment, and end-to-end compilation and analysis workflows. + +## Definitions of key words + +The key words *MUST*, *MUST NOT*, *REQUIRED*, *SHALL*, *SHALL NOT*, *SHOULD*, *SHOULD NOT*, *RECOMMENDED*, *MAY*, and *OPTIONAL* in this document are to be interpreted as described in [RFC 2119](https://www.rfc-editor.org/rfc/rfc2119). + +## Referenced documents + +| Document | Description | +|---|---| +| Software Requirements Specification (SRS) | Defines the requirements verified by the tests in this document | +| Requirements Traceability Matrix (RTM) | Maps requirements to tests and provides coverage metrics | + +# Test environment + +Tests are executed in a container built from the amp-devcontainer image for the applicable flavor. The container provides all necessary compilers, tools, and dependencies. + +| Component | Description | +|---|---| +| Container runtime | OCI-compatible container engine (e.g. Docker, Podman) | +| Gherkin test runner | Playwright with Gherkin step definitions | +| Integration test runner | BATS (Bash Automated Testing System) | +| CI platform | GitHub Actions | + +# Test methodology + +## Gherkin verification tests + +Gherkin scenarios follow the Given-When-Then structure. Each scenario exercises a specific capability of the devcontainer by automating interactions with the development environment. Tests are executed via Playwright inside a running devcontainer instance (GitHub Codespace or local). A scenario passes when all of its steps complete without error. + +## BATS integration tests + +BATS tests verify tool presence, version alignment, and end-to-end workflows (compilation, analysis, formatting) by executing shell commands inside the devcontainer. Each test function runs in isolation. A BATS test passes when the test function exits with code 0. + +## Expected results + +> **Note**: Structured expected-result data per test is not yet available in the current toolchain. This section will be extended when test result rendering is implemented. Currently, a test is considered passed when it completes without error as described above. + +# Tests +{%- set aspects = [] -%} +{%- for elem_id, elem in sbdl.items() -%} +{%- if elem.type == 'aspect' -%} +{%- set _ = aspects.append((elem_id, elem)) -%} +{%- endif -%} +{%- endfor -%} + +{%- for aspect_id, aspect in aspects %} + +## {{ reencode(display_name(aspect_id)) }} + +{%- set ns = namespace(has_requirements=false) -%} + +{%- for req_id, req in sbdl.items() -%} +{%- if req.type == 'requirement' and 'aspect' in req -%} +{%- for asp_ref in req.aspect if asp_ref.identifier == aspect_id -%} +{%- set ns.has_requirements = true %} + +### {{ reencode(display_name(req_id)) }} + +{%- set ns_tests = namespace(has_tests=false) -%} + +{%- for test_id, test_elem in sbdl.items() -%} +{%- if test_elem.type == 'test' -%} +{%- if 'requirement' in test_elem -%} +{%- for req_ref in test_elem.requirement if req_ref.identifier == req_id -%} +{%- set ns_tests.has_tests = true %} +- {{ reencode(display_name(test_id)) }} +{%- endfor -%} +{%- endif -%} +{%- endif -%} +{%- endfor -%} + +{%- if not ns_tests.has_tests %} +- *No tests traced to this requirement.* +{%- endif -%} + +{%- endfor -%} +{%- endif -%} +{%- endfor -%} + +{%- if not ns.has_requirements %} + +*No requirements traced to this area.* +{%- endif -%} + +{%- endfor %} + +## Untraced tests + +The following tests are not traced to any requirement: + +{%- set ns_untraced = namespace(has_untraced=false) -%} + +{%- for test_id, test_elem in sbdl.items() -%} +{%- if test_elem.type == 'test' and 'requirement' not in test_elem -%} +{%- set ns_untraced.has_untraced = true %} +- {{ reencode(display_name(test_id)) }} +{%- if 'tag' in test_elem %} *(tags: {{ test_elem.tag }})* {%- endif -%} +{%- endif -%} +{%- endfor -%} + +{%- if not ns_untraced.has_untraced %} + +*All tests are traced to requirements.* +{%- endif %} + +# Anomaly handling + +Test failures are tracked as issues in the project's issue tracker. Each failure is triaged, assigned a severity, and linked to the affected requirement(s). Resolution is verified by re-running the affected test(s). diff --git a/test/base/integration-tests.bats b/test/base/integration-tests.bats index 39b82d41..0d482420 100644 --- a/test/base/integration-tests.bats +++ b/test/base/integration-tests.bats @@ -7,6 +7,7 @@ setup() { load '/usr/local/bats-assert/load' } +# bats test_tags=security @test "cisco umbrella root certificate is included in system certificate store" { run openssl verify -CAfile /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/cisco-umbrella-root.pem assert_success diff --git a/test/cpp/features/compatibility.feature b/test/cpp/features/compatibility.feature index 65cdebd8..e0a7f8a2 100644 --- a/test/cpp/features/compatibility.feature +++ b/test/cpp/features/compatibility.feature @@ -4,11 +4,13 @@ Feature: Compatibility to ensure that my development environment works well with a variety of tools and systems, I want my development environment to be compatible with commonly used tools and systems. + @REQ-COMPAT-0001 Rule: Open Container Initiative (OCI) Image Specification amp-devcontainer images *SHALL* be compatible with the [OCI image specification](https://github.com/opencontainers/image-spec/blob/main/spec.md) To guarantee compatibility with container runtimes and container- and image tooling; amp-devcontainer should be compatible with the OCI image specification. + @REQ-COMPAT-0002 Rule: Host architecture amp-devcontainer *SHALL* be compatible with both the x86-64 (AMD64) *and* AArch64 (ARM64) host architectures. @@ -17,6 +19,7 @@ Feature: Compatibility - Increasing useability on a wide range of host machines, from PC hardware using the x86-64 architecture to Apple Silicon using the AArch64 architecture - Unlocking the power efficiency of the AArch64 architecture, potentially reducing energy consumption and cost for metered ci-systems + @REQ-COMPAT-0003 Rule: Integrated Development Environment (IDE) amp-devcontainer *SHOULD* be compatible with [VS Code](https://code.visualstudio.com/) *and* [GitHub Codespaces](https://github.com/features/codespaces). @@ -24,6 +27,7 @@ Feature: Compatibility Where minimal effort means: with the least amount of additional set-up, user intervention or configuration for all functionality that is provided by amp-devcontainer. Features and functions should work "out-of-the-box" without being overly opinionated. + @REQ-COMPAT-0004 Rule: GitHub Actions amp-devcontainer *SHOULD* support seamless integration with [GitHub Actions](https://github.com/features/actions) without additional configuration. diff --git a/test/cpp/features/compilation.feature b/test/cpp/features/compilation.feature index de0e3e3c..2a3069fb 100644 --- a/test/cpp/features/compilation.feature +++ b/test/cpp/features/compilation.feature @@ -4,6 +4,7 @@ Feature: Compilation to generate a working product, when using compiled languages, source code needs to be compiled into working software. + @REQ-COMP-0001 Rule: Compile for container host architecture and operating system amp-devcontainer *SHALL* be able to compile valid source code into a working executable targeting the container host architecture and operating system. @@ -21,6 +22,7 @@ Feature: Compilation When the selected target is built Then the output should contain "Build finished with exit code 0" + @REQ-COMP-0002 Rule: Compile for ARM Cortex target architecture amp-devcontainer *SHOULD* be able to compile valid source-code into a working ELF executable targeting the ARM Cortex architecture. @@ -28,6 +30,7 @@ Feature: Compilation is a primary function for amp-devcontainer. It enables building firmware for micro-controllers based on the ARM Cortex architecture. + @REQ-COMP-0003 Rule: Compile for Microsoft® Windows operating system amp-devcontainer *SHOULD* be able to compile valid source-code into a working executable targeting the Microsoft® Windows operating system. @@ -36,6 +39,7 @@ Feature: Compilation - Cross platform code is written and needs to be compiled and deployed - Executables needs to be deployed outside of container context to a host running the Microsoft® Windows operating system + @REQ-COMP-0004 Rule: Compilation cache amp-devcontainer *MAY* be able to cache the results of a compilation to speed-up subsequent compilations. diff --git a/test/cpp/features/debugging.feature b/test/cpp/features/debugging.feature index 58a64287..7ecfbb57 100644 --- a/test/cpp/features/debugging.feature +++ b/test/cpp/features/debugging.feature @@ -4,6 +4,7 @@ Feature: Debugging to efficiently identify and resolve issues in my code, I want to be able to debug my source code within the development environment. + @REQ-DBG-0001 Rule: Debugging support amp-devcontainer *SHALL* provide debugging support for the primary programming language(s) used within the container. @@ -12,6 +13,7 @@ Feature: Debugging This capability is essential for diagnosing complex problems, understanding code flow, and ensuring the correctness of software. By having integrated debugging tools, developers can streamline their workflow and reduce the time spent on troubleshooting and fixing bugs. + @REQ-DBG-0002 Rule: Upload firmware to micro-controller amp-devcontainer *MAY* provide tools to upload compiled firmware to a connected micro-controller. diff --git a/test/cpp/features/maintainability.feature b/test/cpp/features/maintainability.feature index aa384582..6dd96b0e 100644 --- a/test/cpp/features/maintainability.feature +++ b/test/cpp/features/maintainability.feature @@ -4,6 +4,7 @@ Feature: Maintainability to ensure that I have access to a stable and reliable development environment, I want my development environment to be maintainable over time. + @REQ-MAINT-0001 Rule: Tool and dependency updates amp-devcontainer *SHOULD* contain up-to-date tools and dependencies. @@ -11,6 +12,7 @@ Feature: Maintainability It also helps prevent issues related to deprecated or unsupported software versions, reducing maintenance overhead and improving overall developer productivity. Regular updates can also introduce new features and improvements that enhance the development experience. + @REQ-MAINT-0002 Rule: Automatic updates amp-devcontainer *SHOULD* provide support for automatic updates when consumed as a dependency. @@ -18,12 +20,14 @@ Feature: Maintainability This reduces the maintenance burden on users, as they do not need to manually track and apply updates. Automatic updates can also help ensure compatibility with other dependencies and tools, improving the overall stability and reliability of the development environment. + @REQ-MAINT-0003 Rule: Re-usable build system amp-devcontainer *SHOULD* provide re-usable building blocks to enable building, publishing and testing derived containers. Providing re-usable building blocks for building, publishing and testing derived containers reduces duplication, and ensures consistent application of practices. Derived containers (i.e. containers using amp-devcontainer as a base for further extension) should be able to build, push and test in the same way that amp-devcontainer does, without the need to duplicate the build system. + @REQ-MAINT-0004 Rule: Architectural decisions amp-devcontainer *SHOULD* document its architectural decisions. @@ -31,6 +35,7 @@ Feature: Maintainability This information can be valuable for future maintainers, as it helps them understand the reasoning behind certain implementations and can guide them in making informed decisions when modifying or extending the environment. Clear documentation of architectural decisions can also facilitate collaboration among team members and improve overall maintainability. + @REQ-MAINT-0005 Rule: Container image size amp-devcontainer *SHOULD* aim to keep its container image size as small as possible without compromising functionality. diff --git a/test/cpp/features/security.feature b/test/cpp/features/security.feature index 061a0053..de322709 100644 --- a/test/cpp/features/security.feature +++ b/test/cpp/features/security.feature @@ -4,6 +4,7 @@ Feature: Security to have control over the security posture of my development environment, I want to have controls in place to identify and mitigate supply-chain vulnerabilities. + @REQ-SEC-0001 Rule: Build provenance amp-devcontainer *SHALL* include build provenance as specified in [SLSA v1.0 Build L3](https://slsa.dev/spec/v1.0/levels). @@ -12,6 +13,7 @@ Feature: Security Consumers have some way of knowing what the expected provenance should look like for a given container image and then compare each container image's actual provenance to those expectations. Doing so prevents several classes of [supply chain threats](https://slsa.dev/spec/v1.0/threats). + @REQ-SEC-0002 Rule: Signing amp-devcontainer *SHALL* cryptographically sign its released container images. @@ -19,6 +21,7 @@ Feature: Security It enables consumers to verify that the container image hasn't been tampered with and that it indeed originates from the expected publisher. This helps mitigate several classes of [supply chain threats](https://slsa.dev/spec/v1.0/threats). + @REQ-SEC-0003 Rule: Software Bill of Materials (SBOM) amp-devcontainer *SHOULD* provide a Software Bill of Materials (SBOM) for its released container images. diff --git a/test/cpp/features/static-dynamic-analysis.feature b/test/cpp/features/static-dynamic-analysis.feature index 3254790a..2f955b47 100644 --- a/test/cpp/features/static-dynamic-analysis.feature +++ b/test/cpp/features/static-dynamic-analysis.feature @@ -4,6 +4,7 @@ Feature: Static and dynamic analysis to maintain consistent, high-quality and bug-free code, I want my source code to be statically and dynamically analyzed. + @REQ-SDA-0001 Rule: Code formatting amp-devcontainer *MAY* provide code formatting tools for the primary programming language(s) used within the container. @@ -18,6 +19,7 @@ Feature: Static and dynamic analysis And the active document is saved Then the contents of "clang-tools/unformatted.cpp" should match the contents of "clang-tools/formatted.cpp" + @REQ-SDA-0002 Rule: Static analysis amp-devcontainer *MAY* provide static analysis tools for the primary programming language(s) used within the container. @@ -25,6 +27,7 @@ Feature: Static and dynamic analysis These tools can analyze the code for common pitfalls, coding standards violations, and potential bugs, providing developers with valuable feedback early in the development process. By integrating static analysis tools into the development environment, developers can catch issues before they become more significant problems, streamlining the development workflow and improving overall code quality. + @REQ-SDA-0003 Rule: Coverage analysis amp-devcontainer *SHOULD* provide code coverage analysis tools for the primary programming language(s) used within the container. @@ -32,6 +35,7 @@ Feature: Static and dynamic analysis This information can help identify gaps in test coverage, ensuring that critical parts of the code are adequately tested. By integrating code coverage analysis tools into the development environment, developers can improve their test suites, leading to higher code quality and increased confidence in the software's correctness. + @REQ-SDA-0004 Rule: Mutation testing amp-devcontainer *MAY* provide mutation testing tools for the primary programming language(s) used within the container. @@ -39,6 +43,7 @@ Feature: Static and dynamic analysis This process helps identify gaps in test coverage and ensures that the tests are robust enough to catch potential issues in the code. By integrating mutation testing tools into the development environment, developers can improve their test suites, leading to higher code quality and increased confidence in the software's correctness. + @REQ-SDA-0005 Rule: Fuzz testing amp-devcontainer *MAY* provide fuzz testing tools for the primary programming language(s) used within the container. diff --git a/test/cpp/integration-tests.bats b/test/cpp/integration-tests.bats index fd6cfd2f..7b185b94 100644 --- a/test/cpp/integration-tests.bats +++ b/test/cpp/integration-tests.bats @@ -28,7 +28,7 @@ teardown() { # that the tools are compatible with each other. E.g. that the host and embedded toolchains # are aligned in terms of major and minor versions. -# bats test_tags=Compatibility,Version,Clang +# bats test_tags=compatibility,version,clang @test "clang toolchain versions should be aligned with expected versions" { EXPECTED_VERSION=$(get_expected_semver_for clang) @@ -38,7 +38,7 @@ teardown() { done } -# bats test_tags=Compatibility,Version,GCC +# bats test_tags=compatibility,version,gcc @test "host gcc toolchain versions and alternatives should be aligned with expected versions" { EXPECTED_VERSION=$(get_expected_semver_for g++) @@ -48,14 +48,14 @@ teardown() { done } -# bats test_tags=Compatibility,Version,HostGCCArmGCC +# bats test_tags=compatibility,version,hostgccarmgcc @test "host and embedded gcc toolchain versions should be the same major and minor version" { EXPECTED_MAJOR_MINOR_VERSION=$(get_expected_semver_for g++ | cut -d. -f1,2) INSTALLED_MAJOR_MINOR_VERSION=$(arm-none-eabi-gcc -dumpfullversion | cut -d. -f1,2) assert_equal_print "$EXPECTED_MAJOR_MINOR_VERSION" "$INSTALLED_MAJOR_MINOR_VERSION" "Host and ARM GCC major and minor version" } -# bats test_tags=Compatibility,Version,Tools +# bats test_tags=compatibility,version,tools @test "supporting tool versions should be aligned with expected versions" { for TOOL in gdb gdb-multiarch git ninja; do EXPECTED_VERSION=$(get_expected_semver_for ${TOOL}) @@ -72,6 +72,7 @@ teardown() { done } +# bats test_tags=compilation,REQ-COMP-0001 @test "valid code input should result in working executable using host compiler" { cmake --preset gcc cmake --build --preset gcc @@ -81,6 +82,7 @@ teardown() { assert_output "Hello World!" } +# bats test_tags=compilation,REQ-COMP-0002 @test "valid code input should result in elf executable using arm-none-eabi compiler" { cmake --preset gcc-arm-none-eabi cmake --build --preset gcc-arm-none-eabi @@ -90,6 +92,7 @@ teardown() { assert_output --partial "Machine: ARM" } +# bats test_tags=compilation,REQ-COMP-0003 @test "valid code input should result in Windows executable using clang-cl compiler" { install_win_sdk_when_ci_unset @@ -97,6 +100,7 @@ teardown() { cmake --build --preset clang-cl } +# bats test_tags=compilation,REQ-COMP-0001 @test "compilation database should be generated on CMake configure" { cmake --preset gcc assert [ -e build/gcc/compile_commands.json ] @@ -105,21 +109,25 @@ teardown() { assert [ -e build/gcc-arm-none-eabi/compile_commands.json ] } +# bats test_tags=compilation,REQ-COMP-0001 @test "invalid code input should result in failing build" { cmake --preset gcc run ! cmake --build --preset gcc-fail } +# bats test_tags=compilation,REQ-COMP-0004 @test "using ccache as a compiler launcher should result in cached build using gcc compiler" { configure_and_build_with_ccache gcc } +# bats test_tags=compilation,REQ-COMP-0004 @test "using ccache as a compiler launcher should result in cached build using clang-cl compiler" { install_win_sdk_when_ci_unset configure_and_build_with_ccache clang-cl } +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0002 @test "running clang-tidy as part of the build should result in warning diagnostics" { cmake --preset clang @@ -128,6 +136,7 @@ teardown() { assert_output --partial "warning: use a trailing return type for this function" } +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0002 @test "running include-what-you-use as part of the build should result in warning diagnostics" { cmake --preset clang @@ -136,12 +145,14 @@ teardown() { assert_output --partial "Warning: include-what-you-use reported diagnostics:" } +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0001 @test "running clang-format should result in re-formatted code" { run clang-format clang-tools/unformatted.cpp assert_success assert_output "int main() {}" } +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0003 @test "coverage information should be generated when running a testsuite" { cmake --preset coverage cmake --build --preset coverage @@ -155,6 +166,7 @@ teardown() { assert_output --partial "GCC Code Coverage Report" } +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0005 @test "crashes should be detected when fuzzing an executable" { cmake --preset clang cmake --build --preset fuzzing @@ -164,6 +176,7 @@ teardown() { assert_output --partial "SUMMARY: libFuzzer: deadly signal" } +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0004 @test "a mutation score should be calculated when mutation testing a testsuite" { cmake --preset mutation cmake --build --preset mutation @@ -172,17 +185,20 @@ teardown() { assert_output --partial "[info] Mutation score:" } +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0002 @test "clangd should be able to analyze source files" { run clangd --check=gcc/main.cpp assert_success assert_output --partial "All checks completed, 0 errors" } +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0002 @test "clangd should start with a specified compile commands path" { run timeout 1s clangd --compile-commands-dir=/root/.amp refute_output --partial "Path specified by --compile-commands-dir does not exist. The argument will be ignored." } +# bats test_tags=compilation,REQ-COMP-0001 @test "using lld as an alternative linker should result in working host executable" { cmake --preset gcc cmake --build --preset gcc-lld @@ -195,14 +211,17 @@ teardown() { assert_output "Hello World!" } +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0002 @test "sanitizers should detect undefined or suspicious behavior in code compiled with gcc" { build_and_run_with_sanitizers gcc } +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0002 @test "sanitizers should detect undefined or suspicious behavior in code compiled with clang" { build_and_run_with_sanitizers clang } +# bats test_tags=compilation,REQ-COMP-0001 @test "using Conan as package manager should resolve external dependencies" { pushd package-managers/conan @@ -214,6 +233,7 @@ teardown() { popd } +# bats test_tags=compilation,REQ-COMP-0001 @test "using CPM as package manager should resolve external dependencies" { cmake --preset cpm cmake --build --preset cpm diff --git a/test/rust/integration-tests.bats b/test/rust/integration-tests.bats index 0c803fcb..daa7c85a 100644 --- a/test/rust/integration-tests.bats +++ b/test/rust/integration-tests.bats @@ -15,6 +15,7 @@ teardown() { popd } +# bats test_tags=compilation,REQ-COMP-0001 @test "valid code input should result in working executable targeting the host architecture" { rustc --out-dir build rust/hello.rs @@ -23,6 +24,7 @@ teardown() { assert_output "Hello, world!" } +# bats test_tags=compilation,REQ-COMP-0002 @test "valid code input should result in working elf executable targeting the cortex-m architecture" { pushd cortex-m @@ -36,6 +38,7 @@ teardown() { popd } +# bats test_tags=compilation,REQ-COMP-0002 @test "valid code input should result in working elf executable targeting the cortex-mf architecture" { pushd cortex-mf @@ -49,6 +52,7 @@ teardown() { popd } +# bats test_tags=compilation,REQ-COMP-0001 @test "using cargo run should result in working executable" { pushd cargo @@ -59,12 +63,14 @@ teardown() { popd } +# bats test_tags=compilation,REQ-COMP-0001 @test "invalid code input should result in failing build" { run rustc rust/fail.rs assert_failure assert_output --partial "error: " } +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0002 @test "running clippy should result in warning diagnostics" { pushd clippy @@ -75,6 +81,7 @@ teardown() { popd } +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0001 @test "running rustfmt should result in re-formatted code" { run rustfmt --color=never --check rust/unformatted.rs assert_failure @@ -88,6 +95,7 @@ teardown() { EOF } +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0003 @test "coverage information should be generated when running a testsuite" { pushd test @@ -104,6 +112,7 @@ EOF popd } +# bats test_tags=static-and-dynamic-analysis,REQ-SDA-0004 @test "mutation testing a test executable should be supported" { pushd test