diff --git a/.gitignore b/.gitignore index a561059..cafbf19 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,7 @@ _SoftwareClearing/ !SBOM/*.json !tests/fixtures/* +!capycli/data/*.json SoftwareClearing/*.html SoftwareClearing/*.json diff --git a/.reuse/dep5 b/.reuse/dep5 index c1bcf88..9755ae1 100644 --- a/.reuse/dep5 +++ b/.reuse/dep5 @@ -24,8 +24,8 @@ Files: tests/fixtures/* Copyright: 2018-2023 Siemens License: MIT -Files: capycli/data/granularity_list.csv requirements.txt -Copyright: 2018-2023 Siemens +Files: capycli/data/granularity_list.csv requirements.txt capycli/data/component_checks.json +Copyright: 2018-2026 Siemens License: MIT Files: Legacy/code.siemens.com-gitlab-ci.yml.txt Legacy/Pipfile.lock diff --git a/ChangeLog.md b/ChangeLog.md index e903b48..b3c6d10 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -5,6 +5,18 @@ # CaPyCli - Clearing Automation Python Command Line Tool for SW360 +## NEXT + +* Two new commands: `bom componentcheck` and `project componentcheck`. The first one + checks a given SBOM for special components, the second one does the same for + an existing SW360 project. **Special components** are components that should not be + part of license compliance checks. These are for example unit test tools like + `junit` or `pytest`, linter like `eslint`, mocking frameworks like `Moq`, etc. + CaPyCLI has a list of these components (data/component_checks.json), but you + can also provide your own list. For more information please have a look at [this documentation](documentation/Readme_Componentcheck.md). +* New folder `documentation` where we want to keep all more detailed documentation + on the way CaPyCLI works. + ## 2.10.1 * `bom show` now also shows the group, if it exists. diff --git a/Readme.md b/Readme.md index 7495366..da26796 100644 --- a/Readme.md +++ b/Readme.md @@ -152,14 +152,14 @@ These options are not available for all commands. At the moment Over the time we implemented more and more commands with more and more parameters. We understand that it is hard for beginners to find the right command for the task -they want to do. Have a look at our [Use Case Overview](UseCaseOverview.md). +they want to do. Have a look at our [Use Case Overview](documentation/UseCaseOverview.md). ## Software Clearing Approaches From time to time there are questions **why** a command has been implemented in this specific way or why a command exists at all. Not all organization have the same approach when doing license compliance. Have a look at our -[Software Clearing Approach Overview](SoftwareClearingApproachOverview.md) to see our +[Software Clearing Approach Overview](documentation/SoftwareClearingApproachOverview.md) to see our approaches. ## Note about Python Dependency Detection @@ -233,7 +233,7 @@ The software bill of materials (SBOM) is a crucial information for most operatio There is no common description what a bill of materials should contain. There are different formats available, for example the SBOM of CyCloneDX, nevertheless most tools have their own SBOM format. -We have decided also to have our own flavor of CycloneDX, see [SBOM](Readme_BOM.md), +We have decided also to have our own flavor of CycloneDX, see [SBOM](documentation/Readme_BOM.md), focused on the information we need to handle components, releases and projects on SW360. It is a simple JSON format. CaPyCli reads or writes exactly the information that is needed. @@ -243,7 +243,7 @@ to refer you to the open source tools from [CycloneDX](https://cyclonedx.org/). ## Mapping a SBOM to SW360 -SBOM mapping is described in an extra file, see [SBOM Mapping](Readme_Mapping.md). +SBOM mapping is described in an extra file, see [SBOM Mapping](documentation/Readme_Mapping.md). ## Project Management diff --git a/capycli/bom/component_check.py b/capycli/bom/component_check.py new file mode 100644 index 0000000..31b5987 --- /dev/null +++ b/capycli/bom/component_check.py @@ -0,0 +1,212 @@ +# ------------------------------------------------------------------------------- +# Copyright (c) 2026 Siemens +# All Rights Reserved. +# Author: thomas.graf@siemens.com +# +# SPDX-License-Identifier: MIT +# ------------------------------------------------------------------------------- + +try: + import importlib.resources as pkg_resources +except ImportError: + # Try backported to PY<37 `importlib_resources`. + import importlib_resources as pkg_resources # type: ignore + +import json +import os +import sys +from typing import Any, Dict, List + +import requests + +# from cyclonedx.model import ExternalReferenceType +from cyclonedx.model.bom import Bom +from cyclonedx.model.component import Component + +import capycli.common.script_base +from capycli.common.capycli_bom_support import CaPyCliBom +from capycli.common.print import print_red, print_text, print_yellow +from capycli.main.result_codes import ResultCode + +LOG = capycli.get_logger(__name__) + + +class ComponentCheck(capycli.common.script_base.ScriptBase): + """ + Check the SBOM for special components. + """ + def __init__(self) -> None: + self.component_check_list: Dict[str, Any] = {} + self.files_to_ignore: List[Dict[str, Any]] = [] + + @staticmethod + def get_component_check_list(download_url: str) -> None: + """This will only download the component check file from a public repository.""" + response = requests.get(download_url) + response.raise_for_status() + with open("component_checks.json", "wb") as f1: + f1.write(response.content) + + def read_component_check_list(self, download_url: str = "", local_check_list_file: str = "") -> None: + """Reads the granularity list from file.""" + self.component_check_list = {} + text_list = "" + if local_check_list_file: + try: + with open(local_check_list_file, "r", encoding="utf-8") as fin: + self.component_check_list = json.load(fin) + except FileNotFoundError as e: + print_yellow(f"File not found: {e} \n Reading the default component check list") + except Exception as e: + print_red(f"An unexpected error occurred: {e}") + if download_url: + try: + ComponentCheck.get_component_check_list(download_url) + with open("component_checks.json", "r", encoding="utf-8") as fin: + self.component_check_list = json.load(fin) + except FileNotFoundError as e: + print_yellow(f"File not found: {e} \n Reading the default component check list") + except Exception as e: + print_red(f"An unexpected error occurred: {e}") + if not self.component_check_list: + if sys.version_info >= (3, 9): + resources = pkg_resources.files("capycli.data") + text_list = (resources / "component_checks.json").read_text() + else: + text_list = pkg_resources.read_text("capycli.data", "component_checks.json") + + self.component_check_list = json.loads(text_list) + + def get_dev_dependencies(self, ecosystem: str) -> List[Dict[str, Any]]: + """Get the list of dependencies for a specific eco-system.""" + dd = self.component_check_list.get("dev_dependencies", []) + if not dd: + return [] + + return dd.get(ecosystem, []) + + def is_dev_dependency(self, comp: Component) -> bool: + """Check whether the given component matches any known development dependency.""" + if comp.purl: + # preferred: check by package-url + pd = comp.purl.to_dict() + ecosystem = pd.get("type", "").lower() + for entry in self.get_dev_dependencies(ecosystem): + if comp.group: + if comp.group.lower() != entry.get("namespace", ""): + continue + + if comp.name.lower() == entry.get("name", ""): + return True + else: + # fallback: only check by name + dd = self.component_check_list.get("dev_dependencies", []) + for ecosystem in dd: + for entry in dd.get(ecosystem, []): + if comp.name.lower() == entry.get("name", ""): + return True + + return False + + def is_python_binary_component(self, comp: Component) -> bool: + """Check whether the given component matches any known python component + with additional binary dependencies.""" + if comp.purl: + pd = comp.purl.to_dict() + if pd.get("type", "").lower() != "pypi": + return False + + pbc = self.component_check_list.get("python_binary_components", []) + for entry in pbc: + if comp.name.lower() == entry.get("name", ""): + return True + + return False + + def is_file_to_ignore(self, comp: Component) -> bool: + """Check whether the given component is to be ignored.""" + for entry in self.files_to_ignore: + if comp.name.lower() == entry.get("name", ""): + return True + + return False + + def check_bom_items(self, sbom: Bom) -> int: + """Check the SBOM for special components.""" + special_component_count = 0 + for component in sbom.components: + if self.is_file_to_ignore(component): + continue + + if self.is_dev_dependency(component): + special_component_count += 1 + print_yellow(" ", component.name, component.version, "seems to be a development dependency") + + if self.is_python_binary_component(component): + special_component_count += 1 + print_yellow(" ", component.name, component.version, + "is known as a Python component that has additional binary dependencies") + + return special_component_count + + def run(self, args: Any) -> None: + """Main method()""" + if args.debug: + global LOG + LOG = capycli.get_logger(__name__) + + print_text( + "\n" + capycli.get_app_signature() + + " - Check the SBOM for special components.\n") + + if args.help: + print("usage: CaPyCli bom componentcheck [-h] [-v] -i bomfile [-rcl URL] [-lcl FILE]") + print("") + print("optional arguments:") + print(" -h, --help show this help message and exit") + print(" -i INPUTFILE SBOM file to read from (JSON)") + print(" -v be verbose") + print(" -rcl read the component check list file from the URL specified") + print(" -lcl read the component check list file from local") + print(" --forceerror force an error exit code in case of validation errors or warnings") + return + + if not args.inputfile: + print_red("No input file specified!") + sys.exit(ResultCode.RESULT_COMMAND_ERROR) + + if not os.path.isfile(args.inputfile): + print_red("Input file not found!") + sys.exit(ResultCode.RESULT_FILE_NOT_FOUND) + + print_text("Reading component check list from component_checks.json...") + try: + self.read_component_check_list(args.remote_check_list, args.local_checklist_list) + except Exception as ex: + print_red("Error reading component check list " + repr(ex)) + sys.exit(ResultCode.RESULT_GENERAL_ERROR) + if len(self.component_check_list) > 0: + print(" Got component checklist.") + + self.files_to_ignore = self.component_check_list.get("files_to_ignore", []) + print_text(f" {len(self.files_to_ignore)} components will be ignored.") + + print_text("\nLoading SBOM file", args.inputfile) + try: + sbom = CaPyCliBom.read_sbom(args.inputfile) + # for c in sbom.components: + # print(c) + except Exception as ex: + print_red("Error reading SBOM: " + repr(ex)) + sys.exit(ResultCode.RESULT_ERROR_READING_BOM) + + if args.verbose: + print_text(" ", self.get_comp_count_text(sbom), "read from SBOM") + + result = self.check_bom_items(sbom) + print_text("\nDone.") + if result > 0: + if args.force_error: + if args.verbose: + print_yellow("Special component(s) found, exiting with code != 0") + sys.exit(ResultCode.RESULT_SPECIAL_COMPONENT_FOUND) diff --git a/capycli/bom/handle_bom.py b/capycli/bom/handle_bom.py index a4804f1..e2f613d 100644 --- a/capycli/bom/handle_bom.py +++ b/capycli/bom/handle_bom.py @@ -1,5 +1,5 @@ # ------------------------------------------------------------------------------- -# Copyright (c) 2019-2025 Siemens +# Copyright (c) 2019-2026 Siemens # All Rights Reserved. # Author: thomas.graf@siemens.com # @@ -15,6 +15,7 @@ import capycli.bom.check_bom import capycli.bom.check_bom_item_status import capycli.bom.check_granularity +import capycli.bom.component_check import capycli.bom.create_components import capycli.bom.diff_bom import capycli.bom.download_sources @@ -53,6 +54,7 @@ def run_bom_command(args: Any) -> None: print(" Findsources determine the source code for SBOM items") print(" Validate validate an SBOM") print(" BomPackage create a single archive that contains the SBOM and all source and binary files") + print(" ComponentCheck check the SBOM for special components") return subcommand = args.command[1].lower() @@ -147,5 +149,11 @@ def run_bom_command(args: Any) -> None: app15.run(args) return + if subcommand == "componentcheck": + """Check the SBOM for special components.""" + app16 = capycli.bom.component_check.ComponentCheck() + app16.run(args) + return + print_red("Unknown sub-command: ") sys.exit(ResultCode.RESULT_COMMAND_ERROR) diff --git a/capycli/data/component_checks.json b/capycli/data/component_checks.json new file mode 100644 index 0000000..0107d72 --- /dev/null +++ b/capycli/data/component_checks.json @@ -0,0 +1,213 @@ +{ + "dev_dependencies": { + "maven": [ + { "namespace": "org.eclipse.jdt", "name": "junit" }, + { "namespace": "org.junit.jupiter", "name": "junit-jupiter" }, + { "namespace": "org.mockito", "name": "mockito-core" }, + { "namespace": "org.assertj", "name": "assertj-core" }, + { "namespace": "org.hamcrest", "name": "hamcrest" }, + { "namespace": "org.spockframework", "name": "spock-core" }, + { "namespace": "org.easymock", "name": "easymock" }, + { "namespace": "org.powermock", "name": "powermock-module-junit4" }, + { "namespace": "org.mockito", "name": "mockito-junit-jupiter" }, + { "namespace": "org.jacoco", "name": "org.jacoco.core" }, + { "namespace": "org.scoverage", "name": "scalac-scoverage-runtime_2.12" }, + { "namespace": "org.sonarsource.scanner.maven", "name": "sonar-maven-plugin" }, + { "namespace": "com.puppycrawl.tools", "name": "checkstyle" }, + { "namespace": "pmd", "name": "pmd" }, + { "namespace": "org.codehaus.mojo", "name": "animal-sniffer-maven-plugin" }, + { "namespace": "org.apache.maven.plugins", "name": "maven-surefire-plugin" }, + { "namespace": "org.apache.maven.plugins", "name": "maven-failsafe-plugin" }, + { "namespace": "org.apache.maven.plugins", "name": "maven-checkstyle-plugin" }, + { "namespace": "org.apache.maven.plugins", "name": "maven-pmd-plugin" } + ], + "npm": [ + { "namespace": "", "name": "mocha" }, + { "namespace": "", "name": "jest" }, + { "namespace": "", "name": "eslint" }, + { "namespace": "", "name": "tslint" }, + { "namespace": "", "name": "typescript" }, + { "namespace": "", "name": "ts-node" }, + { "namespace": "@types", "name": "node" }, + { "namespace": "", "name": "babel-jest" }, + { "namespace": "@babel", "name": "core" }, + { "namespace": "", "name": "react-testing-library" }, + { "namespace": "@testing-library", "name": "react" }, + { "namespace": "", "name": "jasmine" }, + { "namespace": "", "name": "sinon" }, + { "namespace": "", "name": "chai" }, + { "namespace": "", "name": "nyc" }, + { "namespace": "", "name": "ava" }, + { "namespace": "", "name": "standard" }, + { "namespace": "", "name": "prettier" }, + { "namespace": "", "name": "webpack" }, + { "namespace": "", "name": "rollup" }, + { "namespace": "", "name": "esbuild" }, + { "namespace": "", "name": "nodemon" }, + { "namespace": "", "name": "parcel" }, + { "namespace": "", "name": "stylelint" }, + { "namespace": "", "name": "husky" }, + { "namespace": "", "name": "lint-staged" }, + { "namespace": "", "name": "cypress" }, + { "namespace": "", "name": "protractor" }, + { "namespace": "", "name": "karma" }, + { "namespace": "", "name": "karma-jasmine" }, + { "namespace": "", "name": "dtslint" }, + { "namespace": "", "name": "ts-jest" }, + { "namespace": "@typescript-eslint", "name": "eslint-plugin" }, + { "namespace": "@typescript-eslint", "name": "parser" }, + { "namespace": "", "name": "jshint" }, + { "namespace": "", "name": "gulp" }, + { "namespace": "@eslint", "name": "js" }, + { "namespace": "", "name": "chai" }, + { "namespace": "", "name": "jest" } + ], + "pypi": [ + { "namespace": "", "name": "pytest" }, + { "namespace": "", "name": "tox" }, + { "namespace": "", "name": "coverage" }, + { "namespace": "", "name": "mypy" }, + { "namespace": "", "name": "flake8" }, + { "namespace": "", "name": "black" }, + { "namespace": "", "name": "isort" }, + { "namespace": "", "name": "pylint" }, + { "namespace": "", "name": "pytest-cov" }, + { "namespace": "", "name": "nose" }, + { "namespace": "", "name": "hypothesis" }, + { "namespace": "", "name": "pre-commit" }, + { "namespace": "", "name": "pytest-mock" }, + { "namespace": "", "name": "mock" }, + { "namespace": "", "name": "Sphinx" }, + { "namespace": "", "name": "sphinx-autodoc-typehints" }, + { "namespace": "", "name": "sphinx-rtd-theme" }, + { "namespace": "", "name": "bandit" }, + { "namespace": "", "name": "pytest-xdist" }, + { "namespace": "", "name": "pytest-asyncio" }, + { "namespace": "", "name": "pytest-django" }, + { "namespace": "", "name": "faker" }, + { "namespace": "", "name": "radon" }, + { "namespace": "", "name": "pyflakes" }, + { "namespace": "", "name": "pep8" }, + { "namespace": "", "name": "autopep8" }, + { "namespace": "", "name": "pydocstyle" }, + { "namespace": "", "name": "pycodestyle" }, + { "namespace": "", "name": "pytest-runner" }, + { "namespace": "", "name": "pytest-timeout" }, + { "namespace": "", "name": "pytest-html" }, + { "namespace": "", "name": "pytest-ordering" }, + { "namespace": "", "name": "pytest-sugar" }, + { "namespace": "", "name": "pytest-testmon" }, + { "namespace": "", "name": "twine" }, + { "namespace": "", "name": "wheel" }, + { "namespace": "", "name": "setuptools" }, + { "namespace": "", "name": "pip-tools" }, + { "namespace": "", "name": "check-manifest" }, + { "namespace": "", "name": "types-mock" }, + { "namespace": "", "name": "types-requests" }, + { "namespace": "", "name": "types-setuptools" } + ], + "nuget": [ + { "namespace": "", "name": "NUnit" }, + { "namespace": "", "name": "xunit" }, + { "namespace": "", "name": "Moq" }, + { "namespace": "", "name": "coverlet.collector" }, + { "namespace": "", "name": "Microsoft.NET.Test.Sdk" }, + { "namespace": "", "name": "FluentAssertions" }, + { "namespace": "", "name": "StyleCop.Analyzers" }, + { "namespace": "", "name": "JetBrains.Annotations" }, + { "namespace": "", "name": "FakeItEasy" }, + { "namespace": "", "name": "NSubstitute" }, + { "namespace": "", "name": "AutoFixture" }, + { "namespace": "", "name": "Selenium.WebDriver" }, + { "namespace": "", "name": "Microsoft.CodeAnalysis.FxCopAnalyzers" }, + { "namespace": "", "name": "nunit3testadapter" }, + { "namespace": "", "name": "xunit.runner.visualstudio" }, + { "namespace": "", "name": "Coverlet.MSBuild" }, + { "namespace": "", "name": "BenchmarkDotNet" }, + { "namespace": "", "name": "Microsoft.CodeAnalysis.Workspaces.MSBuild" }, + { "namespace": "", "name": "Microsoft.VisualStudio.SolutionPersistence" }, + { "namespace": "", "name": "msbuild" }, + { "namespace": "", "name": "Microsoft.CodeAnalysis.Analyzers" }, + { "namespace": "", "name": "Microsoft.CodeAnalysis.CSharp.Analyzers" }, + { "namespace": "", "name": "Microsoft.CodeAnalysis.CSharp.NetAnalyzers" }, + { "namespace": "", "name": "Microsoft.CodeAnalysis.NetAnalyzers" }, + { "namespace": "", "name": "Microsoft.Interop.ComInterfaceGenerator" }, + { "namespace": "", "name": "Microsoft.Interop.JavaScript.JSImportGenerator" }, + { "namespace": "", "name": "Microsoft.Interop.LibraryImportGenerator" }, + { "namespace": "", "name": "Microsoft.Interop.SourceGeneration" }, + { "namespace": "", "name": "StyleCop.Analyzers.CodeFixes" }, + { "namespace": "", "name": "System.Text.Json.SourceGeneration" }, + { "namespace": "", "name": "System.Text.RegularExpressions.Generator" }, + { "namespace": "", "name": "System.Windows.Forms.Analyzers" }, + { "namespace": "", "name": "System.Windows.Forms.Analyzers.CSharp" } + ], + "gem": [ + { "namespace": "", "name": "rspec" }, + { "namespace": "", "name": "rubocop" }, + { "namespace": "", "name": "simplecov" }, + { "namespace": "", "name": "factory_bot" }, + { "namespace": "", "name": "cucumber" }, + { "namespace": "", "name": "capybara" }, + { "namespace": "", "name": "minitest" }, + { "namespace": "", "name": "faker" }, + { "namespace": "", "name": "shoulda" }, + { "namespace": "", "name": "shoulda-matchers" }, + { "namespace": "", "name": "yard" }, + { "namespace": "", "name": "guard" }, + { "namespace": "", "name": "guard-rspec" }, + { "namespace": "", "name": "bundler-audit" }, + { "namespace": "", "name": "reek" }, + { "namespace": "", "name": "pry" }, + { "namespace": "", "name": "test-unit" }, + { "namespace": "", "name": "database_cleaner" }, + { "namespace": "", "name": "rspec-rails" }, + { "namespace": "", "name": "rspec-mocks" } + ], + "composer": [ + { "namespace": "phpunit", "name": "phpunit" }, + { "namespace": "squizlabs", "name": "php_codesniffer" }, + { "namespace": "phpstan", "name": "phpstan" }, + { "namespace": "friendsofphp", "name": "php-cs-fixer" }, + { "namespace": "phpmd", "name": "phpmd" }, + { "namespace": "symfony", "name": "var-dumper" }, + { "namespace": "phpspec", "name": "phpspec" }, + { "namespace": "php-parallel-lint", "name": "php-parallel-lint" }, + { "namespace": "mockery", "name": "mockery" }, + { "namespace": "sebastian", "name": "phpcpd" }, + { "namespace": "liuggio", "name": "statsd-php-client" }, + { "namespace": "hamcrest", "name": "hamcrest-php" }, + { "namespace": "psy", "name": "psysh" }, + { "namespace": "fakerphp", "name": "faker" }, + { "namespace": "codeception", "name": "codeception" } + ], + "golang": [ + { "namespace": "github.com/stretchr", "name": "testify" }, + { "namespace": "github.com/golang", "name": "mock" }, + { "namespace": "github.com/onsi", "name": "ginkgo" }, + { "namespace": "github.com/onsi", "name": "gomega" }, + { "namespace": "golang.org/x", "name": "lint" }, + { "namespace": "github.com/goreleaser", "name": "goreleaser" }, + { "namespace": "github.com/axw", "name": "gocov" }, + { "namespace": "github.com/mattn", "name": "goveralls" }, + { "namespace": "github.com/securego", "name": "gosec" }, + { "namespace": "github.com/uudashr", "name": "gocognit" }, + { "namespace": "github.com/DATA-DOG", "name": "go-sqlmock" }, + { "namespace": "honnef.co/go", "name": "tools" }, + { "namespace": "github.com/bouk", "name": "monkey" }, + { "namespace": "github.com/cweill", "name": "gotests" }, + { "namespace": "github.com/vektra", "name": "mockery" } + ] + }, + "python_binary_components": [ + { "namespace": "", "name": "numpy" }, + { "namespace": "", "name": "opencv_python_headless" }, + { "namespace": "", "name": "pandas" }, + { "namespace": "", "name": "pillow" }, + { "namespace": "", "name": "scikit_learn" }, + { "namespace": "", "name": "scipy" }, + { "namespace": "", "name": "shapely" }, + { "namespace": "", "name": "torch" }, + { "namespace": "", "name": "torchvision" } + ], + "files_to_ignore": [] +} diff --git a/capycli/main/options.py b/capycli/main/options.py index f653f41..76ffcd4 100644 --- a/capycli/main/options.py +++ b/capycli/main/options.py @@ -1,5 +1,5 @@ # ------------------------------------------------------------------------------- -# Copyright (c) 2019-2025 Siemens +# Copyright (c) 2019-2026 Siemens # All Rights Reserved. # Author: thomas.graf@siemens.com # @@ -398,6 +398,22 @@ def register_options(self) -> None: help="read the granularity list file from local" ) + # used by bom ComponentCheck + self.parser.add_argument( + "-rcl", + "--remote-checklist", + dest="remote_check_list", + help="read the component check list file from the download URL specified" + ) + + # used by bom ComponentCheck + self.parser.add_argument( + "-lcl", + "--local-checklist", + dest="local_checklist_list", + help="read the component check list file from local" + ) + self.parser.add_argument( "-X", dest="debug", action="store_true", diff --git a/capycli/main/result_codes.py b/capycli/main/result_codes.py index c779dcc..9cc71cf 100644 --- a/capycli/main/result_codes.py +++ b/capycli/main/result_codes.py @@ -1,5 +1,5 @@ # ------------------------------------------------------------------------------- -# Copyright 2023-2024 Siemens +# Copyright 2023-2026 Siemens # All Rights Reserved. # Author: thomas.graf@siemens.com # @@ -41,3 +41,4 @@ class ResultCode(object): RESULT_FILTER_ERROR = 96 RESULT_PREREQUISITE_ERROR = 97 RESULT_LICENSE_INFO_ERROR = 98 + RESULT_SPECIAL_COMPONENT_FOUND = 99 diff --git a/capycli/project/handle_project.py b/capycli/project/handle_project.py index 9eed171..371c19f 100644 --- a/capycli/project/handle_project.py +++ b/capycli/project/handle_project.py @@ -1,5 +1,5 @@ # ------------------------------------------------------------------------------- -# Copyright (c) 2019-24 Siemens +# Copyright (c) 2019-2026 Siemens # All Rights Reserved. # Author: thomas.graf@siemens.com # @@ -15,6 +15,7 @@ import capycli.project.create_readme import capycli.project.find_project import capycli.project.get_license_info +import capycli.project.project_component_check import capycli.project.show_ecc import capycli.project.show_licenses import capycli.project.show_project @@ -115,5 +116,11 @@ def run_project_command(args: Any) -> None: app11.run(args) return + if subcommand == "componentcheck": + """Check the project for special components.""" + app12 = capycli.project.project_component_check.ProjectComponentCheck() + app12.run(args) + return + print_red("Unknown sub-command: " + subcommand) sys.exit(ResultCode.RESULT_COMMAND_ERROR) diff --git a/capycli/project/project_component_check.py b/capycli/project/project_component_check.py new file mode 100644 index 0000000..7d98607 --- /dev/null +++ b/capycli/project/project_component_check.py @@ -0,0 +1,175 @@ +# ------------------------------------------------------------------------------- +# Copyright (c) 2026 Siemens +# All Rights Reserved. +# Author: thomas.graf@siemens.com +# +# SPDX-License-Identifier: MIT +# ------------------------------------------------------------------------------- + +import logging +import sys +from typing import Any + +import sw360 + +import capycli.common.script_base +from capycli.bom.component_check import ComponentCheck +from capycli.common.print import print_red, print_text, print_yellow +from capycli.main.result_codes import ResultCode + +LOG = capycli.get_logger(__name__) + + +class ProjectComponentCheck(capycli.common.script_base.ScriptBase): + """ + Check a project for special components. + """ + def __init__(self) -> None: + self.ComponentCheck = ComponentCheck() + + def is_dev_dependency(self, name: str) -> bool: + """Check whether the given component matches any known development dependency.""" + dd = self.ComponentCheck.component_check_list.get("dev_dependencies", []) + for ecosystem in dd: + for entry in dd.get(ecosystem, []): + if name.lower() == entry.get("name", ""): + return True + + return False + + def is_python_binary_component(self, name: str) -> bool: + """Check whether the given component matches any known python component + with additional binary dependencies.""" + pbc = self.ComponentCheck.component_check_list.get("python_binary_components", []) + for entry in pbc: + if name.lower() == entry.get("name", ""): + return True + + return False + + def is_file_to_ignore(self, name: str) -> bool: + """Check whether the given component is to be ignored.""" + for entry in self.ComponentCheck.files_to_ignore: + if name.lower() == entry.get("name", ""): + return True + + return False + + def check_bom_items(self, project_id: str) -> int: + """Check the given project for special components.""" + print_text("\nRetrieving project details...") + + special_component_count = 0 + + if not self.client: + print_red(" No client!") + sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360) + + try: + self.project = self.client.get_project(project_id) + except sw360.SW360Error as swex: + print_red(" ERROR: unable to access project: " + repr(swex)) + sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360) + + if not self.project: + print_red(" ERROR: unable to read project data!") + sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360) + + if "sw360:releases" in self.project["_embedded"]: + releases = self.project["_embedded"]["sw360:releases"] + releases.sort(key=lambda s: s["name"].lower()) + for release in releases: + name = release.get("name", "") + version = release.get("version", "") + # print(name, version) + if self.is_file_to_ignore(name): + continue + + if self.is_dev_dependency(name): + special_component_count += 1 + print_yellow(" ", name, version, "seems to be a development dependency") + + if self.is_python_binary_component(name): + special_component_count += 1 + print_yellow(" ", name, version, + "is known as a Python component that has additional binary dependencies") + + return special_component_count + + def run(self, args: Any) -> None: + """Main method()""" + if args.debug: + global LOG + LOG = capycli.get_logger(__name__) + else: + # suppress (debug) log output from requests and urllib + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("urllib3.connectionpool").setLevel(logging.WARNING) + + print_text( + "\n" + capycli.get_app_signature() + + " - Check the project for special components\n") + + if args.help: + print("usage: CaPyCli project componentcheck [-h] -t TOKEN -name NAME -version VERSION" + "[-v] [-id PROJECT_ID] [-rcl URL] [-lcl FILE]") + print("") + print("optional arguments:") + print(" -h, --help show this help message and exit") + print(" -name NAME name of the project") + print(" -version VERSION version of the project") + print(" -id PROJECT_ID SW360 id of the project, supersedes name and version parameters") + print(" -t SW360_TOKEN use this token for access to SW360") + print(" -oa, this is an oauth2 token") + print(" -url SW360_URL use this URL for access to SW360") + print(" -v be verbose") + print(" -rcl read the component check list file from the URL specified") + print(" -lcl read the component check list file from local") + print(" --forceerror force an error exit code in case of validation errors or warnings") + return + + print_text("Reading component check list from component_checks.json...") + try: + self.ComponentCheck.read_component_check_list(args.remote_check_list, args.local_checklist_list) + except Exception as ex: + print_red("Error reading component check list " + repr(ex)) + sys.exit(ResultCode.RESULT_GENERAL_ERROR) + if len(self.ComponentCheck.component_check_list) > 0: + print(" Got component checklist.") + + self.ComponentCheck.files_to_ignore = self.ComponentCheck.component_check_list.get("files_to_ignore", []) + print_text(f" {len(self.ComponentCheck.files_to_ignore)} components will be ignored.") + + if not self.login(token=args.sw360_token, url=args.sw360_url, oauth2=args.oauth2): + print_red("ERROR: login failed!") + sys.exit(ResultCode.RESULT_AUTH_ERROR) + + name: str = args.name + version: str = "" + pid: str = "" + if args.version: + version = args.version + + if args.id: + pid = args.id + elif (args.name and args.version): + # find_project() is part of script_base.py + pid = self.find_project(name, version) + else: + print_red("Neither name and version nor project id specified!") + sys.exit(ResultCode.RESULT_COMMAND_ERROR) + + result = 0 + if pid: + result = self.check_bom_items(pid) + + print_text("\nDone.") + else: + print_yellow(" No matching project found") + + if result > 0: + if args.force_error: + if args.verbose: + print_yellow("Special component(s) found, exiting with code != 0") + sys.exit(ResultCode.RESULT_SPECIAL_COMPONENT_FOUND) diff --git a/Dependency Detection.md b/documentation/Dependency Detection.md similarity index 100% rename from Dependency Detection.md rename to documentation/Dependency Detection.md diff --git a/Readme_BOM.md b/documentation/Readme_BOM.md similarity index 100% rename from Readme_BOM.md rename to documentation/Readme_BOM.md diff --git a/documentation/Readme_Componentcheck.md b/documentation/Readme_Componentcheck.md new file mode 100644 index 0000000..3133d7a --- /dev/null +++ b/documentation/Readme_Componentcheck.md @@ -0,0 +1,78 @@ + + +# Component Check + +The command `bom componentcheck` checks a given SBOM for special components. +The command `project componentcheck`. The first one does the same for +an existing SW360 project. + +## Why are these commands helpful? + +The primary goal of CaPyCLI is to support license compliance. In most cases +license compliance focuses on the third-party software components that are +shipped to customers as part of a product or at least made available for customers. + +Obviously this includes only components that are actively used by the application. +Unit test tools like `junit` or `pytest`, linter like `eslint`, mocking frameworks +like `Moq`, etc. are not used by the application. Therefore they should not appear +in a SBOMs or in a SW360 project. These are these **Special components** which +are reported by the `componentcheck` commands. + +A second category are Python components that contain additional binary dependencies +in the wheel files. They are reported, because normal license compliance checks +may not show all licenses that apply. + +## More Details + +Both `componentcheck` commands use a list of special components which is part of +CaPyCLI (see data/component_checks.json). This list has a section for development +dependencies, grouped by package manager, a section for python binary components +and a section for files to get excluded from these checks. + +´´´json +{ + "dev_dependencies": { + "maven": [ + { "namespace": "org.eclipse.jdt", "name": "junit" }, + ... + ], + "npm": [ + { "namespace": "", "name": "mocha" }, + ... + ], + "pypi": [ + { "namespace": "", "name": "pytest" }, + ... + ], + "nuget": [ + { "namespace": "", "name": "NUnit" }, + ... + ], + "gem": [ ... ], + "composer": [ ... ], + "golang": [ ... ] + }, + "python_binary_components": [ + { "namespace": "", "name": "numpy" }, + ... + ], + "files_to_ignore": [] +} +´´´ + +The current list is only a starting point and we are also aware that there are too +many different projects out there with too many different use cases and components. +Also the list covers only some of the existing software ecosystems. +Therefore it is possible for projects to provide there own lists, either as a local +file (parameter `-lcl` or `--local-checklist`) or as a download URL (parameter +`-lcr` or `--remote-checklist`). + +## How is the check done? + +For SBOMs the preferred information to determine a component is the package-url. +If it is not available, only the component name is used. + +For SW360 project always only the component name is used. diff --git a/Readme_Filtering.md b/documentation/Readme_Filtering.md similarity index 100% rename from Readme_Filtering.md rename to documentation/Readme_Filtering.md diff --git a/Readme_Mapping.md b/documentation/Readme_Mapping.md similarity index 100% rename from Readme_Mapping.md rename to documentation/Readme_Mapping.md diff --git a/Readme_Workflow.md b/documentation/Readme_Workflow.md similarity index 100% rename from Readme_Workflow.md rename to documentation/Readme_Workflow.md diff --git a/SoftwareClearingApproachOverview.md b/documentation/SoftwareClearingApproachOverview.md similarity index 100% rename from SoftwareClearingApproachOverview.md rename to documentation/SoftwareClearingApproachOverview.md diff --git a/UseCaseOverview.md b/documentation/UseCaseOverview.md similarity index 100% rename from UseCaseOverview.md rename to documentation/UseCaseOverview.md diff --git a/tests/fixtures/component_checks_extra.json b/tests/fixtures/component_checks_extra.json new file mode 100644 index 0000000..0545afd --- /dev/null +++ b/tests/fixtures/component_checks_extra.json @@ -0,0 +1,215 @@ +{ + "dev_dependencies": { + "maven": [ + { "namespace": "org.eclipse.jdt", "name": "junit" }, + { "namespace": "org.junit.jupiter", "name": "junit-jupiter" }, + { "namespace": "org.mockito", "name": "mockito-core" }, + { "namespace": "org.assertj", "name": "assertj-core" }, + { "namespace": "org.hamcrest", "name": "hamcrest" }, + { "namespace": "org.spockframework", "name": "spock-core" }, + { "namespace": "org.easymock", "name": "easymock" }, + { "namespace": "org.powermock", "name": "powermock-module-junit4" }, + { "namespace": "org.mockito", "name": "mockito-junit-jupiter" }, + { "namespace": "org.jacoco", "name": "org.jacoco.core" }, + { "namespace": "org.scoverage", "name": "scalac-scoverage-runtime_2.12" }, + { "namespace": "org.sonarsource.scanner.maven", "name": "sonar-maven-plugin" }, + { "namespace": "com.puppycrawl.tools", "name": "checkstyle" }, + { "namespace": "pmd", "name": "pmd" }, + { "namespace": "org.codehaus.mojo", "name": "animal-sniffer-maven-plugin" }, + { "namespace": "org.apache.maven.plugins", "name": "maven-surefire-plugin" }, + { "namespace": "org.apache.maven.plugins", "name": "maven-failsafe-plugin" }, + { "namespace": "org.apache.maven.plugins", "name": "maven-checkstyle-plugin" }, + { "namespace": "org.apache.maven.plugins", "name": "maven-pmd-plugin" } + ], + "npm": [ + { "namespace": "", "name": "mocha" }, + { "namespace": "", "name": "jest" }, + { "namespace": "", "name": "eslint" }, + { "namespace": "", "name": "tslint" }, + { "namespace": "", "name": "typescript" }, + { "namespace": "", "name": "ts-node" }, + { "namespace": "@types", "name": "node" }, + { "namespace": "", "name": "babel-jest" }, + { "namespace": "@babel", "name": "core" }, + { "namespace": "", "name": "react-testing-library" }, + { "namespace": "@testing-library", "name": "react" }, + { "namespace": "", "name": "jasmine" }, + { "namespace": "", "name": "sinon" }, + { "namespace": "", "name": "chai" }, + { "namespace": "", "name": "nyc" }, + { "namespace": "", "name": "ava" }, + { "namespace": "", "name": "standard" }, + { "namespace": "", "name": "prettier" }, + { "namespace": "", "name": "webpack" }, + { "namespace": "", "name": "rollup" }, + { "namespace": "", "name": "esbuild" }, + { "namespace": "", "name": "nodemon" }, + { "namespace": "", "name": "parcel" }, + { "namespace": "", "name": "stylelint" }, + { "namespace": "", "name": "husky" }, + { "namespace": "", "name": "lint-staged" }, + { "namespace": "", "name": "cypress" }, + { "namespace": "", "name": "protractor" }, + { "namespace": "", "name": "karma" }, + { "namespace": "", "name": "karma-jasmine" }, + { "namespace": "", "name": "dtslint" }, + { "namespace": "", "name": "ts-jest" }, + { "namespace": "@typescript-eslint", "name": "eslint-plugin" }, + { "namespace": "@typescript-eslint", "name": "parser" }, + { "namespace": "", "name": "jshint" }, + { "namespace": "", "name": "gulp" }, + { "namespace": "@eslint", "name": "js" }, + { "namespace": "", "name": "chai" }, + { "namespace": "", "name": "jest" } + ], + "pypi": [ + { "namespace": "", "name": "pytest" }, + { "namespace": "", "name": "tox" }, + { "namespace": "", "name": "coverage" }, + { "namespace": "", "name": "mypy" }, + { "namespace": "", "name": "flake8" }, + { "namespace": "", "name": "black" }, + { "namespace": "", "name": "isort" }, + { "namespace": "", "name": "pylint" }, + { "namespace": "", "name": "pytest-cov" }, + { "namespace": "", "name": "nose" }, + { "namespace": "", "name": "hypothesis" }, + { "namespace": "", "name": "pre-commit" }, + { "namespace": "", "name": "pytest-mock" }, + { "namespace": "", "name": "mock" }, + { "namespace": "", "name": "Sphinx" }, + { "namespace": "", "name": "sphinx-autodoc-typehints" }, + { "namespace": "", "name": "sphinx-rtd-theme" }, + { "namespace": "", "name": "bandit" }, + { "namespace": "", "name": "pytest-xdist" }, + { "namespace": "", "name": "pytest-asyncio" }, + { "namespace": "", "name": "pytest-django" }, + { "namespace": "", "name": "faker" }, + { "namespace": "", "name": "radon" }, + { "namespace": "", "name": "pyflakes" }, + { "namespace": "", "name": "pep8" }, + { "namespace": "", "name": "autopep8" }, + { "namespace": "", "name": "pydocstyle" }, + { "namespace": "", "name": "pycodestyle" }, + { "namespace": "", "name": "pytest-runner" }, + { "namespace": "", "name": "pytest-timeout" }, + { "namespace": "", "name": "pytest-html" }, + { "namespace": "", "name": "pytest-ordering" }, + { "namespace": "", "name": "pytest-sugar" }, + { "namespace": "", "name": "pytest-testmon" }, + { "namespace": "", "name": "twine" }, + { "namespace": "", "name": "wheel" }, + { "namespace": "", "name": "setuptools" }, + { "namespace": "", "name": "pip-tools" }, + { "namespace": "", "name": "check-manifest" }, + { "namespace": "", "name": "types-mock" }, + { "namespace": "", "name": "types-requests" }, + { "namespace": "", "name": "types-setuptools" } + ], + "nuget": [ + { "namespace": "", "name": "NUnit" }, + { "namespace": "", "name": "xunit" }, + { "namespace": "", "name": "Moq" }, + { "namespace": "", "name": "coverlet.collector" }, + { "namespace": "", "name": "Microsoft.NET.Test.Sdk" }, + { "namespace": "", "name": "FluentAssertions" }, + { "namespace": "", "name": "StyleCop.Analyzers" }, + { "namespace": "", "name": "JetBrains.Annotations" }, + { "namespace": "", "name": "FakeItEasy" }, + { "namespace": "", "name": "NSubstitute" }, + { "namespace": "", "name": "AutoFixture" }, + { "namespace": "", "name": "Selenium.WebDriver" }, + { "namespace": "", "name": "Microsoft.CodeAnalysis.FxCopAnalyzers" }, + { "namespace": "", "name": "nunit3testadapter" }, + { "namespace": "", "name": "xunit.runner.visualstudio" }, + { "namespace": "", "name": "Coverlet.MSBuild" }, + { "namespace": "", "name": "BenchmarkDotNet" }, + { "namespace": "", "name": "Microsoft.CodeAnalysis.Workspaces.MSBuild" }, + { "namespace": "", "name": "Microsoft.VisualStudio.SolutionPersistence" }, + { "namespace": "", "name": "msbuild" }, + { "namespace": "", "name": "Microsoft.CodeAnalysis.Analyzers" }, + { "namespace": "", "name": "Microsoft.CodeAnalysis.CSharp.Analyzers" }, + { "namespace": "", "name": "Microsoft.CodeAnalysis.CSharp.NetAnalyzers" }, + { "namespace": "", "name": "Microsoft.CodeAnalysis.NetAnalyzers" }, + { "namespace": "", "name": "Microsoft.Interop.ComInterfaceGenerator" }, + { "namespace": "", "name": "Microsoft.Interop.JavaScript.JSImportGenerator" }, + { "namespace": "", "name": "Microsoft.Interop.LibraryImportGenerator" }, + { "namespace": "", "name": "Microsoft.Interop.SourceGeneration" }, + { "namespace": "", "name": "StyleCop.Analyzers.CodeFixes" }, + { "namespace": "", "name": "System.Text.Json.SourceGeneration" }, + { "namespace": "", "name": "System.Text.RegularExpressions.Generator" }, + { "namespace": "", "name": "System.Windows.Forms.Analyzers" }, + { "namespace": "", "name": "System.Windows.Forms.Analyzers.CSharp" } + ], + "gem": [ + { "namespace": "", "name": "rspec" }, + { "namespace": "", "name": "rubocop" }, + { "namespace": "", "name": "simplecov" }, + { "namespace": "", "name": "factory_bot" }, + { "namespace": "", "name": "cucumber" }, + { "namespace": "", "name": "capybara" }, + { "namespace": "", "name": "minitest" }, + { "namespace": "", "name": "faker" }, + { "namespace": "", "name": "shoulda" }, + { "namespace": "", "name": "shoulda-matchers" }, + { "namespace": "", "name": "yard" }, + { "namespace": "", "name": "guard" }, + { "namespace": "", "name": "guard-rspec" }, + { "namespace": "", "name": "bundler-audit" }, + { "namespace": "", "name": "reek" }, + { "namespace": "", "name": "pry" }, + { "namespace": "", "name": "test-unit" }, + { "namespace": "", "name": "database_cleaner" }, + { "namespace": "", "name": "rspec-rails" }, + { "namespace": "", "name": "rspec-mocks" } + ], + "composer": [ + { "namespace": "phpunit", "name": "phpunit" }, + { "namespace": "squizlabs", "name": "php_codesniffer" }, + { "namespace": "phpstan", "name": "phpstan" }, + { "namespace": "friendsofphp", "name": "php-cs-fixer" }, + { "namespace": "phpmd", "name": "phpmd" }, + { "namespace": "symfony", "name": "var-dumper" }, + { "namespace": "phpspec", "name": "phpspec" }, + { "namespace": "php-parallel-lint", "name": "php-parallel-lint" }, + { "namespace": "mockery", "name": "mockery" }, + { "namespace": "sebastian", "name": "phpcpd" }, + { "namespace": "liuggio", "name": "statsd-php-client" }, + { "namespace": "hamcrest", "name": "hamcrest-php" }, + { "namespace": "psy", "name": "psysh" }, + { "namespace": "fakerphp", "name": "faker" }, + { "namespace": "codeception", "name": "codeception" } + ], + "golang": [ + { "namespace": "github.com/stretchr", "name": "testify" }, + { "namespace": "github.com/golang", "name": "mock" }, + { "namespace": "github.com/onsi", "name": "ginkgo" }, + { "namespace": "github.com/onsi", "name": "gomega" }, + { "namespace": "golang.org/x", "name": "lint" }, + { "namespace": "github.com/goreleaser", "name": "goreleaser" }, + { "namespace": "github.com/axw", "name": "gocov" }, + { "namespace": "github.com/mattn", "name": "goveralls" }, + { "namespace": "github.com/securego", "name": "gosec" }, + { "namespace": "github.com/uudashr", "name": "gocognit" }, + { "namespace": "github.com/DATA-DOG", "name": "go-sqlmock" }, + { "namespace": "honnef.co/go", "name": "tools" }, + { "namespace": "github.com/bouk", "name": "monkey" }, + { "namespace": "github.com/cweill", "name": "gotests" }, + { "namespace": "github.com/vektra", "name": "mockery" } + ] + }, + "python_binary_components": [ + { "namespace": "", "name": "numpy" }, + { "namespace": "", "name": "opencv_python_headless" }, + { "namespace": "", "name": "pandasx" }, + { "namespace": "", "name": "pillow" }, + { "namespace": "", "name": "scikit_learn" }, + { "namespace": "", "name": "scipy" }, + { "namespace": "", "name": "shapely" }, + { "namespace": "", "name": "torch" }, + { "namespace": "", "name": "torchvision" } + ], + "files_to_ignore": [ + { "namespace": "", "name": "pytest" } + ] +} diff --git a/tests/fixtures/pyproject.toml b/tests/fixtures/pyproject.toml index 628174d..b98d36d 100644 --- a/tests/fixtures/pyproject.toml +++ b/tests/fixtures/pyproject.toml @@ -14,8 +14,10 @@ keywords = ["sw360", "cli, automation", "license", "compliance", "clearing"] include = [ "LICENSE.md", { path = "capycli/data/granularity_list.csv", format = "wheel" }, + { path = "capycli/data/component_checks.json", format = "wheel" }, { path = "capycli/data/__init__.py", format = "wheel" }, { path = "capycli/data/granularity_list.csv", format = "sdist" }, + { path = "capycli/data/component_checks.json", format = "sdist" }, { path = "capycli/data/__init__.py", format = "sdist" }, ] classifiers = [ diff --git a/tests/fixtures/sbom_for_component_check.json b/tests/fixtures/sbom_for_component_check.json new file mode 100644 index 0000000..db7882d --- /dev/null +++ b/tests/fixtures/sbom_for_component_check.json @@ -0,0 +1,351 @@ +{ + "components": [ + { + "bom-ref": "pkg:pypi/pytest@7.4.3", + "description": "xxx", + "externalReferences": [ + { + "comment": "binary (download location)", + "hashes": [ + { + "alg": "SHA-256", + "content": "16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11" + } + ], + "type": "distribution", + "url": "https://pypi.org/project/attrs/25.4.0" + }, + { + "comment": "binary (download location)", + "hashes": [ + { + "alg": "SHA-256", + "content": "adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373" + } + ], + "type": "distribution", + "url": "https://pypi.org/project/attrs/25.4.0" + } + ], + "name": "pytest", + "properties": [ + { + "name": "siemens:primaryLanguage", + "value": "Python" + } + ], + "purl": "pkg:pypi/attrs@25.4.0", + "type": "library", + "version": "7.4.3" + }, + { + "bom-ref": "pkg:pypi/beautifulsoup4@4.14.3", + "description": "Screen-scraping library", + "externalReferences": [ + { + "comment": "binary (download location)", + "hashes": [ + { + "alg": "SHA-256", + "content": "0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb" + } + ], + "type": "distribution", + "url": "https://pypi.org/project/beautifulsoup4/4.14.3" + }, + { + "comment": "binary (download location)", + "hashes": [ + { + "alg": "SHA-256", + "content": "6292b1c5186d356bba669ef9f7f051757099565ad9ada5dd630bd9de5fa7fb86" + } + ], + "type": "distribution", + "url": "https://pypi.org/project/beautifulsoup4/4.14.3" + } + ], + "name": "beautifulsoup4", + "properties": [ + { + "name": "siemens:primaryLanguage", + "value": "Python" + } + ], + "purl": "pkg:pypi/beautifulsoup4@4.14.3", + "type": "library", + "version": "4.14.3" + }, + { + "bom-ref": "pkg:pypi/pandas@5.0", + "description": "Define boolean algebras, create and parse boolean expressions and create custom boolean DSL.", + "externalReferences": [ + { + "comment": "binary (download location)", + "hashes": [ + { + "alg": "SHA-256", + "content": "60cbc4bad079753721d32649545505362c754e121570ada4658b852a3a318d95" + } + ], + "type": "distribution", + "url": "https://pypi.org/project/boolean-py/5.0" + }, + { + "comment": "binary (download location)", + "hashes": [ + { + "alg": "SHA-256", + "content": "ef28a70bd43115208441b53a045d1549e2f0ec6e3d08a9d142cbc41c1938e8d9" + } + ], + "type": "distribution", + "url": "https://pypi.org/project/boolean-py/5.0" + } + ], + "name": "pandas", + "properties": [ + { + "name": "siemens:primaryLanguage", + "value": "Python" + } + ], + "purl": "pkg:pypi/pandas@5.0", + "type": "library", + "version": "5.0" + }, + { + "bom-ref": "pkg:maven/org.eclipse.jdt/junit@1.2.3", + "name": "junit", + "properties": [ + { + "name": "siemens:primaryLanguage", + "value": "Java" + } + ], + "purl": "pkg:maven/org.eclipse.jdt/junit@1.2.3", + "type": "library", + "version": "123" + }, + { + "bom-ref": "pkg:npm/gulp@1.2.3", + "name": "gulp", + "properties": [ + { + "name": "siemens:primaryLanguage", + "value": "JavaScript" + } + ], + "type": "library", + "version": "123" + }, + { + "bom-ref": "pkg:github.com/stretchr/testify@0.38.4", + "externalReferences": [], + "name": "testify", + "group": "github.com/stretchr", + "properties": [ + { + "name": "siemens:primaryLanguage", + "value": "Go" + } + ], + "purl": "pkg:nuget/github.com/stretchr/testify@9.9.9", + "type": "library", + "version": "9.9.9" + } + ], + "definitions": { + "standards": [ + { + "bom-ref": "standard-bom", + "description": "The Standard for Software Bills of Materials in Siemens", + "externalReferences": [ + { + "type": "website", + "url": "https://sbom.siemens.io/" + } + ], + "name": "Standard BOM", + "owner": "Siemens AG", + "version": "3.0.0" + } + ] + }, + "dependencies": [ + { + "ref": "pkg:pypi/attrs@25.4.0" + }, + { + "ref": "pkg:pypi/beautifulsoup4@4.14.3" + }, + { + "ref": "pkg:pypi/boolean-py@5.0" + }, + { + "ref": "pkg:pypi/certifi@2025.11.12" + }, + { + "ref": "pkg:pypi/chardet@5.2.0" + }, + { + "ref": "pkg:pypi/charset-normalizer@3.4.4" + }, + { + "ref": "pkg:pypi/cli-support@2.0.1" + }, + { + "ref": "pkg:pypi/colorama@0.4.6" + }, + { + "ref": "pkg:pypi/cyclonedx-python-lib@11.6.0" + }, + { + "ref": "pkg:pypi/dateparser@1.2.2" + }, + { + "ref": "pkg:pypi/defusedxml@0.7.1" + }, + { + "ref": "pkg:pypi/et-xmlfile@2.0.0" + }, + { + "ref": "pkg:pypi/halo@0.0.31" + }, + { + "ref": "pkg:pypi/idna@3.11" + }, + { + "ref": "pkg:pypi/importlib-resources@5.13.0" + }, + { + "ref": "pkg:pypi/jsonschema-specifications@2025.9.1" + }, + { + "ref": "pkg:pypi/jsonschema@4.25.1" + }, + { + "ref": "pkg:pypi/license-expression@30.4.4" + }, + { + "ref": "pkg:pypi/log-symbols@0.0.14" + }, + { + "ref": "pkg:pypi/openpyxl@3.1.5" + }, + { + "ref": "pkg:pypi/packageurl-python@0.15.6" + }, + { + "ref": "pkg:pypi/packaging@25.0" + }, + { + "ref": "pkg:pypi/py-serializable@2.1.0" + }, + { + "ref": "pkg:pypi/pyjwt@2.10.1" + }, + { + "ref": "pkg:pypi/python-dateutil@2.9.0.post0" + }, + { + "ref": "pkg:pypi/pytz@2025.2" + }, + { + "ref": "pkg:pypi/referencing@0.37.0" + }, + { + "ref": "pkg:pypi/regex@2025.11.3" + }, + { + "ref": "pkg:pypi/requests@2.32.5" + }, + { + "ref": "pkg:pypi/requirements-parser@0.11.0" + }, + { + "ref": "pkg:pypi/rpds-py@0.30.0" + }, + { + "ref": "pkg:pypi/semver@3.0.2" + }, + { + "ref": "pkg:pypi/six@1.17.0" + }, + { + "ref": "pkg:pypi/sortedcontainers@2.4.0" + }, + { + "ref": "pkg:pypi/soupsieve@2.8.1" + }, + { + "ref": "pkg:pypi/spinners@0.0.24" + }, + { + "ref": "pkg:pypi/sw360@1.10.1" + }, + { + "ref": "pkg:pypi/termcolor@3.3.0" + }, + { + "ref": "pkg:pypi/types-setuptools@80.9.0.20251223" + }, + { + "ref": "pkg:pypi/typing-extensions@4.15.0" + }, + { + "ref": "pkg:pypi/tzdata@2025.3" + }, + { + "ref": "pkg:pypi/tzlocal@5.3.1" + }, + { + "ref": "pkg:pypi/urllib3@2.6.2" + }, + { + "ref": "pkg:pypi/validation@0.8.3" + }, + { + "ref": "pkg:pypi/wheel@0.38.4" + } + ], + "metadata": { + "licenses": [ + { + "license": { + "id": "CC0-1.0" + } + } + ], + "properties": [ + { + "name": "siemens:profile", + "value": "clearing" + } + ], + "timestamp": "2025-12-30T08:44:56.941306+00:00", + "tools": { + "components": [ + { + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/sw360/capycli" + } + ], + "name": "CaPyCLI", + "supplier": { + "name": "Siemens AG" + }, + "type": "application", + "version": "2.10.0.dev1" + } + ] + } + }, + "serialNumber": "urn:uuid:1e5f9a27-074b-457c-9db0-a55151346fee", + "version": 1, + "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.6" +} diff --git a/tests/test_base.py b/tests/test_base.py index cda8108..390248d 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -1,5 +1,5 @@ # ------------------------------------------------------------------------------- -# Copyright (c) 2021-2025 Siemens +# Copyright (c) 2021-2026 Siemens # All Rights Reserved. # Author: thomas.graf@siemens.com, manuel.schaffer@siemens.com # @@ -69,6 +69,8 @@ def __init__(self) -> None: self.force_error: bool = False self.project_mainline_state = "" self.copy_from = "" + self.remote_check_list: str = "" + self.local_checklist_list: str = "" class TestBasePytest: diff --git a/tests/test_check_components.py b/tests/test_check_components.py new file mode 100644 index 0000000..643b7df --- /dev/null +++ b/tests/test_check_components.py @@ -0,0 +1,220 @@ +# ------------------------------------------------------------------------------- +# Copyright (c) 2026 Siemens +# All Rights Reserved. +# Author: thomas.graf@siemens.com +# +# SPDX-License-Identifier: MIT +# ------------------------------------------------------------------------------- + +import os +from unittest.mock import mock_open, patch + +import responses + +from capycli.bom.component_check import ComponentCheck +from capycli.main.result_codes import ResultCode +from tests.test_base import AppArguments, TestBase + + +class TestComponentCheck(TestBase): + INPUTFILE1 = "sbom_for_component_check.json" + INPUTFILE2 = "component_checks_extra.json" + INPUT_INVALID = "plaintext.txt" + + def test_show_help(self) -> None: + sut = ComponentCheck() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("bom") + args.command.append("componentcheck") + args.help = True + + out = self.capture_stdout(sut.run, args) + self.assertTrue("usage: CaPyCli bom componentcheck" in out) + + def test_no_input_file_specified(self) -> None: + try: + sut = ComponentCheck() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("bom") + args.command.append("componentcheck") + + sut.run(args) + self.assertTrue(False, "Failed to report missing argument") + except SystemExit as ex: + self.assertEqual(ResultCode.RESULT_COMMAND_ERROR, ex.code) + + def test_file_not_found(self) -> None: + try: + sut = ComponentCheck() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("bom") + args.command.append("componentcheck") + args.inputfile = "DOESNOTEXIST" + + sut.run(args) + self.assertTrue(False, "Failed to report missing file") + except SystemExit as ex: + self.assertEqual(ResultCode.RESULT_FILE_NOT_FOUND, ex.code) + + def test_error_reading_bom(self) -> None: + try: + sut = ComponentCheck() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("bom") + args.command.append("componentcheck") + args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", self.INPUT_INVALID) + + sut.run(args) + self.assertTrue(False, "Failed to report missing file") + except SystemExit as ex: + self.assertEqual(ResultCode.RESULT_ERROR_READING_BOM, ex.code) + + def test_real_bom1(self) -> None: + sut = ComponentCheck() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("bom") + args.command.append("componentcheck") + args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", self.INPUTFILE1) + args.verbose = True + + out = self.capture_stdout(sut.run, args) + self.assertTrue(self.INPUTFILE1 in out) + self.assertTrue("Reading component check list from component_checks.json..." in out) + self.assertTrue("Got component checklist." in out) + self.assertTrue("0 components will be ignored." in out) + self.assertTrue("6 components read from SBOM" in out) + self.assertTrue("pandas 5.0 is known as a Python component that has additional binary dependencies" in out) + self.assertTrue("pytest 7.4.3 seems to be a development dependency" in out) + + def test_real_bom2(self) -> None: + sut = ComponentCheck() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("bom") + args.command.append("componentcheck") + args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", self.INPUTFILE1) + args.local_checklist_list = os.path.join(os.path.dirname(__file__), "fixtures", self.INPUTFILE2) + args.debug = True + args.search_meta_data = True + + out = self.capture_stdout(sut.run, args) + self.assertTrue(self.INPUTFILE1 in out) + self.assertTrue("Reading component check list from component_checks.json..." in out) + self.assertTrue("Got component checklist." in out) + self.assertTrue("1 components will be ignored." in out) + self.assertTrue("gulp 123 seems to be a development dependency" in out) + self.assertTrue("junit 123 seems to be a development dependency" in out) + self.assertFalse("pytest 7.4.3 seems to be a development dependency" in out) + + @responses.activate + def test_read_granularity_list_local(self) -> None: + check_components = ComponentCheck() + read_data = ''' +{ + "dev_dependencies": { + "maven": [ + { "namespace": "org.eclipse.jdt", "name": "zjunitz" } + ] + }, + "python_binary_components": [], + "files_to_ignore": [] +} + ''' + # with patch('builtins.open', new_callable=mock_open(read_data=read_data)) as mock_file: + with patch("builtins.open", mock_open(read_data=read_data)) as mock_file: + + check_components.read_component_check_list(local_check_list_file="component_checks.json") + mock_file.assert_called_once_with("component_checks.json", "r", encoding="utf-8") + self.assertEqual(check_components.component_check_list["dev_dependencies"]["maven"][0]["name"], "zjunitz") + + self.delete_file("component_checks.json") + + @responses.activate + def test_read_granularity_list_local_file_not_found(self) -> None: + sut = ComponentCheck() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("bom") + args.command.append("componentcheck") + args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", self.INPUTFILE1) + args.local_checklist_list = "DOESNOTEXIST" + + sut.run(args) + out = self.capture_stdout(sut.run, args) + self.assertTrue("File not found: [Errno 2] No such file or directory: 'DOESNOTEXIST'" in out) + + @responses.activate + def test_read_granularity_list_local_invalid_file(self) -> None: + sut = ComponentCheck() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("bom") + args.command.append("componentcheck") + args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", self.INPUTFILE1) + args.local_checklist_list = os.path.join(os.path.dirname(__file__), "fixtures", "maven-raw.txt") + + sut.run(args) + out = self.capture_stdout(sut.run, args) + self.assertTrue("An unexpected error occurred: " in out) + + @responses.activate + def test_read_granularity_list_download(self) -> None: + check_components = ComponentCheck() + body_data = ''' +{ + "dev_dependencies": { + "maven": [ + { "namespace": "org.eclipse.jdt", "name": "xjunitx" } + ] + }, + "python_binary_components": [], + "files_to_ignore": [] +} + ''' + responses.add(responses.GET, 'http://example.com/component_checks_extra.json', body=body_data) + + check_components.read_component_check_list(download_url='http://example.com/component_checks_extra.json') + self.assertEqual(len(responses.calls), 1) + self.assertEqual(responses.calls[0].request.url, 'http://example.com/component_checks_extra.json') + self.assertEqual(check_components.component_check_list["dev_dependencies"]["maven"][0]["name"], "xjunitx") + + self.delete_file("component_checks.json") + + @responses.activate + def test_read_component_check_download_error(self) -> None: + responses.add(responses.GET, 'http://wrongurl.com/granularity.csv', status=500) + check_granularity = ComponentCheck() + + check_granularity.read_component_check_list(download_url='http://wrongurl.com/granularity.csv') + + self.assertEqual(len(responses.calls), 1) + self.assertEqual(responses.calls[0].request.url, 'http://wrongurl.com/granularity.csv') + + # check default fallback + self.assertEqual(check_granularity.component_check_list["dev_dependencies"]["maven"][0]["name"], "junit") + + +if __name__ == '__main__': + APP = TestComponentCheck() + APP.test_read_granularity_list_local_file_not_found() diff --git a/tests/test_project_component_check.py b/tests/test_project_component_check.py new file mode 100644 index 0000000..10fe258 --- /dev/null +++ b/tests/test_project_component_check.py @@ -0,0 +1,265 @@ +# ------------------------------------------------------------------------------- +# Copyright (c) 2026 Siemens +# All Rights Reserved. +# Author: thomas.graf@siemens.com +# +# SPDX-License-Identifier: MIT +# ------------------------------------------------------------------------------- + +from typing import Any, Dict +from unittest.mock import MagicMock + +import responses + +from capycli.main.result_codes import ResultCode +from capycli.project.project_component_check import ProjectComponentCheck +from tests.test_base import AppArguments, TestBase + + +class TestProjectComponentCheck(TestBase): + def setUp(self) -> None: + self.client_mock = MagicMock() + self.sut = ProjectComponentCheck() + self.sut.client = self.client_mock + + def test_show_help(self) -> None: + sut = ProjectComponentCheck() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("project") + args.command.append("componentcheck") + args.help = True + + out = self.capture_stdout(sut.run, args) + self.assertTrue("usage: CaPyCli project componentcheck" in out) + + @responses.activate + def test_no_login(self) -> None: + sut = ProjectComponentCheck() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("project") + args.command.append("componentcheck") + args.sw360_url = "https://myserver.com" + args.debug = True + args.verbose = True + + try: + sut.run(args) + self.assertTrue(False, "Failed to report login failure") + except SystemExit as ex: + self.assertEqual(ResultCode.RESULT_AUTH_ERROR, ex.code) + + @responses.activate + def test_no_project_identification(self) -> None: + sut = ProjectComponentCheck() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("project") + args.command.append("componentcheck") + args.debug = True + args.verbose = True + args.sw360_token = TestBase.MYTOKEN + args.sw360_url = TestBase.MYURL + + self.add_login_response() + + try: + sut.run(args) + self.assertTrue(False, "Failed to report login failure") + except SystemExit as ex: + self.assertEqual(ResultCode.RESULT_COMMAND_ERROR, ex.code) + + @responses.activate + def test_project_not_found(self) -> None: + sut = ProjectComponentCheck() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("project") + args.command.append("componentcheck") + args.sw360_token = TestBase.MYTOKEN + args.sw360_url = TestBase.MYURL + args.debug = True + args.verbose = True + args.id = "34ef5c5452014c52aa9ce4bc180624d8" + + self.add_login_response() + + # purl cache: components + responses.add( + responses.GET, + url=self.MYURL + "resource/api/projects/34ef5c5452014c52aa9ce4bc180624d8", + body="""{}""", + status=404, + content_type="application/json", + adding_headers={"Authorization": "Token " + self.MYTOKEN}, + ) + + try: + sut.run(args) + self.assertTrue(False, "Failed to report login failure") + except SystemExit as ex: + self.assertEqual(ResultCode.RESULT_ERROR_ACCESSING_SW360, ex.code) + + @staticmethod + def get_project_for_test() -> Dict[str, Any]: + """ + Return a SW360 project for unit testing. + """ + project = { + "name": "CaPyCLI", + "description": "Software clearing for CaPyCLI, the clearing automation scripts for Python", + "version": "1.9.0", + "externalIds": { + "com.siemens.code.project.id": "69287" + }, + "additionalData": {}, + "createdOn": "2023-03-14", + "businessUnit": "SI", + "state": "ACTIVE", + "tag": "Demo", + "clearingState": "IN_PROGRESS", + "projectResponsible": "thomas.graf@siemens.com", + "roles": {}, + "securityResponsibles": [ + "thomas.graf@siemens.com" + ], + "projectOwner": "thomas.graf@siemens.com", + "ownerAccountingUnit": "", + "ownerGroup": "", + "ownerCountry": "", + "preevaluationDeadline": "", + "systemTestStart": "", + "systemTestEnd": "", + "deliveryStart": "", + "phaseOutSince": "", + "enableSvm": True, + "considerReleasesFromExternalList": False, + "licenseInfoHeaderText": "dummy", + "enableVulnerabilitiesDisplay": True, + "clearingSummary": "", + "specialRisksOSS": "", + "generalRisks3rdParty": "", + "specialRisks3rdParty": "", + "deliveryChannels": "", + "remarksAdditionalRequirements": "", + "projectType": "INNER_SOURCE", + "visibility": "EVERYONE", + "linkedProjects": [], + "linkedReleases": [ + { + "createdBy": "thomas.graf@siemens.com", + "release": "https://my.server.com/resource/api/releases/r001", + "mainlineState": "SPECIFIC", + "comment": "Automatically updated by SCC", + "createdOn": "2023-03-14", + "relation": "UNKNOWN" + }, + { + "createdBy": "thomas.graf@siemens.com", + "release": "https://my.server.com/resource/api/releases/r002", + "mainlineState": "MAINLINE", + "comment": "Automatically updated by SCC", + "createdOn": "2023-03-14", + "relation": "DYNAMICALLY_LINKED" + } + ], + "_links": { + "self": { + "href": "https://my.server.com/resource/api/projects/p001" + } + }, + "_embedded": { + "createdBy": { + "email": "thomas.graf@siemens.com", + "deactivated": False, + "fullName": "Thomas Graf", + "_links": { + "self": { + "href": "https://my.server.com/resource/api/users/byid/thomas.graf%2540siemens.com" + } + } + }, + "sw360:releases": [ + { + "name": "wheelxxx", + "version": "0.38.4", + "_links": { + "self": { + "href": "https://my.server.com/resource/api/releases/r001" + } + } + }, + { + "name": "cli-support", + "version": "1.3", + "_links": { + "self": { + "href": "https://my.server.com/resource/api/releases/r002" + } + } + } + ] + } + } + + return project + + @responses.activate + def test_project_show_by_id(self) -> None: + sut = ProjectComponentCheck() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("project") + args.command.append("componentcheck") + args.sw360_token = TestBase.MYTOKEN + args.sw360_url = TestBase.MYURL + args.debug = True + args.verbose = True + args.id = "p001" + + self.add_login_response() + + # the project + responses.add( + responses.GET, + url=self.MYURL + "resource/api/projects/p001", + json=self.get_project_for_test(), + status=200, + content_type="application/json", + adding_headers={"Authorization": "Token " + self.MYTOKEN}, + ) + + # the release + responses.add( + responses.GET, + url=self.MYURL + "resource/api/releases/r002", + json=self.get_release_cli_for_test(), + status=200, + content_type="application/json", + adding_headers={"Authorization": "Token " + self.MYTOKEN}, + ) + + out = self.capture_stdout(sut.run, args) + # self.dump_textfile(out, "DUMP.TXT") + self.assertTrue("Reading component check list from component_checks.json" in out) + self.assertTrue("Got component checklist." in out) + self.assertTrue("0 components will be ignored." in out) + + self.assertTrue("Retrieving project details..." in out) + + +if __name__ == "__main__": + APP = TestProjectComponentCheck() + APP.setUp() + APP.test_project_show_by_id()