From 25ad3ec12e8fc002c46d62d81986934308c150e8 Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Fri, 7 Mar 2025 10:33:21 +0100 Subject: [PATCH 01/19] CM-45153, CM-45154, CM-45155, CM-45156, CM-45546 - Migrate CLI from Click to Typer (#283) --- .github/workflows/docker-image.yml | 4 +- .github/workflows/pre_release.yml | 6 +- .github/workflows/release.yml | 6 +- .github/workflows/ruff.yml | 4 +- .github/workflows/tests.yml | 4 +- .github/workflows/tests_full.yml | 2 +- CONTRIBUTING.md | 2 +- Dockerfile | 2 +- README.md | 4 +- cycode/cli/__main__.py | 3 + cycode/cli/app.py | 87 ++++++++ cycode/cli/{commands => apps}/__init__.py | 0 cycode/cli/apps/ai_remediation/__init__.py | 6 + .../ai_remediation/ai_remediation_command.py | 32 +++ cycode/cli/apps/ai_remediation/apply_fix.py | 25 +++ .../apps/ai_remediation/print_remediation.py | 15 ++ cycode/cli/apps/auth/__init__.py | 11 ++ cycode/cli/apps/auth/auth_command.py | 28 +++ .../{commands => apps/auth}/auth_common.py | 16 +- .../{commands => apps}/auth/auth_manager.py | 0 cycode/cli/apps/auth/check_command.py | 25 +++ cycode/cli/apps/auth/models.py | 6 + cycode/cli/apps/configure/__init__.py | 8 + .../cli/apps/configure/configure_command.py | 57 ++++++ cycode/cli/apps/configure/consts.py | 19 ++ cycode/cli/apps/configure/messages.py | 37 ++++ cycode/cli/apps/configure/prompts.py | 48 +++++ cycode/cli/apps/ignore/__init__.py | 6 + .../ignore/ignore_command.py | 114 +++++------ cycode/cli/apps/report/__init__.py | 8 + cycode/cli/apps/report/report_command.py | 11 ++ cycode/cli/apps/report/sbom/__init__.py | 12 ++ .../{commands => apps}/report/sbom/common.py | 2 +- .../report/sbom/path}/__init__.py | 0 .../report/sbom/path/path_command.py | 31 +-- .../report/sbom/repository_url}/__init__.py | 0 .../repository_url/repository_url_command.py | 23 +-- cycode/cli/apps/report/sbom/sbom_command.py | 68 +++++++ .../report/sbom/sbom_report_file.py | 0 cycode/cli/apps/scan/__init__.py | 33 ++++ .../{commands => apps}/scan/code_scanner.py | 146 +++++++------- .../scan/commit_history}/__init__.py | 0 .../commit_history/commit_history_command.py | 33 ++++ .../ignore => apps/scan/path}/__init__.py | 0 cycode/cli/apps/scan/path/path_command.py | 25 +++ .../scan/pre_commit}/__init__.py | 0 .../scan/pre_commit/pre_commit_command.py | 24 +-- .../scan/pre_receive}/__init__.py | 0 .../scan/pre_receive/pre_receive_command.py | 27 +-- .../path => apps/scan/repository}/__init__.py | 0 .../scan/repository/repository_command.py | 44 ++--- .../scan/scan_ci}/__init__.py | 0 .../scan/scan_ci/ci_integrations.py | 0 .../cli/apps/scan/scan_ci/scan_ci_command.py | 20 ++ cycode/cli/apps/scan/scan_command.py | 148 ++++++++++++++ cycode/cli/apps/status/__init__.py | 8 + cycode/cli/apps/status/get_cli_status.py | 45 +++++ cycode/cli/apps/status/models.py | 62 ++++++ cycode/cli/apps/status/status_command.py | 15 ++ cycode/cli/apps/status/version_command.py | 15 ++ cycode/cli/cli_types.py | 53 +++++ .../ai_remediation/ai_remediation_command.py | 67 ------- cycode/cli/commands/auth/auth_command.py | 82 -------- .../commands/configure/configure_command.py | 140 ------------- cycode/cli/commands/main_cli.py | 117 ----------- cycode/cli/commands/report/report_command.py | 21 -- .../cli/commands/report/sbom/sbom_command.py | 87 -------- cycode/cli/commands/scan/__init__.py | 0 .../commands/scan/commit_history/__init__.py | 0 .../commit_history/commit_history_command.py | 27 --- cycode/cli/commands/scan/path/__init__.py | 0 cycode/cli/commands/scan/path/path_command.py | 20 -- .../cli/commands/scan/pre_commit/__init__.py | 0 .../cli/commands/scan/pre_receive/__init__.py | 0 .../cli/commands/scan/repository/__init__.py | 0 cycode/cli/commands/scan/scan_ci/__init__.py | 0 .../commands/scan/scan_ci/scan_ci_command.py | 19 -- cycode/cli/commands/scan/scan_command.py | 187 ------------------ cycode/cli/commands/status/__init__.py | 0 cycode/cli/commands/status/status_command.py | 122 ------------ cycode/cli/commands/version/__init__.py | 0 .../cli/commands/version/version_command.py | 22 --- cycode/cli/consts.py | 13 +- .../handle_ai_remediation_errors.py | 8 +- cycode/cli/exceptions/handle_auth_errors.py | 18 ++ .../{common.py => handle_errors.py} | 15 +- .../exceptions/handle_report_sbom_errors.py | 8 +- cycode/cli/exceptions/handle_scan_errors.py | 12 +- cycode/cli/files_collector/excluder.py | 4 +- .../iac/tf_content_generator.py | 2 +- cycode/cli/files_collector/path_documents.py | 4 +- .../sca/base_restore_dependencies.py | 10 +- .../sca/go/restore_go_dependencies.py | 6 +- .../sca/maven/restore_gradle_dependencies.py | 12 +- .../sca/maven/restore_maven_dependencies.py | 6 +- .../sca/npm/restore_npm_dependencies.py | 6 +- .../sca/nuget/restore_nuget_dependencies.py | 6 +- .../files_collector/sca/sca_code_scanner.py | 36 ++-- cycode/cli/main.py | 13 +- cycode/cli/models.py | 29 --- cycode/cli/printers/console_printer.py | 24 +-- cycode/cli/printers/json_printer.py | 2 +- cycode/cli/printers/printer_base.py | 5 +- .../cli/printers/tables/sca_table_printer.py | 12 +- cycode/cli/printers/tables/table_printer.py | 6 +- .../cli/printers/tables/table_printer_base.py | 15 +- cycode/cli/printers/text_printer.py | 13 +- .../cli/user_settings/credentials_manager.py | 2 +- cycode/cli/utils/path_utils.py | 10 +- cycode/cli/utils/scan_batch.py | 77 +++++++- cycode/cli/utils/scan_utils.py | 12 +- cycode/cli/{ => utils}/sentry.py | 0 .../version => utils}/version_checker.py | 13 +- cycode/cyclient/headers.py | 2 +- cycode/cyclient/scan_client.py | 15 +- cycode/cyclient/scan_config_base.py | 6 +- poetry.lock | 125 ++++++++++-- pyproject.toml | 6 +- .../configure/test_configure_command.py | 32 +-- tests/cli/commands/scan/test_code_scanner.py | 6 +- .../test_check_latest_version_on_close.py | 19 +- tests/cli/commands/test_main_command.py | 15 +- .../commands/version/test_version_checker.py | 2 +- .../cli/exceptions/test_handle_scan_errors.py | 18 +- tests/cli/models/test_severity.py | 27 +-- .../scan_config/test_default_scan_config.py | 2 +- .../scan_config/test_dev_scan_config.py | 2 +- tests/cyclient/test_auth_client.py | 2 +- tests/cyclient/test_scan_client.py | 2 +- tests/test_code_scanner.py | 2 +- 130 files changed, 1606 insertions(+), 1417 deletions(-) create mode 100644 cycode/cli/__main__.py create mode 100644 cycode/cli/app.py rename cycode/cli/{commands => apps}/__init__.py (100%) create mode 100644 cycode/cli/apps/ai_remediation/__init__.py create mode 100644 cycode/cli/apps/ai_remediation/ai_remediation_command.py create mode 100644 cycode/cli/apps/ai_remediation/apply_fix.py create mode 100644 cycode/cli/apps/ai_remediation/print_remediation.py create mode 100644 cycode/cli/apps/auth/__init__.py create mode 100644 cycode/cli/apps/auth/auth_command.py rename cycode/cli/{commands => apps/auth}/auth_common.py (75%) rename cycode/cli/{commands => apps}/auth/auth_manager.py (100%) create mode 100644 cycode/cli/apps/auth/check_command.py create mode 100644 cycode/cli/apps/auth/models.py create mode 100644 cycode/cli/apps/configure/__init__.py create mode 100644 cycode/cli/apps/configure/configure_command.py create mode 100644 cycode/cli/apps/configure/consts.py create mode 100644 cycode/cli/apps/configure/messages.py create mode 100644 cycode/cli/apps/configure/prompts.py create mode 100644 cycode/cli/apps/ignore/__init__.py rename cycode/cli/{commands => apps}/ignore/ignore_command.py (60%) create mode 100644 cycode/cli/apps/report/__init__.py create mode 100644 cycode/cli/apps/report/report_command.py create mode 100644 cycode/cli/apps/report/sbom/__init__.py rename cycode/cli/{commands => apps}/report/sbom/common.py (97%) rename cycode/cli/{commands/ai_remediation => apps/report/sbom/path}/__init__.py (100%) rename cycode/cli/{commands => apps}/report/sbom/path/path_command.py (74%) rename cycode/cli/{commands/auth => apps/report/sbom/repository_url}/__init__.py (100%) rename cycode/cli/{commands => apps}/report/sbom/repository_url/repository_url_command.py (73%) create mode 100644 cycode/cli/apps/report/sbom/sbom_command.py rename cycode/cli/{commands => apps}/report/sbom/sbom_report_file.py (100%) create mode 100644 cycode/cli/apps/scan/__init__.py rename cycode/cli/{commands => apps}/scan/code_scanner.py (89%) rename cycode/cli/{commands/configure => apps/scan/commit_history}/__init__.py (100%) create mode 100644 cycode/cli/apps/scan/commit_history/commit_history_command.py rename cycode/cli/{commands/ignore => apps/scan/path}/__init__.py (100%) create mode 100644 cycode/cli/apps/scan/path/path_command.py rename cycode/cli/{commands/report => apps/scan/pre_commit}/__init__.py (100%) rename cycode/cli/{commands => apps}/scan/pre_commit/pre_commit_command.py (64%) rename cycode/cli/{commands/report/sbom => apps/scan/pre_receive}/__init__.py (100%) rename cycode/cli/{commands => apps}/scan/pre_receive/pre_receive_command.py (73%) rename cycode/cli/{commands/report/sbom/path => apps/scan/repository}/__init__.py (100%) rename cycode/cli/{commands => apps}/scan/repository/repository_command.py (66%) rename cycode/cli/{commands/report/sbom/repository_url => apps/scan/scan_ci}/__init__.py (100%) rename cycode/cli/{commands => apps}/scan/scan_ci/ci_integrations.py (100%) create mode 100644 cycode/cli/apps/scan/scan_ci/scan_ci_command.py create mode 100644 cycode/cli/apps/scan/scan_command.py create mode 100644 cycode/cli/apps/status/__init__.py create mode 100644 cycode/cli/apps/status/get_cli_status.py create mode 100644 cycode/cli/apps/status/models.py create mode 100644 cycode/cli/apps/status/status_command.py create mode 100644 cycode/cli/apps/status/version_command.py create mode 100644 cycode/cli/cli_types.py delete mode 100644 cycode/cli/commands/ai_remediation/ai_remediation_command.py delete mode 100644 cycode/cli/commands/auth/auth_command.py delete mode 100644 cycode/cli/commands/configure/configure_command.py delete mode 100644 cycode/cli/commands/main_cli.py delete mode 100644 cycode/cli/commands/report/report_command.py delete mode 100644 cycode/cli/commands/report/sbom/sbom_command.py delete mode 100644 cycode/cli/commands/scan/__init__.py delete mode 100644 cycode/cli/commands/scan/commit_history/__init__.py delete mode 100644 cycode/cli/commands/scan/commit_history/commit_history_command.py delete mode 100644 cycode/cli/commands/scan/path/__init__.py delete mode 100644 cycode/cli/commands/scan/path/path_command.py delete mode 100644 cycode/cli/commands/scan/pre_commit/__init__.py delete mode 100644 cycode/cli/commands/scan/pre_receive/__init__.py delete mode 100644 cycode/cli/commands/scan/repository/__init__.py delete mode 100644 cycode/cli/commands/scan/scan_ci/__init__.py delete mode 100644 cycode/cli/commands/scan/scan_ci/scan_ci_command.py delete mode 100644 cycode/cli/commands/scan/scan_command.py delete mode 100644 cycode/cli/commands/status/__init__.py delete mode 100644 cycode/cli/commands/status/status_command.py delete mode 100644 cycode/cli/commands/version/__init__.py delete mode 100644 cycode/cli/commands/version/version_command.py create mode 100644 cycode/cli/exceptions/handle_auth_errors.py rename cycode/cli/exceptions/{common.py => handle_errors.py} (61%) rename cycode/cli/{ => utils}/sentry.py (100%) rename cycode/cli/{commands/version => utils}/version_checker.py (95%) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 42467e02..4101ded8 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -27,10 +27,10 @@ jobs: run: | git checkout ${{ steps.latest_tag.outputs.LATEST_TAG }} - - name: Set up Python 3.8 + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Load cached Poetry setup id: cached_poetry diff --git a/.github/workflows/pre_release.yml b/.github/workflows/pre_release.yml index 7aca89c1..9b665d73 100644 --- a/.github/workflows/pre_release.yml +++ b/.github/workflows/pre_release.yml @@ -32,10 +32,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Python 3.8 - uses: actions/setup-python@v4 + - name: Set up Python + uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Load cached Poetry setup id: cached-poetry diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 40913767..aacbae5e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,10 +31,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Python 3.8 - uses: actions/setup-python@v4 + - name: Set up Python + uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Load cached Poetry setup id: cached-poetry diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index 3a91d0f3..575abfd0 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -21,9 +21,9 @@ jobs: uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Load cached Poetry setup id: cached-poetry diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0b5ddb58..a62a01b5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,9 +26,9 @@ jobs: uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Load cached Poetry setup id: cached-poetry diff --git a/.github/workflows/tests_full.yml b/.github/workflows/tests_full.yml index 985a3d36..a9ddd4f6 100644 --- a/.github/workflows/tests_full.yml +++ b/.github/workflows/tests_full.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os: [ macos-latest, ubuntu-latest, windows-latest ] - python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12", "3.13" ] + python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13" ] runs-on: ${{matrix.os}} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a95c8c28..75b8e85f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ ## How to contribute to Cycode CLI -The minimum version of Python that we support is 3.8. +The minimum version of Python that we support is 3.9. We recommend using this version for local development. But it’s fine to use a higher version without using new features from these versions. diff --git a/Dockerfile b/Dockerfile index 641b829d..8867d1f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ FROM base AS builder ENV POETRY_VERSION=1.8.3 # deps are required to build cffi -RUN apk add --no-cache --virtual .build-deps gcc=14.2.0-r4 libffi-dev=3.4.6-r0 musl-dev=1.2.5-r9 && \ +RUN apk add --no-cache --virtual .build-deps gcc=14.2.0-r4 libffi-dev=3.4.7-r0 musl-dev=1.2.5-r9 && \ pip install --no-cache-dir "poetry==$POETRY_VERSION" "poetry-dynamic-versioning[plugin]" && \ apk del .build-deps gcc libffi-dev musl-dev diff --git a/README.md b/README.md index 189d69f2..b30bc533 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ This guide walks you through both installation and usage. # Prerequisites -- The Cycode CLI application requires Python version 3.8 or later. +- The Cycode CLI application requires Python version 3.9 or later. - Use the [`cycode auth` command](#using-the-auth-command) to authenticate to Cycode with the CLI - Alternatively, you can get a Cycode Client ID and Client Secret Key by following the steps detailed in the [Service Account Token](https://docs.cycode.com/docs/en/service-accounts) and [Personal Access Token](https://docs.cycode.com/v1/docs/managing-personal-access-tokens) pages, which contain details on getting these values. @@ -208,7 +208,7 @@ Cycode’s pre-commit hook can be set up within your local repository so that th Perform the following steps to install the pre-commit hook: -1. Install the pre-commit framework (Python 3.8 or higher must be installed): +1. Install the pre-commit framework (Python 3.9 or higher must be installed): ```bash pip3 install pre-commit diff --git a/cycode/cli/__main__.py b/cycode/cli/__main__.py new file mode 100644 index 00000000..dad7ac12 --- /dev/null +++ b/cycode/cli/__main__.py @@ -0,0 +1,3 @@ +from cycode.cli.main import app + +app() diff --git a/cycode/cli/app.py b/cycode/cli/app.py new file mode 100644 index 00000000..d3fc10ca --- /dev/null +++ b/cycode/cli/app.py @@ -0,0 +1,87 @@ +import logging +from typing import Annotated, Optional + +import typer + +from cycode import __version__ +from cycode.cli.apps import ai_remediation, auth, configure, ignore, report, scan, status +from cycode.cli.cli_types import OutputTypeOption +from cycode.cli.consts import CLI_CONTEXT_SETTINGS +from cycode.cli.user_settings.configuration_manager import ConfigurationManager +from cycode.cli.utils.progress_bar import SCAN_PROGRESS_BAR_SECTIONS, get_progress_bar +from cycode.cli.utils.sentry import add_breadcrumb, init_sentry +from cycode.cli.utils.version_checker import version_checker +from cycode.cyclient.config import set_logging_level +from cycode.cyclient.cycode_client_base import CycodeClientBase +from cycode.cyclient.models import UserAgentOptionScheme + +app = typer.Typer( + pretty_exceptions_show_locals=False, + pretty_exceptions_short=True, + context_settings=CLI_CONTEXT_SETTINGS, + rich_markup_mode='rich', +) + +app.add_typer(ai_remediation.app) +app.add_typer(auth.app) +app.add_typer(configure.app) +app.add_typer(ignore.app) +app.add_typer(report.app) +app.add_typer(scan.app) +app.add_typer(status.app) + + +def check_latest_version_on_close(ctx: typer.Context) -> None: + output = ctx.obj.get('output') + # don't print anything if the output is JSON + if output == OutputTypeOption.JSON: + return + + # we always want to check the latest version for "version" and "status" commands + should_use_cache = ctx.invoked_subcommand not in {'version', 'status'} + version_checker.check_and_notify_update( + current_version=__version__, use_color=ctx.color, use_cache=should_use_cache + ) + + +@app.callback() +def app_callback( + ctx: typer.Context, + verbose: Annotated[bool, typer.Option('--verbose', '-v', help='Show detailed logs.')] = False, + no_progress_meter: Annotated[ + bool, typer.Option('--no-progress-meter', help='Do not show the progress meter.') + ] = False, + no_update_notifier: Annotated[ + bool, typer.Option('--no-update-notifier', help='Do not check CLI for updates.') + ] = False, + output: Annotated[ + OutputTypeOption, typer.Option('--output', '-o', case_sensitive=False, help='Specify the output type.') + ] = OutputTypeOption.TEXT, + user_agent: Annotated[ + Optional[str], + typer.Option(hidden=True, help='Characteristic JSON object that lets servers identify the application.'), + ] = None, +) -> None: + init_sentry() + add_breadcrumb('cycode') + + ctx.ensure_object(dict) + configuration_manager = ConfigurationManager() + + verbose = verbose or configuration_manager.get_verbose_flag() + ctx.obj['verbose'] = verbose + if verbose: + set_logging_level(logging.DEBUG) + + ctx.obj['output'] = output + if output == OutputTypeOption.JSON: + no_progress_meter = True + + ctx.obj['progress_bar'] = get_progress_bar(hidden=no_progress_meter, sections=SCAN_PROGRESS_BAR_SECTIONS) + + if user_agent: + user_agent_option = UserAgentOptionScheme().loads(user_agent) + CycodeClientBase.enrich_user_agent(user_agent_option.user_agent_suffix) + + if not no_update_notifier: + ctx.call_on_close(lambda: check_latest_version_on_close(ctx)) diff --git a/cycode/cli/commands/__init__.py b/cycode/cli/apps/__init__.py similarity index 100% rename from cycode/cli/commands/__init__.py rename to cycode/cli/apps/__init__.py diff --git a/cycode/cli/apps/ai_remediation/__init__.py b/cycode/cli/apps/ai_remediation/__init__.py new file mode 100644 index 00000000..2ccba382 --- /dev/null +++ b/cycode/cli/apps/ai_remediation/__init__.py @@ -0,0 +1,6 @@ +import typer + +from cycode.cli.apps.ai_remediation.ai_remediation_command import ai_remediation_command + +app = typer.Typer() +app.command(name='ai_remediation', short_help='Get AI remediation (INTERNAL).', hidden=True)(ai_remediation_command) diff --git a/cycode/cli/apps/ai_remediation/ai_remediation_command.py b/cycode/cli/apps/ai_remediation/ai_remediation_command.py new file mode 100644 index 00000000..0a82b815 --- /dev/null +++ b/cycode/cli/apps/ai_remediation/ai_remediation_command.py @@ -0,0 +1,32 @@ +from typing import Annotated +from uuid import UUID + +import typer + +from cycode.cli.apps.ai_remediation.apply_fix import apply_fix +from cycode.cli.apps.ai_remediation.print_remediation import print_remediation +from cycode.cli.exceptions.handle_ai_remediation_errors import handle_ai_remediation_exception +from cycode.cli.utils.get_api_client import get_scan_cycode_client + + +def ai_remediation_command( + ctx: typer.Context, + detection_id: Annotated[UUID, typer.Argument(help='Detection ID to get remediation for', show_default=False)], + fix: Annotated[ + bool, typer.Option('--fix', help='Apply fixes to resolve violations. Note: fix could be not available.') + ] = False, +) -> None: + """Get AI remediation (INTERNAL).""" + client = get_scan_cycode_client() + + try: + remediation_markdown = client.get_ai_remediation(detection_id) + fix_diff = client.get_ai_remediation(detection_id, fix=True) + is_fix_available = bool(fix_diff) # exclude empty string, None, etc. + + if fix: + apply_fix(ctx, fix_diff, is_fix_available) + else: + print_remediation(ctx, remediation_markdown, is_fix_available) + except Exception as err: + handle_ai_remediation_exception(ctx, err) diff --git a/cycode/cli/apps/ai_remediation/apply_fix.py b/cycode/cli/apps/ai_remediation/apply_fix.py new file mode 100644 index 00000000..e0c2599b --- /dev/null +++ b/cycode/cli/apps/ai_remediation/apply_fix.py @@ -0,0 +1,25 @@ +import os + +import typer +from patch_ng import fromstring + +from cycode.cli.models import CliResult +from cycode.cli.printers import ConsolePrinter + + +def apply_fix(ctx: typer.Context, diff: str, is_fix_available: bool) -> None: + printer = ConsolePrinter(ctx) + if not is_fix_available: + printer.print_result(CliResult(success=False, message='Fix is not available for this violation')) + return + + patch = fromstring(diff.encode('UTF-8')) + if patch is False: + printer.print_result(CliResult(success=False, message='Failed to parse fix diff')) + return + + is_fix_applied = patch.apply(root=os.getcwd(), strip=0) + if is_fix_applied: + printer.print_result(CliResult(success=True, message='Fix applied successfully')) + else: + printer.print_result(CliResult(success=False, message='Failed to apply fix')) diff --git a/cycode/cli/apps/ai_remediation/print_remediation.py b/cycode/cli/apps/ai_remediation/print_remediation.py new file mode 100644 index 00000000..c706c13f --- /dev/null +++ b/cycode/cli/apps/ai_remediation/print_remediation.py @@ -0,0 +1,15 @@ +import typer +from rich.console import Console +from rich.markdown import Markdown + +from cycode.cli.models import CliResult +from cycode.cli.printers import ConsolePrinter + + +def print_remediation(ctx: typer.Context, remediation_markdown: str, is_fix_available: bool) -> None: + printer = ConsolePrinter(ctx) + if printer.is_json_printer: + data = {'remediation': remediation_markdown, 'is_fix_available': is_fix_available} + printer.print_result(CliResult(success=True, message='Remediation fetched successfully', data=data)) + else: # text or table + Console().print(Markdown(remediation_markdown)) diff --git a/cycode/cli/apps/auth/__init__.py b/cycode/cli/apps/auth/__init__.py new file mode 100644 index 00000000..82e71fbc --- /dev/null +++ b/cycode/cli/apps/auth/__init__.py @@ -0,0 +1,11 @@ +import typer + +from cycode.cli.apps.auth.auth_command import auth_command +from cycode.cli.apps.auth.check_command import check_command + +app = typer.Typer( + name='auth', + help='Authenticate your machine to associate the CLI with your Cycode account.', +) +app.callback(invoke_without_command=True)(auth_command) +app.command(name='check')(check_command) diff --git a/cycode/cli/apps/auth/auth_command.py b/cycode/cli/apps/auth/auth_command.py new file mode 100644 index 00000000..c0b2fb89 --- /dev/null +++ b/cycode/cli/apps/auth/auth_command.py @@ -0,0 +1,28 @@ +import typer + +from cycode.cli.apps.auth.auth_manager import AuthManager +from cycode.cli.exceptions.handle_auth_errors import handle_auth_exception +from cycode.cli.models import CliResult +from cycode.cli.printers import ConsolePrinter +from cycode.cli.utils.sentry import add_breadcrumb +from cycode.cyclient import logger + + +def auth_command(ctx: typer.Context) -> None: + """Authenticates your machine.""" + add_breadcrumb('auth') + + if ctx.invoked_subcommand is not None: + # if it is a subcommand, do nothing + return + + try: + logger.debug('Starting authentication process') + + auth_manager = AuthManager() + auth_manager.authenticate() + + result = CliResult(success=True, message='Successfully logged into cycode') + ConsolePrinter(ctx).print_result(result) + except Exception as err: + handle_auth_exception(ctx, err) diff --git a/cycode/cli/commands/auth_common.py b/cycode/cli/apps/auth/auth_common.py similarity index 75% rename from cycode/cli/commands/auth_common.py rename to cycode/cli/apps/auth/auth_common.py index bf8d5d41..fffee388 100644 --- a/cycode/cli/commands/auth_common.py +++ b/cycode/cli/apps/auth/auth_common.py @@ -1,7 +1,8 @@ -from typing import NamedTuple, Optional +from typing import Optional -import click +import typer +from cycode.cli.apps.auth.models import AuthInfo from cycode.cli.exceptions.custom_exceptions import HttpUnauthorizedError, RequestHttpError from cycode.cli.printers import ConsolePrinter from cycode.cli.user_settings.credentials_manager import CredentialsManager @@ -9,12 +10,7 @@ from cycode.cyclient.cycode_token_based_client import CycodeTokenBasedClient -class AuthInfo(NamedTuple): - user_id: str - tenant_id: str - - -def get_authorization_info(context: Optional[click.Context] = None) -> Optional[AuthInfo]: +def get_authorization_info(ctx: Optional[typer.Context] = None) -> Optional[AuthInfo]: client_id, client_secret = CredentialsManager().get_credentials() if not client_id or not client_secret: return None @@ -27,7 +23,7 @@ def get_authorization_info(context: Optional[click.Context] = None) -> Optional[ user_id, tenant_id = get_user_and_tenant_ids_from_access_token(access_token) return AuthInfo(user_id=user_id, tenant_id=tenant_id) except (RequestHttpError, HttpUnauthorizedError): - if context: - ConsolePrinter(context).print_exception() + if ctx: + ConsolePrinter(ctx).print_exception() return None diff --git a/cycode/cli/commands/auth/auth_manager.py b/cycode/cli/apps/auth/auth_manager.py similarity index 100% rename from cycode/cli/commands/auth/auth_manager.py rename to cycode/cli/apps/auth/auth_manager.py diff --git a/cycode/cli/apps/auth/check_command.py b/cycode/cli/apps/auth/check_command.py new file mode 100644 index 00000000..cfa57f1c --- /dev/null +++ b/cycode/cli/apps/auth/check_command.py @@ -0,0 +1,25 @@ +import typer + +from cycode.cli.apps.auth.auth_common import get_authorization_info +from cycode.cli.models import CliResult +from cycode.cli.printers import ConsolePrinter +from cycode.cli.utils.sentry import add_breadcrumb + + +def check_command(ctx: typer.Context) -> None: + """Checks that your machine is associating the CLI with your Cycode account.""" + add_breadcrumb('check') + + printer = ConsolePrinter(ctx) + auth_info = get_authorization_info(ctx) + if auth_info is None: + printer.print_result(CliResult(success=False, message='Cycode authentication failed')) + return + + printer.print_result( + CliResult( + success=True, + message='Cycode authentication verified', + data={'user_id': auth_info.user_id, 'tenant_id': auth_info.tenant_id}, + ) + ) diff --git a/cycode/cli/apps/auth/models.py b/cycode/cli/apps/auth/models.py new file mode 100644 index 00000000..4b41dd3e --- /dev/null +++ b/cycode/cli/apps/auth/models.py @@ -0,0 +1,6 @@ +from typing import NamedTuple + + +class AuthInfo(NamedTuple): + user_id: str + tenant_id: str diff --git a/cycode/cli/apps/configure/__init__.py b/cycode/cli/apps/configure/__init__.py new file mode 100644 index 00000000..815874d1 --- /dev/null +++ b/cycode/cli/apps/configure/__init__.py @@ -0,0 +1,8 @@ +import typer + +from cycode.cli.apps.configure.configure_command import configure_command + +app = typer.Typer() +app.command(name='configure', short_help='Initial command to configure your CLI client authentication.')( + configure_command +) diff --git a/cycode/cli/apps/configure/configure_command.py b/cycode/cli/apps/configure/configure_command.py new file mode 100644 index 00000000..9c631641 --- /dev/null +++ b/cycode/cli/apps/configure/configure_command.py @@ -0,0 +1,57 @@ +from typing import Optional + +import typer + +from cycode.cli.apps.configure.consts import CONFIGURATION_MANAGER, CREDENTIALS_MANAGER +from cycode.cli.apps.configure.messages import get_credentials_update_result_message, get_urls_update_result_message +from cycode.cli.apps.configure.prompts import ( + get_api_url_input, + get_app_url_input, + get_client_id_input, + get_client_secret_input, +) +from cycode.cli.utils.sentry import add_breadcrumb + + +def _should_update_value( + old_value: Optional[str], + new_value: Optional[str], +) -> bool: + if not new_value: + return False + + return old_value != new_value + + +def configure_command() -> None: + """Configure your CLI client authentication manually.""" + add_breadcrumb('configure') + + global_config_manager = CONFIGURATION_MANAGER.global_config_file_manager + + current_api_url = global_config_manager.get_api_url() + current_app_url = global_config_manager.get_app_url() + api_url = get_api_url_input(current_api_url) + app_url = get_app_url_input(current_app_url) + + config_updated = False + if _should_update_value(current_api_url, api_url): + global_config_manager.update_api_base_url(api_url) + config_updated = True + if _should_update_value(current_app_url, app_url): + global_config_manager.update_app_base_url(app_url) + config_updated = True + + current_client_id, current_client_secret = CREDENTIALS_MANAGER.get_credentials_from_file() + client_id = get_client_id_input(current_client_id) + client_secret = get_client_secret_input(current_client_secret) + + credentials_updated = False + if _should_update_value(current_client_id, client_id) or _should_update_value(current_client_secret, client_secret): + credentials_updated = True + CREDENTIALS_MANAGER.update_credentials(client_id, client_secret) + + if config_updated: + typer.echo(get_urls_update_result_message()) + if credentials_updated: + typer.echo(get_credentials_update_result_message()) diff --git a/cycode/cli/apps/configure/consts.py b/cycode/cli/apps/configure/consts.py new file mode 100644 index 00000000..15c9b7a5 --- /dev/null +++ b/cycode/cli/apps/configure/consts.py @@ -0,0 +1,19 @@ +from cycode.cli import config, consts +from cycode.cli.user_settings.configuration_manager import ConfigurationManager +from cycode.cli.user_settings.credentials_manager import CredentialsManager + +URLS_UPDATED_SUCCESSFULLY_MESSAGE = 'Successfully configured Cycode URLs! Saved to: {filename}' +URLS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE = ( + 'Note that the URLs (APP and API) that already exist in environment variables ' + f'({consts.CYCODE_API_URL_ENV_VAR_NAME} and {consts.CYCODE_APP_URL_ENV_VAR_NAME}) ' + 'take precedent over these URLs; either update or remove the environment variables.' +) +CREDENTIALS_UPDATED_SUCCESSFULLY_MESSAGE = 'Successfully configured CLI credentials! Saved to: {filename}' +CREDENTIALS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE = ( + 'Note that the credentials that already exist in environment variables ' + f'({config.CYCODE_CLIENT_ID_ENV_VAR_NAME} and {config.CYCODE_CLIENT_SECRET_ENV_VAR_NAME}) ' + 'take precedent over these credentials; either update or remove the environment variables.' +) + +CREDENTIALS_MANAGER = CredentialsManager() +CONFIGURATION_MANAGER = ConfigurationManager() diff --git a/cycode/cli/apps/configure/messages.py b/cycode/cli/apps/configure/messages.py new file mode 100644 index 00000000..36ce807b --- /dev/null +++ b/cycode/cli/apps/configure/messages.py @@ -0,0 +1,37 @@ +from cycode.cli.apps.configure.consts import ( + CONFIGURATION_MANAGER, + CREDENTIALS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE, + CREDENTIALS_MANAGER, + CREDENTIALS_UPDATED_SUCCESSFULLY_MESSAGE, + URLS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE, + URLS_UPDATED_SUCCESSFULLY_MESSAGE, +) + + +def _are_credentials_exist_in_environment_variables() -> bool: + client_id, client_secret = CREDENTIALS_MANAGER.get_credentials_from_environment_variables() + return any([client_id, client_secret]) + + +def get_credentials_update_result_message() -> str: + success_message = CREDENTIALS_UPDATED_SUCCESSFULLY_MESSAGE.format(filename=CREDENTIALS_MANAGER.get_filename()) + if _are_credentials_exist_in_environment_variables(): + return f'{success_message}. {CREDENTIALS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE}' + + return success_message + + +def _are_urls_exist_in_environment_variables() -> bool: + api_url = CONFIGURATION_MANAGER.get_api_url_from_environment_variables() + app_url = CONFIGURATION_MANAGER.get_app_url_from_environment_variables() + return any([api_url, app_url]) + + +def get_urls_update_result_message() -> str: + success_message = URLS_UPDATED_SUCCESSFULLY_MESSAGE.format( + filename=CONFIGURATION_MANAGER.global_config_file_manager.get_filename() + ) + if _are_urls_exist_in_environment_variables(): + return f'{success_message}. {URLS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE}' + + return success_message diff --git a/cycode/cli/apps/configure/prompts.py b/cycode/cli/apps/configure/prompts.py new file mode 100644 index 00000000..3025688d --- /dev/null +++ b/cycode/cli/apps/configure/prompts.py @@ -0,0 +1,48 @@ +from typing import Optional + +import typer + +from cycode.cli import consts +from cycode.cli.utils.string_utils import obfuscate_text + + +def get_client_id_input(current_client_id: Optional[str]) -> Optional[str]: + prompt_text = 'Cycode Client ID' + + prompt_suffix = ' []: ' + if current_client_id: + prompt_suffix = f' [{obfuscate_text(current_client_id)}]: ' + + new_client_id = typer.prompt(text=prompt_text, prompt_suffix=prompt_suffix, default='', show_default=False) + return new_client_id or current_client_id + + +def get_client_secret_input(current_client_secret: Optional[str]) -> Optional[str]: + prompt_text = 'Cycode Client Secret' + + prompt_suffix = ' []: ' + if current_client_secret: + prompt_suffix = f' [{obfuscate_text(current_client_secret)}]: ' + + new_client_secret = typer.prompt(text=prompt_text, prompt_suffix=prompt_suffix, default='', show_default=False) + return new_client_secret or current_client_secret + + +def get_app_url_input(current_app_url: Optional[str]) -> str: + prompt_text = 'Cycode APP URL' + + default = consts.DEFAULT_CYCODE_APP_URL + if current_app_url: + default = current_app_url + + return typer.prompt(text=prompt_text, default=default, type=str) + + +def get_api_url_input(current_api_url: Optional[str]) -> str: + prompt_text = 'Cycode API URL' + + default = consts.DEFAULT_CYCODE_API_URL + if current_api_url: + default = current_api_url + + return typer.prompt(text=prompt_text, default=default, type=str) diff --git a/cycode/cli/apps/ignore/__init__.py b/cycode/cli/apps/ignore/__init__.py new file mode 100644 index 00000000..3c51d38a --- /dev/null +++ b/cycode/cli/apps/ignore/__init__.py @@ -0,0 +1,6 @@ +import typer + +from cycode.cli.apps.ignore.ignore_command import ignore_command + +app = typer.Typer() +app.command(name='ignore', short_help='Ignores a specific value, path or rule ID.')(ignore_command) diff --git a/cycode/cli/commands/ignore/ignore_command.py b/cycode/cli/apps/ignore/ignore_command.py similarity index 60% rename from cycode/cli/commands/ignore/ignore_command.py rename to cycode/cli/apps/ignore/ignore_command.py index b94c5612..3ac3ffff 100644 --- a/cycode/cli/commands/ignore/ignore_command.py +++ b/cycode/cli/apps/ignore/ignore_command.py @@ -1,12 +1,14 @@ import re -from typing import Optional +from typing import Annotated, Optional import click +import typer from cycode.cli import consts -from cycode.cli.config import config, configuration_manager -from cycode.cli.sentry import add_breadcrumb +from cycode.cli.cli_types import ScanTypeOption +from cycode.cli.config import configuration_manager from cycode.cli.utils.path_utils import get_absolute_path, is_path_exists +from cycode.cli.utils.sentry import add_breadcrumb from cycode.cli.utils.string_utils import hash_string_to_sha256 from cycode.cyclient import logger @@ -15,66 +17,54 @@ def _is_package_pattern_valid(package: str) -> bool: return re.search('^[^@]+@[^@]+$', package) is not None -@click.command(short_help='Ignores a specific value, path or rule ID.') -@click.option( - '--by-value', type=click.STRING, required=False, help='Ignore a specific value while scanning for Secrets.' -) -@click.option( - '--by-sha', - type=click.STRING, - required=False, - help='Ignore a specific SHA512 representation of a string while scanning for Secrets.', -) -@click.option( - '--by-path', - type=click.STRING, - required=False, - help='Avoid scanning a specific path. You`ll need to specify the scan type.', -) -@click.option( - '--by-rule', - type=click.STRING, - required=False, - help='Ignore scanning a specific secret rule ID or IaC rule ID. You`ll to specify the scan type.', -) -@click.option( - '--by-package', - type=click.STRING, - required=False, - help='Ignore scanning a specific package version while running an SCA scan. Expected pattern: name@version.', -) -@click.option( - '--by-cve', - type=click.STRING, - required=False, - help='Ignore scanning a specific CVE while running an SCA scan. Expected pattern: CVE-YYYY-NNN.', -) -@click.option( - '--scan-type', - '-t', - default=consts.SECRET_SCAN_TYPE, - help='Specify the type of scan you wish to execute (the default is Secrets).', - type=click.Choice(config['scans']['supported_scans']), - required=False, -) -@click.option( - '--global', - '-g', - 'is_global', - is_flag=True, - default=False, - required=False, - help='Add an ignore rule to the global CLI config.', -) def ignore_command( # noqa: C901 - by_value: Optional[str], - by_sha: Optional[str], - by_path: Optional[str], - by_rule: Optional[str], - by_package: Optional[str], - by_cve: Optional[str], - scan_type: str = consts.SECRET_SCAN_TYPE, - is_global: bool = False, + by_value: Annotated[ + Optional[str], typer.Option(help='Ignore a specific value while scanning for Secrets.', show_default=False) + ] = None, + by_sha: Annotated[ + Optional[str], + typer.Option( + help='Ignore a specific SHA512 representation of a string while scanning for Secrets.', show_default=False + ), + ] = None, + by_path: Annotated[ + Optional[str], + typer.Option(help='Avoid scanning a specific path. You`ll need to specify the scan type.', show_default=False), + ] = None, + by_rule: Annotated[ + Optional[str], + typer.Option( + help='Ignore scanning a specific secret rule ID or IaC rule ID. You`ll to specify the scan type.', + show_default=False, + ), + ] = None, + by_package: Annotated[ + Optional[str], + typer.Option( + help='Ignore scanning a specific package version while running an SCA scan. ' + 'Expected pattern: name@version.', + show_default=False, + ), + ] = None, + by_cve: Annotated[ + Optional[str], + typer.Option( + help='Ignore scanning a specific CVE while running an SCA scan. Expected pattern: CVE-YYYY-NNN.', + show_default=False, + ), + ] = None, + scan_type: Annotated[ + ScanTypeOption, + typer.Option( + '--scan-type', + '-t', + help='Specify the type of scan you wish to execute.', + case_sensitive=False, + ), + ] = ScanTypeOption.SECRET, + is_global: Annotated[ + bool, typer.Option('--global', '-g', help='Add an ignore rule to the global CLI config.') + ] = False, ) -> None: """Ignores a specific value, path or rule ID.""" add_breadcrumb('ignore') diff --git a/cycode/cli/apps/report/__init__.py b/cycode/cli/apps/report/__init__.py new file mode 100644 index 00000000..f71532c8 --- /dev/null +++ b/cycode/cli/apps/report/__init__.py @@ -0,0 +1,8 @@ +import typer + +from cycode.cli.apps.report import sbom +from cycode.cli.apps.report.report_command import report_command + +app = typer.Typer(name='report') +app.callback(short_help='Generate report. You`ll need to specify which report type to perform.')(report_command) +app.add_typer(sbom.app) diff --git a/cycode/cli/apps/report/report_command.py b/cycode/cli/apps/report/report_command.py new file mode 100644 index 00000000..91a061c3 --- /dev/null +++ b/cycode/cli/apps/report/report_command.py @@ -0,0 +1,11 @@ +import typer + +from cycode.cli.utils.progress_bar import SBOM_REPORT_PROGRESS_BAR_SECTIONS, get_progress_bar +from cycode.cli.utils.sentry import add_breadcrumb + + +def report_command(ctx: typer.Context) -> int: + """Generate report.""" + add_breadcrumb('report') + ctx.obj['progress_bar'] = get_progress_bar(hidden=False, sections=SBOM_REPORT_PROGRESS_BAR_SECTIONS) + return 1 diff --git a/cycode/cli/apps/report/sbom/__init__.py b/cycode/cli/apps/report/sbom/__init__.py new file mode 100644 index 00000000..461b3fe0 --- /dev/null +++ b/cycode/cli/apps/report/sbom/__init__.py @@ -0,0 +1,12 @@ +import typer + +from cycode.cli.apps.report.sbom.path.path_command import path_command +from cycode.cli.apps.report.sbom.repository_url.repository_url_command import repository_url_command +from cycode.cli.apps.report.sbom.sbom_command import sbom_command + +app = typer.Typer(name='sbom') +app.callback(short_help='Generate SBOM report for remote repository by url or local directory by path.')(sbom_command) +app.command(name='path', short_help='Generate SBOM report for provided path in the command.')(path_command) +app.command(name='repository_url', short_help='Generate SBOM report for provided repository URI in the command.')( + repository_url_command +) diff --git a/cycode/cli/commands/report/sbom/common.py b/cycode/cli/apps/report/sbom/common.py similarity index 97% rename from cycode/cli/commands/report/sbom/common.py rename to cycode/cli/apps/report/sbom/common.py index 6ea843f5..b296e525 100644 --- a/cycode/cli/commands/report/sbom/common.py +++ b/cycode/cli/apps/report/sbom/common.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Optional from cycode.cli import consts -from cycode.cli.commands.report.sbom.sbom_report_file import SbomReportFile +from cycode.cli.apps.report.sbom.sbom_report_file import SbomReportFile from cycode.cli.config import configuration_manager from cycode.cli.exceptions.custom_exceptions import ReportAsyncError from cycode.cli.utils.progress_bar import SbomReportProgressBarSection diff --git a/cycode/cli/commands/ai_remediation/__init__.py b/cycode/cli/apps/report/sbom/path/__init__.py similarity index 100% rename from cycode/cli/commands/ai_remediation/__init__.py rename to cycode/cli/apps/report/sbom/path/__init__.py diff --git a/cycode/cli/commands/report/sbom/path/path_command.py b/cycode/cli/apps/report/sbom/path/path_command.py similarity index 74% rename from cycode/cli/commands/report/sbom/path/path_command.py rename to cycode/cli/apps/report/sbom/path/path_command.py index c52bc611..20e82848 100644 --- a/cycode/cli/commands/report/sbom/path/path_command.py +++ b/cycode/cli/apps/report/sbom/path/path_command.py @@ -1,30 +1,35 @@ import time +from pathlib import Path +from typing import Annotated -import click +import typer from cycode.cli import consts -from cycode.cli.commands.report.sbom.common import create_sbom_report, send_report_feedback +from cycode.cli.apps.report.sbom.common import create_sbom_report, send_report_feedback from cycode.cli.exceptions.handle_report_sbom_errors import handle_report_exception from cycode.cli.files_collector.path_documents import get_relevant_documents from cycode.cli.files_collector.sca.sca_code_scanner import perform_pre_scan_documents_actions from cycode.cli.files_collector.zip_documents import zip_documents -from cycode.cli.sentry import add_breadcrumb from cycode.cli.utils.get_api_client import get_report_cycode_client from cycode.cli.utils.progress_bar import SbomReportProgressBarSection +from cycode.cli.utils.sentry import add_breadcrumb -@click.command(short_help='Generate SBOM report for provided path in the command.') -@click.argument('path', nargs=1, type=click.Path(exists=True, resolve_path=True), required=True) -@click.pass_context -def path_command(context: click.Context, path: str) -> None: +def path_command( + ctx: typer.Context, + path: Annotated[ + Path, + typer.Argument(exists=True, resolve_path=True, help='Path to generate SBOM report for.', show_default=False), + ], +) -> None: add_breadcrumb('path') client = get_report_cycode_client() - report_parameters = context.obj['report_parameters'] + report_parameters = ctx.obj['report_parameters'] output_format = report_parameters.output_format - output_file = context.obj['output_file'] + output_file = ctx.obj['output_file'] - progress_bar = context.obj['progress_bar'] + progress_bar = ctx.obj['progress_bar'] progress_bar.start() start_scan_time = time.time() @@ -32,11 +37,11 @@ def path_command(context: click.Context, path: str) -> None: try: documents = get_relevant_documents( - progress_bar, SbomReportProgressBarSection.PREPARE_LOCAL_FILES, consts.SCA_SCAN_TYPE, (path,) + progress_bar, SbomReportProgressBarSection.PREPARE_LOCAL_FILES, consts.SCA_SCAN_TYPE, (str(path),) ) # TODO(MarshalX): combine perform_pre_scan_documents_actions with get_relevant_document. # unhardcode usage of context in perform_pre_scan_documents_actions - perform_pre_scan_documents_actions(context, consts.SCA_SCAN_TYPE, documents) + perform_pre_scan_documents_actions(ctx, consts.SCA_SCAN_TYPE, documents) zipped_documents = zip_documents(consts.SCA_SCAN_TYPE, documents) report_execution = client.request_sbom_report_execution(report_parameters, zip_file=zipped_documents) @@ -66,4 +71,4 @@ def path_command(context: click.Context, path: str) -> None: error_message=str(e), ) - handle_report_exception(context, e) + handle_report_exception(ctx, e) diff --git a/cycode/cli/commands/auth/__init__.py b/cycode/cli/apps/report/sbom/repository_url/__init__.py similarity index 100% rename from cycode/cli/commands/auth/__init__.py rename to cycode/cli/apps/report/sbom/repository_url/__init__.py diff --git a/cycode/cli/commands/report/sbom/repository_url/repository_url_command.py b/cycode/cli/apps/report/sbom/repository_url/repository_url_command.py similarity index 73% rename from cycode/cli/commands/report/sbom/repository_url/repository_url_command.py rename to cycode/cli/apps/report/sbom/repository_url/repository_url_command.py index 189fd961..28be0114 100644 --- a/cycode/cli/commands/report/sbom/repository_url/repository_url_command.py +++ b/cycode/cli/apps/report/sbom/repository_url/repository_url_command.py @@ -1,27 +1,28 @@ import time +from typing import Annotated -import click +import typer -from cycode.cli.commands.report.sbom.common import create_sbom_report, send_report_feedback +from cycode.cli.apps.report.sbom.common import create_sbom_report, send_report_feedback from cycode.cli.exceptions.handle_report_sbom_errors import handle_report_exception -from cycode.cli.sentry import add_breadcrumb from cycode.cli.utils.get_api_client import get_report_cycode_client from cycode.cli.utils.progress_bar import SbomReportProgressBarSection +from cycode.cli.utils.sentry import add_breadcrumb -@click.command(short_help='Generate SBOM report for provided repository URI in the command.') -@click.argument('uri', nargs=1, type=str, required=True) -@click.pass_context -def repository_url_command(context: click.Context, uri: str) -> None: +def repository_url_command( + ctx: typer.Context, + uri: Annotated[str, typer.Argument(help='Repository URL to generate SBOM report for.', show_default=False)], +) -> None: add_breadcrumb('repository_url') - progress_bar = context.obj['progress_bar'] + progress_bar = ctx.obj['progress_bar'] progress_bar.start() progress_bar.set_section_length(SbomReportProgressBarSection.PREPARE_LOCAL_FILES) client = get_report_cycode_client() - report_parameters = context.obj['report_parameters'] - output_file = context.obj['output_file'] + report_parameters = ctx.obj['report_parameters'] + output_file = ctx.obj['output_file'] output_format = report_parameters.output_format start_scan_time = time.time() @@ -56,4 +57,4 @@ def repository_url_command(context: click.Context, uri: str) -> None: repository_uri=uri, ) - handle_report_exception(context, e) + handle_report_exception(ctx, e) diff --git a/cycode/cli/apps/report/sbom/sbom_command.py b/cycode/cli/apps/report/sbom/sbom_command.py new file mode 100644 index 00000000..65dc3fd9 --- /dev/null +++ b/cycode/cli/apps/report/sbom/sbom_command.py @@ -0,0 +1,68 @@ +from pathlib import Path +from typing import Annotated, Optional + +import click +import typer + +from cycode.cli.cli_types import SbomFormatOption, SbomOutputFormatOption +from cycode.cli.utils.sentry import add_breadcrumb +from cycode.cyclient.report_client import ReportParameters + + +def sbom_command( + ctx: typer.Context, + sbom_format: Annotated[ + SbomFormatOption, + typer.Option( + '--format', + '-f', + help='SBOM format.', + case_sensitive=False, + show_default=False, + ), + ], + output_format: Annotated[ + SbomOutputFormatOption, + typer.Option( + '--output-format', + '-o', + help='Specify the output file format.', + ), + ] = SbomOutputFormatOption.JSON, + output_file: Annotated[ + Optional[Path], + typer.Option( + help='Output file.', + show_default='Autogenerated filename saved to the current directory', + dir_okay=False, + writable=True, + ), + ] = None, + include_vulnerabilities: Annotated[ + bool, typer.Option('--include-vulnerabilities', help='Include vulnerabilities.', show_default=False) + ] = False, + include_dev_dependencies: Annotated[ + bool, typer.Option('--include-dev-dependencies', help='Include dev dependencies.', show_default=False) + ] = False, +) -> int: + """Generate SBOM report.""" + add_breadcrumb('sbom') + + sbom_format_parts = sbom_format.split('-') + if len(sbom_format_parts) != 2: + raise click.ClickException('Invalid SBOM format.') + + sbom_format, sbom_format_version = sbom_format_parts + + report_parameters = ReportParameters( + entity_type='SbomCli', + sbom_report_type=sbom_format, + sbom_version=sbom_format_version, + output_format=output_format, + include_vulnerabilities=include_vulnerabilities, + include_dev_dependencies=include_dev_dependencies, + ) + ctx.obj['report_parameters'] = report_parameters + ctx.obj['output_file'] = output_file + + return 1 diff --git a/cycode/cli/commands/report/sbom/sbom_report_file.py b/cycode/cli/apps/report/sbom/sbom_report_file.py similarity index 100% rename from cycode/cli/commands/report/sbom/sbom_report_file.py rename to cycode/cli/apps/report/sbom/sbom_report_file.py diff --git a/cycode/cli/apps/scan/__init__.py b/cycode/cli/apps/scan/__init__.py new file mode 100644 index 00000000..e8602091 --- /dev/null +++ b/cycode/cli/apps/scan/__init__.py @@ -0,0 +1,33 @@ +import typer + +from cycode.cli.apps.scan.commit_history.commit_history_command import commit_history_command +from cycode.cli.apps.scan.path.path_command import path_command +from cycode.cli.apps.scan.pre_commit.pre_commit_command import pre_commit_command +from cycode.cli.apps.scan.pre_receive.pre_receive_command import pre_receive_command +from cycode.cli.apps.scan.repository.repository_command import repository_command +from cycode.cli.apps.scan.scan_command import scan_command, scan_command_result_callback + +app = typer.Typer(name='scan') + +app.callback( + short_help='Scan the content for Secrets, IaC, SCA, and SAST violations.', + result_callback=scan_command_result_callback, +)(scan_command) + +app.command(name='path', short_help='Scan the files in the paths provided in the command.')(path_command) +app.command(name='repository', short_help='Scan the Git repository included files.')(repository_command) +app.command(name='commit_history', short_help='Scan all the commits history in this git repository.')( + commit_history_command +) + +app.command( + name='pre_commit', + short_help='Use this command in pre-commit hook to scan any content that was not committed yet.', + rich_help_panel='Automation commands', +)(pre_commit_command) +app.command( + name='pre_receive', + short_help='Use this command in pre-receive hook ' + 'to scan commits on the server side before pushing them to the repository.', + rich_help_panel='Automation commands', +)(pre_receive_command) diff --git a/cycode/cli/commands/scan/code_scanner.py b/cycode/cli/apps/scan/code_scanner.py similarity index 89% rename from cycode/cli/commands/scan/code_scanner.py rename to cycode/cli/apps/scan/code_scanner.py index b3fddf59..535507d7 100644 --- a/cycode/cli/commands/scan/code_scanner.py +++ b/cycode/cli/apps/scan/code_scanner.py @@ -7,8 +7,10 @@ from uuid import UUID, uuid4 import click +import typer from cycode.cli import consts +from cycode.cli.cli_types import SeverityOption from cycode.cli.config import configuration_manager from cycode.cli.exceptions import custom_exceptions from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception @@ -24,7 +26,7 @@ from cycode.cli.files_collector.sca import sca_code_scanner from cycode.cli.files_collector.sca.sca_code_scanner import perform_pre_scan_documents_actions from cycode.cli.files_collector.zip_documents import zip_documents -from cycode.cli.models import CliError, Document, DocumentDetections, LocalScanResult, Severity +from cycode.cli.models import CliError, Document, DocumentDetections, LocalScanResult from cycode.cli.printers import ConsolePrinter from cycode.cli.utils import scan_utils from cycode.cli.utils.git_proxy import git_proxy @@ -43,17 +45,17 @@ start_scan_time = time.time() -def scan_sca_pre_commit(context: click.Context) -> None: - scan_type = context.obj['scan_type'] - scan_parameters = get_default_scan_parameters(context) +def scan_sca_pre_commit(ctx: typer.Context) -> None: + scan_type = ctx.obj['scan_type'] + scan_parameters = get_default_scan_parameters(ctx) git_head_documents, pre_committed_documents = get_pre_commit_modified_documents( - context.obj['progress_bar'], ScanProgressBarSection.PREPARE_LOCAL_FILES + ctx.obj['progress_bar'], ScanProgressBarSection.PREPARE_LOCAL_FILES ) git_head_documents = exclude_irrelevant_documents_to_scan(scan_type, git_head_documents) pre_committed_documents = exclude_irrelevant_documents_to_scan(scan_type, pre_committed_documents) sca_code_scanner.perform_pre_hook_range_scan_actions(git_head_documents, pre_committed_documents) scan_commit_range_documents( - context, + ctx, git_head_documents, pre_committed_documents, scan_parameters, @@ -61,11 +63,11 @@ def scan_sca_pre_commit(context: click.Context) -> None: ) -def scan_sca_commit_range(context: click.Context, path: str, commit_range: str) -> None: - scan_type = context.obj['scan_type'] - progress_bar = context.obj['progress_bar'] +def scan_sca_commit_range(ctx: typer.Context, path: str, commit_range: str) -> None: + scan_type = ctx.obj['scan_type'] + progress_bar = ctx.obj['progress_bar'] - scan_parameters = get_scan_parameters(context, (path,)) + scan_parameters = get_scan_parameters(ctx, (path,)) from_commit_rev, to_commit_rev = parse_commit_range(commit_range, path) from_commit_documents, to_commit_documents = get_commit_range_modified_documents( progress_bar, ScanProgressBarSection.PREPARE_LOCAL_FILES, path, from_commit_rev, to_commit_rev @@ -76,24 +78,24 @@ def scan_sca_commit_range(context: click.Context, path: str, commit_range: str) path, from_commit_documents, from_commit_rev, to_commit_documents, to_commit_rev ) - scan_commit_range_documents(context, from_commit_documents, to_commit_documents, scan_parameters=scan_parameters) + scan_commit_range_documents(ctx, from_commit_documents, to_commit_documents, scan_parameters=scan_parameters) -def scan_disk_files(context: click.Context, paths: Tuple[str]) -> None: - scan_parameters = get_scan_parameters(context, paths) - scan_type = context.obj['scan_type'] - progress_bar = context.obj['progress_bar'] +def scan_disk_files(ctx: typer.Context, paths: Tuple[str, ...]) -> None: + scan_parameters = get_scan_parameters(ctx, paths) + scan_type = ctx.obj['scan_type'] + progress_bar = ctx.obj['progress_bar'] try: documents = get_relevant_documents(progress_bar, ScanProgressBarSection.PREPARE_LOCAL_FILES, scan_type, paths) - perform_pre_scan_documents_actions(context, scan_type, documents) - scan_documents(context, documents, scan_parameters=scan_parameters) + perform_pre_scan_documents_actions(ctx, scan_type, documents) + scan_documents(ctx, documents, scan_parameters=scan_parameters) except Exception as e: - handle_scan_exception(context, e) + handle_scan_exception(ctx, e) -def set_issue_detected_by_scan_results(context: click.Context, scan_results: List[LocalScanResult]) -> None: - set_issue_detected(context, any(scan_result.issue_detected for scan_result in scan_results)) +def set_issue_detected_by_scan_results(ctx: typer.Context, scan_results: List[LocalScanResult]) -> None: + set_issue_detected(ctx, any(scan_result.issue_detected for scan_result in scan_results)) def _should_use_scan_service(scan_type: str, scan_parameters: dict) -> bool: @@ -150,13 +152,13 @@ def _enrich_scan_result_with_data_from_detection_rules( def _get_scan_documents_thread_func( - context: click.Context, is_git_diff: bool, is_commit_range: bool, scan_parameters: dict + ctx: typer.Context, is_git_diff: bool, is_commit_range: bool, scan_parameters: dict ) -> Tuple[Callable[[List[Document]], Tuple[str, CliError, LocalScanResult]], str]: - cycode_client = context.obj['client'] - scan_type = context.obj['scan_type'] - severity_threshold = context.obj['severity_threshold'] - sync_option = context.obj['sync'] - command_scan_type = context.info_name + cycode_client = ctx.obj['client'] + scan_type = ctx.obj['scan_type'] + severity_threshold = ctx.obj['severity_threshold'] + sync_option = ctx.obj['sync'] + command_scan_type = ctx.info_name aggregation_id = str(_generate_unique_id()) scan_parameters['aggregation_id'] = aggregation_id @@ -171,7 +173,7 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local should_use_sync_flow = _should_use_sync_flow(command_scan_type, scan_type, sync_option, scan_parameters) try: - logger.debug('Preparing local files, %s', {'batch_size': len(batch)}) + logger.debug('Preparing local files, %s', {'batch_files_count': len(batch)}) zipped_documents = zip_documents(scan_type, batch) zip_file_size = zipped_documents.size scan_result = perform_scan( @@ -194,7 +196,7 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local scan_completed = True except Exception as e: - error = handle_scan_exception(context, e, return_exception=True) + error = handle_scan_exception(ctx, e, return_exception=True) error_message = str(e) if local_scan_result: @@ -231,18 +233,18 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local def scan_commit_range( - context: click.Context, path: str, commit_range: str, max_commits_count: Optional[int] = None + ctx: typer.Context, path: str, commit_range: str, max_commits_count: Optional[int] = None ) -> None: - scan_type = context.obj['scan_type'] + scan_type = ctx.obj['scan_type'] - progress_bar = context.obj['progress_bar'] + progress_bar = ctx.obj['progress_bar'] progress_bar.start() if scan_type not in consts.COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES: raise click.ClickException(f'Commit range scanning for {str.upper(scan_type)} is not supported') if scan_type == consts.SCA_SCAN_TYPE: - return scan_sca_commit_range(context, path, commit_range) + return scan_sca_commit_range(ctx, path, commit_range) documents_to_scan = [] commit_ids_to_scan = [] @@ -287,26 +289,26 @@ def scan_commit_range( logger.debug('List of commit ids to scan, %s', {'commit_ids': commit_ids_to_scan}) logger.debug('Starting to scan commit range (it may take a few minutes)') - scan_documents(context, documents_to_scan, is_git_diff=True, is_commit_range=True) + scan_documents(ctx, documents_to_scan, is_git_diff=True, is_commit_range=True) return None def scan_documents( - context: click.Context, + ctx: typer.Context, documents_to_scan: List[Document], is_git_diff: bool = False, is_commit_range: bool = False, scan_parameters: Optional[dict] = None, ) -> None: if not scan_parameters: - scan_parameters = get_default_scan_parameters(context) + scan_parameters = get_default_scan_parameters(ctx) - scan_type = context.obj['scan_type'] - progress_bar = context.obj['progress_bar'] + scan_type = ctx.obj['scan_type'] + progress_bar = ctx.obj['progress_bar'] if not documents_to_scan: progress_bar.stop() - ConsolePrinter(context).print_error( + ConsolePrinter(ctx).print_error( CliError( code='no_relevant_files', message='Error: The scan could not be completed - relevant files to scan are not found. ' @@ -316,7 +318,7 @@ def scan_documents( return scan_batch_thread_func, aggregation_id = _get_scan_documents_thread_func( - context, is_git_diff, is_commit_range, scan_parameters + ctx, is_git_diff, is_commit_range, scan_parameters ) errors, local_scan_results = run_parallel_batched_scan( scan_batch_thread_func, scan_type, documents_to_scan, progress_bar=progress_bar @@ -325,20 +327,20 @@ def scan_documents( if len(local_scan_results) > 1: # if we used more than one batch, we need to fetch aggregate report url aggregation_report_url = _try_get_aggregation_report_url_if_needed( - scan_parameters, context.obj['client'], scan_type + scan_parameters, ctx.obj['client'], scan_type ) - set_aggregation_report_url(context, aggregation_report_url) + set_aggregation_report_url(ctx, aggregation_report_url) progress_bar.set_section_length(ScanProgressBarSection.GENERATE_REPORT, 1) progress_bar.update(ScanProgressBarSection.GENERATE_REPORT) progress_bar.stop() - set_issue_detected_by_scan_results(context, local_scan_results) - print_results(context, local_scan_results, errors) + set_issue_detected_by_scan_results(ctx, local_scan_results) + print_results(ctx, local_scan_results, errors) -def set_aggregation_report_url(context: click.Context, aggregation_report_url: Optional[str] = None) -> None: - context.obj['aggregation_report_url'] = aggregation_report_url +def set_aggregation_report_url(ctx: typer.Context, aggregation_report_url: Optional[str] = None) -> None: + ctx.obj['aggregation_report_url'] = aggregation_report_url def _try_get_aggregation_report_url_if_needed( @@ -357,7 +359,7 @@ def _try_get_aggregation_report_url_if_needed( def scan_commit_range_documents( - context: click.Context, + ctx: typer.Context, from_documents_to_scan: List[Document], to_documents_to_scan: List[Document], scan_parameters: Optional[dict] = None, @@ -365,11 +367,11 @@ def scan_commit_range_documents( ) -> None: """Used by SCA only""" - cycode_client = context.obj['client'] - scan_type = context.obj['scan_type'] - severity_threshold = context.obj['severity_threshold'] - scan_command_type = context.info_name - progress_bar = context.obj['progress_bar'] + cycode_client = ctx.obj['client'] + scan_type = ctx.obj['scan_type'] + severity_threshold = ctx.obj['severity_threshold'] + scan_command_type = ctx.info_name + progress_bar = ctx.obj['progress_bar'] local_scan_result = error_message = None scan_completed = False @@ -403,17 +405,17 @@ def scan_commit_range_documents( local_scan_result = create_local_scan_result( scan_result, to_documents_to_scan, scan_command_type, scan_type, severity_threshold ) - set_issue_detected_by_scan_results(context, [local_scan_result]) + set_issue_detected_by_scan_results(ctx, [local_scan_result]) progress_bar.update(ScanProgressBarSection.GENERATE_REPORT) progress_bar.stop() # errors will be handled with try-except block; printing will not occur on errors - print_results(context, [local_scan_result]) + print_results(ctx, [local_scan_result]) scan_completed = True except Exception as e: - handle_scan_exception(context, e) + handle_scan_exception(ctx, e) error_message = str(e) zip_file_size = from_commit_zipped_documents.size + to_commit_zipped_documents.size @@ -601,9 +603,9 @@ def print_debug_scan_details(scan_details_response: 'ScanDetailsResponse') -> No def print_results( - context: click.Context, local_scan_results: List[LocalScanResult], errors: Optional[Dict[str, 'CliError']] = None + ctx: typer.Context, local_scan_results: List[LocalScanResult], errors: Optional[Dict[str, 'CliError']] = None ) -> None: - printer = ConsolePrinter(context) + printer = ConsolePrinter(ctx) printer.print_scan_results(local_scan_results, errors) @@ -671,18 +673,18 @@ def parse_pre_receive_input() -> str: return pre_receive_input.splitlines()[0] -def get_default_scan_parameters(context: click.Context) -> dict: +def get_default_scan_parameters(ctx: typer.Context) -> dict: return { - 'monitor': context.obj.get('monitor'), - 'report': context.obj.get('report'), - 'package_vulnerabilities': context.obj.get('package-vulnerabilities'), - 'license_compliance': context.obj.get('license-compliance'), - 'command_type': context.info_name, + 'monitor': ctx.obj.get('monitor'), + 'report': ctx.obj.get('report'), + 'package_vulnerabilities': ctx.obj.get('package-vulnerabilities'), + 'license_compliance': ctx.obj.get('license-compliance'), + 'command_type': ctx.info_name, } -def get_scan_parameters(context: click.Context, paths: Tuple[str]) -> dict: - scan_parameters = get_default_scan_parameters(context) +def get_scan_parameters(ctx: typer.Context, paths: Tuple[str, ...]) -> dict: + scan_parameters = get_default_scan_parameters(ctx) if not paths: return scan_parameters @@ -696,7 +698,7 @@ def get_scan_parameters(context: click.Context, paths: Tuple[str]) -> dict: remote_url = try_get_git_remote_url(paths[0]) if remote_url: # TODO(MarshalX): remove hardcode in context - context.obj['remote_url'] = remote_url + ctx.obj['remote_url'] = remote_url scan_parameters['remote_url'] = remote_url return scan_parameters @@ -877,9 +879,9 @@ def _generate_unique_id() -> UUID: def _does_severity_match_severity_threshold(severity: str, severity_threshold: str) -> bool: - detection_severity_value = Severity.try_get_value(severity) - severity_threshold_value = Severity.try_get_value(severity_threshold) - if detection_severity_value is None or severity_threshold_value is None: + detection_severity_value = SeverityOption.get_member_weight(severity) + severity_threshold_value = SeverityOption.get_member_weight(severity_threshold) + if detection_severity_value < 0 or severity_threshold_value < 0: return True return detection_severity_value >= severity_threshold_value @@ -997,13 +999,13 @@ def _normalize_file_path(path: str) -> str: return path -def perform_post_pre_receive_scan_actions(context: click.Context) -> None: - if scan_utils.is_scan_failed(context): +def perform_post_pre_receive_scan_actions(ctx: typer.Context) -> None: + if scan_utils.is_scan_failed(ctx): click.echo(consts.PRE_RECEIVE_REMEDIATION_MESSAGE) -def enable_verbose_mode(context: click.Context) -> None: - context.obj['verbose'] = True +def enable_verbose_mode(ctx: typer.Context) -> None: + ctx.obj['verbose'] = True set_logging_level(logging.DEBUG) diff --git a/cycode/cli/commands/configure/__init__.py b/cycode/cli/apps/scan/commit_history/__init__.py similarity index 100% rename from cycode/cli/commands/configure/__init__.py rename to cycode/cli/apps/scan/commit_history/__init__.py diff --git a/cycode/cli/apps/scan/commit_history/commit_history_command.py b/cycode/cli/apps/scan/commit_history/commit_history_command.py new file mode 100644 index 00000000..dd74a4f0 --- /dev/null +++ b/cycode/cli/apps/scan/commit_history/commit_history_command.py @@ -0,0 +1,33 @@ +from pathlib import Path +from typing import Annotated + +import typer + +from cycode.cli.apps.scan.code_scanner import scan_commit_range +from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception +from cycode.cli.utils.sentry import add_breadcrumb +from cycode.cyclient import logger + + +def commit_history_command( + ctx: typer.Context, + path: Annotated[ + Path, typer.Argument(exists=True, resolve_path=True, help='Path to git repository to scan', show_default=False) + ], + commit_range: Annotated[ + str, + typer.Option( + '--commit_range', + '-r', + help='Scan a commit range in this git repository (example: HEAD~1)', + show_default='cycode scans all commit history', + ), + ] = '--all', +) -> None: + try: + add_breadcrumb('commit_history') + + logger.debug('Starting commit history scan process, %s', {'path': path, 'commit_range': commit_range}) + scan_commit_range(ctx, path=str(path), commit_range=commit_range) + except Exception as e: + handle_scan_exception(ctx, e) diff --git a/cycode/cli/commands/ignore/__init__.py b/cycode/cli/apps/scan/path/__init__.py similarity index 100% rename from cycode/cli/commands/ignore/__init__.py rename to cycode/cli/apps/scan/path/__init__.py diff --git a/cycode/cli/apps/scan/path/path_command.py b/cycode/cli/apps/scan/path/path_command.py new file mode 100644 index 00000000..4c841444 --- /dev/null +++ b/cycode/cli/apps/scan/path/path_command.py @@ -0,0 +1,25 @@ +from pathlib import Path +from typing import Annotated, List + +import typer + +from cycode.cli.apps.scan.code_scanner import scan_disk_files +from cycode.cli.utils.sentry import add_breadcrumb +from cycode.cyclient import logger + + +def path_command( + ctx: typer.Context, + paths: Annotated[ + List[Path], typer.Argument(exists=True, resolve_path=True, help='Paths to scan', show_default=False) + ], +) -> None: + add_breadcrumb('path') + + progress_bar = ctx.obj['progress_bar'] + progress_bar.start() + + logger.debug('Starting path scan process, %s', {'paths': paths}) + + tuple_paths = tuple(str(path) for path in paths) + scan_disk_files(ctx, tuple_paths) diff --git a/cycode/cli/commands/report/__init__.py b/cycode/cli/apps/scan/pre_commit/__init__.py similarity index 100% rename from cycode/cli/commands/report/__init__.py rename to cycode/cli/apps/scan/pre_commit/__init__.py diff --git a/cycode/cli/commands/scan/pre_commit/pre_commit_command.py b/cycode/cli/apps/scan/pre_commit/pre_commit_command.py similarity index 64% rename from cycode/cli/commands/scan/pre_commit/pre_commit_command.py rename to cycode/cli/apps/scan/pre_commit/pre_commit_command.py index fa4b295a..d88db8cc 100644 --- a/cycode/cli/commands/scan/pre_commit/pre_commit_command.py +++ b/cycode/cli/apps/scan/pre_commit/pre_commit_command.py @@ -1,37 +1,37 @@ import os -from typing import List +from typing import Annotated, List, Optional -import click +import typer from cycode.cli import consts -from cycode.cli.commands.scan.code_scanner import scan_documents, scan_sca_pre_commit +from cycode.cli.apps.scan.code_scanner import scan_documents, scan_sca_pre_commit from cycode.cli.files_collector.excluder import exclude_irrelevant_documents_to_scan from cycode.cli.files_collector.repository_documents import ( get_diff_file_content, get_diff_file_path, ) from cycode.cli.models import Document -from cycode.cli.sentry import add_breadcrumb from cycode.cli.utils.git_proxy import git_proxy from cycode.cli.utils.path_utils import ( get_path_by_os, ) from cycode.cli.utils.progress_bar import ScanProgressBarSection +from cycode.cli.utils.sentry import add_breadcrumb -@click.command(short_help='Use this command to scan any content that was not committed yet.') -@click.argument('ignored_args', nargs=-1, type=click.UNPROCESSED) -@click.pass_context -def pre_commit_command(context: click.Context, ignored_args: List[str]) -> None: +def pre_commit_command( + ctx: typer.Context, + _: Annotated[Optional[List[str]], typer.Argument(help='Ignored arguments', hidden=True)] = None, +) -> None: add_breadcrumb('pre_commit') - scan_type = context.obj['scan_type'] + scan_type = ctx.obj['scan_type'] - progress_bar = context.obj['progress_bar'] + progress_bar = ctx.obj['progress_bar'] progress_bar.start() if scan_type == consts.SCA_SCAN_TYPE: - scan_sca_pre_commit(context) + scan_sca_pre_commit(ctx) return diff_files = git_proxy.get_repo(os.getcwd()).index.diff('HEAD', create_patch=True, R=True) @@ -44,4 +44,4 @@ def pre_commit_command(context: click.Context, ignored_args: List[str]) -> None: documents_to_scan.append(Document(get_path_by_os(get_diff_file_path(file)), get_diff_file_content(file))) documents_to_scan = exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan) - scan_documents(context, documents_to_scan, is_git_diff=True) + scan_documents(ctx, documents_to_scan, is_git_diff=True) diff --git a/cycode/cli/commands/report/sbom/__init__.py b/cycode/cli/apps/scan/pre_receive/__init__.py similarity index 100% rename from cycode/cli/commands/report/sbom/__init__.py rename to cycode/cli/apps/scan/pre_receive/__init__.py diff --git a/cycode/cli/commands/scan/pre_receive/pre_receive_command.py b/cycode/cli/apps/scan/pre_receive/pre_receive_command.py similarity index 73% rename from cycode/cli/commands/scan/pre_receive/pre_receive_command.py rename to cycode/cli/apps/scan/pre_receive/pre_receive_command.py index 3ad59bad..92c152e6 100644 --- a/cycode/cli/commands/scan/pre_receive/pre_receive_command.py +++ b/cycode/cli/apps/scan/pre_receive/pre_receive_command.py @@ -1,10 +1,11 @@ import os -from typing import List +from typing import Annotated, List, Optional import click +import typer from cycode.cli import consts -from cycode.cli.commands.scan.code_scanner import ( +from cycode.cli.apps.scan.code_scanner import ( enable_verbose_mode, is_verbose_mode_requested_in_pre_receive_scan, parse_pre_receive_input, @@ -17,19 +18,19 @@ from cycode.cli.files_collector.repository_documents import ( calculate_pre_receive_commit_range, ) -from cycode.cli.sentry import add_breadcrumb +from cycode.cli.utils.sentry import add_breadcrumb from cycode.cli.utils.task_timer import TimeoutAfter from cycode.cyclient import logger -@click.command(short_help='Use this command to scan commits on the server side before pushing them to the repository.') -@click.argument('ignored_args', nargs=-1, type=click.UNPROCESSED) -@click.pass_context -def pre_receive_command(context: click.Context, ignored_args: List[str]) -> None: +def pre_receive_command( + ctx: typer.Context, + _: Annotated[Optional[List[str]], typer.Argument(help='Ignored arguments', hidden=True)] = None, +) -> None: try: add_breadcrumb('pre_receive') - scan_type = context.obj['scan_type'] + scan_type = ctx.obj['scan_type'] if scan_type != consts.SECRET_SCAN_TYPE: raise click.ClickException(f'Commit range scanning for {scan_type.upper()} is not supported') @@ -41,10 +42,10 @@ def pre_receive_command(context: click.Context, ignored_args: List[str]) -> None return if is_verbose_mode_requested_in_pre_receive_scan(): - enable_verbose_mode(context) + enable_verbose_mode(ctx) logger.debug('Verbose mode enabled: all log levels will be displayed.') - command_scan_type = context.info_name + command_scan_type = ctx.info_name timeout = configuration_manager.get_pre_receive_command_timeout(command_scan_type) with TimeoutAfter(timeout): if scan_type not in consts.COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES: @@ -60,7 +61,7 @@ def pre_receive_command(context: click.Context, ignored_args: List[str]) -> None return max_commits_to_scan = configuration_manager.get_pre_receive_max_commits_to_scan_count(command_scan_type) - scan_commit_range(context, os.getcwd(), commit_range, max_commits_count=max_commits_to_scan) - perform_post_pre_receive_scan_actions(context) + scan_commit_range(ctx, os.getcwd(), commit_range, max_commits_count=max_commits_to_scan) + perform_post_pre_receive_scan_actions(ctx) except Exception as e: - handle_scan_exception(context, e) + handle_scan_exception(ctx, e) diff --git a/cycode/cli/commands/report/sbom/path/__init__.py b/cycode/cli/apps/scan/repository/__init__.py similarity index 100% rename from cycode/cli/commands/report/sbom/path/__init__.py rename to cycode/cli/apps/scan/repository/__init__.py diff --git a/cycode/cli/commands/scan/repository/repository_command.py b/cycode/cli/apps/scan/repository/repository_command.py similarity index 66% rename from cycode/cli/commands/scan/repository/repository_command.py rename to cycode/cli/apps/scan/repository/repository_command.py index 9485c31c..0503c237 100644 --- a/cycode/cli/commands/scan/repository/repository_command.py +++ b/cycode/cli/apps/scan/repository/repository_command.py @@ -1,46 +1,46 @@ import os +from pathlib import Path +from typing import Annotated, Optional import click +import typer from cycode.cli import consts -from cycode.cli.commands.scan.code_scanner import get_scan_parameters, scan_documents +from cycode.cli.apps.scan.code_scanner import get_scan_parameters, scan_documents from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception from cycode.cli.files_collector.excluder import exclude_irrelevant_documents_to_scan from cycode.cli.files_collector.repository_documents import get_git_repository_tree_file_entries from cycode.cli.files_collector.sca.sca_code_scanner import perform_pre_scan_documents_actions from cycode.cli.models import Document -from cycode.cli.sentry import add_breadcrumb from cycode.cli.utils.path_utils import get_path_by_os from cycode.cli.utils.progress_bar import ScanProgressBarSection +from cycode.cli.utils.sentry import add_breadcrumb from cycode.cyclient import logger -@click.command(short_help='Scan the git repository including its history.') -@click.argument('path', nargs=1, type=click.Path(exists=True, resolve_path=True), required=True) -@click.option( - '--branch', - '-b', - default=None, - help='Branch to scan, if not set scanning the default branch', - type=str, - required=False, -) -@click.pass_context -def repository_command(context: click.Context, path: str, branch: str) -> None: +def repository_command( + ctx: typer.Context, + path: Annotated[ + Path, typer.Argument(exists=True, resolve_path=True, help='Path to git repository to scan.', show_default=False) + ], + branch: Annotated[ + Optional[str], typer.Option('--branch', '-b', help='Branch to scan.', show_default='default branch') + ] = None, +) -> None: try: add_breadcrumb('repository') logger.debug('Starting repository scan process, %s', {'path': path, 'branch': branch}) - scan_type = context.obj['scan_type'] - monitor = context.obj.get('monitor') + scan_type = ctx.obj['scan_type'] + monitor = ctx.obj.get('monitor') if monitor and scan_type != consts.SCA_SCAN_TYPE: raise click.ClickException('Monitor flag is currently supported for SCA scan type only') - progress_bar = context.obj['progress_bar'] + progress_bar = ctx.obj['progress_bar'] progress_bar.start() - file_entries = list(get_git_repository_tree_file_entries(path, branch)) + file_entries = list(get_git_repository_tree_file_entries(str(path), branch)) progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, len(file_entries)) documents_to_scan = [] @@ -60,10 +60,10 @@ def repository_command(context: click.Context, path: str, branch: str) -> None: documents_to_scan = exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan) - perform_pre_scan_documents_actions(context, scan_type, documents_to_scan) + perform_pre_scan_documents_actions(ctx, scan_type, documents_to_scan) logger.debug('Found all relevant files for scanning %s', {'path': path, 'branch': branch}) - scan_parameters = get_scan_parameters(context, (path,)) - scan_documents(context, documents_to_scan, scan_parameters=scan_parameters) + scan_parameters = get_scan_parameters(ctx, (str(path),)) + scan_documents(ctx, documents_to_scan, scan_parameters=scan_parameters) except Exception as e: - handle_scan_exception(context, e) + handle_scan_exception(ctx, e) diff --git a/cycode/cli/commands/report/sbom/repository_url/__init__.py b/cycode/cli/apps/scan/scan_ci/__init__.py similarity index 100% rename from cycode/cli/commands/report/sbom/repository_url/__init__.py rename to cycode/cli/apps/scan/scan_ci/__init__.py diff --git a/cycode/cli/commands/scan/scan_ci/ci_integrations.py b/cycode/cli/apps/scan/scan_ci/ci_integrations.py similarity index 100% rename from cycode/cli/commands/scan/scan_ci/ci_integrations.py rename to cycode/cli/apps/scan/scan_ci/ci_integrations.py diff --git a/cycode/cli/apps/scan/scan_ci/scan_ci_command.py b/cycode/cli/apps/scan/scan_ci/scan_ci_command.py new file mode 100644 index 00000000..cbfebb72 --- /dev/null +++ b/cycode/cli/apps/scan/scan_ci/scan_ci_command.py @@ -0,0 +1,20 @@ +import os + +import click +import typer + +from cycode.cli.apps.scan.code_scanner import scan_commit_range +from cycode.cli.apps.scan.scan_ci.ci_integrations import get_commit_range +from cycode.cli.utils.sentry import add_breadcrumb + +# This command is not finished yet. It is not used in the codebase. + + +@click.command( + short_help='Execute scan in a CI environment which relies on the ' + 'CYCODE_TOKEN and CYCODE_REPO_LOCATION environment variables' +) +@click.pass_context +def scan_ci_command(ctx: typer.Context) -> None: + add_breadcrumb('ci') + scan_commit_range(ctx, path=os.getcwd(), commit_range=get_commit_range()) diff --git a/cycode/cli/apps/scan/scan_command.py b/cycode/cli/apps/scan/scan_command.py new file mode 100644 index 00000000..dffbf34f --- /dev/null +++ b/cycode/cli/apps/scan/scan_command.py @@ -0,0 +1,148 @@ +from typing import Annotated, List, Optional + +import click +import typer + +from cycode.cli.cli_types import ScanTypeOption, ScaScanTypeOption, SeverityOption +from cycode.cli.config import config +from cycode.cli.consts import ( + ISSUE_DETECTED_STATUS_CODE, + NO_ISSUES_STATUS_CODE, + SCA_GRADLE_ALL_SUB_PROJECTS_FLAG, + SCA_SKIP_RESTORE_DEPENDENCIES_FLAG, +) +from cycode.cli.utils import scan_utils +from cycode.cli.utils.get_api_client import get_scan_cycode_client +from cycode.cli.utils.sentry import add_breadcrumb + + +def scan_command( + ctx: typer.Context, + scan_type: Annotated[ + ScanTypeOption, + typer.Option( + '--scan-type', + '-t', + help='Specify the type of scan you wish to execute.', + case_sensitive=False, + ), + ] = ScanTypeOption.SECRET, + client_secret: Annotated[ + Optional[str], + typer.Option( + help='Specify a Cycode client secret for this specific scan execution.', + rich_help_panel='Authentication options', + ), + ] = None, + client_id: Annotated[ + Optional[str], + typer.Option( + help='Specify a Cycode client ID for this specific scan execution.', + rich_help_panel='Authentication options', + ), + ] = None, + show_secret: Annotated[bool, typer.Option('--show-secret', help='Show Secrets in plain text.')] = False, + soft_fail: Annotated[ + bool, typer.Option('--soft-fail', help='Run the scan without failing; always return a non-error status code.') + ] = False, + severity_threshold: Annotated[ + SeverityOption, + typer.Option( + help='Show violations only for the specified level or higher.', + case_sensitive=False, + ), + ] = SeverityOption.INFO, + sync: Annotated[ + bool, + typer.Option('--sync', help='Run scan synchronously.', show_default='asynchronously'), + ] = False, + report: Annotated[ + bool, + typer.Option( + '--report', + help='When specified, generates a violations report. ' + 'A link to the report will be displayed in the console output.', + ), + ] = False, + sca_scan: Annotated[ + List[ScaScanTypeOption], + typer.Option( + help='Specify the type of SCA scan you wish to execute.', + rich_help_panel='SCA options', + ), + ] = (ScaScanTypeOption.PACKAGE_VULNERABILITIES, ScaScanTypeOption.LICENSE_COMPLIANCE), + monitor: Annotated[ + bool, + typer.Option( + '--monitor', + help='When specified, the scan results are recorded in the Discovery module.', + rich_help_panel='SCA options', + ), + ] = False, + no_restore: Annotated[ + bool, + typer.Option( + f'--{SCA_SKIP_RESTORE_DEPENDENCIES_FLAG}', + help='When specified, Cycode will not run restore command. ' + 'Will scan direct dependencies [bold]only[/bold]!', + rich_help_panel='SCA options', + ), + ] = False, + gradle_all_sub_projects: Annotated[ + bool, + typer.Option( + f'--{SCA_GRADLE_ALL_SUB_PROJECTS_FLAG}', + help='When specified, Cycode will run gradle restore command for all sub projects. ' + 'Should run from root project directory [bold]only[/bold]!', + rich_help_panel='SCA options', + ), + ] = False, +) -> None: + """:magnifying_glass_tilted_right: Scan the content for Secrets, IaC, SCA, and SAST violations. + You'll need to specify which scan type to perform: + [cyan]path[/cyan]/[cyan]repository[/cyan]/[cyan]commit_history[/cyan].""" + add_breadcrumb('scan') + + if show_secret: + ctx.obj['show_secret'] = show_secret + else: + ctx.obj['show_secret'] = config['result_printer']['default']['show_secret'] + + if soft_fail: + ctx.obj['soft_fail'] = soft_fail + else: + ctx.obj['soft_fail'] = config['soft_fail'] + + ctx.obj['client'] = get_scan_cycode_client(client_id, client_secret, not ctx.obj['show_secret']) + ctx.obj['scan_type'] = scan_type + ctx.obj['sync'] = sync + ctx.obj['severity_threshold'] = severity_threshold + ctx.obj['monitor'] = monitor + ctx.obj['report'] = report + ctx.obj[SCA_SKIP_RESTORE_DEPENDENCIES_FLAG] = no_restore + ctx.obj[SCA_GRADLE_ALL_SUB_PROJECTS_FLAG] = gradle_all_sub_projects + + _sca_scan_to_context(ctx, sca_scan) + + +def _sca_scan_to_context(ctx: typer.Context, sca_scan_user_selected: List[str]) -> None: + for sca_scan_option_selected in sca_scan_user_selected: + ctx.obj[sca_scan_option_selected] = True + + +@click.pass_context +def scan_command_result_callback(ctx: click.Context, *_, **__) -> None: + add_breadcrumb('scan_finalize') + + progress_bar = ctx.obj.get('progress_bar') + if progress_bar: + progress_bar.stop() + + if ctx.obj['soft_fail']: + raise typer.Exit(0) + + exit_code = NO_ISSUES_STATUS_CODE + if scan_utils.is_scan_failed(ctx): + exit_code = ISSUE_DETECTED_STATUS_CODE + + raise typer.Exit(exit_code) diff --git a/cycode/cli/apps/status/__init__.py b/cycode/cli/apps/status/__init__.py new file mode 100644 index 00000000..f01e3b30 --- /dev/null +++ b/cycode/cli/apps/status/__init__.py @@ -0,0 +1,8 @@ +import typer + +from cycode.cli.apps.status.status_command import status_command +from cycode.cli.apps.status.version_command import version_command + +app = typer.Typer() +app.command(name='status', short_help='Show the CLI status and exit.')(status_command) +app.command(name='version', hidden=True, short_help='Alias to status command.')(version_command) diff --git a/cycode/cli/apps/status/get_cli_status.py b/cycode/cli/apps/status/get_cli_status.py new file mode 100644 index 00000000..e58e910b --- /dev/null +++ b/cycode/cli/apps/status/get_cli_status.py @@ -0,0 +1,45 @@ +import platform + +from cycode import __version__ +from cycode.cli.apps.auth.auth_common import get_authorization_info +from cycode.cli.apps.status.models import CliStatus, CliSupportedModulesStatus +from cycode.cli.consts import PROGRAM_NAME +from cycode.cli.user_settings.configuration_manager import ConfigurationManager +from cycode.cli.utils.get_api_client import get_scan_cycode_client +from cycode.cyclient import logger + + +def get_cli_status() -> CliStatus: + configuration_manager = ConfigurationManager() + + auth_info = get_authorization_info() + is_authenticated = auth_info is not None + + supported_modules_status = CliSupportedModulesStatus() + if is_authenticated: + try: + client = get_scan_cycode_client() + supported_modules_preferences = client.get_supported_modules_preferences() + + supported_modules_status.secret_scanning = supported_modules_preferences.secret_scanning + supported_modules_status.sca_scanning = supported_modules_preferences.sca_scanning + supported_modules_status.iac_scanning = supported_modules_preferences.iac_scanning + supported_modules_status.sast_scanning = supported_modules_preferences.sast_scanning + supported_modules_status.ai_large_language_model = supported_modules_preferences.ai_large_language_model + except Exception as e: + logger.debug('Failed to get supported modules preferences', exc_info=e) + + return CliStatus( + program=PROGRAM_NAME, + version=__version__, + os=platform.system(), + arch=platform.machine(), + python_version=platform.python_version(), + installation_id=configuration_manager.get_or_create_installation_id(), + app_url=configuration_manager.get_cycode_app_url(), + api_url=configuration_manager.get_cycode_api_url(), + is_authenticated=is_authenticated, + user_id=auth_info.user_id if auth_info else None, + tenant_id=auth_info.tenant_id if auth_info else None, + supported_modules=supported_modules_status, + ) diff --git a/cycode/cli/apps/status/models.py b/cycode/cli/apps/status/models.py new file mode 100644 index 00000000..50182ecd --- /dev/null +++ b/cycode/cli/apps/status/models.py @@ -0,0 +1,62 @@ +import json +from dataclasses import asdict, dataclass +from typing import Dict + + +class CliStatusBase: + def as_dict(self) -> Dict[str, any]: + return asdict(self) + + def _get_text_message_part(self, key: str, value: any, intent: int = 0) -> str: + message_parts = [] + + intent_prefix = ' ' * intent * 2 + human_readable_key = key.replace('_', ' ').capitalize() + + if isinstance(value, dict): + message_parts.append(f'{intent_prefix}{human_readable_key}:') + for sub_key, sub_value in value.items(): + message_parts.append(self._get_text_message_part(sub_key, sub_value, intent=intent + 1)) + elif isinstance(value, (list, set, tuple)): + message_parts.append(f'{intent_prefix}{human_readable_key}:') + for index, sub_value in enumerate(value): + message_parts.append(self._get_text_message_part(f'#{index}', sub_value, intent=intent + 1)) + else: + message_parts.append(f'{intent_prefix}{human_readable_key}: {value}') + + return '\n'.join(message_parts) + + def as_text(self) -> str: + message_parts = [] + for key, value in self.as_dict().items(): + message_parts.append(self._get_text_message_part(key, value)) + + return '\n'.join(message_parts) + + def as_json(self) -> str: + return json.dumps(self.as_dict()) + + +@dataclass +class CliSupportedModulesStatus(CliStatusBase): + secret_scanning: bool = False + sca_scanning: bool = False + iac_scanning: bool = False + sast_scanning: bool = False + ai_large_language_model: bool = False + + +@dataclass +class CliStatus(CliStatusBase): + program: str + version: str + os: str + arch: str + python_version: str + installation_id: str + app_url: str + api_url: str + is_authenticated: bool + user_id: str = None + tenant_id: str = None + supported_modules: CliSupportedModulesStatus = None diff --git a/cycode/cli/apps/status/status_command.py b/cycode/cli/apps/status/status_command.py new file mode 100644 index 00000000..edffd24c --- /dev/null +++ b/cycode/cli/apps/status/status_command.py @@ -0,0 +1,15 @@ +import typer + +from cycode.cli.apps.status.get_cli_status import get_cli_status +from cycode.cli.cli_types import OutputTypeOption + + +def status_command(ctx: typer.Context) -> None: + output = ctx.obj['output'] + + cli_status = get_cli_status() + message = cli_status.as_text() + if output == OutputTypeOption.JSON: + message = cli_status.as_json() + + typer.echo(message, color=ctx.color) diff --git a/cycode/cli/apps/status/version_command.py b/cycode/cli/apps/status/version_command.py new file mode 100644 index 00000000..c36aad4b --- /dev/null +++ b/cycode/cli/apps/status/version_command.py @@ -0,0 +1,15 @@ +import typer + +from cycode.cli.apps.status.status_command import status_command + + +def version_command(ctx: typer.Context) -> None: + typer.echo( + typer.style( + text='The "version" command is deprecated. Please use the "status" command instead.', + fg=typer.colors.YELLOW, + bold=True, + ), + color=ctx.color, + ) + status_command(ctx) diff --git a/cycode/cli/cli_types.py b/cycode/cli/cli_types.py new file mode 100644 index 00000000..92c36fa2 --- /dev/null +++ b/cycode/cli/cli_types.py @@ -0,0 +1,53 @@ +from enum import Enum + +from cycode.cli import consts + + +class OutputTypeOption(str, Enum): + TEXT = 'text' + JSON = 'json' + TABLE = 'table' + + +class ScanTypeOption(str, Enum): + SECRET = consts.SECRET_SCAN_TYPE + SCA = consts.SCA_SCAN_TYPE + IAC = consts.IAC_SCAN_TYPE + SAST = consts.SAST_SCAN_TYPE + + +class ScaScanTypeOption(str, Enum): + PACKAGE_VULNERABILITIES = 'package-vulnerabilities' + LICENSE_COMPLIANCE = 'license-compliance' + + +class SbomFormatOption(str, Enum): + SPDX_2_2 = 'spdx-2.2' + SPDX_2_3 = 'spdx-2.3' + CYCLONEDX_1_4 = 'cyclonedx-1.4' + + +class SbomOutputFormatOption(str, Enum): + JSON = 'json' + + +class SeverityOption(str, Enum): + INFO = 'info' + LOW = 'low' + MEDIUM = 'medium' + HIGH = 'high' + CRITICAL = 'critical' + + @staticmethod + def get_member_weight(name: str) -> int: + return _SEVERITY_WEIGHTS.get(name.lower(), _SEVERITY_DEFAULT_WEIGHT) + + +_SEVERITY_DEFAULT_WEIGHT = -1 +_SEVERITY_WEIGHTS = { + SeverityOption.INFO.value: 0, + SeverityOption.LOW.value: 1, + SeverityOption.MEDIUM.value: 2, + SeverityOption.HIGH.value: 3, + SeverityOption.CRITICAL.value: 4, +} diff --git a/cycode/cli/commands/ai_remediation/ai_remediation_command.py b/cycode/cli/commands/ai_remediation/ai_remediation_command.py deleted file mode 100644 index 608fc9f4..00000000 --- a/cycode/cli/commands/ai_remediation/ai_remediation_command.py +++ /dev/null @@ -1,67 +0,0 @@ -import os - -import click -from patch_ng import fromstring -from rich.console import Console -from rich.markdown import Markdown - -from cycode.cli.exceptions.handle_ai_remediation_errors import handle_ai_remediation_exception -from cycode.cli.models import CliResult -from cycode.cli.printers import ConsolePrinter -from cycode.cli.utils.get_api_client import get_scan_cycode_client - - -def _echo_remediation(context: click.Context, remediation_markdown: str, is_fix_available: bool) -> None: - printer = ConsolePrinter(context) - if printer.is_json_printer: - data = {'remediation': remediation_markdown, 'is_fix_available': is_fix_available} - printer.print_result(CliResult(success=True, message='Remediation fetched successfully', data=data)) - else: # text or table - Console().print(Markdown(remediation_markdown)) - - -def _apply_fix(context: click.Context, diff: str, is_fix_available: bool) -> None: - printer = ConsolePrinter(context) - if not is_fix_available: - printer.print_result(CliResult(success=False, message='Fix is not available for this violation')) - return - - patch = fromstring(diff.encode('UTF-8')) - if patch is False: - printer.print_result(CliResult(success=False, message='Failed to parse fix diff')) - return - - is_fix_applied = patch.apply(root=os.getcwd(), strip=0) - if is_fix_applied: - printer.print_result(CliResult(success=True, message='Fix applied successfully')) - else: - printer.print_result(CliResult(success=False, message='Failed to apply fix')) - - -@click.command(short_help='Get AI remediation (INTERNAL).', hidden=True) -@click.argument('detection_id', nargs=1, type=click.UUID, required=True) -@click.option( - '--fix', - is_flag=True, - default=False, - help='Apply fixes to resolve violations. Fix is not available for all violations.', - type=click.BOOL, - required=False, -) -@click.pass_context -def ai_remediation_command(context: click.Context, detection_id: str, fix: bool) -> None: - client = get_scan_cycode_client() - - try: - remediation_markdown = client.get_ai_remediation(detection_id) - fix_diff = client.get_ai_remediation(detection_id, fix=True) - is_fix_available = bool(fix_diff) # exclude empty string, None, etc. - - if fix: - _apply_fix(context, fix_diff, is_fix_available) - else: - _echo_remediation(context, remediation_markdown, is_fix_available) - except Exception as err: - handle_ai_remediation_exception(context, err) - - context.exit() diff --git a/cycode/cli/commands/auth/auth_command.py b/cycode/cli/commands/auth/auth_command.py deleted file mode 100644 index 0862db2b..00000000 --- a/cycode/cli/commands/auth/auth_command.py +++ /dev/null @@ -1,82 +0,0 @@ -import click - -from cycode.cli.commands.auth.auth_manager import AuthManager -from cycode.cli.commands.auth_common import get_authorization_info -from cycode.cli.exceptions.custom_exceptions import ( - KNOWN_USER_FRIENDLY_REQUEST_ERRORS, - AuthProcessError, -) -from cycode.cli.models import CliError, CliErrors, CliResult -from cycode.cli.printers import ConsolePrinter -from cycode.cli.sentry import add_breadcrumb, capture_exception -from cycode.cyclient import logger - - -@click.group( - invoke_without_command=True, short_help='Authenticate your machine to associate the CLI with your Cycode account.' -) -@click.pass_context -def auth_command(context: click.Context) -> None: - """Authenticates your machine.""" - add_breadcrumb('auth') - - if context.invoked_subcommand is not None: - # if it is a subcommand, do nothing - return - - try: - logger.debug('Starting authentication process') - - auth_manager = AuthManager() - auth_manager.authenticate() - - result = CliResult(success=True, message='Successfully logged into cycode') - ConsolePrinter(context).print_result(result) - except Exception as e: - _handle_exception(context, e) - - -@auth_command.command( - name='check', short_help='Checks that your machine is associating the CLI with your Cycode account.' -) -@click.pass_context -def authorization_check(context: click.Context) -> None: - """Validates that your Cycode account has permission to work with the CLI.""" - add_breadcrumb('check') - - printer = ConsolePrinter(context) - auth_info = get_authorization_info(context) - if auth_info is None: - printer.print_result(CliResult(success=False, message='Cycode authentication failed')) - return - - printer.print_result( - CliResult( - success=True, - message='Cycode authentication verified', - data={'user_id': auth_info.user_id, 'tenant_id': auth_info.tenant_id}, - ) - ) - - -def _handle_exception(context: click.Context, e: Exception) -> None: - ConsolePrinter(context).print_exception() - - errors: CliErrors = { - **KNOWN_USER_FRIENDLY_REQUEST_ERRORS, - AuthProcessError: CliError( - code='auth_error', message='Authentication failed. Please try again later using the command `cycode auth`' - ), - } - - error = errors.get(type(e)) - if error: - ConsolePrinter(context).print_error(error) - return - - if isinstance(e, click.ClickException): - raise e - - capture_exception(e) - - raise click.ClickException(str(e)) diff --git a/cycode/cli/commands/configure/configure_command.py b/cycode/cli/commands/configure/configure_command.py deleted file mode 100644 index 8f76d159..00000000 --- a/cycode/cli/commands/configure/configure_command.py +++ /dev/null @@ -1,140 +0,0 @@ -from typing import Optional - -import click - -from cycode.cli import config, consts -from cycode.cli.sentry import add_breadcrumb -from cycode.cli.user_settings.configuration_manager import ConfigurationManager -from cycode.cli.user_settings.credentials_manager import CredentialsManager -from cycode.cli.utils.string_utils import obfuscate_text - -_URLS_UPDATED_SUCCESSFULLY_MESSAGE = 'Successfully configured Cycode URLs! Saved to: {filename}' -_URLS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE = ( - 'Note that the URLs (APP and API) that already exist in environment variables ' - f'({consts.CYCODE_API_URL_ENV_VAR_NAME} and {consts.CYCODE_APP_URL_ENV_VAR_NAME}) ' - 'take precedent over these URLs; either update or remove the environment variables.' -) -_CREDENTIALS_UPDATED_SUCCESSFULLY_MESSAGE = 'Successfully configured CLI credentials! Saved to: {filename}' -_CREDENTIALS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE = ( - 'Note that the credentials that already exist in environment variables ' - f'({config.CYCODE_CLIENT_ID_ENV_VAR_NAME} and {config.CYCODE_CLIENT_SECRET_ENV_VAR_NAME}) ' - 'take precedent over these credentials; either update or remove the environment variables.' -) -_CREDENTIALS_MANAGER = CredentialsManager() -_CONFIGURATION_MANAGER = ConfigurationManager() - - -@click.command(short_help='Initial command to configure your CLI client authentication.') -def configure_command() -> None: - """Configure your CLI client authentication manually.""" - add_breadcrumb('configure') - - global_config_manager = _CONFIGURATION_MANAGER.global_config_file_manager - - current_api_url = global_config_manager.get_api_url() - current_app_url = global_config_manager.get_app_url() - api_url = _get_api_url_input(current_api_url) - app_url = _get_app_url_input(current_app_url) - - config_updated = False - if _should_update_value(current_api_url, api_url): - global_config_manager.update_api_base_url(api_url) - config_updated = True - if _should_update_value(current_app_url, app_url): - global_config_manager.update_app_base_url(app_url) - config_updated = True - - current_client_id, current_client_secret = _CREDENTIALS_MANAGER.get_credentials_from_file() - client_id = _get_client_id_input(current_client_id) - client_secret = _get_client_secret_input(current_client_secret) - - credentials_updated = False - if _should_update_value(current_client_id, client_id) or _should_update_value(current_client_secret, client_secret): - credentials_updated = True - _CREDENTIALS_MANAGER.update_credentials(client_id, client_secret) - - if config_updated: - click.echo(_get_urls_update_result_message()) - if credentials_updated: - click.echo(_get_credentials_update_result_message()) - - -def _get_client_id_input(current_client_id: Optional[str]) -> Optional[str]: - prompt_text = 'Cycode Client ID' - - prompt_suffix = ' []: ' - if current_client_id: - prompt_suffix = f' [{obfuscate_text(current_client_id)}]: ' - - new_client_id = click.prompt(text=prompt_text, prompt_suffix=prompt_suffix, default='', show_default=False) - return new_client_id or current_client_id - - -def _get_client_secret_input(current_client_secret: Optional[str]) -> Optional[str]: - prompt_text = 'Cycode Client Secret' - - prompt_suffix = ' []: ' - if current_client_secret: - prompt_suffix = f' [{obfuscate_text(current_client_secret)}]: ' - - new_client_secret = click.prompt(text=prompt_text, prompt_suffix=prompt_suffix, default='', show_default=False) - return new_client_secret or current_client_secret - - -def _get_app_url_input(current_app_url: Optional[str]) -> str: - prompt_text = 'Cycode APP URL' - - default = consts.DEFAULT_CYCODE_APP_URL - if current_app_url: - default = current_app_url - - return click.prompt(text=prompt_text, default=default, type=click.STRING) - - -def _get_api_url_input(current_api_url: Optional[str]) -> str: - prompt_text = 'Cycode API URL' - - default = consts.DEFAULT_CYCODE_API_URL - if current_api_url: - default = current_api_url - - return click.prompt(text=prompt_text, default=default, type=click.STRING) - - -def _get_credentials_update_result_message() -> str: - success_message = _CREDENTIALS_UPDATED_SUCCESSFULLY_MESSAGE.format(filename=_CREDENTIALS_MANAGER.get_filename()) - if _are_credentials_exist_in_environment_variables(): - return f'{success_message}. {_CREDENTIALS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE}' - - return success_message - - -def _are_credentials_exist_in_environment_variables() -> bool: - client_id, client_secret = _CREDENTIALS_MANAGER.get_credentials_from_environment_variables() - return any([client_id, client_secret]) - - -def _get_urls_update_result_message() -> str: - success_message = _URLS_UPDATED_SUCCESSFULLY_MESSAGE.format( - filename=_CONFIGURATION_MANAGER.global_config_file_manager.get_filename() - ) - if _are_urls_exist_in_environment_variables(): - return f'{success_message}. {_URLS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE}' - - return success_message - - -def _are_urls_exist_in_environment_variables() -> bool: - api_url = _CONFIGURATION_MANAGER.get_api_url_from_environment_variables() - app_url = _CONFIGURATION_MANAGER.get_app_url_from_environment_variables() - return any([api_url, app_url]) - - -def _should_update_value( - old_value: Optional[str], - new_value: Optional[str], -) -> bool: - if not new_value: - return False - - return old_value != new_value diff --git a/cycode/cli/commands/main_cli.py b/cycode/cli/commands/main_cli.py deleted file mode 100644 index 59b8625f..00000000 --- a/cycode/cli/commands/main_cli.py +++ /dev/null @@ -1,117 +0,0 @@ -import logging -from typing import Optional - -import click - -from cycode import __version__ -from cycode.cli.commands.ai_remediation.ai_remediation_command import ai_remediation_command -from cycode.cli.commands.auth.auth_command import auth_command -from cycode.cli.commands.configure.configure_command import configure_command -from cycode.cli.commands.ignore.ignore_command import ignore_command -from cycode.cli.commands.report.report_command import report_command -from cycode.cli.commands.scan.scan_command import scan_command -from cycode.cli.commands.status.status_command import status_command -from cycode.cli.commands.version.version_checker import version_checker -from cycode.cli.commands.version.version_command import version_command -from cycode.cli.consts import ( - CLI_CONTEXT_SETTINGS, -) -from cycode.cli.sentry import add_breadcrumb, init_sentry -from cycode.cli.user_settings.configuration_manager import ConfigurationManager -from cycode.cli.utils.progress_bar import SCAN_PROGRESS_BAR_SECTIONS, get_progress_bar -from cycode.cyclient.config import set_logging_level -from cycode.cyclient.cycode_client_base import CycodeClientBase -from cycode.cyclient.models import UserAgentOptionScheme - - -@click.group( - commands={ - 'scan': scan_command, - 'report': report_command, - 'configure': configure_command, - 'ignore': ignore_command, - 'auth': auth_command, - 'version': version_command, - 'status': status_command, - 'ai_remediation': ai_remediation_command, - }, - context_settings=CLI_CONTEXT_SETTINGS, -) -@click.option( - '--verbose', - '-v', - is_flag=True, - default=False, - help='Show detailed logs.', -) -@click.option( - '--no-progress-meter', - is_flag=True, - default=False, - help='Do not show the progress meter.', -) -@click.option( - '--no-update-notifier', - is_flag=True, - default=False, - help='Do not check CLI for updates.', -) -@click.option( - '--output', - '-o', - default='text', - help='Specify the output type (the default is text).', - type=click.Choice(['text', 'json', 'table']), -) -@click.option( - '--user-agent', - default=None, - help='Characteristic JSON object that lets servers identify the application.', - type=str, -) -@click.pass_context -def main_cli( - context: click.Context, - verbose: bool, - no_progress_meter: bool, - no_update_notifier: bool, - output: str, - user_agent: Optional[str], -) -> None: - init_sentry() - add_breadcrumb('cycode') - - context.ensure_object(dict) - configuration_manager = ConfigurationManager() - - verbose = verbose or configuration_manager.get_verbose_flag() - context.obj['verbose'] = verbose - if verbose: - set_logging_level(logging.DEBUG) - - context.obj['output'] = output - if output == 'json': - no_progress_meter = True - - context.obj['progress_bar'] = get_progress_bar(hidden=no_progress_meter, sections=SCAN_PROGRESS_BAR_SECTIONS) - - if user_agent: - user_agent_option = UserAgentOptionScheme().loads(user_agent) - CycodeClientBase.enrich_user_agent(user_agent_option.user_agent_suffix) - - if not no_update_notifier: - context.call_on_close(lambda: check_latest_version_on_close()) - - -@click.pass_context -def check_latest_version_on_close(context: click.Context) -> None: - output = context.obj.get('output') - # don't print anything if the output is JSON - if output == 'json': - return - - # we always want to check the latest version for "version" and "status" commands - should_use_cache = context.invoked_subcommand not in {'version', 'status'} - version_checker.check_and_notify_update( - current_version=__version__, use_color=context.color, use_cache=should_use_cache - ) diff --git a/cycode/cli/commands/report/report_command.py b/cycode/cli/commands/report/report_command.py deleted file mode 100644 index 9e92a64f..00000000 --- a/cycode/cli/commands/report/report_command.py +++ /dev/null @@ -1,21 +0,0 @@ -import click - -from cycode.cli.commands.report.sbom.sbom_command import sbom_command -from cycode.cli.sentry import add_breadcrumb -from cycode.cli.utils.progress_bar import SBOM_REPORT_PROGRESS_BAR_SECTIONS, get_progress_bar - - -@click.group( - commands={ - 'sbom': sbom_command, - }, - short_help='Generate report. You`ll need to specify which report type to perform.', -) -@click.pass_context -def report_command( - context: click.Context, -) -> int: - """Generate report.""" - add_breadcrumb('report') - context.obj['progress_bar'] = get_progress_bar(hidden=False, sections=SBOM_REPORT_PROGRESS_BAR_SECTIONS) - return 1 diff --git a/cycode/cli/commands/report/sbom/sbom_command.py b/cycode/cli/commands/report/sbom/sbom_command.py deleted file mode 100644 index a938fd90..00000000 --- a/cycode/cli/commands/report/sbom/sbom_command.py +++ /dev/null @@ -1,87 +0,0 @@ -import pathlib -from typing import Optional - -import click - -from cycode.cli.commands.report.sbom.path.path_command import path_command -from cycode.cli.commands.report.sbom.repository_url.repository_url_command import repository_url_command -from cycode.cli.config import config -from cycode.cli.sentry import add_breadcrumb -from cycode.cyclient.report_client import ReportParameters - - -@click.group( - commands={ - 'path': path_command, - 'repository_url': repository_url_command, - }, - short_help='Generate SBOM report for remote repository by url or local directory by path.', -) -@click.option( - '--format', - '-f', - help='SBOM format.', - type=click.Choice(config['scans']['supported_sbom_formats']), - required=True, -) -@click.option( - '--output-format', - '-o', - default='json', - help='Specify the output file format (the default is json).', - type=click.Choice(['json']), - required=False, -) -@click.option( - '--output-file', - help='Output file (the default is autogenerated filename saved to the current directory).', - default=None, - type=click.Path(resolve_path=True, writable=True, path_type=pathlib.Path), - required=False, -) -@click.option( - '--include-vulnerabilities', - is_flag=True, - default=False, - help='Include vulnerabilities.', - type=bool, - required=False, -) -@click.option( - '--include-dev-dependencies', - is_flag=True, - default=False, - help='Include dev dependencies.', - type=bool, - required=False, -) -@click.pass_context -def sbom_command( - context: click.Context, - format: str, - output_format: Optional[str], - output_file: Optional[pathlib.Path], - include_vulnerabilities: bool, - include_dev_dependencies: bool, -) -> int: - """Generate SBOM report.""" - add_breadcrumb('sbom') - - sbom_format_parts = format.split('-') - if len(sbom_format_parts) != 2: - raise click.ClickException('Invalid SBOM format.') - - sbom_format, sbom_format_version = sbom_format_parts - - report_parameters = ReportParameters( - entity_type='SbomCli', - sbom_report_type=sbom_format, - sbom_version=sbom_format_version, - output_format=output_format, - include_vulnerabilities=include_vulnerabilities, - include_dev_dependencies=include_dev_dependencies, - ) - context.obj['report_parameters'] = report_parameters - context.obj['output_file'] = output_file - - return 1 diff --git a/cycode/cli/commands/scan/__init__.py b/cycode/cli/commands/scan/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cycode/cli/commands/scan/commit_history/__init__.py b/cycode/cli/commands/scan/commit_history/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cycode/cli/commands/scan/commit_history/commit_history_command.py b/cycode/cli/commands/scan/commit_history/commit_history_command.py deleted file mode 100644 index bfb57c29..00000000 --- a/cycode/cli/commands/scan/commit_history/commit_history_command.py +++ /dev/null @@ -1,27 +0,0 @@ -import click - -from cycode.cli.commands.scan.code_scanner import scan_commit_range -from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception -from cycode.cli.sentry import add_breadcrumb -from cycode.cyclient import logger - - -@click.command(short_help='Scan all the commits history in this git repository.') -@click.argument('path', nargs=1, type=click.Path(exists=True, resolve_path=True), required=True) -@click.option( - '--commit_range', - '-r', - help='Scan a commit range in this git repository, by default cycode scans all commit history (example: HEAD~1)', - type=click.STRING, - default='--all', - required=False, -) -@click.pass_context -def commit_history_command(context: click.Context, path: str, commit_range: str) -> None: - try: - add_breadcrumb('commit_history') - - logger.debug('Starting commit history scan process, %s', {'path': path, 'commit_range': commit_range}) - scan_commit_range(context, path=path, commit_range=commit_range) - except Exception as e: - handle_scan_exception(context, e) diff --git a/cycode/cli/commands/scan/path/__init__.py b/cycode/cli/commands/scan/path/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cycode/cli/commands/scan/path/path_command.py b/cycode/cli/commands/scan/path/path_command.py deleted file mode 100644 index ec62b224..00000000 --- a/cycode/cli/commands/scan/path/path_command.py +++ /dev/null @@ -1,20 +0,0 @@ -from typing import Tuple - -import click - -from cycode.cli.commands.scan.code_scanner import scan_disk_files -from cycode.cli.sentry import add_breadcrumb -from cycode.cyclient import logger - - -@click.command(short_help='Scan the files in the path provided in the command.') -@click.argument('paths', nargs=-1, type=click.Path(exists=True, resolve_path=True), required=True) -@click.pass_context -def path_command(context: click.Context, paths: Tuple[str]) -> None: - add_breadcrumb('path') - - progress_bar = context.obj['progress_bar'] - progress_bar.start() - - logger.debug('Starting path scan process, %s', {'paths': paths}) - scan_disk_files(context, paths) diff --git a/cycode/cli/commands/scan/pre_commit/__init__.py b/cycode/cli/commands/scan/pre_commit/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cycode/cli/commands/scan/pre_receive/__init__.py b/cycode/cli/commands/scan/pre_receive/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cycode/cli/commands/scan/repository/__init__.py b/cycode/cli/commands/scan/repository/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cycode/cli/commands/scan/scan_ci/__init__.py b/cycode/cli/commands/scan/scan_ci/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cycode/cli/commands/scan/scan_ci/scan_ci_command.py b/cycode/cli/commands/scan/scan_ci/scan_ci_command.py deleted file mode 100644 index 6d4fbd36..00000000 --- a/cycode/cli/commands/scan/scan_ci/scan_ci_command.py +++ /dev/null @@ -1,19 +0,0 @@ -import os - -import click - -from cycode.cli.commands.scan.code_scanner import scan_commit_range -from cycode.cli.commands.scan.scan_ci.ci_integrations import get_commit_range -from cycode.cli.sentry import add_breadcrumb - -# This command is not finished yet. It is not used in the codebase. - - -@click.command( - short_help='Execute scan in a CI environment which relies on the ' - 'CYCODE_TOKEN and CYCODE_REPO_LOCATION environment variables' -) -@click.pass_context -def scan_ci_command(context: click.Context) -> None: - add_breadcrumb('ci') - scan_commit_range(context, path=os.getcwd(), commit_range=get_commit_range()) diff --git a/cycode/cli/commands/scan/scan_command.py b/cycode/cli/commands/scan/scan_command.py deleted file mode 100644 index 95259f4a..00000000 --- a/cycode/cli/commands/scan/scan_command.py +++ /dev/null @@ -1,187 +0,0 @@ -import sys -from typing import List - -import click - -from cycode.cli import consts -from cycode.cli.commands.scan.commit_history.commit_history_command import commit_history_command -from cycode.cli.commands.scan.path.path_command import path_command -from cycode.cli.commands.scan.pre_commit.pre_commit_command import pre_commit_command -from cycode.cli.commands.scan.pre_receive.pre_receive_command import pre_receive_command -from cycode.cli.commands.scan.repository.repository_command import repository_command -from cycode.cli.config import config -from cycode.cli.consts import ( - ISSUE_DETECTED_STATUS_CODE, - NO_ISSUES_STATUS_CODE, - SCA_GRADLE_ALL_SUB_PROJECTS_FLAG, - SCA_SKIP_RESTORE_DEPENDENCIES_FLAG, -) -from cycode.cli.models import Severity -from cycode.cli.sentry import add_breadcrumb -from cycode.cli.utils import scan_utils -from cycode.cli.utils.get_api_client import get_scan_cycode_client - - -@click.group( - commands={ - 'repository': repository_command, - 'commit_history': commit_history_command, - 'path': path_command, - 'pre_commit': pre_commit_command, - 'pre_receive': pre_receive_command, - }, - short_help='Scan the content for Secrets/IaC/SCA/SAST violations. ' - 'You`ll need to specify which scan type to perform: commit_history/path/repository/etc.', -) -@click.option( - '--scan-type', - '-t', - default=consts.SECRET_SCAN_TYPE, - help='Specify the type of scan you wish to execute (the default is Secrets).', - type=click.Choice(config['scans']['supported_scans']), -) -@click.option( - '--secret', - default=None, - help='Specify a Cycode client secret for this specific scan execution.', - type=str, - required=False, -) -@click.option( - '--client-id', - default=None, - help='Specify a Cycode client ID for this specific scan execution.', - type=str, - required=False, -) -@click.option( - '--show-secret', is_flag=True, default=False, help='Show Secrets in plain text.', type=bool, required=False -) -@click.option( - '--soft-fail', - is_flag=True, - default=False, - help='Run the scan without failing; always return a non-error status code.', - type=bool, - required=False, -) -@click.option( - '--severity-threshold', - default=Severity.INFO.name, - help='Show violations only for the specified level or higher.', - type=click.Choice([e.name for e in Severity]), - required=False, -) -@click.option( - '--sca-scan', - default=None, - help='Specify the type of SCA scan you wish to execute (the default is both).', - multiple=True, - type=click.Choice(config['scans']['supported_sca_scans']), -) -@click.option( - '--monitor', - is_flag=True, - default=False, - help='Used for SCA scan types only; when specified, the scan results are recorded in the Discovery module.', - type=bool, - required=False, -) -@click.option( - '--report', - is_flag=True, - default=False, - help='When specified, generates a violations report. A link to the report will be displayed in the console output.', - type=bool, - required=False, -) -@click.option( - f'--{SCA_SKIP_RESTORE_DEPENDENCIES_FLAG}', - is_flag=True, - default=False, - help='When specified, Cycode will not run restore command. Will scan direct dependencies ONLY!', - type=bool, - required=False, -) -@click.option( - '--sync', - is_flag=True, - default=False, - help='Run scan synchronously (the default is asynchronous).', - type=bool, - required=False, -) -@click.option( - f'--{SCA_GRADLE_ALL_SUB_PROJECTS_FLAG}', - is_flag=True, - default=False, - help='When specified, Cycode will run gradle restore command for all sub projects. ' - 'Should run from root project directory ONLY!', - type=bool, - required=False, -) -@click.pass_context -def scan_command( - context: click.Context, - scan_type: str, - secret: str, - client_id: str, - show_secret: bool, - soft_fail: bool, - severity_threshold: str, - sca_scan: List[str], - monitor: bool, - report: bool, - no_restore: bool, - sync: bool, - gradle_all_sub_projects: bool, -) -> int: - """Scans for Secrets, IaC, SCA or SAST violations.""" - add_breadcrumb('scan') - - if show_secret: - context.obj['show_secret'] = show_secret - else: - context.obj['show_secret'] = config['result_printer']['default']['show_secret'] - - if soft_fail: - context.obj['soft_fail'] = soft_fail - else: - context.obj['soft_fail'] = config['soft_fail'] - - context.obj['client'] = get_scan_cycode_client(client_id, secret, not context.obj['show_secret']) - context.obj['scan_type'] = scan_type - context.obj['sync'] = sync - context.obj['severity_threshold'] = severity_threshold - context.obj['monitor'] = monitor - context.obj['report'] = report - context.obj[SCA_SKIP_RESTORE_DEPENDENCIES_FLAG] = no_restore - context.obj[SCA_GRADLE_ALL_SUB_PROJECTS_FLAG] = gradle_all_sub_projects - - _sca_scan_to_context(context, sca_scan) - - return 1 - - -def _sca_scan_to_context(context: click.Context, sca_scan_user_selected: List[str]) -> None: - for sca_scan_option_selected in sca_scan_user_selected: - context.obj[sca_scan_option_selected] = True - - -@scan_command.result_callback() -@click.pass_context -def finalize(context: click.Context, *_, **__) -> None: - add_breadcrumb('scan_finalize') - - progress_bar = context.obj.get('progress_bar') - if progress_bar: - progress_bar.stop() - - if context.obj['soft_fail']: - sys.exit(0) - - exit_code = NO_ISSUES_STATUS_CODE - if scan_utils.is_scan_failed(context): - exit_code = ISSUE_DETECTED_STATUS_CODE - - sys.exit(exit_code) diff --git a/cycode/cli/commands/status/__init__.py b/cycode/cli/commands/status/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cycode/cli/commands/status/status_command.py b/cycode/cli/commands/status/status_command.py deleted file mode 100644 index f5d9aec3..00000000 --- a/cycode/cli/commands/status/status_command.py +++ /dev/null @@ -1,122 +0,0 @@ -import dataclasses -import json -import platform -from typing import Dict - -import click - -from cycode import __version__ -from cycode.cli.commands.auth_common import get_authorization_info -from cycode.cli.consts import PROGRAM_NAME -from cycode.cli.user_settings.configuration_manager import ConfigurationManager -from cycode.cli.utils.get_api_client import get_scan_cycode_client -from cycode.cyclient import logger - - -class CliStatusBase: - def as_dict(self) -> Dict[str, any]: - return dataclasses.asdict(self) - - def _get_text_message_part(self, key: str, value: any, intent: int = 0) -> str: - message_parts = [] - - intent_prefix = ' ' * intent * 2 - human_readable_key = key.replace('_', ' ').capitalize() - - if isinstance(value, dict): - message_parts.append(f'{intent_prefix}{human_readable_key}:') - for sub_key, sub_value in value.items(): - message_parts.append(self._get_text_message_part(sub_key, sub_value, intent=intent + 1)) - elif isinstance(value, (list, set, tuple)): - message_parts.append(f'{intent_prefix}{human_readable_key}:') - for index, sub_value in enumerate(value): - message_parts.append(self._get_text_message_part(f'#{index}', sub_value, intent=intent + 1)) - else: - message_parts.append(f'{intent_prefix}{human_readable_key}: {value}') - - return '\n'.join(message_parts) - - def as_text(self) -> str: - message_parts = [] - for key, value in self.as_dict().items(): - message_parts.append(self._get_text_message_part(key, value)) - - return '\n'.join(message_parts) - - def as_json(self) -> str: - return json.dumps(self.as_dict()) - - -@dataclasses.dataclass -class CliSupportedModulesStatus(CliStatusBase): - secret_scanning: bool = False - sca_scanning: bool = False - iac_scanning: bool = False - sast_scanning: bool = False - ai_large_language_model: bool = False - - -@dataclasses.dataclass -class CliStatus(CliStatusBase): - program: str - version: str - os: str - arch: str - python_version: str - installation_id: str - app_url: str - api_url: str - is_authenticated: bool - user_id: str = None - tenant_id: str = None - supported_modules: CliSupportedModulesStatus = None - - -def get_cli_status() -> CliStatus: - configuration_manager = ConfigurationManager() - - auth_info = get_authorization_info() - is_authenticated = auth_info is not None - - supported_modules_status = CliSupportedModulesStatus() - if is_authenticated: - try: - client = get_scan_cycode_client() - supported_modules_preferences = client.get_supported_modules_preferences() - - supported_modules_status.secret_scanning = supported_modules_preferences.secret_scanning - supported_modules_status.sca_scanning = supported_modules_preferences.sca_scanning - supported_modules_status.iac_scanning = supported_modules_preferences.iac_scanning - supported_modules_status.sast_scanning = supported_modules_preferences.sast_scanning - supported_modules_status.ai_large_language_model = supported_modules_preferences.ai_large_language_model - except Exception as e: - logger.debug('Failed to get supported modules preferences', exc_info=e) - - return CliStatus( - program=PROGRAM_NAME, - version=__version__, - os=platform.system(), - arch=platform.machine(), - python_version=platform.python_version(), - installation_id=configuration_manager.get_or_create_installation_id(), - app_url=configuration_manager.get_cycode_app_url(), - api_url=configuration_manager.get_cycode_api_url(), - is_authenticated=is_authenticated, - user_id=auth_info.user_id if auth_info else None, - tenant_id=auth_info.tenant_id if auth_info else None, - supported_modules=supported_modules_status, - ) - - -@click.command(short_help='Show the CLI status and exit.') -@click.pass_context -def status_command(context: click.Context) -> None: - output = context.obj['output'] - - status = get_cli_status() - message = status.as_text() - if output == 'json': - message = status.as_json() - - click.echo(message, color=context.color) - context.exit() diff --git a/cycode/cli/commands/version/__init__.py b/cycode/cli/commands/version/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cycode/cli/commands/version/version_command.py b/cycode/cli/commands/version/version_command.py deleted file mode 100644 index 107aedbc..00000000 --- a/cycode/cli/commands/version/version_command.py +++ /dev/null @@ -1,22 +0,0 @@ -import json - -import click - -from cycode import __version__ -from cycode.cli.consts import PROGRAM_NAME - - -@click.command(short_help='Show the CLI version and exit. Use `cycode status` instead.', deprecated=True) -@click.pass_context -def version_command(context: click.Context) -> None: - output = context.obj['output'] - - prog = PROGRAM_NAME - ver = __version__ - - message = f'{prog}, version {ver}' - if output == 'json': - message = json.dumps({'name': prog, 'version': ver}) - - click.echo(message, color=context.color) - context.exit() diff --git a/cycode/cli/consts.py b/cycode/cli/consts.py index 558f5b7b..1bca08db 100644 --- a/cycode/cli/consts.py +++ b/cycode/cli/consts.py @@ -1,20 +1,17 @@ PROGRAM_NAME = 'cycode' APP_NAME = 'CycodeCLI' -CLI_CONTEXT_SETTINGS = { - 'terminal_width': 10**9, - 'max_content_width': 10**9, -} +CLI_CONTEXT_SETTINGS = {'terminal_width': 10**9, 'max_content_width': 10**9, 'help_option_names': ['-h', '--help']} PRE_COMMIT_COMMAND_SCAN_TYPE = 'pre_commit' PRE_RECEIVE_COMMAND_SCAN_TYPE = 'pre_receive' COMMIT_HISTORY_COMMAND_SCAN_TYPE = 'commit_history' SECRET_SCAN_TYPE = 'secret' # noqa: S105 -INFRA_CONFIGURATION_SCAN_TYPE = 'iac' +IAC_SCAN_TYPE = 'iac' SCA_SCAN_TYPE = 'sca' SAST_SCAN_TYPE = 'sast' -INFRA_CONFIGURATION_SCAN_SUPPORTED_FILES = ('.tf', '.tf.json', '.json', '.yaml', '.yml', 'dockerfile') +IAC_SCAN_SUPPORTED_FILES = ('.tf', '.tf.json', '.json', '.yaml', '.yml', 'dockerfile') SECRET_SCAN_FILE_EXTENSIONS_TO_IGNORE = ( '.7z', @@ -145,7 +142,11 @@ # scan in batches DEFAULT_SCAN_BATCH_MAX_SIZE_IN_BYTES = 9 * 1024 * 1024 SCAN_BATCH_MAX_SIZE_IN_BYTES = {SAST_SCAN_TYPE: 50 * 1024 * 1024} +SCAN_BATCH_MAX_SIZE_IN_BYTES_ENV_VAR_NAME = 'SCAN_BATCH_MAX_SIZE_IN_BYTES' + DEFAULT_SCAN_BATCH_MAX_FILES_COUNT = 1000 +SCAN_BATCH_MAX_FILES_COUNT_ENV_VAR_NAME = 'SCAN_BATCH_MAX_FILES_COUNT' + # if we increase this values, the server doesn't allow connecting (ConnectionError) SCAN_BATCH_MAX_PARALLEL_SCANS = 5 SCAN_BATCH_SCANS_PER_CPU = 1 diff --git a/cycode/cli/exceptions/handle_ai_remediation_errors.py b/cycode/cli/exceptions/handle_ai_remediation_errors.py index ba46cbf7..961acd62 100644 --- a/cycode/cli/exceptions/handle_ai_remediation_errors.py +++ b/cycode/cli/exceptions/handle_ai_remediation_errors.py @@ -1,14 +1,14 @@ -import click +import typer -from cycode.cli.exceptions.common import handle_errors from cycode.cli.exceptions.custom_exceptions import KNOWN_USER_FRIENDLY_REQUEST_ERRORS, RequestHttpError +from cycode.cli.exceptions.handle_errors import handle_errors from cycode.cli.models import CliError, CliErrors class AiRemediationNotFoundError(Exception): ... -def handle_ai_remediation_exception(context: click.Context, err: Exception) -> None: +def handle_ai_remediation_exception(ctx: typer.Context, err: Exception) -> None: if isinstance(err, RequestHttpError) and err.status_code == 404: err = AiRemediationNotFoundError() @@ -19,4 +19,4 @@ def handle_ai_remediation_exception(context: click.Context, err: Exception) -> N message='The AI remediation was not found. Please try different detection ID', ), } - handle_errors(context, err, errors) + handle_errors(ctx, err, errors) diff --git a/cycode/cli/exceptions/handle_auth_errors.py b/cycode/cli/exceptions/handle_auth_errors.py new file mode 100644 index 00000000..72e18c88 --- /dev/null +++ b/cycode/cli/exceptions/handle_auth_errors.py @@ -0,0 +1,18 @@ +import typer + +from cycode.cli.exceptions.custom_exceptions import ( + KNOWN_USER_FRIENDLY_REQUEST_ERRORS, + AuthProcessError, +) +from cycode.cli.exceptions.handle_errors import handle_errors +from cycode.cli.models import CliError, CliErrors + + +def handle_auth_exception(ctx: typer.Context, err: Exception) -> None: + errors: CliErrors = { + **KNOWN_USER_FRIENDLY_REQUEST_ERRORS, + AuthProcessError: CliError( + code='auth_error', message='Authentication failed. Please try again later using the command `cycode auth`' + ), + } + handle_errors(ctx, err, errors) diff --git a/cycode/cli/exceptions/common.py b/cycode/cli/exceptions/handle_errors.py similarity index 61% rename from cycode/cli/exceptions/common.py rename to cycode/cli/exceptions/handle_errors.py index 51433af7..db102773 100644 --- a/cycode/cli/exceptions/common.py +++ b/cycode/cli/exceptions/handle_errors.py @@ -1,27 +1,28 @@ from typing import Optional import click +import typer from cycode.cli.models import CliError, CliErrors from cycode.cli.printers import ConsolePrinter -from cycode.cli.sentry import capture_exception +from cycode.cli.utils.sentry import capture_exception def handle_errors( - context: click.Context, err: BaseException, cli_errors: CliErrors, *, return_exception: bool = False + ctx: typer.Context, err: BaseException, cli_errors: CliErrors, *, return_exception: bool = False ) -> Optional['CliError']: - ConsolePrinter(context).print_exception(err) + ConsolePrinter(ctx).print_exception(err) if type(err) in cli_errors: error = cli_errors[type(err)] if error.soft_fail is True: - context.obj['soft_fail'] = True + ctx.obj['soft_fail'] = True if return_exception: return error - ConsolePrinter(context).print_error(error) + ConsolePrinter(ctx).print_error(error) return None if isinstance(err, click.ClickException): @@ -33,5 +34,5 @@ def handle_errors( if return_exception: return unknown_error - ConsolePrinter(context).print_error(unknown_error) - exit(1) + ConsolePrinter(ctx).print_error(unknown_error) + raise typer.Exit(1) diff --git a/cycode/cli/exceptions/handle_report_sbom_errors.py b/cycode/cli/exceptions/handle_report_sbom_errors.py index 70cf6277..22707c8c 100644 --- a/cycode/cli/exceptions/handle_report_sbom_errors.py +++ b/cycode/cli/exceptions/handle_report_sbom_errors.py @@ -1,12 +1,12 @@ -import click +import typer from cycode.cli.exceptions import custom_exceptions -from cycode.cli.exceptions.common import handle_errors from cycode.cli.exceptions.custom_exceptions import KNOWN_USER_FRIENDLY_REQUEST_ERRORS +from cycode.cli.exceptions.handle_errors import handle_errors from cycode.cli.models import CliError, CliErrors -def handle_report_exception(context: click.Context, err: Exception) -> None: +def handle_report_exception(ctx: typer.Context, err: Exception) -> None: errors: CliErrors = { **KNOWN_USER_FRIENDLY_REQUEST_ERRORS, custom_exceptions.ScanAsyncError: CliError( @@ -20,4 +20,4 @@ def handle_report_exception(context: click.Context, err: Exception) -> None: 'Please try again by executing the `cycode report` command', ), } - handle_errors(context, err, errors) + handle_errors(ctx, err, errors) diff --git a/cycode/cli/exceptions/handle_scan_errors.py b/cycode/cli/exceptions/handle_scan_errors.py index 550e6879..09890247 100644 --- a/cycode/cli/exceptions/handle_scan_errors.py +++ b/cycode/cli/exceptions/handle_scan_errors.py @@ -1,18 +1,16 @@ from typing import Optional -import click +import typer from cycode.cli.exceptions import custom_exceptions -from cycode.cli.exceptions.common import handle_errors from cycode.cli.exceptions.custom_exceptions import KNOWN_USER_FRIENDLY_REQUEST_ERRORS +from cycode.cli.exceptions.handle_errors import handle_errors from cycode.cli.models import CliError, CliErrors from cycode.cli.utils.git_proxy import git_proxy -def handle_scan_exception( - context: click.Context, err: Exception, *, return_exception: bool = False -) -> Optional[CliError]: - context.obj['did_fail'] = True +def handle_scan_exception(ctx: typer.Context, err: Exception, *, return_exception: bool = False) -> Optional[CliError]: + ctx.obj['did_fail'] = True errors: CliErrors = { **KNOWN_USER_FRIENDLY_REQUEST_ERRORS, @@ -45,4 +43,4 @@ def handle_scan_exception( ), } - return handle_errors(context, err, errors, return_exception=return_exception) + return handle_errors(ctx, err, errors, return_exception=return_exception) diff --git a/cycode/cli/files_collector/excluder.py b/cycode/cli/files_collector/excluder.py index b8cb7920..2d8243cb 100644 --- a/cycode/cli/files_collector/excluder.py +++ b/cycode/cli/files_collector/excluder.py @@ -149,8 +149,8 @@ def _is_relevant_document_to_scan(scan_type: str, filename: str, content: str) - def _is_file_extension_supported(scan_type: str, filename: str) -> bool: filename = filename.lower() - if scan_type == consts.INFRA_CONFIGURATION_SCAN_TYPE: - return filename.endswith(consts.INFRA_CONFIGURATION_SCAN_SUPPORTED_FILES) + if scan_type == consts.IAC_SCAN_TYPE: + return filename.endswith(consts.IAC_SCAN_SUPPORTED_FILES) if scan_type == consts.SCA_SCAN_TYPE: return filename.endswith(consts.SCA_CONFIGURATION_SCAN_SUPPORTED_FILES) diff --git a/cycode/cli/files_collector/iac/tf_content_generator.py b/cycode/cli/files_collector/iac/tf_content_generator.py index 57ebb4b1..8f4cb4d0 100644 --- a/cycode/cli/files_collector/iac/tf_content_generator.py +++ b/cycode/cli/files_collector/iac/tf_content_generator.py @@ -17,7 +17,7 @@ def generate_tfplan_document_name(path: str) -> str: def is_iac(scan_type: str) -> bool: - return scan_type == consts.INFRA_CONFIGURATION_SCAN_TYPE + return scan_type == consts.IAC_SCAN_TYPE def is_tfplan_file(file: str, content: str) -> bool: diff --git a/cycode/cli/files_collector/path_documents.py b/cycode/cli/files_collector/path_documents.py index 14f88888..571773a0 100644 --- a/cycode/cli/files_collector/path_documents.py +++ b/cycode/cli/files_collector/path_documents.py @@ -42,7 +42,7 @@ def _get_relevant_files_in_path(path: str) -> List[str]: def _get_relevant_files( - progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection', scan_type: str, paths: Tuple[str] + progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection', scan_type: str, paths: Tuple[str, ...] ) -> List[str]: all_files_to_scan = [] for path in paths: @@ -89,7 +89,7 @@ def get_relevant_documents( progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection', scan_type: str, - paths: Tuple[str], + paths: Tuple[str, ...], *, is_git_diff: bool = False, ) -> List[Document]: diff --git a/cycode/cli/files_collector/sca/base_restore_dependencies.py b/cycode/cli/files_collector/sca/base_restore_dependencies.py index 81caea1d..b1bf9e80 100644 --- a/cycode/cli/files_collector/sca/base_restore_dependencies.py +++ b/cycode/cli/files_collector/sca/base_restore_dependencies.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import List, Optional -import click +import typer from cycode.cli.models import Document from cycode.cli.utils.path_utils import get_file_content, get_file_dir, get_path_from_context, join_paths @@ -43,9 +43,9 @@ def execute_commands( class BaseRestoreDependencies(ABC): def __init__( - self, context: click.Context, is_git_diff: bool, command_timeout: int, create_output_file_manually: bool = False + self, ctx: typer.Context, is_git_diff: bool, command_timeout: int, create_output_file_manually: bool = False ) -> None: - self.context = context + self.ctx = ctx self.is_git_diff = is_git_diff self.command_timeout = command_timeout self.create_output_file_manually = create_output_file_manually @@ -55,9 +55,7 @@ def restore(self, document: Document) -> Optional[Document]: def get_manifest_file_path(self, document: Document) -> str: return ( - join_paths(get_path_from_context(self.context), document.path) - if self.context.obj.get('monitor') - else document.path + join_paths(get_path_from_context(self.ctx), document.path) if self.ctx.obj.get('monitor') else document.path ) def try_restore_dependencies(self, document: Document) -> Optional[Document]: diff --git a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py index 1986b3a2..77af2a57 100644 --- a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py +++ b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py @@ -2,7 +2,7 @@ import os from typing import List, Optional -import click +import typer from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.models import Document @@ -14,8 +14,8 @@ class RestoreGoDependencies(BaseRestoreDependencies): - def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: - super().__init__(context, is_git_diff, command_timeout, create_output_file_manually=True) + def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(ctx, is_git_diff, command_timeout, create_output_file_manually=True) def try_restore_dependencies(self, document: Document) -> Optional[Document]: manifest_exists = os.path.isfile(self.get_working_directory(document) + os.sep + BUILD_GO_FILE_NAME) diff --git a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py index 85dc9e20..a5c6d48b 100644 --- a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py @@ -2,7 +2,7 @@ import re from typing import List, Optional, Set -import click +import typer from cycode.cli.consts import SCA_GRADLE_ALL_SUB_PROJECTS_FLAG from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies @@ -20,15 +20,15 @@ class RestoreGradleDependencies(BaseRestoreDependencies): def __init__( - self, context: click.Context, is_git_diff: bool, command_timeout: int, projects: Optional[Set[str]] = None + self, ctx: typer.Context, is_git_diff: bool, command_timeout: int, projects: Optional[Set[str]] = None ) -> None: - super().__init__(context, is_git_diff, command_timeout, create_output_file_manually=True) + super().__init__(ctx, is_git_diff, command_timeout, create_output_file_manually=True) if projects is None: projects = set() self.projects = self.get_all_projects() if self.is_gradle_sub_projects() else projects def is_gradle_sub_projects(self) -> bool: - return self.context.obj.get(SCA_GRADLE_ALL_SUB_PROJECTS_FLAG) + return self.ctx.obj.get(SCA_GRADLE_ALL_SUB_PROJECTS_FLAG) def is_project(self, document: Document) -> bool: return document.path.endswith(BUILD_GRADLE_FILE_NAME) or document.path.endswith(BUILD_GRADLE_KTS_FILE_NAME) @@ -47,13 +47,13 @@ def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: return os.path.isfile(restore_file_path) def get_working_directory(self, document: Document) -> Optional[str]: - return get_path_from_context(self.context) if self.is_gradle_sub_projects() else None + return get_path_from_context(self.ctx) if self.is_gradle_sub_projects() else None def get_all_projects(self) -> Set[str]: projects_output = shell( command=BUILD_GRADLE_ALL_PROJECTS_COMMAND, timeout=BUILD_GRADLE_ALL_PROJECTS_TIMEOUT, - working_directory=get_path_from_context(self.context), + working_directory=get_path_from_context(self.ctx), ) projects = re.findall(ALL_PROJECTS_REGEX, projects_output) diff --git a/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py index a44a27e0..d90bbe71 100644 --- a/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py @@ -2,7 +2,7 @@ from os import path from typing import List, Optional -import click +import typer from cycode.cli.files_collector.sca.base_restore_dependencies import ( BaseRestoreDependencies, @@ -18,8 +18,8 @@ class RestoreMavenDependencies(BaseRestoreDependencies): - def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: - super().__init__(context, is_git_diff, command_timeout) + def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(ctx, is_git_diff, command_timeout) def is_project(self, document: Document) -> bool: return path.basename(document.path).split('/')[-1] == BUILD_MAVEN_FILE_NAME diff --git a/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py b/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py index c3026938..68175d88 100644 --- a/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py +++ b/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py @@ -1,7 +1,7 @@ import os from typing import List -import click +import typer from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.models import Document @@ -12,8 +12,8 @@ class RestoreNpmDependencies(BaseRestoreDependencies): - def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: - super().__init__(context, is_git_diff, command_timeout) + def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(ctx, is_git_diff, command_timeout) def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in NPM_PROJECT_FILE_EXTENSIONS) diff --git a/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py b/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py index 0e2ed83d..b4f5a248 100644 --- a/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py +++ b/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py @@ -1,7 +1,7 @@ import os from typing import List -import click +import typer from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.models import Document @@ -11,8 +11,8 @@ class RestoreNugetDependencies(BaseRestoreDependencies): - def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: - super().__init__(context, is_git_diff, command_timeout) + def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(ctx, is_git_diff, command_timeout) def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in NUGET_PROJECT_FILE_EXTENSIONS) diff --git a/cycode/cli/files_collector/sca/sca_code_scanner.py b/cycode/cli/files_collector/sca/sca_code_scanner.py index ca6908b6..3f54eb12 100644 --- a/cycode/cli/files_collector/sca/sca_code_scanner.py +++ b/cycode/cli/files_collector/sca/sca_code_scanner.py @@ -1,7 +1,7 @@ import os from typing import TYPE_CHECKING, Dict, List, Optional -import click +import typer from cycode.cli import consts from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies @@ -89,7 +89,7 @@ def get_project_file_ecosystem(document: Document) -> Optional[str]: def try_restore_dependencies( - context: click.Context, + ctx: typer.Context, documents_to_add: Dict[str, Document], restore_dependencies: 'BaseRestoreDependencies', document: Document, @@ -106,8 +106,8 @@ def try_restore_dependencies( logger.warning('Error occurred while trying to generate dependencies tree, %s', {'filename': document.path}) restore_dependencies_document.content = '' else: - is_monitor_action = context.obj.get('monitor', False) - project_path = get_path_from_context(context) + is_monitor_action = ctx.obj.get('monitor', False) + project_path = get_path_from_context(ctx) manifest_file_path = get_manifest_file_path(document, is_monitor_action, project_path) logger.debug('Succeeded to generate dependencies tree on path: %s', manifest_file_path) @@ -119,27 +119,27 @@ def try_restore_dependencies( def add_dependencies_tree_document( - context: click.Context, documents_to_scan: List[Document], is_git_diff: bool = False + ctx: typer.Context, documents_to_scan: List[Document], is_git_diff: bool = False ) -> None: documents_to_add: Dict[str, Document] = {} - restore_dependencies_list = restore_handlers(context, is_git_diff) + restore_dependencies_list = restore_handlers(ctx, is_git_diff) for restore_dependencies in restore_dependencies_list: for document in documents_to_scan: - try_restore_dependencies(context, documents_to_add, restore_dependencies, document) + try_restore_dependencies(ctx, documents_to_add, restore_dependencies, document) documents_to_scan.extend(list(documents_to_add.values())) -def restore_handlers(context: click.Context, is_git_diff: bool) -> List[BaseRestoreDependencies]: +def restore_handlers(ctx: typer.Context, is_git_diff: bool) -> List[BaseRestoreDependencies]: return [ - RestoreGradleDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), - RestoreMavenDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), - RestoreSbtDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), - RestoreGoDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), - RestoreNugetDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), - RestoreNpmDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), - RestoreRubyDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreGradleDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreMavenDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreSbtDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreGoDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreNugetDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreNpmDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreRubyDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT), ] @@ -155,8 +155,8 @@ def get_file_content_from_commit(repo: 'Repo', commit: str, file_path: str) -> O def perform_pre_scan_documents_actions( - context: click.Context, scan_type: str, documents_to_scan: List[Document], is_git_diff: bool = False + ctx: typer.Context, scan_type: str, documents_to_scan: List[Document], is_git_diff: bool = False ) -> None: - if scan_type == consts.SCA_SCAN_TYPE and not context.obj.get(consts.SCA_SKIP_RESTORE_DEPENDENCIES_FLAG): + if scan_type == consts.SCA_SCAN_TYPE and not ctx.obj.get(consts.SCA_SKIP_RESTORE_DEPENDENCIES_FLAG): logger.debug('Perform pre-scan document add_dependencies_tree_document action') - add_dependencies_tree_document(context, documents_to_scan, is_git_diff) + add_dependencies_tree_document(ctx, documents_to_scan, is_git_diff) diff --git a/cycode/cli/main.py b/cycode/cli/main.py index dd2d1fa7..c6a857a4 100644 --- a/cycode/cli/main.py +++ b/cycode/cli/main.py @@ -1,11 +1,10 @@ from multiprocessing import freeze_support -from cycode.cli.commands.main_cli import main_cli +from cycode.cli.app import app -if __name__ == '__main__': - # DO NOT REMOVE OR MOVE THIS LINE - # this is required to support multiprocessing in executables files packaged with PyInstaller - # see https://pyinstaller.org/en/latest/common-issues-and-pitfalls.html#multi-processing - freeze_support() +# DO NOT REMOVE OR MOVE THIS LINE +# this is required to support multiprocessing in executables files packaged with PyInstaller +# see https://pyinstaller.org/en/latest/common-issues-and-pitfalls.html#multi-processing +freeze_support() - main_cli() +app() diff --git a/cycode/cli/models.py b/cycode/cli/models.py index 25b2347f..df62583a 100644 --- a/cycode/cli/models.py +++ b/cycode/cli/models.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from enum import Enum from typing import Dict, List, NamedTuple, Optional, Type from cycode.cyclient.models import Detection @@ -33,34 +32,6 @@ def __repr__(self) -> str: return 'document:{0}, detections:{1}'.format(self.document, self.detections) -SEVERITY_UNKNOWN_WEIGHT = -2 - - -class Severity(Enum): - INFO = -1 - LOW = 0 - MEDIUM = 1 - MODERATE = 1 # noqa: PIE796. TODO(MarshalX): rework. should not be Enum - HIGH = 2 - CRITICAL = 3 - - @staticmethod - def try_get_value(name: str) -> Optional[int]: - name = name.upper() - if name not in Severity.__members__: - return None - - return Severity[name].value - - @staticmethod - def get_member_weight(name: str) -> int: - weight = Severity.try_get_value(name) - if weight is None: # unknown severity - return SEVERITY_UNKNOWN_WEIGHT - - return weight - - class CliError(NamedTuple): code: str message: str diff --git a/cycode/cli/printers/console_printer.py b/cycode/cli/printers/console_printer.py index 1f70836c..64efa9d5 100644 --- a/cycode/cli/printers/console_printer.py +++ b/cycode/cli/printers/console_printer.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING, ClassVar, Dict, List, Optional, Type -import click +import typer from cycode.cli.exceptions.custom_exceptions import CycodeError from cycode.cli.models import CliError, CliResult @@ -24,11 +24,13 @@ class ConsolePrinter: 'text_sca': ScaTablePrinter, } - def __init__(self, context: click.Context) -> None: - self.context = context - self.scan_type = self.context.obj.get('scan_type') - self.output_type = self.context.obj.get('output') - self.aggregation_report_url = self.context.obj.get('aggregation_report_url') + def __init__(self, ctx: typer.Context) -> None: + self.ctx = ctx + + self.scan_type = self.ctx.obj.get('scan_type') + self.output_type = self.ctx.obj.get('output') + self.aggregation_report_url = self.ctx.obj.get('aggregation_report_url') + self._printer_class = self._AVAILABLE_PRINTERS.get(self.output_type) if self._printer_class is None: raise CycodeError(f'"{self.output_type}" output type is not supported.') @@ -48,18 +50,18 @@ def _get_scan_printer(self) -> 'PrinterBase': if composite_printer: printer_class = composite_printer - return printer_class(self.context) + return printer_class(self.ctx) def print_result(self, result: CliResult) -> None: - self._printer_class(self.context).print_result(result) + self._printer_class(self.ctx).print_result(result) def print_error(self, error: CliError) -> None: - self._printer_class(self.context).print_error(error) + self._printer_class(self.ctx).print_error(error) def print_exception(self, e: Optional[BaseException] = None, force_print: bool = False) -> None: """Print traceback message in stderr if verbose mode is set.""" - if force_print or self.context.obj.get('verbose', False): - self._printer_class(self.context).print_exception(e) + if force_print or self.ctx.obj.get('verbose', False): + self._printer_class(self.ctx).print_exception(e) @property def is_json_printer(self) -> bool: diff --git a/cycode/cli/printers/json_printer.py b/cycode/cli/printers/json_printer.py index b682b8c7..c8fbacb3 100644 --- a/cycode/cli/printers/json_printer.py +++ b/cycode/cli/printers/json_printer.py @@ -28,7 +28,7 @@ def print_scan_results( scan_ids = [] report_urls = [] detections = [] - aggregation_report_url = self.context.obj.get('aggregation_report_url') + aggregation_report_url = self.ctx.obj.get('aggregation_report_url') if aggregation_report_url: report_urls.append(aggregation_report_url) diff --git a/cycode/cli/printers/printer_base.py b/cycode/cli/printers/printer_base.py index fa5bf435..45419aec 100644 --- a/cycode/cli/printers/printer_base.py +++ b/cycode/cli/printers/printer_base.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Dict, List, Optional import click +import typer from cycode.cli.models import CliError, CliResult from cycode.cyclient.headers import get_correlation_id @@ -16,8 +17,8 @@ class PrinterBase(ABC): WHITE_COLOR_NAME = 'white' GREEN_COLOR_NAME = 'green' - def __init__(self, context: click.Context) -> None: - self.context = context + def __init__(self, ctx: typer.Context) -> None: + self.ctx = ctx @abstractmethod def print_scan_results( diff --git a/cycode/cli/printers/tables/sca_table_printer.py b/cycode/cli/printers/tables/sca_table_printer.py index 5a6ec726..063e80e8 100644 --- a/cycode/cli/printers/tables/sca_table_printer.py +++ b/cycode/cli/printers/tables/sca_table_printer.py @@ -3,8 +3,9 @@ import click +from cycode.cli.cli_types import SeverityOption from cycode.cli.consts import LICENSE_COMPLIANCE_POLICY_ID, PACKAGE_VULNERABILITY_POLICY_ID -from cycode.cli.models import SEVERITY_UNKNOWN_WEIGHT, Detection, Severity +from cycode.cli.models import Detection from cycode.cli.printers.tables.table import Table from cycode.cli.printers.tables.table_models import ColumnInfoBuilder, ColumnWidths from cycode.cli.printers.tables.table_printer_base import TablePrinterBase @@ -40,7 +41,7 @@ class ScaTablePrinter(TablePrinterBase): def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: - aggregation_report_url = self.context.obj.get('aggregation_report_url') + aggregation_report_url = self.ctx.obj.get('aggregation_report_url') detections_per_policy_id = self._extract_detections_per_policy_id(local_scan_results) for policy_id, detections in detections_per_policy_id.items(): table = self._get_table(policy_id) @@ -72,11 +73,8 @@ def __group_by(detections: List[Detection], details_field_name: str) -> Dict[str @staticmethod def __severity_sort_key(detection: Detection) -> int: - severity = detection.detection_details.get('advisory_severity') - if severity: - return Severity.get_member_weight(severity) - - return SEVERITY_UNKNOWN_WEIGHT + severity = detection.detection_details.get('advisory_severity', 'unknown') + return SeverityOption.get_member_weight(severity) def _sort_detections_by_severity(self, detections: List[Detection]) -> List[Detection]: return sorted(detections, key=self.__severity_sort_key, reverse=True) diff --git a/cycode/cli/printers/tables/table_printer.py b/cycode/cli/printers/tables/table_printer.py index f2153e56..61234066 100644 --- a/cycode/cli/printers/tables/table_printer.py +++ b/cycode/cli/printers/tables/table_printer.py @@ -2,7 +2,7 @@ import click -from cycode.cli.consts import INFRA_CONFIGURATION_SCAN_TYPE, SAST_SCAN_TYPE, SECRET_SCAN_TYPE +from cycode.cli.consts import IAC_SCAN_TYPE, SAST_SCAN_TYPE, SECRET_SCAN_TYPE from cycode.cli.models import Detection, Document from cycode.cli.printers.tables.table import Table from cycode.cli.printers.tables.table_models import ColumnInfoBuilder, ColumnWidthsConfig @@ -35,7 +35,7 @@ VIOLATION_COLUMN: 2, SCAN_ID_COLUMN: 2, }, - INFRA_CONFIGURATION_SCAN_TYPE: { + IAC_SCAN_TYPE: { ISSUE_TYPE_COLUMN: 4, RULE_ID_COLUMN: 3, FILE_PATH_COLUMN: 3, @@ -63,7 +63,7 @@ def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: self._enrich_table_with_values(table, detection, document_detections.document) self._print_table(table) - self._print_report_urls(local_scan_results, self.context.obj.get('aggregation_report_url')) + self._print_report_urls(local_scan_results, self.ctx.obj.get('aggregation_report_url')) def _get_table(self) -> Table: table = Table() diff --git a/cycode/cli/printers/tables/table_printer_base.py b/cycode/cli/printers/tables/table_printer_base.py index be41454f..abbc8251 100644 --- a/cycode/cli/printers/tables/table_printer_base.py +++ b/cycode/cli/printers/tables/table_printer_base.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING, Dict, List, Optional import click +import typer from cycode.cli.models import CliError, CliResult from cycode.cli.printers.printer_base import PrinterBase @@ -13,16 +14,16 @@ class TablePrinterBase(PrinterBase, abc.ABC): - def __init__(self, context: click.Context) -> None: - super().__init__(context) - self.scan_type: str = context.obj.get('scan_type') - self.show_secret: bool = context.obj.get('show_secret', False) + def __init__(self, ctx: typer.Context) -> None: + super().__init__(ctx) + self.scan_type: str = ctx.obj.get('scan_type') + self.show_secret: bool = ctx.obj.get('show_secret', False) def print_result(self, result: CliResult) -> None: - TextPrinter(self.context).print_result(result) + TextPrinter(self.ctx).print_result(result) def print_error(self, error: CliError) -> None: - TextPrinter(self.context).print_error(error) + TextPrinter(self.ctx).print_error(error) def print_scan_results( self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None @@ -46,7 +47,7 @@ def print_scan_results( self.print_error(error) def _is_git_repository(self) -> bool: - return self.context.obj.get('remote_url') is not None + return self.ctx.obj.get('remote_url') is not None @abc.abstractmethod def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: diff --git a/cycode/cli/printers/text_printer.py b/cycode/cli/printers/text_printer.py index 0b503207..0f617f8c 100644 --- a/cycode/cli/printers/text_printer.py +++ b/cycode/cli/printers/text_printer.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING, Dict, List, Optional import click +import typer from cycode.cli.config import config from cycode.cli.consts import COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES, SECRET_SCAN_TYPE @@ -14,11 +15,11 @@ class TextPrinter(PrinterBase): - def __init__(self, context: click.Context) -> None: - super().__init__(context) - self.scan_type: str = context.obj.get('scan_type') - self.command_scan_type: str = context.info_name - self.show_secret: bool = context.obj.get('show_secret', False) + def __init__(self, ctx: typer.Context) -> None: + super().__init__(ctx) + self.scan_type: str = ctx.obj.get('scan_type') + self.command_scan_type: str = ctx.info_name + self.show_secret: bool = ctx.obj.get('show_secret', False) def print_result(self, result: CliResult) -> None: color = None @@ -50,7 +51,7 @@ def print_scan_results( report_urls = [scan_result.report_url for scan_result in local_scan_results if scan_result.report_url] - self._print_report_urls(report_urls, self.context.obj.get('aggregation_report_url')) + self._print_report_urls(report_urls, self.ctx.obj.get('aggregation_report_url')) if not errors: return diff --git a/cycode/cli/user_settings/credentials_manager.py b/cycode/cli/user_settings/credentials_manager.py index ad380e8a..86a84ba6 100644 --- a/cycode/cli/user_settings/credentials_manager.py +++ b/cycode/cli/user_settings/credentials_manager.py @@ -3,9 +3,9 @@ from typing import Optional, Tuple from cycode.cli.config import CYCODE_CLIENT_ID_ENV_VAR_NAME, CYCODE_CLIENT_SECRET_ENV_VAR_NAME -from cycode.cli.sentry import setup_scope_from_access_token from cycode.cli.user_settings.base_file_manager import BaseFileManager from cycode.cli.user_settings.jwt_creator import JwtCreator +from cycode.cli.utils.sentry import setup_scope_from_access_token class CredentialsManager(BaseFileManager): diff --git a/cycode/cli/utils/path_utils.py b/cycode/cli/utils/path_utils.py index a2d8816b..c6d2001d 100644 --- a/cycode/cli/utils/path_utils.py +++ b/cycode/cli/utils/path_utils.py @@ -3,7 +3,7 @@ from functools import lru_cache from typing import TYPE_CHECKING, AnyStr, List, Optional, Union -import click +import typer from binaryornot.helpers import is_binary_string from cycode.cyclient import logger @@ -106,8 +106,8 @@ def concat_unique_id(filename: str, unique_id: str) -> str: return os.path.join(unique_id, filename) -def get_path_from_context(context: click.Context) -> Optional[str]: - path = context.params.get('path') - if path is None and 'paths' in context.params: - path = context.params['paths'][0] +def get_path_from_context(ctx: typer.Context) -> Optional[str]: + path = ctx.params.get('path') + if path is None and 'paths' in ctx.params: + path = ctx.params['paths'][0] return path diff --git a/cycode/cli/utils/scan_batch.py b/cycode/cli/utils/scan_batch.py index 3d2d83dc..4019b7b0 100644 --- a/cycode/cli/utils/scan_batch.py +++ b/cycode/cli/utils/scan_batch.py @@ -5,17 +5,53 @@ from cycode.cli import consts from cycode.cli.models import Document from cycode.cli.utils.progress_bar import ScanProgressBarSection +from cycode.cyclient import logger if TYPE_CHECKING: from cycode.cli.models import CliError, LocalScanResult from cycode.cli.utils.progress_bar import BaseProgressBar +def _get_max_batch_size(scan_type: str) -> int: + logger.debug( + 'You can customize the batch size by setting the environment variable "%s"', + consts.SCAN_BATCH_MAX_SIZE_IN_BYTES_ENV_VAR_NAME, + ) + + custom_size = os.environ.get(consts.SCAN_BATCH_MAX_SIZE_IN_BYTES_ENV_VAR_NAME) + if custom_size: + logger.debug('Custom batch size is set, %s', {'custom_size': custom_size}) + return int(custom_size) + + return consts.SCAN_BATCH_MAX_SIZE_IN_BYTES.get(scan_type, consts.DEFAULT_SCAN_BATCH_MAX_SIZE_IN_BYTES) + + +def _get_max_batch_files_count(_: str) -> int: + logger.debug( + 'You can customize the batch files count by setting the environment variable "%s"', + consts.SCAN_BATCH_MAX_FILES_COUNT_ENV_VAR_NAME, + ) + + custom_files_count = os.environ.get(consts.SCAN_BATCH_MAX_FILES_COUNT_ENV_VAR_NAME) + if custom_files_count: + logger.debug('Custom batch files count is set, %s', {'custom_files_count': custom_files_count}) + return int(custom_files_count) + + return consts.DEFAULT_SCAN_BATCH_MAX_FILES_COUNT + + def split_documents_into_batches( + scan_type: str, documents: List[Document], - max_size: int = consts.DEFAULT_SCAN_BATCH_MAX_SIZE_IN_BYTES, - max_files_count: int = consts.DEFAULT_SCAN_BATCH_MAX_FILES_COUNT, ) -> List[List[Document]]: + max_size = _get_max_batch_size(scan_type) + max_files_count = _get_max_batch_files_count(scan_type) + + logger.debug( + 'Splitting documents into batches, %s', + {'document_count': len(documents), 'max_batch_size': max_size, 'max_files_count': max_files_count}, + ) + batches = [] current_size = 0 @@ -23,7 +59,29 @@ def split_documents_into_batches( for document in documents: document_size = len(document.content.encode('UTF-8')) - if (current_size + document_size > max_size) or (len(current_batch) >= max_files_count): + exceeds_max_size = current_size + document_size > max_size + if exceeds_max_size: + logger.debug( + 'Going to create new batch because current batch size exceeds the limit, %s', + { + 'batch_index': len(batches), + 'current_batch_size': current_size + document_size, + 'max_batch_size': max_size, + }, + ) + + exceeds_max_files_count = len(current_batch) >= max_files_count + if exceeds_max_files_count: + logger.debug( + 'Going to create new batch because current batch files count exceeds the limit, %s', + { + 'batch_index': len(batches), + 'current_batch_files_count': len(current_batch), + 'max_batch_files_count': max_files_count, + }, + ) + + if exceeds_max_size or exceeds_max_files_count: batches.append(current_batch) current_batch = [document] @@ -35,6 +93,8 @@ def split_documents_into_batches( if current_batch: batches.append(current_batch) + logger.debug('Documents were split into batches %s', {'batches_count': len(batches)}) + return batches @@ -49,9 +109,8 @@ def run_parallel_batched_scan( documents: List[Document], progress_bar: 'BaseProgressBar', ) -> Tuple[Dict[str, 'CliError'], List['LocalScanResult']]: - max_size = consts.SCAN_BATCH_MAX_SIZE_IN_BYTES.get(scan_type, consts.DEFAULT_SCAN_BATCH_MAX_SIZE_IN_BYTES) - - batches = [documents] if scan_type == consts.SCA_SCAN_TYPE else split_documents_into_batches(documents, max_size) + # batching is disabled for SCA; requested by Mor + batches = [documents] if scan_type == consts.SCA_SCAN_TYPE else split_documents_into_batches(scan_type, documents) progress_bar.set_section_length(ScanProgressBarSection.SCAN, len(batches)) # * 3 # TODO(MarshalX): we should multiply the count of batches in SCAN section because each batch has 3 steps: @@ -61,9 +120,13 @@ def run_parallel_batched_scan( # it's not possible yet because not all scan types moved to polling mechanism # the progress bar could be significant improved (be more dynamic) in the future + threads_count = _get_threads_count() local_scan_results: List['LocalScanResult'] = [] cli_errors: Dict[str, 'CliError'] = {} - with ThreadPool(processes=_get_threads_count()) as pool: + + logger.debug('Running parallel batched scan, %s', {'threads_count': threads_count, 'batches_count': len(batches)}) + + with ThreadPool(processes=threads_count) as pool: for scan_id, err, result in pool.imap(scan_function, batches): if result: local_scan_results.append(result) diff --git a/cycode/cli/utils/scan_utils.py b/cycode/cli/utils/scan_utils.py index 77866c4b..8c9dcca7 100644 --- a/cycode/cli/utils/scan_utils.py +++ b/cycode/cli/utils/scan_utils.py @@ -1,11 +1,11 @@ -import click +import typer -def set_issue_detected(context: click.Context, issue_detected: bool) -> None: - context.obj['issue_detected'] = issue_detected +def set_issue_detected(ctx: typer.Context, issue_detected: bool) -> None: + ctx.obj['issue_detected'] = issue_detected -def is_scan_failed(context: click.Context) -> bool: - did_fail = context.obj.get('did_fail') - issue_detected = context.obj.get('issue_detected') +def is_scan_failed(ctx: typer.Context) -> bool: + did_fail = ctx.obj.get('did_fail') + issue_detected = ctx.obj.get('issue_detected') return did_fail or issue_detected diff --git a/cycode/cli/sentry.py b/cycode/cli/utils/sentry.py similarity index 100% rename from cycode/cli/sentry.py rename to cycode/cli/utils/sentry.py diff --git a/cycode/cli/commands/version/version_checker.py b/cycode/cli/utils/version_checker.py similarity index 95% rename from cycode/cli/commands/version/version_checker.py rename to cycode/cli/utils/version_checker.py index c5ec9d4f..40022cbd 100644 --- a/cycode/cli/commands/version/version_checker.py +++ b/cycode/cli/utils/version_checker.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import List, Optional, Tuple -import click +import typer from cycode.cli.user_settings.configuration_manager import ConfigurationManager from cycode.cli.utils.path_utils import get_file_content @@ -56,6 +56,7 @@ def _compare_versions( class VersionChecker(CycodeClientBase): PYPI_API_URL = 'https://pypi.org/pypi' PYPI_PACKAGE_NAME = 'cycode' + PYPI_REQUEST_TIMEOUT = 1 GIT_CHANGELOG_URL_PREFIX = 'https://github.com/cycodehq/cycode-cli/releases/tag/v' @@ -84,7 +85,7 @@ def get_latest_version(self) -> Optional[str]: or the version information is not available. """ try: - response = self.get(f'{self.PYPI_PACKAGE_NAME}/json') + response = self.get(f'{self.PYPI_PACKAGE_NAME}/json', timeout=self.PYPI_REQUEST_TIMEOUT) data = response.json() return data.get('info', {}).get('version') except Exception: @@ -199,11 +200,11 @@ def check_and_notify_update(self, current_version: str, use_color: bool = True, if should_update: update_message = ( '\nNew version of cycode available! ' - f"{click.style(current_version, fg='yellow')} → {click.style(latest_version, fg='bright_blue')}\n" - f"Changelog: {click.style(f'{self.GIT_CHANGELOG_URL_PREFIX}{latest_version}', fg='bright_blue')}\n" - f"Run {click.style('pip install --upgrade cycode', fg='green')} to update\n" + f"{typer.style(current_version, fg='yellow')} → {typer.style(latest_version, fg='bright_blue')}\n" + f"Changelog: {typer.style(f'{self.GIT_CHANGELOG_URL_PREFIX}{latest_version}', fg='bright_blue')}\n" + f"Run {typer.style('pip install --upgrade cycode', fg='green')} to update\n" ) - click.echo(update_message, color=use_color) + typer.echo(update_message, color=use_color) version_checker = VersionChecker() diff --git a/cycode/cyclient/headers.py b/cycode/cyclient/headers.py index c6983d32..4e2434e9 100644 --- a/cycode/cyclient/headers.py +++ b/cycode/cyclient/headers.py @@ -4,8 +4,8 @@ from cycode import __version__ from cycode.cli import consts -from cycode.cli.sentry import add_correlation_id_to_scope from cycode.cli.user_settings.configuration_manager import ConfigurationManager +from cycode.cli.utils.sentry import add_correlation_id_to_scope from cycode.cyclient import logger diff --git a/cycode/cyclient/scan_client.py b/cycode/cyclient/scan_client.py index 31abba17..c6bfc57c 100644 --- a/cycode/cyclient/scan_client.py +++ b/cycode/cyclient/scan_client.py @@ -1,5 +1,6 @@ import json from typing import TYPE_CHECKING, List, Optional, Set, Union +from uuid import UUID from requests import Response @@ -32,7 +33,7 @@ def __init__( self._hide_response_log = hide_response_log def get_scan_controller_path(self, scan_type: str, should_use_scan_service: bool = False) -> str: - if not should_use_scan_service and scan_type == consts.INFRA_CONFIGURATION_SCAN_TYPE: + if not should_use_scan_service and scan_type == consts.IAC_SCAN_TYPE: # we don't use async flow for IaC scan yet return self._SCAN_SERVICE_CONTROLLER_PATH if not should_use_scan_service and scan_type == consts.SECRET_SCAN_TYPE: @@ -43,7 +44,7 @@ def get_scan_controller_path(self, scan_type: str, should_use_scan_service: bool return self._SCAN_SERVICE_CLI_CONTROLLER_PATH def get_detections_service_controller_path(self, scan_type: str) -> str: - if scan_type == consts.INFRA_CONFIGURATION_SCAN_TYPE: + if scan_type == consts.IAC_SCAN_TYPE: # we don't use async flow for IaC scan yet return self._DETECTIONS_SERVICE_CONTROLLER_PATH @@ -210,8 +211,8 @@ def get_supported_modules_preferences(self) -> models.SupportedModulesPreference def get_ai_remediation_path(detection_id: str) -> str: return f'scm-remediator/api/v1/ContentRemediation/preview/{detection_id}' - def get_ai_remediation(self, detection_id: str, *, fix: bool = False) -> str: - path = self.get_ai_remediation_path(detection_id) + def get_ai_remediation(self, detection_id: UUID, *, fix: bool = False) -> str: + path = self.get_ai_remediation_path(detection_id.hex) data = { 'resolving_parameters': { @@ -231,7 +232,7 @@ def get_ai_remediation(self, detection_id: str, *, fix: bool = False) -> str: @staticmethod def _get_policy_type_by_scan_type(scan_type: str) -> str: scan_type_to_policy_type = { - consts.INFRA_CONFIGURATION_SCAN_TYPE: 'IaC', + consts.IAC_SCAN_TYPE: 'IaC', consts.SCA_SCAN_TYPE: 'SCA', consts.SECRET_SCAN_TYPE: 'SecretDetection', consts.SAST_SCAN_TYPE: 'SAST', @@ -261,7 +262,7 @@ def get_scan_detections_path(self, scan_type: str) -> str: @staticmethod def get_scan_detections_list_path_suffix(scan_type: str) -> str: # we don't use async flow for IaC scan yet - if scan_type == consts.INFRA_CONFIGURATION_SCAN_TYPE: + if scan_type == consts.IAC_SCAN_TYPE: return '' return '/detections' @@ -330,7 +331,7 @@ def get_service_name(scan_type: str) -> Optional[str]: # TODO(MarshalX): get_service_name should be removed from ScanClient? Because it exists in ScanConfig if scan_type == consts.SECRET_SCAN_TYPE: return 'secret' - if scan_type == consts.INFRA_CONFIGURATION_SCAN_TYPE: + if scan_type == consts.IAC_SCAN_TYPE: return 'iac' if scan_type == consts.SCA_SCAN_TYPE or scan_type == consts.SAST_SCAN_TYPE: return 'scans' diff --git a/cycode/cyclient/scan_config_base.py b/cycode/cyclient/scan_config_base.py index 1ff1da6c..6dfa97ef 100644 --- a/cycode/cyclient/scan_config_base.py +++ b/cycode/cyclient/scan_config_base.py @@ -11,7 +11,7 @@ def get_service_name(self, scan_type: str, should_use_scan_service: bool = False def get_async_scan_type(scan_type: str) -> str: if scan_type == consts.SECRET_SCAN_TYPE: return 'Secrets' - if scan_type == consts.INFRA_CONFIGURATION_SCAN_TYPE: + if scan_type == consts.IAC_SCAN_TYPE: return 'InfraConfiguration' return scan_type.upper() @@ -33,7 +33,7 @@ def get_service_name(self, scan_type: str, should_use_scan_service: bool = False return '5004' if scan_type == consts.SECRET_SCAN_TYPE: return '5025' - if scan_type == consts.INFRA_CONFIGURATION_SCAN_TYPE: + if scan_type == consts.IAC_SCAN_TYPE: return '5026' # sca and sast @@ -49,7 +49,7 @@ def get_service_name(self, scan_type: str, should_use_scan_service: bool = False return 'scans' if scan_type == consts.SECRET_SCAN_TYPE: return 'secret' - if scan_type == consts.INFRA_CONFIGURATION_SCAN_TYPE: + if scan_type == consts.IAC_SCAN_TYPE: return 'iac' # sca and sast diff --git a/poetry.lock b/poetry.lock index c97b44a9..f104cc28 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.0 and should not be changed by hand. [[package]] name = "altgraph" @@ -6,6 +6,8 @@ version = "0.17.4" description = "Python graph (network) package" optional = false python-versions = "*" +groups = ["executable"] +markers = "python_version < \"3.13\"" files = [ {file = "altgraph-0.17.4-py2.py3-none-any.whl", hash = "sha256:642743b4750de17e655e6711601b077bc6598dbfa3ba5fa2b2a35ce12b508dff"}, {file = "altgraph-0.17.4.tar.gz", hash = "sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406"}, @@ -17,6 +19,7 @@ version = "1.3.0" description = "Better dates & times for Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "arrow-1.3.0-py3-none-any.whl", hash = "sha256:c728b120ebc00eb84e01882a6f5e7927a53960aa990ce7dd2b10f39005a67f80"}, {file = "arrow-1.3.0.tar.gz", hash = "sha256:d4540617648cb5f895730f1ad8c82a65f2dad0166f57b75f3ca54759c4d67a85"}, @@ -36,6 +39,7 @@ version = "0.4.4" description = "Ultra-lightweight pure Python package to check if a file is binary or text." optional = false python-versions = "*" +groups = ["main"] files = [ {file = "binaryornot-0.4.4-py2.py3-none-any.whl", hash = "sha256:b8b71173c917bddcd2c16070412e369c3ed7f0528926f70cac18a6c97fd563e4"}, {file = "binaryornot-0.4.4.tar.gz", hash = "sha256:359501dfc9d40632edc9fac890e19542db1a287bbcfa58175b66658392018061"}, @@ -50,6 +54,7 @@ version = "2024.8.30" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" +groups = ["main", "test"] files = [ {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, @@ -61,6 +66,7 @@ version = "5.2.0" description = "Universal encoding detector for Python 3" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970"}, {file = "chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7"}, @@ -72,6 +78,7 @@ version = "3.4.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" +groups = ["main", "test"] files = [ {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6"}, {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b"}, @@ -186,6 +193,7 @@ version = "8.1.7" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, @@ -200,6 +208,7 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main", "test"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -211,6 +220,7 @@ version = "7.2.7" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.7" +groups = ["test"] files = [ {file = "coverage-7.2.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8"}, {file = "coverage-7.2.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb"}, @@ -275,7 +285,7 @@ files = [ ] [package.extras] -toml = ["tomli"] +toml = ["tomli ; python_full_version <= \"3.11.0a6\""] [[package]] name = "dunamai" @@ -283,6 +293,7 @@ version = "1.21.2" description = "Dynamic version generation" optional = false python-versions = ">=3.5" +groups = ["executable"] files = [ {file = "dunamai-1.21.2-py3-none-any.whl", hash = "sha256:87db76405bf9366f9b4925ff5bb1db191a9a1bd9f9693f81c4d3abb8298be6f0"}, {file = "dunamai-1.21.2.tar.gz", hash = "sha256:05827fb5f032f5596bfc944b23f613c147e676de118681f3bb1559533d8a65c4"}, @@ -297,6 +308,8 @@ version = "1.2.2" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" +groups = ["test"] +markers = "python_version < \"3.11\"" files = [ {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, @@ -311,6 +324,7 @@ version = "4.0.11" description = "Git Object Database" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4"}, {file = "gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b"}, @@ -325,6 +339,7 @@ version = "3.1.43" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff"}, {file = "GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c"}, @@ -335,7 +350,7 @@ gitdb = ">=4.0.1,<5" [package.extras] doc = ["sphinx (==4.3.2)", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "sphinxcontrib-applehelp (>=1.0.2,<=1.0.4)", "sphinxcontrib-devhelp (==1.0.2)", "sphinxcontrib-htmlhelp (>=2.0.0,<=2.0.1)", "sphinxcontrib-qthelp (==1.0.3)", "sphinxcontrib-serializinghtml (==1.1.5)"] -test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] +test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] [[package]] name = "idna" @@ -343,6 +358,7 @@ version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" +groups = ["main", "test"] files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, @@ -357,6 +373,8 @@ version = "8.5.0" description = "Read metadata from Python packages" optional = false python-versions = ">=3.8" +groups = ["executable"] +markers = "python_version < \"3.10\"" files = [ {file = "importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b"}, {file = "importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7"}, @@ -366,12 +384,12 @@ files = [ zipp = ">=3.20" [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] enabler = ["pytest-enabler (>=2.2)"] perf = ["ipython"] -test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] +test = ["flufl.flake8", "importlib-resources (>=1.3) ; python_version < \"3.9\"", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] type = ["pytest-mypy"] [[package]] @@ -380,6 +398,7 @@ version = "2.0.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.7" +groups = ["test"] files = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, @@ -391,6 +410,8 @@ version = "1.16.3" description = "Mach-O header analysis and editing" optional = false python-versions = "*" +groups = ["executable"] +markers = "python_version < \"3.13\" and sys_platform == \"darwin\"" files = [ {file = "macholib-1.16.3-py2.py3-none-any.whl", hash = "sha256:0e315d7583d38b8c77e815b1ecbdbf504a8258d8b3e17b61165c6feb60d18f2c"}, {file = "macholib-1.16.3.tar.gz", hash = "sha256:07ae9e15e8e4cd9a788013d81f5908b3609aa76f9b1421bae9c4d7606ec86a30"}, @@ -405,6 +426,7 @@ version = "3.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, @@ -429,6 +451,7 @@ version = "3.22.0" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "marshmallow-3.22.0-py3-none-any.whl", hash = "sha256:71a2dce49ef901c3f97ed296ae5051135fd3febd2bf43afe0ae9a82143a494d9"}, {file = "marshmallow-3.22.0.tar.gz", hash = "sha256:4972f529104a220bb8637d595aa4c9762afbe7f7a77d82dc58c1615d70c5823e"}, @@ -448,6 +471,7 @@ version = "0.1.2" description = "Markdown URL utilities" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -459,6 +483,7 @@ version = "4.0.3" description = "Rolling backport of unittest.mock for all Pythons" optional = false python-versions = ">=3.6" +groups = ["test"] files = [ {file = "mock-4.0.3-py3-none-any.whl", hash = "sha256:122fcb64ee37cfad5b3f48d7a7d51875d7031aaf3d8be7c42e2bee25044eee62"}, {file = "mock-4.0.3.tar.gz", hash = "sha256:7d3fbbde18228f4ff2f1f119a45cdffa458b4c0dee32eb4d2bb2f82554bac7bc"}, @@ -475,6 +500,7 @@ version = "24.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" +groups = ["main", "executable", "test"] files = [ {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, @@ -486,6 +512,7 @@ version = "1.18.1" description = "Library to parse and apply unified diffs." optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "patch-ng-1.18.1.tar.gz", hash = "sha256:52fd46ee46f6c8667692682c1fd7134edc65a2d2d084ebec1d295a6087fc0291"}, ] @@ -496,6 +523,8 @@ version = "2024.8.26" description = "Python PE parsing module" optional = false python-versions = ">=3.6.0" +groups = ["executable"] +markers = "python_version < \"3.13\" and sys_platform == \"win32\"" files = [ {file = "pefile-2024.8.26-py3-none-any.whl", hash = "sha256:76f8b485dcd3b1bb8166f1128d395fa3d87af26360c2358fb75b80019b957c6f"}, {file = "pefile-2024.8.26.tar.gz", hash = "sha256:3ff6c5d8b43e8c37bb6e6dd5085658d658a7a0bdcd20b6a07b1fcfc1c4e9d632"}, @@ -507,6 +536,7 @@ version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" +groups = ["test"] files = [ {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, @@ -522,6 +552,7 @@ version = "5.7.2" description = "pyfakefs implements a fake file system that mocks the Python file system modules." optional = false python-versions = ">=3.7" +groups = ["test"] files = [ {file = "pyfakefs-5.7.2-py3-none-any.whl", hash = "sha256:e1527b0e8e4b33be52f0b024ca1deb269c73eecd68457c6b0bf608d6dab12ebd"}, {file = "pyfakefs-5.7.2.tar.gz", hash = "sha256:40da84175c5af8d9c4f3b31800b8edc4af1e74a212671dd658b21cc881c60000"}, @@ -533,6 +564,7 @@ version = "2.18.0" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, @@ -547,6 +579,8 @@ version = "5.13.2" description = "PyInstaller bundles a Python application and all its dependencies into a single package." optional = false python-versions = "<3.13,>=3.7" +groups = ["executable"] +markers = "python_version < \"3.13\"" files = [ {file = "pyinstaller-5.13.2-py3-none-macosx_10_13_universal2.whl", hash = "sha256:16cbd66b59a37f4ee59373a003608d15df180a0d9eb1a29ff3bfbfae64b23d0f"}, {file = "pyinstaller-5.13.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8f6dd0e797ae7efdd79226f78f35eb6a4981db16c13325e962a83395c0ec7420"}, @@ -580,6 +614,8 @@ version = "2024.10" description = "Community maintained hooks for PyInstaller" optional = false python-versions = ">=3.8" +groups = ["executable"] +markers = "python_version < \"3.13\"" files = [ {file = "pyinstaller_hooks_contrib-2024.10-py3-none-any.whl", hash = "sha256:ad47db0e153683b4151e10d231cb91f2d93c85079e78d76d9e0f57ac6c8a5e10"}, {file = "pyinstaller_hooks_contrib-2024.10.tar.gz", hash = "sha256:8a46655e5c5b0186b5e527399118a9b342f10513eb1425c483fa4f6d02e8800c"}, @@ -596,6 +632,7 @@ version = "2.9.0" description = "JSON Web Token implementation in Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850"}, {file = "pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"}, @@ -613,6 +650,7 @@ version = "7.3.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" +groups = ["test"] files = [ {file = "pytest-7.3.2-py3-none-any.whl", hash = "sha256:cdcbd012c9312258922f8cd3f1b62a6580fdced17db6014896053d47cddf9295"}, {file = "pytest-7.3.2.tar.gz", hash = "sha256:ee990a3cc55ba808b80795a79944756f315c67c12b56abd3ac993a7b8c17030b"}, @@ -635,6 +673,7 @@ version = "3.10.0" description = "Thin-wrapper around the mock package for easier use with pytest" optional = false python-versions = ">=3.7" +groups = ["test"] files = [ {file = "pytest-mock-3.10.0.tar.gz", hash = "sha256:fbbdb085ef7c252a326fd8cdcac0aa3b1333d8811f131bdcc701002e1be7ed4f"}, {file = "pytest_mock-3.10.0-py3-none-any.whl", hash = "sha256:f4c973eeae0282963eb293eb173ce91b091a79c1334455acfac9ddee8a1c784b"}, @@ -652,6 +691,7 @@ version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -666,6 +706,8 @@ version = "0.2.3" description = "A (partial) reimplementation of pywin32 using ctypes/cffi" optional = false python-versions = ">=3.6" +groups = ["executable"] +markers = "python_version < \"3.13\" and sys_platform == \"win32\"" files = [ {file = "pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755"}, {file = "pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8"}, @@ -677,6 +719,7 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" +groups = ["main", "test"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -739,6 +782,7 @@ version = "2.32.3" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" +groups = ["main", "test"] files = [ {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, @@ -760,6 +804,7 @@ version = "0.23.3" description = "A utility library for mocking out the `requests` Python library." optional = false python-versions = ">=3.7" +groups = ["test"] files = [ {file = "responses-0.23.3-py3-none-any.whl", hash = "sha256:e6fbcf5d82172fecc0aa1860fd91e58cbfd96cee5e96da5b63fa6eb3caa10dd3"}, {file = "responses-0.23.3.tar.gz", hash = "sha256:205029e1cb334c21cb4ec64fc7599be48b859a0fd381a42443cdd600bfe8b16a"}, @@ -772,7 +817,7 @@ types-PyYAML = "*" urllib3 = ">=1.25.10,<3.0" [package.extras] -tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-requests"] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli ; python_version < \"3.11\"", "tomli-w", "types-requests"] [[package]] name = "rich" @@ -780,6 +825,7 @@ version = "13.9.4" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.8.0" +groups = ["main"] files = [ {file = "rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90"}, {file = "rich-13.9.4.tar.gz", hash = "sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098"}, @@ -799,6 +845,7 @@ version = "0.6.9" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "ruff-0.6.9-py3-none-linux_armv6l.whl", hash = "sha256:064df58d84ccc0ac0fcd63bc3090b251d90e2a372558c0f057c3f75ed73e1ccd"}, {file = "ruff-0.6.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:140d4b5c9f5fc7a7b074908a78ab8d384dd7f6510402267bc76c37195c02a7ec"}, @@ -826,6 +873,7 @@ version = "2.19.2" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "sentry_sdk-2.19.2-py2.py3-none-any.whl", hash = "sha256:ebdc08228b4d131128e568d696c210d846e5b9d70aa0327dec6b1272d9d40b84"}, {file = "sentry_sdk-2.19.2.tar.gz", hash = "sha256:467df6e126ba242d39952375dd816fbee0f217d119bf454a8ce74cf1e7909e8d"}, @@ -880,19 +928,33 @@ version = "75.3.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" +groups = ["executable"] +markers = "python_version < \"3.13\"" files = [ {file = "setuptools-75.3.0-py3-none-any.whl", hash = "sha256:f2504966861356aa38616760c0f66568e535562374995367b4e69c7143cf6bcd"}, {file = "setuptools-75.3.0.tar.gz", hash = "sha256:fba5dd4d766e97be1b1681d98712680ae8f2f26d7881245f2ce9e40714f1a686"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.5.2)"] -core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.collections", "jaraco.functools", "jaraco.text (>=3.7)", "more-itertools", "more-itertools (>=8.8)", "packaging", "packaging (>=24)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.5.2) ; sys_platform != \"cygwin\""] +core = ["importlib-metadata (>=6) ; python_version < \"3.10\"", "importlib-resources (>=5.10.2) ; python_version < \"3.9\"", "jaraco.collections", "jaraco.functools", "jaraco.text (>=3.7)", "more-itertools", "more-itertools (>=8.8)", "packaging", "packaging (>=24)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib-metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.12.*)", "pytest-mypy"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib-metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.12.*)", "pytest-mypy"] + +[[package]] +name = "shellingham" +version = "1.5.4" +description = "Tool to Detect Surrounding Shell" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, + {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, +] [[package]] name = "six" @@ -900,6 +962,7 @@ version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -911,6 +974,7 @@ version = "5.0.1" description = "A pure Python implementation of a sliding window memory map manager" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"}, {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, @@ -922,6 +986,7 @@ version = "1.7.0" description = "module to create simple ASCII tables" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "texttable-1.7.0-py2.py3-none-any.whl", hash = "sha256:72227d592c82b3d7f672731ae73e4d1f88cd8e2ef5b075a7a7f01a23a3743917"}, {file = "texttable-1.7.0.tar.gz", hash = "sha256:2d2068fb55115807d3ac77a4ca68fa48803e84ebb0ee2340f858107a36522638"}, @@ -933,6 +998,8 @@ version = "2.2.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" +groups = ["test"] +markers = "python_version < \"3.11\"" files = [ {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, @@ -968,12 +1035,31 @@ files = [ {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, ] +[[package]] +name = "typer" +version = "0.15.2" +description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "typer-0.15.2-py3-none-any.whl", hash = "sha256:46a499c6107d645a9c13f7ee46c5d5096cae6f5fc57dd11eccbbb9ae3e44ddfc"}, + {file = "typer-0.15.2.tar.gz", hash = "sha256:ab2fab47533a813c49fe1f16b1a370fd5819099c00b119e0633df65f22144ba5"}, +] + +[package.dependencies] +click = ">=8.0.0" +rich = ">=10.11.0" +shellingham = ">=1.3.0" +typing-extensions = ">=3.7.4.3" + [[package]] name = "types-python-dateutil" version = "2.9.0.20241206" description = "Typing stubs for python-dateutil" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "types_python_dateutil-2.9.0.20241206-py3-none-any.whl", hash = "sha256:e248a4bc70a486d3e3ec84d0dc30eec3a5f979d6e7ee4123ae043eedbb987f53"}, {file = "types_python_dateutil-2.9.0.20241206.tar.gz", hash = "sha256:18f493414c26ffba692a72369fea7a154c502646301ebfe3d56a04b3767284cb"}, @@ -985,6 +1071,7 @@ version = "6.0.12.20240917" description = "Typing stubs for PyYAML" optional = false python-versions = ">=3.8" +groups = ["test"] files = [ {file = "types-PyYAML-6.0.12.20240917.tar.gz", hash = "sha256:d1405a86f9576682234ef83bcb4e6fff7c9305c8b1fbad5e0bcd4f7dbdc9c587"}, {file = "types_PyYAML-6.0.12.20240917-py3-none-any.whl", hash = "sha256:392b267f1c0fe6022952462bf5d6523f31e37f6cea49b14cee7ad634b6301570"}, @@ -996,6 +1083,7 @@ version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, @@ -1007,14 +1095,15 @@ version = "1.26.19" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +groups = ["main", "test"] files = [ {file = "urllib3-1.26.19-py2.py3-none-any.whl", hash = "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3"}, {file = "urllib3-1.26.19.tar.gz", hash = "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429"}, ] [package.extras] -brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +brotli = ["brotli (==1.0.9) ; os_name != \"nt\" and python_version < \"3\" and platform_python_implementation == \"CPython\"", "brotli (>=1.0.9) ; python_version >= \"3\" and platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; (os_name != \"nt\" or python_version >= \"3\") and platform_python_implementation != \"CPython\"", "brotlipy (>=0.6.0) ; os_name == \"nt\" and python_version < \"3\""] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress ; python_version == \"2.7\"", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] @@ -1023,20 +1112,22 @@ version = "3.20.2" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" +groups = ["executable"] +markers = "python_version < \"3.10\"" files = [ {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] enabler = ["pytest-enabler (>=2.2)"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +test = ["big-O", "importlib-resources ; python_version < \"3.9\"", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] type = ["pytest-mypy"] [metadata] -lock-version = "2.0" -python-versions = ">=3.8,<3.14" -content-hash = "e91a6f9b7e080cea351f9073ef333afe026df6172b95fba5477af67f15c96000" +lock-version = "2.1" +python-versions = ">=3.9,<3.14" +content-hash = "c0140dc408f1e3827b51357d74b05274297c233de11dbca85d4b6f3a909f4191" diff --git a/pyproject.toml b/pyproject.toml index 42511ec8..6baffad9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,6 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -24,10 +23,10 @@ classifiers = [ ] [tool.poetry.scripts] -cycode = "cycode.cli.main:main_cli" +cycode = "cycode.cli.app:app" [tool.poetry.dependencies] -python = ">=3.8,<3.14" +python = ">=3.9,<3.14" click = ">=8.1.0,<8.2.0" colorama = ">=0.4.3,<0.5.0" pyyaml = ">=6.0,<7.0" @@ -42,6 +41,7 @@ sentry-sdk = ">=2.8.0,<3.0" pyjwt = ">=2.8.0,<3.0" rich = ">=13.9.4, <14" patch-ng = "1.18.1" +typer = "^0.15.2" [tool.poetry.group.test.dependencies] mock = ">=4.0.3,<4.1.0" diff --git a/tests/cli/commands/configure/test_configure_command.py b/tests/cli/commands/configure/test_configure_command.py index c5ae2b9c..5ed94c1d 100644 --- a/tests/cli/commands/configure/test_configure_command.py +++ b/tests/cli/commands/configure/test_configure_command.py @@ -1,8 +1,8 @@ from typing import TYPE_CHECKING -from click.testing import CliRunner +from typer.testing import CliRunner -from cycode.cli.commands.configure.configure_command import configure_command +from cycode.cli.app import app if TYPE_CHECKING: from pytest_mock import MockerFixture @@ -30,7 +30,7 @@ def test_configure_command_no_exist_values_in_file(mocker: 'MockerFixture') -> N # side effect - multiple return values, each item in the list represents return of a call mocker.patch( - 'click.prompt', + 'typer.prompt', side_effect=[api_url_user_input, app_url_user_input, client_id_user_input, client_secret_user_input], ) @@ -45,7 +45,7 @@ def test_configure_command_no_exist_values_in_file(mocker: 'MockerFixture') -> N ) # Act - CliRunner().invoke(configure_command) + CliRunner().invoke(app, ['configure']) # Assert mocked_update_credentials.assert_called_once_with(client_id_user_input, client_secret_user_input) @@ -75,7 +75,7 @@ def test_configure_command_update_current_configs_in_files(mocker: 'MockerFixtur # side effect - multiple return values, each item in the list represents return of a call mocker.patch( - 'click.prompt', + 'typer.prompt', side_effect=[api_url_user_input, app_url_user_input, client_id_user_input, client_secret_user_input], ) @@ -90,7 +90,7 @@ def test_configure_command_update_current_configs_in_files(mocker: 'MockerFixtur ) # Act - CliRunner().invoke(configure_command) + CliRunner().invoke(app, ['configure']) # Assert mocked_update_credentials.assert_called_once_with(client_id_user_input, client_secret_user_input) @@ -108,13 +108,13 @@ def test_set_credentials_update_only_client_id(mocker: 'MockerFixture') -> None: ) # side effect - multiple return values, each item in the list represents return of a call - mocker.patch('click.prompt', side_effect=['', '', client_id_user_input, '']) + mocker.patch('typer.prompt', side_effect=['', '', client_id_user_input, '']) mocked_update_credentials = mocker.patch( 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_credentials' ) # Act - CliRunner().invoke(configure_command) + CliRunner().invoke(app, ['configure']) # Assert mocked_update_credentials.assert_called_once_with(client_id_user_input, current_client_id) @@ -131,13 +131,13 @@ def test_configure_command_update_only_client_secret(mocker: 'MockerFixture') -> ) # side effect - multiple return values, each item in the list represents return of a call - mocker.patch('click.prompt', side_effect=['', '', '', client_secret_user_input]) + mocker.patch('typer.prompt', side_effect=['', '', '', client_secret_user_input]) mocked_update_credentials = mocker.patch( 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_credentials' ) # Act - CliRunner().invoke(configure_command) + CliRunner().invoke(app, ['configure']) # Assert mocked_update_credentials.assert_called_once_with(current_client_id, client_secret_user_input) @@ -154,13 +154,13 @@ def test_configure_command_update_only_api_url(mocker: 'MockerFixture') -> None: ) # side effect - multiple return values, each item in the list represents return of a call - mocker.patch('click.prompt', side_effect=[api_url_user_input, '', '', '']) + mocker.patch('typer.prompt', side_effect=[api_url_user_input, '', '', '']) mocked_update_api_base_url = mocker.patch( 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.update_api_base_url' ) # Act - CliRunner().invoke(configure_command) + CliRunner().invoke(app, ['configure']) # Assert mocked_update_api_base_url.assert_called_once_with(api_url_user_input) @@ -177,13 +177,13 @@ def test_configure_command_should_not_update_credentials(mocker: 'MockerFixture' ) # side effect - multiple return values, each item in the list represents return of a call - mocker.patch('click.prompt', side_effect=['', '', client_id_user_input, client_secret_user_input]) + mocker.patch('typer.prompt', side_effect=['', '', client_id_user_input, client_secret_user_input]) mocked_update_credentials = mocker.patch( 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_credentials' ) # Act - CliRunner().invoke(configure_command) + CliRunner().invoke(app, ['configure']) # Assert assert not mocked_update_credentials.called @@ -204,7 +204,7 @@ def test_configure_command_should_not_update_config_file(mocker: 'MockerFixture' ) # side effect - multiple return values, each item in the list represents return of a call - mocker.patch('click.prompt', side_effect=[api_url_user_input, app_url_user_input, '', '']) + mocker.patch('typer.prompt', side_effect=[api_url_user_input, app_url_user_input, '', '']) mocked_update_api_base_url = mocker.patch( 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.update_api_base_url' ) @@ -213,7 +213,7 @@ def test_configure_command_should_not_update_config_file(mocker: 'MockerFixture' ) # Act - CliRunner().invoke(configure_command) + CliRunner().invoke(app, ['configure']) # Assert assert not mocked_update_api_base_url.called diff --git a/tests/cli/commands/scan/test_code_scanner.py b/tests/cli/commands/scan/test_code_scanner.py index 2c15dd3d..3151684e 100644 --- a/tests/cli/commands/scan/test_code_scanner.py +++ b/tests/cli/commands/scan/test_code_scanner.py @@ -1,7 +1,7 @@ import os from cycode.cli import consts -from cycode.cli.commands.scan.code_scanner import _does_severity_match_severity_threshold +from cycode.cli.apps.scan.code_scanner import _does_severity_match_severity_threshold from cycode.cli.files_collector.excluder import _is_file_relevant_for_sca_scan from cycode.cli.files_collector.path_documents import _generate_document from cycode.cli.models import Document @@ -42,7 +42,7 @@ def test_generate_document() -> None: }""" iac_document = Document(path, content, is_git_diff) - generated_document = _generate_document(path, consts.INFRA_CONFIGURATION_SCAN_TYPE, content, is_git_diff) + generated_document = _generate_document(path, consts.IAC_SCAN_TYPE, content, is_git_diff) assert iac_document.path == generated_document.path assert iac_document.content == generated_document.content assert iac_document.is_git_diff_format == generated_document.is_git_diff_format @@ -68,7 +68,7 @@ def test_generate_document() -> None: } """ - generated_tfplan_document = _generate_document(path, consts.INFRA_CONFIGURATION_SCAN_TYPE, content, is_git_diff) + generated_tfplan_document = _generate_document(path, consts.IAC_SCAN_TYPE, content, is_git_diff) assert isinstance(generated_tfplan_document, Document) assert generated_tfplan_document.path.endswith('.tf') diff --git a/tests/cli/commands/test_check_latest_version_on_close.py b/tests/cli/commands/test_check_latest_version_on_close.py index 189973b4..b1f11e24 100644 --- a/tests/cli/commands/test_check_latest_version_on_close.py +++ b/tests/cli/commands/test_check_latest_version_on_close.py @@ -1,11 +1,12 @@ from unittest.mock import patch import pytest -from click.testing import CliRunner +from typer.testing import CliRunner from cycode import __version__ -from cycode.cli.commands.main_cli import main_cli -from cycode.cli.commands.version.version_checker import VersionChecker +from cycode.cli.app import app +from cycode.cli.cli_types import OutputTypeOption +from cycode.cli.utils.version_checker import VersionChecker from tests.conftest import CLI_ENV_VARS _NEW_LATEST_VERSION = '999.0.0' # Simulate a newer version available @@ -17,8 +18,8 @@ def test_version_check_with_json_output(mock_check_update: patch) -> None: # When output is JSON, version check should be skipped mock_check_update.return_value = _NEW_LATEST_VERSION - args = ['--output', 'json', 'version'] - result = CliRunner().invoke(main_cli, args, env=CLI_ENV_VARS) + args = ['--output', OutputTypeOption.JSON, 'version'] + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) # Version check message should not be present in JSON output assert _UPDATE_MESSAGE_PART not in result.output.lower() @@ -28,7 +29,7 @@ def test_version_check_with_json_output(mock_check_update: patch) -> None: @pytest.fixture def mock_auth_info() -> 'patch': # Mock the authorization info to avoid API calls - with patch('cycode.cli.commands.status.status_command.get_authorization_info', return_value=None) as mock: + with patch('cycode.cli.apps.auth.auth_common.get_authorization_info', return_value=None) as mock: yield mock @@ -38,7 +39,7 @@ def test_version_check_for_special_commands(mock_check_update: patch, mock_auth_ # Version and status commands should always check the version without cache mock_check_update.return_value = _NEW_LATEST_VERSION - result = CliRunner().invoke(main_cli, [command], env=CLI_ENV_VARS) + result = CliRunner().invoke(app, [command], env=CLI_ENV_VARS) # Version information should be present in output assert _UPDATE_MESSAGE_PART in result.output.lower() @@ -52,7 +53,7 @@ def test_version_check_with_text_output(mock_check_update: patch) -> None: mock_check_update.return_value = _NEW_LATEST_VERSION args = ['version'] - result = CliRunner().invoke(main_cli, args, env=CLI_ENV_VARS) + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) # Version check message should be present in JSON output assert _UPDATE_MESSAGE_PART in result.output.lower() @@ -64,7 +65,7 @@ def test_version_check_disabled(mock_check_update: patch) -> None: mock_check_update.return_value = _NEW_LATEST_VERSION args = ['--no-update-notifier', 'version'] - result = CliRunner().invoke(main_cli, args, env=CLI_ENV_VARS) + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) # Version check message should not be present assert _UPDATE_MESSAGE_PART not in result.output.lower() diff --git a/tests/cli/commands/test_main_command.py b/tests/cli/commands/test_main_command.py index 7e588cf2..d7575ddb 100644 --- a/tests/cli/commands/test_main_command.py +++ b/tests/cli/commands/test_main_command.py @@ -4,10 +4,11 @@ import pytest import responses -from click.testing import CliRunner +from typer.testing import CliRunner from cycode.cli import consts -from cycode.cli.commands.main_cli import main_cli +from cycode.cli.app import app +from cycode.cli.cli_types import OutputTypeOption from cycode.cli.utils.git_proxy import git_proxy from tests.conftest import CLI_ENV_VARS, TEST_FILES_PATH, ZIP_CONTENT_PATH from tests.cyclient.mocked_responses.scan_client import mock_scan_responses @@ -28,7 +29,7 @@ def _is_json(plain: str) -> bool: @responses.activate -@pytest.mark.parametrize('output', ['text', 'json']) +@pytest.mark.parametrize('output', [OutputTypeOption.TEXT, OutputTypeOption.JSON]) def test_passing_output_option(output: str, scan_client: 'ScanClient', api_token_response: responses.Response) -> None: scan_type = consts.SECRET_SCAN_TYPE scan_id = uuid4() @@ -38,7 +39,7 @@ def test_passing_output_option(output: str, scan_client: 'ScanClient', api_token responses.add(api_token_response) args = ['--output', output, 'scan', '--soft-fail', 'path', str(_PATH_TO_SCAN)] - result = CliRunner().invoke(main_cli, args, env=CLI_ENV_VARS) + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) except_json = output == 'json' @@ -63,7 +64,7 @@ def test_optional_git_with_path_scan(scan_client: 'ScanClient', api_token_respon git_proxy._set_dummy_git_proxy() args = ['--output', 'json', 'scan', 'path', str(_PATH_TO_SCAN)] - result = CliRunner().invoke(main_cli, args, env=CLI_ENV_VARS) + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) # do NOT expect error about not found Git executable assert 'GIT_PYTHON_GIT_EXECUTABLE' not in result.output @@ -79,8 +80,8 @@ def test_required_git_with_path_repository(scan_client: 'ScanClient', api_token_ # fake env without Git executable git_proxy._set_dummy_git_proxy() - args = ['--output', 'json', 'scan', 'repository', str(_PATH_TO_SCAN)] - result = CliRunner().invoke(main_cli, args, env=CLI_ENV_VARS) + args = ['--output', OutputTypeOption.JSON, 'scan', 'repository', str(_PATH_TO_SCAN)] + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) # expect error about not found Git executable assert 'GIT_PYTHON_GIT_EXECUTABLE' in result.output diff --git a/tests/cli/commands/version/test_version_checker.py b/tests/cli/commands/version/test_version_checker.py index eb0b9bd9..926a21e8 100644 --- a/tests/cli/commands/version/test_version_checker.py +++ b/tests/cli/commands/version/test_version_checker.py @@ -7,7 +7,7 @@ if TYPE_CHECKING: from pathlib import Path -from cycode.cli.commands.version.version_checker import VersionChecker +from cycode.cli.utils.version_checker import VersionChecker @pytest.fixture diff --git a/tests/cli/exceptions/test_handle_scan_errors.py b/tests/cli/exceptions/test_handle_scan_errors.py index 82a44bb0..c1d34306 100644 --- a/tests/cli/exceptions/test_handle_scan_errors.py +++ b/tests/cli/exceptions/test_handle_scan_errors.py @@ -2,9 +2,11 @@ import click import pytest +import typer from click import ClickException from requests import Response +from cycode.cli.cli_types import OutputTypeOption from cycode.cli.exceptions import custom_exceptions from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception from cycode.cli.utils.git_proxy import git_proxy @@ -14,8 +16,8 @@ @pytest.fixture() -def ctx() -> click.Context: - return click.Context(click.Command('path'), obj={'verbose': False, 'output': 'text'}) +def ctx() -> typer.Context: + return typer.Context(click.Command('path'), obj={'verbose': False, 'output': OutputTypeOption.TEXT}) @pytest.mark.parametrize( @@ -30,7 +32,7 @@ def ctx() -> click.Context: ], ) def test_handle_exception_soft_fail( - ctx: click.Context, exception: custom_exceptions.CycodeError, expected_soft_fail: bool + ctx: typer.Context, exception: custom_exceptions.CycodeError, expected_soft_fail: bool ) -> None: with ctx: handle_scan_exception(ctx, exception) @@ -39,15 +41,15 @@ def test_handle_exception_soft_fail( assert ctx.obj.get('soft_fail') is expected_soft_fail -def test_handle_exception_unhandled_error(ctx: click.Context) -> None: - with ctx, pytest.raises(SystemExit): +def test_handle_exception_unhandled_error(ctx: typer.Context) -> None: + with ctx, pytest.raises(typer.Exit): handle_scan_exception(ctx, ValueError('test')) assert ctx.obj.get('did_fail') is True assert ctx.obj.get('soft_fail') is None -def test_handle_exception_click_error(ctx: click.Context) -> None: +def test_handle_exception_click_error(ctx: typer.Context) -> None: with ctx, pytest.raises(ClickException): handle_scan_exception(ctx, click.ClickException('test')) @@ -56,7 +58,7 @@ def test_handle_exception_click_error(ctx: click.Context) -> None: def test_handle_exception_verbose(monkeypatch: 'MonkeyPatch') -> None: - ctx = click.Context(click.Command('path'), obj={'verbose': True, 'output': 'text'}) + ctx = typer.Context(click.Command('path'), obj={'verbose': True, 'output': OutputTypeOption.TEXT}) error_text = 'test' @@ -65,5 +67,5 @@ def mock_secho(msg: str, *_, **__) -> None: monkeypatch.setattr(click, 'secho', mock_secho) - with pytest.raises(SystemExit): + with pytest.raises(typer.Exit): handle_scan_exception(ctx, ValueError(error_text)) diff --git a/tests/cli/models/test_severity.py b/tests/cli/models/test_severity.py index 332f987c..a59d5751 100644 --- a/tests/cli/models/test_severity.py +++ b/tests/cli/models/test_severity.py @@ -1,24 +1,11 @@ -from cycode.cli.models import Severity - - -def test_try_get_value() -> None: - assert Severity.try_get_value('info') == -1 - assert Severity.try_get_value('iNfO') == -1 - - assert Severity.try_get_value('INFO') == -1 - assert Severity.try_get_value('LOW') == 0 - assert Severity.try_get_value('MEDIUM') == 1 - assert Severity.try_get_value('HIGH') == 2 - assert Severity.try_get_value('CRITICAL') == 3 - - assert Severity.try_get_value('NON_EXISTENT') is None +from cycode.cli.cli_types import SeverityOption def test_get_member_weight() -> None: - assert Severity.get_member_weight('INFO') == -1 - assert Severity.get_member_weight('LOW') == 0 - assert Severity.get_member_weight('MEDIUM') == 1 - assert Severity.get_member_weight('HIGH') == 2 - assert Severity.get_member_weight('CRITICAL') == 3 + assert SeverityOption.get_member_weight('INFO') == 0 + assert SeverityOption.get_member_weight('LOW') == 1 + assert SeverityOption.get_member_weight('MEDIUM') == 2 + assert SeverityOption.get_member_weight('HIGH') == 3 + assert SeverityOption.get_member_weight('CRITICAL') == 4 - assert Severity.get_member_weight('NON_EXISTENT') == -2 + assert SeverityOption.get_member_weight('NON_EXISTENT') == -1 diff --git a/tests/cyclient/scan_config/test_default_scan_config.py b/tests/cyclient/scan_config/test_default_scan_config.py index 75b305b5..7371250c 100644 --- a/tests/cyclient/scan_config/test_default_scan_config.py +++ b/tests/cyclient/scan_config/test_default_scan_config.py @@ -6,7 +6,7 @@ def test_get_service_name() -> None: default_scan_config = DefaultScanConfig() assert default_scan_config.get_service_name(consts.SECRET_SCAN_TYPE) == 'secret' - assert default_scan_config.get_service_name(consts.INFRA_CONFIGURATION_SCAN_TYPE) == 'iac' + assert default_scan_config.get_service_name(consts.IAC_SCAN_TYPE) == 'iac' assert default_scan_config.get_service_name(consts.SCA_SCAN_TYPE) == 'scans' assert default_scan_config.get_service_name(consts.SAST_SCAN_TYPE) == 'scans' assert default_scan_config.get_service_name(consts.SECRET_SCAN_TYPE, True) == 'scans' diff --git a/tests/cyclient/scan_config/test_dev_scan_config.py b/tests/cyclient/scan_config/test_dev_scan_config.py index 63c99169..6ebb368b 100644 --- a/tests/cyclient/scan_config/test_dev_scan_config.py +++ b/tests/cyclient/scan_config/test_dev_scan_config.py @@ -6,7 +6,7 @@ def test_get_service_name() -> None: dev_scan_config = DevScanConfig() assert dev_scan_config.get_service_name(consts.SECRET_SCAN_TYPE) == '5025' - assert dev_scan_config.get_service_name(consts.INFRA_CONFIGURATION_SCAN_TYPE) == '5026' + assert dev_scan_config.get_service_name(consts.IAC_SCAN_TYPE) == '5026' assert dev_scan_config.get_service_name(consts.SCA_SCAN_TYPE) == '5004' assert dev_scan_config.get_service_name(consts.SAST_SCAN_TYPE) == '5004' assert dev_scan_config.get_service_name(consts.SECRET_SCAN_TYPE, should_use_scan_service=True) == '5004' diff --git a/tests/cyclient/test_auth_client.py b/tests/cyclient/test_auth_client.py index 1d7e6683..67147a6e 100644 --- a/tests/cyclient/test_auth_client.py +++ b/tests/cyclient/test_auth_client.py @@ -3,7 +3,7 @@ import responses from requests import Timeout -from cycode.cli.commands.auth.auth_manager import AuthManager +from cycode.cli.apps.auth.auth_manager import AuthManager from cycode.cli.exceptions.custom_exceptions import CycodeError, RequestTimeout from cycode.cyclient.auth_client import AuthClient from cycode.cyclient.models import ( diff --git a/tests/cyclient/test_scan_client.py b/tests/cyclient/test_scan_client.py index 2b8fc3f3..1b1724ca 100644 --- a/tests/cyclient/test_scan_client.py +++ b/tests/cyclient/test_scan_client.py @@ -51,7 +51,7 @@ def get_test_zip_file(scan_type: str) -> InMemoryZip: def test_get_service_name(scan_client: ScanClient) -> None: # TODO(MarshalX): get_service_name should be removed from ScanClient? Because it exists in ScanConfig assert scan_client.get_service_name(consts.SECRET_SCAN_TYPE) == 'secret' - assert scan_client.get_service_name(consts.INFRA_CONFIGURATION_SCAN_TYPE) == 'iac' + assert scan_client.get_service_name(consts.IAC_SCAN_TYPE) == 'iac' assert scan_client.get_service_name(consts.SCA_SCAN_TYPE) == 'scans' assert scan_client.get_service_name(consts.SAST_SCAN_TYPE) == 'scans' diff --git a/tests/test_code_scanner.py b/tests/test_code_scanner.py index 10726a65..d6341a7c 100644 --- a/tests/test_code_scanner.py +++ b/tests/test_code_scanner.py @@ -5,7 +5,7 @@ import responses from cycode.cli import consts -from cycode.cli.commands.scan.code_scanner import ( +from cycode.cli.apps.scan.code_scanner import ( _try_get_aggregation_report_url_if_needed, _try_get_report_url_if_needed, ) From 4ee8185678f87a83e07245118dbd6be9d65fa2f7 Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Wed, 12 Mar 2025 13:19:01 +0100 Subject: [PATCH 02/19] CM-45717 - Add formatted and colorized text in logs (`--verbose` mode) (#285) --- cycode/cli/app.py | 2 +- cycode/cli/apps/auth/auth_command.py | 2 +- cycode/cli/apps/auth/auth_manager.py | 5 +- cycode/cli/apps/ignore/ignore_command.py | 2 +- cycode/cli/apps/report/sbom/common.py | 2 +- cycode/cli/apps/scan/code_scanner.py | 6 +- .../commit_history/commit_history_command.py | 2 +- cycode/cli/apps/scan/path/path_command.py | 2 +- .../scan/pre_receive/pre_receive_command.py | 2 +- .../scan/repository/repository_command.py | 2 +- cycode/cli/apps/scan/scan_command.py | 23 ++--- cycode/cli/apps/status/get_cli_status.py | 2 +- cycode/cli/config.py | 6 -- cycode/cli/config.yaml | 25 ----- cycode/cli/consts.py | 4 - cycode/cli/files_collector/excluder.py | 5 +- cycode/cli/files_collector/path_documents.py | 2 +- .../sca/base_restore_dependencies.py | 2 +- .../sca/go/restore_go_dependencies.py | 4 +- .../sca/maven/restore_gradle_dependencies.py | 3 +- .../files_collector/sca/sca_code_scanner.py | 8 +- cycode/cli/files_collector/walk_ignore.py | 2 +- cycode/cli/files_collector/zip_documents.py | 13 ++- cycode/cli/logger.py | 3 + cycode/cli/printers/printer_base.py | 20 ++-- cycode/cli/printers/text_printer.py | 35 +++---- cycode/cli/user_settings/base_file_manager.py | 6 +- cycode/cli/utils/path_utils.py | 2 +- cycode/cli/utils/progress_bar.py | 4 +- cycode/cli/utils/scan_batch.py | 5 +- cycode/cli/utils/sentry.py | 2 +- cycode/cli/utils/shell_executor.py | 8 +- cycode/cli/utils/yaml_utils.py | 28 +++--- cycode/config.py | 45 +++++++++ cycode/cyclient/config.py | 97 +------------------ cycode/cyclient/config.yaml | 5 - cycode/cyclient/logger.py | 3 + cycode/logger.py | 66 +++++++++++++ pyinstaller.spec | 1 - tests/cyclient/test_scan_client.py | 34 ++++--- tests/test_code_scanner.py | 22 ++--- 41 files changed, 251 insertions(+), 261 deletions(-) delete mode 100644 cycode/cli/config.yaml create mode 100644 cycode/cli/logger.py create mode 100644 cycode/config.py delete mode 100644 cycode/cyclient/config.yaml create mode 100644 cycode/cyclient/logger.py create mode 100644 cycode/logger.py diff --git a/cycode/cli/app.py b/cycode/cli/app.py index d3fc10ca..96389ef1 100644 --- a/cycode/cli/app.py +++ b/cycode/cli/app.py @@ -11,9 +11,9 @@ from cycode.cli.utils.progress_bar import SCAN_PROGRESS_BAR_SECTIONS, get_progress_bar from cycode.cli.utils.sentry import add_breadcrumb, init_sentry from cycode.cli.utils.version_checker import version_checker -from cycode.cyclient.config import set_logging_level from cycode.cyclient.cycode_client_base import CycodeClientBase from cycode.cyclient.models import UserAgentOptionScheme +from cycode.logger import set_logging_level app = typer.Typer( pretty_exceptions_show_locals=False, diff --git a/cycode/cli/apps/auth/auth_command.py b/cycode/cli/apps/auth/auth_command.py index c0b2fb89..8150be01 100644 --- a/cycode/cli/apps/auth/auth_command.py +++ b/cycode/cli/apps/auth/auth_command.py @@ -2,10 +2,10 @@ from cycode.cli.apps.auth.auth_manager import AuthManager from cycode.cli.exceptions.handle_auth_errors import handle_auth_exception +from cycode.cli.logger import logger from cycode.cli.models import CliResult from cycode.cli.printers import ConsolePrinter from cycode.cli.utils.sentry import add_breadcrumb -from cycode.cyclient import logger def auth_command(ctx: typer.Context) -> None: diff --git a/cycode/cli/apps/auth/auth_manager.py b/cycode/cli/apps/auth/auth_manager.py index ab621842..ee064f3c 100644 --- a/cycode/cli/apps/auth/auth_manager.py +++ b/cycode/cli/apps/auth/auth_manager.py @@ -8,14 +8,17 @@ from cycode.cli.user_settings.configuration_manager import ConfigurationManager from cycode.cli.user_settings.credentials_manager import CredentialsManager from cycode.cli.utils.string_utils import generate_random_string, hash_string_to_sha256 -from cycode.cyclient import logger from cycode.cyclient.auth_client import AuthClient from cycode.cyclient.models import ApiTokenGenerationPollingResponse +from cycode.logger import get_logger if TYPE_CHECKING: from cycode.cyclient.models import ApiToken +logger = get_logger('Auth Manager') + + class AuthManager: CODE_VERIFIER_LENGTH = 101 POLLING_WAIT_INTERVAL_IN_SECONDS = 3 diff --git a/cycode/cli/apps/ignore/ignore_command.py b/cycode/cli/apps/ignore/ignore_command.py index 3ac3ffff..47d4fa0d 100644 --- a/cycode/cli/apps/ignore/ignore_command.py +++ b/cycode/cli/apps/ignore/ignore_command.py @@ -7,10 +7,10 @@ from cycode.cli import consts from cycode.cli.cli_types import ScanTypeOption from cycode.cli.config import configuration_manager +from cycode.cli.logger import logger from cycode.cli.utils.path_utils import get_absolute_path, is_path_exists from cycode.cli.utils.sentry import add_breadcrumb from cycode.cli.utils.string_utils import hash_string_to_sha256 -from cycode.cyclient import logger def _is_package_pattern_valid(package: str) -> bool: diff --git a/cycode/cli/apps/report/sbom/common.py b/cycode/cli/apps/report/sbom/common.py index b296e525..dabaffef 100644 --- a/cycode/cli/apps/report/sbom/common.py +++ b/cycode/cli/apps/report/sbom/common.py @@ -7,8 +7,8 @@ from cycode.cli.apps.report.sbom.sbom_report_file import SbomReportFile from cycode.cli.config import configuration_manager from cycode.cli.exceptions.custom_exceptions import ReportAsyncError +from cycode.cli.logger import logger from cycode.cli.utils.progress_bar import SbomReportProgressBarSection -from cycode.cyclient import logger from cycode.cyclient.models import ReportExecutionSchema if TYPE_CHECKING: diff --git a/cycode/cli/apps/scan/code_scanner.py b/cycode/cli/apps/scan/code_scanner.py index 535507d7..a1a5d440 100644 --- a/cycode/cli/apps/scan/code_scanner.py +++ b/cycode/cli/apps/scan/code_scanner.py @@ -34,9 +34,8 @@ from cycode.cli.utils.progress_bar import ScanProgressBarSection from cycode.cli.utils.scan_batch import run_parallel_batched_scan from cycode.cli.utils.scan_utils import set_issue_detected -from cycode.cyclient import logger -from cycode.cyclient.config import set_logging_level from cycode.cyclient.models import Detection, DetectionSchema, DetectionsPerFile, ZippedFileScanResult +from cycode.logger import get_logger, set_logging_level if TYPE_CHECKING: from cycode.cyclient.models import ScanDetailsResponse @@ -45,6 +44,9 @@ start_scan_time = time.time() +logger = get_logger('Code Scanner') + + def scan_sca_pre_commit(ctx: typer.Context) -> None: scan_type = ctx.obj['scan_type'] scan_parameters = get_default_scan_parameters(ctx) diff --git a/cycode/cli/apps/scan/commit_history/commit_history_command.py b/cycode/cli/apps/scan/commit_history/commit_history_command.py index dd74a4f0..f7992a92 100644 --- a/cycode/cli/apps/scan/commit_history/commit_history_command.py +++ b/cycode/cli/apps/scan/commit_history/commit_history_command.py @@ -5,8 +5,8 @@ from cycode.cli.apps.scan.code_scanner import scan_commit_range from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception +from cycode.cli.logger import logger from cycode.cli.utils.sentry import add_breadcrumb -from cycode.cyclient import logger def commit_history_command( diff --git a/cycode/cli/apps/scan/path/path_command.py b/cycode/cli/apps/scan/path/path_command.py index 4c841444..48db40ac 100644 --- a/cycode/cli/apps/scan/path/path_command.py +++ b/cycode/cli/apps/scan/path/path_command.py @@ -4,8 +4,8 @@ import typer from cycode.cli.apps.scan.code_scanner import scan_disk_files +from cycode.cli.logger import logger from cycode.cli.utils.sentry import add_breadcrumb -from cycode.cyclient import logger def path_command( diff --git a/cycode/cli/apps/scan/pre_receive/pre_receive_command.py b/cycode/cli/apps/scan/pre_receive/pre_receive_command.py index 92c152e6..01242b24 100644 --- a/cycode/cli/apps/scan/pre_receive/pre_receive_command.py +++ b/cycode/cli/apps/scan/pre_receive/pre_receive_command.py @@ -18,9 +18,9 @@ from cycode.cli.files_collector.repository_documents import ( calculate_pre_receive_commit_range, ) +from cycode.cli.logger import logger from cycode.cli.utils.sentry import add_breadcrumb from cycode.cli.utils.task_timer import TimeoutAfter -from cycode.cyclient import logger def pre_receive_command( diff --git a/cycode/cli/apps/scan/repository/repository_command.py b/cycode/cli/apps/scan/repository/repository_command.py index 0503c237..045448e6 100644 --- a/cycode/cli/apps/scan/repository/repository_command.py +++ b/cycode/cli/apps/scan/repository/repository_command.py @@ -11,11 +11,11 @@ from cycode.cli.files_collector.excluder import exclude_irrelevant_documents_to_scan from cycode.cli.files_collector.repository_documents import get_git_repository_tree_file_entries from cycode.cli.files_collector.sca.sca_code_scanner import perform_pre_scan_documents_actions +from cycode.cli.logger import logger from cycode.cli.models import Document from cycode.cli.utils.path_utils import get_path_by_os from cycode.cli.utils.progress_bar import ScanProgressBarSection from cycode.cli.utils.sentry import add_breadcrumb -from cycode.cyclient import logger def repository_command( diff --git a/cycode/cli/apps/scan/scan_command.py b/cycode/cli/apps/scan/scan_command.py index dffbf34f..5b9c43c6 100644 --- a/cycode/cli/apps/scan/scan_command.py +++ b/cycode/cli/apps/scan/scan_command.py @@ -4,12 +4,9 @@ import typer from cycode.cli.cli_types import ScanTypeOption, ScaScanTypeOption, SeverityOption -from cycode.cli.config import config from cycode.cli.consts import ( ISSUE_DETECTED_STATUS_CODE, NO_ISSUES_STATUS_CODE, - SCA_GRADLE_ALL_SUB_PROJECTS_FLAG, - SCA_SKIP_RESTORE_DEPENDENCIES_FLAG, ) from cycode.cli.utils import scan_utils from cycode.cli.utils.get_api_client import get_scan_cycode_client @@ -82,7 +79,7 @@ def scan_command( no_restore: Annotated[ bool, typer.Option( - f'--{SCA_SKIP_RESTORE_DEPENDENCIES_FLAG}', + '--no-restore', help='When specified, Cycode will not run restore command. ' 'Will scan direct dependencies [bold]only[/bold]!', rich_help_panel='SCA options', @@ -91,7 +88,7 @@ def scan_command( gradle_all_sub_projects: Annotated[ bool, typer.Option( - f'--{SCA_GRADLE_ALL_SUB_PROJECTS_FLAG}', + '--gradle-all-sub-projects', help='When specified, Cycode will run gradle restore command for all sub projects. ' 'Should run from root project directory [bold]only[/bold]!', rich_help_panel='SCA options', @@ -103,24 +100,16 @@ def scan_command( [cyan]path[/cyan]/[cyan]repository[/cyan]/[cyan]commit_history[/cyan].""" add_breadcrumb('scan') - if show_secret: - ctx.obj['show_secret'] = show_secret - else: - ctx.obj['show_secret'] = config['result_printer']['default']['show_secret'] - - if soft_fail: - ctx.obj['soft_fail'] = soft_fail - else: - ctx.obj['soft_fail'] = config['soft_fail'] - + ctx.obj['show_secret'] = show_secret + ctx.obj['soft_fail'] = soft_fail ctx.obj['client'] = get_scan_cycode_client(client_id, client_secret, not ctx.obj['show_secret']) ctx.obj['scan_type'] = scan_type ctx.obj['sync'] = sync ctx.obj['severity_threshold'] = severity_threshold ctx.obj['monitor'] = monitor ctx.obj['report'] = report - ctx.obj[SCA_SKIP_RESTORE_DEPENDENCIES_FLAG] = no_restore - ctx.obj[SCA_GRADLE_ALL_SUB_PROJECTS_FLAG] = gradle_all_sub_projects + + _ = no_restore, gradle_all_sub_projects # they are actually used; via ctx.params _sca_scan_to_context(ctx, sca_scan) diff --git a/cycode/cli/apps/status/get_cli_status.py b/cycode/cli/apps/status/get_cli_status.py index e58e910b..4a3dc4b0 100644 --- a/cycode/cli/apps/status/get_cli_status.py +++ b/cycode/cli/apps/status/get_cli_status.py @@ -4,9 +4,9 @@ from cycode.cli.apps.auth.auth_common import get_authorization_info from cycode.cli.apps.status.models import CliStatus, CliSupportedModulesStatus from cycode.cli.consts import PROGRAM_NAME +from cycode.cli.logger import logger from cycode.cli.user_settings.configuration_manager import ConfigurationManager from cycode.cli.utils.get_api_client import get_scan_cycode_client -from cycode.cyclient import logger def get_cli_status() -> CliStatus: diff --git a/cycode/cli/config.py b/cycode/cli/config.py index 71f354ad..a1ddbbaf 100644 --- a/cycode/cli/config.py +++ b/cycode/cli/config.py @@ -1,11 +1,5 @@ -import os - from cycode.cli.user_settings.configuration_manager import ConfigurationManager -from cycode.cli.utils.yaml_utils import read_file -relative_path = os.path.dirname(__file__) -config_file_path = os.path.join(relative_path, 'config.yaml') -config = read_file(config_file_path) configuration_manager = ConfigurationManager() # env vars diff --git a/cycode/cli/config.yaml b/cycode/cli/config.yaml deleted file mode 100644 index 875f37c1..00000000 --- a/cycode/cli/config.yaml +++ /dev/null @@ -1,25 +0,0 @@ -soft_fail: False -scans: - supported_scans: - - secret - - iac - - sca - - sast - supported_sca_scans: - - package-vulnerabilities - - license-compliance - supported_sbom_formats: - - spdx-2.2 - - spdx-2.3 - - cyclonedx-1.4 -result_printer: - default: - lines_to_display: 3 - show_secret: False - secret: - pre_receive: - lines_to_display: 1 - show_secret: False - commit_history: - lines_to_display: 1 - show_secret: False \ No newline at end of file diff --git a/cycode/cli/consts.py b/cycode/cli/consts.py index 1bca08db..60953143 100644 --- a/cycode/cli/consts.py +++ b/cycode/cli/consts.py @@ -223,7 +223,3 @@ # Example: A -> B -> C # Result: A -> ... -> C SCA_SHORTCUT_DEPENDENCY_PATHS = 2 - -SCA_SKIP_RESTORE_DEPENDENCIES_FLAG = 'no-restore' - -SCA_GRADLE_ALL_SUB_PROJECTS_FLAG = 'gradle-all-sub-projects' diff --git a/cycode/cli/files_collector/excluder.py b/cycode/cli/files_collector/excluder.py index 2d8243cb..f16e9710 100644 --- a/cycode/cli/files_collector/excluder.py +++ b/cycode/cli/files_collector/excluder.py @@ -5,13 +5,16 @@ from cycode.cli.user_settings.config_file_manager import ConfigFileManager from cycode.cli.utils.path_utils import get_file_size, is_binary_file, is_sub_path from cycode.cli.utils.string_utils import get_content_size, is_binary_content -from cycode.cyclient import logger +from cycode.logger import get_logger if TYPE_CHECKING: from cycode.cli.models import Document from cycode.cli.utils.progress_bar import BaseProgressBar, ProgressBarSection +logger = get_logger('File Excluder') + + def exclude_irrelevant_files( progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection', scan_type: str, filenames: List[str] ) -> List[str]: diff --git a/cycode/cli/files_collector/path_documents.py b/cycode/cli/files_collector/path_documents.py index 571773a0..469e6ce7 100644 --- a/cycode/cli/files_collector/path_documents.py +++ b/cycode/cli/files_collector/path_documents.py @@ -9,9 +9,9 @@ is_tfplan_file, ) from cycode.cli.files_collector.walk_ignore import walk_ignore +from cycode.cli.logger import logger from cycode.cli.models import Document from cycode.cli.utils.path_utils import get_absolute_path, get_file_content -from cycode.cyclient import logger if TYPE_CHECKING: from cycode.cli.utils.progress_bar import BaseProgressBar, ProgressBarSection diff --git a/cycode/cli/files_collector/sca/base_restore_dependencies.py b/cycode/cli/files_collector/sca/base_restore_dependencies.py index b1bf9e80..2e6c0993 100644 --- a/cycode/cli/files_collector/sca/base_restore_dependencies.py +++ b/cycode/cli/files_collector/sca/base_restore_dependencies.py @@ -3,10 +3,10 @@ import typer +from cycode.cli.logger import logger from cycode.cli.models import Document from cycode.cli.utils.path_utils import get_file_content, get_file_dir, get_path_from_context, join_paths from cycode.cli.utils.shell_executor import shell -from cycode.cyclient import logger def build_dep_tree_path(path: str, generated_file_name: str) -> str: diff --git a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py index 77af2a57..5d56644a 100644 --- a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py +++ b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py @@ -1,10 +1,10 @@ -import logging import os from typing import List, Optional import typer from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies +from cycode.cli.logger import logger from cycode.cli.models import Document GO_PROJECT_FILE_EXTENSIONS = ['.mod', '.sum'] @@ -22,7 +22,7 @@ def try_restore_dependencies(self, document: Document) -> Optional[Document]: lock_exists = os.path.isfile(self.get_working_directory(document) + os.sep + BUILD_GO_LOCK_FILE_NAME) if not manifest_exists or not lock_exists: - logging.info('No manifest go.mod file found' if not manifest_exists else 'No manifest go.sum file found') + logger.info('No manifest go.mod file found' if not manifest_exists else 'No manifest go.sum file found') manifest_files_exists = manifest_exists & lock_exists diff --git a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py index a5c6d48b..3995da90 100644 --- a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py @@ -4,7 +4,6 @@ import typer -from cycode.cli.consts import SCA_GRADLE_ALL_SUB_PROJECTS_FLAG from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.models import Document from cycode.cli.utils.path_utils import get_path_from_context @@ -28,7 +27,7 @@ def __init__( self.projects = self.get_all_projects() if self.is_gradle_sub_projects() else projects def is_gradle_sub_projects(self) -> bool: - return self.ctx.obj.get(SCA_GRADLE_ALL_SUB_PROJECTS_FLAG) + return self.ctx.params.get('gradle-all-sub-projects', False) def is_project(self, document: Document) -> bool: return document.path.endswith(BUILD_GRADLE_FILE_NAME) or document.path.endswith(BUILD_GRADLE_KTS_FILE_NAME) diff --git a/cycode/cli/files_collector/sca/sca_code_scanner.py b/cycode/cli/files_collector/sca/sca_code_scanner.py index 3f54eb12..fc8c3809 100644 --- a/cycode/cli/files_collector/sca/sca_code_scanner.py +++ b/cycode/cli/files_collector/sca/sca_code_scanner.py @@ -15,7 +15,7 @@ from cycode.cli.models import Document from cycode.cli.utils.git_proxy import git_proxy from cycode.cli.utils.path_utils import get_file_content, get_file_dir, get_path_from_context, join_paths -from cycode.cyclient import logger +from cycode.logger import get_logger if TYPE_CHECKING: from git import Repo @@ -23,6 +23,9 @@ BUILD_DEP_TREE_TIMEOUT = 180 +logger = get_logger('SCA Code Scanner') + + def perform_pre_commit_range_scan_actions( path: str, from_commit_documents: List[Document], @@ -157,6 +160,7 @@ def get_file_content_from_commit(repo: 'Repo', commit: str, file_path: str) -> O def perform_pre_scan_documents_actions( ctx: typer.Context, scan_type: str, documents_to_scan: List[Document], is_git_diff: bool = False ) -> None: - if scan_type == consts.SCA_SCAN_TYPE and not ctx.obj.get(consts.SCA_SKIP_RESTORE_DEPENDENCIES_FLAG): + no_restore = ctx.params.get('no-restore', False) + if scan_type == consts.SCA_SCAN_TYPE and not no_restore: logger.debug('Perform pre-scan document add_dependencies_tree_document action') add_dependencies_tree_document(ctx, documents_to_scan, is_git_diff) diff --git a/cycode/cli/files_collector/walk_ignore.py b/cycode/cli/files_collector/walk_ignore.py index 93286c87..0ba2b93d 100644 --- a/cycode/cli/files_collector/walk_ignore.py +++ b/cycode/cli/files_collector/walk_ignore.py @@ -1,8 +1,8 @@ import os from typing import Generator, Iterable, List, Tuple +from cycode.cli.logger import logger from cycode.cli.utils.ignore_utils import IgnoreFilterManager -from cycode.cyclient import logger _SUPPORTED_IGNORE_PATTERN_FILES = { # oneday we will bring .cycodeignore or something like that '.gitignore', diff --git a/cycode/cli/files_collector/zip_documents.py b/cycode/cli/files_collector/zip_documents.py index 9547f7fb..b9a272e1 100644 --- a/cycode/cli/files_collector/zip_documents.py +++ b/cycode/cli/files_collector/zip_documents.py @@ -6,7 +6,9 @@ from cycode.cli.exceptions import custom_exceptions from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip from cycode.cli.models import Document -from cycode.cyclient import logger +from cycode.logger import get_logger + +logger = get_logger('ZIP') def _validate_zip_file_size(scan_type: str, zip_file_size: int) -> None: @@ -25,7 +27,7 @@ def zip_documents(scan_type: str, documents: List[Document], zip_file: Optional[ _validate_zip_file_size(scan_type, zip_file.size) logger.debug( - 'Adding file to ZIP, %s', + 'Adding file, %s', {'index': index, 'filename': document.path, 'unique_id': document.unique_id}, ) zip_file.append(document.path, document.unique_id, document.content) @@ -34,11 +36,14 @@ def zip_documents(scan_type: str, documents: List[Document], zip_file: Optional[ end_zip_creation_time = timeit.default_timer() zip_creation_time = int(end_zip_creation_time - start_zip_creation_time) - logger.debug('Finished to create ZIP file, %s', {'zip_creation_time': zip_creation_time}) + logger.debug( + 'Finished to create file, %s', + {'zip_creation_time': zip_creation_time, 'zip_size': zip_file.size, 'documents_count': len(documents)}, + ) if zip_file.configuration_manager.get_debug_flag(): zip_file_path = Path.joinpath(Path.cwd(), f'{scan_type}_scan_{end_zip_creation_time}.zip') - logger.debug('Writing ZIP file to disk, %s', {'zip_file_path': zip_file_path}) + logger.debug('Writing file to disk, %s', {'zip_file_path': zip_file_path}) zip_file.write_on_disk(zip_file_path) return zip_file diff --git a/cycode/cli/logger.py b/cycode/cli/logger.py new file mode 100644 index 00000000..46748bff --- /dev/null +++ b/cycode/cli/logger.py @@ -0,0 +1,3 @@ +from cycode.logger import get_logger + +logger = get_logger('CLI') diff --git a/cycode/cli/printers/printer_base.py b/cycode/cli/printers/printer_base.py index 45419aec..ee9a7793 100644 --- a/cycode/cli/printers/printer_base.py +++ b/cycode/cli/printers/printer_base.py @@ -1,8 +1,7 @@ -import traceback +import sys from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Dict, List, Optional -import click import typer from cycode.cli.models import CliError, CliResult @@ -12,6 +11,10 @@ from cycode.cli.models import LocalScanResult +from rich.console import Console +from rich.traceback import Traceback + + class PrinterBase(ABC): RED_COLOR_NAME = 'red' WHITE_COLOR_NAME = 'white' @@ -40,14 +43,9 @@ def print_exception(self, e: Optional[BaseException] = None) -> None: Note: Called only when the verbose flag is set. """ - if e is None: - # gets the most recent exception caught by an except clause - message = f'Error: {traceback.format_exc()}' - else: - traceback_message = ''.join(traceback.format_exception(None, e, e.__traceback__)) - message = f'Error: {traceback_message}' + console = Console(stderr=True) - click.secho(message, err=True, fg=self.RED_COLOR_NAME) + traceback = Traceback.from_exception(type(e), e, None) if e else Traceback.from_exception(*sys.exc_info()) + console.print(traceback) - correlation_message = f'Correlation ID: {get_correlation_id()}' - click.secho(correlation_message, err=True, fg=self.RED_COLOR_NAME) + console.print(f'Correlation ID: {get_correlation_id()}', style=self.RED_COLOR_NAME) diff --git a/cycode/cli/printers/text_printer.py b/cycode/cli/printers/text_printer.py index 0f617f8c..7828d909 100644 --- a/cycode/cli/printers/text_printer.py +++ b/cycode/cli/printers/text_printer.py @@ -4,7 +4,6 @@ import click import typer -from cycode.cli.config import config from cycode.cli.consts import COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES, SECRET_SCAN_TYPE from cycode.cli.models import CliError, CliResult, Detection, Document, DocumentDetections from cycode.cli.printers.printer_base import PrinterBase @@ -17,7 +16,7 @@ class TextPrinter(PrinterBase): def __init__(self, ctx: typer.Context) -> None: super().__init__(ctx) - self.scan_type: str = ctx.obj.get('scan_type') + self.scan_type = ctx.obj.get('scan_type') self.command_scan_type: str = ctx.info_name self.show_secret: bool = ctx.obj.get('show_secret', False) @@ -66,10 +65,9 @@ def print_scan_results( def _print_document_detections(self, document_detections: DocumentDetections, scan_id: str) -> None: document = document_detections.document - lines_to_display = self._get_lines_to_display_count() for detection in document_detections.detections: self._print_detection_summary(detection, document.path, scan_id) - self._print_detection_code_segment(detection, document, lines_to_display) + self._print_detection_code_segment(detection, document) def _print_detection_summary(self, detection: Detection, document_path: str, scan_id: str) -> None: detection_name = detection.type if self.scan_type == SECRET_SCAN_TYPE else detection.message @@ -96,12 +94,15 @@ def _print_detection_summary(self, detection: Detection, document_path: str, sca f' ⛔' ) - def _print_detection_code_segment(self, detection: Detection, document: Document, code_segment_size: int) -> None: + def _print_detection_code_segment( + self, detection: Detection, document: Document, lines_to_display: int = 3 + ) -> None: if self._is_git_diff_based_scan(): + # it will print just one line self._print_detection_from_git_diff(detection, document) return - self._print_detection_from_file(detection, document, code_segment_size) + self._print_detection_from_file(detection, document, lines_to_display) @staticmethod def _print_report_urls(report_urls: List[str], aggregation_report_url: Optional[str] = None) -> None: @@ -116,8 +117,8 @@ def _print_report_urls(report_urls: List[str], aggregation_report_url: Optional[ click.echo(f'- {report_url}') @staticmethod - def _get_code_segment_start_line(detection_line: int, code_segment_size: int) -> int: - start_line = detection_line - math.ceil(code_segment_size / 2) + def _get_code_segment_start_line(detection_line: int, lines_to_display: int) -> int: + start_line = detection_line - math.ceil(lines_to_display / 2) return 0 if start_line < 0 else start_line def _print_line_of_code_segment( @@ -193,17 +194,7 @@ def _get_line_number_style(self, line_number: int) -> str: f'{click.style("|", fg=self.RED_COLOR_NAME, bold=False)}' ) - def _get_lines_to_display_count(self) -> int: - result_printer_configuration = config.get('result_printer') - lines_to_display_of_scan = ( - result_printer_configuration.get(self.scan_type, {}).get(self.command_scan_type, {}).get('lines_to_display') - ) - if lines_to_display_of_scan: - return lines_to_display_of_scan - - return result_printer_configuration.get('default').get('lines_to_display') - - def _print_detection_from_file(self, detection: Detection, document: Document, code_segment_size: int) -> None: + def _print_detection_from_file(self, detection: Detection, document: Document, lines_to_display: int) -> None: detection_details = detection.detection_details detection_line = ( detection_details.get('line', -1) @@ -215,12 +206,12 @@ def _print_detection_from_file(self, detection: Detection, document: Document, c file_content = document.content file_lines = file_content.splitlines() - start_line = self._get_code_segment_start_line(detection_line, code_segment_size) + start_line = self._get_code_segment_start_line(detection_line, lines_to_display) detection_position_in_line = get_position_in_line(file_content, detection_position) click.echo() - for i in range(code_segment_size): - current_line_index = start_line + i + for line_index in range(lines_to_display): + current_line_index = start_line + line_index if current_line_index >= len(file_lines): break diff --git a/cycode/cli/user_settings/base_file_manager.py b/cycode/cli/user_settings/base_file_manager.py index b7be273d..4eb15e2a 100644 --- a/cycode/cli/user_settings/base_file_manager.py +++ b/cycode/cli/user_settings/base_file_manager.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from typing import Any, Dict, Hashable -from cycode.cli.utils.yaml_utils import read_file, update_file +from cycode.cli.utils.yaml_utils import read_yaml_file, update_yaml_file class BaseFileManager(ABC): @@ -10,9 +10,9 @@ class BaseFileManager(ABC): def get_filename(self) -> str: ... def read_file(self) -> Dict[Hashable, Any]: - return read_file(self.get_filename()) + return read_yaml_file(self.get_filename()) def write_content_to_file(self, content: Dict[Hashable, Any]) -> None: filename = self.get_filename() os.makedirs(os.path.dirname(filename), exist_ok=True) - update_file(filename, content) + update_yaml_file(filename, content) diff --git a/cycode/cli/utils/path_utils.py b/cycode/cli/utils/path_utils.py index c6d2001d..3f670dd4 100644 --- a/cycode/cli/utils/path_utils.py +++ b/cycode/cli/utils/path_utils.py @@ -6,7 +6,7 @@ import typer from binaryornot.helpers import is_binary_string -from cycode.cyclient import logger +from cycode.cli.logger import logger if TYPE_CHECKING: from os import PathLike diff --git a/cycode/cli/utils/progress_bar.py b/cycode/cli/utils/progress_bar.py index 623222d7..3d798131 100644 --- a/cycode/cli/utils/progress_bar.py +++ b/cycode/cli/utils/progress_bar.py @@ -5,14 +5,14 @@ import click from cycode.cli.utils.enum_utils import AutoCountEnum -from cycode.cyclient.config import get_logger +from cycode.logger import get_logger if TYPE_CHECKING: from click._termui_impl import ProgressBar from click.termui import V as ProgressBarValue # use LOGGING_LEVEL=DEBUG env var to see debug logs of this module -logger = get_logger('progress bar', control_level_in_runtime=False) +logger = get_logger('Progress Bar', control_level_in_runtime=False) class ProgressBarSection(AutoCountEnum): diff --git a/cycode/cli/utils/scan_batch.py b/cycode/cli/utils/scan_batch.py index 4019b7b0..45e4d120 100644 --- a/cycode/cli/utils/scan_batch.py +++ b/cycode/cli/utils/scan_batch.py @@ -5,13 +5,16 @@ from cycode.cli import consts from cycode.cli.models import Document from cycode.cli.utils.progress_bar import ScanProgressBarSection -from cycode.cyclient import logger +from cycode.logger import get_logger if TYPE_CHECKING: from cycode.cli.models import CliError, LocalScanResult from cycode.cli.utils.progress_bar import BaseProgressBar +logger = get_logger('Batching') + + def _get_max_batch_size(scan_type: str) -> int: logger.debug( 'You can customize the batch size by setting the environment variable "%s"', diff --git a/cycode/cli/utils/sentry.py b/cycode/cli/utils/sentry.py index e132bcf8..16b2a982 100644 --- a/cycode/cli/utils/sentry.py +++ b/cycode/cli/utils/sentry.py @@ -11,8 +11,8 @@ from cycode import __version__ from cycode.cli import consts +from cycode.cli.logger import logger from cycode.cli.utils.jwt_utils import get_user_and_tenant_ids_from_access_token -from cycode.cyclient import logger from cycode.cyclient.config import on_premise_installation # when Sentry is blocked on the machine, we want to keep clean output without retries warnings diff --git a/cycode/cli/utils/shell_executor.py b/cycode/cli/utils/shell_executor.py index 5ac79518..a7a537e6 100644 --- a/cycode/cli/utils/shell_executor.py +++ b/cycode/cli/utils/shell_executor.py @@ -3,11 +3,14 @@ import click -from cycode.cyclient import logger +from cycode.logger import get_logger _SUBPROCESS_DEFAULT_TIMEOUT_SEC = 60 +logger = get_logger('SHELL') + + def shell( command: Union[str, List[str]], timeout: int = _SUBPROCESS_DEFAULT_TIMEOUT_SEC, @@ -19,13 +22,16 @@ def shell( result = subprocess.run( # noqa: S603 command, cwd=working_directory, timeout=timeout, check=True, capture_output=True ) + logger.debug('Shell command executed successfully') return result.stdout.decode('UTF-8').strip() except subprocess.CalledProcessError as e: logger.debug('Error occurred while running shell command', exc_info=e) except subprocess.TimeoutExpired as e: + logger.debug('Command timed out', exc_info=e) raise click.Abort(f'Command "{command}" timed out') from e except Exception as e: + logger.debug('Unhandled exception occurred while running shell command', exc_info=e) raise click.ClickException(f'Unhandled exception: {e}') from e return None diff --git a/cycode/cli/utils/yaml_utils.py b/cycode/cli/utils/yaml_utils.py index 251b6c24..388f3498 100644 --- a/cycode/cli/utils/yaml_utils.py +++ b/cycode/cli/utils/yaml_utils.py @@ -4,6 +4,16 @@ import yaml +def _deep_update(source: Dict[Hashable, Any], overrides: Dict[Hashable, Any]) -> Dict[Hashable, Any]: + for key, value in overrides.items(): + if isinstance(value, dict) and value: + source[key] = _deep_update(source.get(key, {}), value) + else: + source[key] = overrides[key] + + return source + + def _yaml_safe_load(file: TextIO) -> Dict[Hashable, Any]: # loader.get_single_data could return None loaded_file = yaml.safe_load(file) @@ -13,7 +23,7 @@ def _yaml_safe_load(file: TextIO) -> Dict[Hashable, Any]: return loaded_file -def read_file(filename: str) -> Dict[Hashable, Any]: +def read_yaml_file(filename: str) -> Dict[Hashable, Any]: if not os.path.exists(filename): return {} @@ -21,20 +31,10 @@ def read_file(filename: str) -> Dict[Hashable, Any]: return _yaml_safe_load(file) -def write_file(filename: str, content: Dict[Hashable, Any]) -> None: +def write_yaml_file(filename: str, content: Dict[Hashable, Any]) -> None: with open(filename, 'w', encoding='UTF-8') as file: yaml.safe_dump(content, file) -def update_file(filename: str, content: Dict[Hashable, Any]) -> None: - write_file(filename, _deep_update(read_file(filename), content)) - - -def _deep_update(source: Dict[Hashable, Any], overrides: Dict[Hashable, Any]) -> Dict[Hashable, Any]: - for key, value in overrides.items(): - if isinstance(value, dict) and value: - source[key] = _deep_update(source.get(key, {}), value) - else: - source[key] = overrides[key] - - return source +def update_yaml_file(filename: str, content: Dict[Hashable, Any]) -> None: + write_yaml_file(filename, _deep_update(read_yaml_file(filename), content)) diff --git a/cycode/config.py b/cycode/config.py new file mode 100644 index 00000000..f4306b31 --- /dev/null +++ b/cycode/config.py @@ -0,0 +1,45 @@ +import logging +import os +from typing import Optional +from urllib.parse import urlparse + +from cycode.cli import consts +from cycode.cyclient import config_dev + +DEFAULT_CONFIGURATION = { + consts.TIMEOUT_ENV_VAR_NAME: 300, + consts.LOGGING_LEVEL_ENV_VAR_NAME: logging.INFO, + config_dev.DEV_MODE_ENV_VAR_NAME: 'false', +} + +configuration = dict(DEFAULT_CONFIGURATION, **os.environ) + + +def get_val_as_string(key: str) -> str: + return configuration.get(key) + + +def get_val_as_bool(key: str, default: bool = False) -> bool: + if key not in configuration: + return default + + return configuration[key].lower() in {'true', '1', 'yes', 'y', 'on', 'enabled'} + + +def get_val_as_int(key: str) -> Optional[int]: + val = configuration.get(key) + if not val: + return None + + try: + return int(val) + except ValueError: + return None + + +def is_valid_url(url: str) -> bool: + try: + parsed_url = urlparse(url) + return all([parsed_url.scheme, parsed_url.netloc]) + except ValueError: + return False diff --git a/cycode/cyclient/config.py b/cycode/cyclient/config.py index 37183195..2b278bf4 100644 --- a/cycode/cyclient/config.py +++ b/cycode/cyclient/config.py @@ -1,102 +1,9 @@ -import logging -import os -import sys -from typing import NamedTuple, Optional, Set, Union -from urllib.parse import urlparse - from cycode.cli import consts from cycode.cli.user_settings.configuration_manager import ConfigurationManager +from cycode.config import get_val_as_bool, get_val_as_int, get_val_as_string, is_valid_url from cycode.cyclient import config_dev +from cycode.cyclient.logger import logger - -def _set_io_encodings() -> None: - # set io encoding (for Windows) - sys.stdout.reconfigure(encoding='UTF-8') - sys.stderr.reconfigure(encoding='UTF-8') - - -_set_io_encodings() - -# logs -logging.basicConfig( - stream=sys.stderr, - level=logging.INFO, - format='%(asctime)s [%(name)s] %(levelname)s: %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', -) -logging.getLogger('urllib3').setLevel(logging.WARNING) -logging.getLogger('werkzeug').setLevel(logging.WARNING) -logging.getLogger('schedule').setLevel(logging.WARNING) -logging.getLogger('kubernetes').setLevel(logging.WARNING) -logging.getLogger('binaryornot').setLevel(logging.WARNING) -logging.getLogger('chardet').setLevel(logging.WARNING) -logging.getLogger('git.cmd').setLevel(logging.WARNING) -logging.getLogger('git.util').setLevel(logging.WARNING) - -# configs -DEFAULT_CONFIGURATION = { - consts.TIMEOUT_ENV_VAR_NAME: 300, - consts.LOGGING_LEVEL_ENV_VAR_NAME: logging.INFO, - config_dev.DEV_MODE_ENV_VAR_NAME: 'false', -} - -configuration = dict(DEFAULT_CONFIGURATION, **os.environ) - - -class CreatedLogger(NamedTuple): - logger: logging.Logger - control_level_in_runtime: bool - - -_CREATED_LOGGERS: Set[CreatedLogger] = set() - - -def get_logger_level() -> Optional[Union[int, str]]: - config_level = get_val_as_string(consts.LOGGING_LEVEL_ENV_VAR_NAME) - return logging.getLevelName(config_level) - - -def get_logger(logger_name: Optional[str] = None, control_level_in_runtime: bool = True) -> logging.Logger: - new_logger = logging.getLogger(logger_name) - new_logger.setLevel(get_logger_level()) - - _CREATED_LOGGERS.add(CreatedLogger(logger=new_logger, control_level_in_runtime=control_level_in_runtime)) - - return new_logger - - -def set_logging_level(level: int) -> None: - for created_logger in _CREATED_LOGGERS: - if created_logger.control_level_in_runtime: - created_logger.logger.setLevel(level) - - -def get_val_as_string(key: str) -> str: - return configuration.get(key) - - -def get_val_as_bool(key: str, default: str = '') -> bool: - val = configuration.get(key, default) - return val.lower() in {'true', '1'} - - -def get_val_as_int(key: str) -> Optional[int]: - val = configuration.get(key) - if val: - return int(val) - - return None - - -def is_valid_url(url: str) -> bool: - try: - parsed_url = urlparse(url) - return all([parsed_url.scheme, parsed_url.netloc]) - except ValueError: - return False - - -logger = get_logger('cycode cli') configuration_manager = ConfigurationManager() cycode_api_url = configuration_manager.get_cycode_api_url() diff --git a/cycode/cyclient/config.yaml b/cycode/cyclient/config.yaml deleted file mode 100644 index 7b8c1bdc..00000000 --- a/cycode/cyclient/config.yaml +++ /dev/null @@ -1,5 +0,0 @@ -cycode: - api: - base_url: http://api.cycode.com - time_out: 30 -# base_url: http://localhost:5048 #local configuration diff --git a/cycode/cyclient/logger.py b/cycode/cyclient/logger.py new file mode 100644 index 00000000..b36f036f --- /dev/null +++ b/cycode/cyclient/logger.py @@ -0,0 +1,3 @@ +from cycode.logger import get_logger + +logger = get_logger('CyClient') diff --git a/cycode/logger.py b/cycode/logger.py new file mode 100644 index 00000000..684d296c --- /dev/null +++ b/cycode/logger.py @@ -0,0 +1,66 @@ +import logging +import sys +from typing import NamedTuple, Optional, Set, Union + +import click +import typer +from rich.console import Console +from rich.logging import RichHandler + +from cycode.cli import consts +from cycode.config import get_val_as_string + + +def _set_io_encodings() -> None: + # set io encoding (for Windows) + sys.stdout.reconfigure(encoding='UTF-8') + sys.stderr.reconfigure(encoding='UTF-8') + + +_set_io_encodings() + +_ERROR_CONSOLE = Console(stderr=True) +_RICH_LOGGING_HANDLER = RichHandler(console=_ERROR_CONSOLE, rich_tracebacks=True, tracebacks_suppress=[click, typer]) + +logging.basicConfig( + level=logging.INFO, + format='[%(name)s] %(message)s', + handlers=[_RICH_LOGGING_HANDLER], +) + +logging.getLogger('urllib3').setLevel(logging.WARNING) +logging.getLogger('werkzeug').setLevel(logging.WARNING) +logging.getLogger('schedule').setLevel(logging.WARNING) +logging.getLogger('kubernetes').setLevel(logging.WARNING) +logging.getLogger('binaryornot').setLevel(logging.WARNING) +logging.getLogger('chardet').setLevel(logging.WARNING) +logging.getLogger('git.cmd').setLevel(logging.WARNING) +logging.getLogger('git.util').setLevel(logging.WARNING) + + +class CreatedLogger(NamedTuple): + logger: logging.Logger + control_level_in_runtime: bool + + +_CREATED_LOGGERS: Set[CreatedLogger] = set() + + +def get_logger_level() -> Optional[Union[int, str]]: + config_level = get_val_as_string(consts.LOGGING_LEVEL_ENV_VAR_NAME) + return logging.getLevelName(config_level) + + +def get_logger(logger_name: Optional[str] = None, control_level_in_runtime: bool = True) -> logging.Logger: + new_logger = logging.getLogger(logger_name) + new_logger.setLevel(get_logger_level()) + + _CREATED_LOGGERS.add(CreatedLogger(logger=new_logger, control_level_in_runtime=control_level_in_runtime)) + + return new_logger + + +def set_logging_level(level: int) -> None: + for created_logger in _CREATED_LOGGERS: + if created_logger.control_level_in_runtime: + created_logger.logger.setLevel(level) diff --git a/pyinstaller.spec b/pyinstaller.spec index cb3382d4..39b8588f 100644 --- a/pyinstaller.spec +++ b/pyinstaller.spec @@ -23,7 +23,6 @@ with open(_INIT_FILE_PATH, 'w', encoding='UTF-8') as file: a = Analysis( scripts=['cycode/cli/main.py'], - datas=[('cycode/cli/config.yaml', 'cycode/cli'), ('cycode/cyclient/config.yaml', 'cycode/cyclient')], excludes=['tests'], ) diff --git a/tests/cyclient/test_scan_client.py b/tests/cyclient/test_scan_client.py index 1b1724ca..a1c0d151 100644 --- a/tests/cyclient/test_scan_client.py +++ b/tests/cyclient/test_scan_client.py @@ -9,7 +9,7 @@ from requests.exceptions import ProxyError from cycode.cli import consts -from cycode.cli.config import config +from cycode.cli.cli_types import ScanTypeOption from cycode.cli.exceptions.custom_exceptions import ( CycodeError, HttpUnauthorizedError, @@ -29,14 +29,14 @@ ) -def zip_scan_resources(scan_type: str, scan_client: ScanClient) -> Tuple[str, InMemoryZip]: +def zip_scan_resources(scan_type: ScanTypeOption, scan_client: ScanClient) -> Tuple[str, InMemoryZip]: url = get_zipped_file_scan_url(scan_type, scan_client) zip_file = get_test_zip_file(scan_type) return url, zip_file -def get_test_zip_file(scan_type: str) -> InMemoryZip: +def get_test_zip_file(scan_type: ScanTypeOption) -> InMemoryZip: # TODO(MarshalX): refactor scan_disk_files in code_scanner.py to reuse method here instead of this test_documents: List[Document] = [] for root, _, files in os.walk(ZIP_CONTENT_PATH): @@ -56,9 +56,11 @@ def test_get_service_name(scan_client: ScanClient) -> None: assert scan_client.get_service_name(consts.SAST_SCAN_TYPE) == 'scans' -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate -def test_zipped_file_scan(scan_type: str, scan_client: ScanClient, api_token_response: responses.Response) -> None: +def test_zipped_file_scan( + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response +) -> None: url, zip_file = zip_scan_resources(scan_type, scan_client) expected_scan_id = uuid4() @@ -71,9 +73,11 @@ def test_zipped_file_scan(scan_type: str, scan_client: ScanClient, api_token_res assert zipped_file_scan_response.scan_id == str(expected_scan_id) -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate -def test_get_scan_report_url(scan_type: str, scan_client: ScanClient, api_token_response: responses.Response) -> None: +def test_get_scan_report_url( + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response +) -> None: scan_id = uuid4() url = get_scan_report_url(scan_id, scan_client, scan_type) @@ -84,10 +88,10 @@ def test_get_scan_report_url(scan_type: str, scan_client: ScanClient, api_token_ assert scan_report_url_response.report_url == 'https://app.domain/on-demand-scans/{scan_id}'.format(scan_id=scan_id) -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate def test_zipped_file_scan_unauthorized_error( - scan_type: str, scan_client: ScanClient, api_token_response: responses.Response + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: url, zip_file = zip_scan_resources(scan_type, scan_client) expected_scan_id = uuid4().hex @@ -101,10 +105,10 @@ def test_zipped_file_scan_unauthorized_error( assert e_info.value.status_code == 401 -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate def test_zipped_file_scan_bad_request_error( - scan_type: str, scan_client: ScanClient, api_token_response: responses.Response + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: url, zip_file = zip_scan_resources(scan_type, scan_client) expected_scan_id = uuid4().hex @@ -122,10 +126,10 @@ def test_zipped_file_scan_bad_request_error( assert e_info.value.error_message == expected_response_text -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate def test_zipped_file_scan_timeout_error( - scan_type: str, scan_client: ScanClient, api_token_response: responses.Response + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: scan_url, zip_file = zip_scan_resources(scan_type, scan_client) expected_scan_id = uuid4().hex @@ -148,10 +152,10 @@ def test_zipped_file_scan_timeout_error( scan_client.zipped_file_scan(scan_type, zip_file, scan_id=expected_scan_id, scan_parameters={}) -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate def test_zipped_file_scan_connection_error( - scan_type: str, scan_client: ScanClient, api_token_response: responses.Response + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: url, zip_file = zip_scan_resources(scan_type, scan_client) expected_scan_id = uuid4().hex diff --git a/tests/test_code_scanner.py b/tests/test_code_scanner.py index d6341a7c..709fe70e 100644 --- a/tests/test_code_scanner.py +++ b/tests/test_code_scanner.py @@ -9,7 +9,7 @@ _try_get_aggregation_report_url_if_needed, _try_get_report_url_if_needed, ) -from cycode.cli.config import config +from cycode.cli.cli_types import ScanTypeOption from cycode.cli.files_collector.excluder import _is_relevant_file_to_scan from cycode.cyclient.scan_client import ScanClient from tests.conftest import TEST_FILES_PATH @@ -26,17 +26,17 @@ def test_is_relevant_file_to_scan_sca() -> None: assert _is_relevant_file_to_scan(consts.SCA_SCAN_TYPE, path) is True -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) -def test_try_get_report_url_if_needed_return_none(scan_type: str, scan_client: ScanClient) -> None: +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) +def test_try_get_report_url_if_needed_return_none(scan_type: ScanTypeOption, scan_client: ScanClient) -> None: scan_id = uuid4().hex result = _try_get_report_url_if_needed(scan_client, False, scan_id, consts.SECRET_SCAN_TYPE) assert result is None -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate def test_try_get_report_url_if_needed_return_result( - scan_type: str, scan_client: ScanClient, api_token_response: responses.Response + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: scan_id = uuid4() url = get_scan_report_url(scan_id, scan_client, scan_type) @@ -48,9 +48,9 @@ def test_try_get_report_url_if_needed_return_result( assert result == scan_report_url_response.report_url -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) def test_try_get_aggregation_report_url_if_no_report_command_needed_return_none( - scan_type: str, scan_client: ScanClient + scan_type: ScanTypeOption, scan_client: ScanClient ) -> None: aggregation_id = uuid4().hex scan_parameter = {'aggregation_id': aggregation_id} @@ -58,19 +58,19 @@ def test_try_get_aggregation_report_url_if_no_report_command_needed_return_none( assert result is None -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) def test_try_get_aggregation_report_url_if_no_aggregation_id_needed_return_none( - scan_type: str, scan_client: ScanClient + scan_type: ScanTypeOption, scan_client: ScanClient ) -> None: scan_parameter = {'report': True} result = _try_get_aggregation_report_url_if_needed(scan_parameter, scan_client, scan_type) assert result is None -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate def test_try_get_aggregation_report_url_if_needed_return_result( - scan_type: str, scan_client: ScanClient, api_token_response: responses.Response + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: aggregation_id = uuid4() scan_parameter = {'report': True, 'aggregation_id': aggregation_id} From b64c67e2ce6d0ea7e96822ee3d6a9191d89627db Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Fri, 14 Mar 2025 15:55:13 +0100 Subject: [PATCH 03/19] CM-45715 - Add rich progress bar with spinner and elapsed time (#286) --- cycode/cli/apps/report/sbom/common.py | 2 +- cycode/cli/utils/progress_bar.py | 94 ++++++++++++--------------- cycode/cyclient/__init__.py | 5 -- cycode/cyclient/cycode_client_base.py | 3 +- cycode/cyclient/headers.py | 2 +- 5 files changed, 44 insertions(+), 62 deletions(-) diff --git a/cycode/cli/apps/report/sbom/common.py b/cycode/cli/apps/report/sbom/common.py index dabaffef..067a9fa6 100644 --- a/cycode/cli/apps/report/sbom/common.py +++ b/cycode/cli/apps/report/sbom/common.py @@ -30,7 +30,7 @@ def _poll_report_execution_until_completed( report_execution = client.get_report_execution(report_execution_id) report_label = report_execution.error_message or report_execution.status_message - progress_bar.update_label(report_label) + progress_bar.update_right_side_label(report_label) if report_execution.status == consts.REPORT_STATUS_COMPLETED: return report_execution diff --git a/cycode/cli/utils/progress_bar.py b/cycode/cli/utils/progress_bar.py index 3d798131..90a19801 100644 --- a/cycode/cli/utils/progress_bar.py +++ b/cycode/cli/utils/progress_bar.py @@ -1,16 +1,12 @@ from abc import ABC, abstractmethod from enum import auto -from typing import TYPE_CHECKING, Dict, NamedTuple, Optional +from typing import Dict, NamedTuple, Optional -import click +from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn, TimeElapsedColumn from cycode.cli.utils.enum_utils import AutoCountEnum from cycode.logger import get_logger -if TYPE_CHECKING: - from click._termui_impl import ProgressBar - from click.termui import V as ProgressBarValue - # use LOGGING_LEVEL=DEBUG env var to see debug logs of this module logger = get_logger('Progress Bar', control_level_in_runtime=False) @@ -32,6 +28,14 @@ class ProgressBarSectionInfo(NamedTuple): _PROGRESS_BAR_LENGTH = 100 +_PROGRESS_BAR_COLUMNS = ( + SpinnerColumn(), + TextColumn('[progress.description]{task.description}'), + TextColumn('{task.fields[right_side_label]}'), + BarColumn(bar_width=None), + TaskProgressColumn(), + TimeElapsedColumn(), +) ProgressBarSections = Dict[ProgressBarSection, ProgressBarSectionInfo] @@ -91,12 +95,6 @@ class BaseProgressBar(ABC): def __init__(self, *args, **kwargs) -> None: pass - @abstractmethod - def __enter__(self) -> 'BaseProgressBar': ... - - @abstractmethod - def __exit__(self, *args, **kwargs) -> None: ... - @abstractmethod def start(self) -> None: ... @@ -110,19 +108,13 @@ def set_section_length(self, section: 'ProgressBarSection', length: int = 0) -> def update(self, section: 'ProgressBarSection') -> None: ... @abstractmethod - def update_label(self, label: Optional[str] = None) -> None: ... + def update_right_side_label(self, label: Optional[str] = None) -> None: ... class DummyProgressBar(BaseProgressBar): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - def __enter__(self) -> 'DummyProgressBar': - return self - - def __exit__(self, *args, **kwargs) -> None: - pass - def start(self) -> None: pass @@ -135,7 +127,7 @@ def set_section_length(self, section: 'ProgressBarSection', length: int = 0) -> def update(self, section: 'ProgressBarSection') -> None: pass - def update_label(self, label: Optional[str] = None) -> None: + def update_right_side_label(self, label: Optional[str] = None) -> None: pass @@ -143,38 +135,41 @@ class CompositeProgressBar(BaseProgressBar): def __init__(self, progress_bar_sections: ProgressBarSections) -> None: super().__init__() - self._progress_bar_sections = progress_bar_sections - - self._progress_bar_context_manager = click.progressbar( - length=_PROGRESS_BAR_LENGTH, - item_show_func=self._progress_bar_item_show_func, - update_min_steps=0, - ) - self._progress_bar: Optional['ProgressBar'] = None self._run = False + self._progress_bar_sections = progress_bar_sections self._section_lengths: Dict[ProgressBarSection, int] = {} self._section_values: Dict[ProgressBarSection, int] = {} self._current_section_value = 0 self._current_section: ProgressBarSectionInfo = _get_initial_section(self._progress_bar_sections) + self._current_right_side_label = '' - def __enter__(self) -> 'CompositeProgressBar': - self._progress_bar = self._progress_bar_context_manager.__enter__() - self._run = True - return self + self._progress_bar = Progress( + *_PROGRESS_BAR_COLUMNS, + transient=True, + ) + self._progress_bar_task_id = self._progress_bar.add_task( + description=self._current_section.label, + total=_PROGRESS_BAR_LENGTH, + right_side_label=self._current_right_side_label, + ) - def __exit__(self, *args, **kwargs) -> None: - self._progress_bar_context_manager.__exit__(*args, **kwargs) - self._run = False + def _progress_bar_update(self, advance: int = 0) -> None: + self._progress_bar.update( + self._progress_bar_task_id, + advance=advance, + description=self._current_section.label, + right_side_label=self._current_right_side_label, + ) def start(self) -> None: if not self._run: - self.__enter__() + self._progress_bar.start() def stop(self) -> None: if self._run: - self.__exit__(None, None, None) + self._progress_bar.stop() def set_section_length(self, section: 'ProgressBarSection', length: int = 0) -> None: logger.debug('Calling set_section_length, %s', {'section': str(section), 'length': length}) @@ -190,7 +185,7 @@ def _get_section_length(self, section: 'ProgressBarSection') -> int: return section_info.stop_percent - section_info.start_percent def _skip_section(self, section: 'ProgressBarSection') -> None: - self._progress_bar.update(self._get_section_length(section)) + self._progress_bar_update(self._get_section_length(section)) self._maybe_update_current_section() def _increment_section_value(self, section: 'ProgressBarSection', value: int) -> None: @@ -205,13 +200,13 @@ def _increment_section_value(self, section: 'ProgressBarSection', value: int) -> def _rerender_progress_bar(self) -> None: """Used to update label right after changing the progress bar section.""" - self._progress_bar.update(0) + self._progress_bar_update() def _increment_progress(self, section: 'ProgressBarSection') -> None: increment_value = self._get_increment_progress_value(section) self._current_section_value += increment_value - self._progress_bar.update(increment_value) + self._progress_bar_update(increment_value) def _maybe_update_current_section(self) -> None: if not self._current_section.section.has_next(): @@ -237,13 +232,7 @@ def _get_increment_progress_value(self, section: 'ProgressBarSection') -> int: return expected_value - self._current_section_value - def _progress_bar_item_show_func(self, _: Optional['ProgressBarValue'] = None) -> str: - return self._current_section.label - def update(self, section: 'ProgressBarSection', value: int = 1) -> None: - if not self._progress_bar: - raise ValueError('Progress bar is not initialized. Call start() first or use "with" statement.') - if section not in self._section_lengths: raise ValueError(f'{section} section is not initialized. Call set_section_length() first.') if section is not self._current_section.section: @@ -255,12 +244,9 @@ def update(self, section: 'ProgressBarSection', value: int = 1) -> None: self._increment_progress(section) self._maybe_update_current_section() - def update_label(self, label: Optional[str] = None) -> None: - if not self._progress_bar: - raise ValueError('Progress bar is not initialized. Call start() first or use "with" statement.') - - self._progress_bar.label = label or '' - self._progress_bar.render_progress() + def update_right_side_label(self, label: Optional[str] = None) -> None: + self._current_right_side_label = f'({label})' or '' + self._progress_bar_update() def get_progress_bar(*, hidden: bool, sections: ProgressBarSections) -> BaseProgressBar: @@ -284,9 +270,9 @@ def get_progress_bar(*, hidden: bool, sections: ProgressBarSections) -> BaseProg for _i in range(section_capacity): time.sleep(0.01) - bar.update_label(f'{bar_section} {_i}/{section_capacity}') + bar.update_right_side_label(f'{bar_section} {_i}/{section_capacity}') bar.update(bar_section) - bar.update_label() + bar.update_right_side_label() bar.stop() diff --git a/cycode/cyclient/__init__.py b/cycode/cyclient/__init__.py index 9bea26e9..e69de29b 100644 --- a/cycode/cyclient/__init__.py +++ b/cycode/cyclient/__init__.py @@ -1,5 +0,0 @@ -from cycode.cyclient.config import logger - -__all__ = [ - 'logger', -] diff --git a/cycode/cyclient/cycode_client_base.py b/cycode/cyclient/cycode_client_base.py index 3024de89..f2eb77bf 100644 --- a/cycode/cyclient/cycode_client_base.py +++ b/cycode/cyclient/cycode_client_base.py @@ -15,8 +15,9 @@ RequestSslError, RequestTimeout, ) -from cycode.cyclient import config, logger +from cycode.cyclient import config from cycode.cyclient.headers import get_cli_user_agent, get_correlation_id +from cycode.cyclient.logger import logger class SystemStorageSslContext(HTTPAdapter): diff --git a/cycode/cyclient/headers.py b/cycode/cyclient/headers.py index 4e2434e9..76716826 100644 --- a/cycode/cyclient/headers.py +++ b/cycode/cyclient/headers.py @@ -6,7 +6,7 @@ from cycode.cli import consts from cycode.cli.user_settings.configuration_manager import ConfigurationManager from cycode.cli.utils.sentry import add_correlation_id_to_scope -from cycode.cyclient import logger +from cycode.cyclient.logger import logger def get_cli_user_agent() -> str: From da80ead8f337a93fe22c7dc5a662907c88aa9173 Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Thu, 20 Mar 2025 13:43:47 +0100 Subject: [PATCH 04/19] CM-45716 - Add rich tables with more useful information, colorful values, and clickable paths (#287) --- cycode/cli/cli_types.py | 13 ++ .../cli/printers/tables/sca_table_printer.py | 100 ++++++++------ cycode/cli/printers/tables/table.py | 53 ++++---- cycode/cli/printers/tables/table_models.py | 15 +- cycode/cli/printers/tables/table_printer.py | 128 ++++++++++-------- .../cli/printers/tables/table_printer_base.py | 18 +-- cycode/cli/utils/string_utils.py | 2 +- poetry.lock | 16 +-- pyproject.toml | 1 - tests/utils/test_string_utils.py | 2 +- 10 files changed, 189 insertions(+), 159 deletions(-) diff --git a/cycode/cli/cli_types.py b/cycode/cli/cli_types.py index 92c36fa2..83451df2 100644 --- a/cycode/cli/cli_types.py +++ b/cycode/cli/cli_types.py @@ -42,6 +42,10 @@ class SeverityOption(str, Enum): def get_member_weight(name: str) -> int: return _SEVERITY_WEIGHTS.get(name.lower(), _SEVERITY_DEFAULT_WEIGHT) + @staticmethod + def get_member_color(name: str) -> str: + return _SEVERITY_COLORS.get(name.lower(), _SEVERITY_DEFAULT_COLOR) + _SEVERITY_DEFAULT_WEIGHT = -1 _SEVERITY_WEIGHTS = { @@ -51,3 +55,12 @@ def get_member_weight(name: str) -> int: SeverityOption.HIGH.value: 3, SeverityOption.CRITICAL.value: 4, } + +_SEVERITY_DEFAULT_COLOR = 'white' +_SEVERITY_COLORS = { + SeverityOption.INFO.value: 'deep_sky_blue1', + SeverityOption.LOW.value: 'gold1', + SeverityOption.MEDIUM.value: 'dark_orange', + SeverityOption.HIGH.value: 'red1', + SeverityOption.CRITICAL.value: 'red3', +} diff --git a/cycode/cli/printers/tables/sca_table_printer.py b/cycode/cli/printers/tables/sca_table_printer.py index 063e80e8..b59a33ef 100644 --- a/cycode/cli/printers/tables/sca_table_printer.py +++ b/cycode/cli/printers/tables/sca_table_printer.py @@ -1,13 +1,13 @@ from collections import defaultdict from typing import TYPE_CHECKING, Dict, List -import click +import typer from cycode.cli.cli_types import SeverityOption from cycode.cli.consts import LICENSE_COMPLIANCE_POLICY_ID, PACKAGE_VULNERABILITY_POLICY_ID from cycode.cli.models import Detection from cycode.cli.printers.tables.table import Table -from cycode.cli.printers.tables.table_models import ColumnInfoBuilder, ColumnWidths +from cycode.cli.printers.tables.table_models import ColumnInfoBuilder from cycode.cli.printers.tables.table_printer_base import TablePrinterBase from cycode.cli.utils.string_utils import shortcut_dependency_paths @@ -19,25 +19,16 @@ # Building must have strict order. Represents the order of the columns in the table (from left to right) SEVERITY_COLUMN = column_builder.build(name='Severity') REPOSITORY_COLUMN = column_builder.build(name='Repository') -CODE_PROJECT_COLUMN = column_builder.build(name='Code Project') # File path to manifest file -ECOSYSTEM_COLUMN = column_builder.build(name='Ecosystem') -PACKAGE_COLUMN = column_builder.build(name='Package') -CVE_COLUMNS = column_builder.build(name='CVE') +CODE_PROJECT_COLUMN = column_builder.build(name='Code Project', highlight=False) # File path to the manifest file +ECOSYSTEM_COLUMN = column_builder.build(name='Ecosystem', highlight=False) +PACKAGE_COLUMN = column_builder.build(name='Package', highlight=False) +CVE_COLUMNS = column_builder.build(name='CVE', highlight=False) DEPENDENCY_PATHS_COLUMN = column_builder.build(name='Dependency Paths') UPGRADE_COLUMN = column_builder.build(name='Upgrade') -LICENSE_COLUMN = column_builder.build(name='License') +LICENSE_COLUMN = column_builder.build(name='License', highlight=False) DIRECT_DEPENDENCY_COLUMN = column_builder.build(name='Direct Dependency') DEVELOPMENT_DEPENDENCY_COLUMN = column_builder.build(name='Development Dependency') -COLUMN_WIDTHS_CONFIG: ColumnWidths = { - REPOSITORY_COLUMN: 2, - CODE_PROJECT_COLUMN: 2, - PACKAGE_COLUMN: 3, - CVE_COLUMNS: 5, - UPGRADE_COLUMN: 3, - LICENSE_COLUMN: 2, -} - class ScaTablePrinter(TablePrinterBase): def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: @@ -45,10 +36,9 @@ def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: detections_per_policy_id = self._extract_detections_per_policy_id(local_scan_results) for policy_id, detections in detections_per_policy_id.items(): table = self._get_table(policy_id) - table.set_cols_width(COLUMN_WIDTHS_CONFIG) for detection in self._sort_and_group_detections(detections): - self._enrich_table_with_values(table, detection) + self._enrich_table_with_values(policy_id, table, detection) self._print_summary_issues(len(detections), self._get_title(policy_id)) self._print_table(table) @@ -90,7 +80,7 @@ def _sort_and_group_detections(self, detections: List[Detection]) -> List[Detect """Sort detections by severity and group by repository, code project and package name. Note: - Code Project is path to manifest file. + Code Project is path to the manifest file. Grouping by code projects also groups by ecosystem. Because manifest files are unique per ecosystem. @@ -114,55 +104,77 @@ def _get_table(self, policy_id: str) -> Table: table = Table() if policy_id == PACKAGE_VULNERABILITY_POLICY_ID: - table.add(SEVERITY_COLUMN) - table.add(CVE_COLUMNS) - table.add(UPGRADE_COLUMN) + table.add_column(CVE_COLUMNS) + table.add_column(UPGRADE_COLUMN) elif policy_id == LICENSE_COMPLIANCE_POLICY_ID: - table.add(LICENSE_COLUMN) + table.add_column(LICENSE_COLUMN) if self._is_git_repository(): - table.add(REPOSITORY_COLUMN) + table.add_column(REPOSITORY_COLUMN) - table.add(CODE_PROJECT_COLUMN) - table.add(ECOSYSTEM_COLUMN) - table.add(PACKAGE_COLUMN) - table.add(DIRECT_DEPENDENCY_COLUMN) - table.add(DEVELOPMENT_DEPENDENCY_COLUMN) - table.add(DEPENDENCY_PATHS_COLUMN) + table.add_column(SEVERITY_COLUMN) + table.add_column(CODE_PROJECT_COLUMN) + table.add_column(ECOSYSTEM_COLUMN) + table.add_column(PACKAGE_COLUMN) + table.add_column(DIRECT_DEPENDENCY_COLUMN) + table.add_column(DEVELOPMENT_DEPENDENCY_COLUMN) + table.add_column(DEPENDENCY_PATHS_COLUMN) return table @staticmethod - def _enrich_table_with_values(table: Table, detection: Detection) -> None: + def _enrich_table_with_values(policy_id: str, table: Table, detection: Detection) -> None: detection_details = detection.detection_details - table.set(SEVERITY_COLUMN, detection_details.get('advisory_severity')) - table.set(REPOSITORY_COLUMN, detection_details.get('repository_name')) - - table.set(CODE_PROJECT_COLUMN, detection_details.get('file_name')) - table.set(ECOSYSTEM_COLUMN, detection_details.get('ecosystem')) - table.set(PACKAGE_COLUMN, detection_details.get('package_name')) - table.set(DIRECT_DEPENDENCY_COLUMN, detection_details.get('is_direct_dependency_str')) - table.set(DEVELOPMENT_DEPENDENCY_COLUMN, detection_details.get('is_dev_dependency_str')) + severity = None + if policy_id == PACKAGE_VULNERABILITY_POLICY_ID: + severity = detection_details.get('advisory_severity') + elif policy_id == LICENSE_COMPLIANCE_POLICY_ID: + severity = detection.severity + + if not severity: + severity = 'N/A' + + table.add_cell(SEVERITY_COLUMN, severity, SeverityOption.get_member_color(severity)) + + table.add_cell(REPOSITORY_COLUMN, detection_details.get('repository_name')) + table.add_file_path_cell(CODE_PROJECT_COLUMN, detection_details.get('file_name')) + table.add_cell(ECOSYSTEM_COLUMN, detection_details.get('ecosystem')) + table.add_cell(PACKAGE_COLUMN, detection_details.get('package_name')) + + dependency_bool_to_color = { + True: 'green', + False: 'red', + } # by default, not colored (None) + table.add_cell( + column=DIRECT_DEPENDENCY_COLUMN, + value=detection_details.get('is_direct_dependency_str'), + color=dependency_bool_to_color.get(detection_details.get('is_direct_dependency')), + ) + table.add_cell( + column=DEVELOPMENT_DEPENDENCY_COLUMN, + value=detection_details.get('is_dev_dependency_str'), + color=dependency_bool_to_color.get(detection_details.get('is_dev_dependency')), + ) dependency_paths = 'N/A' dependency_paths_raw = detection_details.get('dependency_paths') if dependency_paths_raw: dependency_paths = shortcut_dependency_paths(dependency_paths_raw) - table.set(DEPENDENCY_PATHS_COLUMN, dependency_paths) + table.add_cell(DEPENDENCY_PATHS_COLUMN, dependency_paths) upgrade = '' alert = detection_details.get('alert') if alert and alert.get('first_patched_version'): upgrade = f'{alert.get("vulnerable_requirements")} -> {alert.get("first_patched_version")}' - table.set(UPGRADE_COLUMN, upgrade) + table.add_cell(UPGRADE_COLUMN, upgrade) - table.set(CVE_COLUMNS, detection_details.get('vulnerability_id')) - table.set(LICENSE_COLUMN, detection_details.get('license')) + table.add_cell(CVE_COLUMNS, detection_details.get('vulnerability_id')) + table.add_cell(LICENSE_COLUMN, detection_details.get('license')) @staticmethod def _print_summary_issues(detections_count: int, title: str) -> None: - click.echo(f'⛔ Found {detections_count} issues of type: {click.style(title, bold=True)}') + typer.echo(f'⛔ Found {detections_count} issues of type: {typer.style(title, bold=True)}') @staticmethod def _extract_detections_per_policy_id( diff --git a/cycode/cli/printers/tables/table.py b/cycode/cli/printers/tables/table.py index 2017b9c8..a071c9b4 100644 --- a/cycode/cli/printers/tables/table.py +++ b/cycode/cli/printers/tables/table.py @@ -1,29 +1,40 @@ +import urllib.parse from typing import TYPE_CHECKING, Dict, List, Optional -from texttable import Texttable +from rich.markup import escape +from rich.table import Table as RichTable if TYPE_CHECKING: - from cycode.cli.printers.tables.table_models import ColumnInfo, ColumnWidths + from cycode.cli.printers.tables.table_models import ColumnInfo class Table: """Helper class to manage columns and their values in the right order and only if the column should be presented.""" def __init__(self, column_infos: Optional[List['ColumnInfo']] = None) -> None: - self._column_widths = None - self._columns: Dict['ColumnInfo', List[str]] = {} if column_infos: - self._columns: Dict['ColumnInfo', List[str]] = {columns: [] for columns in column_infos} + self._columns = {columns: [] for columns in column_infos} - def add(self, column: 'ColumnInfo') -> None: + def add_column(self, column: 'ColumnInfo') -> None: self._columns[column] = [] - def set(self, column: 'ColumnInfo', value: str) -> None: + def _add_cell_no_error(self, column: 'ColumnInfo', value: str) -> None: # we push values only for existing columns what were added before if column in self._columns: self._columns[column].append(value) + def add_cell(self, column: 'ColumnInfo', value: str, color: Optional[str] = None) -> None: + if color: + value = f'[{color}]{value}[/{color}]' + + self._add_cell_no_error(column, value) + + def add_file_path_cell(self, column: 'ColumnInfo', path: str) -> None: + encoded_path = urllib.parse.quote(path) + escaped_path = escape(encoded_path) + self._add_cell_no_error(column, f'[link file://{escaped_path}]{path}') + def _get_ordered_columns(self) -> List['ColumnInfo']: # we are sorting columns by index to make sure that columns will be printed in the right order return sorted(self._columns, key=lambda column_info: column_info.index) @@ -31,32 +42,18 @@ def _get_ordered_columns(self) -> List['ColumnInfo']: def get_columns_info(self) -> List['ColumnInfo']: return self._get_ordered_columns() - def get_headers(self) -> List[str]: - return [header.name for header in self._get_ordered_columns()] - def get_rows(self) -> List[str]: column_values = [self._columns[column_info] for column_info in self._get_ordered_columns()] return list(zip(*column_values)) - def set_cols_width(self, column_widths: 'ColumnWidths') -> None: - header_width_size = [] - for header in self.get_columns_info(): - width_multiplier = 1 - if header in column_widths: - width_multiplier = column_widths[header] - - header_width_size.append(len(header.name) * width_multiplier) - - self._column_widths = header_width_size - - def get_table(self, max_width: int = 80) -> Texttable: - table = Texttable(max_width) - table.header(self.get_headers()) + def get_table(self) -> 'RichTable': + table = RichTable(expand=True, highlight=True) - for row in self.get_rows(): - table.add_row(row) + for column in self.get_columns_info(): + extra_args = column.column_opts if column.column_opts else {} + table.add_column(header=column.name, overflow='fold', **extra_args) - if self._column_widths: - table.set_cols_width(self._column_widths) + for raw in self.get_rows(): + table.add_row(*raw) return table diff --git a/cycode/cli/printers/tables/table_models.py b/cycode/cli/printers/tables/table_models.py index c162a8ce..42e3b1fb 100644 --- a/cycode/cli/printers/tables/table_models.py +++ b/cycode/cli/printers/tables/table_models.py @@ -1,12 +1,12 @@ -from typing import Dict, NamedTuple +from typing import Any, Dict, NamedTuple, Optional class ColumnInfoBuilder: def __init__(self) -> None: self._index = 0 - def build(self, name: str) -> 'ColumnInfo': - column_info = ColumnInfo(name, self._index) + def build(self, name: str, **column_opts) -> 'ColumnInfo': + column_info = ColumnInfo(name, self._index, column_opts) self._index += 1 return column_info @@ -14,7 +14,12 @@ def build(self, name: str) -> 'ColumnInfo': class ColumnInfo(NamedTuple): name: str index: int # Represents the order of the columns, starting from the left + column_opts: Optional[Dict] = None + def __hash__(self) -> int: + return hash((self.name, self.index)) -ColumnWidths = Dict[ColumnInfo, int] -ColumnWidthsConfig = Dict[str, ColumnWidths] + def __eq__(self, other: Any) -> bool: + if not isinstance(other, ColumnInfo): + return NotImplemented + return (self.name, self.index) == (other.name, other.index) diff --git a/cycode/cli/printers/tables/table_printer.py b/cycode/cli/printers/tables/table_printer.py index 61234066..728e2baf 100644 --- a/cycode/cli/printers/tables/table_printer.py +++ b/cycode/cli/printers/tables/table_printer.py @@ -1,11 +1,11 @@ -from typing import TYPE_CHECKING, List +from collections import defaultdict +from typing import TYPE_CHECKING, List, Tuple -import click - -from cycode.cli.consts import IAC_SCAN_TYPE, SAST_SCAN_TYPE, SECRET_SCAN_TYPE +from cycode.cli.cli_types import SeverityOption +from cycode.cli.consts import SECRET_SCAN_TYPE from cycode.cli.models import Detection, Document from cycode.cli.printers.tables.table import Table -from cycode.cli.printers.tables.table_models import ColumnInfoBuilder, ColumnWidthsConfig +from cycode.cli.printers.tables.table_models import ColumnInfoBuilder from cycode.cli.printers.tables.table_printer_base import TablePrinterBase from cycode.cli.utils.string_utils import get_position_in_line, obfuscate_text @@ -15,73 +15,89 @@ column_builder = ColumnInfoBuilder() # Building must have strict order. Represents the order of the columns in the table (from left to right) +SEVERITY_COLUMN = column_builder.build(name='Severity') ISSUE_TYPE_COLUMN = column_builder.build(name='Issue Type') -RULE_ID_COLUMN = column_builder.build(name='Rule ID') -FILE_PATH_COLUMN = column_builder.build(name='File Path') +FILE_PATH_COLUMN = column_builder.build(name='File Path', highlight=False) SECRET_SHA_COLUMN = column_builder.build(name='Secret SHA') COMMIT_SHA_COLUMN = column_builder.build(name='Commit SHA') LINE_NUMBER_COLUMN = column_builder.build(name='Line Number') COLUMN_NUMBER_COLUMN = column_builder.build(name='Column Number') VIOLATION_LENGTH_COLUMN = column_builder.build(name='Violation Length') -VIOLATION_COLUMN = column_builder.build(name='Violation') -SCAN_ID_COLUMN = column_builder.build(name='Scan ID') - -COLUMN_WIDTHS_CONFIG: ColumnWidthsConfig = { - SECRET_SCAN_TYPE: { - ISSUE_TYPE_COLUMN: 2, - RULE_ID_COLUMN: 2, - FILE_PATH_COLUMN: 2, - SECRET_SHA_COLUMN: 2, - VIOLATION_COLUMN: 2, - SCAN_ID_COLUMN: 2, - }, - IAC_SCAN_TYPE: { - ISSUE_TYPE_COLUMN: 4, - RULE_ID_COLUMN: 3, - FILE_PATH_COLUMN: 3, - SCAN_ID_COLUMN: 2, - }, - SAST_SCAN_TYPE: { - ISSUE_TYPE_COLUMN: 7, - RULE_ID_COLUMN: 2, - FILE_PATH_COLUMN: 3, - SCAN_ID_COLUMN: 2, - }, -} +VIOLATION_COLUMN = column_builder.build(name='Violation', highlight=False) class TablePrinter(TablePrinterBase): def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: table = self._get_table() - if self.scan_type in COLUMN_WIDTHS_CONFIG: - table.set_cols_width(COLUMN_WIDTHS_CONFIG[self.scan_type]) + detections_with_documents = [] for local_scan_result in local_scan_results: for document_detections in local_scan_result.document_detections: - for detection in document_detections.detections: - table.set(SCAN_ID_COLUMN, local_scan_result.scan_id) - self._enrich_table_with_values(table, detection, document_detections.document) + detections_with_documents.extend( + [(detection, document_detections.document) for detection in document_detections.detections] + ) + + for detection, document in self._sort_and_group_detections(detections_with_documents): + self._enrich_table_with_values(table, detection, document) self._print_table(table) self._print_report_urls(local_scan_results, self.ctx.obj.get('aggregation_report_url')) + @staticmethod + def __severity_sort_key(detection_with_document: Tuple[Detection, Document]) -> int: + detection, _ = detection_with_document + severity = detection.severity if detection.severity else '' + return SeverityOption.get_member_weight(severity) + + def _sort_detections_by_severity( + self, detections_with_documents: List[Tuple[Detection, Document]] + ) -> List[Tuple[Detection, Document]]: + return sorted(detections_with_documents, key=self.__severity_sort_key, reverse=True) + + @staticmethod + def __file_path_sort_key(detection_with_document: Tuple[Detection, Document]) -> str: + _, document = detection_with_document + return document.path + + def _sort_detections_by_file_path( + self, detections_with_documents: List[Tuple[Detection, Document]] + ) -> List[Tuple[Detection, Document]]: + return sorted(detections_with_documents, key=self.__file_path_sort_key) + + def _sort_and_group_detections( + self, detections_with_documents: List[Tuple[Detection, Document]] + ) -> List[Tuple[Detection, Document]]: + """Sort detections by severity and group by file name.""" + result = [] + + # we sort detections by file path to make persist output order + sorted_detections = self._sort_detections_by_file_path(detections_with_documents) + + grouped_by_file_path = defaultdict(list) + for detection, document in sorted_detections: + grouped_by_file_path[document.path].append((detection, document)) + + for file_path_group in grouped_by_file_path.values(): + result.extend(self._sort_detections_by_severity(file_path_group)) + + return result + def _get_table(self) -> Table: table = Table() - table.add(ISSUE_TYPE_COLUMN) - table.add(RULE_ID_COLUMN) - table.add(FILE_PATH_COLUMN) - table.add(LINE_NUMBER_COLUMN) - table.add(COLUMN_NUMBER_COLUMN) - table.add(SCAN_ID_COLUMN) + table.add_column(SEVERITY_COLUMN) + table.add_column(ISSUE_TYPE_COLUMN) + table.add_column(FILE_PATH_COLUMN) + table.add_column(LINE_NUMBER_COLUMN) + table.add_column(COLUMN_NUMBER_COLUMN) if self._is_git_repository(): - table.add(COMMIT_SHA_COLUMN) + table.add_column(COMMIT_SHA_COLUMN) if self.scan_type == SECRET_SCAN_TYPE: - table.add(SECRET_SHA_COLUMN) - table.add(VIOLATION_LENGTH_COLUMN) - table.add(VIOLATION_COLUMN) + table.add_column(SECRET_SHA_COLUMN) + table.add_column(VIOLATION_LENGTH_COLUMN) + table.add_column(VIOLATION_COLUMN) return table @@ -96,11 +112,11 @@ def _enrich_table_with_detection_summary_values( if self.scan_type == SECRET_SCAN_TYPE: issue_type = detection.type - table.set(ISSUE_TYPE_COLUMN, issue_type) - table.set(RULE_ID_COLUMN, detection.detection_rule_id) - table.set(FILE_PATH_COLUMN, click.format_filename(document.path)) - table.set(SECRET_SHA_COLUMN, detection.detection_details.get('sha512', '')) - table.set(COMMIT_SHA_COLUMN, detection.detection_details.get('commit_id', '')) + table.add_cell(SEVERITY_COLUMN, detection.severity, SeverityOption.get_member_color(detection.severity)) + table.add_cell(ISSUE_TYPE_COLUMN, issue_type) + table.add_file_path_cell(FILE_PATH_COLUMN, document.path) + table.add_cell(SECRET_SHA_COLUMN, detection.detection_details.get('sha512', '')) + table.add_cell(COMMIT_SHA_COLUMN, detection.detection_details.get('commit_id', '')) def _enrich_table_with_detection_code_segment_values( self, table: Table, detection: Detection, document: Document @@ -123,7 +139,7 @@ def _enrich_table_with_detection_code_segment_values( if not self.show_secret: violation = obfuscate_text(violation) - table.set(LINE_NUMBER_COLUMN, str(detection_line)) - table.set(COLUMN_NUMBER_COLUMN, str(detection_column)) - table.set(VIOLATION_LENGTH_COLUMN, f'{violation_length} chars') - table.set(VIOLATION_COLUMN, violation) + table.add_cell(LINE_NUMBER_COLUMN, str(detection_line)) + table.add_cell(COLUMN_NUMBER_COLUMN, str(detection_column)) + table.add_cell(VIOLATION_LENGTH_COLUMN, f'{violation_length} chars') + table.add_cell(VIOLATION_COLUMN, violation) diff --git a/cycode/cli/printers/tables/table_printer_base.py b/cycode/cli/printers/tables/table_printer_base.py index abbc8251..71b4f399 100644 --- a/cycode/cli/printers/tables/table_printer_base.py +++ b/cycode/cli/printers/tables/table_printer_base.py @@ -1,8 +1,8 @@ import abc from typing import TYPE_CHECKING, Dict, List, Optional -import click import typer +from rich.console import Console from cycode.cli.models import CliError, CliResult from cycode.cli.printers.printer_base import PrinterBase @@ -29,7 +29,7 @@ def print_scan_results( self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None ) -> None: if not errors and all(result.issue_detected == 0 for result in local_scan_results): - click.secho('Good job! No issues were found!!! 👏👏👏', fg=self.GREEN_COLOR_NAME) + typer.secho('Good job! No issues were found!!! 👏👏👏', fg=self.GREEN_COLOR_NAME) return self._print_results(local_scan_results) @@ -37,17 +37,17 @@ def print_scan_results( if not errors: return - click.secho( + typer.secho( 'Unfortunately, Cycode was unable to complete the full scan. ' 'Please note that not all results may be available:', fg='red', ) for scan_id, error in errors.items(): - click.echo(f'- {scan_id}: ', nl=False) + typer.echo(f'- {scan_id}: ', nl=False) self.print_error(error) def _is_git_repository(self) -> bool: - return self.ctx.obj.get('remote_url') is not None + return self.ctx.info_name in {'commit_history', 'pre_commit', 'pre_receive'} and 'remote_url' in self.ctx.obj @abc.abstractmethod def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: @@ -56,7 +56,7 @@ def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: @staticmethod def _print_table(table: 'Table') -> None: if table.get_rows(): - click.echo(table.get_table().draw()) + Console().print(table.get_table()) @staticmethod def _print_report_urls( @@ -67,9 +67,9 @@ def _print_report_urls( if not report_urls and not aggregation_report_url: return if aggregation_report_url: - click.echo(f'Report URL: {aggregation_report_url}') + typer.echo(f'Report URL: {aggregation_report_url}') return - click.echo('Report URLs:') + typer.echo('Report URLs:') for report_url in report_urls: - click.echo(f'- {report_url}') + typer.echo(f'- {report_url}') diff --git a/cycode/cli/utils/string_utils.py b/cycode/cli/utils/string_utils.py index 9dce0026..c3c0c6c6 100644 --- a/cycode/cli/utils/string_utils.py +++ b/cycode/cli/utils/string_utils.py @@ -62,6 +62,6 @@ def shortcut_dependency_paths(dependency_paths_list: str) -> str: result += dependency_paths else: result += f'{dependencies[0]} -> ... -> {dependencies[-1]}' - result += '\n\n' + result += '\n' return result.rstrip().rstrip(',') diff --git a/poetry.lock b/poetry.lock index f104cc28..186fee3f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. [[package]] name = "altgraph" @@ -980,18 +980,6 @@ files = [ {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, ] -[[package]] -name = "texttable" -version = "1.7.0" -description = "module to create simple ASCII tables" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "texttable-1.7.0-py2.py3-none-any.whl", hash = "sha256:72227d592c82b3d7f672731ae73e4d1f88cd8e2ef5b075a7a7f01a23a3743917"}, - {file = "texttable-1.7.0.tar.gz", hash = "sha256:2d2068fb55115807d3ac77a4ca68fa48803e84ebb0ee2340f858107a36522638"}, -] - [[package]] name = "tomli" version = "2.2.1" @@ -1130,4 +1118,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<3.14" -content-hash = "c0140dc408f1e3827b51357d74b05274297c233de11dbca85d4b6f3a909f4191" +content-hash = "2c45abb3ea36096f8dbe862ba8bf5e064994ed9cdf86a8256643b9822c40168f" diff --git a/pyproject.toml b/pyproject.toml index 6baffad9..ef555019 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,6 @@ marshmallow = ">=3.15.0,<3.23.0" # 3.23 dropped support for Python 3.8 gitpython = ">=3.1.30,<3.2.0" arrow = ">=1.0.0,<1.4.0" binaryornot = ">=0.4.4,<0.5.0" -texttable = ">=1.6.7,<1.8.0" requests = ">=2.32.2,<3.0" urllib3 = "1.26.19" # lock v1 to avoid issues with openssl and old Python versions (<3.9.11) on macOS sentry-sdk = ">=2.8.0,<3.0" diff --git a/tests/utils/test_string_utils.py b/tests/utils/test_string_utils.py index 60d10efa..8c94fceb 100644 --- a/tests/utils/test_string_utils.py +++ b/tests/utils/test_string_utils.py @@ -3,5 +3,5 @@ def test_shortcut_dependency_paths_list_single_dependencies() -> None: dependency_paths = 'A, A -> B, A -> B -> C' - expected_result = 'A\n\nA -> B\n\nA -> ... -> C' + expected_result = 'A\nA -> B\nA -> ... -> C' assert shortcut_dependency_paths(dependency_paths) == expected_result From 72e8b777ab8d4a34a71960c35af808a243b0978c Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Mon, 24 Mar 2025 13:40:59 +0100 Subject: [PATCH 05/19] CM-46137 - Add visual separators of row groups; reorder columns (#289) --- cycode/cli/apps/scan/code_scanner.py | 124 +++++++++--------- .../scan/pre_commit/pre_commit_command.py | 4 +- .../scan/repository/repository_command.py | 3 +- .../cli/printers/tables/sca_table_printer.py | 17 ++- cycode/cli/printers/tables/table.py | 11 +- cycode/cli/printers/tables/table_printer.py | 25 ++-- tests/test_code_scanner.py | 4 +- 7 files changed, 103 insertions(+), 85 deletions(-) diff --git a/cycode/cli/apps/scan/code_scanner.py b/cycode/cli/apps/scan/code_scanner.py index a1a5d440..a3b1201e 100644 --- a/cycode/cli/apps/scan/code_scanner.py +++ b/cycode/cli/apps/scan/code_scanner.py @@ -49,7 +49,7 @@ def scan_sca_pre_commit(ctx: typer.Context) -> None: scan_type = ctx.obj['scan_type'] - scan_parameters = get_default_scan_parameters(ctx) + scan_parameters = get_scan_parameters(ctx) git_head_documents, pre_committed_documents = get_pre_commit_modified_documents( ctx.obj['progress_bar'], ScanProgressBarSection.PREPARE_LOCAL_FILES ) @@ -83,15 +83,14 @@ def scan_sca_commit_range(ctx: typer.Context, path: str, commit_range: str) -> N scan_commit_range_documents(ctx, from_commit_documents, to_commit_documents, scan_parameters=scan_parameters) -def scan_disk_files(ctx: typer.Context, paths: Tuple[str, ...]) -> None: - scan_parameters = get_scan_parameters(ctx, paths) +def scan_disk_files(ctx: click.Context, paths: Tuple[str]) -> None: scan_type = ctx.obj['scan_type'] progress_bar = ctx.obj['progress_bar'] try: documents = get_relevant_documents(progress_bar, ScanProgressBarSection.PREPARE_LOCAL_FILES, scan_type, paths) perform_pre_scan_documents_actions(ctx, scan_type, documents) - scan_documents(ctx, documents, scan_parameters=scan_parameters) + scan_documents(ctx, documents, get_scan_parameters(ctx, paths)) except Exception as e: handle_scan_exception(ctx, e) @@ -155,14 +154,12 @@ def _enrich_scan_result_with_data_from_detection_rules( def _get_scan_documents_thread_func( ctx: typer.Context, is_git_diff: bool, is_commit_range: bool, scan_parameters: dict -) -> Tuple[Callable[[List[Document]], Tuple[str, CliError, LocalScanResult]], str]: +) -> Callable[[List[Document]], Tuple[str, CliError, LocalScanResult]]: cycode_client = ctx.obj['client'] scan_type = ctx.obj['scan_type'] severity_threshold = ctx.obj['severity_threshold'] sync_option = ctx.obj['sync'] command_scan_type = ctx.info_name - aggregation_id = str(_generate_unique_id()) - scan_parameters['aggregation_id'] = aggregation_id def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, LocalScanResult]: local_scan_result = error = error_message = None @@ -231,7 +228,7 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local return scan_id, error, local_scan_result - return _scan_batch_thread_func, aggregation_id + return _scan_batch_thread_func def scan_commit_range( @@ -291,20 +288,17 @@ def scan_commit_range( logger.debug('List of commit ids to scan, %s', {'commit_ids': commit_ids_to_scan}) logger.debug('Starting to scan commit range (it may take a few minutes)') - scan_documents(ctx, documents_to_scan, is_git_diff=True, is_commit_range=True) + scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx, (path,)), is_git_diff=True, is_commit_range=True) return None def scan_documents( ctx: typer.Context, documents_to_scan: List[Document], + scan_parameters: dict, is_git_diff: bool = False, is_commit_range: bool = False, - scan_parameters: Optional[dict] = None, ) -> None: - if not scan_parameters: - scan_parameters = get_default_scan_parameters(ctx) - scan_type = ctx.obj['scan_type'] progress_bar = ctx.obj['progress_bar'] @@ -319,19 +313,13 @@ def scan_documents( ) return - scan_batch_thread_func, aggregation_id = _get_scan_documents_thread_func( - ctx, is_git_diff, is_commit_range, scan_parameters - ) + scan_batch_thread_func = _get_scan_documents_thread_func(ctx, is_git_diff, is_commit_range, scan_parameters) errors, local_scan_results = run_parallel_batched_scan( scan_batch_thread_func, scan_type, documents_to_scan, progress_bar=progress_bar ) - if len(local_scan_results) > 1: - # if we used more than one batch, we need to fetch aggregate report url - aggregation_report_url = _try_get_aggregation_report_url_if_needed( - scan_parameters, ctx.obj['client'], scan_type - ) - set_aggregation_report_url(ctx, aggregation_report_url) + aggregation_report_url = _try_get_aggregation_report_url_if_needed(scan_parameters, ctx.obj['client'], scan_type) + _set_aggregation_report_url(ctx, aggregation_report_url) progress_bar.set_section_length(ScanProgressBarSection.GENERATE_REPORT, 1) progress_bar.update(ScanProgressBarSection.GENERATE_REPORT) @@ -341,25 +329,6 @@ def scan_documents( print_results(ctx, local_scan_results, errors) -def set_aggregation_report_url(ctx: typer.Context, aggregation_report_url: Optional[str] = None) -> None: - ctx.obj['aggregation_report_url'] = aggregation_report_url - - -def _try_get_aggregation_report_url_if_needed( - scan_parameters: dict, cycode_client: 'ScanClient', scan_type: str -) -> Optional[str]: - aggregation_id = scan_parameters.get('aggregation_id') - if not scan_parameters.get('report'): - return None - if aggregation_id is None: - return None - try: - report_url_response = cycode_client.get_scan_aggregation_report_url(aggregation_id, scan_type) - return report_url_response.report_url - except Exception as e: - logger.debug('Failed to get aggregation report url: %s', str(e)) - - def scan_commit_range_documents( ctx: typer.Context, from_documents_to_scan: List[Document], @@ -384,7 +353,7 @@ def scan_commit_range_documents( try: progress_bar.set_section_length(ScanProgressBarSection.SCAN, 1) - scan_result = init_default_scan_result(cycode_client, scan_id, scan_type) + scan_result = init_default_scan_result(scan_id) if should_scan_documents(from_documents_to_scan, to_documents_to_scan): logger.debug('Preparing from-commit zip') from_commit_zipped_documents = zip_documents(scan_type, from_documents_to_scan) @@ -522,7 +491,7 @@ def perform_scan_async( cycode_client, scan_async_result.scan_id, scan_type, - scan_parameters.get('report'), + scan_parameters, ) @@ -557,16 +526,14 @@ def perform_commit_range_scan_async( logger.debug( 'Async commit range scan request has been triggered successfully, %s', {'scan_id': scan_async_result.scan_id} ) - return poll_scan_results( - cycode_client, scan_async_result.scan_id, scan_type, scan_parameters.get('report'), timeout - ) + return poll_scan_results(cycode_client, scan_async_result.scan_id, scan_type, scan_parameters, timeout) def poll_scan_results( cycode_client: 'ScanClient', scan_id: str, scan_type: str, - should_get_report: bool = False, + scan_parameters: dict, polling_timeout: Optional[int] = None, ) -> ZippedFileScanResult: if polling_timeout is None: @@ -583,7 +550,7 @@ def poll_scan_results( print_debug_scan_details(scan_details) if scan_details.scan_status == consts.SCAN_STATUS_COMPLETED: - return _get_scan_result(cycode_client, scan_type, scan_id, scan_details, should_get_report) + return _get_scan_result(cycode_client, scan_type, scan_id, scan_details, scan_parameters) if scan_details.scan_status == consts.SCAN_STATUS_ERROR: raise custom_exceptions.ScanAsyncError( @@ -675,18 +642,19 @@ def parse_pre_receive_input() -> str: return pre_receive_input.splitlines()[0] -def get_default_scan_parameters(ctx: typer.Context) -> dict: +def _get_default_scan_parameters(ctx: click.Context) -> dict: return { 'monitor': ctx.obj.get('monitor'), 'report': ctx.obj.get('report'), 'package_vulnerabilities': ctx.obj.get('package-vulnerabilities'), 'license_compliance': ctx.obj.get('license-compliance'), 'command_type': ctx.info_name, + 'aggregation_id': str(_generate_unique_id()), } -def get_scan_parameters(ctx: typer.Context, paths: Tuple[str, ...]) -> dict: - scan_parameters = get_default_scan_parameters(ctx) +def get_scan_parameters(ctx: typer.Context, paths: Optional[Tuple[str]] = None) -> dict: + scan_parameters = _get_default_scan_parameters(ctx) if not paths: return scan_parameters @@ -894,10 +862,10 @@ def _get_scan_result( scan_type: str, scan_id: str, scan_details: 'ScanDetailsResponse', - should_get_report: bool = False, + scan_parameters: dict, ) -> ZippedFileScanResult: if not scan_details.detections_count: - return init_default_scan_result(cycode_client, scan_id, scan_type, should_get_report) + return init_default_scan_result(scan_id) scan_raw_detections = cycode_client.get_scan_raw_detections(scan_type, scan_id) @@ -905,25 +873,40 @@ def _get_scan_result( did_detect=True, detections_per_file=_map_detections_per_file_and_commit_id(scan_type, scan_raw_detections), scan_id=scan_id, - report_url=_try_get_report_url_if_needed(cycode_client, should_get_report, scan_id, scan_type), + report_url=_try_get_any_report_url_if_needed(cycode_client, scan_id, scan_type, scan_parameters), ) -def init_default_scan_result( - cycode_client: 'ScanClient', scan_id: str, scan_type: str, should_get_report: bool = False -) -> ZippedFileScanResult: +def init_default_scan_result(scan_id: str) -> ZippedFileScanResult: return ZippedFileScanResult( did_detect=False, detections_per_file=[], scan_id=scan_id, - report_url=_try_get_report_url_if_needed(cycode_client, should_get_report, scan_id, scan_type), ) +def _try_get_any_report_url_if_needed( + cycode_client: 'ScanClient', + scan_id: str, + scan_type: str, + scan_parameters: dict, +) -> Optional[str]: + """Tries to get aggregation report URL if needed, otherwise tries to get report URL.""" + aggregation_report_url = None + if scan_parameters: + _try_get_report_url_if_needed(cycode_client, scan_id, scan_type, scan_parameters) + aggregation_report_url = _try_get_aggregation_report_url_if_needed(scan_parameters, cycode_client, scan_type) + + if aggregation_report_url: + return aggregation_report_url + + return _try_get_report_url_if_needed(cycode_client, scan_id, scan_type, scan_parameters) + + def _try_get_report_url_if_needed( - cycode_client: 'ScanClient', should_get_report: bool, scan_id: str, scan_type: str + cycode_client: 'ScanClient', scan_id: str, scan_type: str, scan_parameters: dict ) -> Optional[str]: - if not should_get_report: + if not scan_parameters.get('report', False): return None try: @@ -933,6 +916,27 @@ def _try_get_report_url_if_needed( logger.debug('Failed to get report URL', exc_info=e) +def _set_aggregation_report_url(ctx: click.Context, aggregation_report_url: Optional[str] = None) -> None: + ctx.obj['aggregation_report_url'] = aggregation_report_url + + +def _try_get_aggregation_report_url_if_needed( + scan_parameters: dict, cycode_client: 'ScanClient', scan_type: str +) -> Optional[str]: + if not scan_parameters.get('report', False): + return None + + aggregation_id = scan_parameters.get('aggregation_id') + if aggregation_id is None: + return None + + try: + report_url_response = cycode_client.get_scan_aggregation_report_url(aggregation_id, scan_type) + return report_url_response.report_url + except Exception as e: + logger.debug('Failed to get aggregation report url: %s', str(e)) + + def _map_detections_per_file_and_commit_id(scan_type: str, raw_detections: List[dict]) -> List[DetectionsPerFile]: """Converts list of detections (async flow) to list of DetectionsPerFile objects (sync flow). diff --git a/cycode/cli/apps/scan/pre_commit/pre_commit_command.py b/cycode/cli/apps/scan/pre_commit/pre_commit_command.py index d88db8cc..8e528d15 100644 --- a/cycode/cli/apps/scan/pre_commit/pre_commit_command.py +++ b/cycode/cli/apps/scan/pre_commit/pre_commit_command.py @@ -4,7 +4,7 @@ import typer from cycode.cli import consts -from cycode.cli.apps.scan.code_scanner import scan_documents, scan_sca_pre_commit +from cycode.cli.apps.scan.code_scanner import get_scan_parameters, scan_documents, scan_sca_pre_commit from cycode.cli.files_collector.excluder import exclude_irrelevant_documents_to_scan from cycode.cli.files_collector.repository_documents import ( get_diff_file_content, @@ -44,4 +44,4 @@ def pre_commit_command( documents_to_scan.append(Document(get_path_by_os(get_diff_file_path(file)), get_diff_file_content(file))) documents_to_scan = exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan) - scan_documents(ctx, documents_to_scan, is_git_diff=True) + scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx), is_git_diff=True) diff --git a/cycode/cli/apps/scan/repository/repository_command.py b/cycode/cli/apps/scan/repository/repository_command.py index 045448e6..7d6b421a 100644 --- a/cycode/cli/apps/scan/repository/repository_command.py +++ b/cycode/cli/apps/scan/repository/repository_command.py @@ -63,7 +63,6 @@ def repository_command( perform_pre_scan_documents_actions(ctx, scan_type, documents_to_scan) logger.debug('Found all relevant files for scanning %s', {'path': path, 'branch': branch}) - scan_parameters = get_scan_parameters(ctx, (str(path),)) - scan_documents(ctx, documents_to_scan, scan_parameters=scan_parameters) + scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx, (path,))) except Exception as e: handle_scan_exception(ctx, e) diff --git a/cycode/cli/printers/tables/sca_table_printer.py b/cycode/cli/printers/tables/sca_table_printer.py index b59a33ef..b77f55ed 100644 --- a/cycode/cli/printers/tables/sca_table_printer.py +++ b/cycode/cli/printers/tables/sca_table_printer.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import TYPE_CHECKING, Dict, List +from typing import TYPE_CHECKING, Dict, List, Set, Tuple import typer @@ -37,9 +37,12 @@ def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: for policy_id, detections in detections_per_policy_id.items(): table = self._get_table(policy_id) - for detection in self._sort_and_group_detections(detections): + resulting_detections, group_separator_indexes = self._sort_and_group_detections(detections) + for detection in resulting_detections: self._enrich_table_with_values(policy_id, table, detection) + table.set_group_separator_indexes(group_separator_indexes) + self._print_summary_issues(len(detections), self._get_title(policy_id)) self._print_table(table) @@ -76,7 +79,7 @@ def __package_sort_key(detection: Detection) -> int: def _sort_detections_by_package(self, detections: List[Detection]) -> List[Detection]: return sorted(detections, key=self.__package_sort_key) - def _sort_and_group_detections(self, detections: List[Detection]) -> List[Detection]: + def _sort_and_group_detections(self, detections: List[Detection]) -> Tuple[List[Detection], Set[int]]: """Sort detections by severity and group by repository, code project and package name. Note: @@ -85,7 +88,8 @@ def _sort_and_group_detections(self, detections: List[Detection]) -> List[Detect Grouping by code projects also groups by ecosystem. Because manifest files are unique per ecosystem. """ - result = [] + resulting_detections = [] + group_separator_indexes = set() # we sort detections by package name to make persist output order sorted_detections = self._sort_detections_by_package(detections) @@ -96,9 +100,10 @@ def _sort_and_group_detections(self, detections: List[Detection]) -> List[Detect for code_project_group in grouped_by_code_project.values(): grouped_by_package = self.__group_by(code_project_group, 'package_name') for package_group in grouped_by_package.values(): - result.extend(self._sort_detections_by_severity(package_group)) + group_separator_indexes.add(len(resulting_detections) - 1) # indexing starts from 0 + resulting_detections.extend(self._sort_detections_by_severity(package_group)) - return result + return resulting_detections, group_separator_indexes def _get_table(self, policy_id: str) -> Table: table = Table() diff --git a/cycode/cli/printers/tables/table.py b/cycode/cli/printers/tables/table.py index a071c9b4..23022b2d 100644 --- a/cycode/cli/printers/tables/table.py +++ b/cycode/cli/printers/tables/table.py @@ -1,5 +1,5 @@ import urllib.parse -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING, Dict, List, Optional, Set from rich.markup import escape from rich.table import Table as RichTable @@ -12,6 +12,8 @@ class Table: """Helper class to manage columns and their values in the right order and only if the column should be presented.""" def __init__(self, column_infos: Optional[List['ColumnInfo']] = None) -> None: + self._group_separator_indexes: Set[int] = set() + self._columns: Dict['ColumnInfo', List[str]] = {} if column_infos: self._columns = {columns: [] for columns in column_infos} @@ -35,6 +37,9 @@ def add_file_path_cell(self, column: 'ColumnInfo', path: str) -> None: escaped_path = escape(encoded_path) self._add_cell_no_error(column, f'[link file://{escaped_path}]{path}') + def set_group_separator_indexes(self, group_separator_indexes: Set[int]) -> None: + self._group_separator_indexes = group_separator_indexes + def _get_ordered_columns(self) -> List['ColumnInfo']: # we are sorting columns by index to make sure that columns will be printed in the right order return sorted(self._columns, key=lambda column_info: column_info.index) @@ -53,7 +58,7 @@ def get_table(self) -> 'RichTable': extra_args = column.column_opts if column.column_opts else {} table.add_column(header=column.name, overflow='fold', **extra_args) - for raw in self.get_rows(): - table.add_row(*raw) + for index, raw in enumerate(self.get_rows()): + table.add_row(*raw, end_section=index in self._group_separator_indexes) return table diff --git a/cycode/cli/printers/tables/table_printer.py b/cycode/cli/printers/tables/table_printer.py index 728e2baf..853e2465 100644 --- a/cycode/cli/printers/tables/table_printer.py +++ b/cycode/cli/printers/tables/table_printer.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import TYPE_CHECKING, List, Tuple +from typing import TYPE_CHECKING, List, Set, Tuple from cycode.cli.cli_types import SeverityOption from cycode.cli.consts import SECRET_SCAN_TYPE @@ -18,12 +18,12 @@ SEVERITY_COLUMN = column_builder.build(name='Severity') ISSUE_TYPE_COLUMN = column_builder.build(name='Issue Type') FILE_PATH_COLUMN = column_builder.build(name='File Path', highlight=False) +LINE_NUMBER_COLUMN = column_builder.build(name='Line') +COLUMN_NUMBER_COLUMN = column_builder.build(name='Column') +VIOLATION_COLUMN = column_builder.build(name='Violation', highlight=False) +VIOLATION_LENGTH_COLUMN = column_builder.build(name='Length') SECRET_SHA_COLUMN = column_builder.build(name='Secret SHA') COMMIT_SHA_COLUMN = column_builder.build(name='Commit SHA') -LINE_NUMBER_COLUMN = column_builder.build(name='Line Number') -COLUMN_NUMBER_COLUMN = column_builder.build(name='Column Number') -VIOLATION_LENGTH_COLUMN = column_builder.build(name='Violation Length') -VIOLATION_COLUMN = column_builder.build(name='Violation', highlight=False) class TablePrinter(TablePrinterBase): @@ -37,9 +37,12 @@ def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: [(detection, document_detections.document) for detection in document_detections.detections] ) - for detection, document in self._sort_and_group_detections(detections_with_documents): + detections, group_separator_indexes = self._sort_and_group_detections(detections_with_documents) + for detection, document in detections: self._enrich_table_with_values(table, detection, document) + table.set_group_separator_indexes(group_separator_indexes) + self._print_table(table) self._print_report_urls(local_scan_results, self.ctx.obj.get('aggregation_report_url')) @@ -66,9 +69,10 @@ def _sort_detections_by_file_path( def _sort_and_group_detections( self, detections_with_documents: List[Tuple[Detection, Document]] - ) -> List[Tuple[Detection, Document]]: + ) -> Tuple[List[Tuple[Detection, Document]], Set[int]]: """Sort detections by severity and group by file name.""" - result = [] + detections = [] + group_separator_indexes = set() # we sort detections by file path to make persist output order sorted_detections = self._sort_detections_by_file_path(detections_with_documents) @@ -78,9 +82,10 @@ def _sort_and_group_detections( grouped_by_file_path[document.path].append((detection, document)) for file_path_group in grouped_by_file_path.values(): - result.extend(self._sort_detections_by_severity(file_path_group)) + group_separator_indexes.add(len(detections) - 1) # indexing starts from 0 + detections.extend(self._sort_detections_by_severity(file_path_group)) - return result + return detections, group_separator_indexes def _get_table(self) -> Table: table = Table() diff --git a/tests/test_code_scanner.py b/tests/test_code_scanner.py index 709fe70e..d16aad82 100644 --- a/tests/test_code_scanner.py +++ b/tests/test_code_scanner.py @@ -29,7 +29,7 @@ def test_is_relevant_file_to_scan_sca() -> None: @pytest.mark.parametrize('scan_type', list(ScanTypeOption)) def test_try_get_report_url_if_needed_return_none(scan_type: ScanTypeOption, scan_client: ScanClient) -> None: scan_id = uuid4().hex - result = _try_get_report_url_if_needed(scan_client, False, scan_id, consts.SECRET_SCAN_TYPE) + result = _try_get_report_url_if_needed(scan_client, scan_id, consts.SECRET_SCAN_TYPE, scan_parameters={}) assert result is None @@ -44,7 +44,7 @@ def test_try_get_report_url_if_needed_return_result( responses.add(get_scan_report_url_response(url, scan_id)) scan_report_url_response = scan_client.get_scan_report_url(str(scan_id), scan_type) - result = _try_get_report_url_if_needed(scan_client, True, str(scan_id), scan_type) + result = _try_get_report_url_if_needed(scan_client, str(scan_id), scan_type, scan_parameters={'report': True}) assert result == scan_report_url_response.report_url From f4ae0fae56f9539d21f9eeb47cd3aff04a655a74 Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Wed, 26 Mar 2025 12:14:44 +0100 Subject: [PATCH 06/19] CM-45719 - Add syntax highlight for code snippets in text output (#290) --- cycode/cli/printers/printer_base.py | 1 - cycode/cli/printers/text_printer.py | 222 ++++++++++-------------- cycode/cli/utils/progress_bar.py | 7 +- tests/cli/commands/test_main_command.py | 2 +- 4 files changed, 90 insertions(+), 142 deletions(-) diff --git a/cycode/cli/printers/printer_base.py b/cycode/cli/printers/printer_base.py index ee9a7793..633a2ccc 100644 --- a/cycode/cli/printers/printer_base.py +++ b/cycode/cli/printers/printer_base.py @@ -17,7 +17,6 @@ class PrinterBase(ABC): RED_COLOR_NAME = 'red' - WHITE_COLOR_NAME = 'white' GREEN_COLOR_NAME = 'green' def __init__(self, ctx: typer.Context) -> None: diff --git a/cycode/cli/printers/text_printer.py b/cycode/cli/printers/text_printer.py index 7828d909..73eaccc2 100644 --- a/cycode/cli/printers/text_printer.py +++ b/cycode/cli/printers/text_printer.py @@ -1,9 +1,13 @@ import math +import urllib.parse from typing import TYPE_CHECKING, Dict, List, Optional -import click import typer +from rich.console import Console +from rich.markup import escape +from rich.syntax import Syntax +from cycode.cli.cli_types import SeverityOption from cycode.cli.consts import COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES, SECRET_SCAN_TYPE from cycode.cli.models import CliError, CliResult, Detection, Document, DocumentDetections from cycode.cli.printers.printer_base import PrinterBase @@ -25,28 +29,28 @@ def print_result(self, result: CliResult) -> None: if not result.success: color = self.RED_COLOR_NAME - click.secho(result.message, fg=color) + typer.secho(result.message, fg=color) if not result.data: return - click.secho('\nAdditional data:', fg=color) + typer.secho('\nAdditional data:', fg=color) for name, value in result.data.items(): - click.secho(f'- {name}: {value}', fg=color) + typer.secho(f'- {name}: {value}', fg=color) def print_error(self, error: CliError) -> None: - click.secho(error.message, fg=self.RED_COLOR_NAME) + typer.secho(error.message, fg=self.RED_COLOR_NAME) def print_scan_results( self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None ) -> None: if not errors and all(result.issue_detected == 0 for result in local_scan_results): - click.secho('Good job! No issues were found!!! 👏👏👏', fg=self.GREEN_COLOR_NAME) + typer.secho('Good job! No issues were found!!! 👏👏👏', fg=self.GREEN_COLOR_NAME) return for local_scan_result in local_scan_results: for document_detections in local_scan_result.document_detections: - self._print_document_detections(document_detections, local_scan_result.scan_id) + self._print_document_detections(document_detections) report_urls = [scan_result.report_url for scan_result in local_scan_results if scan_result.report_url] @@ -54,44 +58,51 @@ def print_scan_results( if not errors: return - click.secho( + typer.secho( 'Unfortunately, Cycode was unable to complete the full scan. ' 'Please note that not all results may be available:', fg='red', ) for scan_id, error in errors.items(): - click.echo(f'- {scan_id}: ', nl=False) + typer.echo(f'- {scan_id}: ', nl=False) self.print_error(error) - def _print_document_detections(self, document_detections: DocumentDetections, scan_id: str) -> None: + def _print_document_detections(self, document_detections: DocumentDetections) -> None: document = document_detections.document for detection in document_detections.detections: - self._print_detection_summary(detection, document.path, scan_id) + self._print_detection_summary(detection, document.path) + self._print_new_line() self._print_detection_code_segment(detection, document) + self._print_new_line() - def _print_detection_summary(self, detection: Detection, document_path: str, scan_id: str) -> None: + @staticmethod + def _print_new_line() -> None: + typer.echo() + + def _print_detection_summary(self, detection: Detection, document_path: str) -> None: detection_name = detection.type if self.scan_type == SECRET_SCAN_TYPE else detection.message - detection_name_styled = click.style(detection_name, fg='bright_red', bold=True) - detection_sha = detection.detection_details.get('sha512') - detection_sha_message = f'\nSecret SHA: {detection_sha}' if detection_sha else '' + detection_severity = detection.severity or 'N/A' + detection_severity_color = SeverityOption.get_member_color(detection_severity) + detection_severity = f'[{detection_severity_color}]{detection_severity.upper()}[/{detection_severity_color}]' + + escaped_document_path = escape(urllib.parse.quote(document_path)) + clickable_document_path = f'[link file://{escaped_document_path}]{document_path}' - scan_id_message = f'\nScan ID: {scan_id}' detection_commit_id = detection.detection_details.get('commit_id') detection_commit_id_message = f'\nCommit SHA: {detection_commit_id}' if detection_commit_id else '' company_guidelines = detection.detection_details.get('custom_remediation_guidelines') company_guidelines_message = f'\nCompany Guideline: {company_guidelines}' if company_guidelines else '' - click.echo( - f'⛔ ' - f'Found issue of type: {detection_name_styled} ' - f'(rule ID: {detection.detection_rule_id}) in file: {click.format_filename(document_path)} ' - f'{detection_sha_message}' - f'{scan_id_message}' + Console().print( + f':no_entry: ' + f'Found {detection_severity} issue of type: [bright_red][bold]{detection_name}[/bold][/bright_red] ' + f'in file: {clickable_document_path} ' f'{detection_commit_id_message}' f'{company_guidelines_message}' - f' ⛔' + f' :no_entry:', + highlight=True, ) def _print_detection_code_segment( @@ -109,145 +120,86 @@ def _print_report_urls(report_urls: List[str], aggregation_report_url: Optional[ if not report_urls and not aggregation_report_url: return if aggregation_report_url: - click.echo(f'Report URL: {aggregation_report_url}') + typer.echo(f'Report URL: {aggregation_report_url}') return - click.echo('Report URLs:') + typer.echo('Report URLs:') for report_url in report_urls: - click.echo(f'- {report_url}') + typer.echo(f'- {report_url}') @staticmethod def _get_code_segment_start_line(detection_line: int, lines_to_display: int) -> int: start_line = detection_line - math.ceil(lines_to_display / 2) return 0 if start_line < 0 else start_line - def _print_line_of_code_segment( - self, - document: Document, - line: str, - line_number: int, - detection_position_in_line: int, - violation_length: int, - is_detection_line: bool, - ) -> None: - if is_detection_line: - self._print_detection_line(document, line, line_number, detection_position_in_line, violation_length) - else: - self._print_line(document, line, line_number) - - def _print_detection_line( - self, document: Document, line: str, line_number: int, detection_position_in_line: int, violation_length: int - ) -> None: - detection_line = self._get_detection_line_style( - line, document.is_git_diff_format, detection_position_in_line, violation_length - ) - - click.echo(f'{self._get_line_number_style(line_number)} {detection_line}') - - def _print_line(self, document: Document, line: str, line_number: int) -> None: - line_no = self._get_line_number_style(line_number) - line = self._get_line_style(line, document.is_git_diff_format) - - click.echo(f'{line_no} {line}') - - def _get_detection_line_style(self, line: str, is_git_diff: bool, start_position: int, length: int) -> str: - line_color = self._get_line_color(line, is_git_diff) - if self.scan_type != SECRET_SCAN_TYPE or start_position < 0 or length < 0: - return self._get_line_style(line, is_git_diff, line_color) - - violation = line[start_position : start_position + length] - if not self.show_secret: - violation = obfuscate_text(violation) - - line_to_violation = line[0:start_position] - line_from_violation = line[start_position + length :] - - return ( - f'{self._get_line_style(line_to_violation, is_git_diff, line_color)}' - f'{self._get_line_style(violation, is_git_diff, line_color, underline=True)}' - f'{self._get_line_style(line_from_violation, is_git_diff, line_color)}' - ) - - def _get_line_style( - self, line: str, is_git_diff: bool, color: Optional[str] = None, underline: bool = False - ) -> str: - if color is None: - color = self._get_line_color(line, is_git_diff) - - return click.style(line, fg=color, bold=False, underline=underline) - - def _get_line_color(self, line: str, is_git_diff: bool) -> str: - if not is_git_diff: - return self.WHITE_COLOR_NAME - - if line.startswith('+'): - return self.GREEN_COLOR_NAME - - if line.startswith('-'): - return self.RED_COLOR_NAME - - return self.WHITE_COLOR_NAME - - def _get_line_number_style(self, line_number: int) -> str: + def _get_detection_line(self, detection: Detection) -> int: return ( - f'{click.style(str(line_number), fg=self.WHITE_COLOR_NAME, bold=False)} ' - f'{click.style("|", fg=self.RED_COLOR_NAME, bold=False)}' + detection.detection_details.get('line', -1) + if self.scan_type == SECRET_SCAN_TYPE + else detection.detection_details.get('line_in_file', -1) - 1 ) def _print_detection_from_file(self, detection: Detection, document: Document, lines_to_display: int) -> None: detection_details = detection.detection_details - detection_line = ( - detection_details.get('line', -1) - if self.scan_type == SECRET_SCAN_TYPE - else detection_details.get('line_in_file', -1) - ) - detection_position = detection_details.get('start_position', -1) + detection_line = self._get_detection_line(detection) + start_line_index = self._get_code_segment_start_line(detection_line, lines_to_display) + detection_position = get_position_in_line(document.content, detection_details.get('start_position', -1)) violation_length = detection_details.get('length', -1) - file_content = document.content - file_lines = file_content.splitlines() - start_line = self._get_code_segment_start_line(detection_line, lines_to_display) - detection_position_in_line = get_position_in_line(file_content, detection_position) - - click.echo() + code_lines_to_render = [] + document_content_lines = document.content.splitlines() for line_index in range(lines_to_display): - current_line_index = start_line + line_index - if current_line_index >= len(file_lines): + current_line_index = start_line_index + line_index + if current_line_index >= len(document_content_lines): break - current_line = file_lines[current_line_index] - is_detection_line = current_line_index == detection_line - self._print_line_of_code_segment( - document, - current_line, - current_line_index + 1, - detection_position_in_line, - violation_length, - is_detection_line, + line_content = document_content_lines[current_line_index] + + line_with_detection = current_line_index == detection_line + if self.scan_type == SECRET_SCAN_TYPE and line_with_detection and not self.show_secret: + violation = line_content[detection_position : detection_position + violation_length] + code_lines_to_render.append(line_content.replace(violation, obfuscate_text(violation))) + else: + code_lines_to_render.append(line_content) + + code_to_render = '\n'.join(code_lines_to_render) + Console().print( + Syntax( + code=code_to_render, + lexer=Syntax.guess_lexer(document.path, code=code_to_render), + line_numbers=True, + dedent=True, + tab_size=2, + start_line=start_line_index + 1, + highlight_lines={ + detection_line + 1, + }, ) - click.echo() + ) def _print_detection_from_git_diff(self, detection: Detection, document: Document) -> None: detection_details = detection.detection_details - detection_line_number = detection_details.get('line', -1) - detection_line_number_in_original_file = detection_details.get('line_in_file', -1) + detection_line = self._get_detection_line(detection) detection_position = detection_details.get('start_position', -1) violation_length = detection_details.get('length', -1) - git_diff_content = document.content - git_diff_lines = git_diff_content.splitlines() - detection_line = git_diff_lines[detection_line_number] - detection_position_in_line = get_position_in_line(git_diff_content, detection_position) - - click.echo() - self._print_detection_line( - document, - detection_line, - detection_line_number_in_original_file, - detection_position_in_line, - violation_length, + line_content = document.content.splitlines()[detection_line] + detection_position_in_line = get_position_in_line(document.content, detection_position) + if self.scan_type == SECRET_SCAN_TYPE and not self.show_secret: + violation = line_content[detection_position_in_line : detection_position_in_line + violation_length] + line_content = line_content.replace(violation, obfuscate_text(violation)) + + Console().print( + Syntax( + line_content, + lexer='diff', + line_numbers=True, + start_line=detection_line, + dedent=True, + tab_size=2, + highlight_lines={detection_line + 1}, + ) ) - click.echo() def _is_git_diff_based_scan(self) -> bool: return self.command_scan_type in COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES and self.scan_type == SECRET_SCAN_TYPE diff --git a/cycode/cli/utils/progress_bar.py b/cycode/cli/utils/progress_bar.py index 90a19801..e0fec5aa 100644 --- a/cycode/cli/utils/progress_bar.py +++ b/cycode/cli/utils/progress_bar.py @@ -145,10 +145,7 @@ def __init__(self, progress_bar_sections: ProgressBarSections) -> None: self._current_section: ProgressBarSectionInfo = _get_initial_section(self._progress_bar_sections) self._current_right_side_label = '' - self._progress_bar = Progress( - *_PROGRESS_BAR_COLUMNS, - transient=True, - ) + self._progress_bar = Progress(*_PROGRESS_BAR_COLUMNS) self._progress_bar_task_id = self._progress_bar.add_task( description=self._current_section.label, total=_PROGRESS_BAR_LENGTH, @@ -245,7 +242,7 @@ def update(self, section: 'ProgressBarSection', value: int = 1) -> None: self._maybe_update_current_section() def update_right_side_label(self, label: Optional[str] = None) -> None: - self._current_right_side_label = f'({label})' or '' + self._current_right_side_label = f'({label})' if label else '' self._progress_bar_update() diff --git a/tests/cli/commands/test_main_command.py b/tests/cli/commands/test_main_command.py index d7575ddb..04bc3e01 100644 --- a/tests/cli/commands/test_main_command.py +++ b/tests/cli/commands/test_main_command.py @@ -49,7 +49,7 @@ def test_passing_output_option(output: str, scan_client: 'ScanClient', api_token output = json.loads(result.output) assert 'scan_id' in output else: - assert 'Scan ID' in result.output + assert 'issue of type:' in result.output @responses.activate From ba16609d2384cb5d91009fb2b857707007108eb3 Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Tue, 1 Apr 2025 15:49:26 +0200 Subject: [PATCH 07/19] CM-46371 - Add retry behavior for HTTP requests (#291) --- cycode/cli/apps/auth/auth_manager.py | 14 +------ cycode/cyclient/auth_client.py | 9 ++++- cycode/cyclient/config.py | 8 ++++ cycode/cyclient/cycode_client_base.py | 54 ++++++++++++++++++++++++++- poetry.lock | 18 ++++++++- pyproject.toml | 1 + 6 files changed, 88 insertions(+), 16 deletions(-) diff --git a/cycode/cli/apps/auth/auth_manager.py b/cycode/cli/apps/auth/auth_manager.py index ee064f3c..2652bfe1 100644 --- a/cycode/cli/apps/auth/auth_manager.py +++ b/cycode/cli/apps/auth/auth_manager.py @@ -2,8 +2,6 @@ import webbrowser from typing import TYPE_CHECKING, Tuple -from requests import Request - from cycode.cli.exceptions.custom_exceptions import AuthProcessError from cycode.cli.user_settings.configuration_manager import ConfigurationManager from cycode.cli.user_settings.credentials_manager import CredentialsManager @@ -53,7 +51,7 @@ def start_session(self, code_challenge: str) -> str: return auth_session.session_id def redirect_to_login_page(self, code_challenge: str, session_id: str) -> None: - login_url = self._build_login_url(code_challenge, session_id) + login_url = self.auth_client.build_login_url(code_challenge, session_id) webbrowser.open(login_url) def get_api_token(self, session_id: str, code_verifier: str) -> 'ApiToken': @@ -75,19 +73,11 @@ def get_api_token_polling(self, session_id: str, code_verifier: str) -> 'ApiToke raise AuthProcessError('Error while obtaining API token') time.sleep(self.POLLING_WAIT_INTERVAL_IN_SECONDS) - raise AuthProcessError('session expired') + raise AuthProcessError('Timeout while obtaining API token (session expired)') def save_api_token(self, api_token: 'ApiToken') -> None: self.credentials_manager.update_credentials(api_token.client_id, api_token.secret) - def _build_login_url(self, code_challenge: str, session_id: str) -> str: - app_url = self.configuration_manager.get_cycode_app_url() - login_url = f'{app_url}/account/sign-in' - query_params = {'source': 'cycode_cli', 'code_challenge': code_challenge, 'session_id': session_id} - # TODO(MarshalX). Use auth_client instead and don't depend on "requests" lib here - request = Request(url=login_url, params=query_params) - return request.prepare().url - def _generate_pkce_code_pair(self) -> Tuple[str, str]: code_verifier = generate_random_string(self.CODE_VERIFIER_LENGTH) code_challenge = hash_string_to_sha256(code_verifier) diff --git a/cycode/cyclient/auth_client.py b/cycode/cyclient/auth_client.py index 20c80d13..1df7ad9b 100644 --- a/cycode/cyclient/auth_client.py +++ b/cycode/cyclient/auth_client.py @@ -1,9 +1,9 @@ from typing import Optional -from requests import Response +from requests import Request, Response from cycode.cli.exceptions.custom_exceptions import HttpUnauthorizedError, RequestHttpError -from cycode.cyclient import models +from cycode.cyclient import config, models from cycode.cyclient.cycode_client import CycodeClient @@ -13,6 +13,11 @@ class AuthClient: def __init__(self) -> None: self.cycode_client = CycodeClient() + @staticmethod + def build_login_url(code_challenge: str, session_id: str) -> str: + query_params = {'source': 'cycode_cli', 'code_challenge': code_challenge, 'session_id': session_id} + return Request(url=f'{config.cycode_app_url}/account/sign-in', params=query_params).prepare().url + def start_session(self, code_challenge: str) -> models.AuthenticationSession: path = f'{self.AUTH_CONTROLLER_PATH}/start' body = {'code_challenge': code_challenge} diff --git a/cycode/cyclient/config.py b/cycode/cyclient/config.py index 2b278bf4..ec21efb4 100644 --- a/cycode/cyclient/config.py +++ b/cycode/cyclient/config.py @@ -14,6 +14,14 @@ cycode_api_url = consts.DEFAULT_CYCODE_API_URL +cycode_app_url = configuration_manager.get_cycode_app_url() +if not is_valid_url(cycode_app_url): + logger.warning( + 'Invalid Cycode APP URL: %s, using default value (%s)', cycode_app_url, consts.DEFAULT_CYCODE_APP_URL + ) + cycode_app_url = consts.DEFAULT_CYCODE_APP_URL + + def _is_on_premise_installation(cycode_domain: str) -> bool: return not cycode_api_url.endswith(cycode_domain) diff --git a/cycode/cyclient/cycode_client_base.py b/cycode/cyclient/cycode_client_base.py index f2eb77bf..37e9d4f6 100644 --- a/cycode/cyclient/cycode_client_base.py +++ b/cycode/cyclient/cycode_client_base.py @@ -1,11 +1,12 @@ import os import platform import ssl -from typing import Callable, ClassVar, Dict, Optional +from typing import TYPE_CHECKING, Callable, ClassVar, Dict, Optional import requests from requests import Response, exceptions from requests.adapters import HTTPAdapter +from tenacity import retry, retry_if_exception, stop_after_attempt, wait_random_exponential from cycode.cli.exceptions.custom_exceptions import ( HttpUnauthorizedError, @@ -19,6 +20,9 @@ from cycode.cyclient.headers import get_cli_user_agent, get_correlation_id from cycode.cyclient.logger import logger +if TYPE_CHECKING: + from tenacity import RetryCallState + class SystemStorageSslContext(HTTPAdapter): def init_poolmanager(self, *args, **kwargs) -> None: @@ -45,6 +49,47 @@ def _get_request_function() -> Callable: return session.request +_REQUEST_ERRORS_TO_RETRY = ( + RequestTimeout, + RequestConnectionError, + exceptions.ChunkedEncodingError, + exceptions.ContentDecodingError, +) +_RETRY_MAX_ATTEMPTS = 3 +_RETRY_STOP_STRATEGY = stop_after_attempt(_RETRY_MAX_ATTEMPTS) +_RETRY_WAIT_STRATEGY = wait_random_exponential(multiplier=1, min=2, max=10) + + +def _retry_before_sleep(retry_state: 'RetryCallState') -> None: + exception_name = 'None' + if retry_state.outcome.failed: + exception = retry_state.outcome.exception() + exception_name = f'{exception.__class__.__name__}' + + logger.debug( + 'Retrying request after error: %s. Attempt %s of %s. Upcoming sleep: %s', + exception_name, + retry_state.attempt_number, + _RETRY_MAX_ATTEMPTS, + retry_state.upcoming_sleep, + ) + + +def _should_retry_exception(exception: BaseException) -> bool: + if 'PYTEST_CURRENT_TEST' in os.environ: + # We are running under pytest, don't retry + return False + + # Don't retry client errors (400, 401, etc.) + if isinstance(exception, RequestHttpError): + return not exception.status_code < 500 + + is_request_error = isinstance(exception, _REQUEST_ERRORS_TO_RETRY) + is_server_error = isinstance(exception, RequestHttpError) and exception.status_code >= 500 + + return is_request_error or is_server_error + + class CycodeClientBase: MANDATORY_HEADERS: ClassVar[Dict[str, str]] = { 'User-Agent': get_cli_user_agent(), @@ -72,6 +117,13 @@ def put(self, url_path: str, body: Optional[dict] = None, headers: Optional[dict def get(self, url_path: str, headers: Optional[dict] = None, **kwargs) -> Response: return self._execute(method='get', endpoint=url_path, headers=headers, **kwargs) + @retry( + retry=retry_if_exception(_should_retry_exception), + stop=_RETRY_STOP_STRATEGY, + wait=_RETRY_WAIT_STRATEGY, + reraise=True, + before_sleep=_retry_before_sleep, + ) def _execute( self, method: str, diff --git a/poetry.lock b/poetry.lock index 186fee3f..d0b6503d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -980,6 +980,22 @@ files = [ {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, ] +[[package]] +name = "tenacity" +version = "9.0.0" +description = "Retry code until it succeeds" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "tenacity-9.0.0-py3-none-any.whl", hash = "sha256:93de0c98785b27fcf659856aa9f54bfbd399e29969b0621bc7f762bd441b4539"}, + {file = "tenacity-9.0.0.tar.gz", hash = "sha256:807f37ca97d62aa361264d497b0e31e92b8027044942bfa756160d908320d73b"}, +] + +[package.extras] +doc = ["reno", "sphinx"] +test = ["pytest", "tornado (>=4.5)", "typeguard"] + [[package]] name = "tomli" version = "2.2.1" @@ -1118,4 +1134,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<3.14" -content-hash = "2c45abb3ea36096f8dbe862ba8bf5e064994ed9cdf86a8256643b9822c40168f" +content-hash = "590be7f6a392d52a8d298596ef95e6ee664a8a3515530b01d727fe268e15fb0d" diff --git a/pyproject.toml b/pyproject.toml index ef555019..cde794b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ pyjwt = ">=2.8.0,<3.0" rich = ">=13.9.4, <14" patch-ng = "1.18.1" typer = "^0.15.2" +tenacity = ">=9.0.0,<9.1.0" [tool.poetry.group.test.dependencies] mock = ">=4.0.3,<4.1.0" From 355d1c05bf6110c05fc35985acbf8d42c2afe460 Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Tue, 8 Apr 2025 13:20:19 +0200 Subject: [PATCH 08/19] CM-46563 - Migrate to rich Console, add help rich panels, add syntax highlighting for light themes, fix progress bar, fix traceback printing, fix repository scan (#293) --- cycode/cli/app.py | 31 ++++++++++- cycode/cli/apps/ai_remediation/__init__.py | 5 +- .../apps/ai_remediation/print_remediation.py | 4 +- .../cli/apps/configure/configure_command.py | 7 +-- cycode/cli/apps/ignore/ignore_command.py | 39 +++++++++---- cycode/cli/apps/report/sbom/__init__.py | 5 +- cycode/cli/apps/report/sbom/sbom_command.py | 4 ++ .../cli/apps/report/sbom/sbom_report_file.py | 8 ++- cycode/cli/apps/scan/__init__.py | 12 ++-- cycode/cli/apps/scan/code_scanner.py | 9 +-- .../scan/repository/repository_command.py | 2 +- .../cli/apps/scan/scan_ci/ci_integrations.py | 6 +- cycode/cli/apps/scan/scan_command.py | 15 +++-- cycode/cli/apps/status/status_command.py | 8 +-- cycode/cli/apps/status/version_command.py | 11 +--- cycode/cli/cli_types.py | 13 +++++ cycode/cli/console.py | 47 ++++++++++++++++ cycode/cli/printers/json_printer.py | 12 ++-- cycode/cli/printers/printer_base.py | 31 +++++++---- .../cli/printers/tables/sca_table_printer.py | 13 ++--- cycode/cli/printers/tables/table_printer.py | 2 +- .../cli/printers/tables/table_printer_base.py | 20 +++---- cycode/cli/printers/text_printer.py | 55 +++++++++---------- cycode/cli/utils/progress_bar.py | 11 ++-- cycode/cli/utils/shell_executor.py | 3 +- cycode/cli/utils/version_checker.py | 14 ++--- cycode/logger.py | 5 +- tests/cli/commands/test_main_command.py | 2 +- .../cli/exceptions/test_handle_scan_errors.py | 17 ++++-- 29 files changed, 264 insertions(+), 147 deletions(-) create mode 100644 cycode/cli/console.py diff --git a/cycode/cli/app.py b/cycode/cli/app.py index 96389ef1..80742bab 100644 --- a/cycode/cli/app.py +++ b/cycode/cli/app.py @@ -2,6 +2,7 @@ from typing import Annotated, Optional import typer +from typer.completion import install_callback, show_callback from cycode import __version__ from cycode.cli.apps import ai_remediation, auth, configure, ignore, report, scan, status @@ -20,6 +21,7 @@ pretty_exceptions_short=True, context_settings=CLI_CONTEXT_SETTINGS, rich_markup_mode='rich', + add_completion=False, # we add it manually to control the rich help panel ) app.add_typer(ai_remediation.app) @@ -39,9 +41,10 @@ def check_latest_version_on_close(ctx: typer.Context) -> None: # we always want to check the latest version for "version" and "status" commands should_use_cache = ctx.invoked_subcommand not in {'version', 'status'} - version_checker.check_and_notify_update( - current_version=__version__, use_color=ctx.color, use_cache=should_use_cache - ) + version_checker.check_and_notify_update(current_version=__version__, use_cache=should_use_cache) + + +_COMPLETION_RICH_HELP_PANEL = 'Completion options' @app.callback() @@ -61,6 +64,28 @@ def app_callback( Optional[str], typer.Option(hidden=True, help='Characteristic JSON object that lets servers identify the application.'), ] = None, + _: Annotated[ + Optional[bool], + typer.Option( + '--install-completion', + callback=install_callback, + is_eager=True, + expose_value=False, + help='Install completion for the current shell.', + rich_help_panel=_COMPLETION_RICH_HELP_PANEL, + ), + ] = False, + __: Annotated[ + Optional[bool], + typer.Option( + '--show-completion', + callback=show_callback, + is_eager=True, + expose_value=False, + help='Show completion for the current shell, to copy it or customize the installation.', + rich_help_panel=_COMPLETION_RICH_HELP_PANEL, + ), + ] = False, ) -> None: init_sentry() add_breadcrumb('cycode') diff --git a/cycode/cli/apps/ai_remediation/__init__.py b/cycode/cli/apps/ai_remediation/__init__.py index 2ccba382..6b5a3013 100644 --- a/cycode/cli/apps/ai_remediation/__init__.py +++ b/cycode/cli/apps/ai_remediation/__init__.py @@ -3,4 +3,7 @@ from cycode.cli.apps.ai_remediation.ai_remediation_command import ai_remediation_command app = typer.Typer() -app.command(name='ai_remediation', short_help='Get AI remediation (INTERNAL).', hidden=True)(ai_remediation_command) +app.command(name='ai-remediation', short_help='Get AI remediation (INTERNAL).', hidden=True)(ai_remediation_command) + +# backward compatibility +app.command(hidden=True, name='ai_remediation')(ai_remediation_command) diff --git a/cycode/cli/apps/ai_remediation/print_remediation.py b/cycode/cli/apps/ai_remediation/print_remediation.py index c706c13f..c0109341 100644 --- a/cycode/cli/apps/ai_remediation/print_remediation.py +++ b/cycode/cli/apps/ai_remediation/print_remediation.py @@ -1,7 +1,7 @@ import typer -from rich.console import Console from rich.markdown import Markdown +from cycode.cli.console import console from cycode.cli.models import CliResult from cycode.cli.printers import ConsolePrinter @@ -12,4 +12,4 @@ def print_remediation(ctx: typer.Context, remediation_markdown: str, is_fix_avai data = {'remediation': remediation_markdown, 'is_fix_available': is_fix_available} printer.print_result(CliResult(success=True, message='Remediation fetched successfully', data=data)) else: # text or table - Console().print(Markdown(remediation_markdown)) + console.print(Markdown(remediation_markdown)) diff --git a/cycode/cli/apps/configure/configure_command.py b/cycode/cli/apps/configure/configure_command.py index 9c631641..2aa86a8f 100644 --- a/cycode/cli/apps/configure/configure_command.py +++ b/cycode/cli/apps/configure/configure_command.py @@ -1,7 +1,5 @@ from typing import Optional -import typer - from cycode.cli.apps.configure.consts import CONFIGURATION_MANAGER, CREDENTIALS_MANAGER from cycode.cli.apps.configure.messages import get_credentials_update_result_message, get_urls_update_result_message from cycode.cli.apps.configure.prompts import ( @@ -10,6 +8,7 @@ get_client_id_input, get_client_secret_input, ) +from cycode.cli.console import console from cycode.cli.utils.sentry import add_breadcrumb @@ -52,6 +51,6 @@ def configure_command() -> None: CREDENTIALS_MANAGER.update_credentials(client_id, client_secret) if config_updated: - typer.echo(get_urls_update_result_message()) + console.print(get_urls_update_result_message()) if credentials_updated: - typer.echo(get_credentials_update_result_message()) + console.print(get_credentials_update_result_message()) diff --git a/cycode/cli/apps/ignore/ignore_command.py b/cycode/cli/apps/ignore/ignore_command.py index 47d4fa0d..cc6ecd25 100644 --- a/cycode/cli/apps/ignore/ignore_command.py +++ b/cycode/cli/apps/ignore/ignore_command.py @@ -12,45 +12,62 @@ from cycode.cli.utils.sentry import add_breadcrumb from cycode.cli.utils.string_utils import hash_string_to_sha256 +_FILTER_BY_RICH_HELP_PANEL = 'Filter options' +_SECRETS_FILTER_BY_RICH_HELP_PANEL = 'Secrets filter options' +_SCA_FILTER_BY_RICH_HELP_PANEL = 'SCA filter options' + def _is_package_pattern_valid(package: str) -> bool: return re.search('^[^@]+@[^@]+$', package) is not None def ignore_command( # noqa: C901 - by_value: Annotated[ - Optional[str], typer.Option(help='Ignore a specific value while scanning for Secrets.', show_default=False) + by_path: Annotated[ + Optional[str], + typer.Option( + help='Ignore a specific file or directory while scanning.', + show_default=False, + rich_help_panel=_FILTER_BY_RICH_HELP_PANEL, + ), ] = None, - by_sha: Annotated[ + by_rule: Annotated[ Optional[str], typer.Option( - help='Ignore a specific SHA512 representation of a string while scanning for Secrets.', show_default=False + help='Ignore scanning a specific Secrets rule ID or IaC rule ID.', + show_default=False, + rich_help_panel=_FILTER_BY_RICH_HELP_PANEL, ), ] = None, - by_path: Annotated[ + by_value: Annotated[ Optional[str], - typer.Option(help='Avoid scanning a specific path. You`ll need to specify the scan type.', show_default=False), + typer.Option( + help='Ignore a specific value.', + show_default=False, + rich_help_panel=_SECRETS_FILTER_BY_RICH_HELP_PANEL, + ), ] = None, - by_rule: Annotated[ + by_sha: Annotated[ Optional[str], typer.Option( - help='Ignore scanning a specific secret rule ID or IaC rule ID. You`ll to specify the scan type.', + help='Ignore a specific SHA512 representation of a string.', show_default=False, + rich_help_panel=_SECRETS_FILTER_BY_RICH_HELP_PANEL, ), ] = None, by_package: Annotated[ Optional[str], typer.Option( - help='Ignore scanning a specific package version while running an SCA scan. ' - 'Expected pattern: name@version.', + help='Ignore scanning a specific package version. Expected pattern: [cyan]name@version[/cyan].', show_default=False, + rich_help_panel=_SCA_FILTER_BY_RICH_HELP_PANEL, ), ] = None, by_cve: Annotated[ Optional[str], typer.Option( - help='Ignore scanning a specific CVE while running an SCA scan. Expected pattern: CVE-YYYY-NNN.', + help='Ignore scanning a specific CVE. Expected pattern: [cyan]CVE-YYYY-NNN[/cyan].', show_default=False, + rich_help_panel=_SCA_FILTER_BY_RICH_HELP_PANEL, ), ] = None, scan_type: Annotated[ diff --git a/cycode/cli/apps/report/sbom/__init__.py b/cycode/cli/apps/report/sbom/__init__.py index 461b3fe0..77d081e8 100644 --- a/cycode/cli/apps/report/sbom/__init__.py +++ b/cycode/cli/apps/report/sbom/__init__.py @@ -7,6 +7,9 @@ app = typer.Typer(name='sbom') app.callback(short_help='Generate SBOM report for remote repository by url or local directory by path.')(sbom_command) app.command(name='path', short_help='Generate SBOM report for provided path in the command.')(path_command) -app.command(name='repository_url', short_help='Generate SBOM report for provided repository URI in the command.')( +app.command(name='repository-url', short_help='Generate SBOM report for provided repository URI in the command.')( repository_url_command ) + +# backward compatibility +app.command(hidden=True, name='repository_url')(repository_url_command) diff --git a/cycode/cli/apps/report/sbom/sbom_command.py b/cycode/cli/apps/report/sbom/sbom_command.py index 65dc3fd9..06126dd0 100644 --- a/cycode/cli/apps/report/sbom/sbom_command.py +++ b/cycode/cli/apps/report/sbom/sbom_command.py @@ -8,6 +8,8 @@ from cycode.cli.utils.sentry import add_breadcrumb from cycode.cyclient.report_client import ReportParameters +_OUTPUT_RICH_HELP_PANEL = 'Output options' + def sbom_command( ctx: typer.Context, @@ -27,6 +29,7 @@ def sbom_command( '--output-format', '-o', help='Specify the output file format.', + rich_help_panel=_OUTPUT_RICH_HELP_PANEL, ), ] = SbomOutputFormatOption.JSON, output_file: Annotated[ @@ -36,6 +39,7 @@ def sbom_command( show_default='Autogenerated filename saved to the current directory', dir_okay=False, writable=True, + rich_help_panel=_OUTPUT_RICH_HELP_PANEL, ), ] = None, include_vulnerabilities: Annotated[ diff --git a/cycode/cli/apps/report/sbom/sbom_report_file.py b/cycode/cli/apps/report/sbom/sbom_report_file.py index 4d58f89f..f3178b44 100644 --- a/cycode/cli/apps/report/sbom/sbom_report_file.py +++ b/cycode/cli/apps/report/sbom/sbom_report_file.py @@ -3,7 +3,9 @@ import re from typing import Optional -import click +import typer + +from cycode.cli.console import console class SbomReportFile: @@ -21,14 +23,14 @@ def is_exists(self) -> bool: return self._file_path.exists() def _prompt_overwrite(self) -> bool: - return click.confirm(f'File {self._file_path} already exists. Save with a different filename?', default=True) + return typer.confirm(f'File {self._file_path} already exists. Save with a different filename?', default=True) def _write(self, content: str) -> None: with open(self._file_path, 'w', encoding='UTF-8') as f: f.write(content) def _notify_about_saved_file(self) -> None: - click.echo(f'Report saved to {self._file_path}') + console.print(f'Report saved to {self._file_path}') def _find_and_set_unique_filename(self) -> None: attempt_no = 0 diff --git a/cycode/cli/apps/scan/__init__.py b/cycode/cli/apps/scan/__init__.py index e8602091..07c15978 100644 --- a/cycode/cli/apps/scan/__init__.py +++ b/cycode/cli/apps/scan/__init__.py @@ -16,18 +16,22 @@ app.command(name='path', short_help='Scan the files in the paths provided in the command.')(path_command) app.command(name='repository', short_help='Scan the Git repository included files.')(repository_command) -app.command(name='commit_history', short_help='Scan all the commits history in this git repository.')( +app.command(name='commit-history', short_help='Scan all the commits history in this git repository.')( commit_history_command ) - app.command( - name='pre_commit', + name='pre-commit', short_help='Use this command in pre-commit hook to scan any content that was not committed yet.', rich_help_panel='Automation commands', )(pre_commit_command) app.command( - name='pre_receive', + name='pre-receive', short_help='Use this command in pre-receive hook ' 'to scan commits on the server side before pushing them to the repository.', rich_help_panel='Automation commands', )(pre_receive_command) + +# backward compatibility +app.command(hidden=True, name='commit_history')(commit_history_command) +app.command(hidden=True, name='pre_commit')(pre_commit_command) +app.command(hidden=True, name='pre_receive')(pre_receive_command) diff --git a/cycode/cli/apps/scan/code_scanner.py b/cycode/cli/apps/scan/code_scanner.py index a3b1201e..a44d9c15 100644 --- a/cycode/cli/apps/scan/code_scanner.py +++ b/cycode/cli/apps/scan/code_scanner.py @@ -12,6 +12,7 @@ from cycode.cli import consts from cycode.cli.cli_types import SeverityOption from cycode.cli.config import configuration_manager +from cycode.cli.console import console from cycode.cli.exceptions import custom_exceptions from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception from cycode.cli.files_collector.excluder import exclude_irrelevant_documents_to_scan @@ -83,7 +84,7 @@ def scan_sca_commit_range(ctx: typer.Context, path: str, commit_range: str) -> N scan_commit_range_documents(ctx, from_commit_documents, to_commit_documents, scan_parameters=scan_parameters) -def scan_disk_files(ctx: click.Context, paths: Tuple[str]) -> None: +def scan_disk_files(ctx: typer.Context, paths: Tuple[str]) -> None: scan_type = ctx.obj['scan_type'] progress_bar = ctx.obj['progress_bar'] @@ -642,7 +643,7 @@ def parse_pre_receive_input() -> str: return pre_receive_input.splitlines()[0] -def _get_default_scan_parameters(ctx: click.Context) -> dict: +def _get_default_scan_parameters(ctx: typer.Context) -> dict: return { 'monitor': ctx.obj.get('monitor'), 'report': ctx.obj.get('report'), @@ -916,7 +917,7 @@ def _try_get_report_url_if_needed( logger.debug('Failed to get report URL', exc_info=e) -def _set_aggregation_report_url(ctx: click.Context, aggregation_report_url: Optional[str] = None) -> None: +def _set_aggregation_report_url(ctx: typer.Context, aggregation_report_url: Optional[str] = None) -> None: ctx.obj['aggregation_report_url'] = aggregation_report_url @@ -1007,7 +1008,7 @@ def _normalize_file_path(path: str) -> str: def perform_post_pre_receive_scan_actions(ctx: typer.Context) -> None: if scan_utils.is_scan_failed(ctx): - click.echo(consts.PRE_RECEIVE_REMEDIATION_MESSAGE) + console.print(consts.PRE_RECEIVE_REMEDIATION_MESSAGE) def enable_verbose_mode(ctx: typer.Context) -> None: diff --git a/cycode/cli/apps/scan/repository/repository_command.py b/cycode/cli/apps/scan/repository/repository_command.py index 7d6b421a..a99cc2d1 100644 --- a/cycode/cli/apps/scan/repository/repository_command.py +++ b/cycode/cli/apps/scan/repository/repository_command.py @@ -63,6 +63,6 @@ def repository_command( perform_pre_scan_documents_actions(ctx, scan_type, documents_to_scan) logger.debug('Found all relevant files for scanning %s', {'path': path, 'branch': branch}) - scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx, (path,))) + scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx, (str(path),))) except Exception as e: handle_scan_exception(ctx, e) diff --git a/cycode/cli/apps/scan/scan_ci/ci_integrations.py b/cycode/cli/apps/scan/scan_ci/ci_integrations.py index f2869b2f..3cb617a9 100644 --- a/cycode/cli/apps/scan/scan_ci/ci_integrations.py +++ b/cycode/cli/apps/scan/scan_ci/ci_integrations.py @@ -2,6 +2,8 @@ import click +from cycode.cli.console import console + def github_action_range() -> str: before_sha = os.getenv('BEFORE_SHA') @@ -11,7 +13,7 @@ def github_action_range() -> str: head_sha = os.getenv('GITHUB_SHA') ref = os.getenv('GITHUB_REF') - click.echo(f'{before_sha}, {push_base_sha}, {pr_base_sha}, {default_branch}, {head_sha}, {ref}') + console.print(f'{before_sha}, {push_base_sha}, {pr_base_sha}, {default_branch}, {head_sha}, {ref}') if before_sha and before_sha != NO_COMMITS: return f'{before_sha}...' @@ -26,7 +28,7 @@ def circleci_range() -> str: before_sha = os.getenv('BEFORE_SHA') current_sha = os.getenv('CURRENT_SHA') commit_range = f'{before_sha}...{current_sha}' - click.echo(f'commit range: {commit_range}') + console.print(f'commit range: {commit_range}') if not commit_range.startswith('...'): return commit_range diff --git a/cycode/cli/apps/scan/scan_command.py b/cycode/cli/apps/scan/scan_command.py index 5b9c43c6..6b776689 100644 --- a/cycode/cli/apps/scan/scan_command.py +++ b/cycode/cli/apps/scan/scan_command.py @@ -12,6 +12,9 @@ from cycode.cli.utils.get_api_client import get_scan_cycode_client from cycode.cli.utils.sentry import add_breadcrumb +_AUTH_RICH_HELP_PANEL = 'Authentication options' +_SCA_RICH_HELP_PANEL = 'SCA options' + def scan_command( ctx: typer.Context, @@ -28,14 +31,14 @@ def scan_command( Optional[str], typer.Option( help='Specify a Cycode client secret for this specific scan execution.', - rich_help_panel='Authentication options', + rich_help_panel=_AUTH_RICH_HELP_PANEL, ), ] = None, client_id: Annotated[ Optional[str], typer.Option( help='Specify a Cycode client ID for this specific scan execution.', - rich_help_panel='Authentication options', + rich_help_panel=_AUTH_RICH_HELP_PANEL, ), ] = None, show_secret: Annotated[bool, typer.Option('--show-secret', help='Show Secrets in plain text.')] = False, @@ -65,7 +68,7 @@ def scan_command( List[ScaScanTypeOption], typer.Option( help='Specify the type of SCA scan you wish to execute.', - rich_help_panel='SCA options', + rich_help_panel=_SCA_RICH_HELP_PANEL, ), ] = (ScaScanTypeOption.PACKAGE_VULNERABILITIES, ScaScanTypeOption.LICENSE_COMPLIANCE), monitor: Annotated[ @@ -73,7 +76,7 @@ def scan_command( typer.Option( '--monitor', help='When specified, the scan results are recorded in the Discovery module.', - rich_help_panel='SCA options', + rich_help_panel=_SCA_RICH_HELP_PANEL, ), ] = False, no_restore: Annotated[ @@ -82,7 +85,7 @@ def scan_command( '--no-restore', help='When specified, Cycode will not run restore command. ' 'Will scan direct dependencies [bold]only[/bold]!', - rich_help_panel='SCA options', + rich_help_panel=_SCA_RICH_HELP_PANEL, ), ] = False, gradle_all_sub_projects: Annotated[ @@ -91,7 +94,7 @@ def scan_command( '--gradle-all-sub-projects', help='When specified, Cycode will run gradle restore command for all sub projects. ' 'Should run from root project directory [bold]only[/bold]!', - rich_help_panel='SCA options', + rich_help_panel=_SCA_RICH_HELP_PANEL, ), ] = False, ) -> None: diff --git a/cycode/cli/apps/status/status_command.py b/cycode/cli/apps/status/status_command.py index edffd24c..28f8cfba 100644 --- a/cycode/cli/apps/status/status_command.py +++ b/cycode/cli/apps/status/status_command.py @@ -2,14 +2,14 @@ from cycode.cli.apps.status.get_cli_status import get_cli_status from cycode.cli.cli_types import OutputTypeOption +from cycode.cli.console import console def status_command(ctx: typer.Context) -> None: output = ctx.obj['output'] cli_status = get_cli_status() - message = cli_status.as_text() if output == OutputTypeOption.JSON: - message = cli_status.as_json() - - typer.echo(message, color=ctx.color) + console.print_json(cli_status.as_json()) + else: + console.print(cli_status.as_text()) diff --git a/cycode/cli/apps/status/version_command.py b/cycode/cli/apps/status/version_command.py index c36aad4b..272b4a17 100644 --- a/cycode/cli/apps/status/version_command.py +++ b/cycode/cli/apps/status/version_command.py @@ -1,15 +1,10 @@ import typer from cycode.cli.apps.status.status_command import status_command +from cycode.cli.console import console def version_command(ctx: typer.Context) -> None: - typer.echo( - typer.style( - text='The "version" command is deprecated. Please use the "status" command instead.', - fg=typer.colors.YELLOW, - bold=True, - ), - color=ctx.color, - ) + console.print('[yellow][bold]This command is deprecated. Please use the "status" command instead.[/bold][/yellow]') + console.print() # print an empty line status_command(ctx) diff --git a/cycode/cli/cli_types.py b/cycode/cli/cli_types.py index 83451df2..87c6346b 100644 --- a/cycode/cli/cli_types.py +++ b/cycode/cli/cli_types.py @@ -38,6 +38,15 @@ class SeverityOption(str, Enum): HIGH = 'high' CRITICAL = 'critical' + @classmethod + def _missing_(cls, value: str) -> str: + value = value.lower() + for member in cls: + if member.lower() == value: + return member + + return cls.INFO # fallback to INFO if no match is found + @staticmethod def get_member_weight(name: str) -> int: return _SEVERITY_WEIGHTS.get(name.lower(), _SEVERITY_DEFAULT_WEIGHT) @@ -46,6 +55,10 @@ def get_member_weight(name: str) -> int: def get_member_color(name: str) -> str: return _SEVERITY_COLORS.get(name.lower(), _SEVERITY_DEFAULT_COLOR) + def __rich__(self) -> str: + color = self.get_member_color(self.value) + return f'[{color}]{self.value.upper()}[/{color}]' + _SEVERITY_DEFAULT_WEIGHT = -1 _SEVERITY_WEIGHTS = { diff --git a/cycode/cli/console.py b/cycode/cli/console.py new file mode 100644 index 00000000..159f4733 --- /dev/null +++ b/cycode/cli/console.py @@ -0,0 +1,47 @@ +import os +from typing import Optional + +from rich.console import Console + +console_out = Console() +console_err = Console(stderr=True) + +console = console_out # alias + + +def is_dark_console() -> Optional[bool]: + """Detect if the console is dark or light. + + This function checks the environment variables and terminal type to determine if the console is dark or light. + + Used approaches: + 1. Check the `LC_DARK_BG` environment variable. + 2. Check the `COLORFGBG` environment variable for background color. + + And it still could be wrong in some cases. + + TODO(MarshalX): migrate to https://github.com/dalance/termbg when someone will implement it for Python. + """ + dark = None + + dark_bg = os.environ.get('LC_DARK_BG') + if dark_bg is not None: + return dark_bg != '0' + + # If BG color in {0, 1, 2, 3, 4, 5, 6, 8} then dark, else light. + try: + color = os.environ.get('COLORFGBG') + *_, bg = color.split(';') + bg = int(bg) + dark = bool(0 <= bg <= 6 or bg == 8) + except Exception: # noqa: S110 + pass + + return dark + + +_SYNTAX_HIGHLIGHT_DARK_THEME = 'monokai' +_SYNTAX_HIGHLIGHT_LIGHT_THEME = 'default' + +# when we could not detect it, use dark theme as most terminals are dark +_SYNTAX_HIGHLIGHT_THEME = _SYNTAX_HIGHLIGHT_LIGHT_THEME if is_dark_console() is False else _SYNTAX_HIGHLIGHT_DARK_THEME diff --git a/cycode/cli/printers/json_printer.py b/cycode/cli/printers/json_printer.py index c8fbacb3..76b1f7c7 100644 --- a/cycode/cli/printers/json_printer.py +++ b/cycode/cli/printers/json_printer.py @@ -1,8 +1,7 @@ import json from typing import TYPE_CHECKING, Dict, List, Optional -import click - +from cycode.cli.console import console from cycode.cli.models import CliError, CliResult from cycode.cli.printers.printer_base import PrinterBase from cycode.cyclient.models import DetectionSchema @@ -15,12 +14,12 @@ class JsonPrinter(PrinterBase): def print_result(self, result: CliResult) -> None: result = {'result': result.success, 'message': result.message, 'data': result.data} - click.echo(self.get_data_json(result)) + console.print_json(self.get_data_json(result)) def print_error(self, error: CliError) -> None: result = {'error': error.code, 'message': error.message} - click.echo(self.get_data_json(result)) + console.print_json(self.get_data_json(result)) def print_scan_results( self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None @@ -47,13 +46,12 @@ def print_scan_results( # FIXME(MarshalX): we don't care about scan IDs in JSON output due to clumsy JSON root structure inlined_errors = [err._asdict() for err in errors.values()] - click.echo(self._get_json_scan_result(scan_ids, detections_dict, report_urls, inlined_errors)) + console.print_json(self._get_json_scan_result(scan_ids, detections_dict, report_urls, inlined_errors)) def _get_json_scan_result( self, scan_ids: List[str], detections: dict, report_urls: List[str], errors: List[dict] ) -> str: result = { - 'scan_id': 'DEPRECATED', # backward compatibility 'scan_ids': scan_ids, 'detections': detections, 'report_urls': report_urls, @@ -65,4 +63,4 @@ def _get_json_scan_result( @staticmethod def get_data_json(data: dict) -> str: # ensure_ascii is disabled for symbols like "`". Eg: `cycode scan` - return json.dumps(data, indent=4, ensure_ascii=False) + return json.dumps(data, ensure_ascii=False) diff --git a/cycode/cli/printers/printer_base.py b/cycode/cli/printers/printer_base.py index 633a2ccc..c6c5120b 100644 --- a/cycode/cli/printers/printer_base.py +++ b/cycode/cli/printers/printer_base.py @@ -4,6 +4,7 @@ import typer +from cycode.cli.console import console_err from cycode.cli.models import CliError, CliResult from cycode.cyclient.headers import get_correlation_id @@ -11,13 +12,17 @@ from cycode.cli.models import LocalScanResult -from rich.console import Console -from rich.traceback import Traceback +from rich.traceback import Traceback as RichTraceback class PrinterBase(ABC): - RED_COLOR_NAME = 'red' - GREEN_COLOR_NAME = 'green' + NO_DETECTIONS_MESSAGE = ( + '[green]Good job! No issues were found!!! :clapping_hands::clapping_hands::clapping_hands:[/green]' + ) + FAILED_SCAN_MESSAGE = ( + '[red]Unfortunately, Cycode was unable to complete the full scan. ' + 'Please note that not all results may be available:[/red]' + ) def __init__(self, ctx: typer.Context) -> None: self.ctx = ctx @@ -36,15 +41,19 @@ def print_result(self, result: CliResult) -> None: def print_error(self, error: CliError) -> None: pass - def print_exception(self, e: Optional[BaseException] = None) -> None: + @staticmethod + def print_exception(e: Optional[BaseException] = None) -> None: """We are printing it in stderr so, we don't care about supporting JSON and TABLE outputs. Note: Called only when the verbose flag is set. """ - console = Console(stderr=True) - - traceback = Traceback.from_exception(type(e), e, None) if e else Traceback.from_exception(*sys.exc_info()) - console.print(traceback) - - console.print(f'Correlation ID: {get_correlation_id()}', style=self.RED_COLOR_NAME) + rich_traceback = ( + RichTraceback.from_exception(type(e), e, e.__traceback__) + if e + else RichTraceback.from_exception(*sys.exc_info()) + ) + rich_traceback.show_locals = False + console_err.print(rich_traceback) + + console_err.print(f'[red]Correlation ID:[/red] {get_correlation_id()}') diff --git a/cycode/cli/printers/tables/sca_table_printer.py b/cycode/cli/printers/tables/sca_table_printer.py index b77f55ed..f5c38279 100644 --- a/cycode/cli/printers/tables/sca_table_printer.py +++ b/cycode/cli/printers/tables/sca_table_printer.py @@ -1,9 +1,8 @@ from collections import defaultdict from typing import TYPE_CHECKING, Dict, List, Set, Tuple -import typer - from cycode.cli.cli_types import SeverityOption +from cycode.cli.console import console from cycode.cli.consts import LICENSE_COMPLIANCE_POLICY_ID, PACKAGE_VULNERABILITY_POLICY_ID from cycode.cli.models import Detection from cycode.cli.printers.tables.table import Table @@ -137,10 +136,10 @@ def _enrich_table_with_values(policy_id: str, table: Table, detection: Detection elif policy_id == LICENSE_COMPLIANCE_POLICY_ID: severity = detection.severity - if not severity: - severity = 'N/A' - - table.add_cell(SEVERITY_COLUMN, severity, SeverityOption.get_member_color(severity)) + if severity: + table.add_cell(SEVERITY_COLUMN, SeverityOption(severity)) + else: + table.add_cell(SEVERITY_COLUMN, 'N/A') table.add_cell(REPOSITORY_COLUMN, detection_details.get('repository_name')) table.add_file_path_cell(CODE_PROJECT_COLUMN, detection_details.get('file_name')) @@ -179,7 +178,7 @@ def _enrich_table_with_values(policy_id: str, table: Table, detection: Detection @staticmethod def _print_summary_issues(detections_count: int, title: str) -> None: - typer.echo(f'⛔ Found {detections_count} issues of type: {typer.style(title, bold=True)}') + console.print(f':no_entry: Found {detections_count} issues of type: [bold]{title}[/bold]') @staticmethod def _extract_detections_per_policy_id( diff --git a/cycode/cli/printers/tables/table_printer.py b/cycode/cli/printers/tables/table_printer.py index 853e2465..12e2dbf3 100644 --- a/cycode/cli/printers/tables/table_printer.py +++ b/cycode/cli/printers/tables/table_printer.py @@ -117,7 +117,7 @@ def _enrich_table_with_detection_summary_values( if self.scan_type == SECRET_SCAN_TYPE: issue_type = detection.type - table.add_cell(SEVERITY_COLUMN, detection.severity, SeverityOption.get_member_color(detection.severity)) + table.add_cell(SEVERITY_COLUMN, SeverityOption(detection.severity)) table.add_cell(ISSUE_TYPE_COLUMN, issue_type) table.add_file_path_cell(FILE_PATH_COLUMN, document.path) table.add_cell(SECRET_SHA_COLUMN, detection.detection_details.get('sha512', '')) diff --git a/cycode/cli/printers/tables/table_printer_base.py b/cycode/cli/printers/tables/table_printer_base.py index 71b4f399..73ab7f88 100644 --- a/cycode/cli/printers/tables/table_printer_base.py +++ b/cycode/cli/printers/tables/table_printer_base.py @@ -2,8 +2,8 @@ from typing import TYPE_CHECKING, Dict, List, Optional import typer -from rich.console import Console +from cycode.cli.console import console from cycode.cli.models import CliError, CliResult from cycode.cli.printers.printer_base import PrinterBase from cycode.cli.printers.text_printer import TextPrinter @@ -29,7 +29,7 @@ def print_scan_results( self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None ) -> None: if not errors and all(result.issue_detected == 0 for result in local_scan_results): - typer.secho('Good job! No issues were found!!! 👏👏👏', fg=self.GREEN_COLOR_NAME) + console.print(self.NO_DETECTIONS_MESSAGE) return self._print_results(local_scan_results) @@ -37,13 +37,9 @@ def print_scan_results( if not errors: return - typer.secho( - 'Unfortunately, Cycode was unable to complete the full scan. ' - 'Please note that not all results may be available:', - fg='red', - ) + console.print(self.FAILED_SCAN_MESSAGE) for scan_id, error in errors.items(): - typer.echo(f'- {scan_id}: ', nl=False) + console.print(f'- {scan_id}: ', end='') self.print_error(error) def _is_git_repository(self) -> bool: @@ -56,7 +52,7 @@ def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: @staticmethod def _print_table(table: 'Table') -> None: if table.get_rows(): - Console().print(table.get_table()) + console.print(table.get_table()) @staticmethod def _print_report_urls( @@ -67,9 +63,9 @@ def _print_report_urls( if not report_urls and not aggregation_report_url: return if aggregation_report_url: - typer.echo(f'Report URL: {aggregation_report_url}') + console.print(f'Report URL: {aggregation_report_url}') return - typer.echo('Report URLs:') + console.print('Report URLs:') for report_url in report_urls: - typer.echo(f'- {report_url}') + console.print(f'- {report_url}') diff --git a/cycode/cli/printers/text_printer.py b/cycode/cli/printers/text_printer.py index 73eaccc2..75725aac 100644 --- a/cycode/cli/printers/text_printer.py +++ b/cycode/cli/printers/text_printer.py @@ -3,11 +3,11 @@ from typing import TYPE_CHECKING, Dict, List, Optional import typer -from rich.console import Console from rich.markup import escape from rich.syntax import Syntax from cycode.cli.cli_types import SeverityOption +from cycode.cli.console import _SYNTAX_HIGHLIGHT_THEME, console from cycode.cli.consts import COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES, SECRET_SCAN_TYPE from cycode.cli.models import CliError, CliResult, Detection, Document, DocumentDetections from cycode.cli.printers.printer_base import PrinterBase @@ -25,27 +25,27 @@ def __init__(self, ctx: typer.Context) -> None: self.show_secret: bool = ctx.obj.get('show_secret', False) def print_result(self, result: CliResult) -> None: - color = None + color = 'default' if not result.success: - color = self.RED_COLOR_NAME + color = 'red' - typer.secho(result.message, fg=color) + console.print(result.message, style=color) if not result.data: return - typer.secho('\nAdditional data:', fg=color) + console.print('\nAdditional data:', style=color) for name, value in result.data.items(): - typer.secho(f'- {name}: {value}', fg=color) + console.print(f'- {name}: {value}', style=color) def print_error(self, error: CliError) -> None: - typer.secho(error.message, fg=self.RED_COLOR_NAME) + console.print(f'[red]Error: {error.message}[/red]') def print_scan_results( self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None ) -> None: if not errors and all(result.issue_detected == 0 for result in local_scan_results): - typer.secho('Good job! No issues were found!!! 👏👏👏', fg=self.GREEN_COLOR_NAME) + console.print(self.NO_DETECTIONS_MESSAGE) return for local_scan_result in local_scan_results: @@ -58,13 +58,9 @@ def print_scan_results( if not errors: return - typer.secho( - 'Unfortunately, Cycode was unable to complete the full scan. ' - 'Please note that not all results may be available:', - fg='red', - ) + console.print(self.FAILED_SCAN_MESSAGE) for scan_id, error in errors.items(): - typer.echo(f'- {scan_id}: ', nl=False) + console.print(f'- {scan_id}: ', end='') self.print_error(error) def _print_document_detections(self, document_detections: DocumentDetections) -> None: @@ -77,14 +73,11 @@ def _print_document_detections(self, document_detections: DocumentDetections) -> @staticmethod def _print_new_line() -> None: - typer.echo() + console.print() def _print_detection_summary(self, detection: Detection, document_path: str) -> None: - detection_name = detection.type if self.scan_type == SECRET_SCAN_TYPE else detection.message - - detection_severity = detection.severity or 'N/A' - detection_severity_color = SeverityOption.get_member_color(detection_severity) - detection_severity = f'[{detection_severity_color}]{detection_severity.upper()}[/{detection_severity_color}]' + name = detection.type if self.scan_type == SECRET_SCAN_TYPE else detection.message + severity = SeverityOption(detection.severity) if detection.severity else 'N/A' escaped_document_path = escape(urllib.parse.quote(document_path)) clickable_document_path = f'[link file://{escaped_document_path}]{document_path}' @@ -95,14 +88,14 @@ def _print_detection_summary(self, detection: Detection, document_path: str) -> company_guidelines = detection.detection_details.get('custom_remediation_guidelines') company_guidelines_message = f'\nCompany Guideline: {company_guidelines}' if company_guidelines else '' - Console().print( - f':no_entry: ' - f'Found {detection_severity} issue of type: [bright_red][bold]{detection_name}[/bold][/bright_red] ' + console.print( + ':no_entry: Found', + severity, + f'issue of type: [bright_red][bold]{name}[/bold][/bright_red] ' f'in file: {clickable_document_path} ' f'{detection_commit_id_message}' f'{company_guidelines_message}' f' :no_entry:', - highlight=True, ) def _print_detection_code_segment( @@ -120,12 +113,12 @@ def _print_report_urls(report_urls: List[str], aggregation_report_url: Optional[ if not report_urls and not aggregation_report_url: return if aggregation_report_url: - typer.echo(f'Report URL: {aggregation_report_url}') + console.print(f'Report URL: {aggregation_report_url}') return - typer.echo('Report URLs:') + console.print('Report URLs:') for report_url in report_urls: - typer.echo(f'- {report_url}') + console.print(f'- {report_url}') @staticmethod def _get_code_segment_start_line(detection_line: int, lines_to_display: int) -> int: @@ -163,8 +156,9 @@ def _print_detection_from_file(self, detection: Detection, document: Document, l code_lines_to_render.append(line_content) code_to_render = '\n'.join(code_lines_to_render) - Console().print( + console.print( Syntax( + theme=_SYNTAX_HIGHLIGHT_THEME, code=code_to_render, lexer=Syntax.guess_lexer(document.path, code=code_to_render), line_numbers=True, @@ -189,9 +183,10 @@ def _print_detection_from_git_diff(self, detection: Detection, document: Documen violation = line_content[detection_position_in_line : detection_position_in_line + violation_length] line_content = line_content.replace(violation, obfuscate_text(violation)) - Console().print( + console.print( Syntax( - line_content, + theme=_SYNTAX_HIGHLIGHT_THEME, + code=line_content, lexer='diff', line_numbers=True, start_line=detection_line, diff --git a/cycode/cli/utils/progress_bar.py b/cycode/cli/utils/progress_bar.py index e0fec5aa..054d5cf8 100644 --- a/cycode/cli/utils/progress_bar.py +++ b/cycode/cli/utils/progress_bar.py @@ -4,6 +4,7 @@ from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn, TimeElapsedColumn +from cycode.cli.console import console from cycode.cli.utils.enum_utils import AutoCountEnum from cycode.logger import get_logger @@ -135,7 +136,6 @@ class CompositeProgressBar(BaseProgressBar): def __init__(self, progress_bar_sections: ProgressBarSections) -> None: super().__init__() - self._run = False self._progress_bar_sections = progress_bar_sections self._section_lengths: Dict[ProgressBarSection, int] = {} @@ -145,7 +145,7 @@ def __init__(self, progress_bar_sections: ProgressBarSections) -> None: self._current_section: ProgressBarSectionInfo = _get_initial_section(self._progress_bar_sections) self._current_right_side_label = '' - self._progress_bar = Progress(*_PROGRESS_BAR_COLUMNS) + self._progress_bar = Progress(*_PROGRESS_BAR_COLUMNS, console=console, refresh_per_second=5, transient=True) self._progress_bar_task_id = self._progress_bar.add_task( description=self._current_section.label, total=_PROGRESS_BAR_LENGTH, @@ -158,15 +158,14 @@ def _progress_bar_update(self, advance: int = 0) -> None: advance=advance, description=self._current_section.label, right_side_label=self._current_right_side_label, + refresh=True, ) def start(self) -> None: - if not self._run: - self._progress_bar.start() + self._progress_bar.start() def stop(self) -> None: - if self._run: - self._progress_bar.stop() + self._progress_bar.stop() def set_section_length(self, section: 'ProgressBarSection', length: int = 0) -> None: logger.debug('Calling set_section_length, %s', {'section': str(section), 'length': length}) diff --git a/cycode/cli/utils/shell_executor.py b/cycode/cli/utils/shell_executor.py index a7a537e6..812fee1f 100644 --- a/cycode/cli/utils/shell_executor.py +++ b/cycode/cli/utils/shell_executor.py @@ -2,6 +2,7 @@ from typing import List, Optional, Union import click +import typer from cycode.logger import get_logger @@ -29,7 +30,7 @@ def shell( logger.debug('Error occurred while running shell command', exc_info=e) except subprocess.TimeoutExpired as e: logger.debug('Command timed out', exc_info=e) - raise click.Abort(f'Command "{command}" timed out') from e + raise typer.Abort(f'Command "{command}" timed out') from e except Exception as e: logger.debug('Unhandled exception occurred while running shell command', exc_info=e) raise click.ClickException(f'Unhandled exception: {e}') from e diff --git a/cycode/cli/utils/version_checker.py b/cycode/cli/utils/version_checker.py index 40022cbd..f35e53a2 100644 --- a/cycode/cli/utils/version_checker.py +++ b/cycode/cli/utils/version_checker.py @@ -4,8 +4,7 @@ from pathlib import Path from typing import List, Optional, Tuple -import typer - +from cycode.cli.console import console from cycode.cli.user_settings.configuration_manager import ConfigurationManager from cycode.cli.utils.path_utils import get_file_content from cycode.cyclient.cycode_client_base import CycodeClientBase @@ -181,7 +180,7 @@ def check_for_update(self, current_version: str, use_cache: bool = True) -> Opti latest_parts, latest_is_pre = self._parse_version(latest_version) return _compare_versions(current_parts, latest_parts, current_is_pre, latest_is_pre, latest_version) - def check_and_notify_update(self, current_version: str, use_color: bool = True, use_cache: bool = True) -> None: + def check_and_notify_update(self, current_version: str, use_cache: bool = True) -> None: """Check for updates and display a notification if a new version is available. Performs the version check and displays a formatted message with update instructions @@ -192,7 +191,6 @@ def check_and_notify_update(self, current_version: str, use_color: bool = True, Args: current_version: Current version of the CLI - use_color: If True, use colored output in the terminal use_cache: If True, use the cached timestamp to determine if an update check is needed """ latest_version = self.check_for_update(current_version, use_cache) @@ -200,11 +198,11 @@ def check_and_notify_update(self, current_version: str, use_color: bool = True, if should_update: update_message = ( '\nNew version of cycode available! ' - f"{typer.style(current_version, fg='yellow')} → {typer.style(latest_version, fg='bright_blue')}\n" - f"Changelog: {typer.style(f'{self.GIT_CHANGELOG_URL_PREFIX}{latest_version}', fg='bright_blue')}\n" - f"Run {typer.style('pip install --upgrade cycode', fg='green')} to update\n" + f'[yellow]{current_version}[/yellow] → [bright_blue]{latest_version}[/bright_blue]\n' + f'Changelog: [bright_blue]{self.GIT_CHANGELOG_URL_PREFIX}{latest_version}[/bright_blue]\n' + f'Run [green]pip install --upgrade cycode[/green] to update\n' ) - typer.echo(update_message, color=use_color) + console.print(update_message) version_checker = VersionChecker() diff --git a/cycode/logger.py b/cycode/logger.py index 684d296c..b63c796f 100644 --- a/cycode/logger.py +++ b/cycode/logger.py @@ -4,10 +4,10 @@ import click import typer -from rich.console import Console from rich.logging import RichHandler from cycode.cli import consts +from cycode.cli.console import console_err from cycode.config import get_val_as_string @@ -19,8 +19,7 @@ def _set_io_encodings() -> None: _set_io_encodings() -_ERROR_CONSOLE = Console(stderr=True) -_RICH_LOGGING_HANDLER = RichHandler(console=_ERROR_CONSOLE, rich_tracebacks=True, tracebacks_suppress=[click, typer]) +_RICH_LOGGING_HANDLER = RichHandler(console=console_err, rich_tracebacks=True, tracebacks_suppress=[click, typer]) logging.basicConfig( level=logging.INFO, diff --git a/tests/cli/commands/test_main_command.py b/tests/cli/commands/test_main_command.py index 04bc3e01..d9890268 100644 --- a/tests/cli/commands/test_main_command.py +++ b/tests/cli/commands/test_main_command.py @@ -47,7 +47,7 @@ def test_passing_output_option(output: str, scan_client: 'ScanClient', api_token if except_json: output = json.loads(result.output) - assert 'scan_id' in output + assert 'scan_ids' in output else: assert 'issue of type:' in result.output diff --git a/tests/cli/exceptions/test_handle_scan_errors.py b/tests/cli/exceptions/test_handle_scan_errors.py index c1d34306..a6b2a9ec 100644 --- a/tests/cli/exceptions/test_handle_scan_errors.py +++ b/tests/cli/exceptions/test_handle_scan_errors.py @@ -1,12 +1,13 @@ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import click import pytest import typer -from click import ClickException from requests import Response +from rich.traceback import Traceback from cycode.cli.cli_types import OutputTypeOption +from cycode.cli.console import console_err from cycode.cli.exceptions import custom_exceptions from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception from cycode.cli.utils.git_proxy import git_proxy @@ -50,7 +51,7 @@ def test_handle_exception_unhandled_error(ctx: typer.Context) -> None: def test_handle_exception_click_error(ctx: typer.Context) -> None: - with ctx, pytest.raises(ClickException): + with ctx, pytest.raises(click.ClickException): handle_scan_exception(ctx, click.ClickException('test')) assert ctx.obj.get('did_fail') is True @@ -62,10 +63,14 @@ def test_handle_exception_verbose(monkeypatch: 'MonkeyPatch') -> None: error_text = 'test' - def mock_secho(msg: str, *_, **__) -> None: - assert error_text in msg or 'Correlation ID:' in msg + def mock_console_print(obj: Any, *_, **__) -> None: + if isinstance(obj, str): + assert 'Correlation ID:' in obj + else: + assert isinstance(obj, Traceback) + assert error_text in str(obj.trace) - monkeypatch.setattr(click, 'secho', mock_secho) + monkeypatch.setattr(console_err, 'print', mock_console_print) with pytest.raises(typer.Exit): handle_scan_exception(ctx, ValueError(error_text)) From 0b32c0d4c5c170b462d236ad464af958ecd84377 Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Wed, 9 Apr 2025 10:46:23 +0200 Subject: [PATCH 09/19] CM-46370 - Add the error message from the server responses to the user space (#294) --- cycode/cli/apps/ignore/ignore_command.py | 4 ++-- cycode/cli/apps/scan/code_scanner.py | 2 +- cycode/cli/apps/scan/scan_command.py | 7 +++---- cycode/cli/apps/status/version_command.py | 2 +- cycode/cli/cli_types.py | 2 +- cycode/cli/exceptions/custom_exceptions.py | 17 +++++++++-------- cycode/cli/exceptions/handle_errors.py | 2 +- cycode/cli/models.py | 4 ++++ cycode/cli/printers/printer_base.py | 6 +++--- cycode/cli/printers/tables/sca_table_printer.py | 2 +- cycode/cli/printers/tables/table.py | 2 +- cycode/cli/printers/text_printer.py | 4 ++-- cycode/cli/utils/version_checker.py | 6 +++--- 13 files changed, 32 insertions(+), 28 deletions(-) diff --git a/cycode/cli/apps/ignore/ignore_command.py b/cycode/cli/apps/ignore/ignore_command.py index cc6ecd25..079a3c2d 100644 --- a/cycode/cli/apps/ignore/ignore_command.py +++ b/cycode/cli/apps/ignore/ignore_command.py @@ -57,7 +57,7 @@ def ignore_command( # noqa: C901 by_package: Annotated[ Optional[str], typer.Option( - help='Ignore scanning a specific package version. Expected pattern: [cyan]name@version[/cyan].', + help='Ignore scanning a specific package version. Expected pattern: [cyan]name@version[/].', show_default=False, rich_help_panel=_SCA_FILTER_BY_RICH_HELP_PANEL, ), @@ -65,7 +65,7 @@ def ignore_command( # noqa: C901 by_cve: Annotated[ Optional[str], typer.Option( - help='Ignore scanning a specific CVE. Expected pattern: [cyan]CVE-YYYY-NNN[/cyan].', + help='Ignore scanning a specific CVE. Expected pattern: [cyan]CVE-YYYY-NNN[/].', show_default=False, rich_help_panel=_SCA_FILTER_BY_RICH_HELP_PANEL, ), diff --git a/cycode/cli/apps/scan/code_scanner.py b/cycode/cli/apps/scan/code_scanner.py index a44d9c15..35208d59 100644 --- a/cycode/cli/apps/scan/code_scanner.py +++ b/cycode/cli/apps/scan/code_scanner.py @@ -649,7 +649,7 @@ def _get_default_scan_parameters(ctx: typer.Context) -> dict: 'report': ctx.obj.get('report'), 'package_vulnerabilities': ctx.obj.get('package-vulnerabilities'), 'license_compliance': ctx.obj.get('license-compliance'), - 'command_type': ctx.info_name, + 'command_type': ctx.info_name.replace('-', '_'), # save backward compatibility 'aggregation_id': str(_generate_unique_id()), } diff --git a/cycode/cli/apps/scan/scan_command.py b/cycode/cli/apps/scan/scan_command.py index 6b776689..3ba7699b 100644 --- a/cycode/cli/apps/scan/scan_command.py +++ b/cycode/cli/apps/scan/scan_command.py @@ -83,8 +83,7 @@ def scan_command( bool, typer.Option( '--no-restore', - help='When specified, Cycode will not run restore command. ' - 'Will scan direct dependencies [bold]only[/bold]!', + help='When specified, Cycode will not run restore command. ' 'Will scan direct dependencies [b]only[/]!', rich_help_panel=_SCA_RICH_HELP_PANEL, ), ] = False, @@ -93,14 +92,14 @@ def scan_command( typer.Option( '--gradle-all-sub-projects', help='When specified, Cycode will run gradle restore command for all sub projects. ' - 'Should run from root project directory [bold]only[/bold]!', + 'Should run from root project directory [b]only[/]!', rich_help_panel=_SCA_RICH_HELP_PANEL, ), ] = False, ) -> None: """:magnifying_glass_tilted_right: Scan the content for Secrets, IaC, SCA, and SAST violations. You'll need to specify which scan type to perform: - [cyan]path[/cyan]/[cyan]repository[/cyan]/[cyan]commit_history[/cyan].""" + [cyan]path[/]/[cyan]repository[/]/[cyan]commit_history[/].""" add_breadcrumb('scan') ctx.obj['show_secret'] = show_secret diff --git a/cycode/cli/apps/status/version_command.py b/cycode/cli/apps/status/version_command.py index 272b4a17..e5982548 100644 --- a/cycode/cli/apps/status/version_command.py +++ b/cycode/cli/apps/status/version_command.py @@ -5,6 +5,6 @@ def version_command(ctx: typer.Context) -> None: - console.print('[yellow][bold]This command is deprecated. Please use the "status" command instead.[/bold][/yellow]') + console.print('[b yellow]This command is deprecated. Please use the "status" command instead.[/]') console.print() # print an empty line status_command(ctx) diff --git a/cycode/cli/cli_types.py b/cycode/cli/cli_types.py index 87c6346b..67ae2fb2 100644 --- a/cycode/cli/cli_types.py +++ b/cycode/cli/cli_types.py @@ -57,7 +57,7 @@ def get_member_color(name: str) -> str: def __rich__(self) -> str: color = self.get_member_color(self.value) - return f'[{color}]{self.value.upper()}[/{color}]' + return f'[{color}]{self.value.upper()}[/]' _SEVERITY_DEFAULT_WEIGHT = -1 diff --git a/cycode/cli/exceptions/custom_exceptions.py b/cycode/cli/exceptions/custom_exceptions.py index 40abed63..4d692812 100644 --- a/cycode/cli/exceptions/custom_exceptions.py +++ b/cycode/cli/exceptions/custom_exceptions.py @@ -6,6 +6,10 @@ class CycodeError(Exception): """Base class for all custom exceptions""" + def __str__(self) -> str: + class_name = self.__class__.__name__ + return f'{class_name} error occurred.' + class RequestError(CycodeError): ... @@ -27,10 +31,7 @@ def __init__(self, status_code: int, error_message: str, response: Response) -> super().__init__(self.error_message) def __str__(self) -> str: - return ( - f'error occurred during the request. status code: {self.status_code}, error message: ' - f'{self.error_message}' - ) + return f'HTTP error occurred during the request (code {self.status_code}). Message: {self.error_message}' class ScanAsyncError(CycodeError): @@ -39,7 +40,7 @@ def __init__(self, error_message: str) -> None: super().__init__(self.error_message) def __str__(self) -> str: - return f'error occurred during the scan. error message: {self.error_message}' + return f'Async scan error occurred during the scan. Message: {self.error_message}' class ReportAsyncError(CycodeError): @@ -54,7 +55,7 @@ def __init__(self, error_message: str, response: Response) -> None: super().__init__(self.error_message) def __str__(self) -> str: - return 'Http Unauthorized Error' + return f'HTTP unauthorized error occurred during the request. Message: {self.error_message}' class ZipTooLargeError(CycodeError): @@ -72,7 +73,7 @@ def __init__(self, error_message: str) -> None: super().__init__() def __str__(self) -> str: - return f'Something went wrong during the authentication process, error message: {self.error_message}' + return f'Something went wrong during the authentication process. Message: {self.error_message}' class TfplanKeyError(CycodeError): @@ -106,6 +107,6 @@ def __str__(self) -> str: code='ssl_error', message='An SSL error occurred when trying to connect to the Cycode API. ' 'If you use an on-premises installation or a proxy that intercepts SSL traffic ' - 'you should use the CURL_CA_BUNDLE environment variable to specify path to a valid .pem or similar.', + 'you should use the CURL_CA_BUNDLE environment variable to specify path to a valid .pem or similar', ), } diff --git a/cycode/cli/exceptions/handle_errors.py b/cycode/cli/exceptions/handle_errors.py index db102773..b9cb9c80 100644 --- a/cycode/cli/exceptions/handle_errors.py +++ b/cycode/cli/exceptions/handle_errors.py @@ -14,7 +14,7 @@ def handle_errors( ConsolePrinter(ctx).print_exception(err) if type(err) in cli_errors: - error = cli_errors[type(err)] + error = cli_errors[type(err)].enrich(additional_message=str(err)) if error.soft_fail is True: ctx.obj['soft_fail'] = True diff --git a/cycode/cli/models.py b/cycode/cli/models.py index df62583a..14058f0c 100644 --- a/cycode/cli/models.py +++ b/cycode/cli/models.py @@ -37,6 +37,10 @@ class CliError(NamedTuple): message: str soft_fail: bool = False + def enrich(self, additional_message: str) -> 'CliError': + message = f'{self.message} ({additional_message})' + return CliError(self.code, message, self.soft_fail) + CliErrors = Dict[Type[BaseException], CliError] diff --git a/cycode/cli/printers/printer_base.py b/cycode/cli/printers/printer_base.py index c6c5120b..9a86c3d6 100644 --- a/cycode/cli/printers/printer_base.py +++ b/cycode/cli/printers/printer_base.py @@ -17,11 +17,11 @@ class PrinterBase(ABC): NO_DETECTIONS_MESSAGE = ( - '[green]Good job! No issues were found!!! :clapping_hands::clapping_hands::clapping_hands:[/green]' + '[green]Good job! No issues were found!!! :clapping_hands::clapping_hands::clapping_hands:[/]' ) FAILED_SCAN_MESSAGE = ( '[red]Unfortunately, Cycode was unable to complete the full scan. ' - 'Please note that not all results may be available:[/red]' + 'Please note that not all results may be available:[/]' ) def __init__(self, ctx: typer.Context) -> None: @@ -56,4 +56,4 @@ def print_exception(e: Optional[BaseException] = None) -> None: rich_traceback.show_locals = False console_err.print(rich_traceback) - console_err.print(f'[red]Correlation ID:[/red] {get_correlation_id()}') + console_err.print(f'[red]Correlation ID:[/] {get_correlation_id()}') diff --git a/cycode/cli/printers/tables/sca_table_printer.py b/cycode/cli/printers/tables/sca_table_printer.py index f5c38279..1ba1c3a3 100644 --- a/cycode/cli/printers/tables/sca_table_printer.py +++ b/cycode/cli/printers/tables/sca_table_printer.py @@ -178,7 +178,7 @@ def _enrich_table_with_values(policy_id: str, table: Table, detection: Detection @staticmethod def _print_summary_issues(detections_count: int, title: str) -> None: - console.print(f':no_entry: Found {detections_count} issues of type: [bold]{title}[/bold]') + console.print(f':no_entry: Found {detections_count} issues of type: [b]{title}[/]') @staticmethod def _extract_detections_per_policy_id( diff --git a/cycode/cli/printers/tables/table.py b/cycode/cli/printers/tables/table.py index 23022b2d..b89df4af 100644 --- a/cycode/cli/printers/tables/table.py +++ b/cycode/cli/printers/tables/table.py @@ -28,7 +28,7 @@ def _add_cell_no_error(self, column: 'ColumnInfo', value: str) -> None: def add_cell(self, column: 'ColumnInfo', value: str, color: Optional[str] = None) -> None: if color: - value = f'[{color}]{value}[/{color}]' + value = f'[{color}]{value}[/]' self._add_cell_no_error(column, value) diff --git a/cycode/cli/printers/text_printer.py b/cycode/cli/printers/text_printer.py index 75725aac..7f1db10f 100644 --- a/cycode/cli/printers/text_printer.py +++ b/cycode/cli/printers/text_printer.py @@ -39,7 +39,7 @@ def print_result(self, result: CliResult) -> None: console.print(f'- {name}: {value}', style=color) def print_error(self, error: CliError) -> None: - console.print(f'[red]Error: {error.message}[/red]') + console.print(f'[red]Error: {error.message}[/]', highlight=False) def print_scan_results( self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None @@ -91,7 +91,7 @@ def _print_detection_summary(self, detection: Detection, document_path: str) -> console.print( ':no_entry: Found', severity, - f'issue of type: [bright_red][bold]{name}[/bold][/bright_red] ' + f'issue of type: [b bright_red]{name}[/] ' f'in file: {clickable_document_path} ' f'{detection_commit_id_message}' f'{company_guidelines_message}' diff --git a/cycode/cli/utils/version_checker.py b/cycode/cli/utils/version_checker.py index f35e53a2..035b3595 100644 --- a/cycode/cli/utils/version_checker.py +++ b/cycode/cli/utils/version_checker.py @@ -198,9 +198,9 @@ def check_and_notify_update(self, current_version: str, use_cache: bool = True) if should_update: update_message = ( '\nNew version of cycode available! ' - f'[yellow]{current_version}[/yellow] → [bright_blue]{latest_version}[/bright_blue]\n' - f'Changelog: [bright_blue]{self.GIT_CHANGELOG_URL_PREFIX}{latest_version}[/bright_blue]\n' - f'Run [green]pip install --upgrade cycode[/green] to update\n' + f'[yellow]{current_version}[/] → [bright_blue]{latest_version}[/]\n' + f'Changelog: [bright_blue]{self.GIT_CHANGELOG_URL_PREFIX}{latest_version}[/]\n' + f'Run [green]pip install --upgrade cycode[/] to update\n' ) console.print(update_message) From dad785913731fd3f523805c8ec08272f8a1dbe6b Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Tue, 15 Apr 2025 14:22:11 +0200 Subject: [PATCH 10/19] CM-46732 - Add rich output; improve text output (#295) --- cycode/cli/app.py | 2 +- cycode/cli/apps/status/version_command.py | 2 +- cycode/cli/cli_types.py | 14 ++ cycode/cli/console.py | 26 ++- cycode/cli/consts.py | 16 +- cycode/cli/printers/console_printer.py | 7 + cycode/cli/printers/rich_printer.py | 141 +++++++++++++++ .../cli/printers/tables/sca_table_printer.py | 53 +----- cycode/cli/printers/tables/table_printer.py | 54 +----- cycode/cli/printers/text_printer.py | 168 +++++------------- cycode/cli/printers/utils/__init__.py | 0 .../cli/printers/utils/code_snippet_syntax.py | 113 ++++++++++++ cycode/cli/printers/utils/detection_data.py | 16 ++ .../utils/detection_ordering/__init__.py | 0 .../detection_ordering/common_ordering.py | 66 +++++++ .../utils/detection_ordering/sca_ordering.py | 58 ++++++ cycode/cli/printers/utils/rich_helpers.py | 37 ++++ tests/cli/commands/test_main_command.py | 2 +- 18 files changed, 540 insertions(+), 235 deletions(-) create mode 100644 cycode/cli/printers/rich_printer.py create mode 100644 cycode/cli/printers/utils/__init__.py create mode 100644 cycode/cli/printers/utils/code_snippet_syntax.py create mode 100644 cycode/cli/printers/utils/detection_data.py create mode 100644 cycode/cli/printers/utils/detection_ordering/__init__.py create mode 100644 cycode/cli/printers/utils/detection_ordering/common_ordering.py create mode 100644 cycode/cli/printers/utils/detection_ordering/sca_ordering.py create mode 100644 cycode/cli/printers/utils/rich_helpers.py diff --git a/cycode/cli/app.py b/cycode/cli/app.py index 80742bab..aed0e172 100644 --- a/cycode/cli/app.py +++ b/cycode/cli/app.py @@ -59,7 +59,7 @@ def app_callback( ] = False, output: Annotated[ OutputTypeOption, typer.Option('--output', '-o', case_sensitive=False, help='Specify the output type.') - ] = OutputTypeOption.TEXT, + ] = OutputTypeOption.RICH, user_agent: Annotated[ Optional[str], typer.Option(hidden=True, help='Characteristic JSON object that lets servers identify the application.'), diff --git a/cycode/cli/apps/status/version_command.py b/cycode/cli/apps/status/version_command.py index e5982548..ef117fc7 100644 --- a/cycode/cli/apps/status/version_command.py +++ b/cycode/cli/apps/status/version_command.py @@ -6,5 +6,5 @@ def version_command(ctx: typer.Context) -> None: console.print('[b yellow]This command is deprecated. Please use the "status" command instead.[/]') - console.print() # print an empty line + console.line() status_command(ctx) diff --git a/cycode/cli/cli_types.py b/cycode/cli/cli_types.py index 67ae2fb2..2a576ace 100644 --- a/cycode/cli/cli_types.py +++ b/cycode/cli/cli_types.py @@ -4,6 +4,7 @@ class OutputTypeOption(str, Enum): + RICH = 'rich' TEXT = 'text' JSON = 'json' TABLE = 'table' @@ -55,6 +56,10 @@ def get_member_weight(name: str) -> int: def get_member_color(name: str) -> str: return _SEVERITY_COLORS.get(name.lower(), _SEVERITY_DEFAULT_COLOR) + @staticmethod + def get_member_emoji(name: str) -> str: + return _SEVERITY_EMOJIS.get(name.lower(), _SEVERITY_DEFAULT_EMOJI) + def __rich__(self) -> str: color = self.get_member_color(self.value) return f'[{color}]{self.value.upper()}[/]' @@ -77,3 +82,12 @@ def __rich__(self) -> str: SeverityOption.HIGH.value: 'red1', SeverityOption.CRITICAL.value: 'red3', } + +_SEVERITY_DEFAULT_EMOJI = ':white_circle:' +_SEVERITY_EMOJIS = { + SeverityOption.INFO.value: ':blue_circle:', + SeverityOption.LOW.value: ':yellow_circle:', + SeverityOption.MEDIUM.value: ':orange_circle:', + SeverityOption.HIGH.value: ':heavy_large_circle:', + SeverityOption.CRITICAL.value: ':red_circle:', +} diff --git a/cycode/cli/console.py b/cycode/cli/console.py index 159f4733..5d78fc36 100644 --- a/cycode/cli/console.py +++ b/cycode/cli/console.py @@ -1,7 +1,12 @@ import os -from typing import Optional +from typing import TYPE_CHECKING, Optional -from rich.console import Console +from rich.console import Console, RenderResult +from rich.markdown import Heading, Markdown +from rich.text import Text + +if TYPE_CHECKING: + from rich.console import ConsoleOptions console_out = Console() console_err = Console(stderr=True) @@ -45,3 +50,20 @@ def is_dark_console() -> Optional[bool]: # when we could not detect it, use dark theme as most terminals are dark _SYNTAX_HIGHLIGHT_THEME = _SYNTAX_HIGHLIGHT_LIGHT_THEME if is_dark_console() is False else _SYNTAX_HIGHLIGHT_DARK_THEME + + +class CycodeHeading(Heading): + """Custom Rich Heading for Markdown. + + Changes: + - remove justify to 'center' + - remove the box for h1 + """ + + def __rich_console__(self, console: 'Console', options: 'ConsoleOptions') -> RenderResult: + if self.tag == 'h2': + yield Text('') + yield self.text + + +Markdown.elements['heading_open'] = CycodeHeading diff --git a/cycode/cli/consts.py b/cycode/cli/consts.py index 60953143..9d7a619d 100644 --- a/cycode/cli/consts.py +++ b/cycode/cli/consts.py @@ -2,9 +2,12 @@ APP_NAME = 'CycodeCLI' CLI_CONTEXT_SETTINGS = {'terminal_width': 10**9, 'max_content_width': 10**9, 'help_option_names': ['-h', '--help']} -PRE_COMMIT_COMMAND_SCAN_TYPE = 'pre_commit' -PRE_RECEIVE_COMMAND_SCAN_TYPE = 'pre_receive' -COMMIT_HISTORY_COMMAND_SCAN_TYPE = 'commit_history' +PRE_COMMIT_COMMAND_SCAN_TYPE = 'pre-commit' +PRE_COMMIT_COMMAND_SCAN_TYPE_OLD = 'pre_commit' +PRE_RECEIVE_COMMAND_SCAN_TYPE = 'pre-receive' +PRE_RECEIVE_COMMAND_SCAN_TYPE_OLD = 'pre_receive' +COMMIT_HISTORY_COMMAND_SCAN_TYPE = 'commit-history' +COMMIT_HISTORY_COMMAND_SCAN_TYPE_OLD = 'commit_history' SECRET_SCAN_TYPE = 'secret' # noqa: S105 IAC_SCAN_TYPE = 'iac' @@ -105,7 +108,12 @@ COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES = [SECRET_SCAN_TYPE, SCA_SCAN_TYPE] -COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES = [PRE_RECEIVE_COMMAND_SCAN_TYPE, COMMIT_HISTORY_COMMAND_SCAN_TYPE] +COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES = [ + PRE_RECEIVE_COMMAND_SCAN_TYPE, + PRE_RECEIVE_COMMAND_SCAN_TYPE_OLD, + COMMIT_HISTORY_COMMAND_SCAN_TYPE, + COMMIT_HISTORY_COMMAND_SCAN_TYPE_OLD, +] DEFAULT_CYCODE_DOMAIN = 'cycode.com' DEFAULT_CYCODE_API_URL = f'https://api.{DEFAULT_CYCODE_DOMAIN}' diff --git a/cycode/cli/printers/console_printer.py b/cycode/cli/printers/console_printer.py index 64efa9d5..5ad5dac2 100644 --- a/cycode/cli/printers/console_printer.py +++ b/cycode/cli/printers/console_printer.py @@ -5,6 +5,7 @@ from cycode.cli.exceptions.custom_exceptions import CycodeError from cycode.cli.models import CliError, CliResult from cycode.cli.printers.json_printer import JsonPrinter +from cycode.cli.printers.rich_printer import RichPrinter from cycode.cli.printers.tables.sca_table_printer import ScaTablePrinter from cycode.cli.printers.tables.table_printer import TablePrinter from cycode.cli.printers.text_printer import TextPrinter @@ -16,12 +17,14 @@ class ConsolePrinter: _AVAILABLE_PRINTERS: ClassVar[Dict[str, Type['PrinterBase']]] = { + 'rich': RichPrinter, 'text': TextPrinter, 'json': JsonPrinter, 'table': TablePrinter, # overrides 'table_sca': ScaTablePrinter, 'text_sca': ScaTablePrinter, + 'rich_sca': ScaTablePrinter, } def __init__(self, ctx: typer.Context) -> None: @@ -74,3 +77,7 @@ def is_table_printer(self) -> bool: @property def is_text_printer(self) -> bool: return self._printer_class == TextPrinter + + @property + def is_rich_printer(self) -> bool: + return self._printer_class == RichPrinter diff --git a/cycode/cli/printers/rich_printer.py b/cycode/cli/printers/rich_printer.py new file mode 100644 index 00000000..61cb14ef --- /dev/null +++ b/cycode/cli/printers/rich_printer.py @@ -0,0 +1,141 @@ +from pathlib import Path +from typing import TYPE_CHECKING, Dict, List, Optional + +from rich.console import Group +from rich.panel import Panel +from rich.table import Table +from rich.text import Text + +from cycode.cli import consts +from cycode.cli.cli_types import SeverityOption +from cycode.cli.console import console +from cycode.cli.printers.text_printer import TextPrinter +from cycode.cli.printers.utils.code_snippet_syntax import get_code_snippet_syntax +from cycode.cli.printers.utils.detection_data import get_detection_title +from cycode.cli.printers.utils.detection_ordering.common_ordering import sort_and_group_detections_from_scan_result +from cycode.cli.printers.utils.rich_helpers import get_columns_in_1_to_3_ratio, get_markdown_panel, get_panel + +if TYPE_CHECKING: + from cycode.cli.models import CliError, Detection, Document, LocalScanResult + + +class RichPrinter(TextPrinter): + def print_scan_results( + self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None + ) -> None: + if not errors and all(result.issue_detected == 0 for result in local_scan_results): + console.print(self.NO_DETECTIONS_MESSAGE) + return + + current_file = None + detections, _ = sort_and_group_detections_from_scan_result(local_scan_results) + detections_count = len(detections) + for detection_number, (detection, document) in enumerate(detections, start=1): + if current_file != document.path: + current_file = document.path + self._print_file_header(current_file) + + self._print_violation_card( + document, + detection, + detection_number, + detections_count, + ) + + self.print_report_urls_and_errors(local_scan_results, errors) + + @staticmethod + def _print_file_header(file_path: str) -> None: + clickable_path = f'[link=file://{file_path}]{file_path}[/link]' + file_header = Panel( + Text.from_markup(f'[b purple3]:file_folder: File: {clickable_path}[/]', justify='center'), + border_style='dim', + ) + console.print(file_header) + + def _get_details_table(self, detection: 'Detection') -> Table: + details_table = Table(show_header=False, box=None, padding=(0, 1)) + + details_table.add_column('Key', style='dim') + details_table.add_column('Value', style='', overflow='fold') + + severity = detection.severity if detection.severity else 'N/A' + severity_icon = SeverityOption.get_member_emoji(severity.lower()) + details_table.add_row('Severity', f'{severity_icon} {SeverityOption(severity).__rich__()}') + + detection_details = detection.detection_details + path = Path(detection_details.get('file_name', '')) + details_table.add_row('In file', path.name) # it is name already except for IaC :) + + # we do not allow using rich output with SCA; SCA designed to be used with table output + if self.scan_type == consts.IAC_SCAN_TYPE: + details_table.add_row('IaC Provider', detection_details.get('infra_provider')) + elif self.scan_type == consts.SECRET_SCAN_TYPE: + details_table.add_row('Secret SHA', detection_details.get('sha512')) + elif self.scan_type == consts.SAST_SCAN_TYPE: + details_table.add_row('Subcategory', detection_details.get('category')) + details_table.add_row('Language', ', '.join(detection_details.get('languages', []))) + + engine_id_to_display_name = { + '5db84696-88dc-11ec-a8a3-0242ac120002': 'Semgrep OSS (Orchestrated by Cycode)', + '560a0abd-d7da-4e6d-a3f1-0ed74895295c': 'Bearer (Powered by Cycode)', + } + engine_id = detection.detection_details.get('external_scanner_id') + details_table.add_row('Security Tool', engine_id_to_display_name.get(engine_id, 'N/A')) + + details_table.add_row('Rule ID', detection.detection_rule_id) + + return details_table + + def _print_violation_card( + self, document: 'Document', detection: 'Detection', detection_number: int, detections_count: int + ) -> None: + details_table = self._get_details_table(detection) + details_panel = get_panel( + details_table, + title=':mag: Details', + ) + + code_snippet_panel = get_panel( + get_code_snippet_syntax( + self.scan_type, + self.command_scan_type, + detection, + document, + obfuscate=not self.show_secret, + ), + title=':computer: Code Snippet', + ) + + guidelines_panel = None + guidelines = detection.detection_details.get('remediation_guidelines') + if guidelines: + guidelines_panel = get_markdown_panel( + guidelines, + title=':clipboard: Cycode Guidelines', + ) + + custom_guidelines_panel = None + custom_guidelines = detection.detection_details.get('custom_remediation_guidelines') + if custom_guidelines: + custom_guidelines_panel = get_markdown_panel( + custom_guidelines, + title=':office: Company Guidelines', + ) + + navigation = Text(f'Violation {detection_number} of {detections_count}', style='dim', justify='right') + + renderables = [navigation, get_columns_in_1_to_3_ratio(details_panel, code_snippet_panel)] + if guidelines_panel: + renderables.append(guidelines_panel) + if custom_guidelines_panel: + renderables.append(custom_guidelines_panel) + + violation_card_panel = Panel( + Group(*renderables), + title=get_detection_title(self.scan_type, detection), + border_style=SeverityOption.get_member_color(detection.severity), + title_align='center', + ) + + console.print(violation_card_panel) diff --git a/cycode/cli/printers/tables/sca_table_printer.py b/cycode/cli/printers/tables/sca_table_printer.py index 1ba1c3a3..70965be2 100644 --- a/cycode/cli/printers/tables/sca_table_printer.py +++ b/cycode/cli/printers/tables/sca_table_printer.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import TYPE_CHECKING, Dict, List, Set, Tuple +from typing import TYPE_CHECKING, Dict, List from cycode.cli.cli_types import SeverityOption from cycode.cli.console import console @@ -8,6 +8,7 @@ from cycode.cli.printers.tables.table import Table from cycode.cli.printers.tables.table_models import ColumnInfoBuilder from cycode.cli.printers.tables.table_printer_base import TablePrinterBase +from cycode.cli.printers.utils.detection_ordering.sca_ordering import sort_and_group_detections from cycode.cli.utils.string_utils import shortcut_dependency_paths if TYPE_CHECKING: @@ -36,7 +37,7 @@ def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: for policy_id, detections in detections_per_policy_id.items(): table = self._get_table(policy_id) - resulting_detections, group_separator_indexes = self._sort_and_group_detections(detections) + resulting_detections, group_separator_indexes = sort_and_group_detections(detections) for detection in resulting_detections: self._enrich_table_with_values(policy_id, table, detection) @@ -56,54 +57,6 @@ def _get_title(policy_id: str) -> str: return 'Unknown' - @staticmethod - def __group_by(detections: List[Detection], details_field_name: str) -> Dict[str, List[Detection]]: - grouped = defaultdict(list) - for detection in detections: - grouped[detection.detection_details.get(details_field_name)].append(detection) - return grouped - - @staticmethod - def __severity_sort_key(detection: Detection) -> int: - severity = detection.detection_details.get('advisory_severity', 'unknown') - return SeverityOption.get_member_weight(severity) - - def _sort_detections_by_severity(self, detections: List[Detection]) -> List[Detection]: - return sorted(detections, key=self.__severity_sort_key, reverse=True) - - @staticmethod - def __package_sort_key(detection: Detection) -> int: - return detection.detection_details.get('package_name') - - def _sort_detections_by_package(self, detections: List[Detection]) -> List[Detection]: - return sorted(detections, key=self.__package_sort_key) - - def _sort_and_group_detections(self, detections: List[Detection]) -> Tuple[List[Detection], Set[int]]: - """Sort detections by severity and group by repository, code project and package name. - - Note: - Code Project is path to the manifest file. - - Grouping by code projects also groups by ecosystem. - Because manifest files are unique per ecosystem. - """ - resulting_detections = [] - group_separator_indexes = set() - - # we sort detections by package name to make persist output order - sorted_detections = self._sort_detections_by_package(detections) - - grouped_by_repository = self.__group_by(sorted_detections, 'repository_name') - for repository_group in grouped_by_repository.values(): - grouped_by_code_project = self.__group_by(repository_group, 'file_name') - for code_project_group in grouped_by_code_project.values(): - grouped_by_package = self.__group_by(code_project_group, 'package_name') - for package_group in grouped_by_package.values(): - group_separator_indexes.add(len(resulting_detections) - 1) # indexing starts from 0 - resulting_detections.extend(self._sort_detections_by_severity(package_group)) - - return resulting_detections, group_separator_indexes - def _get_table(self, policy_id: str) -> Table: table = Table() diff --git a/cycode/cli/printers/tables/table_printer.py b/cycode/cli/printers/tables/table_printer.py index 12e2dbf3..e36b1b01 100644 --- a/cycode/cli/printers/tables/table_printer.py +++ b/cycode/cli/printers/tables/table_printer.py @@ -1,5 +1,4 @@ -from collections import defaultdict -from typing import TYPE_CHECKING, List, Set, Tuple +from typing import TYPE_CHECKING, List from cycode.cli.cli_types import SeverityOption from cycode.cli.consts import SECRET_SCAN_TYPE @@ -7,6 +6,7 @@ from cycode.cli.printers.tables.table import Table from cycode.cli.printers.tables.table_models import ColumnInfoBuilder from cycode.cli.printers.tables.table_printer_base import TablePrinterBase +from cycode.cli.printers.utils.detection_ordering.common_ordering import sort_and_group_detections_from_scan_result from cycode.cli.utils.string_utils import get_position_in_line, obfuscate_text if TYPE_CHECKING: @@ -30,14 +30,7 @@ class TablePrinter(TablePrinterBase): def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: table = self._get_table() - detections_with_documents = [] - for local_scan_result in local_scan_results: - for document_detections in local_scan_result.document_detections: - detections_with_documents.extend( - [(detection, document_detections.document) for detection in document_detections.detections] - ) - - detections, group_separator_indexes = self._sort_and_group_detections(detections_with_documents) + detections, group_separator_indexes = sort_and_group_detections_from_scan_result(local_scan_results) for detection, document in detections: self._enrich_table_with_values(table, detection, document) @@ -46,47 +39,6 @@ def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: self._print_table(table) self._print_report_urls(local_scan_results, self.ctx.obj.get('aggregation_report_url')) - @staticmethod - def __severity_sort_key(detection_with_document: Tuple[Detection, Document]) -> int: - detection, _ = detection_with_document - severity = detection.severity if detection.severity else '' - return SeverityOption.get_member_weight(severity) - - def _sort_detections_by_severity( - self, detections_with_documents: List[Tuple[Detection, Document]] - ) -> List[Tuple[Detection, Document]]: - return sorted(detections_with_documents, key=self.__severity_sort_key, reverse=True) - - @staticmethod - def __file_path_sort_key(detection_with_document: Tuple[Detection, Document]) -> str: - _, document = detection_with_document - return document.path - - def _sort_detections_by_file_path( - self, detections_with_documents: List[Tuple[Detection, Document]] - ) -> List[Tuple[Detection, Document]]: - return sorted(detections_with_documents, key=self.__file_path_sort_key) - - def _sort_and_group_detections( - self, detections_with_documents: List[Tuple[Detection, Document]] - ) -> Tuple[List[Tuple[Detection, Document]], Set[int]]: - """Sort detections by severity and group by file name.""" - detections = [] - group_separator_indexes = set() - - # we sort detections by file path to make persist output order - sorted_detections = self._sort_detections_by_file_path(detections_with_documents) - - grouped_by_file_path = defaultdict(list) - for detection, document in sorted_detections: - grouped_by_file_path[document.path].append((detection, document)) - - for file_path_group in grouped_by_file_path.values(): - group_separator_indexes.add(len(detections) - 1) # indexing starts from 0 - detections.extend(self._sort_detections_by_severity(file_path_group)) - - return detections, group_separator_indexes - def _get_table(self) -> Table: table = Table() diff --git a/cycode/cli/printers/text_printer.py b/cycode/cli/printers/text_printer.py index 7f1db10f..b1eeb38e 100644 --- a/cycode/cli/printers/text_printer.py +++ b/cycode/cli/printers/text_printer.py @@ -1,20 +1,19 @@ -import math import urllib.parse from typing import TYPE_CHECKING, Dict, List, Optional import typer from rich.markup import escape -from rich.syntax import Syntax from cycode.cli.cli_types import SeverityOption -from cycode.cli.console import _SYNTAX_HIGHLIGHT_THEME, console -from cycode.cli.consts import COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES, SECRET_SCAN_TYPE -from cycode.cli.models import CliError, CliResult, Detection, Document, DocumentDetections +from cycode.cli.console import console +from cycode.cli.models import CliError, CliResult, Document from cycode.cli.printers.printer_base import PrinterBase -from cycode.cli.utils.string_utils import get_position_in_line, obfuscate_text +from cycode.cli.printers.utils.code_snippet_syntax import get_code_snippet_syntax +from cycode.cli.printers.utils.detection_data import get_detection_title +from cycode.cli.printers.utils.detection_ordering.common_ordering import sort_and_group_detections_from_scan_result if TYPE_CHECKING: - from cycode.cli.models import LocalScanResult + from cycode.cli.models import Detection, LocalScanResult class TextPrinter(PrinterBase): @@ -48,36 +47,26 @@ def print_scan_results( console.print(self.NO_DETECTIONS_MESSAGE) return - for local_scan_result in local_scan_results: - for document_detections in local_scan_result.document_detections: - self._print_document_detections(document_detections) + detections, _ = sort_and_group_detections_from_scan_result(local_scan_results) + for detection, document in detections: + self.__print_document_detection(document, detection) - report_urls = [scan_result.report_url for scan_result in local_scan_results if scan_result.report_url] - - self._print_report_urls(report_urls, self.ctx.obj.get('aggregation_report_url')) - if not errors: - return - - console.print(self.FAILED_SCAN_MESSAGE) - for scan_id, error in errors.items(): - console.print(f'- {scan_id}: ', end='') - self.print_error(error) + self.print_report_urls_and_errors(local_scan_results, errors) - def _print_document_detections(self, document_detections: DocumentDetections) -> None: - document = document_detections.document - for detection in document_detections.detections: - self._print_detection_summary(detection, document.path) - self._print_new_line() - self._print_detection_code_segment(detection, document) - self._print_new_line() + def __print_document_detection(self, document: 'Document', detection: 'Detection') -> None: + self.__print_detection_summary(detection, document.path) + self.__print_detection_code_segment(detection, document) + self._print_new_line() @staticmethod def _print_new_line() -> None: - console.print() + console.line() + + def __print_detection_summary(self, detection: 'Detection', document_path: str) -> None: + title = get_detection_title(self.scan_type, detection) - def _print_detection_summary(self, detection: Detection, document_path: str) -> None: - name = detection.type if self.scan_type == SECRET_SCAN_TYPE else detection.message severity = SeverityOption(detection.severity) if detection.severity else 'N/A' + severity_icon = SeverityOption.get_member_emoji(detection.severity) if detection.severity else '' escaped_document_path = escape(urllib.parse.quote(document_path)) clickable_document_path = f'[link file://{escaped_document_path}]{document_path}' @@ -85,31 +74,39 @@ def _print_detection_summary(self, detection: Detection, document_path: str) -> detection_commit_id = detection.detection_details.get('commit_id') detection_commit_id_message = f'\nCommit SHA: {detection_commit_id}' if detection_commit_id else '' - company_guidelines = detection.detection_details.get('custom_remediation_guidelines') - company_guidelines_message = f'\nCompany Guideline: {company_guidelines}' if company_guidelines else '' - console.print( - ':no_entry: Found', + f'{severity_icon}', severity, - f'issue of type: [b bright_red]{name}[/] ' - f'in file: {clickable_document_path} ' - f'{detection_commit_id_message}' - f'{company_guidelines_message}' - f' :no_entry:', + f'violation: [b bright_red]{title}[/]{detection_commit_id_message}\n{clickable_document_path}:', + ) + + def __print_detection_code_segment(self, detection: 'Detection', document: Document) -> None: + console.print( + get_code_snippet_syntax( + self.scan_type, + self.command_scan_type, + detection, + document, + obfuscate=not self.show_secret, + ) ) - def _print_detection_code_segment( - self, detection: Detection, document: Document, lines_to_display: int = 3 + def print_report_urls_and_errors( + self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None ) -> None: - if self._is_git_diff_based_scan(): - # it will print just one line - self._print_detection_from_git_diff(detection, document) + report_urls = [scan_result.report_url for scan_result in local_scan_results if scan_result.report_url] + + self.print_report_urls(report_urls, self.ctx.obj.get('aggregation_report_url')) + if not errors: return - self._print_detection_from_file(detection, document, lines_to_display) + console.print(self.FAILED_SCAN_MESSAGE) + for scan_id, error in errors.items(): + console.print(f'- {scan_id}: ', end='') + self.print_error(error) @staticmethod - def _print_report_urls(report_urls: List[str], aggregation_report_url: Optional[str] = None) -> None: + def print_report_urls(report_urls: List[str], aggregation_report_url: Optional[str] = None) -> None: if not report_urls and not aggregation_report_url: return if aggregation_report_url: @@ -119,82 +116,3 @@ def _print_report_urls(report_urls: List[str], aggregation_report_url: Optional[ console.print('Report URLs:') for report_url in report_urls: console.print(f'- {report_url}') - - @staticmethod - def _get_code_segment_start_line(detection_line: int, lines_to_display: int) -> int: - start_line = detection_line - math.ceil(lines_to_display / 2) - return 0 if start_line < 0 else start_line - - def _get_detection_line(self, detection: Detection) -> int: - return ( - detection.detection_details.get('line', -1) - if self.scan_type == SECRET_SCAN_TYPE - else detection.detection_details.get('line_in_file', -1) - 1 - ) - - def _print_detection_from_file(self, detection: Detection, document: Document, lines_to_display: int) -> None: - detection_details = detection.detection_details - detection_line = self._get_detection_line(detection) - start_line_index = self._get_code_segment_start_line(detection_line, lines_to_display) - detection_position = get_position_in_line(document.content, detection_details.get('start_position', -1)) - violation_length = detection_details.get('length', -1) - - code_lines_to_render = [] - document_content_lines = document.content.splitlines() - for line_index in range(lines_to_display): - current_line_index = start_line_index + line_index - if current_line_index >= len(document_content_lines): - break - - line_content = document_content_lines[current_line_index] - - line_with_detection = current_line_index == detection_line - if self.scan_type == SECRET_SCAN_TYPE and line_with_detection and not self.show_secret: - violation = line_content[detection_position : detection_position + violation_length] - code_lines_to_render.append(line_content.replace(violation, obfuscate_text(violation))) - else: - code_lines_to_render.append(line_content) - - code_to_render = '\n'.join(code_lines_to_render) - console.print( - Syntax( - theme=_SYNTAX_HIGHLIGHT_THEME, - code=code_to_render, - lexer=Syntax.guess_lexer(document.path, code=code_to_render), - line_numbers=True, - dedent=True, - tab_size=2, - start_line=start_line_index + 1, - highlight_lines={ - detection_line + 1, - }, - ) - ) - - def _print_detection_from_git_diff(self, detection: Detection, document: Document) -> None: - detection_details = detection.detection_details - detection_line = self._get_detection_line(detection) - detection_position = detection_details.get('start_position', -1) - violation_length = detection_details.get('length', -1) - - line_content = document.content.splitlines()[detection_line] - detection_position_in_line = get_position_in_line(document.content, detection_position) - if self.scan_type == SECRET_SCAN_TYPE and not self.show_secret: - violation = line_content[detection_position_in_line : detection_position_in_line + violation_length] - line_content = line_content.replace(violation, obfuscate_text(violation)) - - console.print( - Syntax( - theme=_SYNTAX_HIGHLIGHT_THEME, - code=line_content, - lexer='diff', - line_numbers=True, - start_line=detection_line, - dedent=True, - tab_size=2, - highlight_lines={detection_line + 1}, - ) - ) - - def _is_git_diff_based_scan(self) -> bool: - return self.command_scan_type in COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES and self.scan_type == SECRET_SCAN_TYPE diff --git a/cycode/cli/printers/utils/__init__.py b/cycode/cli/printers/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/printers/utils/code_snippet_syntax.py b/cycode/cli/printers/utils/code_snippet_syntax.py new file mode 100644 index 00000000..c3c9f59b --- /dev/null +++ b/cycode/cli/printers/utils/code_snippet_syntax.py @@ -0,0 +1,113 @@ +import math +from typing import TYPE_CHECKING + +from rich.syntax import Syntax + +from cycode.cli import consts +from cycode.cli.console import _SYNTAX_HIGHLIGHT_THEME +from cycode.cli.utils.string_utils import get_position_in_line, obfuscate_text + +if TYPE_CHECKING: + from cycode.cli.models import Document + from cycode.cyclient.models import Detection + + +def _get_code_segment_start_line(detection_line: int, lines_to_display: int) -> int: + start_line = detection_line - math.ceil(lines_to_display / 2) + return 0 if start_line < 0 else start_line + + +def _get_detection_line(scan_type: str, detection: 'Detection') -> int: + return ( + detection.detection_details.get('line', -1) + if scan_type == consts.SECRET_SCAN_TYPE + else detection.detection_details.get('line_in_file', -1) - 1 + ) + + +def _get_code_snippet_syntax_from_file( + scan_type: str, detection: 'Detection', document: 'Document', lines_to_display: int, obfuscate: bool +) -> Syntax: + detection_details = detection.detection_details + detection_line = _get_detection_line(scan_type, detection) + start_line_index = _get_code_segment_start_line(detection_line, lines_to_display) + detection_position = get_position_in_line(document.content, detection_details.get('start_position', -1)) + violation_length = detection_details.get('length', -1) + + code_lines_to_render = [] + document_content_lines = document.content.splitlines() + for line_index in range(lines_to_display): + current_line_index = start_line_index + line_index + if current_line_index >= len(document_content_lines): + break + + line_content = document_content_lines[current_line_index] + + line_with_detection = current_line_index == detection_line + if scan_type == consts.SECRET_SCAN_TYPE and line_with_detection and obfuscate: + violation = line_content[detection_position : detection_position + violation_length] + code_lines_to_render.append(line_content.replace(violation, obfuscate_text(violation))) + else: + code_lines_to_render.append(line_content) + + code_to_render = '\n'.join(code_lines_to_render) + return Syntax( + theme=_SYNTAX_HIGHLIGHT_THEME, + code=code_to_render, + lexer=Syntax.guess_lexer(document.path, code=code_to_render), + line_numbers=True, + dedent=True, + tab_size=2, + start_line=start_line_index + 1, + highlight_lines={ + detection_line + 1, + }, + ) + + +def _get_code_snippet_syntax_from_git_diff( + scan_type: str, detection: 'Detection', document: 'Document', obfuscate: bool +) -> Syntax: + detection_details = detection.detection_details + detection_line = _get_detection_line(scan_type, detection) + detection_position = detection_details.get('start_position', -1) + violation_length = detection_details.get('length', -1) + + line_content = document.content.splitlines()[detection_line] + detection_position_in_line = get_position_in_line(document.content, detection_position) + if scan_type == consts.SECRET_SCAN_TYPE and obfuscate: + violation = line_content[detection_position_in_line : detection_position_in_line + violation_length] + line_content = line_content.replace(violation, obfuscate_text(violation)) + + return Syntax( + theme=_SYNTAX_HIGHLIGHT_THEME, + code=line_content, + lexer='diff', + line_numbers=True, + start_line=detection_line, + dedent=True, + tab_size=2, + highlight_lines={detection_line + 1}, + ) + + +def _is_git_diff_based_scan(scan_type: str, command_scan_type: str) -> bool: + return ( + command_scan_type in consts.COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES + and scan_type in consts.COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES + ) + + +def get_code_snippet_syntax( + scan_type: str, + command_scan_type: str, + detection: 'Detection', + document: 'Document', + lines_to_display: int = 3, + obfuscate: bool = True, +) -> Syntax: + if _is_git_diff_based_scan(scan_type, command_scan_type): + # it will return syntax with just one line + return _get_code_snippet_syntax_from_git_diff(scan_type, detection, document, obfuscate) + + return _get_code_snippet_syntax_from_file(scan_type, detection, document, lines_to_display, obfuscate) diff --git a/cycode/cli/printers/utils/detection_data.py b/cycode/cli/printers/utils/detection_data.py new file mode 100644 index 00000000..66171226 --- /dev/null +++ b/cycode/cli/printers/utils/detection_data.py @@ -0,0 +1,16 @@ +from typing import TYPE_CHECKING + +from cycode.cli import consts + +if TYPE_CHECKING: + from cycode.cyclient.models import Detection + + +def get_detection_title(scan_type: str, detection: 'Detection') -> str: + title = detection.message + if scan_type == consts.SAST_SCAN_TYPE: + title = detection.detection_details['policy_display_name'] + elif scan_type == consts.SECRET_SCAN_TYPE: + title = f'Hardcoded {detection.type} is used' + + return title diff --git a/cycode/cli/printers/utils/detection_ordering/__init__.py b/cycode/cli/printers/utils/detection_ordering/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/printers/utils/detection_ordering/common_ordering.py b/cycode/cli/printers/utils/detection_ordering/common_ordering.py new file mode 100644 index 00000000..531cbc4c --- /dev/null +++ b/cycode/cli/printers/utils/detection_ordering/common_ordering.py @@ -0,0 +1,66 @@ +from collections import defaultdict +from typing import TYPE_CHECKING, List, Set, Tuple + +from cycode.cli.cli_types import SeverityOption + +if TYPE_CHECKING: + from cycode.cli.models import Document, LocalScanResult + from cycode.cyclient.models import Detection + + +GroupedDetections = Tuple[List[Tuple['Detection', 'Document']], Set[int]] + + +def __severity_sort_key(detection_with_document: Tuple['Detection', 'Document']) -> int: + detection, _ = detection_with_document + severity = detection.severity if detection.severity else '' + return SeverityOption.get_member_weight(severity) + + +def _sort_detections_by_severity( + detections_with_documents: List[Tuple['Detection', 'Document']], +) -> List[Tuple['Detection', 'Document']]: + return sorted(detections_with_documents, key=__severity_sort_key, reverse=True) + + +def __file_path_sort_key(detection_with_document: Tuple['Detection', 'Document']) -> str: + _, document = detection_with_document + return document.path + + +def _sort_detections_by_file_path( + detections_with_documents: List[Tuple['Detection', 'Document']], +) -> List[Tuple['Detection', 'Document']]: + return sorted(detections_with_documents, key=__file_path_sort_key) + + +def sort_and_group_detections( + detections_with_documents: List[Tuple['Detection', 'Document']], +) -> GroupedDetections: + """Sort detections by severity and group by file name.""" + detections = [] + group_separator_indexes = set() + + # we sort detections by file path to make persist output order + sorted_detections = _sort_detections_by_file_path(detections_with_documents) + + grouped_by_file_path = defaultdict(list) + for detection, document in sorted_detections: + grouped_by_file_path[document.path].append((detection, document)) + + for file_path_group in grouped_by_file_path.values(): + group_separator_indexes.add(len(detections) - 1) # indexing starts from 0 + detections.extend(_sort_detections_by_severity(file_path_group)) + + return detections, group_separator_indexes + + +def sort_and_group_detections_from_scan_result(local_scan_results: List['LocalScanResult']) -> GroupedDetections: + detections_with_documents = [] + for local_scan_result in local_scan_results: + for document_detections in local_scan_result.document_detections: + detections_with_documents.extend( + [(detection, document_detections.document) for detection in document_detections.detections] + ) + + return sort_and_group_detections(detections_with_documents) diff --git a/cycode/cli/printers/utils/detection_ordering/sca_ordering.py b/cycode/cli/printers/utils/detection_ordering/sca_ordering.py new file mode 100644 index 00000000..85915c56 --- /dev/null +++ b/cycode/cli/printers/utils/detection_ordering/sca_ordering.py @@ -0,0 +1,58 @@ +from collections import defaultdict +from typing import TYPE_CHECKING, Dict, List, Set, Tuple + +from cycode.cli.cli_types import SeverityOption + +if TYPE_CHECKING: + from cycode.cyclient.models import Detection + + +def __group_by(detections: List['Detection'], details_field_name: str) -> Dict[str, List['Detection']]: + grouped = defaultdict(list) + for detection in detections: + grouped[detection.detection_details.get(details_field_name)].append(detection) + return grouped + + +def __severity_sort_key(detection: 'Detection') -> int: + severity = detection.detection_details.get('advisory_severity', 'unknown') + return SeverityOption.get_member_weight(severity) + + +def _sort_detections_by_severity(detections: List['Detection']) -> List['Detection']: + return sorted(detections, key=__severity_sort_key, reverse=True) + + +def __package_sort_key(detection: 'Detection') -> int: + return detection.detection_details.get('package_name') + + +def _sort_detections_by_package(detections: List['Detection']) -> List['Detection']: + return sorted(detections, key=__package_sort_key) + + +def sort_and_group_detections(detections: List['Detection']) -> Tuple[List['Detection'], Set[int]]: + """Sort detections by severity and group by repository, code project and package name. + + Note: + Code Project is path to the manifest file. + + Grouping by code projects also groups by ecosystem. + Because manifest files are unique per ecosystem. + """ + resulting_detections = [] + group_separator_indexes = set() + + # we sort detections by package name to make persist output order + sorted_detections = _sort_detections_by_package(detections) + + grouped_by_repository = __group_by(sorted_detections, 'repository_name') + for repository_group in grouped_by_repository.values(): + grouped_by_code_project = __group_by(repository_group, 'file_name') + for code_project_group in grouped_by_code_project.values(): + grouped_by_package = __group_by(code_project_group, 'package_name') + for package_group in grouped_by_package.values(): + group_separator_indexes.add(len(resulting_detections) - 1) # indexing starts from 0 + resulting_detections.extend(_sort_detections_by_severity(package_group)) + + return resulting_detections, group_separator_indexes diff --git a/cycode/cli/printers/utils/rich_helpers.py b/cycode/cli/printers/utils/rich_helpers.py new file mode 100644 index 00000000..52d2a0f2 --- /dev/null +++ b/cycode/cli/printers/utils/rich_helpers.py @@ -0,0 +1,37 @@ +from typing import TYPE_CHECKING + +from rich.columns import Columns +from rich.markdown import Markdown +from rich.panel import Panel + +from cycode.cli.console import console + +if TYPE_CHECKING: + from rich.console import RenderableType + + +def get_panel(renderable: 'RenderableType', title: str) -> Panel: + return Panel( + renderable, + title=title, + title_align='left', + border_style='dim', + ) + + +def get_markdown_panel(markdown_text: str, title: str) -> Panel: + return get_panel( + Markdown(markdown_text.strip()), + title=title, + ) + + +def get_columns_in_1_to_3_ratio(left: 'Panel', right: 'Panel', panel_border_offset: int = 5) -> Columns: + terminal_width = console.width + one_third_width = terminal_width // 3 + two_thirds_width = terminal_width - one_third_width - panel_border_offset + + left.width = one_third_width + right.width = two_thirds_width + + return Columns([left, right]) diff --git a/tests/cli/commands/test_main_command.py b/tests/cli/commands/test_main_command.py index d9890268..ba791f2e 100644 --- a/tests/cli/commands/test_main_command.py +++ b/tests/cli/commands/test_main_command.py @@ -49,7 +49,7 @@ def test_passing_output_option(output: str, scan_client: 'ScanClient', api_token output = json.loads(result.output) assert 'scan_ids' in output else: - assert 'issue of type:' in result.output + assert 'violation:' in result.output @responses.activate From aadb5905abeb3d0407af2291a87e4bd59013eed6 Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Fri, 18 Apr 2025 22:55:36 +0200 Subject: [PATCH 11/19] CM-46731 - Make all flows use scan service (#296) --- README.md | 1 - cycode/cli/apps/scan/code_scanner.py | 85 ++++------- cycode/cli/apps/scan/scan_command.py | 4 +- cycode/cyclient/models.py | 28 ---- cycode/cyclient/scan_client.py | 132 +++--------------- cycode/cyclient/scan_config_base.py | 28 +--- tests/cli/commands/test_main_command.py | 14 +- .../cyclient/mocked_responses/scan_client.py | 88 +----------- .../scan_config/test_default_scan_config.py | 5 +- .../scan_config/test_dev_scan_config.py | 5 +- tests/cyclient/test_scan_client.py | 110 ++++++++------- tests/test_code_scanner.py | 25 ---- 12 files changed, 127 insertions(+), 398 deletions(-) diff --git a/README.md b/README.md index b30bc533..6218cba8 100644 --- a/README.md +++ b/README.md @@ -303,7 +303,6 @@ The Cycode CLI application offers several types of scans so that you can choose | `--monitor` | When specified, the scan results will be recorded in the knowledge graph. Please note that when working in `monitor` mode, the knowledge graph will not be updated as a result of SCM events (Push, Repo creation). (Supported for SCA scan type only). | | `--report` | When specified, a violations report will be generated. A URL link to the report will be printed as an output to the command execution. | | `--no-restore` | When specified, Cycode will not run restore command. Will scan direct dependencies ONLY! | -| `--sync` | Run scan synchronously (the default is asynchronous). | | `--gradle-all-sub-projects` | When specified, Cycode will run gradle restore command for all sub projects. Should run from root project directory ONLY! | | `--help` | Show options for given command. | diff --git a/cycode/cli/apps/scan/code_scanner.py b/cycode/cli/apps/scan/code_scanner.py index 35208d59..67185dce 100644 --- a/cycode/cli/apps/scan/code_scanner.py +++ b/cycode/cli/apps/scan/code_scanner.py @@ -100,24 +100,30 @@ def set_issue_detected_by_scan_results(ctx: typer.Context, scan_results: List[Lo set_issue_detected(ctx, any(scan_result.issue_detected for scan_result in scan_results)) -def _should_use_scan_service(scan_type: str, scan_parameters: dict) -> bool: - return scan_type == consts.SECRET_SCAN_TYPE and scan_parameters.get('report') is True +def _should_use_sync_flow(command_scan_type: str, scan_type: str, sync_option: bool) -> bool: + """Decide whether to use sync flow or async flow for the scan. + Note: + Passing `--sync` option does not mean that sync flow will be used in all cases. -def _should_use_sync_flow( - command_scan_type: str, scan_type: str, sync_option: bool, scan_parameters: Optional[dict] = None -) -> bool: - if not sync_option: + The logic: + - for IAC scan, sync flow is always used + - for SAST scan, sync flow is not supported + - for SCA and Secrets scan, sync flow is supported only for path/repository scan + """ + if not sync_option and scan_type != consts.IAC_SCAN_TYPE: return False if command_scan_type not in {'path', 'repository'}: - raise ValueError(f'Sync flow is not available for "{command_scan_type}" command type. Remove --sync option.') + return False - if scan_type is consts.SAST_SCAN_TYPE: - raise ValueError('Sync scan is not available for SAST scan type.') + if scan_type == consts.IAC_SCAN_TYPE: + # sync in the only available flow for IAC scan; we do not use detector directly anymore + return True - if scan_parameters.get('report') is True: - raise ValueError('You can not use sync flow with report option. Either remove "report" or "sync" option.') + if scan_type is consts.SAST_SCAN_TYPE: # noqa: SIM103 + # SAST does not support sync flow + return False return True @@ -169,8 +175,7 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local scan_id = str(_generate_unique_id()) scan_completed = False - should_use_scan_service = _should_use_scan_service(scan_type, scan_parameters) - should_use_sync_flow = _should_use_sync_flow(command_scan_type, scan_type, sync_option, scan_parameters) + should_use_sync_flow = _should_use_sync_flow(command_scan_type, scan_type, sync_option) try: logger.debug('Preparing local files, %s', {'batch_files_count': len(batch)}) @@ -180,11 +185,9 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local cycode_client, zipped_documents, scan_type, - scan_id, is_git_diff, is_commit_range, scan_parameters, - should_use_scan_service, should_use_sync_flow, ) @@ -224,7 +227,6 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local zip_file_size, command_scan_type, error_message, - should_use_scan_service or should_use_sync_flow, # sync flow implies scan service ) return scan_id, error, local_scan_result @@ -456,24 +458,16 @@ def perform_scan( cycode_client: 'ScanClient', zipped_documents: 'InMemoryZip', scan_type: str, - scan_id: str, is_git_diff: bool, is_commit_range: bool, scan_parameters: dict, - should_use_scan_service: bool = False, should_use_sync_flow: bool = False, ) -> ZippedFileScanResult: if should_use_sync_flow: # it does not support commit range scans; should_use_sync_flow handles it return perform_scan_sync(cycode_client, zipped_documents, scan_type, scan_parameters, is_git_diff) - if scan_type in (consts.SCA_SCAN_TYPE, consts.SAST_SCAN_TYPE) or should_use_scan_service: - return perform_scan_async(cycode_client, zipped_documents, scan_type, scan_parameters, is_commit_range) - - if is_commit_range: - return cycode_client.commit_range_zipped_file_scan(scan_type, zipped_documents, scan_id) - - return cycode_client.zipped_file_scan(scan_type, zipped_documents, scan_id, scan_parameters, is_git_diff) + return perform_scan_async(cycode_client, zipped_documents, scan_type, scan_parameters, is_commit_range) def perform_scan_async( @@ -823,7 +817,6 @@ def _report_scan_status( zip_size: int, command_scan_type: str, error_message: Optional[str], - should_use_scan_service: bool = False, ) -> None: try: end_scan_time = time.time() @@ -840,12 +833,15 @@ def _report_scan_status( 'scan_type': scan_type, } - cycode_client.report_scan_status(scan_type, scan_id, scan_status, should_use_scan_service) + cycode_client.report_scan_status(scan_type, scan_id, scan_status) except Exception as e: logger.debug('Failed to report scan status', exc_info=e) def _generate_unique_id() -> UUID: + if 'PYTEST_TEST_UNIQUE_ID' in os.environ: + return UUID(os.environ['PYTEST_TEST_UNIQUE_ID']) + return uuid4() @@ -868,13 +864,13 @@ def _get_scan_result( if not scan_details.detections_count: return init_default_scan_result(scan_id) - scan_raw_detections = cycode_client.get_scan_raw_detections(scan_type, scan_id) + scan_raw_detections = cycode_client.get_scan_raw_detections(scan_id) return ZippedFileScanResult( did_detect=True, detections_per_file=_map_detections_per_file_and_commit_id(scan_type, scan_raw_detections), scan_id=scan_id, - report_url=_try_get_any_report_url_if_needed(cycode_client, scan_id, scan_type, scan_parameters), + report_url=_try_get_aggregation_report_url_if_needed(scan_parameters, cycode_client, scan_type), ) @@ -886,37 +882,6 @@ def init_default_scan_result(scan_id: str) -> ZippedFileScanResult: ) -def _try_get_any_report_url_if_needed( - cycode_client: 'ScanClient', - scan_id: str, - scan_type: str, - scan_parameters: dict, -) -> Optional[str]: - """Tries to get aggregation report URL if needed, otherwise tries to get report URL.""" - aggregation_report_url = None - if scan_parameters: - _try_get_report_url_if_needed(cycode_client, scan_id, scan_type, scan_parameters) - aggregation_report_url = _try_get_aggregation_report_url_if_needed(scan_parameters, cycode_client, scan_type) - - if aggregation_report_url: - return aggregation_report_url - - return _try_get_report_url_if_needed(cycode_client, scan_id, scan_type, scan_parameters) - - -def _try_get_report_url_if_needed( - cycode_client: 'ScanClient', scan_id: str, scan_type: str, scan_parameters: dict -) -> Optional[str]: - if not scan_parameters.get('report', False): - return None - - try: - report_url_response = cycode_client.get_scan_report_url(scan_id, scan_type) - return report_url_response.report_url - except Exception as e: - logger.debug('Failed to get report URL', exc_info=e) - - def _set_aggregation_report_url(ctx: typer.Context, aggregation_report_url: Optional[str] = None) -> None: ctx.obj['aggregation_report_url'] = aggregation_report_url diff --git a/cycode/cli/apps/scan/scan_command.py b/cycode/cli/apps/scan/scan_command.py index 3ba7699b..84485c0b 100644 --- a/cycode/cli/apps/scan/scan_command.py +++ b/cycode/cli/apps/scan/scan_command.py @@ -54,7 +54,9 @@ def scan_command( ] = SeverityOption.INFO, sync: Annotated[ bool, - typer.Option('--sync', help='Run scan synchronously.', show_default='asynchronously'), + typer.Option( + '--sync', help='Run scan synchronously (INTERNAL FOR IDEs).', show_default='asynchronously', hidden=True + ), ] = False, report: Annotated[ bool, diff --git a/cycode/cyclient/models.py b/cycode/cyclient/models.py index 2433ef6c..2c0f53d7 100644 --- a/cycode/cyclient/models.py +++ b/cycode/cyclient/models.py @@ -59,19 +59,6 @@ def __init__(self, file_name: str, detections: List[Detection], commit_id: Optio self.commit_id = commit_id -class DetectionsPerFileSchema(Schema): - class Meta: - unknown = EXCLUDE - - file_name = fields.String() - detections = fields.List(fields.Nested(DetectionSchema)) - commit_id = fields.String(allow_none=True) - - @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'DetectionsPerFile': - return DetectionsPerFile(**data) - - class ZippedFileScanResult(Schema): def __init__( self, @@ -89,21 +76,6 @@ def __init__( self.err = err -class ZippedFileScanResultSchema(Schema): - class Meta: - unknown = EXCLUDE - - did_detect = fields.Boolean() - scan_id = fields.String() - report_url = fields.String(allow_none=True) - detections_per_file = fields.List(fields.Nested(DetectionsPerFileSchema)) - err = fields.String() - - @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'ZippedFileScanResult': - return ZippedFileScanResult(**data) - - class ScanResult(Schema): def __init__( self, diff --git a/cycode/cyclient/scan_client.py b/cycode/cyclient/scan_client.py index c6bfc57c..09908943 100644 --- a/cycode/cyclient/scan_client.py +++ b/cycode/cyclient/scan_client.py @@ -1,5 +1,6 @@ import json -from typing import TYPE_CHECKING, List, Optional, Set, Union +from copy import deepcopy +from typing import TYPE_CHECKING, List, Set, Union from uuid import UUID from requests import Response @@ -22,34 +23,12 @@ def __init__( self.scan_cycode_client = scan_cycode_client self.scan_config = scan_config - self._SCAN_SERVICE_CONTROLLER_PATH = 'api/v1/scan' self._SCAN_SERVICE_CLI_CONTROLLER_PATH = 'api/v1/cli-scan' - - self._DETECTIONS_SERVICE_CONTROLLER_PATH = 'api/v1/detections' self._DETECTIONS_SERVICE_CLI_CONTROLLER_PATH = 'api/v1/detections/cli' - self._POLICIES_SERVICE_CONTROLLER_PATH_V3 = 'api/v3/policies' self._hide_response_log = hide_response_log - def get_scan_controller_path(self, scan_type: str, should_use_scan_service: bool = False) -> str: - if not should_use_scan_service and scan_type == consts.IAC_SCAN_TYPE: - # we don't use async flow for IaC scan yet - return self._SCAN_SERVICE_CONTROLLER_PATH - if not should_use_scan_service and scan_type == consts.SECRET_SCAN_TYPE: - # if a secret scan goes to detector directly, we should not use CLI controller. - # CLI controller belongs to the scan service only - return self._SCAN_SERVICE_CONTROLLER_PATH - - return self._SCAN_SERVICE_CLI_CONTROLLER_PATH - - def get_detections_service_controller_path(self, scan_type: str) -> str: - if scan_type == consts.IAC_SCAN_TYPE: - # we don't use async flow for IaC scan yet - return self._DETECTIONS_SERVICE_CONTROLLER_PATH - - return self._DETECTIONS_SERVICE_CLI_CONTROLLER_PATH - @staticmethod def get_scan_flow_type(should_use_sync_flow: bool = False) -> str: if should_use_sync_flow: @@ -57,13 +36,10 @@ def get_scan_flow_type(should_use_sync_flow: bool = False) -> str: return '' - def get_scan_service_url_path( - self, scan_type: str, should_use_scan_service: bool = False, should_use_sync_flow: bool = False - ) -> str: - service_path = self.scan_config.get_service_name(scan_type, should_use_scan_service) - controller_path = self.get_scan_controller_path(scan_type, should_use_scan_service) + def get_scan_service_url_path(self, scan_type: str, should_use_sync_flow: bool = False) -> str: + service_path = self.scan_config.get_service_name(scan_type) flow_type = self.get_scan_flow_type(should_use_sync_flow) - return f'{service_path}/{controller_path}{flow_type}' + return f'{service_path}/{self._SCAN_SERVICE_CLI_CONTROLLER_PATH}{flow_type}' def content_scan(self, scan_type: str, file_name: str, content: str, is_git_diff: bool = True) -> models.ScanResult: path = f'{self.get_scan_service_url_path(scan_type)}/content' @@ -73,27 +49,6 @@ def content_scan(self, scan_type: str, file_name: str, content: str, is_git_diff ) return self.parse_scan_response(response) - def get_zipped_file_scan_url_path(self, scan_type: str) -> str: - return f'{self.get_scan_service_url_path(scan_type)}/zipped-file' - - def zipped_file_scan( - self, scan_type: str, zip_file: InMemoryZip, scan_id: str, scan_parameters: dict, is_git_diff: bool = False - ) -> models.ZippedFileScanResult: - files = {'file': ('multiple_files_scan.zip', zip_file.read())} - - response = self.scan_cycode_client.post( - url_path=self.get_zipped_file_scan_url_path(scan_type), - data={'scan_id': scan_id, 'is_git_diff': is_git_diff, 'scan_parameters': json.dumps(scan_parameters)}, - files=files, - hide_response_content_log=self._hide_response_log, - ) - - return self.parse_zipped_file_scan_response(response) - - def get_scan_report_url(self, scan_id: str, scan_type: str) -> models.ScanReportUrlResponse: - response = self.scan_cycode_client.get(url_path=self.get_scan_report_url_path(scan_id, scan_type)) - return models.ScanReportUrlResponseSchema().build_dto(response.json()) - def get_scan_aggregation_report_url(self, aggregation_id: str, scan_type: str) -> models.ScanReportUrlResponse: response = self.scan_cycode_client.get( url_path=self.get_scan_aggregation_report_url_path(aggregation_id, scan_type) @@ -103,16 +58,12 @@ def get_scan_aggregation_report_url(self, aggregation_id: str, scan_type: str) - def get_zipped_file_scan_async_url_path(self, scan_type: str, should_use_sync_flow: bool = False) -> str: async_scan_type = self.scan_config.get_async_scan_type(scan_type) async_entity_type = self.scan_config.get_async_entity_type(scan_type) - scan_service_url_path = self.get_scan_service_url_path( - scan_type, should_use_scan_service=True, should_use_sync_flow=should_use_sync_flow - ) + scan_service_url_path = self.get_scan_service_url_path(scan_type, should_use_sync_flow=should_use_sync_flow) return f'{scan_service_url_path}/{async_scan_type}/{async_entity_type}' def get_zipped_file_scan_sync_url_path(self, scan_type: str) -> str: server_scan_type = self.scan_config.get_async_scan_type(scan_type) - scan_service_url_path = self.get_scan_service_url_path( - scan_type, should_use_scan_service=True, should_use_sync_flow=True - ) + scan_service_url_path = self.get_scan_service_url_path(scan_type, should_use_sync_flow=True) return f'{scan_service_url_path}/{server_scan_type}/repository' def zipped_file_scan_sync( @@ -124,6 +75,7 @@ def zipped_file_scan_sync( ) -> models.ScanResultsSyncFlow: files = {'file': ('multiple_files_scan.zip', zip_file.read())} + scan_parameters = deepcopy(scan_parameters) # avoid mutating the original dict if 'report' in scan_parameters: del scan_parameters['report'] # BE raises validation error instead of ignoring it @@ -180,16 +132,10 @@ def multiple_zipped_file_scan_async( return models.ScanInitializationResponseSchema().load(response.json()) def get_scan_details_path(self, scan_type: str, scan_id: str) -> str: - return f'{self.get_scan_service_url_path(scan_type, should_use_scan_service=True)}/{scan_id}' - - def get_scan_report_url_path(self, scan_id: str, scan_type: str) -> str: - return f'{self.get_scan_service_url_path(scan_type, should_use_scan_service=True)}/reportUrl/{scan_id}' + return f'{self.get_scan_service_url_path(scan_type)}/{scan_id}' def get_scan_aggregation_report_url_path(self, aggregation_id: str, scan_type: str) -> str: - return ( - f'{self.get_scan_service_url_path(scan_type, should_use_scan_service=True)}' - f'/reportUrlByAggregationId/{aggregation_id}' - ) + return f'{self.get_scan_service_url_path(scan_type)}' f'/reportUrlByAggregationId/{aggregation_id}' def get_scan_details(self, scan_type: str, scan_id: str) -> models.ScanDetailsResponse: path = self.get_scan_details_path(scan_type, scan_id) @@ -256,21 +202,13 @@ def get_detection_rules(self, detection_rules_ids: Union[Set[str], List[str]]) - return self.parse_detection_rules_response(response) - def get_scan_detections_path(self, scan_type: str) -> str: - return f'{self.scan_config.get_detections_prefix()}/{self.get_detections_service_controller_path(scan_type)}' + def get_scan_detections_path(self) -> str: + return f'{self.scan_config.get_detections_prefix()}/{self._DETECTIONS_SERVICE_CLI_CONTROLLER_PATH}' - @staticmethod - def get_scan_detections_list_path_suffix(scan_type: str) -> str: - # we don't use async flow for IaC scan yet - if scan_type == consts.IAC_SCAN_TYPE: - return '' - - return '/detections' + def get_scan_detections_list_path(self) -> str: + return f'{self.get_scan_detections_path()}/detections' - def get_scan_detections_list_path(self, scan_type: str) -> str: - return f'{self.get_scan_detections_path(scan_type)}{self.get_scan_detections_list_path_suffix(scan_type)}' - - def get_scan_raw_detections(self, scan_type: str, scan_id: str) -> List[dict]: + def get_scan_raw_detections(self, scan_id: str) -> List[dict]: params = {'scan_id': scan_id} page_size = 200 @@ -284,7 +222,7 @@ def get_scan_raw_detections(self, scan_type: str, scan_id: str) -> List[dict]: params['page_number'] = page_number response = self.scan_cycode_client.get( - url_path=self.get_scan_detections_list_path(scan_type), + url_path=self.get_scan_detections_list_path(), params=params, hide_response_content_log=self._hide_response_log, ).json() @@ -295,45 +233,15 @@ def get_scan_raw_detections(self, scan_type: str, scan_id: str) -> List[dict]: return raw_detections - def commit_range_zipped_file_scan( - self, scan_type: str, zip_file: InMemoryZip, scan_id: str - ) -> models.ZippedFileScanResult: - url_path = f'{self.get_scan_service_url_path(scan_type)}/commit-range-zipped-file' - files = {'file': ('multiple_files_scan.zip', zip_file.read())} - response = self.scan_cycode_client.post( - url_path=url_path, data={'scan_id': scan_id}, files=files, hide_response_content_log=self._hide_response_log - ) - return self.parse_zipped_file_scan_response(response) + def get_report_scan_status_path(self, scan_type: str, scan_id: str) -> str: + return f'{self.get_scan_service_url_path(scan_type)}/{scan_id}/status' - def get_report_scan_status_path(self, scan_type: str, scan_id: str, should_use_scan_service: bool = False) -> str: - return f'{self.get_scan_service_url_path(scan_type, should_use_scan_service)}/{scan_id}/status' - - def report_scan_status( - self, scan_type: str, scan_id: str, scan_status: dict, should_use_scan_service: bool = False - ) -> None: + def report_scan_status(self, scan_type: str, scan_id: str, scan_status: dict) -> None: self.scan_cycode_client.post( - url_path=self.get_report_scan_status_path( - scan_type, scan_id, should_use_scan_service=should_use_scan_service - ), + url_path=self.get_report_scan_status_path(scan_type, scan_id), body=scan_status, ) @staticmethod def parse_scan_response(response: Response) -> models.ScanResult: return models.ScanResultSchema().load(response.json()) - - @staticmethod - def parse_zipped_file_scan_response(response: Response) -> models.ZippedFileScanResult: - return models.ZippedFileScanResultSchema().load(response.json()) - - @staticmethod - def get_service_name(scan_type: str) -> Optional[str]: - # TODO(MarshalX): get_service_name should be removed from ScanClient? Because it exists in ScanConfig - if scan_type == consts.SECRET_SCAN_TYPE: - return 'secret' - if scan_type == consts.IAC_SCAN_TYPE: - return 'iac' - if scan_type == consts.SCA_SCAN_TYPE or scan_type == consts.SAST_SCAN_TYPE: - return 'scans' - - return None diff --git a/cycode/cyclient/scan_config_base.py b/cycode/cyclient/scan_config_base.py index 6dfa97ef..d60068ce 100644 --- a/cycode/cyclient/scan_config_base.py +++ b/cycode/cyclient/scan_config_base.py @@ -5,7 +5,7 @@ class ScanConfigBase(ABC): @abstractmethod - def get_service_name(self, scan_type: str, should_use_scan_service: bool = False) -> str: ... + def get_service_name(self, scan_type: str) -> str: ... @staticmethod def get_async_scan_type(scan_type: str) -> str: @@ -28,32 +28,16 @@ def get_detections_prefix(self) -> str: ... class DevScanConfig(ScanConfigBase): - def get_service_name(self, scan_type: str, should_use_scan_service: bool = False) -> str: - if should_use_scan_service: - return '5004' - if scan_type == consts.SECRET_SCAN_TYPE: - return '5025' - if scan_type == consts.IAC_SCAN_TYPE: - return '5026' - - # sca and sast - return '5004' + def get_service_name(self, scan_type: str) -> str: + return '5004' # scan service def get_detections_prefix(self) -> str: - return '5016' + return '5016' # detections service class DefaultScanConfig(ScanConfigBase): - def get_service_name(self, scan_type: str, should_use_scan_service: bool = False) -> str: - if should_use_scan_service: - return 'scans' - if scan_type == consts.SECRET_SCAN_TYPE: - return 'secret' - if scan_type == consts.IAC_SCAN_TYPE: - return 'iac' - - # sca and sast - return 'scans' + def get_service_name(self, scan_type: str) -> str: + return 'scans' # scan service def get_detections_prefix(self) -> str: return 'detections' diff --git a/tests/cli/commands/test_main_command.py b/tests/cli/commands/test_main_command.py index ba791f2e..db8fe86b 100644 --- a/tests/cli/commands/test_main_command.py +++ b/tests/cli/commands/test_main_command.py @@ -11,8 +11,7 @@ from cycode.cli.cli_types import OutputTypeOption from cycode.cli.utils.git_proxy import git_proxy from tests.conftest import CLI_ENV_VARS, TEST_FILES_PATH, ZIP_CONTENT_PATH -from tests.cyclient.mocked_responses.scan_client import mock_scan_responses -from tests.cyclient.test_scan_client import get_zipped_file_scan_response, get_zipped_file_scan_url +from tests.cyclient.mocked_responses.scan_client import mock_scan_async_responses _PATH_TO_SCAN = TEST_FILES_PATH.joinpath('zip_content').absolute() @@ -34,12 +33,12 @@ def test_passing_output_option(output: str, scan_client: 'ScanClient', api_token scan_type = consts.SECRET_SCAN_TYPE scan_id = uuid4() - mock_scan_responses(responses, scan_type, scan_client, scan_id, ZIP_CONTENT_PATH) - responses.add(get_zipped_file_scan_response(get_zipped_file_scan_url(scan_type, scan_client), ZIP_CONTENT_PATH)) + mock_scan_async_responses(responses, scan_type, scan_client, scan_id, ZIP_CONTENT_PATH) responses.add(api_token_response) args = ['--output', output, 'scan', '--soft-fail', 'path', str(_PATH_TO_SCAN)] - result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) + env = {'PYTEST_TEST_UNIQUE_ID': str(scan_id), **CLI_ENV_VARS} + result = CliRunner().invoke(app, args, env=env) except_json = output == 'json' @@ -54,10 +53,7 @@ def test_passing_output_option(output: str, scan_client: 'ScanClient', api_token @responses.activate def test_optional_git_with_path_scan(scan_client: 'ScanClient', api_token_response: responses.Response) -> None: - mock_scan_responses(responses, consts.SECRET_SCAN_TYPE, scan_client, uuid4(), ZIP_CONTENT_PATH) - responses.add( - get_zipped_file_scan_response(get_zipped_file_scan_url(consts.SECRET_SCAN_TYPE, scan_client), ZIP_CONTENT_PATH) - ) + mock_scan_async_responses(responses, consts.SECRET_SCAN_TYPE, scan_client, uuid4(), ZIP_CONTENT_PATH) responses.add(api_token_response) # fake env without Git executable diff --git a/tests/cyclient/mocked_responses/scan_client.py b/tests/cyclient/mocked_responses/scan_client.py index 87643001..1726e74c 100644 --- a/tests/cyclient/mocked_responses/scan_client.py +++ b/tests/cyclient/mocked_responses/scan_client.py @@ -9,53 +9,6 @@ from tests.conftest import MOCKED_RESPONSES_PATH -def get_zipped_file_scan_url(scan_type: str, scan_client: ScanClient) -> str: - api_url = scan_client.scan_cycode_client.api_url - service_url = scan_client.get_zipped_file_scan_url_path(scan_type) - return f'{api_url}/{service_url}' - - -def get_zipped_file_scan_response( - url: str, zip_content_path: Path, scan_id: Optional[UUID] = None -) -> responses.Response: - if not scan_id: - scan_id = uuid4() - - json_response = { - 'did_detect': True, - 'scan_id': str(scan_id), # not always as expected due to _get_scan_id and passing scan_id to cxt of CLI - 'detections_per_file': [ - { - 'file_name': str(zip_content_path.joinpath('secrets.py')), - 'commit_id': None, - 'detections': [ - { - 'detection_type_id': '12345678-418f-47ee-abb0-012345678901', - 'detection_rule_id': '12345678-aea1-4304-a6e9-012345678901', - 'message': "Secret of type 'Slack Token' was found in filename 'secrets.py'", - 'type': 'slack-token', - 'is_research': False, - 'detection_details': { - 'sha512': 'sha hash', - 'length': 55, - 'start_position': 19, - 'line': 0, - 'committed_at': '0001-01-01T00:00:00+00:00', - 'file_path': str(zip_content_path), - 'file_name': 'secrets.py', - 'file_extension': '.py', - 'should_resolve_upon_branch_deletion': False, - }, - } - ], - } - ], - 'report_url': None, - } - - return responses.Response(method=responses.POST, url=url, json=json_response, status=200) - - def get_zipped_file_scan_async_url(scan_type: str, scan_client: ScanClient) -> str: api_url = scan_client.scan_cycode_client.api_url service_url = scan_client.get_zipped_file_scan_async_url_path(scan_type) @@ -73,15 +26,9 @@ def get_zipped_file_scan_async_response(url: str, scan_id: Optional[UUID] = None return responses.Response(method=responses.POST, url=url, json=json_response, status=200) -def get_scan_details_url(scan_id: Optional[UUID], scan_client: ScanClient) -> str: - api_url = scan_client.scan_cycode_client.api_url - service_url = scan_client.get_scan_details_path(str(scan_id)) - return f'{api_url}/{service_url}' - - -def get_scan_report_url(scan_id: Optional[UUID], scan_client: ScanClient, scan_type: str) -> str: +def get_scan_details_url(scan_type: str, scan_id: Optional[UUID], scan_client: ScanClient) -> str: api_url = scan_client.scan_cycode_client.api_url - service_url = scan_client.get_scan_report_url_path(str(scan_id), scan_type) + service_url = scan_client.get_scan_details_path(scan_type, str(scan_id)) return f'{api_url}/{service_url}' @@ -91,14 +38,6 @@ def get_scan_aggregation_report_url(aggregation_id: Optional[UUID], scan_client: return f'{api_url}/{service_url}' -def get_scan_report_url_response(url: str, scan_id: Optional[UUID] = None) -> responses.Response: - if not scan_id: - scan_id = uuid4() - json_response = {'report_url': f'https://app.domain/on-demand-scans/{scan_id}'} - - return responses.Response(method=responses.GET, url=url, json=json_response, status=200) - - def get_scan_aggregation_report_url_response(url: str, aggregation_id: Optional[UUID] = None) -> responses.Response: if not aggregation_id: aggregation_id = uuid4() @@ -135,10 +74,10 @@ def get_scan_details_response(url: str, scan_id: Optional[UUID] = None) -> respo return responses.Response(method=responses.GET, url=url, json=json_response, status=200) -def get_scan_detections_url(scan_client: ScanClient, scan_type: str) -> str: +def get_scan_detections_url(scan_client: ScanClient) -> str: api_url = scan_client.scan_cycode_client.api_url - service_url = scan_client.get_scan_detections_path(scan_type) - return f'{api_url}/{service_url}' + path = scan_client.get_scan_detections_list_path() + return f'{api_url}/{path}' def get_scan_detections_response(url: str, scan_id: UUID, zip_content_path: Path) -> responses.Response: @@ -181,20 +120,7 @@ def mock_scan_async_responses( responses_module.add( get_zipped_file_scan_async_response(get_zipped_file_scan_async_url(scan_type, scan_client), scan_id) ) - responses_module.add(get_scan_details_response(get_scan_details_url(scan_id, scan_client), scan_id)) - responses_module.add(get_detection_rules_response(get_detection_rules_url(scan_client))) - responses_module.add( - get_scan_detections_response(get_scan_detections_url(scan_client, scan_type), scan_id, zip_content_path) - ) - responses_module.add(get_report_scan_status_response(get_report_scan_status_url(scan_type, scan_id, scan_client))) - - -def mock_scan_responses( - responses_module: responses, scan_type: str, scan_client: ScanClient, scan_id: UUID, zip_content_path: Path -) -> None: - responses_module.add( - get_zipped_file_scan_response(get_zipped_file_scan_url(scan_type, scan_client), zip_content_path) - ) + responses_module.add(get_scan_details_response(get_scan_details_url(scan_type, scan_id, scan_client), scan_id)) responses_module.add(get_detection_rules_response(get_detection_rules_url(scan_client))) + responses_module.add(get_scan_detections_response(get_scan_detections_url(scan_client), scan_id, zip_content_path)) responses_module.add(get_report_scan_status_response(get_report_scan_status_url(scan_type, scan_id, scan_client))) - responses_module.add(get_scan_report_url_response(get_scan_report_url(scan_id, scan_client, scan_type))) diff --git a/tests/cyclient/scan_config/test_default_scan_config.py b/tests/cyclient/scan_config/test_default_scan_config.py index 7371250c..987c6c78 100644 --- a/tests/cyclient/scan_config/test_default_scan_config.py +++ b/tests/cyclient/scan_config/test_default_scan_config.py @@ -5,11 +5,10 @@ def test_get_service_name() -> None: default_scan_config = DefaultScanConfig() - assert default_scan_config.get_service_name(consts.SECRET_SCAN_TYPE) == 'secret' - assert default_scan_config.get_service_name(consts.IAC_SCAN_TYPE) == 'iac' + assert default_scan_config.get_service_name(consts.SECRET_SCAN_TYPE) == 'scans' + assert default_scan_config.get_service_name(consts.IAC_SCAN_TYPE) == 'scans' assert default_scan_config.get_service_name(consts.SCA_SCAN_TYPE) == 'scans' assert default_scan_config.get_service_name(consts.SAST_SCAN_TYPE) == 'scans' - assert default_scan_config.get_service_name(consts.SECRET_SCAN_TYPE, True) == 'scans' def test_get_detections_prefix() -> None: diff --git a/tests/cyclient/scan_config/test_dev_scan_config.py b/tests/cyclient/scan_config/test_dev_scan_config.py index 6ebb368b..f1cd484c 100644 --- a/tests/cyclient/scan_config/test_dev_scan_config.py +++ b/tests/cyclient/scan_config/test_dev_scan_config.py @@ -5,11 +5,10 @@ def test_get_service_name() -> None: dev_scan_config = DevScanConfig() - assert dev_scan_config.get_service_name(consts.SECRET_SCAN_TYPE) == '5025' - assert dev_scan_config.get_service_name(consts.IAC_SCAN_TYPE) == '5026' + assert dev_scan_config.get_service_name(consts.SECRET_SCAN_TYPE) == '5004' + assert dev_scan_config.get_service_name(consts.IAC_SCAN_TYPE) == '5004' assert dev_scan_config.get_service_name(consts.SCA_SCAN_TYPE) == '5004' assert dev_scan_config.get_service_name(consts.SAST_SCAN_TYPE) == '5004' - assert dev_scan_config.get_service_name(consts.SECRET_SCAN_TYPE, should_use_scan_service=True) == '5004' def test_get_detections_prefix() -> None: diff --git a/tests/cyclient/test_scan_client.py b/tests/cyclient/test_scan_client.py index a1c0d151..d81116fb 100644 --- a/tests/cyclient/test_scan_client.py +++ b/tests/cyclient/test_scan_client.py @@ -5,10 +5,8 @@ import pytest import requests import responses -from requests import Timeout -from requests.exceptions import ProxyError +from requests.exceptions import ConnectionError as RequestsConnectionError -from cycode.cli import consts from cycode.cli.cli_types import ScanTypeOption from cycode.cli.exceptions.custom_exceptions import ( CycodeError, @@ -17,20 +15,21 @@ RequestTimeout, ) from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip -from cycode.cli.files_collector.zip_documents import zip_documents from cycode.cli.models import Document from cycode.cyclient.scan_client import ScanClient from tests.conftest import ZIP_CONTENT_PATH from tests.cyclient.mocked_responses.scan_client import ( - get_scan_report_url, - get_scan_report_url_response, - get_zipped_file_scan_response, - get_zipped_file_scan_url, + get_scan_aggregation_report_url, + get_scan_aggregation_report_url_response, + get_scan_details_response, + get_scan_details_url, + get_zipped_file_scan_async_response, + get_zipped_file_scan_async_url, ) def zip_scan_resources(scan_type: ScanTypeOption, scan_client: ScanClient) -> Tuple[str, InMemoryZip]: - url = get_zipped_file_scan_url(scan_type, scan_client) + url = get_zipped_file_scan_async_url(scan_type, scan_client) zip_file = get_test_zip_file(scan_type) return url, zip_file @@ -45,32 +44,25 @@ def get_test_zip_file(scan_type: ScanTypeOption) -> InMemoryZip: with open(path, 'r', encoding='UTF-8') as f: test_documents.append(Document(path, f.read(), is_git_diff_format=False)) - return zip_documents(scan_type, test_documents) - + from cycode.cli.files_collector.zip_documents import zip_documents -def test_get_service_name(scan_client: ScanClient) -> None: - # TODO(MarshalX): get_service_name should be removed from ScanClient? Because it exists in ScanConfig - assert scan_client.get_service_name(consts.SECRET_SCAN_TYPE) == 'secret' - assert scan_client.get_service_name(consts.IAC_SCAN_TYPE) == 'iac' - assert scan_client.get_service_name(consts.SCA_SCAN_TYPE) == 'scans' - assert scan_client.get_service_name(consts.SAST_SCAN_TYPE) == 'scans' + return zip_documents(scan_type, test_documents) @pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate -def test_zipped_file_scan( +def test_zipped_file_scan_async( scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: + """Test the zipped_file_scan_async method for the async flow.""" url, zip_file = zip_scan_resources(scan_type, scan_client) expected_scan_id = uuid4() responses.add(api_token_response) # mock token based client - responses.add(get_zipped_file_scan_response(url, ZIP_CONTENT_PATH, expected_scan_id)) + responses.add(get_zipped_file_scan_async_response(url, expected_scan_id)) - zipped_file_scan_response = scan_client.zipped_file_scan( - scan_type, zip_file, scan_id=str(expected_scan_id), scan_parameters={} - ) - assert zipped_file_scan_response.scan_id == str(expected_scan_id) + scan_initialization_response = scan_client.zipped_file_scan_async(zip_file, scan_type, scan_parameters={}) + assert scan_initialization_response.scan_id == str(expected_scan_id) @pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @@ -78,40 +70,41 @@ def test_zipped_file_scan( def test_get_scan_report_url( scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: + """Test getting the scan report URL for the async flow.""" scan_id = uuid4() - url = get_scan_report_url(scan_id, scan_client, scan_type) + url = get_scan_aggregation_report_url(scan_id, scan_client, scan_type) responses.add(api_token_response) # mock token based client - responses.add(get_scan_report_url_response(url, scan_id)) + responses.add(get_scan_aggregation_report_url_response(url, scan_id)) - scan_report_url_response = scan_client.get_scan_report_url(str(scan_id), scan_type) - assert scan_report_url_response.report_url == 'https://app.domain/on-demand-scans/{scan_id}'.format(scan_id=scan_id) + scan_report_url_response = scan_client.get_scan_aggregation_report_url(str(scan_id), scan_type) + assert scan_report_url_response.report_url == f'https://app.domain/cli-logs-aggregation/{scan_id}' @pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate -def test_zipped_file_scan_unauthorized_error( +def test_zipped_file_scan_async_unauthorized_error( scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: + """Test handling of unauthorized errors in the async flow.""" url, zip_file = zip_scan_resources(scan_type, scan_client) - expected_scan_id = uuid4().hex responses.add(api_token_response) # mock token based client - responses.add(method=responses.POST, url=url, status=401) + responses.add(method=responses.POST, url=url, status=401, body='Unauthorized') with pytest.raises(HttpUnauthorizedError) as e_info: - scan_client.zipped_file_scan(scan_type, zip_file, scan_id=expected_scan_id, scan_parameters={}) + scan_client.zipped_file_scan_async(zip_file=zip_file, scan_type=scan_type, scan_parameters={}) assert e_info.value.status_code == 401 @pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate -def test_zipped_file_scan_bad_request_error( +def test_zipped_file_scan_async_bad_request_error( scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: + """Test handling of bad request errors in the async flow.""" url, zip_file = zip_scan_resources(scan_type, scan_client) - expected_scan_id = uuid4().hex expected_status_code = 400 expected_response_text = 'Bad Request' @@ -120,7 +113,7 @@ def test_zipped_file_scan_bad_request_error( responses.add(method=responses.POST, url=url, status=expected_status_code, body=expected_response_text) with pytest.raises(CycodeError) as e_info: - scan_client.zipped_file_scan(scan_type, zip_file, scan_id=expected_scan_id, scan_parameters={}) + scan_client.zipped_file_scan_async(zip_file=zip_file, scan_type=scan_type, scan_parameters={}) assert e_info.value.status_code == expected_status_code assert e_info.value.error_message == expected_response_text @@ -128,40 +121,51 @@ def test_zipped_file_scan_bad_request_error( @pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate -def test_zipped_file_scan_timeout_error( +def test_zipped_file_scan_async_timeout_error( scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: - scan_url, zip_file = zip_scan_resources(scan_type, scan_client) - expected_scan_id = uuid4().hex - - responses.add(responses.POST, scan_url, status=504) - - timeout_response = requests.post(scan_url, timeout=5) - if timeout_response.status_code == 504: - """bypass SAST""" - - responses.reset() + """Test handling of timeout errors in the async flow.""" + url, zip_file = zip_scan_resources(scan_type, scan_client) - timeout_error = Timeout() - timeout_error.response = timeout_response + timeout_error = requests.exceptions.Timeout('Connection timed out') responses.add(api_token_response) # mock token based client - responses.add(method=responses.POST, url=scan_url, body=timeout_error, status=504) + responses.add(method=responses.POST, url=url, body=timeout_error) with pytest.raises(RequestTimeout): - scan_client.zipped_file_scan(scan_type, zip_file, scan_id=expected_scan_id, scan_parameters={}) + scan_client.zipped_file_scan_async(zip_file=zip_file, scan_type=scan_type, scan_parameters={}) @pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate -def test_zipped_file_scan_connection_error( +def test_zipped_file_scan_async_connection_error( scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: + """Test handling of connection errors in the async flow.""" url, zip_file = zip_scan_resources(scan_type, scan_client) - expected_scan_id = uuid4().hex + + # Create a connection error response + connection_error = RequestsConnectionError('Connection refused') responses.add(api_token_response) # mock token based client - responses.add(method=responses.POST, url=url, body=ProxyError()) + responses.add(method=responses.POST, url=url, body=connection_error) with pytest.raises(RequestConnectionError): - scan_client.zipped_file_scan(scan_type, zip_file, scan_id=expected_scan_id, scan_parameters={}) + scan_client.zipped_file_scan_async(zip_file=zip_file, scan_type=scan_type, scan_parameters={}) + + +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) +@responses.activate +def test_get_scan_details( + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response +) -> None: + """Test getting scan details in the async flow.""" + scan_id = uuid4() + url = get_scan_details_url(scan_type, scan_id, scan_client) + + responses.add(api_token_response) # mock token based client + responses.add(get_scan_details_response(url, scan_id)) + + scan_details_response = scan_client.get_scan_details(scan_type, str(scan_id)) + assert scan_details_response.id == str(scan_id) + assert scan_details_response.scan_status == 'Completed' diff --git a/tests/test_code_scanner.py b/tests/test_code_scanner.py index d16aad82..9ef09123 100644 --- a/tests/test_code_scanner.py +++ b/tests/test_code_scanner.py @@ -7,7 +7,6 @@ from cycode.cli import consts from cycode.cli.apps.scan.code_scanner import ( _try_get_aggregation_report_url_if_needed, - _try_get_report_url_if_needed, ) from cycode.cli.cli_types import ScanTypeOption from cycode.cli.files_collector.excluder import _is_relevant_file_to_scan @@ -16,8 +15,6 @@ from tests.cyclient.mocked_responses.scan_client import ( get_scan_aggregation_report_url, get_scan_aggregation_report_url_response, - get_scan_report_url, - get_scan_report_url_response, ) @@ -26,28 +23,6 @@ def test_is_relevant_file_to_scan_sca() -> None: assert _is_relevant_file_to_scan(consts.SCA_SCAN_TYPE, path) is True -@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) -def test_try_get_report_url_if_needed_return_none(scan_type: ScanTypeOption, scan_client: ScanClient) -> None: - scan_id = uuid4().hex - result = _try_get_report_url_if_needed(scan_client, scan_id, consts.SECRET_SCAN_TYPE, scan_parameters={}) - assert result is None - - -@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) -@responses.activate -def test_try_get_report_url_if_needed_return_result( - scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response -) -> None: - scan_id = uuid4() - url = get_scan_report_url(scan_id, scan_client, scan_type) - responses.add(api_token_response) # mock token based client - responses.add(get_scan_report_url_response(url, scan_id)) - - scan_report_url_response = scan_client.get_scan_report_url(str(scan_id), scan_type) - result = _try_get_report_url_if_needed(scan_client, str(scan_id), scan_type, scan_parameters={'report': True}) - assert result == scan_report_url_response.report_url - - @pytest.mark.parametrize('scan_type', list(ScanTypeOption)) def test_try_get_aggregation_report_url_if_no_report_command_needed_return_none( scan_type: ScanTypeOption, scan_client: ScanClient From 6c70c2118ccdf43ce41bd090744474fad728ec97 Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Wed, 23 Apr 2025 12:30:34 +0200 Subject: [PATCH 12/19] CM-46733 - Add CLI output exporting in HTML, SVG, and JSON formats (#297) --- cycode/cli/app.py | 37 +++++- cycode/cli/apps/ai_remediation/apply_fix.py | 3 +- .../apps/ai_remediation/print_remediation.py | 3 +- cycode/cli/apps/auth/auth_command.py | 4 +- cycode/cli/apps/auth/auth_common.py | 5 +- cycode/cli/apps/auth/check_command.py | 3 +- cycode/cli/apps/scan/code_scanner.py | 6 +- cycode/cli/cli_types.py | 6 + cycode/cli/exceptions/handle_errors.py | 8 +- cycode/cli/printers/console_printer.py | 105 ++++++++++++++---- cycode/cli/printers/json_printer.py | 7 +- cycode/cli/printers/printer_base.py | 19 +++- cycode/cli/printers/rich_printer.py | 10 +- .../cli/printers/tables/sca_table_printer.py | 6 +- .../cli/printers/tables/table_printer_base.py | 24 ++-- cycode/cli/printers/text_printer.py | 37 +++--- .../cli/exceptions/test_handle_scan_errors.py | 6 +- 17 files changed, 196 insertions(+), 93 deletions(-) diff --git a/cycode/cli/app.py b/cycode/cli/app.py index aed0e172..5c83be9e 100644 --- a/cycode/cli/app.py +++ b/cycode/cli/app.py @@ -1,4 +1,5 @@ import logging +from pathlib import Path from typing import Annotated, Optional import typer @@ -6,8 +7,9 @@ from cycode import __version__ from cycode.cli.apps import ai_remediation, auth, configure, ignore, report, scan, status -from cycode.cli.cli_types import OutputTypeOption +from cycode.cli.cli_types import ExportTypeOption, OutputTypeOption from cycode.cli.consts import CLI_CONTEXT_SETTINGS +from cycode.cli.printers import ConsolePrinter from cycode.cli.user_settings.configuration_manager import ConfigurationManager from cycode.cli.utils.progress_bar import SCAN_PROGRESS_BAR_SECTIONS, get_progress_bar from cycode.cli.utils.sentry import add_breadcrumb, init_sentry @@ -44,7 +46,14 @@ def check_latest_version_on_close(ctx: typer.Context) -> None: version_checker.check_and_notify_update(current_version=__version__, use_cache=should_use_cache) +def export_if_needed_on_close(ctx: typer.Context) -> None: + printer = ctx.obj.get('console_printer') + if printer.is_recording: + printer.export() + + _COMPLETION_RICH_HELP_PANEL = 'Completion options' +_EXPORT_RICH_HELP_PANEL = 'Export options' @app.callback() @@ -64,6 +73,27 @@ def app_callback( Optional[str], typer.Option(hidden=True, help='Characteristic JSON object that lets servers identify the application.'), ] = None, + export_type: Annotated[ + ExportTypeOption, + typer.Option( + '--export-type', + case_sensitive=False, + help='Specify the export type. ' + 'HTML and SVG will export terminal output and rely on --output option. ' + 'JSON always exports JSON.', + rich_help_panel=_EXPORT_RICH_HELP_PANEL, + ), + ] = ExportTypeOption.JSON, + export_file: Annotated[ + Optional[Path], + typer.Option( + '--export-file', + help='Export file. Path to the file where the export will be saved. ', + dir_okay=False, + writable=True, + rich_help_panel=_EXPORT_RICH_HELP_PANEL, + ), + ] = None, _: Annotated[ Optional[bool], typer.Option( @@ -104,6 +134,11 @@ def app_callback( ctx.obj['progress_bar'] = get_progress_bar(hidden=no_progress_meter, sections=SCAN_PROGRESS_BAR_SECTIONS) + ctx.obj['export_type'] = export_type + ctx.obj['export_file'] = export_file + ctx.obj['console_printer'] = ConsolePrinter(ctx) + ctx.call_on_close(lambda: export_if_needed_on_close(ctx)) + if user_agent: user_agent_option = UserAgentOptionScheme().loads(user_agent) CycodeClientBase.enrich_user_agent(user_agent_option.user_agent_suffix) diff --git a/cycode/cli/apps/ai_remediation/apply_fix.py b/cycode/cli/apps/ai_remediation/apply_fix.py index e0c2599b..bd840411 100644 --- a/cycode/cli/apps/ai_remediation/apply_fix.py +++ b/cycode/cli/apps/ai_remediation/apply_fix.py @@ -4,11 +4,10 @@ from patch_ng import fromstring from cycode.cli.models import CliResult -from cycode.cli.printers import ConsolePrinter def apply_fix(ctx: typer.Context, diff: str, is_fix_available: bool) -> None: - printer = ConsolePrinter(ctx) + printer = ctx.obj.get('console_printer') if not is_fix_available: printer.print_result(CliResult(success=False, message='Fix is not available for this violation')) return diff --git a/cycode/cli/apps/ai_remediation/print_remediation.py b/cycode/cli/apps/ai_remediation/print_remediation.py index c0109341..92272b76 100644 --- a/cycode/cli/apps/ai_remediation/print_remediation.py +++ b/cycode/cli/apps/ai_remediation/print_remediation.py @@ -3,11 +3,10 @@ from cycode.cli.console import console from cycode.cli.models import CliResult -from cycode.cli.printers import ConsolePrinter def print_remediation(ctx: typer.Context, remediation_markdown: str, is_fix_available: bool) -> None: - printer = ConsolePrinter(ctx) + printer = ctx.obj.get('console_printer') if printer.is_json_printer: data = {'remediation': remediation_markdown, 'is_fix_available': is_fix_available} printer.print_result(CliResult(success=True, message='Remediation fetched successfully', data=data)) diff --git a/cycode/cli/apps/auth/auth_command.py b/cycode/cli/apps/auth/auth_command.py index 8150be01..a402b0c2 100644 --- a/cycode/cli/apps/auth/auth_command.py +++ b/cycode/cli/apps/auth/auth_command.py @@ -4,13 +4,13 @@ from cycode.cli.exceptions.handle_auth_errors import handle_auth_exception from cycode.cli.logger import logger from cycode.cli.models import CliResult -from cycode.cli.printers import ConsolePrinter from cycode.cli.utils.sentry import add_breadcrumb def auth_command(ctx: typer.Context) -> None: """Authenticates your machine.""" add_breadcrumb('auth') + printer = ctx.obj.get('console_printer') if ctx.invoked_subcommand is not None: # if it is a subcommand, do nothing @@ -23,6 +23,6 @@ def auth_command(ctx: typer.Context) -> None: auth_manager.authenticate() result = CliResult(success=True, message='Successfully logged into cycode') - ConsolePrinter(ctx).print_result(result) + printer.print_result(result) except Exception as err: handle_auth_exception(ctx, err) diff --git a/cycode/cli/apps/auth/auth_common.py b/cycode/cli/apps/auth/auth_common.py index fffee388..f6120d94 100644 --- a/cycode/cli/apps/auth/auth_common.py +++ b/cycode/cli/apps/auth/auth_common.py @@ -4,13 +4,14 @@ from cycode.cli.apps.auth.models import AuthInfo from cycode.cli.exceptions.custom_exceptions import HttpUnauthorizedError, RequestHttpError -from cycode.cli.printers import ConsolePrinter from cycode.cli.user_settings.credentials_manager import CredentialsManager from cycode.cli.utils.jwt_utils import get_user_and_tenant_ids_from_access_token from cycode.cyclient.cycode_token_based_client import CycodeTokenBasedClient def get_authorization_info(ctx: Optional[typer.Context] = None) -> Optional[AuthInfo]: + printer = ctx.obj.get('console_printer') + client_id, client_secret = CredentialsManager().get_credentials() if not client_id or not client_secret: return None @@ -24,6 +25,6 @@ def get_authorization_info(ctx: Optional[typer.Context] = None) -> Optional[Auth return AuthInfo(user_id=user_id, tenant_id=tenant_id) except (RequestHttpError, HttpUnauthorizedError): if ctx: - ConsolePrinter(ctx).print_exception() + printer.print_exception() return None diff --git a/cycode/cli/apps/auth/check_command.py b/cycode/cli/apps/auth/check_command.py index cfa57f1c..0a5ea5b3 100644 --- a/cycode/cli/apps/auth/check_command.py +++ b/cycode/cli/apps/auth/check_command.py @@ -2,7 +2,6 @@ from cycode.cli.apps.auth.auth_common import get_authorization_info from cycode.cli.models import CliResult -from cycode.cli.printers import ConsolePrinter from cycode.cli.utils.sentry import add_breadcrumb @@ -10,7 +9,7 @@ def check_command(ctx: typer.Context) -> None: """Checks that your machine is associating the CLI with your Cycode account.""" add_breadcrumb('check') - printer = ConsolePrinter(ctx) + printer = ctx.obj.get('console_printer') auth_info = get_authorization_info(ctx) if auth_info is None: printer.print_result(CliResult(success=False, message='Cycode authentication failed')) diff --git a/cycode/cli/apps/scan/code_scanner.py b/cycode/cli/apps/scan/code_scanner.py index 67185dce..a3cae6b5 100644 --- a/cycode/cli/apps/scan/code_scanner.py +++ b/cycode/cli/apps/scan/code_scanner.py @@ -28,7 +28,6 @@ from cycode.cli.files_collector.sca.sca_code_scanner import perform_pre_scan_documents_actions from cycode.cli.files_collector.zip_documents import zip_documents from cycode.cli.models import CliError, Document, DocumentDetections, LocalScanResult -from cycode.cli.printers import ConsolePrinter from cycode.cli.utils import scan_utils from cycode.cli.utils.git_proxy import git_proxy from cycode.cli.utils.path_utils import get_path_by_os @@ -304,10 +303,11 @@ def scan_documents( ) -> None: scan_type = ctx.obj['scan_type'] progress_bar = ctx.obj['progress_bar'] + printer = ctx.obj.get('console_printer') if not documents_to_scan: progress_bar.stop() - ConsolePrinter(ctx).print_error( + printer.print_error( CliError( code='no_relevant_files', message='Error: The scan could not be completed - relevant files to scan are not found. ' @@ -569,7 +569,7 @@ def print_debug_scan_details(scan_details_response: 'ScanDetailsResponse') -> No def print_results( ctx: typer.Context, local_scan_results: List[LocalScanResult], errors: Optional[Dict[str, 'CliError']] = None ) -> None: - printer = ConsolePrinter(ctx) + printer = ctx.obj.get('console_printer') printer.print_scan_results(local_scan_results, errors) diff --git a/cycode/cli/cli_types.py b/cycode/cli/cli_types.py index 2a576ace..b8d4b8ee 100644 --- a/cycode/cli/cli_types.py +++ b/cycode/cli/cli_types.py @@ -10,6 +10,12 @@ class OutputTypeOption(str, Enum): TABLE = 'table' +class ExportTypeOption(str, Enum): + JSON = 'json' + HTML = 'html' + SVG = 'svg' + + class ScanTypeOption(str, Enum): SECRET = consts.SECRET_SCAN_TYPE SCA = consts.SCA_SCAN_TYPE diff --git a/cycode/cli/exceptions/handle_errors.py b/cycode/cli/exceptions/handle_errors.py index b9cb9c80..8d230902 100644 --- a/cycode/cli/exceptions/handle_errors.py +++ b/cycode/cli/exceptions/handle_errors.py @@ -4,14 +4,14 @@ import typer from cycode.cli.models import CliError, CliErrors -from cycode.cli.printers import ConsolePrinter from cycode.cli.utils.sentry import capture_exception def handle_errors( ctx: typer.Context, err: BaseException, cli_errors: CliErrors, *, return_exception: bool = False ) -> Optional['CliError']: - ConsolePrinter(ctx).print_exception(err) + printer = ctx.obj.get('console_printer') + printer.print_exception(err) if type(err) in cli_errors: error = cli_errors[type(err)].enrich(additional_message=str(err)) @@ -22,7 +22,7 @@ def handle_errors( if return_exception: return error - ConsolePrinter(ctx).print_error(error) + printer.print_error(error) return None if isinstance(err, click.ClickException): @@ -34,5 +34,5 @@ def handle_errors( if return_exception: return unknown_error - ConsolePrinter(ctx).print_error(unknown_error) + printer.print_error(unknown_error) raise typer.Exit(1) diff --git a/cycode/cli/printers/console_printer.py b/cycode/cli/printers/console_printer.py index 5ad5dac2..1f6af3ab 100644 --- a/cycode/cli/printers/console_printer.py +++ b/cycode/cli/printers/console_printer.py @@ -1,7 +1,12 @@ +import io from typing import TYPE_CHECKING, ClassVar, Dict, List, Optional, Type import typer +from rich.console import Console +from cycode.cli import consts +from cycode.cli.cli_types import ExportTypeOption +from cycode.cli.console import console, console_err from cycode.cli.exceptions.custom_exceptions import CycodeError from cycode.cli.models import CliError, CliResult from cycode.cli.printers.json_printer import JsonPrinter @@ -27,57 +32,115 @@ class ConsolePrinter: 'rich_sca': ScaTablePrinter, } - def __init__(self, ctx: typer.Context) -> None: + def __init__( + self, + ctx: typer.Context, + console_override: Optional['Console'] = None, + console_err_override: Optional['Console'] = None, + output_type_override: Optional[str] = None, + ) -> None: self.ctx = ctx + self.console = console_override or console + self.console_err = console_err_override or console_err self.scan_type = self.ctx.obj.get('scan_type') - self.output_type = self.ctx.obj.get('output') + self.output_type = output_type_override or self.ctx.obj.get('output') self.aggregation_report_url = self.ctx.obj.get('aggregation_report_url') - self._printer_class = self._AVAILABLE_PRINTERS.get(self.output_type) - if self._printer_class is None: - raise CycodeError(f'"{self.output_type}" output type is not supported.') + self.printer = self._get_scan_printer() - def print_scan_results( - self, - local_scan_results: List['LocalScanResult'], - errors: Optional[Dict[str, 'CliError']] = None, - ) -> None: - printer = self._get_scan_printer() - printer.print_scan_results(local_scan_results, errors) + self.console_record = None + + self.export_type = self.ctx.obj.get('export_type') + self.export_file = self.ctx.obj.get('export_file') + if console_override is None and self.export_type and self.export_file: + self.console_record = ConsolePrinter( + ctx, + console_override=Console(record=True, file=io.StringIO()), + console_err_override=Console(stderr=True, record=True, file=io.StringIO()), + output_type_override='json' if self.export_type == 'json' else self.output_type, + ) def _get_scan_printer(self) -> 'PrinterBase': - printer_class = self._printer_class + printer_class = self._AVAILABLE_PRINTERS.get(self.output_type) composite_printer = self._AVAILABLE_PRINTERS.get(f'{self.output_type}_{self.scan_type}') if composite_printer: printer_class = composite_printer - return printer_class(self.ctx) + if not printer_class: + raise CycodeError(f'"{self.output_type}" output type is not supported.') + + return printer_class(self.ctx, self.console, self.console_err) + + def print_scan_results( + self, + local_scan_results: List['LocalScanResult'], + errors: Optional[Dict[str, 'CliError']] = None, + ) -> None: + if self.console_record: + self.console_record.print_scan_results(local_scan_results, errors) + self.printer.print_scan_results(local_scan_results, errors) def print_result(self, result: CliResult) -> None: - self._printer_class(self.ctx).print_result(result) + if self.console_record: + self.console_record.print_result(result) + self.printer.print_result(result) def print_error(self, error: CliError) -> None: - self._printer_class(self.ctx).print_error(error) + if self.console_record: + self.console_record.print_error(error) + self.printer.print_error(error) def print_exception(self, e: Optional[BaseException] = None, force_print: bool = False) -> None: """Print traceback message in stderr if verbose mode is set.""" if force_print or self.ctx.obj.get('verbose', False): - self._printer_class(self.ctx).print_exception(e) + if self.console_record: + self.console_record.print_exception(e) + self.printer.print_exception(e) + + def export(self) -> None: + if self.console_record is None: + raise CycodeError('Console recording was not enabled. Cannot export.') + + if not self.export_file.suffix: + # resolve file extension based on the export type if not provided in the file name + self.export_file = self.export_file.with_suffix(f'.{self.export_type.lower()}') + + if self.export_type is ExportTypeOption.HTML: + self.console_record.console.save_html(self.export_file) + elif self.export_type is ExportTypeOption.SVG: + self.console_record.console.save_svg(self.export_file, title=consts.APP_NAME) + elif self.export_type is ExportTypeOption.JSON: + with open(self.export_file, 'w', encoding='UTF-8') as f: + self.console_record.console.file.seek(0) + f.write(self.console_record.console.file.read()) + else: + raise CycodeError(f'Export type "{self.export_type}" is not supported.') + + export_format_msg = f'{self.export_type.upper()} format' + if self.export_type in {ExportTypeOption.HTML, ExportTypeOption.SVG}: + export_format_msg += f' with {self.output_type.upper()} output type' + + clickable_path = f'[link=file://{self.export_file}]{self.export_file}[/link]' + self.console.print(f'[b green]Cycode CLI output exported to {clickable_path} in {export_format_msg}[/]') + + @property + def is_recording(self) -> bool: + return self.console_record is not None @property def is_json_printer(self) -> bool: - return self._printer_class == JsonPrinter + return isinstance(self.printer, JsonPrinter) @property def is_table_printer(self) -> bool: - return self._printer_class == TablePrinter + return isinstance(self.printer, TablePrinter) @property def is_text_printer(self) -> bool: - return self._printer_class == TextPrinter + return isinstance(self.printer, TextPrinter) @property def is_rich_printer(self) -> bool: - return self._printer_class == RichPrinter + return isinstance(self.printer, RichPrinter) diff --git a/cycode/cli/printers/json_printer.py b/cycode/cli/printers/json_printer.py index 76b1f7c7..6ad14e22 100644 --- a/cycode/cli/printers/json_printer.py +++ b/cycode/cli/printers/json_printer.py @@ -1,7 +1,6 @@ import json from typing import TYPE_CHECKING, Dict, List, Optional -from cycode.cli.console import console from cycode.cli.models import CliError, CliResult from cycode.cli.printers.printer_base import PrinterBase from cycode.cyclient.models import DetectionSchema @@ -14,12 +13,12 @@ class JsonPrinter(PrinterBase): def print_result(self, result: CliResult) -> None: result = {'result': result.success, 'message': result.message, 'data': result.data} - console.print_json(self.get_data_json(result)) + self.console.print_json(self.get_data_json(result)) def print_error(self, error: CliError) -> None: result = {'error': error.code, 'message': error.message} - console.print_json(self.get_data_json(result)) + self.console.print_json(self.get_data_json(result)) def print_scan_results( self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None @@ -46,7 +45,7 @@ def print_scan_results( # FIXME(MarshalX): we don't care about scan IDs in JSON output due to clumsy JSON root structure inlined_errors = [err._asdict() for err in errors.values()] - console.print_json(self._get_json_scan_result(scan_ids, detections_dict, report_urls, inlined_errors)) + self.console.print_json(self._get_json_scan_result(scan_ids, detections_dict, report_urls, inlined_errors)) def _get_json_scan_result( self, scan_ids: List[str], detections: dict, report_urls: List[str], errors: List[dict] diff --git a/cycode/cli/printers/printer_base.py b/cycode/cli/printers/printer_base.py index 9a86c3d6..f461a446 100644 --- a/cycode/cli/printers/printer_base.py +++ b/cycode/cli/printers/printer_base.py @@ -4,11 +4,12 @@ import typer -from cycode.cli.console import console_err from cycode.cli.models import CliError, CliResult from cycode.cyclient.headers import get_correlation_id if TYPE_CHECKING: + from rich.console import Console + from cycode.cli.models import LocalScanResult @@ -24,8 +25,15 @@ class PrinterBase(ABC): 'Please note that not all results may be available:[/]' ) - def __init__(self, ctx: typer.Context) -> None: + def __init__( + self, + ctx: typer.Context, + console: 'Console', + console_err: 'Console', + ) -> None: self.ctx = ctx + self.console = console + self.console_err = console_err @abstractmethod def print_scan_results( @@ -41,8 +49,7 @@ def print_result(self, result: CliResult) -> None: def print_error(self, error: CliError) -> None: pass - @staticmethod - def print_exception(e: Optional[BaseException] = None) -> None: + def print_exception(self, e: Optional[BaseException] = None) -> None: """We are printing it in stderr so, we don't care about supporting JSON and TABLE outputs. Note: @@ -54,6 +61,6 @@ def print_exception(e: Optional[BaseException] = None) -> None: else RichTraceback.from_exception(*sys.exc_info()) ) rich_traceback.show_locals = False - console_err.print(rich_traceback) + self.console_err.print(rich_traceback) - console_err.print(f'[red]Correlation ID:[/] {get_correlation_id()}') + self.console_err.print(f'[red]Correlation ID:[/] {get_correlation_id()}') diff --git a/cycode/cli/printers/rich_printer.py b/cycode/cli/printers/rich_printer.py index 61cb14ef..6693351a 100644 --- a/cycode/cli/printers/rich_printer.py +++ b/cycode/cli/printers/rich_printer.py @@ -8,7 +8,6 @@ from cycode.cli import consts from cycode.cli.cli_types import SeverityOption -from cycode.cli.console import console from cycode.cli.printers.text_printer import TextPrinter from cycode.cli.printers.utils.code_snippet_syntax import get_code_snippet_syntax from cycode.cli.printers.utils.detection_data import get_detection_title @@ -24,7 +23,7 @@ def print_scan_results( self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None ) -> None: if not errors and all(result.issue_detected == 0 for result in local_scan_results): - console.print(self.NO_DETECTIONS_MESSAGE) + self.console.print(self.NO_DETECTIONS_MESSAGE) return current_file = None @@ -44,14 +43,13 @@ def print_scan_results( self.print_report_urls_and_errors(local_scan_results, errors) - @staticmethod - def _print_file_header(file_path: str) -> None: + def _print_file_header(self, file_path: str) -> None: clickable_path = f'[link=file://{file_path}]{file_path}[/link]' file_header = Panel( Text.from_markup(f'[b purple3]:file_folder: File: {clickable_path}[/]', justify='center'), border_style='dim', ) - console.print(file_header) + self.console.print(file_header) def _get_details_table(self, detection: 'Detection') -> Table: details_table = Table(show_header=False, box=None, padding=(0, 1)) @@ -138,4 +136,4 @@ def _print_violation_card( title_align='center', ) - console.print(violation_card_panel) + self.console.print(violation_card_panel) diff --git a/cycode/cli/printers/tables/sca_table_printer.py b/cycode/cli/printers/tables/sca_table_printer.py index 70965be2..e334209c 100644 --- a/cycode/cli/printers/tables/sca_table_printer.py +++ b/cycode/cli/printers/tables/sca_table_printer.py @@ -2,7 +2,6 @@ from typing import TYPE_CHECKING, Dict, List from cycode.cli.cli_types import SeverityOption -from cycode.cli.console import console from cycode.cli.consts import LICENSE_COMPLIANCE_POLICY_ID, PACKAGE_VULNERABILITY_POLICY_ID from cycode.cli.models import Detection from cycode.cli.printers.tables.table import Table @@ -129,9 +128,8 @@ def _enrich_table_with_values(policy_id: str, table: Table, detection: Detection table.add_cell(CVE_COLUMNS, detection_details.get('vulnerability_id')) table.add_cell(LICENSE_COLUMN, detection_details.get('license')) - @staticmethod - def _print_summary_issues(detections_count: int, title: str) -> None: - console.print(f':no_entry: Found {detections_count} issues of type: [b]{title}[/]') + def _print_summary_issues(self, detections_count: int, title: str) -> None: + self.console.print(f':no_entry: Found {detections_count} issues of type: [b]{title}[/]') @staticmethod def _extract_detections_per_policy_id( diff --git a/cycode/cli/printers/tables/table_printer_base.py b/cycode/cli/printers/tables/table_printer_base.py index 73ab7f88..f36e489d 100644 --- a/cycode/cli/printers/tables/table_printer_base.py +++ b/cycode/cli/printers/tables/table_printer_base.py @@ -3,7 +3,6 @@ import typer -from cycode.cli.console import console from cycode.cli.models import CliError, CliResult from cycode.cli.printers.printer_base import PrinterBase from cycode.cli.printers.text_printer import TextPrinter @@ -14,8 +13,8 @@ class TablePrinterBase(PrinterBase, abc.ABC): - def __init__(self, ctx: typer.Context) -> None: - super().__init__(ctx) + def __init__(self, ctx: typer.Context, *args, **kwargs) -> None: + super().__init__(ctx, *args, **kwargs) self.scan_type: str = ctx.obj.get('scan_type') self.show_secret: bool = ctx.obj.get('show_secret', False) @@ -29,7 +28,7 @@ def print_scan_results( self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None ) -> None: if not errors and all(result.issue_detected == 0 for result in local_scan_results): - console.print(self.NO_DETECTIONS_MESSAGE) + self.console.print(self.NO_DETECTIONS_MESSAGE) return self._print_results(local_scan_results) @@ -37,9 +36,9 @@ def print_scan_results( if not errors: return - console.print(self.FAILED_SCAN_MESSAGE) + self.console.print(self.FAILED_SCAN_MESSAGE) for scan_id, error in errors.items(): - console.print(f'- {scan_id}: ', end='') + self.console.print(f'- {scan_id}: ', end='') self.print_error(error) def _is_git_repository(self) -> bool: @@ -49,13 +48,12 @@ def _is_git_repository(self) -> bool: def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: raise NotImplementedError - @staticmethod - def _print_table(table: 'Table') -> None: + def _print_table(self, table: 'Table') -> None: if table.get_rows(): - console.print(table.get_table()) + self.console.print(table.get_table()) - @staticmethod def _print_report_urls( + self, local_scan_results: List['LocalScanResult'], aggregation_report_url: Optional[str] = None, ) -> None: @@ -63,9 +61,9 @@ def _print_report_urls( if not report_urls and not aggregation_report_url: return if aggregation_report_url: - console.print(f'Report URL: {aggregation_report_url}') + self.console.print(f'Report URL: {aggregation_report_url}') return - console.print('Report URLs:') + self.console.print('Report URLs:') for report_url in report_urls: - console.print(f'- {report_url}') + self.console.print(f'- {report_url}') diff --git a/cycode/cli/printers/text_printer.py b/cycode/cli/printers/text_printer.py index b1eeb38e..f4dcf19a 100644 --- a/cycode/cli/printers/text_printer.py +++ b/cycode/cli/printers/text_printer.py @@ -5,7 +5,6 @@ from rich.markup import escape from cycode.cli.cli_types import SeverityOption -from cycode.cli.console import console from cycode.cli.models import CliError, CliResult, Document from cycode.cli.printers.printer_base import PrinterBase from cycode.cli.printers.utils.code_snippet_syntax import get_code_snippet_syntax @@ -17,8 +16,8 @@ class TextPrinter(PrinterBase): - def __init__(self, ctx: typer.Context) -> None: - super().__init__(ctx) + def __init__(self, ctx: typer.Context, *args, **kwargs) -> None: + super().__init__(ctx, *args, **kwargs) self.scan_type = ctx.obj.get('scan_type') self.command_scan_type: str = ctx.info_name self.show_secret: bool = ctx.obj.get('show_secret', False) @@ -28,23 +27,23 @@ def print_result(self, result: CliResult) -> None: if not result.success: color = 'red' - console.print(result.message, style=color) + self.console.print(result.message, style=color) if not result.data: return - console.print('\nAdditional data:', style=color) + self.console.print('\nAdditional data:', style=color) for name, value in result.data.items(): - console.print(f'- {name}: {value}', style=color) + self.console.print(f'- {name}: {value}', style=color) def print_error(self, error: CliError) -> None: - console.print(f'[red]Error: {error.message}[/]', highlight=False) + self.console.print(f'[red]Error: {error.message}[/]', highlight=False) def print_scan_results( self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None ) -> None: if not errors and all(result.issue_detected == 0 for result in local_scan_results): - console.print(self.NO_DETECTIONS_MESSAGE) + self.console.print(self.NO_DETECTIONS_MESSAGE) return detections, _ = sort_and_group_detections_from_scan_result(local_scan_results) @@ -58,9 +57,8 @@ def __print_document_detection(self, document: 'Document', detection: 'Detection self.__print_detection_code_segment(detection, document) self._print_new_line() - @staticmethod - def _print_new_line() -> None: - console.line() + def _print_new_line(self) -> None: + self.console.line() def __print_detection_summary(self, detection: 'Detection', document_path: str) -> None: title = get_detection_title(self.scan_type, detection) @@ -74,14 +72,14 @@ def __print_detection_summary(self, detection: 'Detection', document_path: str) detection_commit_id = detection.detection_details.get('commit_id') detection_commit_id_message = f'\nCommit SHA: {detection_commit_id}' if detection_commit_id else '' - console.print( + self.console.print( f'{severity_icon}', severity, f'violation: [b bright_red]{title}[/]{detection_commit_id_message}\n{clickable_document_path}:', ) def __print_detection_code_segment(self, detection: 'Detection', document: Document) -> None: - console.print( + self.console.print( get_code_snippet_syntax( self.scan_type, self.command_scan_type, @@ -100,19 +98,18 @@ def print_report_urls_and_errors( if not errors: return - console.print(self.FAILED_SCAN_MESSAGE) + self.console.print(self.FAILED_SCAN_MESSAGE) for scan_id, error in errors.items(): - console.print(f'- {scan_id}: ', end='') + self.console.print(f'- {scan_id}: ', end='') self.print_error(error) - @staticmethod - def print_report_urls(report_urls: List[str], aggregation_report_url: Optional[str] = None) -> None: + def print_report_urls(self, report_urls: List[str], aggregation_report_url: Optional[str] = None) -> None: if not report_urls and not aggregation_report_url: return if aggregation_report_url: - console.print(f'Report URL: {aggregation_report_url}') + self.console.print(f'Report URL: {aggregation_report_url}') return - console.print('Report URLs:') + self.console.print('Report URLs:') for report_url in report_urls: - console.print(f'- {report_url}') + self.console.print(f'- {report_url}') diff --git a/tests/cli/exceptions/test_handle_scan_errors.py b/tests/cli/exceptions/test_handle_scan_errors.py index a6b2a9ec..abd297db 100644 --- a/tests/cli/exceptions/test_handle_scan_errors.py +++ b/tests/cli/exceptions/test_handle_scan_errors.py @@ -10,6 +10,7 @@ from cycode.cli.console import console_err from cycode.cli.exceptions import custom_exceptions from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception +from cycode.cli.printers import ConsolePrinter from cycode.cli.utils.git_proxy import git_proxy if TYPE_CHECKING: @@ -18,7 +19,9 @@ @pytest.fixture() def ctx() -> typer.Context: - return typer.Context(click.Command('path'), obj={'verbose': False, 'output': OutputTypeOption.TEXT}) + ctx = typer.Context(click.Command('path'), obj={'verbose': False, 'output': OutputTypeOption.TEXT}) + ctx.obj['console_printer'] = ConsolePrinter(ctx) + return ctx @pytest.mark.parametrize( @@ -60,6 +63,7 @@ def test_handle_exception_click_error(ctx: typer.Context) -> None: def test_handle_exception_verbose(monkeypatch: 'MonkeyPatch') -> None: ctx = typer.Context(click.Command('path'), obj={'verbose': True, 'output': OutputTypeOption.TEXT}) + ctx.obj['console_printer'] = ConsolePrinter(ctx) error_text = 'test' From 0405c6b8729667f1b279dde7df6260d0a79884fc Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Fri, 25 Apr 2025 14:25:25 +0200 Subject: [PATCH 13/19] CM-47493 - Make changes in CLI v3.0.0 after feedback (#299) --- cycode/cli/app.py | 8 ++ cycode/cli/apps/ai_remediation/__init__.py | 2 +- cycode/cli/apps/auth/__init__.py | 1 + cycode/cli/apps/configure/__init__.py | 2 +- cycode/cli/apps/ignore/__init__.py | 2 +- cycode/cli/apps/report/__init__.py | 2 +- cycode/cli/apps/scan/__init__.py | 2 +- cycode/cli/apps/status/__init__.py | 2 +- cycode/cli/cli_types.py | 4 +- .../sca/npm/restore_npm_dependencies.py | 3 +- .../files_collector/sca/sca_code_scanner.py | 5 +- cycode/cli/printers/console_printer.py | 17 +++-- cycode/cli/printers/printer_base.py | 45 +++++++++++ cycode/cli/printers/rich_printer.py | 73 +++++++++++------- .../cli/printers/tables/sca_table_printer.py | 3 +- cycode/cli/printers/tables/table_printer.py | 1 + .../cli/printers/tables/table_printer_base.py | 11 +-- cycode/cli/printers/text_printer.py | 22 ++---- .../cli/printers/utils/code_snippet_syntax.py | 6 +- cycode/cli/printers/utils/detection_data.py | 76 ++++++++++++++++++- .../detection_ordering/common_ordering.py | 17 +---- 21 files changed, 214 insertions(+), 90 deletions(-) diff --git a/cycode/cli/app.py b/cycode/cli/app.py index 5c83be9e..507c03c8 100644 --- a/cycode/cli/app.py +++ b/cycode/cli/app.py @@ -3,6 +3,7 @@ from typing import Annotated, Optional import typer +from typer import rich_utils from typer.completion import install_callback, show_callback from cycode import __version__ @@ -18,11 +19,18 @@ from cycode.cyclient.models import UserAgentOptionScheme from cycode.logger import set_logging_level +# By default, it uses dim style which is hard to read with the combination of color from RICH_HELP +rich_utils.STYLE_ERRORS_SUGGESTION = 'bold' +# By default, it uses blue color which is too dark for some terminals +rich_utils.RICH_HELP = "Try [cyan]'{command_path} {help_option}'[/] for help." + + app = typer.Typer( pretty_exceptions_show_locals=False, pretty_exceptions_short=True, context_settings=CLI_CONTEXT_SETTINGS, rich_markup_mode='rich', + no_args_is_help=True, add_completion=False, # we add it manually to control the rich help panel ) diff --git a/cycode/cli/apps/ai_remediation/__init__.py b/cycode/cli/apps/ai_remediation/__init__.py index 6b5a3013..0f017cf7 100644 --- a/cycode/cli/apps/ai_remediation/__init__.py +++ b/cycode/cli/apps/ai_remediation/__init__.py @@ -2,7 +2,7 @@ from cycode.cli.apps.ai_remediation.ai_remediation_command import ai_remediation_command -app = typer.Typer() +app = typer.Typer(no_args_is_help=True) app.command(name='ai-remediation', short_help='Get AI remediation (INTERNAL).', hidden=True)(ai_remediation_command) # backward compatibility diff --git a/cycode/cli/apps/auth/__init__.py b/cycode/cli/apps/auth/__init__.py index 82e71fbc..951a9f1f 100644 --- a/cycode/cli/apps/auth/__init__.py +++ b/cycode/cli/apps/auth/__init__.py @@ -6,6 +6,7 @@ app = typer.Typer( name='auth', help='Authenticate your machine to associate the CLI with your Cycode account.', + no_args_is_help=True, ) app.callback(invoke_without_command=True)(auth_command) app.command(name='check')(check_command) diff --git a/cycode/cli/apps/configure/__init__.py b/cycode/cli/apps/configure/__init__.py index 815874d1..039c6f2e 100644 --- a/cycode/cli/apps/configure/__init__.py +++ b/cycode/cli/apps/configure/__init__.py @@ -2,7 +2,7 @@ from cycode.cli.apps.configure.configure_command import configure_command -app = typer.Typer() +app = typer.Typer(no_args_is_help=True) app.command(name='configure', short_help='Initial command to configure your CLI client authentication.')( configure_command ) diff --git a/cycode/cli/apps/ignore/__init__.py b/cycode/cli/apps/ignore/__init__.py index 3c51d38a..e6573b69 100644 --- a/cycode/cli/apps/ignore/__init__.py +++ b/cycode/cli/apps/ignore/__init__.py @@ -2,5 +2,5 @@ from cycode.cli.apps.ignore.ignore_command import ignore_command -app = typer.Typer() +app = typer.Typer(no_args_is_help=True) app.command(name='ignore', short_help='Ignores a specific value, path or rule ID.')(ignore_command) diff --git a/cycode/cli/apps/report/__init__.py b/cycode/cli/apps/report/__init__.py index f71532c8..40cc696a 100644 --- a/cycode/cli/apps/report/__init__.py +++ b/cycode/cli/apps/report/__init__.py @@ -3,6 +3,6 @@ from cycode.cli.apps.report import sbom from cycode.cli.apps.report.report_command import report_command -app = typer.Typer(name='report') +app = typer.Typer(name='report', no_args_is_help=True) app.callback(short_help='Generate report. You`ll need to specify which report type to perform.')(report_command) app.add_typer(sbom.app) diff --git a/cycode/cli/apps/scan/__init__.py b/cycode/cli/apps/scan/__init__.py index 07c15978..136e7bef 100644 --- a/cycode/cli/apps/scan/__init__.py +++ b/cycode/cli/apps/scan/__init__.py @@ -7,7 +7,7 @@ from cycode.cli.apps.scan.repository.repository_command import repository_command from cycode.cli.apps.scan.scan_command import scan_command, scan_command_result_callback -app = typer.Typer(name='scan') +app = typer.Typer(name='scan', no_args_is_help=True) app.callback( short_help='Scan the content for Secrets, IaC, SCA, and SAST violations.', diff --git a/cycode/cli/apps/status/__init__.py b/cycode/cli/apps/status/__init__.py index f01e3b30..1161b2e6 100644 --- a/cycode/cli/apps/status/__init__.py +++ b/cycode/cli/apps/status/__init__.py @@ -3,6 +3,6 @@ from cycode.cli.apps.status.status_command import status_command from cycode.cli.apps.status.version_command import version_command -app = typer.Typer() +app = typer.Typer(no_args_is_help=True) app.command(name='status', short_help='Show the CLI status and exit.')(status_command) app.command(name='version', hidden=True, short_help='Alias to status command.')(version_command) diff --git a/cycode/cli/cli_types.py b/cycode/cli/cli_types.py index b8d4b8ee..9b792a01 100644 --- a/cycode/cli/cli_types.py +++ b/cycode/cli/cli_types.py @@ -94,6 +94,6 @@ def __rich__(self) -> str: SeverityOption.INFO.value: ':blue_circle:', SeverityOption.LOW.value: ':yellow_circle:', SeverityOption.MEDIUM.value: ':orange_circle:', - SeverityOption.HIGH.value: ':heavy_large_circle:', - SeverityOption.CRITICAL.value: ':red_circle:', + SeverityOption.HIGH.value: ':red_circle:', + SeverityOption.CRITICAL.value: ':exclamation_mark:', # double_exclamation_mark is not red } diff --git a/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py b/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py index 68175d88..672ee0db 100644 --- a/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py +++ b/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py @@ -37,5 +37,6 @@ def get_lock_file_name(self) -> str: def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: return os.path.isfile(restore_file_path) - def prepare_manifest_file_path_for_command(self, manifest_file_path: str) -> str: + @staticmethod + def prepare_manifest_file_path_for_command(manifest_file_path: str) -> str: return manifest_file_path.replace(os.sep + NPM_MANIFEST_FILE_NAME, '') diff --git a/cycode/cli/files_collector/sca/sca_code_scanner.py b/cycode/cli/files_collector/sca/sca_code_scanner.py index fc8c3809..88626c9c 100644 --- a/cycode/cli/files_collector/sca/sca_code_scanner.py +++ b/cycode/cli/files_collector/sca/sca_code_scanner.py @@ -124,14 +124,15 @@ def try_restore_dependencies( def add_dependencies_tree_document( ctx: typer.Context, documents_to_scan: List[Document], is_git_diff: bool = False ) -> None: - documents_to_add: Dict[str, Document] = {} + documents_to_add: Dict[str, Document] = {document.path: document for document in documents_to_scan} restore_dependencies_list = restore_handlers(ctx, is_git_diff) for restore_dependencies in restore_dependencies_list: for document in documents_to_scan: try_restore_dependencies(ctx, documents_to_add, restore_dependencies, document) - documents_to_scan.extend(list(documents_to_add.values())) + # mutate original list using slice assignment + documents_to_scan[:] = list(documents_to_add.values()) def restore_handlers(ctx: typer.Context, is_git_diff: bool) -> List[BaseRestoreDependencies]: diff --git a/cycode/cli/printers/console_printer.py b/cycode/cli/printers/console_printer.py index 1f6af3ab..00eb38cf 100644 --- a/cycode/cli/printers/console_printer.py +++ b/cycode/cli/printers/console_printer.py @@ -29,7 +29,6 @@ class ConsolePrinter: # overrides 'table_sca': ScaTablePrinter, 'text_sca': ScaTablePrinter, - 'rich_sca': ScaTablePrinter, } def __init__( @@ -42,12 +41,7 @@ def __init__( self.ctx = ctx self.console = console_override or console self.console_err = console_err_override or console_err - - self.scan_type = self.ctx.obj.get('scan_type') self.output_type = output_type_override or self.ctx.obj.get('output') - self.aggregation_report_url = self.ctx.obj.get('aggregation_report_url') - - self.printer = self._get_scan_printer() self.console_record = None @@ -61,7 +55,16 @@ def __init__( output_type_override='json' if self.export_type == 'json' else self.output_type, ) - def _get_scan_printer(self) -> 'PrinterBase': + @property + def scan_type(self) -> str: + return self.ctx.obj.get('scan_type') + + @property + def aggregation_report_url(self) -> str: + return self.ctx.obj.get('aggregation_report_url') + + @property + def printer(self) -> 'PrinterBase': printer_class = self._AVAILABLE_PRINTERS.get(self.output_type) composite_printer = self._AVAILABLE_PRINTERS.get(f'{self.output_type}_{self.scan_type}') diff --git a/cycode/cli/printers/printer_base.py b/cycode/cli/printers/printer_base.py index f461a446..23ba7384 100644 --- a/cycode/cli/printers/printer_base.py +++ b/cycode/cli/printers/printer_base.py @@ -1,9 +1,11 @@ import sys from abc import ABC, abstractmethod +from collections import defaultdict from typing import TYPE_CHECKING, Dict, List, Optional import typer +from cycode.cli.cli_types import SeverityOption from cycode.cli.models import CliError, CliResult from cycode.cyclient.headers import get_correlation_id @@ -35,6 +37,18 @@ def __init__( self.console = console self.console_err = console_err + @property + def scan_type(self) -> str: + return self.ctx.obj.get('scan_type') + + @property + def command_scan_type(self) -> str: + return self.ctx.info_name + + @property + def show_secret(self) -> bool: + return self.ctx.obj.get('show_secret', False) + @abstractmethod def print_scan_results( self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None @@ -64,3 +78,34 @@ def print_exception(self, e: Optional[BaseException] = None) -> None: self.console_err.print(rich_traceback) self.console_err.print(f'[red]Correlation ID:[/] {get_correlation_id()}') + + def print_scan_results_summary(self, local_scan_results: List['LocalScanResult']) -> None: + """Print a summary of scan results based on severity levels. + + Args: + local_scan_results (List['LocalScanResult']): A list of local scan results containing detections. + + The summary includes the count of detections for each severity level + and is displayed in the console in a formatted string. + """ + + detections_count = 0 + severity_counts = defaultdict(int) + for local_scan_result in local_scan_results: + for document_detections in local_scan_result.document_detections: + for detection in document_detections.detections: + if detection.severity: + detections_count += 1 + severity_counts[SeverityOption(detection.severity)] += 1 + + self.console.print(f'[bold]Cycode found {detections_count} violations[/]', end=': ') + + # Example of string: CRITICAL - 6 | HIGH - 0 | MEDIUM - 14 | LOW - 0 | INFO - 0 + for index, severity in enumerate(reversed(SeverityOption), start=1): + end = ' | ' + if index == len(SeverityOption): + end = '\n' + + self.console.print( + SeverityOption.get_member_emoji(severity), severity, '-', severity_counts[severity], end=end + ) diff --git a/cycode/cli/printers/rich_printer.py b/cycode/cli/printers/rich_printer.py index 6693351a..3401b8f5 100644 --- a/cycode/cli/printers/rich_printer.py +++ b/cycode/cli/printers/rich_printer.py @@ -1,4 +1,3 @@ -from pathlib import Path from typing import TYPE_CHECKING, Dict, List, Optional from rich.console import Group @@ -10,7 +9,11 @@ from cycode.cli.cli_types import SeverityOption from cycode.cli.printers.text_printer import TextPrinter from cycode.cli.printers.utils.code_snippet_syntax import get_code_snippet_syntax -from cycode.cli.printers.utils.detection_data import get_detection_title +from cycode.cli.printers.utils.detection_data import ( + get_detection_clickable_cwe_cve, + get_detection_file_path, + get_detection_title, +) from cycode.cli.printers.utils.detection_ordering.common_ordering import sort_and_group_detections_from_scan_result from cycode.cli.printers.utils.rich_helpers import get_columns_in_1_to_3_ratio, get_markdown_panel, get_panel @@ -19,6 +22,8 @@ class RichPrinter(TextPrinter): + MAX_PATH_LENGTH = 60 + def print_scan_results( self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None ) -> None: @@ -26,14 +31,9 @@ def print_scan_results( self.console.print(self.NO_DETECTIONS_MESSAGE) return - current_file = None detections, _ = sort_and_group_detections_from_scan_result(local_scan_results) detections_count = len(detections) for detection_number, (detection, document) in enumerate(detections, start=1): - if current_file != document.path: - current_file = document.path - self._print_file_header(current_file) - self._print_violation_card( document, detection, @@ -41,16 +41,9 @@ def print_scan_results( detections_count, ) + self.print_scan_results_summary(local_scan_results) self.print_report_urls_and_errors(local_scan_results, errors) - def _print_file_header(self, file_path: str) -> None: - clickable_path = f'[link=file://{file_path}]{file_path}[/link]' - file_header = Panel( - Text.from_markup(f'[b purple3]:file_folder: File: {clickable_path}[/]', justify='center'), - border_style='dim', - ) - self.console.print(file_header) - def _get_details_table(self, detection: 'Detection') -> Table: details_table = Table(show_header=False, box=None, padding=(0, 1)) @@ -62,15 +55,32 @@ def _get_details_table(self, detection: 'Detection') -> Table: details_table.add_row('Severity', f'{severity_icon} {SeverityOption(severity).__rich__()}') detection_details = detection.detection_details - path = Path(detection_details.get('file_name', '')) - details_table.add_row('In file', path.name) # it is name already except for IaC :) - # we do not allow using rich output with SCA; SCA designed to be used with table output - if self.scan_type == consts.IAC_SCAN_TYPE: - details_table.add_row('IaC Provider', detection_details.get('infra_provider')) - elif self.scan_type == consts.SECRET_SCAN_TYPE: + path = str(get_detection_file_path(self.scan_type, detection)) + shorten_path = f'...{path[-self.MAX_PATH_LENGTH:]}' if len(path) > self.MAX_PATH_LENGTH else path + details_table.add_row('In file', f'[link=file://{path}]{shorten_path}[/]') + + if self.scan_type == consts.SECRET_SCAN_TYPE: details_table.add_row('Secret SHA', detection_details.get('sha512')) + elif self.scan_type == consts.SCA_SCAN_TYPE: + details_table.add_row('CVEs', get_detection_clickable_cwe_cve(self.scan_type, detection)) + details_table.add_row('Package', detection_details.get('package_name')) + details_table.add_row('Version', detection_details.get('package_version')) + + is_package_vulnerability = 'alert' in detection_details + if is_package_vulnerability: + details_table.add_row( + 'First patched version', detection_details['alert'].get('first_patched_version', 'Not fixed') + ) + + details_table.add_row('Dependency path', detection_details.get('dependency_paths', 'N/A')) + + if not is_package_vulnerability: + details_table.add_row('License', detection_details.get('license')) + elif self.scan_type == consts.IAC_SCAN_TYPE: + details_table.add_row('IaC Provider', detection_details.get('infra_provider')) elif self.scan_type == consts.SAST_SCAN_TYPE: + details_table.add_row('CWE', get_detection_clickable_cwe_cve(self.scan_type, detection)) details_table.add_row('Subcategory', detection_details.get('category')) details_table.add_row('Language', ', '.join(detection_details.get('languages', []))) @@ -105,12 +115,17 @@ def _print_violation_card( title=':computer: Code Snippet', ) - guidelines_panel = None - guidelines = detection.detection_details.get('remediation_guidelines') - if guidelines: - guidelines_panel = get_markdown_panel( - guidelines, - title=':clipboard: Cycode Guidelines', + is_sca_package_vulnerability = self.scan_type == consts.SCA_SCAN_TYPE and 'alert' in detection.detection_details + if is_sca_package_vulnerability: + summary = detection.detection_details['alert'].get('description') + else: + summary = detection.detection_details.get('description') or detection.message + + summary_panel = None + if summary: + summary_panel = get_markdown_panel( + summary, + title=':memo: Summary', ) custom_guidelines_panel = None @@ -124,8 +139,8 @@ def _print_violation_card( navigation = Text(f'Violation {detection_number} of {detections_count}', style='dim', justify='right') renderables = [navigation, get_columns_in_1_to_3_ratio(details_panel, code_snippet_panel)] - if guidelines_panel: - renderables.append(guidelines_panel) + if summary_panel: + renderables.append(summary_panel) if custom_guidelines_panel: renderables.append(custom_guidelines_panel) diff --git a/cycode/cli/printers/tables/sca_table_printer.py b/cycode/cli/printers/tables/sca_table_printer.py index e334209c..74ac2832 100644 --- a/cycode/cli/printers/tables/sca_table_printer.py +++ b/cycode/cli/printers/tables/sca_table_printer.py @@ -45,6 +45,7 @@ def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: self._print_summary_issues(len(detections), self._get_title(policy_id)) self._print_table(table) + self.print_scan_results_summary(local_scan_results) self._print_report_urls(local_scan_results, aggregation_report_url) @staticmethod @@ -129,7 +130,7 @@ def _enrich_table_with_values(policy_id: str, table: Table, detection: Detection table.add_cell(LICENSE_COLUMN, detection_details.get('license')) def _print_summary_issues(self, detections_count: int, title: str) -> None: - self.console.print(f':no_entry: Found {detections_count} issues of type: [b]{title}[/]') + self.console.print(f'[bold]Cycode found {detections_count} violations of type: [cyan]{title}[/]') @staticmethod def _extract_detections_per_policy_id( diff --git a/cycode/cli/printers/tables/table_printer.py b/cycode/cli/printers/tables/table_printer.py index e36b1b01..4f821c7f 100644 --- a/cycode/cli/printers/tables/table_printer.py +++ b/cycode/cli/printers/tables/table_printer.py @@ -37,6 +37,7 @@ def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: table.set_group_separator_indexes(group_separator_indexes) self._print_table(table) + self.print_scan_results_summary(local_scan_results) self._print_report_urls(local_scan_results, self.ctx.obj.get('aggregation_report_url')) def _get_table(self) -> Table: diff --git a/cycode/cli/printers/tables/table_printer_base.py b/cycode/cli/printers/tables/table_printer_base.py index f36e489d..5d2aaa73 100644 --- a/cycode/cli/printers/tables/table_printer_base.py +++ b/cycode/cli/printers/tables/table_printer_base.py @@ -1,8 +1,6 @@ import abc from typing import TYPE_CHECKING, Dict, List, Optional -import typer - from cycode.cli.models import CliError, CliResult from cycode.cli.printers.printer_base import PrinterBase from cycode.cli.printers.text_printer import TextPrinter @@ -13,16 +11,11 @@ class TablePrinterBase(PrinterBase, abc.ABC): - def __init__(self, ctx: typer.Context, *args, **kwargs) -> None: - super().__init__(ctx, *args, **kwargs) - self.scan_type: str = ctx.obj.get('scan_type') - self.show_secret: bool = ctx.obj.get('show_secret', False) - def print_result(self, result: CliResult) -> None: - TextPrinter(self.ctx).print_result(result) + TextPrinter(self.ctx, self.console, self.console_err).print_result(result) def print_error(self, error: CliError) -> None: - TextPrinter(self.ctx).print_error(error) + TextPrinter(self.ctx, self.console, self.console_err).print_error(error) def print_scan_results( self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None diff --git a/cycode/cli/printers/text_printer.py b/cycode/cli/printers/text_printer.py index f4dcf19a..6eb4b78b 100644 --- a/cycode/cli/printers/text_printer.py +++ b/cycode/cli/printers/text_printer.py @@ -1,13 +1,9 @@ -import urllib.parse from typing import TYPE_CHECKING, Dict, List, Optional -import typer -from rich.markup import escape - from cycode.cli.cli_types import SeverityOption from cycode.cli.models import CliError, CliResult, Document from cycode.cli.printers.printer_base import PrinterBase -from cycode.cli.printers.utils.code_snippet_syntax import get_code_snippet_syntax +from cycode.cli.printers.utils.code_snippet_syntax import get_code_snippet_syntax, get_detection_line from cycode.cli.printers.utils.detection_data import get_detection_title from cycode.cli.printers.utils.detection_ordering.common_ordering import sort_and_group_detections_from_scan_result @@ -16,12 +12,6 @@ class TextPrinter(PrinterBase): - def __init__(self, ctx: typer.Context, *args, **kwargs) -> None: - super().__init__(ctx, *args, **kwargs) - self.scan_type = ctx.obj.get('scan_type') - self.command_scan_type: str = ctx.info_name - self.show_secret: bool = ctx.obj.get('show_secret', False) - def print_result(self, result: CliResult) -> None: color = 'default' if not result.success: @@ -50,6 +40,7 @@ def print_scan_results( for detection, document in detections: self.__print_document_detection(document, detection) + self.print_scan_results_summary(local_scan_results) self.print_report_urls_and_errors(local_scan_results, errors) def __print_document_detection(self, document: 'Document', detection: 'Detection') -> None: @@ -66,16 +57,17 @@ def __print_detection_summary(self, detection: 'Detection', document_path: str) severity = SeverityOption(detection.severity) if detection.severity else 'N/A' severity_icon = SeverityOption.get_member_emoji(detection.severity) if detection.severity else '' - escaped_document_path = escape(urllib.parse.quote(document_path)) - clickable_document_path = f'[link file://{escaped_document_path}]{document_path}' + line_no = get_detection_line(self.scan_type, detection) + 1 + clickable_document_path = f'[u]{document_path}:{line_no}[/]' detection_commit_id = detection.detection_details.get('commit_id') detection_commit_id_message = f'\nCommit SHA: {detection_commit_id}' if detection_commit_id else '' self.console.print( - f'{severity_icon}', + severity_icon, severity, - f'violation: [b bright_red]{title}[/]{detection_commit_id_message}\n{clickable_document_path}:', + f'violation: [b bright_red]{title}[/]{detection_commit_id_message}\n' + f'[dodger_blue1]File: {clickable_document_path}[/]', ) def __print_detection_code_segment(self, detection: 'Detection', document: Document) -> None: diff --git a/cycode/cli/printers/utils/code_snippet_syntax.py b/cycode/cli/printers/utils/code_snippet_syntax.py index c3c9f59b..aae33872 100644 --- a/cycode/cli/printers/utils/code_snippet_syntax.py +++ b/cycode/cli/printers/utils/code_snippet_syntax.py @@ -17,7 +17,7 @@ def _get_code_segment_start_line(detection_line: int, lines_to_display: int) -> return 0 if start_line < 0 else start_line -def _get_detection_line(scan_type: str, detection: 'Detection') -> int: +def get_detection_line(scan_type: str, detection: 'Detection') -> int: return ( detection.detection_details.get('line', -1) if scan_type == consts.SECRET_SCAN_TYPE @@ -29,7 +29,7 @@ def _get_code_snippet_syntax_from_file( scan_type: str, detection: 'Detection', document: 'Document', lines_to_display: int, obfuscate: bool ) -> Syntax: detection_details = detection.detection_details - detection_line = _get_detection_line(scan_type, detection) + detection_line = get_detection_line(scan_type, detection) start_line_index = _get_code_segment_start_line(detection_line, lines_to_display) detection_position = get_position_in_line(document.content, detection_details.get('start_position', -1)) violation_length = detection_details.get('length', -1) @@ -69,7 +69,7 @@ def _get_code_snippet_syntax_from_git_diff( scan_type: str, detection: 'Detection', document: 'Document', obfuscate: bool ) -> Syntax: detection_details = detection.detection_details - detection_line = _get_detection_line(scan_type, detection) + detection_line = get_detection_line(scan_type, detection) detection_position = detection_details.get('start_position', -1) violation_length = detection_details.get('length', -1) diff --git a/cycode/cli/printers/utils/detection_data.py b/cycode/cli/printers/utils/detection_data.py index 66171226..358b4c63 100644 --- a/cycode/cli/printers/utils/detection_data.py +++ b/cycode/cli/printers/utils/detection_data.py @@ -1,4 +1,5 @@ -from typing import TYPE_CHECKING +from pathlib import Path +from typing import TYPE_CHECKING, Optional from cycode.cli import consts @@ -6,6 +7,63 @@ from cycode.cyclient.models import Detection +def get_cwe_cve_link(cwe_cve: Optional[str]) -> Optional[str]: + if not cwe_cve: + return None + + if cwe_cve.startswith('GHSA'): + return f'https://github.com/advisories/{cwe_cve}' + + if cwe_cve.startswith('CWE'): + # string example: 'CWE-532: Insertion of Sensitive Information into Log File' + parts = cwe_cve.split('-') + if len(parts) < 1: + return None + + number = '' + for char in parts[1]: + if char.isdigit(): + number += char + else: + break + + return f'https://cwe.mitre.org/data/definitions/{number}' + + if cwe_cve.startswith('CVE'): + return f'https://cve.mitre.org/cgi-bin/cvename.cgi?name={cwe_cve}' + + return None + + +def get_detection_clickable_cwe_cve(scan_type: str, detection: 'Detection') -> str: + def link(url: str, name: str) -> str: + return f'[link={url}]{name}[/]' + + if scan_type == consts.SCA_SCAN_TYPE: + cve = detection.detection_details.get('vulnerability_id') + return link(get_cwe_cve_link(cve), cve) if cve else '' + if scan_type == consts.SAST_SCAN_TYPE: + renderables = [] + for cwe in detection.detection_details.get('cwe', []): + cwe and renderables.append(link(get_cwe_cve_link(cwe), cwe)) + return ', '.join(renderables) + + return '' + + +def get_detection_cwe_cve(scan_type: str, detection: 'Detection') -> Optional[str]: + if scan_type == consts.SCA_SCAN_TYPE: + return detection.detection_details.get('vulnerability_id') + if scan_type == consts.SAST_SCAN_TYPE: + cwes = detection.detection_details.get('cwe') # actually it is List[str] + if not cwes: + return None + + return ' | '.join(cwes) + + return None + + def get_detection_title(scan_type: str, detection: 'Detection') -> str: title = detection.message if scan_type == consts.SAST_SCAN_TYPE: @@ -13,4 +71,18 @@ def get_detection_title(scan_type: str, detection: 'Detection') -> str: elif scan_type == consts.SECRET_SCAN_TYPE: title = f'Hardcoded {detection.type} is used' - return title + is_sca_package_vulnerability = scan_type == consts.SCA_SCAN_TYPE and 'alert' in detection.detection_details + if is_sca_package_vulnerability: + title = detection.detection_details['alert'].get('summary', 'N/A') + + cwe_cve = get_detection_cwe_cve(scan_type, detection) + return f'[{cwe_cve}] {title}' if cwe_cve else title + + +def get_detection_file_path(scan_type: str, detection: 'Detection') -> Path: + if scan_type == consts.SECRET_SCAN_TYPE: + folder_path = detection.detection_details.get('file_path', '') + file_name = detection.detection_details.get('file_name', '') + return Path.joinpath(Path(folder_path), Path(file_name)) + + return Path(detection.detection_details.get('file_name', '')) diff --git a/cycode/cli/printers/utils/detection_ordering/common_ordering.py b/cycode/cli/printers/utils/detection_ordering/common_ordering.py index 531cbc4c..d93b858e 100644 --- a/cycode/cli/printers/utils/detection_ordering/common_ordering.py +++ b/cycode/cli/printers/utils/detection_ordering/common_ordering.py @@ -1,4 +1,3 @@ -from collections import defaultdict from typing import TYPE_CHECKING, List, Set, Tuple from cycode.cli.cli_types import SeverityOption @@ -37,22 +36,14 @@ def _sort_detections_by_file_path( def sort_and_group_detections( detections_with_documents: List[Tuple['Detection', 'Document']], ) -> GroupedDetections: - """Sort detections by severity and group by file name.""" - detections = [] + """Sort detections by severity. We do not have groping here (don't find the best one yet).""" group_separator_indexes = set() # we sort detections by file path to make persist output order - sorted_detections = _sort_detections_by_file_path(detections_with_documents) + sorted_by_path_detections = _sort_detections_by_file_path(detections_with_documents) + sorted_by_severity = _sort_detections_by_severity(sorted_by_path_detections) - grouped_by_file_path = defaultdict(list) - for detection, document in sorted_detections: - grouped_by_file_path[document.path].append((detection, document)) - - for file_path_group in grouped_by_file_path.values(): - group_separator_indexes.add(len(detections) - 1) # indexing starts from 0 - detections.extend(_sort_detections_by_severity(file_path_group)) - - return detections, group_separator_indexes + return sorted_by_severity, group_separator_indexes def sort_and_group_detections_from_scan_result(local_scan_results: List['LocalScanResult']) -> GroupedDetections: From 070763648dbb34e9c8b297379958833e4b84854c Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Tue, 29 Apr 2025 12:10:52 +0200 Subject: [PATCH 14/19] CM-45588 - Improve `--help` (#302) --- .github/workflows/build_executable.yml | 3 +- .pre-commit-hooks.yaml | 4 +- CONTRIBUTING.md | 4 +- README.md | 20 +- cycode/cli/app.py | 10 + cycode/cli/apps/ai_remediation/__init__.py | 15 +- .../ai_remediation/ai_remediation_command.py | 9 +- cycode/cli/apps/auth/__init__.py | 20 +- cycode/cli/apps/auth/auth_command.py | 13 +- cycode/cli/apps/auth/auth_common.py | 9 +- cycode/cli/apps/auth/auth_manager.py | 4 +- cycode/cli/apps/auth/check_command.py | 24 -- cycode/cli/apps/configure/__init__.py | 17 +- .../cli/apps/configure/configure_command.py | 15 +- cycode/cli/apps/ignore/ignore_command.py | 17 +- cycode/cli/apps/report/report_command.py | 6 +- cycode/cli/apps/scan/__init__.py | 11 +- cycode/cli/apps/scan/code_scanner.py | 162 ++++++-- .../commit_history/commit_history_command.py | 6 +- cycode/cli/apps/scan/path/path_command.py | 4 +- .../scan/pre_commit/pre_commit_command.py | 4 +- .../scan/pre_receive/pre_receive_command.py | 4 +- .../scan/repository/repository_command.py | 2 +- cycode/cli/apps/scan/scan_command.py | 25 +- cycode/cli/apps/status/get_cli_status.py | 8 +- cycode/cli/apps/status/models.py | 3 +- cycode/cli/apps/status/status_command.py | 18 +- cycode/cli/config.py | 2 +- cycode/cli/consts.py | 10 +- cycode/cli/exceptions/custom_exceptions.py | 6 +- cycode/cli/exceptions/handle_scan_errors.py | 5 +- cycode/cli/files_collector/excluder.py | 8 +- .../iac/tf_content_generator.py | 7 +- .../files_collector/models/in_memory_zip.py | 2 +- cycode/cli/files_collector/path_documents.py | 18 +- .../files_collector/repository_documents.py | 9 +- .../sca/base_restore_dependencies.py | 6 +- .../sca/go/restore_go_dependencies.py | 4 +- .../sca/maven/restore_gradle_dependencies.py | 10 +- .../sca/maven/restore_maven_dependencies.py | 6 +- .../sca/npm/restore_npm_dependencies.py | 3 +- .../sca/nuget/restore_nuget_dependencies.py | 3 +- .../sca/ruby/restore_ruby_dependencies.py | 4 +- .../sca/sbt/restore_sbt_dependencies.py | 4 +- .../files_collector/sca/sca_code_scanner.py | 30 +- cycode/cli/files_collector/walk_ignore.py | 6 +- cycode/cli/files_collector/zip_documents.py | 4 +- cycode/cli/models.py | 18 +- cycode/cli/printers/console_printer.py | 8 +- cycode/cli/printers/json_printer.py | 6 +- cycode/cli/printers/printer_base.py | 9 +- cycode/cli/printers/rich_printer.py | 6 +- .../cli/printers/tables/sca_table_printer.py | 18 +- cycode/cli/printers/tables/table.py | 16 +- cycode/cli/printers/tables/table_models.py | 6 +- cycode/cli/printers/tables/table_printer.py | 4 +- .../cli/printers/tables/table_printer_base.py | 8 +- cycode/cli/printers/text_printer.py | 8 +- .../detection_ordering/common_ordering.py | 22 +- .../utils/detection_ordering/sca_ordering.py | 13 +- cycode/cli/user_settings/base_file_manager.py | 7 +- .../cli/user_settings/config_file_manager.py | 11 +- .../user_settings/configuration_manager.py | 10 +- .../cli/user_settings/credentials_manager.py | 10 +- cycode/cli/utils/enum_utils.py | 3 +- cycode/cli/utils/get_api_client.py | 4 +- cycode/cli/utils/git_proxy.py | 20 +- cycode/cli/utils/ignore_utils.py | 30 +- cycode/cli/utils/jwt_utils.py | 4 +- cycode/cli/utils/path_utils.py | 12 +- cycode/cli/utils/progress_bar.py | 10 +- cycode/cli/utils/scan_batch.py | 16 +- cycode/cli/utils/shell_executor.py | 4 +- cycode/cli/utils/task_timer.py | 12 +- cycode/cli/utils/version_checker.py | 16 +- cycode/cli/utils/yaml_utils.py | 15 +- cycode/cyclient/cycode_client_base.py | 12 +- cycode/cyclient/cycode_dev_based_client.py | 4 +- cycode/cyclient/headers.py | 1 + cycode/cyclient/models.py | 52 +-- cycode/cyclient/report_client.py | 4 +- cycode/cyclient/scan_client.py | 10 +- cycode/logger.py | 4 +- poetry.lock | 377 +++++++++--------- process_executable_file.py | 10 +- pyproject.toml | 20 +- .../commands/version/test_version_checker.py | 2 +- .../cli/exceptions/test_handle_scan_errors.py | 4 +- tests/cli/files_collector/test_walk_ignore.py | 4 +- tests/cyclient/test_auth_client.py | 4 +- tests/cyclient/test_scan_client.py | 11 +- tests/test_performance_get_all_files.py | 14 +- .../test_configuration_manager.py | 3 +- tests/utils/test_ignore_utils.py | 6 +- 94 files changed, 819 insertions(+), 633 deletions(-) delete mode 100644 cycode/cli/apps/auth/check_command.py diff --git a/.github/workflows/build_executable.yml b/.github/workflows/build_executable.yml index 44c9a02a..41cfa2ed 100644 --- a/.github/workflows/build_executable.yml +++ b/.github/workflows/build_executable.yml @@ -166,6 +166,7 @@ jobs: shell: cmd env: SM_HOST: ${{ secrets.SM_HOST }} + SM_KEYPAIR_ALIAS: ${{ secrets.SM_KEYPAIR_ALIAS }} SM_API_KEY: ${{ secrets.SM_API_KEY }} SM_CLIENT_CERT_PASSWORD: ${{ secrets.SM_CLIENT_CERT_PASSWORD }} SM_CODE_SIGNING_CERT_SHA1_HASH: ${{ secrets.SM_CODE_SIGNING_CERT_SHA1_HASH }} @@ -174,7 +175,7 @@ jobs: curl -X GET https://one.digicert.com/signingmanager/api-ui/v1/releases/smtools-windows-x64.msi/download -H "x-api-key:%SM_API_KEY%" -o smtools-windows-x64.msi msiexec /i smtools-windows-x64.msi /quiet /qn C:\Windows\System32\certutil.exe -csp "DigiCert Signing Manager KSP" -key -user - smksp_cert_sync.exe + smctl windows certsync --keypair-alias=%SM_KEYPAIR_ALIAS% :: sign executable signtool.exe sign /sha1 %SM_CODE_SIGNING_CERT_SHA1_HASH% /tr http://timestamp.digicert.com /td SHA256 /fd SHA256 ".\dist\cycode-cli.exe" diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 40e7a614..02a86db0 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -3,10 +3,10 @@ language: python language_version: python3 entry: cycode - args: [ '--no-progress-meter', 'scan', '--scan-type', 'secret', 'pre_commit' ] + args: [ '--no-progress-meter', 'scan', '--scan-type', 'secret', 'pre-commit' ] - id: cycode-sca name: Cycode SCA pre-commit defender language: python language_version: python3 entry: cycode - args: [ '--no-progress-meter', 'scan', '--scan-type', 'sca', 'pre_commit' ] + args: [ '--no-progress-meter', 'scan', '--scan-type', 'sca', 'pre-commit' ] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 75b8e85f..857a27cd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,7 @@ But it’s fine to use a higher version without using new features from these ve The project is under Poetry project management. To deal with it, you should install it on your system: -Install Poetry (feel free to use Brew, etc): +Install Poetry (feel free to use Brew, etc.): ```shell curl -sSL https://install.python-poetry.org | python - -y @@ -70,6 +70,8 @@ poetry run ruff format . Many rules support auto-fixing. You can run it with the `--fix` flag. +Plugin for JB IDEs with auto formatting on save is available [here](https://plugins.jetbrains.com/plugin/20574-ruff). + ### Branching and versioning We use the `main` branch as the main one. diff --git a/README.md b/README.md index 6218cba8..fbe5c6a6 100644 --- a/README.md +++ b/README.md @@ -281,7 +281,7 @@ The following are the options and commands available with the Cycode CLI applica | [auth](#using-the-auth-command) | Authenticate your machine to associate the CLI with your Cycode account. | | [configure](#using-the-configure-command) | Initial command to configure your CLI client authentication. | | [ignore](#ignoring-scan-results) | Ignores a specific value, path or rule ID. | -| [scan](#running-a-scan) | Scan the content for Secrets/IaC/SCA/SAST violations. You`ll need to specify which scan type to perform: commit_history/path/repository/etc. | +| [scan](#running-a-scan) | Scan the content for Secrets/IaC/SCA/SAST violations. You`ll need to specify which scan type to perform: commit-history/path/repository/etc. | | [report](#report-command) | Generate report. You`ll need to specify which report type to perform. | | status | Show the CLI status and exit. | @@ -294,7 +294,7 @@ The Cycode CLI application offers several types of scans so that you can choose | Option | Description | |------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `-t, --scan-type [secret\|iac\|sca\|sast]` | Specify the scan you wish to execute (`secret`/`iac`/`sca`/`sast`), the default is `secret`. | -| `--secret TEXT` | Specify a Cycode client secret for this specific scan execution. | +| `--client-secret TEXT` | Specify a Cycode client secret for this specific scan execution. | | `--client-id TEXT` | Specify a Cycode client ID for this specific scan execution. | | `--show-secret BOOLEAN` | Show secrets in plain text. See [Show/Hide Secrets](#showhide-secrets) section for more details. | | `--soft-fail BOOLEAN` | Run scan without failing, always return a non-error status code. See [Soft Fail](#soft-fail) section for more details. | @@ -308,9 +308,9 @@ The Cycode CLI application offers several types of scans so that you can choose | Command | Description | |----------------------------------------|-----------------------------------------------------------------| -| [commit_history](#commit-history-scan) | Scan all the commits history in this git repository | +| [commit-history](#commit-history-scan) | Scan all the commits history in this git repository | | [path](#path-scan) | Scan the files in the path supplied in the command | -| [pre_commit](#pre-commit-scan) | Use this command to scan the content that was not committed yet | +| [pre-commit](#pre-commit-scan) | Use this command to scan the content that was not committed yet | | [repository](#repository-scan) | Scan git repository including its history | ### Options @@ -466,25 +466,25 @@ A commit history scan is limited to a local repository’s previous commits, foc To execute a commit history scan, execute the following: -`cycode scan commit_history {{path}}` +`cycode scan commit-history {{path}}` For example, consider a scenario in which you want to scan the commit history for a repository stored in `~/home/git/codebase`. You could then execute the following: -`cycode scan commit_history ~/home/git/codebase` +`cycode scan commit-history ~/home/git/codebase` The following options are available for use with this command: | Option | Description | |---------------------------|----------------------------------------------------------------------------------------------------------| -| `-r, --commit_range TEXT` | Scan a commit range in this git repository, by default cycode scans all commit history (example: HEAD~1) | +| `-r, --commit-range TEXT` | Scan a commit range in this git repository, by default cycode scans all commit history (example: HEAD~1) | #### Commit Range Option -The commit history scan, by default, examines the repository’s entire commit history, all the way back to the initial commit. You can instead limit the scan to a specific commit range by adding the argument `--commit_range` (`-r`) followed by the name you specify. +The commit history scan, by default, examines the repository’s entire commit history, all the way back to the initial commit. You can instead limit the scan to a specific commit range by adding the argument `--commit-range` (`-r`) followed by the name you specify. Consider the previous example. If you wanted to scan only specific commits in your repository, you could execute the following: -`cycode scan commit_history -r {{from-commit-id}}...{{to-commit-id}} ~/home/git/codebase` +`cycode scan commit-history -r {{from-commit-id}}...{{to-commit-id}} ~/home/git/codebase` ### Pre-Commit Scan @@ -823,7 +823,7 @@ The following commands are available for use with this command: | Command | Description | |------------------|-----------------------------------------------------------------| | `path` | Generate SBOM report for provided path in the command | -| `repository_url` | Generate SBOM report for provided repository URI in the command | +| `repository-url` | Generate SBOM report for provided repository URI in the command | ### Repository diff --git a/cycode/cli/app.py b/cycode/cli/app.py index 507c03c8..b07b3221 100644 --- a/cycode/cli/app.py +++ b/cycode/cli/app.py @@ -25,10 +25,19 @@ rich_utils.RICH_HELP = "Try [cyan]'{command_path} {help_option}'[/] for help." +_cycode_cli_docs = 'https://github.com/cycodehq/cycode-cli/blob/main/README.md' +_cycode_cli_epilog = f"""[bold]Documentation[/] + + + +For more details and advanced usage, visit: [link={_cycode_cli_docs}]{_cycode_cli_docs}[/link] +""" + app = typer.Typer( pretty_exceptions_show_locals=False, pretty_exceptions_short=True, context_settings=CLI_CONTEXT_SETTINGS, + epilog=_cycode_cli_epilog, rich_markup_mode='rich', no_args_is_help=True, add_completion=False, # we add it manually to control the rich help panel @@ -125,6 +134,7 @@ def app_callback( ), ] = False, ) -> None: + """[bold cyan]Cycode CLI - Command Line Interface for Cycode.[/]""" init_sentry() add_breadcrumb('cycode') diff --git a/cycode/cli/apps/ai_remediation/__init__.py b/cycode/cli/apps/ai_remediation/__init__.py index 0f017cf7..cd471a08 100644 --- a/cycode/cli/apps/ai_remediation/__init__.py +++ b/cycode/cli/apps/ai_remediation/__init__.py @@ -2,8 +2,19 @@ from cycode.cli.apps.ai_remediation.ai_remediation_command import ai_remediation_command -app = typer.Typer(no_args_is_help=True) -app.command(name='ai-remediation', short_help='Get AI remediation (INTERNAL).', hidden=True)(ai_remediation_command) +app = typer.Typer() + +_ai_remediation_epilog = """ +Note: AI remediation suggestions are generated automatically and should be reviewed before applying. +""" + +app.command( + name='ai-remediation', + short_help='Get AI remediation (INTERNAL).', + epilog=_ai_remediation_epilog, + hidden=True, + no_args_is_help=True, +)(ai_remediation_command) # backward compatibility app.command(hidden=True, name='ai_remediation')(ai_remediation_command) diff --git a/cycode/cli/apps/ai_remediation/ai_remediation_command.py b/cycode/cli/apps/ai_remediation/ai_remediation_command.py index 0a82b815..ea5ef826 100644 --- a/cycode/cli/apps/ai_remediation/ai_remediation_command.py +++ b/cycode/cli/apps/ai_remediation/ai_remediation_command.py @@ -16,7 +16,14 @@ def ai_remediation_command( bool, typer.Option('--fix', help='Apply fixes to resolve violations. Note: fix could be not available.') ] = False, ) -> None: - """Get AI remediation (INTERNAL).""" + """:robot: [bold cyan]Get AI-powered remediation for security issues.[/] + + This command provides AI-generated remediation guidance for detected security issues. + + Example usage: + * `cycode ai-remediation `: View remediation guidance + * `cycode ai-remediation --fix`: Apply suggested fixes + """ client = get_scan_cycode_client() try: diff --git a/cycode/cli/apps/auth/__init__.py b/cycode/cli/apps/auth/__init__.py index 951a9f1f..beecae38 100644 --- a/cycode/cli/apps/auth/__init__.py +++ b/cycode/cli/apps/auth/__init__.py @@ -1,12 +1,14 @@ import typer from cycode.cli.apps.auth.auth_command import auth_command -from cycode.cli.apps.auth.check_command import check_command - -app = typer.Typer( - name='auth', - help='Authenticate your machine to associate the CLI with your Cycode account.', - no_args_is_help=True, -) -app.callback(invoke_without_command=True)(auth_command) -app.command(name='check')(check_command) + +_auth_command_docs = 'https://github.com/cycodehq/cycode-cli/blob/main/README.md#using-the-auth-command' +_auth_command_epilog = f"""[bold]Documentation[/] + + + +For more details and advanced usage, visit: [link={_auth_command_docs}]{_auth_command_docs}[/link] +""" + +app = typer.Typer(no_args_is_help=False) +app.command(name='auth', epilog=_auth_command_epilog, short_help='Authenticate your machine with Cycode.')(auth_command) diff --git a/cycode/cli/apps/auth/auth_command.py b/cycode/cli/apps/auth/auth_command.py index a402b0c2..817e0213 100644 --- a/cycode/cli/apps/auth/auth_command.py +++ b/cycode/cli/apps/auth/auth_command.py @@ -8,14 +8,17 @@ def auth_command(ctx: typer.Context) -> None: - """Authenticates your machine.""" + """:key: [bold cyan]Authenticate your machine with Cycode.[/] + + This command handles authentication with Cycode's security platform. + + Example usage: + * `cycode auth`: Start interactive authentication + * `cycode auth --help`: View authentication options + """ add_breadcrumb('auth') printer = ctx.obj.get('console_printer') - if ctx.invoked_subcommand is not None: - # if it is a subcommand, do nothing - return - try: logger.debug('Starting authentication process') diff --git a/cycode/cli/apps/auth/auth_common.py b/cycode/cli/apps/auth/auth_common.py index f6120d94..52b7b6fa 100644 --- a/cycode/cli/apps/auth/auth_common.py +++ b/cycode/cli/apps/auth/auth_common.py @@ -1,6 +1,4 @@ -from typing import Optional - -import typer +from typing import TYPE_CHECKING, Optional from cycode.cli.apps.auth.models import AuthInfo from cycode.cli.exceptions.custom_exceptions import HttpUnauthorizedError, RequestHttpError @@ -8,8 +6,11 @@ from cycode.cli.utils.jwt_utils import get_user_and_tenant_ids_from_access_token from cycode.cyclient.cycode_token_based_client import CycodeTokenBasedClient +if TYPE_CHECKING: + from typer import Context + -def get_authorization_info(ctx: Optional[typer.Context] = None) -> Optional[AuthInfo]: +def get_authorization_info(ctx: 'Context') -> Optional[AuthInfo]: printer = ctx.obj.get('console_printer') client_id, client_secret = CredentialsManager().get_credentials() diff --git a/cycode/cli/apps/auth/auth_manager.py b/cycode/cli/apps/auth/auth_manager.py index 2652bfe1..56a480e4 100644 --- a/cycode/cli/apps/auth/auth_manager.py +++ b/cycode/cli/apps/auth/auth_manager.py @@ -1,6 +1,6 @@ import time import webbrowser -from typing import TYPE_CHECKING, Tuple +from typing import TYPE_CHECKING from cycode.cli.exceptions.custom_exceptions import AuthProcessError from cycode.cli.user_settings.configuration_manager import ConfigurationManager @@ -78,7 +78,7 @@ def get_api_token_polling(self, session_id: str, code_verifier: str) -> 'ApiToke def save_api_token(self, api_token: 'ApiToken') -> None: self.credentials_manager.update_credentials(api_token.client_id, api_token.secret) - def _generate_pkce_code_pair(self) -> Tuple[str, str]: + def _generate_pkce_code_pair(self) -> tuple[str, str]: code_verifier = generate_random_string(self.CODE_VERIFIER_LENGTH) code_challenge = hash_string_to_sha256(code_verifier) return code_challenge, code_verifier diff --git a/cycode/cli/apps/auth/check_command.py b/cycode/cli/apps/auth/check_command.py deleted file mode 100644 index 0a5ea5b3..00000000 --- a/cycode/cli/apps/auth/check_command.py +++ /dev/null @@ -1,24 +0,0 @@ -import typer - -from cycode.cli.apps.auth.auth_common import get_authorization_info -from cycode.cli.models import CliResult -from cycode.cli.utils.sentry import add_breadcrumb - - -def check_command(ctx: typer.Context) -> None: - """Checks that your machine is associating the CLI with your Cycode account.""" - add_breadcrumb('check') - - printer = ctx.obj.get('console_printer') - auth_info = get_authorization_info(ctx) - if auth_info is None: - printer.print_result(CliResult(success=False, message='Cycode authentication failed')) - return - - printer.print_result( - CliResult( - success=True, - message='Cycode authentication verified', - data={'user_id': auth_info.user_id, 'tenant_id': auth_info.tenant_id}, - ) - ) diff --git a/cycode/cli/apps/configure/__init__.py b/cycode/cli/apps/configure/__init__.py index 039c6f2e..ce73c450 100644 --- a/cycode/cli/apps/configure/__init__.py +++ b/cycode/cli/apps/configure/__init__.py @@ -2,7 +2,18 @@ from cycode.cli.apps.configure.configure_command import configure_command +_configure_command_docs = 'https://github.com/cycodehq/cycode-cli/blob/main/README.md#using-the-configure-command' +_configure_command_epilog = f"""[bold]Documentation[/] + + + +For more details and advanced usage, visit: [link={_configure_command_docs}]{_configure_command_docs}[/link] +""" + + app = typer.Typer(no_args_is_help=True) -app.command(name='configure', short_help='Initial command to configure your CLI client authentication.')( - configure_command -) +app.command( + name='configure', + epilog=_configure_command_epilog, + short_help='Initial command to configure your CLI client authentication.', +)(configure_command) diff --git a/cycode/cli/apps/configure/configure_command.py b/cycode/cli/apps/configure/configure_command.py index 2aa86a8f..348e3ccb 100644 --- a/cycode/cli/apps/configure/configure_command.py +++ b/cycode/cli/apps/configure/configure_command.py @@ -23,7 +23,20 @@ def _should_update_value( def configure_command() -> None: - """Configure your CLI client authentication manually.""" + """:gear: [bold cyan]Configure Cycode CLI settings.[/] + + This command allows you to configure various aspects of the Cycode CLI. + + Configuration options: + * API URL: The base URL for Cycode's API (for on-premise or EU installations) + * APP URL: The base URL for Cycode's web application (for on-premise or EU installations) + * Client ID: Your Cycode client ID for authentication + * Client Secret: Your Cycode client secret for authentication + + Example usage: + * `cycode configure`: Start interactive configuration + * `cycode configure --help`: View configuration options + """ add_breadcrumb('configure') global_config_manager = CONFIGURATION_MANAGER.global_config_file_manager diff --git a/cycode/cli/apps/ignore/ignore_command.py b/cycode/cli/apps/ignore/ignore_command.py index 079a3c2d..1183114a 100644 --- a/cycode/cli/apps/ignore/ignore_command.py +++ b/cycode/cli/apps/ignore/ignore_command.py @@ -83,7 +83,20 @@ def ignore_command( # noqa: C901 bool, typer.Option('--global', '-g', help='Add an ignore rule to the global CLI config.') ] = False, ) -> None: - """Ignores a specific value, path or rule ID.""" + """:no_entry: [bold cyan]Ignore specific findings or paths in scans.[/] + + This command allows you to exclude specific items from Cycode scans, including: + * Paths: Exclude specific files or directories + * Rules: Ignore specific security rules + * Values: Exclude specific sensitive values + * Packages: Ignore specific package versions + * CVEs: Exclude specific vulnerabilities + + Example usage: + * `cycode ignore --by-path .env`: Ignore the tests directory + * `cycode ignore --by-rule GUID`: Ignore rule with the specified GUID + * `cycode ignore --by-package lodash@4.17.21`: Ignore lodash version 4.17.21 + """ add_breadcrumb('ignore') all_by_values = [by_value, by_sha, by_path, by_rule, by_package, by_cve] @@ -145,4 +158,4 @@ def ignore_command( # noqa: C901 'exclusion_value': exclusion_value, }, ) - configuration_manager.add_exclusion(configuration_scope, scan_type, exclusion_type, exclusion_value) + configuration_manager.add_exclusion(configuration_scope, str(scan_type), exclusion_type, exclusion_value) diff --git a/cycode/cli/apps/report/report_command.py b/cycode/cli/apps/report/report_command.py index 91a061c3..75debb33 100644 --- a/cycode/cli/apps/report/report_command.py +++ b/cycode/cli/apps/report/report_command.py @@ -5,7 +5,11 @@ def report_command(ctx: typer.Context) -> int: - """Generate report.""" + """:bar_chart: [bold cyan]Generate security reports.[/] + + Example usage: + * `cycode report sbom`: Generate SBOM report + """ add_breadcrumb('report') ctx.obj['progress_bar'] = get_progress_bar(hidden=False, sections=SBOM_REPORT_PROGRESS_BAR_SECTIONS) return 1 diff --git a/cycode/cli/apps/scan/__init__.py b/cycode/cli/apps/scan/__init__.py index 136e7bef..ada2d105 100644 --- a/cycode/cli/apps/scan/__init__.py +++ b/cycode/cli/apps/scan/__init__.py @@ -9,14 +9,23 @@ app = typer.Typer(name='scan', no_args_is_help=True) +_scan_command_docs = 'https://github.com/cycodehq/cycode-cli/blob/main/README.md#scan-command' +_scan_command_epilog = f"""[bold]Documentation[/] + + + +For more details and advanced usage, visit: [link={_scan_command_docs}]{_scan_command_docs}[/link] +""" + app.callback( short_help='Scan the content for Secrets, IaC, SCA, and SAST violations.', result_callback=scan_command_result_callback, + epilog=_scan_command_epilog, )(scan_command) app.command(name='path', short_help='Scan the files in the paths provided in the command.')(path_command) app.command(name='repository', short_help='Scan the Git repository included files.')(repository_command) -app.command(name='commit-history', short_help='Scan all the commits history in this git repository.')( +app.command(name='commit-history', short_help='Scan all the commits history in this Git repository.')( commit_history_command ) app.command( diff --git a/cycode/cli/apps/scan/code_scanner.py b/cycode/cli/apps/scan/code_scanner.py index a3cae6b5..0209d9da 100644 --- a/cycode/cli/apps/scan/code_scanner.py +++ b/cycode/cli/apps/scan/code_scanner.py @@ -3,7 +3,7 @@ import sys import time from platform import platform -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Callable, Optional from uuid import UUID, uuid4 import click @@ -34,6 +34,7 @@ from cycode.cli.utils.progress_bar import ScanProgressBarSection from cycode.cli.utils.scan_batch import run_parallel_batched_scan from cycode.cli.utils.scan_utils import set_issue_detected +from cycode.cli.utils.shell_executor import shell from cycode.cyclient.models import Detection, DetectionSchema, DetectionsPerFile, ZippedFileScanResult from cycode.logger import get_logger, set_logging_level @@ -83,7 +84,7 @@ def scan_sca_commit_range(ctx: typer.Context, path: str, commit_range: str) -> N scan_commit_range_documents(ctx, from_commit_documents, to_commit_documents, scan_parameters=scan_parameters) -def scan_disk_files(ctx: typer.Context, paths: Tuple[str]) -> None: +def scan_disk_files(ctx: typer.Context, paths: tuple[str, ...]) -> None: scan_type = ctx.obj['scan_type'] progress_bar = ctx.obj['progress_bar'] @@ -95,7 +96,7 @@ def scan_disk_files(ctx: typer.Context, paths: Tuple[str]) -> None: handle_scan_exception(ctx, e) -def set_issue_detected_by_scan_results(ctx: typer.Context, scan_results: List[LocalScanResult]) -> None: +def set_issue_detected_by_scan_results(ctx: typer.Context, scan_results: list[LocalScanResult]) -> None: set_issue_detected(ctx, any(scan_result.issue_detected for scan_result in scan_results)) @@ -109,6 +110,7 @@ def _should_use_sync_flow(command_scan_type: str, scan_type: str, sync_option: b - for IAC scan, sync flow is always used - for SAST scan, sync flow is not supported - for SCA and Secrets scan, sync flow is supported only for path/repository scan + """ if not sync_option and scan_type != consts.IAC_SCAN_TYPE: return False @@ -160,14 +162,14 @@ def _enrich_scan_result_with_data_from_detection_rules( def _get_scan_documents_thread_func( ctx: typer.Context, is_git_diff: bool, is_commit_range: bool, scan_parameters: dict -) -> Callable[[List[Document]], Tuple[str, CliError, LocalScanResult]]: +) -> Callable[[list[Document]], tuple[str, CliError, LocalScanResult]]: cycode_client = ctx.obj['client'] scan_type = ctx.obj['scan_type'] severity_threshold = ctx.obj['severity_threshold'] sync_option = ctx.obj['sync'] command_scan_type = ctx.info_name - def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, LocalScanResult]: + def _scan_batch_thread_func(batch: list[Document]) -> tuple[str, CliError, LocalScanResult]: local_scan_result = error = error_message = None detections_count = relevant_detections_count = zip_file_size = 0 @@ -296,7 +298,7 @@ def scan_commit_range( def scan_documents( ctx: typer.Context, - documents_to_scan: List[Document], + documents_to_scan: list[Document], scan_parameters: dict, is_git_diff: bool = False, is_commit_range: bool = False, @@ -334,13 +336,12 @@ def scan_documents( def scan_commit_range_documents( ctx: typer.Context, - from_documents_to_scan: List[Document], - to_documents_to_scan: List[Document], + from_documents_to_scan: list[Document], + to_documents_to_scan: list[Document], scan_parameters: Optional[dict] = None, timeout: Optional[int] = None, ) -> None: - """Used by SCA only""" - + """In use by SCA only.""" cycode_client = ctx.obj['client'] scan_type = ctx.obj['scan_type'] severity_threshold = ctx.obj['severity_threshold'] @@ -423,13 +424,13 @@ def scan_commit_range_documents( ) -def should_scan_documents(from_documents_to_scan: List[Document], to_documents_to_scan: List[Document]) -> bool: +def should_scan_documents(from_documents_to_scan: list[Document], to_documents_to_scan: list[Document]) -> bool: return len(from_documents_to_scan) > 0 or len(to_documents_to_scan) > 0 def create_local_scan_result( scan_result: ZippedFileScanResult, - documents_to_scan: List[Document], + documents_to_scan: list[Document], command_scan_type: str, scan_type: str, severity_threshold: str, @@ -567,15 +568,15 @@ def print_debug_scan_details(scan_details_response: 'ScanDetailsResponse') -> No def print_results( - ctx: typer.Context, local_scan_results: List[LocalScanResult], errors: Optional[Dict[str, 'CliError']] = None + ctx: typer.Context, local_scan_results: list[LocalScanResult], errors: Optional[dict[str, 'CliError']] = None ) -> None: printer = ctx.obj.get('console_printer') printer.print_scan_results(local_scan_results, errors) def get_document_detections( - scan_result: ZippedFileScanResult, documents_to_scan: List[Document] -) -> List[DocumentDetections]: + scan_result: ZippedFileScanResult, documents_to_scan: list[Document] +) -> list[DocumentDetections]: logger.debug('Getting document detections') document_detections = [] @@ -594,11 +595,11 @@ def get_document_detections( def exclude_irrelevant_document_detections( - document_detections_list: List[DocumentDetections], + document_detections_list: list[DocumentDetections], scan_type: str, command_scan_type: str, severity_threshold: str, -) -> List[DocumentDetections]: +) -> list[DocumentDetections]: relevant_document_detections_list = [] for document_detections in document_detections_list: relevant_detections = exclude_irrelevant_detections( @@ -613,8 +614,7 @@ def exclude_irrelevant_document_detections( def parse_pre_receive_input() -> str: - """ - Parsing input to pushed branch update details + """Parse input to pushed branch update details. Example input: old_value new_value refname @@ -623,7 +623,7 @@ def parse_pre_receive_input() -> str: 973a96d3e925b65941f7c47fa16129f1577d499f 0000000000000000000000000000000000000000 refs/heads/feature-branch 59564ef68745bca38c42fc57a7822efd519a6bd9 3378e52dcfa47fb11ce3a4a520bea5f85d5d0bf3 refs/heads/develop - :return: first branch update details (input's first line) + :return: First branch update details (input's first line) """ # FIXME(MarshalX): this blocks main thread forever if called outside of pre-receive hook pre_receive_input = sys.stdin.read().strip() @@ -648,7 +648,7 @@ def _get_default_scan_parameters(ctx: typer.Context) -> dict: } -def get_scan_parameters(ctx: typer.Context, paths: Optional[Tuple[str]] = None) -> dict: +def get_scan_parameters(ctx: typer.Context, paths: Optional[tuple[str, ...]] = None) -> dict: scan_parameters = _get_default_scan_parameters(ctx) if not paths: @@ -661,6 +661,9 @@ def get_scan_parameters(ctx: typer.Context, paths: Optional[Tuple[str]] = None) return scan_parameters remote_url = try_get_git_remote_url(paths[0]) + if not remote_url: + remote_url = try_to_get_plastic_remote_url(paths[0]) + if remote_url: # TODO(MarshalX): remove hardcode in context ctx.obj['remote_url'] = remote_url @@ -679,20 +682,103 @@ def try_get_git_remote_url(path: str) -> Optional[str]: return None +def _get_plastic_repository_name(path: str) -> Optional[str]: + """Get the name of the Plastic repository from the current working directory. + + The command to execute is: + cm status --header --machinereadable --fieldseparator=":::" + + Example of status header in machine-readable format: + STATUS:::0:::Project/RepoName:::OrgName@ServerInfo + """ + try: + command = [ + 'cm', + 'status', + '--header', + '--machinereadable', + f'--fieldseparator={consts.PLASTIC_VCS_DATA_SEPARATOR}', + ] + + status = shell(command=command, timeout=consts.PLASTIC_VSC_CLI_TIMEOUT, working_directory=path) + if not status: + logger.debug('Failed to get Plastic repository name (command failed)') + return None + + status_parts = status.split(consts.PLASTIC_VCS_DATA_SEPARATOR) + if len(status_parts) < 2: + logger.debug('Failed to parse Plastic repository name (command returned unexpected format)') + return None + + return status_parts[2].strip() + except Exception as e: + logger.debug('Failed to get Plastic repository name', exc_info=e) + return None + + +def _get_plastic_repository_list(working_dir: Optional[str] = None) -> dict[str, str]: + """Get the list of Plastic repositories and their GUIDs. + + The command to execute is: + cm repo list --format="{repname}:::{repguid}" + + Example line with data: + Project/RepoName:::tapo1zqt-wn99-4752-h61m-7d9k79d40r4v + + Each line represents an individual repository. + """ + repo_name_to_guid = {} + + try: + command = ['cm', 'repo', 'ls', f'--format={{repname}}{consts.PLASTIC_VCS_DATA_SEPARATOR}{{repguid}}'] + + status = shell(command=command, timeout=consts.PLASTIC_VSC_CLI_TIMEOUT, working_directory=working_dir) + if not status: + logger.debug('Failed to get Plastic repository list (command failed)') + return repo_name_to_guid + + status_lines = status.splitlines() + for line in status_lines: + data_parts = line.split(consts.PLASTIC_VCS_DATA_SEPARATOR) + if len(data_parts) < 2: + logger.debug('Failed to parse Plastic repository list line (unexpected format), %s', {'line': line}) + continue + + repo_name, repo_guid = data_parts + repo_name_to_guid[repo_name.strip()] = repo_guid.strip() + + return repo_name_to_guid + except Exception as e: + logger.debug('Failed to get Plastic repository list', exc_info=e) + return repo_name_to_guid + + +def try_to_get_plastic_remote_url(path: str) -> Optional[str]: + repository_name = _get_plastic_repository_name(path) + if not repository_name: + return None + + repository_map = _get_plastic_repository_list(path) + if repository_name not in repository_map: + logger.debug('Failed to get Plastic repository GUID (repository not found in the list)') + return None + + repository_guid = repository_map[repository_name] + return f'{consts.PLASTIC_VCS_REMOTE_URI_PREFIX}{repository_guid}' + + def exclude_irrelevant_detections( - detections: List[Detection], scan_type: str, command_scan_type: str, severity_threshold: str -) -> List[Detection]: + detections: list[Detection], scan_type: str, command_scan_type: str, severity_threshold: str +) -> list[Detection]: relevant_detections = _exclude_detections_by_exclusions_configuration(detections, scan_type) relevant_detections = _exclude_detections_by_scan_type(relevant_detections, scan_type, command_scan_type) return _exclude_detections_by_severity(relevant_detections, severity_threshold) -def _exclude_detections_by_severity(detections: List[Detection], severity_threshold: str) -> List[Detection]: +def _exclude_detections_by_severity(detections: list[Detection], severity_threshold: str) -> list[Detection]: relevant_detections = [] for detection in detections: - severity = detection.detection_details.get('advisory_severity') - if not severity: - severity = detection.severity + severity = detection.severity if _does_severity_match_severity_threshold(severity, severity_threshold): relevant_detections.append(detection) @@ -706,8 +792,8 @@ def _exclude_detections_by_severity(detections: List[Detection], severity_thresh def _exclude_detections_by_scan_type( - detections: List[Detection], scan_type: str, command_scan_type: str -) -> List[Detection]: + detections: list[Detection], scan_type: str, command_scan_type: str +) -> list[Detection]: if command_scan_type == consts.PRE_COMMIT_COMMAND_SCAN_TYPE: return exclude_detections_in_deleted_lines(detections) @@ -722,16 +808,16 @@ def _exclude_detections_by_scan_type( return detections -def exclude_detections_in_deleted_lines(detections: List[Detection]) -> List[Detection]: +def exclude_detections_in_deleted_lines(detections: list[Detection]) -> list[Detection]: return [detection for detection in detections if detection.detection_details.get('line_type') != 'Removed'] -def _exclude_detections_by_exclusions_configuration(detections: List[Detection], scan_type: str) -> List[Detection]: +def _exclude_detections_by_exclusions_configuration(detections: list[Detection], scan_type: str) -> list[Detection]: exclusions = configuration_manager.get_exclusions_by_scan_type(scan_type) return [detection for detection in detections if not _should_exclude_detection(detection, exclusions)] -def _should_exclude_detection(detection: Detection, exclusions: Dict) -> bool: +def _should_exclude_detection(detection: Detection, exclusions: dict) -> bool: # FIXME(MarshalX): what the difference between by_value and by_sha? exclusions_by_value = exclusions.get(consts.EXCLUSIONS_BY_VALUE_SECTION_NAME, []) if _is_detection_sha_configured_in_exclusions(detection, exclusions_by_value): @@ -773,7 +859,7 @@ def _should_exclude_detection(detection: Detection, exclusions: Dict) -> bool: return False -def _is_detection_sha_configured_in_exclusions(detection: Detection, exclusions: List[str]) -> bool: +def _is_detection_sha_configured_in_exclusions(detection: Detection, exclusions: list[str]) -> bool: detection_sha = detection.detection_details.get('sha512') return detection_sha in exclusions @@ -797,7 +883,7 @@ def _get_cve_identifier(detection: Detection) -> Optional[str]: def _get_document_by_file_name( - documents: List[Document], file_name: str, unique_id: Optional[str] = None + documents: list[Document], file_name: str, unique_id: Optional[str] = None ) -> Optional[Document]: for document in documents: if _normalize_file_path(document.path) == _normalize_file_path(file_name) and document.unique_id == unique_id: @@ -903,10 +989,11 @@ def _try_get_aggregation_report_url_if_needed( logger.debug('Failed to get aggregation report url: %s', str(e)) -def _map_detections_per_file_and_commit_id(scan_type: str, raw_detections: List[dict]) -> List[DetectionsPerFile]: - """Converts list of detections (async flow) to list of DetectionsPerFile objects (sync flow). +def _map_detections_per_file_and_commit_id(scan_type: str, raw_detections: list[dict]) -> list[DetectionsPerFile]: + """Convert a list of detections (async flow) to list of DetectionsPerFile objects (sync flow). Args: + scan_type: Type of the scan. raw_detections: List of detections as is returned from the server. Note: @@ -915,6 +1002,7 @@ def _map_detections_per_file_and_commit_id(scan_type: str, raw_detections: List[ Note: Aggregation is performed by file name and commit ID (if available) + """ detections_per_files = {} for raw_detection in raw_detections: @@ -956,7 +1044,7 @@ def _get_secret_file_name_from_detection(raw_detection: dict) -> str: return os.path.join(file_path, file_name) -def _does_reach_to_max_commits_to_scan_limit(commit_ids: List[str], max_commits_count: Optional[int]) -> bool: +def _does_reach_to_max_commits_to_scan_limit(commit_ids: list[str], max_commits_count: Optional[int]) -> bool: if max_commits_count is None: return False diff --git a/cycode/cli/apps/scan/commit_history/commit_history_command.py b/cycode/cli/apps/scan/commit_history/commit_history_command.py index f7992a92..fc1ef23f 100644 --- a/cycode/cli/apps/scan/commit_history/commit_history_command.py +++ b/cycode/cli/apps/scan/commit_history/commit_history_command.py @@ -12,14 +12,14 @@ def commit_history_command( ctx: typer.Context, path: Annotated[ - Path, typer.Argument(exists=True, resolve_path=True, help='Path to git repository to scan', show_default=False) + Path, typer.Argument(exists=True, resolve_path=True, help='Path to Git repository to scan', show_default=False) ], commit_range: Annotated[ str, typer.Option( - '--commit_range', + '--commit-range', '-r', - help='Scan a commit range in this git repository (example: HEAD~1)', + help='Scan a commit range in this Git repository (example: HEAD~1)', show_default='cycode scans all commit history', ), ] = '--all', diff --git a/cycode/cli/apps/scan/path/path_command.py b/cycode/cli/apps/scan/path/path_command.py index 48db40ac..3ee87350 100644 --- a/cycode/cli/apps/scan/path/path_command.py +++ b/cycode/cli/apps/scan/path/path_command.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Annotated, List +from typing import Annotated import typer @@ -11,7 +11,7 @@ def path_command( ctx: typer.Context, paths: Annotated[ - List[Path], typer.Argument(exists=True, resolve_path=True, help='Paths to scan', show_default=False) + list[Path], typer.Argument(exists=True, resolve_path=True, help='Paths to scan', show_default=False) ], ) -> None: add_breadcrumb('path') diff --git a/cycode/cli/apps/scan/pre_commit/pre_commit_command.py b/cycode/cli/apps/scan/pre_commit/pre_commit_command.py index 8e528d15..b919d659 100644 --- a/cycode/cli/apps/scan/pre_commit/pre_commit_command.py +++ b/cycode/cli/apps/scan/pre_commit/pre_commit_command.py @@ -1,5 +1,5 @@ import os -from typing import Annotated, List, Optional +from typing import Annotated, Optional import typer @@ -21,7 +21,7 @@ def pre_commit_command( ctx: typer.Context, - _: Annotated[Optional[List[str]], typer.Argument(help='Ignored arguments', hidden=True)] = None, + _: Annotated[Optional[list[str]], typer.Argument(help='Ignored arguments', hidden=True)] = None, ) -> None: add_breadcrumb('pre_commit') diff --git a/cycode/cli/apps/scan/pre_receive/pre_receive_command.py b/cycode/cli/apps/scan/pre_receive/pre_receive_command.py index 01242b24..eb4f1420 100644 --- a/cycode/cli/apps/scan/pre_receive/pre_receive_command.py +++ b/cycode/cli/apps/scan/pre_receive/pre_receive_command.py @@ -1,5 +1,5 @@ import os -from typing import Annotated, List, Optional +from typing import Annotated, Optional import click import typer @@ -25,7 +25,7 @@ def pre_receive_command( ctx: typer.Context, - _: Annotated[Optional[List[str]], typer.Argument(help='Ignored arguments', hidden=True)] = None, + _: Annotated[Optional[list[str]], typer.Argument(help='Ignored arguments', hidden=True)] = None, ) -> None: try: add_breadcrumb('pre_receive') diff --git a/cycode/cli/apps/scan/repository/repository_command.py b/cycode/cli/apps/scan/repository/repository_command.py index a99cc2d1..16ad8611 100644 --- a/cycode/cli/apps/scan/repository/repository_command.py +++ b/cycode/cli/apps/scan/repository/repository_command.py @@ -21,7 +21,7 @@ def repository_command( ctx: typer.Context, path: Annotated[ - Path, typer.Argument(exists=True, resolve_path=True, help='Path to git repository to scan.', show_default=False) + Path, typer.Argument(exists=True, resolve_path=True, help='Path to Git repository to scan.', show_default=False) ], branch: Annotated[ Optional[str], typer.Option('--branch', '-b', help='Branch to scan.', show_default='default branch') diff --git a/cycode/cli/apps/scan/scan_command.py b/cycode/cli/apps/scan/scan_command.py index 84485c0b..38e4a610 100644 --- a/cycode/cli/apps/scan/scan_command.py +++ b/cycode/cli/apps/scan/scan_command.py @@ -1,4 +1,4 @@ -from typing import Annotated, List, Optional +from typing import Annotated, Optional import click import typer @@ -67,7 +67,7 @@ def scan_command( ), ] = False, sca_scan: Annotated[ - List[ScaScanTypeOption], + list[ScaScanTypeOption], typer.Option( help='Specify the type of SCA scan you wish to execute.', rich_help_panel=_SCA_RICH_HELP_PANEL, @@ -85,7 +85,7 @@ def scan_command( bool, typer.Option( '--no-restore', - help='When specified, Cycode will not run restore command. ' 'Will scan direct dependencies [b]only[/]!', + help='When specified, Cycode will not run restore command. Will scan direct dependencies [b]only[/]!', rich_help_panel=_SCA_RICH_HELP_PANEL, ), ] = False, @@ -99,9 +99,20 @@ def scan_command( ), ] = False, ) -> None: - """:magnifying_glass_tilted_right: Scan the content for Secrets, IaC, SCA, and SAST violations. - You'll need to specify which scan type to perform: - [cyan]path[/]/[cyan]repository[/]/[cyan]commit_history[/].""" + """:mag: [bold cyan]Scan code for vulnerabilities (Secrets, IaC, SCA, SAST).[/] + + This command scans your code for various types of security issues, including: + * [yellow]Secrets:[/] Hardcoded credentials and sensitive information. + * [dodger_blue1]Infrastructure as Code (IaC):[/] Misconfigurations in Terraform, CloudFormation, etc. + * [green]Software Composition Analysis (SCA):[/] Vulnerabilities and license issues in dependencies. + * [magenta]Static Application Security Testing (SAST):[/] Code quality and security flaws. + + Example usage: + * `cycode scan path `: Scan a specific local directory or file. + * `cycode scan repository `: Scan Git related files in a local Git repository. + * `cycode scan commit-history `: Scan the commit history of a local Git repository. + + """ add_breadcrumb('scan') ctx.obj['show_secret'] = show_secret @@ -118,7 +129,7 @@ def scan_command( _sca_scan_to_context(ctx, sca_scan) -def _sca_scan_to_context(ctx: typer.Context, sca_scan_user_selected: List[str]) -> None: +def _sca_scan_to_context(ctx: typer.Context, sca_scan_user_selected: list[str]) -> None: for sca_scan_option_selected in sca_scan_user_selected: ctx.obj[sca_scan_option_selected] = True diff --git a/cycode/cli/apps/status/get_cli_status.py b/cycode/cli/apps/status/get_cli_status.py index 4a3dc4b0..0a272c57 100644 --- a/cycode/cli/apps/status/get_cli_status.py +++ b/cycode/cli/apps/status/get_cli_status.py @@ -1,4 +1,5 @@ import platform +from typing import TYPE_CHECKING from cycode import __version__ from cycode.cli.apps.auth.auth_common import get_authorization_info @@ -8,11 +9,14 @@ from cycode.cli.user_settings.configuration_manager import ConfigurationManager from cycode.cli.utils.get_api_client import get_scan_cycode_client +if TYPE_CHECKING: + from typer import Context -def get_cli_status() -> CliStatus: + +def get_cli_status(ctx: 'Context') -> CliStatus: configuration_manager = ConfigurationManager() - auth_info = get_authorization_info() + auth_info = get_authorization_info(ctx) is_authenticated = auth_info is not None supported_modules_status = CliSupportedModulesStatus() diff --git a/cycode/cli/apps/status/models.py b/cycode/cli/apps/status/models.py index 50182ecd..82b9751a 100644 --- a/cycode/cli/apps/status/models.py +++ b/cycode/cli/apps/status/models.py @@ -1,10 +1,9 @@ import json from dataclasses import asdict, dataclass -from typing import Dict class CliStatusBase: - def as_dict(self) -> Dict[str, any]: + def as_dict(self) -> dict[str, any]: return asdict(self) def _get_text_message_part(self, key: str, value: any, intent: int = 0) -> str: diff --git a/cycode/cli/apps/status/status_command.py b/cycode/cli/apps/status/status_command.py index 28f8cfba..4654ef20 100644 --- a/cycode/cli/apps/status/status_command.py +++ b/cycode/cli/apps/status/status_command.py @@ -6,9 +6,25 @@ def status_command(ctx: typer.Context) -> None: + """:information_source: [bold cyan]Show Cycode CLI status and configuration.[/] + + This command displays the current status and configuration of the Cycode CLI, including: + * Authentication status: Whether you're logged in + * Version information: Current CLI version + * Configuration: Current API endpoints and settings + * System information: Operating system and environment details + + Output formats: + * Text: Human-readable format (default) + * JSON: Machine-readable format + + Example usage: + * `cycode status`: Show status in text format + * `cycode -o json status`: Show status in JSON format + """ output = ctx.obj['output'] - cli_status = get_cli_status() + cli_status = get_cli_status(ctx) if output == OutputTypeOption.JSON: console.print_json(cli_status.as_json()) else: diff --git a/cycode/cli/config.py b/cycode/cli/config.py index a1ddbbaf..73491546 100644 --- a/cycode/cli/config.py +++ b/cycode/cli/config.py @@ -4,4 +4,4 @@ # env vars CYCODE_CLIENT_ID_ENV_VAR_NAME = 'CYCODE_CLIENT_ID' -CYCODE_CLIENT_SECRET_ENV_VAR_NAME = 'CYCODE_CLIENT_SECRET' # noqa: S105 +CYCODE_CLIENT_SECRET_ENV_VAR_NAME = 'CYCODE_CLIENT_SECRET' diff --git a/cycode/cli/consts.py b/cycode/cli/consts.py index 9d7a619d..286f1f95 100644 --- a/cycode/cli/consts.py +++ b/cycode/cli/consts.py @@ -9,7 +9,7 @@ COMMIT_HISTORY_COMMAND_SCAN_TYPE = 'commit-history' COMMIT_HISTORY_COMMAND_SCAN_TYPE_OLD = 'commit_history' -SECRET_SCAN_TYPE = 'secret' # noqa: S105 +SECRET_SCAN_TYPE = 'secret' IAC_SCAN_TYPE = 'iac' SCA_SCAN_TYPE = 'sca' SAST_SCAN_TYPE = 'sast' @@ -231,3 +231,11 @@ # Example: A -> B -> C # Result: A -> ... -> C SCA_SHORTCUT_DEPENDENCY_PATHS = 2 + +SCA_SKIP_RESTORE_DEPENDENCIES_FLAG = 'no-restore' + +SCA_GRADLE_ALL_SUB_PROJECTS_FLAG = 'gradle-all-sub-projects' + +PLASTIC_VCS_DATA_SEPARATOR = ':::' +PLASTIC_VSC_CLI_TIMEOUT = 10 +PLASTIC_VCS_REMOTE_URI_PREFIX = 'plastic::' diff --git a/cycode/cli/exceptions/custom_exceptions.py b/cycode/cli/exceptions/custom_exceptions.py index 4d692812..59c0f693 100644 --- a/cycode/cli/exceptions/custom_exceptions.py +++ b/cycode/cli/exceptions/custom_exceptions.py @@ -4,7 +4,7 @@ class CycodeError(Exception): - """Base class for all custom exceptions""" + """Base class for all custom exceptions.""" def __str__(self) -> str: class_name = self.__class__.__name__ @@ -14,7 +14,7 @@ def __str__(self) -> str: class RequestError(CycodeError): ... -class RequestTimeout(RequestError): ... +class RequestTimeoutError(RequestError): ... class RequestConnectionError(RequestError): ... @@ -91,7 +91,7 @@ def __str__(self) -> str: code='cycode_error', message='Cycode was unable to complete this scan. Please try again by executing the `cycode scan` command', ), - RequestTimeout: CliError( + RequestTimeoutError: CliError( soft_fail=True, code='timeout_error', message='The request timed out. Please try again by executing the `cycode scan` command', diff --git a/cycode/cli/exceptions/handle_scan_errors.py b/cycode/cli/exceptions/handle_scan_errors.py index 09890247..229e0f02 100644 --- a/cycode/cli/exceptions/handle_scan_errors.py +++ b/cycode/cli/exceptions/handle_scan_errors.py @@ -17,8 +17,7 @@ def handle_scan_exception(ctx: typer.Context, err: Exception, *, return_exceptio custom_exceptions.ScanAsyncError: CliError( soft_fail=True, code='scan_error', - message='Cycode was unable to complete this scan. ' - 'Please try again by executing the `cycode scan` command', + message='Cycode was unable to complete this scan. Please try again by executing the `cycode scan` command', ), custom_exceptions.ZipTooLargeError: CliError( soft_fail=True, @@ -38,7 +37,7 @@ def handle_scan_exception(ctx: typer.Context, err: Exception, *, return_exceptio git_proxy.get_invalid_git_repository_error(): CliError( soft_fail=False, code='invalid_git_error', - message='The path you supplied does not correlate to a git repository. ' + message='The path you supplied does not correlate to a Git repository. ' 'If you still wish to scan this path, use: `cycode scan path `', ), } diff --git a/cycode/cli/files_collector/excluder.py b/cycode/cli/files_collector/excluder.py index f16e9710..9ef5e3d6 100644 --- a/cycode/cli/files_collector/excluder.py +++ b/cycode/cli/files_collector/excluder.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from cycode.cli import consts from cycode.cli.config import configuration_manager @@ -16,8 +16,8 @@ def exclude_irrelevant_files( - progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection', scan_type: str, filenames: List[str] -) -> List[str]: + progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection', scan_type: str, filenames: list[str] +) -> list[str]: relevant_files = [] for filename in filenames: progress_bar.update(progress_bar_section) @@ -29,7 +29,7 @@ def exclude_irrelevant_files( return relevant_files -def exclude_irrelevant_documents_to_scan(scan_type: str, documents_to_scan: List['Document']) -> List['Document']: +def exclude_irrelevant_documents_to_scan(scan_type: str, documents_to_scan: list['Document']) -> list['Document']: logger.debug('Excluding irrelevant documents to scan') relevant_documents = [] diff --git a/cycode/cli/files_collector/iac/tf_content_generator.py b/cycode/cli/files_collector/iac/tf_content_generator.py index 8f4cb4d0..63be9e47 100644 --- a/cycode/cli/files_collector/iac/tf_content_generator.py +++ b/cycode/cli/files_collector/iac/tf_content_generator.py @@ -1,6 +1,5 @@ import json import time -from typing import List from cycode.cli import consts from cycode.cli.exceptions.custom_exceptions import TfplanKeyError @@ -34,7 +33,7 @@ def generate_tf_content_from_tfplan(filename: str, tfplan: str) -> str: return _generate_tf_content(planned_resources) -def _generate_tf_content(resource_changes: List[ResourceChange]) -> str: +def _generate_tf_content(resource_changes: list[ResourceChange]) -> str: tf_content = '' for resource_change in resource_changes: if not any(item in resource_change.actions for item in ACTIONS_TO_OMIT_RESOURCE): @@ -62,9 +61,9 @@ def _get_resource_name(resource_change: ResourceChange) -> str: return '.'.join(valid_parts) -def _extract_resources(tfplan: str, filename: str) -> List[ResourceChange]: +def _extract_resources(tfplan: str, filename: str) -> list[ResourceChange]: tfplan_json = load_json(tfplan) - resources: List[ResourceChange] = [] + resources: list[ResourceChange] = [] try: resource_changes = tfplan_json['resource_changes'] for resource_change in resource_changes: diff --git a/cycode/cli/files_collector/models/in_memory_zip.py b/cycode/cli/files_collector/models/in_memory_zip.py index a0700f6b..8f58b12b 100644 --- a/cycode/cli/files_collector/models/in_memory_zip.py +++ b/cycode/cli/files_collector/models/in_memory_zip.py @@ -10,7 +10,7 @@ from pathlib import Path -class InMemoryZip(object): +class InMemoryZip: def __init__(self) -> None: self.configuration_manager = ConfigurationManager() diff --git a/cycode/cli/files_collector/path_documents.py b/cycode/cli/files_collector/path_documents.py index 469e6ce7..e0f06312 100644 --- a/cycode/cli/files_collector/path_documents.py +++ b/cycode/cli/files_collector/path_documents.py @@ -1,5 +1,5 @@ import os -from typing import TYPE_CHECKING, List, Tuple +from typing import TYPE_CHECKING from cycode.cli.files_collector.excluder import exclude_irrelevant_files from cycode.cli.files_collector.iac.tf_content_generator import ( @@ -17,8 +17,8 @@ from cycode.cli.utils.progress_bar import BaseProgressBar, ProgressBarSection -def _get_all_existing_files_in_directory(path: str, *, walk_with_ignore_patterns: bool = True) -> List[str]: - files: List[str] = [] +def _get_all_existing_files_in_directory(path: str, *, walk_with_ignore_patterns: bool = True) -> list[str]: + files: list[str] = [] walk_func = walk_ignore if walk_with_ignore_patterns else os.walk for root, _, filenames in walk_func(path): @@ -28,7 +28,7 @@ def _get_all_existing_files_in_directory(path: str, *, walk_with_ignore_patterns return files -def _get_relevant_files_in_path(path: str) -> List[str]: +def _get_relevant_files_in_path(path: str) -> list[str]: absolute_path = get_absolute_path(path) if not os.path.isfile(absolute_path) and not os.path.isdir(absolute_path): @@ -42,8 +42,8 @@ def _get_relevant_files_in_path(path: str) -> List[str]: def _get_relevant_files( - progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection', scan_type: str, paths: Tuple[str, ...] -) -> List[str]: + progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection', scan_type: str, paths: tuple[str, ...] +) -> list[str]: all_files_to_scan = [] for path in paths: all_files_to_scan.extend(_get_relevant_files_in_path(path)) @@ -89,13 +89,13 @@ def get_relevant_documents( progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection', scan_type: str, - paths: Tuple[str, ...], + paths: tuple[str, ...], *, is_git_diff: bool = False, -) -> List[Document]: +) -> list[Document]: relevant_files = _get_relevant_files(progress_bar, progress_bar_section, scan_type, paths) - documents: List[Document] = [] + documents: list[Document] = [] for file in relevant_files: progress_bar.update(progress_bar_section) diff --git a/cycode/cli/files_collector/repository_documents.py b/cycode/cli/files_collector/repository_documents.py index df49aa95..b524ca4c 100644 --- a/cycode/cli/files_collector/repository_documents.py +++ b/cycode/cli/files_collector/repository_documents.py @@ -1,5 +1,6 @@ import os -from typing import TYPE_CHECKING, Iterator, List, Optional, Tuple, Union +from collections.abc import Iterator +from typing import TYPE_CHECKING, Optional, Union from cycode.cli import consts from cycode.cli.files_collector.sca import sca_code_scanner @@ -25,7 +26,7 @@ def get_git_repository_tree_file_entries( return git_proxy.get_repo(path).tree(branch).traverse(predicate=should_process_git_object) -def parse_commit_range(commit_range: str, path: str) -> Tuple[str, str]: +def parse_commit_range(commit_range: str, path: str) -> tuple[str, str]: from_commit_rev = None to_commit_rev = None @@ -47,7 +48,7 @@ def get_diff_file_content(file: 'Diff') -> str: def get_pre_commit_modified_documents( progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection' -) -> Tuple[List[Document], List[Document]]: +) -> tuple[list[Document], list[Document]]: git_head_documents = [] pre_committed_documents = [] @@ -77,7 +78,7 @@ def get_commit_range_modified_documents( path: str, from_commit_rev: str, to_commit_rev: str, -) -> Tuple[List[Document], List[Document]]: +) -> tuple[list[Document], list[Document]]: from_commit_documents = [] to_commit_documents = [] diff --git a/cycode/cli/files_collector/sca/base_restore_dependencies.py b/cycode/cli/files_collector/sca/base_restore_dependencies.py index 2e6c0993..c4364c05 100644 --- a/cycode/cli/files_collector/sca/base_restore_dependencies.py +++ b/cycode/cli/files_collector/sca/base_restore_dependencies.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import List, Optional +from typing import Optional import typer @@ -14,7 +14,7 @@ def build_dep_tree_path(path: str, generated_file_name: str) -> str: def execute_commands( - commands: List[List[str]], + commands: list[list[str]], file_name: str, command_timeout: int, dependencies_file_name: Optional[str] = None, @@ -91,7 +91,7 @@ def is_project(self, document: Document) -> bool: pass @abstractmethod - def get_commands(self, manifest_file_path: str) -> List[List[str]]: + def get_commands(self, manifest_file_path: str) -> list[list[str]]: pass @abstractmethod diff --git a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py index 5d56644a..4f469896 100644 --- a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py +++ b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py @@ -1,5 +1,5 @@ import os -from typing import List, Optional +from typing import Optional import typer @@ -34,7 +34,7 @@ def try_restore_dependencies(self, document: Document) -> Optional[Document]: def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in GO_PROJECT_FILE_EXTENSIONS) - def get_commands(self, manifest_file_path: str) -> List[List[str]]: + def get_commands(self, manifest_file_path: str) -> list[list[str]]: return [ ['go', 'list', '-m', '-json', 'all'], ['echo', '------------------------------------------------------'], diff --git a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py index 3995da90..89595e0e 100644 --- a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py @@ -1,6 +1,6 @@ import os import re -from typing import List, Optional, Set +from typing import Optional import typer @@ -19,7 +19,7 @@ class RestoreGradleDependencies(BaseRestoreDependencies): def __init__( - self, ctx: typer.Context, is_git_diff: bool, command_timeout: int, projects: Optional[Set[str]] = None + self, ctx: typer.Context, is_git_diff: bool, command_timeout: int, projects: Optional[set[str]] = None ) -> None: super().__init__(ctx, is_git_diff, command_timeout, create_output_file_manually=True) if projects is None: @@ -32,7 +32,7 @@ def is_gradle_sub_projects(self) -> bool: def is_project(self, document: Document) -> bool: return document.path.endswith(BUILD_GRADLE_FILE_NAME) or document.path.endswith(BUILD_GRADLE_KTS_FILE_NAME) - def get_commands(self, manifest_file_path: str) -> List[List[str]]: + def get_commands(self, manifest_file_path: str) -> list[list[str]]: return ( self.get_commands_for_sub_projects(manifest_file_path) if self.is_gradle_sub_projects() @@ -48,7 +48,7 @@ def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: def get_working_directory(self, document: Document) -> Optional[str]: return get_path_from_context(self.ctx) if self.is_gradle_sub_projects() else None - def get_all_projects(self) -> Set[str]: + def get_all_projects(self) -> set[str]: projects_output = shell( command=BUILD_GRADLE_ALL_PROJECTS_COMMAND, timeout=BUILD_GRADLE_ALL_PROJECTS_TIMEOUT, @@ -59,7 +59,7 @@ def get_all_projects(self) -> Set[str]: return set(projects) - def get_commands_for_sub_projects(self, manifest_file_path: str) -> List[List[str]]: + def get_commands_for_sub_projects(self, manifest_file_path: str) -> list[list[str]]: project_name = os.path.basename(os.path.dirname(manifest_file_path)) project_name = f':{project_name}' return ( diff --git a/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py index d90bbe71..1c3d860c 100644 --- a/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py @@ -1,6 +1,6 @@ import os from os import path -from typing import List, Optional +from typing import Optional import typer @@ -24,7 +24,7 @@ def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) def is_project(self, document: Document) -> bool: return path.basename(document.path).split('/')[-1] == BUILD_MAVEN_FILE_NAME - def get_commands(self, manifest_file_path: str) -> List[List[str]]: + def get_commands(self, manifest_file_path: str) -> list[list[str]]: return [['mvn', 'org.cyclonedx:cyclonedx-maven-plugin:2.7.4:makeAggregateBom', '-f', manifest_file_path]] def get_lock_file_name(self) -> str: @@ -64,7 +64,7 @@ def restore_from_secondary_command( return restore_dependencies -def create_secondary_restore_command(manifest_file_path: str) -> List[str]: +def create_secondary_restore_command(manifest_file_path: str) -> list[str]: return [ 'mvn', 'dependency:tree', diff --git a/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py b/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py index 672ee0db..ed8e36c2 100644 --- a/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py +++ b/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py @@ -1,5 +1,4 @@ import os -from typing import List import typer @@ -18,7 +17,7 @@ def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in NPM_PROJECT_FILE_EXTENSIONS) - def get_commands(self, manifest_file_path: str) -> List[List[str]]: + def get_commands(self, manifest_file_path: str) -> list[list[str]]: return [ [ 'npm', diff --git a/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py b/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py index b4f5a248..3bd6627f 100644 --- a/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py +++ b/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py @@ -1,5 +1,4 @@ import os -from typing import List import typer @@ -17,7 +16,7 @@ def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in NUGET_PROJECT_FILE_EXTENSIONS) - def get_commands(self, manifest_file_path: str) -> List[List[str]]: + def get_commands(self, manifest_file_path: str) -> list[list[str]]: return [['dotnet', 'restore', manifest_file_path, '--use-lock-file', '--verbosity', 'quiet']] def get_lock_file_name(self) -> str: diff --git a/cycode/cli/files_collector/sca/ruby/restore_ruby_dependencies.py b/cycode/cli/files_collector/sca/ruby/restore_ruby_dependencies.py index 3dfc4a16..4571b1c5 100644 --- a/cycode/cli/files_collector/sca/ruby/restore_ruby_dependencies.py +++ b/cycode/cli/files_collector/sca/ruby/restore_ruby_dependencies.py @@ -1,5 +1,5 @@ import os -from typing import List, Optional +from typing import Optional from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.models import Document @@ -12,7 +12,7 @@ class RestoreRubyDependencies(BaseRestoreDependencies): def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in RUBY_PROJECT_FILE_EXTENSIONS) - def get_commands(self, manifest_file_path: str) -> List[List[str]]: + def get_commands(self, manifest_file_path: str) -> list[list[str]]: return [['bundle', '--quiet']] def get_lock_file_name(self) -> str: diff --git a/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py b/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py index b8e1c41b..d7eeba3b 100644 --- a/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py +++ b/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py @@ -1,5 +1,5 @@ import os -from typing import List, Optional +from typing import Optional from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.models import Document @@ -12,7 +12,7 @@ class RestoreSbtDependencies(BaseRestoreDependencies): def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in SBT_PROJECT_FILE_EXTENSIONS) - def get_commands(self, manifest_file_path: str) -> List[List[str]]: + def get_commands(self, manifest_file_path: str) -> list[list[str]]: return [['sbt', 'dependencyLockWrite', '--verbose']] def get_lock_file_name(self) -> str: diff --git a/cycode/cli/files_collector/sca/sca_code_scanner.py b/cycode/cli/files_collector/sca/sca_code_scanner.py index 88626c9c..e6ec0e9d 100644 --- a/cycode/cli/files_collector/sca/sca_code_scanner.py +++ b/cycode/cli/files_collector/sca/sca_code_scanner.py @@ -1,5 +1,5 @@ import os -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING, Optional import typer @@ -28,9 +28,9 @@ def perform_pre_commit_range_scan_actions( path: str, - from_commit_documents: List[Document], + from_commit_documents: list[Document], from_commit_rev: str, - to_commit_documents: List[Document], + to_commit_documents: list[Document], to_commit_rev: str, ) -> None: repo = git_proxy.get_repo(path) @@ -39,7 +39,7 @@ def perform_pre_commit_range_scan_actions( def perform_pre_hook_range_scan_actions( - git_head_documents: List[Document], pre_committed_documents: List[Document] + git_head_documents: list[Document], pre_committed_documents: list[Document] ) -> None: repo = git_proxy.get_repo(os.getcwd()) add_ecosystem_related_files_if_exists(git_head_documents, repo, consts.GIT_HEAD_COMMIT_REV) @@ -47,9 +47,9 @@ def perform_pre_hook_range_scan_actions( def add_ecosystem_related_files_if_exists( - documents: List[Document], repo: Optional['Repo'] = None, commit_rev: Optional[str] = None + documents: list[Document], repo: Optional['Repo'] = None, commit_rev: Optional[str] = None ) -> None: - documents_to_add: List[Document] = [] + documents_to_add: list[Document] = [] for doc in documents: ecosystem = get_project_file_ecosystem(doc) if ecosystem is None: @@ -62,9 +62,9 @@ def add_ecosystem_related_files_if_exists( def get_doc_ecosystem_related_project_files( - doc: Document, documents: List[Document], ecosystem: str, commit_rev: Optional[str], repo: Optional['Repo'] -) -> List[Document]: - documents_to_add: List[Document] = [] + doc: Document, documents: list[Document], ecosystem: str, commit_rev: Optional[str], repo: Optional['Repo'] +) -> list[Document]: + documents_to_add: list[Document] = [] for ecosystem_project_file in consts.PROJECT_FILES_BY_ECOSYSTEM_MAP.get(ecosystem): file_to_search = join_paths(get_file_dir(doc.path), ecosystem_project_file) if not is_project_file_exists_in_documents(documents, file_to_search): @@ -79,7 +79,7 @@ def get_doc_ecosystem_related_project_files( return documents_to_add -def is_project_file_exists_in_documents(documents: List[Document], file: str) -> bool: +def is_project_file_exists_in_documents(documents: list[Document], file: str) -> bool: return any(doc for doc in documents if file == doc.path) @@ -93,7 +93,7 @@ def get_project_file_ecosystem(document: Document) -> Optional[str]: def try_restore_dependencies( ctx: typer.Context, - documents_to_add: Dict[str, Document], + documents_to_add: dict[str, Document], restore_dependencies: 'BaseRestoreDependencies', document: Document, ) -> None: @@ -122,9 +122,9 @@ def try_restore_dependencies( def add_dependencies_tree_document( - ctx: typer.Context, documents_to_scan: List[Document], is_git_diff: bool = False + ctx: typer.Context, documents_to_scan: list[Document], is_git_diff: bool = False ) -> None: - documents_to_add: Dict[str, Document] = {document.path: document for document in documents_to_scan} + documents_to_add: dict[str, Document] = {document.path: document for document in documents_to_scan} restore_dependencies_list = restore_handlers(ctx, is_git_diff) for restore_dependencies in restore_dependencies_list: @@ -135,7 +135,7 @@ def add_dependencies_tree_document( documents_to_scan[:] = list(documents_to_add.values()) -def restore_handlers(ctx: typer.Context, is_git_diff: bool) -> List[BaseRestoreDependencies]: +def restore_handlers(ctx: typer.Context, is_git_diff: bool) -> list[BaseRestoreDependencies]: return [ RestoreGradleDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT), RestoreMavenDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT), @@ -159,7 +159,7 @@ def get_file_content_from_commit(repo: 'Repo', commit: str, file_path: str) -> O def perform_pre_scan_documents_actions( - ctx: typer.Context, scan_type: str, documents_to_scan: List[Document], is_git_diff: bool = False + ctx: typer.Context, scan_type: str, documents_to_scan: list[Document], is_git_diff: bool = False ) -> None: no_restore = ctx.params.get('no-restore', False) if scan_type == consts.SCA_SCAN_TYPE and not no_restore: diff --git a/cycode/cli/files_collector/walk_ignore.py b/cycode/cli/files_collector/walk_ignore.py index 0ba2b93d..35855ff4 100644 --- a/cycode/cli/files_collector/walk_ignore.py +++ b/cycode/cli/files_collector/walk_ignore.py @@ -1,5 +1,5 @@ import os -from typing import Generator, Iterable, List, Tuple +from collections.abc import Generator, Iterable from cycode.cli.logger import logger from cycode.cli.utils.ignore_utils import IgnoreFilterManager @@ -22,7 +22,7 @@ def _walk_to_top(path: str) -> Iterable[str]: yield path # Include the top-level directory -def _collect_top_level_ignore_files(path: str) -> List[str]: +def _collect_top_level_ignore_files(path: str) -> list[str]: ignore_files = [] top_paths = reversed(list(_walk_to_top(path))) # we must reverse it to make top levels more prioritized for dir_path in top_paths: @@ -34,7 +34,7 @@ def _collect_top_level_ignore_files(path: str) -> List[str]: return ignore_files -def walk_ignore(path: str) -> Generator[Tuple[str, List[str], List[str]], None, None]: +def walk_ignore(path: str) -> Generator[tuple[str, list[str], list[str]], None, None]: ignore_filter_manager = IgnoreFilterManager.build( path=path, global_ignore_file_paths=_collect_top_level_ignore_files(path), diff --git a/cycode/cli/files_collector/zip_documents.py b/cycode/cli/files_collector/zip_documents.py index b9a272e1..770121fa 100644 --- a/cycode/cli/files_collector/zip_documents.py +++ b/cycode/cli/files_collector/zip_documents.py @@ -1,6 +1,6 @@ import timeit from pathlib import Path -from typing import List, Optional +from typing import Optional from cycode.cli import consts from cycode.cli.exceptions import custom_exceptions @@ -17,7 +17,7 @@ def _validate_zip_file_size(scan_type: str, zip_file_size: int) -> None: raise custom_exceptions.ZipTooLargeError(max_size_limit) -def zip_documents(scan_type: str, documents: List[Document], zip_file: Optional[InMemoryZip] = None) -> InMemoryZip: +def zip_documents(scan_type: str, documents: list[Document], zip_file: Optional[InMemoryZip] = None) -> InMemoryZip: if zip_file is None: zip_file = InMemoryZip() diff --git a/cycode/cli/models.py b/cycode/cli/models.py index 14058f0c..3c59eeee 100644 --- a/cycode/cli/models.py +++ b/cycode/cli/models.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Dict, List, NamedTuple, Optional, Type +from typing import NamedTuple, Optional from cycode.cyclient.models import Detection @@ -20,16 +20,16 @@ def __init__( self.absolute_path = absolute_path def __repr__(self) -> str: - return 'path:{0}, content:{1}'.format(self.path, self.content) + return f'path:{self.path}, content:{self.content}' class DocumentDetections: - def __init__(self, document: Document, detections: List[Detection]) -> None: + def __init__(self, document: Document, detections: list[Detection]) -> None: self.document = document self.detections = detections def __repr__(self) -> str: - return 'document:{0}, detections:{1}'.format(self.document, self.detections) + return f'document:{self.document}, detections:{self.detections}' class CliError(NamedTuple): @@ -42,19 +42,19 @@ def enrich(self, additional_message: str) -> 'CliError': return CliError(self.code, message, self.soft_fail) -CliErrors = Dict[Type[BaseException], CliError] +CliErrors = dict[type[BaseException], CliError] class CliResult(NamedTuple): success: bool message: str - data: Optional[Dict[str, any]] = None + data: Optional[dict[str, any]] = None class LocalScanResult(NamedTuple): scan_id: str report_url: Optional[str] - document_detections: List[DocumentDetections] + document_detections: list[DocumentDetections] issue_detected: bool detections_count: int relevant_detections_count: int @@ -66,8 +66,8 @@ class ResourceChange: resource_type: str name: str index: Optional[int] - actions: List[str] - values: Dict[str, str] + actions: list[str] + values: dict[str, str] def __repr__(self) -> str: return f'resource_type: {self.resource_type}, name: {self.name}' diff --git a/cycode/cli/printers/console_printer.py b/cycode/cli/printers/console_printer.py index 00eb38cf..f581c894 100644 --- a/cycode/cli/printers/console_printer.py +++ b/cycode/cli/printers/console_printer.py @@ -1,5 +1,5 @@ import io -from typing import TYPE_CHECKING, ClassVar, Dict, List, Optional, Type +from typing import TYPE_CHECKING, ClassVar, Optional import typer from rich.console import Console @@ -21,7 +21,7 @@ class ConsolePrinter: - _AVAILABLE_PRINTERS: ClassVar[Dict[str, Type['PrinterBase']]] = { + _AVAILABLE_PRINTERS: ClassVar[dict[str, type['PrinterBase']]] = { 'rich': RichPrinter, 'text': TextPrinter, 'json': JsonPrinter, @@ -78,8 +78,8 @@ def printer(self) -> 'PrinterBase': def print_scan_results( self, - local_scan_results: List['LocalScanResult'], - errors: Optional[Dict[str, 'CliError']] = None, + local_scan_results: list['LocalScanResult'], + errors: Optional[dict[str, 'CliError']] = None, ) -> None: if self.console_record: self.console_record.print_scan_results(local_scan_results, errors) diff --git a/cycode/cli/printers/json_printer.py b/cycode/cli/printers/json_printer.py index 6ad14e22..acb7912f 100644 --- a/cycode/cli/printers/json_printer.py +++ b/cycode/cli/printers/json_printer.py @@ -1,5 +1,5 @@ import json -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING, Optional from cycode.cli.models import CliError, CliResult from cycode.cli.printers.printer_base import PrinterBase @@ -21,7 +21,7 @@ def print_error(self, error: CliError) -> None: self.console.print_json(self.get_data_json(result)) def print_scan_results( - self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None + self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None ) -> None: scan_ids = [] report_urls = [] @@ -48,7 +48,7 @@ def print_scan_results( self.console.print_json(self._get_json_scan_result(scan_ids, detections_dict, report_urls, inlined_errors)) def _get_json_scan_result( - self, scan_ids: List[str], detections: dict, report_urls: List[str], errors: List[dict] + self, scan_ids: list[str], detections: dict, report_urls: list[str], errors: list[dict] ) -> str: result = { 'scan_ids': scan_ids, diff --git a/cycode/cli/printers/printer_base.py b/cycode/cli/printers/printer_base.py index 23ba7384..527cc31b 100644 --- a/cycode/cli/printers/printer_base.py +++ b/cycode/cli/printers/printer_base.py @@ -1,7 +1,7 @@ import sys from abc import ABC, abstractmethod from collections import defaultdict -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING, Optional import typer @@ -51,7 +51,7 @@ def show_secret(self) -> bool: @abstractmethod def print_scan_results( - self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None + self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None ) -> None: pass @@ -68,6 +68,7 @@ def print_exception(self, e: Optional[BaseException] = None) -> None: Note: Called only when the verbose flag is set. + """ rich_traceback = ( RichTraceback.from_exception(type(e), e, e.__traceback__) @@ -79,7 +80,7 @@ def print_exception(self, e: Optional[BaseException] = None) -> None: self.console_err.print(f'[red]Correlation ID:[/] {get_correlation_id()}') - def print_scan_results_summary(self, local_scan_results: List['LocalScanResult']) -> None: + def print_scan_results_summary(self, local_scan_results: list['LocalScanResult']) -> None: """Print a summary of scan results based on severity levels. Args: @@ -87,8 +88,8 @@ def print_scan_results_summary(self, local_scan_results: List['LocalScanResult'] The summary includes the count of detections for each severity level and is displayed in the console in a formatted string. - """ + """ detections_count = 0 severity_counts = defaultdict(int) for local_scan_result in local_scan_results: diff --git a/cycode/cli/printers/rich_printer.py b/cycode/cli/printers/rich_printer.py index 3401b8f5..b2ed1a2e 100644 --- a/cycode/cli/printers/rich_printer.py +++ b/cycode/cli/printers/rich_printer.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING, Optional from rich.console import Group from rich.panel import Panel @@ -25,7 +25,7 @@ class RichPrinter(TextPrinter): MAX_PATH_LENGTH = 60 def print_scan_results( - self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None + self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None ) -> None: if not errors and all(result.issue_detected == 0 for result in local_scan_results): self.console.print(self.NO_DETECTIONS_MESSAGE) @@ -57,7 +57,7 @@ def _get_details_table(self, detection: 'Detection') -> Table: detection_details = detection.detection_details path = str(get_detection_file_path(self.scan_type, detection)) - shorten_path = f'...{path[-self.MAX_PATH_LENGTH:]}' if len(path) > self.MAX_PATH_LENGTH else path + shorten_path = f'...{path[-self.MAX_PATH_LENGTH :]}' if len(path) > self.MAX_PATH_LENGTH else path details_table.add_row('In file', f'[link=file://{path}]{shorten_path}[/]') if self.scan_type == consts.SECRET_SCAN_TYPE: diff --git a/cycode/cli/printers/tables/sca_table_printer.py b/cycode/cli/printers/tables/sca_table_printer.py index 74ac2832..0bf59a20 100644 --- a/cycode/cli/printers/tables/sca_table_printer.py +++ b/cycode/cli/printers/tables/sca_table_printer.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import TYPE_CHECKING, Dict, List +from typing import TYPE_CHECKING from cycode.cli.cli_types import SeverityOption from cycode.cli.consts import LICENSE_COMPLIANCE_POLICY_ID, PACKAGE_VULNERABILITY_POLICY_ID @@ -30,7 +30,7 @@ class ScaTablePrinter(TablePrinterBase): - def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: + def _print_results(self, local_scan_results: list['LocalScanResult']) -> None: aggregation_report_url = self.ctx.obj.get('aggregation_report_url') detections_per_policy_id = self._extract_detections_per_policy_id(local_scan_results) for policy_id, detections in detections_per_policy_id.items(): @@ -83,14 +83,8 @@ def _get_table(self, policy_id: str) -> Table: def _enrich_table_with_values(policy_id: str, table: Table, detection: Detection) -> None: detection_details = detection.detection_details - severity = None - if policy_id == PACKAGE_VULNERABILITY_POLICY_ID: - severity = detection_details.get('advisory_severity') - elif policy_id == LICENSE_COMPLIANCE_POLICY_ID: - severity = detection.severity - - if severity: - table.add_cell(SEVERITY_COLUMN, SeverityOption(severity)) + if detection.severity: + table.add_cell(SEVERITY_COLUMN, SeverityOption(detection.severity)) else: table.add_cell(SEVERITY_COLUMN, 'N/A') @@ -134,8 +128,8 @@ def _print_summary_issues(self, detections_count: int, title: str) -> None: @staticmethod def _extract_detections_per_policy_id( - local_scan_results: List['LocalScanResult'], - ) -> Dict[str, List[Detection]]: + local_scan_results: list['LocalScanResult'], + ) -> dict[str, list[Detection]]: detections_to_policy_id = defaultdict(list) for local_scan_result in local_scan_results: diff --git a/cycode/cli/printers/tables/table.py b/cycode/cli/printers/tables/table.py index b89df4af..61e143ca 100644 --- a/cycode/cli/printers/tables/table.py +++ b/cycode/cli/printers/tables/table.py @@ -1,5 +1,5 @@ import urllib.parse -from typing import TYPE_CHECKING, Dict, List, Optional, Set +from typing import TYPE_CHECKING, Optional from rich.markup import escape from rich.table import Table as RichTable @@ -11,10 +11,10 @@ class Table: """Helper class to manage columns and their values in the right order and only if the column should be presented.""" - def __init__(self, column_infos: Optional[List['ColumnInfo']] = None) -> None: - self._group_separator_indexes: Set[int] = set() + def __init__(self, column_infos: Optional[list['ColumnInfo']] = None) -> None: + self._group_separator_indexes: set[int] = set() - self._columns: Dict['ColumnInfo', List[str]] = {} + self._columns: dict[ColumnInfo, list[str]] = {} if column_infos: self._columns = {columns: [] for columns in column_infos} @@ -37,17 +37,17 @@ def add_file_path_cell(self, column: 'ColumnInfo', path: str) -> None: escaped_path = escape(encoded_path) self._add_cell_no_error(column, f'[link file://{escaped_path}]{path}') - def set_group_separator_indexes(self, group_separator_indexes: Set[int]) -> None: + def set_group_separator_indexes(self, group_separator_indexes: set[int]) -> None: self._group_separator_indexes = group_separator_indexes - def _get_ordered_columns(self) -> List['ColumnInfo']: + def _get_ordered_columns(self) -> list['ColumnInfo']: # we are sorting columns by index to make sure that columns will be printed in the right order return sorted(self._columns, key=lambda column_info: column_info.index) - def get_columns_info(self) -> List['ColumnInfo']: + def get_columns_info(self) -> list['ColumnInfo']: return self._get_ordered_columns() - def get_rows(self) -> List[str]: + def get_rows(self) -> list[str]: column_values = [self._columns[column_info] for column_info in self._get_ordered_columns()] return list(zip(*column_values)) diff --git a/cycode/cli/printers/tables/table_models.py b/cycode/cli/printers/tables/table_models.py index 42e3b1fb..58e41aaa 100644 --- a/cycode/cli/printers/tables/table_models.py +++ b/cycode/cli/printers/tables/table_models.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, NamedTuple, Optional +from typing import NamedTuple, Optional class ColumnInfoBuilder: @@ -14,12 +14,12 @@ def build(self, name: str, **column_opts) -> 'ColumnInfo': class ColumnInfo(NamedTuple): name: str index: int # Represents the order of the columns, starting from the left - column_opts: Optional[Dict] = None + column_opts: Optional[dict] = None def __hash__(self) -> int: return hash((self.name, self.index)) - def __eq__(self, other: Any) -> bool: + def __eq__(self, other: object) -> bool: if not isinstance(other, ColumnInfo): return NotImplemented return (self.name, self.index) == (other.name, other.index) diff --git a/cycode/cli/printers/tables/table_printer.py b/cycode/cli/printers/tables/table_printer.py index 4f821c7f..fe9f8dd5 100644 --- a/cycode/cli/printers/tables/table_printer.py +++ b/cycode/cli/printers/tables/table_printer.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from cycode.cli.cli_types import SeverityOption from cycode.cli.consts import SECRET_SCAN_TYPE @@ -27,7 +27,7 @@ class TablePrinter(TablePrinterBase): - def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: + def _print_results(self, local_scan_results: list['LocalScanResult']) -> None: table = self._get_table() detections, group_separator_indexes = sort_and_group_detections_from_scan_result(local_scan_results) diff --git a/cycode/cli/printers/tables/table_printer_base.py b/cycode/cli/printers/tables/table_printer_base.py index 5d2aaa73..d7a2b502 100644 --- a/cycode/cli/printers/tables/table_printer_base.py +++ b/cycode/cli/printers/tables/table_printer_base.py @@ -1,5 +1,5 @@ import abc -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING, Optional from cycode.cli.models import CliError, CliResult from cycode.cli.printers.printer_base import PrinterBase @@ -18,7 +18,7 @@ def print_error(self, error: CliError) -> None: TextPrinter(self.ctx, self.console, self.console_err).print_error(error) def print_scan_results( - self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None + self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None ) -> None: if not errors and all(result.issue_detected == 0 for result in local_scan_results): self.console.print(self.NO_DETECTIONS_MESSAGE) @@ -38,7 +38,7 @@ def _is_git_repository(self) -> bool: return self.ctx.info_name in {'commit_history', 'pre_commit', 'pre_receive'} and 'remote_url' in self.ctx.obj @abc.abstractmethod - def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: + def _print_results(self, local_scan_results: list['LocalScanResult']) -> None: raise NotImplementedError def _print_table(self, table: 'Table') -> None: @@ -47,7 +47,7 @@ def _print_table(self, table: 'Table') -> None: def _print_report_urls( self, - local_scan_results: List['LocalScanResult'], + local_scan_results: list['LocalScanResult'], aggregation_report_url: Optional[str] = None, ) -> None: report_urls = [scan_result.report_url for scan_result in local_scan_results if scan_result.report_url] diff --git a/cycode/cli/printers/text_printer.py b/cycode/cli/printers/text_printer.py index 6eb4b78b..564456ae 100644 --- a/cycode/cli/printers/text_printer.py +++ b/cycode/cli/printers/text_printer.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING, Optional from cycode.cli.cli_types import SeverityOption from cycode.cli.models import CliError, CliResult, Document @@ -30,7 +30,7 @@ def print_error(self, error: CliError) -> None: self.console.print(f'[red]Error: {error.message}[/]', highlight=False) def print_scan_results( - self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None + self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None ) -> None: if not errors and all(result.issue_detected == 0 for result in local_scan_results): self.console.print(self.NO_DETECTIONS_MESSAGE) @@ -82,7 +82,7 @@ def __print_detection_code_segment(self, detection: 'Detection', document: Docum ) def print_report_urls_and_errors( - self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None + self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None ) -> None: report_urls = [scan_result.report_url for scan_result in local_scan_results if scan_result.report_url] @@ -95,7 +95,7 @@ def print_report_urls_and_errors( self.console.print(f'- {scan_id}: ', end='') self.print_error(error) - def print_report_urls(self, report_urls: List[str], aggregation_report_url: Optional[str] = None) -> None: + def print_report_urls(self, report_urls: list[str], aggregation_report_url: Optional[str] = None) -> None: if not report_urls and not aggregation_report_url: return if aggregation_report_url: diff --git a/cycode/cli/printers/utils/detection_ordering/common_ordering.py b/cycode/cli/printers/utils/detection_ordering/common_ordering.py index d93b858e..c4b431ef 100644 --- a/cycode/cli/printers/utils/detection_ordering/common_ordering.py +++ b/cycode/cli/printers/utils/detection_ordering/common_ordering.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Set, Tuple +from typing import TYPE_CHECKING from cycode.cli.cli_types import SeverityOption @@ -7,36 +7,36 @@ from cycode.cyclient.models import Detection -GroupedDetections = Tuple[List[Tuple['Detection', 'Document']], Set[int]] +GroupedDetections = tuple[list[tuple['Detection', 'Document']], set[int]] -def __severity_sort_key(detection_with_document: Tuple['Detection', 'Document']) -> int: +def __severity_sort_key(detection_with_document: tuple['Detection', 'Document']) -> int: detection, _ = detection_with_document severity = detection.severity if detection.severity else '' return SeverityOption.get_member_weight(severity) def _sort_detections_by_severity( - detections_with_documents: List[Tuple['Detection', 'Document']], -) -> List[Tuple['Detection', 'Document']]: + detections_with_documents: list[tuple['Detection', 'Document']], +) -> list[tuple['Detection', 'Document']]: return sorted(detections_with_documents, key=__severity_sort_key, reverse=True) -def __file_path_sort_key(detection_with_document: Tuple['Detection', 'Document']) -> str: +def __file_path_sort_key(detection_with_document: tuple['Detection', 'Document']) -> str: _, document = detection_with_document return document.path def _sort_detections_by_file_path( - detections_with_documents: List[Tuple['Detection', 'Document']], -) -> List[Tuple['Detection', 'Document']]: + detections_with_documents: list[tuple['Detection', 'Document']], +) -> list[tuple['Detection', 'Document']]: return sorted(detections_with_documents, key=__file_path_sort_key) def sort_and_group_detections( - detections_with_documents: List[Tuple['Detection', 'Document']], + detections_with_documents: list[tuple['Detection', 'Document']], ) -> GroupedDetections: - """Sort detections by severity. We do not have groping here (don't find the best one yet).""" + """Sort detections by severity. We do not have grouping here (don't find the best one yet).""" group_separator_indexes = set() # we sort detections by file path to make persist output order @@ -46,7 +46,7 @@ def sort_and_group_detections( return sorted_by_severity, group_separator_indexes -def sort_and_group_detections_from_scan_result(local_scan_results: List['LocalScanResult']) -> GroupedDetections: +def sort_and_group_detections_from_scan_result(local_scan_results: list['LocalScanResult']) -> GroupedDetections: detections_with_documents = [] for local_scan_result in local_scan_results: for document_detections in local_scan_result.document_detections: diff --git a/cycode/cli/printers/utils/detection_ordering/sca_ordering.py b/cycode/cli/printers/utils/detection_ordering/sca_ordering.py index 85915c56..a8be3430 100644 --- a/cycode/cli/printers/utils/detection_ordering/sca_ordering.py +++ b/cycode/cli/printers/utils/detection_ordering/sca_ordering.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import TYPE_CHECKING, Dict, List, Set, Tuple +from typing import TYPE_CHECKING from cycode.cli.cli_types import SeverityOption @@ -7,7 +7,7 @@ from cycode.cyclient.models import Detection -def __group_by(detections: List['Detection'], details_field_name: str) -> Dict[str, List['Detection']]: +def __group_by(detections: list['Detection'], details_field_name: str) -> dict[str, list['Detection']]: grouped = defaultdict(list) for detection in detections: grouped[detection.detection_details.get(details_field_name)].append(detection) @@ -15,11 +15,11 @@ def __group_by(detections: List['Detection'], details_field_name: str) -> Dict[s def __severity_sort_key(detection: 'Detection') -> int: - severity = detection.detection_details.get('advisory_severity', 'unknown') + severity = detection.severity if detection.severity else 'unknown' return SeverityOption.get_member_weight(severity) -def _sort_detections_by_severity(detections: List['Detection']) -> List['Detection']: +def _sort_detections_by_severity(detections: list['Detection']) -> list['Detection']: return sorted(detections, key=__severity_sort_key, reverse=True) @@ -27,11 +27,11 @@ def __package_sort_key(detection: 'Detection') -> int: return detection.detection_details.get('package_name') -def _sort_detections_by_package(detections: List['Detection']) -> List['Detection']: +def _sort_detections_by_package(detections: list['Detection']) -> list['Detection']: return sorted(detections, key=__package_sort_key) -def sort_and_group_detections(detections: List['Detection']) -> Tuple[List['Detection'], Set[int]]: +def sort_and_group_detections(detections: list['Detection']) -> tuple[list['Detection'], set[int]]: """Sort detections by severity and group by repository, code project and package name. Note: @@ -39,6 +39,7 @@ def sort_and_group_detections(detections: List['Detection']) -> Tuple[List['Dete Grouping by code projects also groups by ecosystem. Because manifest files are unique per ecosystem. + """ resulting_detections = [] group_separator_indexes = set() diff --git a/cycode/cli/user_settings/base_file_manager.py b/cycode/cli/user_settings/base_file_manager.py index 4eb15e2a..4f07f11c 100644 --- a/cycode/cli/user_settings/base_file_manager.py +++ b/cycode/cli/user_settings/base_file_manager.py @@ -1,6 +1,7 @@ import os from abc import ABC, abstractmethod -from typing import Any, Dict, Hashable +from collections.abc import Hashable +from typing import Any from cycode.cli.utils.yaml_utils import read_yaml_file, update_yaml_file @@ -9,10 +10,10 @@ class BaseFileManager(ABC): @abstractmethod def get_filename(self) -> str: ... - def read_file(self) -> Dict[Hashable, Any]: + def read_file(self) -> dict[Hashable, Any]: return read_yaml_file(self.get_filename()) - def write_content_to_file(self, content: Dict[Hashable, Any]) -> None: + def write_content_to_file(self, content: dict[Hashable, Any]) -> None: filename = self.get_filename() os.makedirs(os.path.dirname(filename), exist_ok=True) update_yaml_file(filename, content) diff --git a/cycode/cli/user_settings/config_file_manager.py b/cycode/cli/user_settings/config_file_manager.py index e4e5e6b1..5b029e39 100644 --- a/cycode/cli/user_settings/config_file_manager.py +++ b/cycode/cli/user_settings/config_file_manager.py @@ -1,5 +1,6 @@ import os -from typing import TYPE_CHECKING, Any, Dict, Hashable, List, Optional, Union +from collections.abc import Hashable +from typing import TYPE_CHECKING, Any, Optional, Union from cycode.cli.consts import CYCODE_CONFIGURATION_DIRECTORY from cycode.cli.user_settings.base_file_manager import BaseFileManager @@ -37,7 +38,7 @@ def get_app_url(self) -> Optional[Any]: def get_verbose_flag(self) -> Optional[Any]: return self._get_value_from_environment_section(self.VERBOSE_FIELD_NAME) - def get_exclusions_by_scan_type(self, scan_type: str) -> Dict[Hashable, Any]: + def get_exclusions_by_scan_type(self, scan_type: str) -> dict[Hashable, Any]: exclusions_section = self._get_section(self.EXCLUSIONS_SECTION_NAME) return exclusions_section.get(scan_type, {}) @@ -87,7 +88,7 @@ def get_filename(self) -> str: def get_config_file_route() -> str: return os.path.join(ConfigFileManager.CYCODE_HIDDEN_DIRECTORY, ConfigFileManager.FILE_NAME) - def _get_exclusions_by_exclusion_type(self, scan_type: str, exclusion_type: str) -> List[Any]: + def _get_exclusions_by_exclusion_type(self, scan_type: str, exclusion_type: str) -> list[Any]: scan_type_exclusions = self.get_exclusions_by_scan_type(scan_type) return scan_type_exclusions.get(exclusion_type, []) @@ -95,7 +96,7 @@ def _get_value_from_environment_section(self, field_name: str) -> Optional[Any]: environment_section = self._get_section(self.ENVIRONMENT_SECTION_NAME) return environment_section.get(field_name) - def _get_scan_configuration_by_scan_type(self, command_scan_type: str) -> Dict[Hashable, Any]: + def _get_scan_configuration_by_scan_type(self, command_scan_type: str) -> dict[Hashable, Any]: scan_section = self._get_section(self.SCAN_SECTION_NAME) return scan_section.get(command_scan_type, {}) @@ -103,6 +104,6 @@ def _get_value_from_command_scan_type_configuration(self, command_scan_type: str command_scan_type_configuration = self._get_scan_configuration_by_scan_type(command_scan_type) return command_scan_type_configuration.get(field_name) - def _get_section(self, section_name: str) -> Dict[Hashable, Any]: + def _get_section(self, section_name: str) -> dict[Hashable, Any]: file_content = self.read_file() return file_content.get(section_name, {}) diff --git a/cycode/cli/user_settings/configuration_manager.py b/cycode/cli/user_settings/configuration_manager.py index f8d67c42..3b83f1c9 100644 --- a/cycode/cli/user_settings/configuration_manager.py +++ b/cycode/cli/user_settings/configuration_manager.py @@ -1,7 +1,7 @@ import os -from functools import lru_cache +from functools import cache from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Optional from uuid import uuid4 from cycode.cli import consts @@ -69,8 +69,8 @@ def get_verbose_flag_from_environment_variables(self) -> bool: value = self._get_value_from_environment_variables(consts.VERBOSE_ENV_VAR_NAME, '') return value.lower() in ('true', '1') - @lru_cache(maxsize=None) # noqa: B019 - def get_exclusions_by_scan_type(self, scan_type: str) -> Dict: + @cache # noqa: B019 + def get_exclusions_by_scan_type(self, scan_type: str) -> dict: local_exclusions = self.local_config_file_manager.get_exclusions_by_scan_type(scan_type) global_exclusions = self.global_config_file_manager.get_exclusions_by_scan_type(scan_type) return self._merge_exclusions(local_exclusions, global_exclusions) @@ -80,7 +80,7 @@ def add_exclusion(self, scope: str, scan_type: str, exclusion_type: str, value: config_file_manager.add_exclusion(scan_type, exclusion_type, value) @staticmethod - def _merge_exclusions(local_exclusions: Dict, global_exclusions: Dict) -> Dict: + def _merge_exclusions(local_exclusions: dict, global_exclusions: dict) -> dict: keys = set(list(local_exclusions.keys()) + list(global_exclusions.keys())) return {key: local_exclusions.get(key, []) + global_exclusions.get(key, []) for key in keys} diff --git a/cycode/cli/user_settings/credentials_manager.py b/cycode/cli/user_settings/credentials_manager.py index 86a84ba6..7af43569 100644 --- a/cycode/cli/user_settings/credentials_manager.py +++ b/cycode/cli/user_settings/credentials_manager.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from typing import Optional, Tuple +from typing import Optional from cycode.cli.config import CYCODE_CLIENT_ID_ENV_VAR_NAME, CYCODE_CLIENT_SECRET_ENV_VAR_NAME from cycode.cli.user_settings.base_file_manager import BaseFileManager @@ -19,7 +19,7 @@ class CredentialsManager(BaseFileManager): ACCESS_TOKEN_EXPIRES_IN_FIELD_NAME: str = 'cycode_access_token_expires_in' ACCESS_TOKEN_CREATOR_FIELD_NAME: str = 'cycode_access_token_creator' - def get_credentials(self) -> Tuple[str, str]: + def get_credentials(self) -> tuple[str, str]: client_id, client_secret = self.get_credentials_from_environment_variables() if client_id is not None and client_secret is not None: return client_id, client_secret @@ -27,12 +27,12 @@ def get_credentials(self) -> Tuple[str, str]: return self.get_credentials_from_file() @staticmethod - def get_credentials_from_environment_variables() -> Tuple[str, str]: + def get_credentials_from_environment_variables() -> tuple[str, str]: client_id = os.getenv(CYCODE_CLIENT_ID_ENV_VAR_NAME) client_secret = os.getenv(CYCODE_CLIENT_SECRET_ENV_VAR_NAME) return client_id, client_secret - def get_credentials_from_file(self) -> Tuple[Optional[str], Optional[str]]: + def get_credentials_from_file(self) -> tuple[Optional[str], Optional[str]]: file_content = self.read_file() client_id = file_content.get(self.CLIENT_ID_FIELD_NAME) client_secret = file_content.get(self.CLIENT_SECRET_FIELD_NAME) @@ -42,7 +42,7 @@ def update_credentials(self, client_id: str, client_secret: str) -> None: file_content_to_update = {self.CLIENT_ID_FIELD_NAME: client_id, self.CLIENT_SECRET_FIELD_NAME: client_secret} self.write_content_to_file(file_content_to_update) - def get_access_token(self) -> Tuple[Optional[str], Optional[float], Optional[JwtCreator]]: + def get_access_token(self) -> tuple[Optional[str], Optional[float], Optional[JwtCreator]]: file_content = self.read_file() access_token = file_content.get(self.ACCESS_TOKEN_FIELD_NAME) diff --git a/cycode/cli/utils/enum_utils.py b/cycode/cli/utils/enum_utils.py index 6ea9ef72..3280a5bb 100644 --- a/cycode/cli/utils/enum_utils.py +++ b/cycode/cli/utils/enum_utils.py @@ -1,8 +1,7 @@ from enum import Enum -from typing import List class AutoCountEnum(Enum): @staticmethod - def _generate_next_value_(name: str, start: int, count: int, last_values: List[int]) -> int: + def _generate_next_value_(name: str, start: int, count: int, last_values: list[int]) -> int: return count diff --git a/cycode/cli/utils/get_api_client.py b/cycode/cli/utils/get_api_client.py index 7bbfa2d9..91e8f0f7 100644 --- a/cycode/cli/utils/get_api_client.py +++ b/cycode/cli/utils/get_api_client.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional, Tuple, Union +from typing import TYPE_CHECKING, Optional, Union import click @@ -35,6 +35,6 @@ def get_report_cycode_client( return _get_cycode_client(create_report_client, client_id, client_secret, hide_response_log) -def _get_configured_credentials() -> Tuple[str, str]: +def _get_configured_credentials() -> tuple[str, str]: credentials_manager = CredentialsManager() return credentials_manager.get_credentials() diff --git a/cycode/cli/utils/git_proxy.py b/cycode/cli/utils/git_proxy.py index c46d016b..beaafdd0 100644 --- a/cycode/cli/utils/git_proxy.py +++ b/cycode/cli/utils/git_proxy.py @@ -1,9 +1,9 @@ import types from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Optional, Type +from typing import TYPE_CHECKING, Optional _GIT_ERROR_MESSAGE = """ -Cycode CLI needs the git executable to be installed on the system. +Cycode CLI needs the Git executable to be installed on the system. Git executable must be available in the PATH. Git 1.7.x or newer is required. You can help Cycode CLI to locate the Git executable @@ -31,10 +31,10 @@ def get_repo(self, path: Optional['PathLike'] = None, *args, **kwargs) -> 'Repo' def get_null_tree(self) -> object: ... @abstractmethod - def get_invalid_git_repository_error(self) -> Type[BaseException]: ... + def get_invalid_git_repository_error(self) -> type[BaseException]: ... @abstractmethod - def get_git_command_error(self) -> Type[BaseException]: ... + def get_git_command_error(self) -> type[BaseException]: ... class _DummyGitProxy(_AbstractGitProxy): @@ -44,10 +44,10 @@ def get_repo(self, path: Optional['PathLike'] = None, *args, **kwargs) -> 'Repo' def get_null_tree(self) -> object: raise RuntimeError(_GIT_ERROR_MESSAGE) - def get_invalid_git_repository_error(self) -> Type[BaseException]: + def get_invalid_git_repository_error(self) -> type[BaseException]: return GitProxyError - def get_git_command_error(self) -> Type[BaseException]: + def get_git_command_error(self) -> type[BaseException]: return GitProxyError @@ -58,10 +58,10 @@ def get_repo(self, path: Optional['PathLike'] = None, *args, **kwargs) -> 'Repo' def get_null_tree(self) -> object: return git.NULL_TREE - def get_invalid_git_repository_error(self) -> Type[BaseException]: + def get_invalid_git_repository_error(self) -> type[BaseException]: return git.InvalidGitRepositoryError - def get_git_command_error(self) -> Type[BaseException]: + def get_git_command_error(self) -> type[BaseException]: return git.GitCommandError @@ -87,10 +87,10 @@ def get_repo(self, path: Optional['PathLike'] = None, *args, **kwargs) -> 'Repo' def get_null_tree(self) -> object: return self._git_proxy.get_null_tree() - def get_invalid_git_repository_error(self) -> Type[BaseException]: + def get_invalid_git_repository_error(self) -> type[BaseException]: return self._git_proxy.get_invalid_git_repository_error() - def get_git_command_error(self) -> Type[BaseException]: + def get_git_command_error(self) -> type[BaseException]: return self._git_proxy.get_git_command_error() diff --git a/cycode/cli/utils/ignore_utils.py b/cycode/cli/utils/ignore_utils.py index f44b6024..e8994e46 100644 --- a/cycode/cli/utils/ignore_utils.py +++ b/cycode/cli/utils/ignore_utils.py @@ -38,16 +38,12 @@ import contextlib import os.path import re +from collections.abc import Generator, Iterable from os import PathLike from typing import ( Any, BinaryIO, - Dict, - Generator, - Iterable, - List, Optional, - Tuple, Union, ) @@ -98,7 +94,6 @@ def translate(pat: bytes) -> bytes: Originally copied from fnmatch in Python 2.7, but modified for Dulwich to cope with features in Git ignore patterns. """ - res = b'(?ms)' if b'/' not in pat[:-1]: @@ -131,6 +126,7 @@ def read_ignore_patterns(f: BinaryIO) -> Iterable[bytes]: Args: f: File-like object to read from Returns: List of patterns + """ for line in f: line = line.rstrip(b'\r\n') @@ -160,6 +156,7 @@ def match_pattern(path: bytes, pattern: bytes, ignore_case: bool = False) -> boo ignore_case: Whether to do case-sensitive matching Returns: bool indicating whether the pattern matched + """ return Pattern(pattern, ignore_case).match(path) @@ -200,6 +197,7 @@ def match(self, path: bytes) -> bool: Args: path: Path to match (relative to ignore location) Returns: boolean + """ return bool(self._re.match(path)) @@ -219,7 +217,7 @@ def __init__( for pattern in patterns: self.append_pattern(pattern) - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: d = { 'patterns': [str(p) for p in self._patterns], 'ignore_case': self._ignore_case, @@ -242,6 +240,7 @@ def find_matching(self, path: Union[bytes, str]) -> Iterable[Pattern]: path: Path to match Returns: Iterator over iterators + """ if not isinstance(path, bytes): path = os.fsencode(path) @@ -284,7 +283,7 @@ class IgnoreFilterManager: def __init__( self, path: str, - global_filters: List[IgnoreFilter], + global_filters: list[IgnoreFilter], ignore_file_name: Optional[str] = None, ignore_case: bool = False, ) -> None: @@ -303,7 +302,7 @@ def __init__( def __repr__(self) -> str: return f'{type(self).__name__}({self._top_path}, {self._global_filters!r}, {self._ignore_case!r})' - def to_dict(self, include_path_filters: bool = True) -> Dict[str, Any]: + def to_dict(self, include_path_filters: bool = True) -> dict[str, Any]: d = { 'path': self._top_path, 'global_filters': [f.to_dict() for f in self._global_filters], @@ -337,7 +336,7 @@ def _load_path(self, path: str) -> Optional[IgnoreFilter]: p = os.path.join(self._top_path, path, self._ignore_file_name) try: self._path_filters[path] = IgnoreFilter.from_path(p, self._ignore_case) - except IOError: + except OSError: self._path_filters[path] = None return self._path_filters[path] @@ -348,6 +347,7 @@ def _find_matching(self, path: str) -> Iterable[Pattern]: path: Path to check Returns: Iterator over Pattern instances + """ if os.path.isabs(path): raise ValueError(f'{path} is an absolute path') @@ -379,6 +379,7 @@ def is_ignored(self, path: str) -> Optional[bool]: True if the path matches an ignore pattern, False if the path is explicitly not ignored, or None if the file does not match any patterns. + """ if hasattr(path, '__fspath__'): path = path.__fspath__() @@ -387,10 +388,8 @@ def is_ignored(self, path: str) -> Optional[bool]: return matches[-1].is_exclude return None - def walk(self, **kwargs) -> Generator[Tuple[str, List[str], List[str]], None, None]: - """A wrapper for os.walk() without ignored files and subdirectories. - kwargs are passed to walk().""" - + def walk(self, **kwargs) -> Generator[tuple[str, list[str], list[str]], None, None]: + """Wrap os.walk() without ignored files and subdirectories and kwargs are passed to walk.""" for dirpath, dirnames, filenames in os.walk(self.path, topdown=True, **kwargs): rel_dirpath = '' if dirpath == self.path else os.path.relpath(dirpath, self.path) @@ -413,6 +412,7 @@ def build( ignore_case: bool = False, ) -> 'IgnoreFilterManager': """Create a IgnoreFilterManager from patterns and paths. + Args: path: The root path for ignore checks. global_ignore_file_paths: A list of file paths to load patterns from. @@ -421,8 +421,10 @@ def build( global_patterns: Global patterns to ignore. ignore_file_name: The per-directory ignore file name. ignore_case: Whether to ignore case in matching. + Returns: A `IgnoreFilterManager` object + """ if not global_ignore_file_paths: global_ignore_file_paths = [] diff --git a/cycode/cli/utils/jwt_utils.py b/cycode/cli/utils/jwt_utils.py index 7bb7df62..c87b7c48 100644 --- a/cycode/cli/utils/jwt_utils.py +++ b/cycode/cli/utils/jwt_utils.py @@ -1,11 +1,11 @@ -from typing import Optional, Tuple +from typing import Optional import jwt _JWT_PAYLOAD_POSSIBLE_USER_ID_FIELD_NAMES = ('userId', 'internalId', 'token-user-id') -def get_user_and_tenant_ids_from_access_token(access_token: str) -> Tuple[Optional[str], Optional[str]]: +def get_user_and_tenant_ids_from_access_token(access_token: str) -> tuple[Optional[str], Optional[str]]: payload = jwt.decode(access_token, options={'verify_signature': False}) user_id = None diff --git a/cycode/cli/utils/path_utils.py b/cycode/cli/utils/path_utils.py index 3f670dd4..7d525e56 100644 --- a/cycode/cli/utils/path_utils.py +++ b/cycode/cli/utils/path_utils.py @@ -1,7 +1,7 @@ import json import os -from functools import lru_cache -from typing import TYPE_CHECKING, AnyStr, List, Optional, Union +from functools import cache +from typing import TYPE_CHECKING, AnyStr, Optional, Union import typer from binaryornot.helpers import is_binary_string @@ -12,7 +12,7 @@ from os import PathLike -@lru_cache(maxsize=None) +@cache def is_sub_path(path: str, sub_path: str) -> bool: try: common_path = os.path.commonpath([get_absolute_path(path), get_absolute_path(sub_path)]) @@ -35,7 +35,7 @@ def _get_starting_chunk(filename: str, length: int = 1024) -> Optional[bytes]: try: with open(filename, 'rb') as f: return f.read(length) - except IOError as e: + except OSError as e: logger.debug('Failed to read the starting chunk from file: %s', filename, exc_info=e) return None @@ -68,7 +68,7 @@ def get_file_dir(path: str) -> str: return os.path.dirname(path) -def get_immediate_subdirectories(path: str) -> List[str]: +def get_immediate_subdirectories(path: str) -> list[str]: return [f.name for f in os.scandir(path) if f.is_dir()] @@ -78,7 +78,7 @@ def join_paths(path: str, filename: str) -> str: def get_file_content(file_path: Union[str, 'PathLike']) -> Optional[AnyStr]: try: - with open(file_path, 'r', encoding='UTF-8') as f: + with open(file_path, encoding='UTF-8') as f: return f.read() except (FileNotFoundError, UnicodeDecodeError): return None diff --git a/cycode/cli/utils/progress_bar.py b/cycode/cli/utils/progress_bar.py index 054d5cf8..7c2de487 100644 --- a/cycode/cli/utils/progress_bar.py +++ b/cycode/cli/utils/progress_bar.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from enum import auto -from typing import Dict, NamedTuple, Optional +from typing import NamedTuple, Optional from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn, TimeElapsedColumn @@ -38,7 +38,7 @@ class ProgressBarSectionInfo(NamedTuple): TimeElapsedColumn(), ) -ProgressBarSections = Dict[ProgressBarSection, ProgressBarSectionInfo] +ProgressBarSections = dict[ProgressBarSection, ProgressBarSectionInfo] class ScanProgressBarSection(ProgressBarSection): @@ -138,8 +138,8 @@ def __init__(self, progress_bar_sections: ProgressBarSections) -> None: self._progress_bar_sections = progress_bar_sections - self._section_lengths: Dict[ProgressBarSection, int] = {} - self._section_values: Dict[ProgressBarSection, int] = {} + self._section_lengths: dict[ProgressBarSection, int] = {} + self._section_values: dict[ProgressBarSection, int] = {} self._current_section_value = 0 self._current_section: ProgressBarSectionInfo = _get_initial_section(self._progress_bar_sections) @@ -195,7 +195,7 @@ def _increment_section_value(self, section: 'ProgressBarSection', value: int) -> ) def _rerender_progress_bar(self) -> None: - """Used to update label right after changing the progress bar section.""" + """Use to update label right after changing the progress bar section.""" self._progress_bar_update() def _increment_progress(self, section: 'ProgressBarSection') -> None: diff --git a/cycode/cli/utils/scan_batch.py b/cycode/cli/utils/scan_batch.py index 45e4d120..8bfd7ed0 100644 --- a/cycode/cli/utils/scan_batch.py +++ b/cycode/cli/utils/scan_batch.py @@ -1,6 +1,6 @@ import os from multiprocessing.pool import ThreadPool -from typing import TYPE_CHECKING, Callable, Dict, List, Tuple +from typing import TYPE_CHECKING, Callable from cycode.cli import consts from cycode.cli.models import Document @@ -45,8 +45,8 @@ def _get_max_batch_files_count(_: str) -> int: def split_documents_into_batches( scan_type: str, - documents: List[Document], -) -> List[List[Document]]: + documents: list[Document], +) -> list[list[Document]]: max_size = _get_max_batch_size(scan_type) max_files_count = _get_max_batch_files_count(scan_type) @@ -107,11 +107,11 @@ def _get_threads_count() -> int: def run_parallel_batched_scan( - scan_function: Callable[[List[Document]], Tuple[str, 'CliError', 'LocalScanResult']], + scan_function: Callable[[list[Document]], tuple[str, 'CliError', 'LocalScanResult']], scan_type: str, - documents: List[Document], + documents: list[Document], progress_bar: 'BaseProgressBar', -) -> Tuple[Dict[str, 'CliError'], List['LocalScanResult']]: +) -> tuple[dict[str, 'CliError'], list['LocalScanResult']]: # batching is disabled for SCA; requested by Mor batches = [documents] if scan_type == consts.SCA_SCAN_TYPE else split_documents_into_batches(scan_type, documents) @@ -124,8 +124,8 @@ def run_parallel_batched_scan( # the progress bar could be significant improved (be more dynamic) in the future threads_count = _get_threads_count() - local_scan_results: List['LocalScanResult'] = [] - cli_errors: Dict[str, 'CliError'] = {} + local_scan_results: list[LocalScanResult] = [] + cli_errors: dict[str, CliError] = {} logger.debug('Running parallel batched scan, %s', {'threads_count': threads_count, 'batches_count': len(batches)}) diff --git a/cycode/cli/utils/shell_executor.py b/cycode/cli/utils/shell_executor.py index 812fee1f..db0331da 100644 --- a/cycode/cli/utils/shell_executor.py +++ b/cycode/cli/utils/shell_executor.py @@ -1,5 +1,5 @@ import subprocess -from typing import List, Optional, Union +from typing import Optional, Union import click import typer @@ -13,7 +13,7 @@ def shell( - command: Union[str, List[str]], + command: Union[str, list[str]], timeout: int = _SUBPROCESS_DEFAULT_TIMEOUT_SEC, working_directory: Optional[str] = None, ) -> Optional[str]: diff --git a/cycode/cli/utils/task_timer.py b/cycode/cli/utils/task_timer.py index 29e65dc8..4b5e903e 100644 --- a/cycode/cli/utils/task_timer.py +++ b/cycode/cli/utils/task_timer.py @@ -1,19 +1,18 @@ from _thread import interrupt_main from threading import Event, Thread from types import TracebackType -from typing import Callable, Dict, List, Optional, Type +from typing import Callable, Optional class FunctionContext: - def __init__(self, function: Callable, args: Optional[List] = None, kwargs: Optional[Dict] = None) -> None: + def __init__(self, function: Callable, args: Optional[list] = None, kwargs: Optional[dict] = None) -> None: self.function = function self.args = args or [] self.kwargs = kwargs or {} class TimerThread(Thread): - """ - Custom thread class for executing timer in the background + """Custom thread class for executing timer in the background. Members: timeout - the amount of time to count until timeout in seconds @@ -43,8 +42,7 @@ def _call_quit_function(self) -> None: class TimeoutAfter: - """ - A task wrapper for controlling how much time a task should be run before timing out + """A task wrapper for controlling how much time a task should be run before timing out. Use Example: with TimeoutAfter(5, repeat_function=FunctionContext(x), repeat_interval=2): @@ -66,7 +64,7 @@ def __enter__(self) -> None: self.timer.start() def __exit__( - self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType] + self, exc_type: Optional[type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType] ) -> None: if self.timeout: self.timer.stop() diff --git a/cycode/cli/utils/version_checker.py b/cycode/cli/utils/version_checker.py index 035b3595..47da17c4 100644 --- a/cycode/cli/utils/version_checker.py +++ b/cycode/cli/utils/version_checker.py @@ -2,7 +2,7 @@ import re import time from pathlib import Path -from typing import List, Optional, Tuple +from typing import Optional from cycode.cli.console import console from cycode.cli.user_settings.configuration_manager import ConfigurationManager @@ -11,8 +11,8 @@ def _compare_versions( - current_parts: List[int], - latest_parts: List[int], + current_parts: list[int], + latest_parts: list[int], current_is_pre: bool, latest_is_pre: bool, latest_version: str, @@ -33,6 +33,7 @@ def _compare_versions( Returns: str | None: The latest version string if an update is recommended, None if no update is needed + """ # If current is stable and latest is pre-release, don't suggest update if not current_is_pre and latest_is_pre: @@ -82,6 +83,7 @@ def get_latest_version(self) -> Optional[str]: Returns: str | None: The latest version string if successful, None if the request fails or the version information is not available. + """ try: response = self.get(f'{self.PYPI_PACKAGE_NAME}/json', timeout=self.PYPI_REQUEST_TIMEOUT) @@ -91,7 +93,7 @@ def get_latest_version(self) -> Optional[str]: return None @staticmethod - def _parse_version(version: str) -> Tuple[List[int], bool]: + def _parse_version(version: str) -> tuple[list[int], bool]: """Parse version string into components and identify if it's a pre-release. Extracts numeric version components and determines if the version is a pre-release @@ -104,6 +106,7 @@ def _parse_version(version: str) -> Tuple[List[int], bool]: tuple: A tuple containing: - List[int]: List of numeric version components - bool: True if this is a pre-release version, False otherwise + """ version_parts = [int(x) for x in re.findall(r'\d+', version)] is_prerelease = 'dev' in version @@ -122,6 +125,7 @@ def _should_check_update(self, is_prerelease: bool) -> bool: Returns: bool: True if an update check should be performed, False otherwise + """ if not os.path.exists(self.cache_file): return True @@ -148,7 +152,7 @@ def _update_last_check(self) -> None: os.makedirs(os.path.dirname(self.cache_file), exist_ok=True) with open(self.cache_file, 'w', encoding='UTF-8') as f: f.write(str(time.time())) - except IOError: + except OSError: pass def check_for_update(self, current_version: str, use_cache: bool = True) -> Optional[str]: @@ -163,6 +167,7 @@ def check_for_update(self, current_version: str, use_cache: bool = True) -> Opti Returns: str | None: The latest version string if an update is recommended, None if no update is needed or if check should be skipped + """ current_parts, current_is_pre = self._parse_version(current_version) @@ -192,6 +197,7 @@ def check_and_notify_update(self, current_version: str, use_cache: bool = True) Args: current_version: Current version of the CLI use_cache: If True, use the cached timestamp to determine if an update check is needed + """ latest_version = self.check_for_update(current_version, use_cache) should_update = bool(latest_version) diff --git a/cycode/cli/utils/yaml_utils.py b/cycode/cli/utils/yaml_utils.py index 388f3498..c89e1a5c 100644 --- a/cycode/cli/utils/yaml_utils.py +++ b/cycode/cli/utils/yaml_utils.py @@ -1,10 +1,11 @@ import os -from typing import Any, Dict, Hashable, TextIO +from collections.abc import Hashable +from typing import Any, TextIO import yaml -def _deep_update(source: Dict[Hashable, Any], overrides: Dict[Hashable, Any]) -> Dict[Hashable, Any]: +def _deep_update(source: dict[Hashable, Any], overrides: dict[Hashable, Any]) -> dict[Hashable, Any]: for key, value in overrides.items(): if isinstance(value, dict) and value: source[key] = _deep_update(source.get(key, {}), value) @@ -14,7 +15,7 @@ def _deep_update(source: Dict[Hashable, Any], overrides: Dict[Hashable, Any]) -> return source -def _yaml_safe_load(file: TextIO) -> Dict[Hashable, Any]: +def _yaml_safe_load(file: TextIO) -> dict[Hashable, Any]: # loader.get_single_data could return None loaded_file = yaml.safe_load(file) if loaded_file is None: @@ -23,18 +24,18 @@ def _yaml_safe_load(file: TextIO) -> Dict[Hashable, Any]: return loaded_file -def read_yaml_file(filename: str) -> Dict[Hashable, Any]: +def read_yaml_file(filename: str) -> dict[Hashable, Any]: if not os.path.exists(filename): return {} - with open(filename, 'r', encoding='UTF-8') as file: + with open(filename, encoding='UTF-8') as file: return _yaml_safe_load(file) -def write_yaml_file(filename: str, content: Dict[Hashable, Any]) -> None: +def write_yaml_file(filename: str, content: dict[Hashable, Any]) -> None: with open(filename, 'w', encoding='UTF-8') as file: yaml.safe_dump(content, file) -def update_yaml_file(filename: str, content: Dict[Hashable, Any]) -> None: +def update_yaml_file(filename: str, content: dict[Hashable, Any]) -> None: write_yaml_file(filename, _deep_update(read_yaml_file(filename), content)) diff --git a/cycode/cyclient/cycode_client_base.py b/cycode/cyclient/cycode_client_base.py index 37e9d4f6..4b2e2698 100644 --- a/cycode/cyclient/cycode_client_base.py +++ b/cycode/cyclient/cycode_client_base.py @@ -1,7 +1,7 @@ import os import platform import ssl -from typing import TYPE_CHECKING, Callable, ClassVar, Dict, Optional +from typing import TYPE_CHECKING, Callable, ClassVar, Optional import requests from requests import Response, exceptions @@ -14,7 +14,7 @@ RequestError, RequestHttpError, RequestSslError, - RequestTimeout, + RequestTimeoutError, ) from cycode.cyclient import config from cycode.cyclient.headers import get_cli_user_agent, get_correlation_id @@ -50,7 +50,7 @@ def _get_request_function() -> Callable: _REQUEST_ERRORS_TO_RETRY = ( - RequestTimeout, + RequestTimeoutError, RequestConnectionError, exceptions.ChunkedEncodingError, exceptions.ContentDecodingError, @@ -91,7 +91,7 @@ def _should_retry_exception(exception: BaseException) -> bool: class CycodeClientBase: - MANDATORY_HEADERS: ClassVar[Dict[str, str]] = { + MANDATORY_HEADERS: ClassVar[dict[str, str]] = { 'User-Agent': get_cli_user_agent(), 'X-Correlation-Id': get_correlation_id(), } @@ -160,7 +160,7 @@ def _execute( except Exception as e: self._handle_exception(e) - def get_request_headers(self, additional_headers: Optional[dict] = None, **kwargs) -> Dict[str, str]: + def get_request_headers(self, additional_headers: Optional[dict] = None, **kwargs) -> dict[str, str]: if additional_headers is None: return self.MANDATORY_HEADERS.copy() return {**self.MANDATORY_HEADERS, **additional_headers} @@ -170,7 +170,7 @@ def build_full_url(self, url: str, endpoint: str) -> str: def _handle_exception(self, e: Exception) -> None: if isinstance(e, exceptions.Timeout): - raise RequestTimeout from e + raise RequestTimeoutError from e if isinstance(e, exceptions.HTTPError): raise self._get_http_exception(e) from e if isinstance(e, exceptions.SSLError): diff --git a/cycode/cyclient/cycode_dev_based_client.py b/cycode/cyclient/cycode_dev_based_client.py index 347797c3..d8fe1cab 100644 --- a/cycode/cyclient/cycode_dev_based_client.py +++ b/cycode/cyclient/cycode_dev_based_client.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional +from typing import Optional from cycode.cyclient.config import dev_tenant_id from cycode.cyclient.cycode_client_base import CycodeClientBase @@ -12,7 +12,7 @@ class CycodeDevBasedClient(CycodeClientBase): def __init__(self, api_url: str) -> None: super().__init__(api_url) - def get_request_headers(self, additional_headers: Optional[dict] = None, **_) -> Dict[str, str]: + def get_request_headers(self, additional_headers: Optional[dict] = None, **_) -> dict[str, str]: headers = super().get_request_headers(additional_headers=additional_headers) headers['X-Tenant-Id'] = dev_tenant_id diff --git a/cycode/cyclient/headers.py b/cycode/cyclient/headers.py index 76716826..5d10f69b 100644 --- a/cycode/cyclient/headers.py +++ b/cycode/cyclient/headers.py @@ -35,6 +35,7 @@ def get_correlation_id(self) -> str: Used across all requests to correlate logs and metrics. It doesn't depend on client instances. Lifetime is the same as the process. + """ if self._id is None: # example: 16fd2706-8baf-433b-82eb-8c7fada847da diff --git a/cycode/cyclient/models.py b/cycode/cyclient/models.py index 2c0f53d7..70e3e551 100644 --- a/cycode/cyclient/models.py +++ b/cycode/cyclient/models.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Any, Dict, List, Optional +from typing import Any, Optional from marshmallow import EXCLUDE, Schema, fields, post_load @@ -47,12 +47,12 @@ class Meta: detection_rule_id = fields.String() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> Detection: + def build_dto(self, data: dict[str, Any], **_) -> Detection: return Detection(**data) class DetectionsPerFile(Schema): - def __init__(self, file_name: str, detections: List[Detection], commit_id: Optional[str] = None) -> None: + def __init__(self, file_name: str, detections: list[Detection], commit_id: Optional[str] = None) -> None: super().__init__() self.file_name = file_name self.detections = detections @@ -63,7 +63,7 @@ class ZippedFileScanResult(Schema): def __init__( self, did_detect: bool, - detections_per_file: List[DetectionsPerFile], + detections_per_file: list[DetectionsPerFile], report_url: Optional[str] = None, scan_id: Optional[str] = None, err: Optional[str] = None, @@ -81,7 +81,7 @@ def __init__( self, did_detect: bool, scan_id: Optional[str] = None, - detections: Optional[List[Detection]] = None, + detections: Optional[list[Detection]] = None, err: Optional[str] = None, ) -> None: super().__init__() @@ -101,7 +101,7 @@ class Meta: err = fields.String() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'ScanResult': + def build_dto(self, data: dict[str, Any], **_) -> 'ScanResult': return ScanResult(**data) @@ -120,7 +120,7 @@ class Meta: err = fields.String() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'ScanInitializationResponse': + def build_dto(self, data: dict[str, Any], **_) -> 'ScanInitializationResponse': return ScanInitializationResponse(**data) @@ -154,7 +154,7 @@ class ScanReportUrlResponseSchema(Schema): report_url = fields.String() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'ScanReportUrlResponse': + def build_dto(self, data: dict[str, Any], **_) -> 'ScanReportUrlResponse': return ScanReportUrlResponse(**data) @@ -171,12 +171,12 @@ class Meta: err = fields.String() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'ScanDetailsResponse': + def build_dto(self, data: dict[str, Any], **_) -> 'ScanDetailsResponse': return ScanDetailsResponse(**data) class K8SResource: - def __init__(self, name: str, resource_type: str, namespace: str, content: Dict) -> None: + def __init__(self, name: str, resource_type: str, namespace: str, content: dict) -> None: super().__init__() self.name = name self.type = resource_type @@ -201,7 +201,7 @@ def to_json(self) -> dict: # FIXME(MarshalX): rename to to_dict? class ResourcesCollection: - def __init__(self, resource_type: str, namespace: str, resources: List[K8SResource], total_count: int) -> None: + def __init__(self, resource_type: str, namespace: str, resources: list[K8SResource], total_count: int) -> None: super().__init__() self.type = resource_type self.namespace = namespace @@ -240,7 +240,7 @@ def __init__(self, name: str, kind: str) -> None: self.kind = kind def __str__(self) -> str: - return 'Name: {0}, Kind: {1}'.format(self.name, self.kind) + return f'Name: {self.name}, Kind: {self.kind}' class AuthenticationSession(Schema): @@ -256,7 +256,7 @@ class Meta: session_id = fields.String() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'AuthenticationSession': + def build_dto(self, data: dict[str, Any], **_) -> 'AuthenticationSession': return AuthenticationSession(**data) @@ -277,7 +277,7 @@ class Meta: description = fields.String() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'ApiToken': + def build_dto(self, data: dict[str, Any], **_) -> 'ApiToken': return ApiToken(**data) @@ -296,7 +296,7 @@ class Meta: api_token = fields.Nested(ApiTokenSchema, allow_none=True) @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'ApiTokenGenerationPollingResponse': + def build_dto(self, data: dict[str, Any], **_) -> 'ApiTokenGenerationPollingResponse': return ApiTokenGenerationPollingResponse(**data) @@ -307,7 +307,7 @@ class UserAgentOptionScheme(Schema): env_version = fields.String(required=True) # ex. 1.78.2 @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'UserAgentOption': + def build_dto(self, data: dict[str, Any], **_) -> 'UserAgentOption': return UserAgentOption(**data) @@ -349,7 +349,7 @@ class Meta: size = fields.Integer() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> SbomReportStorageDetails: + def build_dto(self, data: dict[str, Any], **_) -> SbomReportStorageDetails: return SbomReportStorageDetails(**data) @@ -373,13 +373,13 @@ class Meta: storage_details = fields.Nested(SbomReportStorageDetailsSchema, allow_none=True) @post_load - def build_dto(self, data: Dict[str, Any], **_) -> ReportExecution: + def build_dto(self, data: dict[str, Any], **_) -> ReportExecution: return ReportExecution(**data) @dataclass class SbomReport: - report_executions: List[ReportExecution] + report_executions: list[ReportExecution] class RequestedSbomReportResultSchema(Schema): @@ -389,7 +389,7 @@ class Meta: report_executions = fields.List(fields.Nested(ReportExecutionSchema)) @post_load - def build_dto(self, data: Dict[str, Any], **_) -> SbomReport: + def build_dto(self, data: dict[str, Any], **_) -> SbomReport: return SbomReport(**data) @@ -405,13 +405,13 @@ class Meta: severity = fields.String() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> ClassificationData: + def build_dto(self, data: dict[str, Any], **_) -> ClassificationData: return ClassificationData(**data) @dataclass class DetectionRule: - classification_data: List[ClassificationData] + classification_data: list[ClassificationData] detection_rule_id: str custom_remediation_guidelines: Optional[str] = None remediation_guidelines: Optional[str] = None @@ -433,14 +433,14 @@ class Meta: display_name = fields.String(allow_none=True) @post_load - def build_dto(self, data: Dict[str, Any], **_) -> DetectionRule: + def build_dto(self, data: dict[str, Any], **_) -> DetectionRule: return DetectionRule(**data) @dataclass class ScanResultsSyncFlow: id: str - detection_messages: List[Dict] + detection_messages: list[dict] class ScanResultsSyncFlowSchema(Schema): @@ -451,7 +451,7 @@ class Meta: detection_messages = fields.List(fields.Dict()) @post_load - def build_dto(self, data: Dict[str, Any], **_) -> ScanResultsSyncFlow: + def build_dto(self, data: dict[str, Any], **_) -> ScanResultsSyncFlow: return ScanResultsSyncFlow(**data) @@ -489,5 +489,5 @@ class Meta: ai_large_language_model = fields.Boolean() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'SupportedModulesPreferences': + def build_dto(self, data: dict[str, Any], **_) -> 'SupportedModulesPreferences': return SupportedModulesPreferences(**data) diff --git a/cycode/cyclient/report_client.py b/cycode/cyclient/report_client.py index fa8e0c3f..e8107827 100644 --- a/cycode/cyclient/report_client.py +++ b/cycode/cyclient/report_client.py @@ -1,6 +1,6 @@ import dataclasses import json -from typing import List, Optional +from typing import Optional from requests import Response @@ -97,5 +97,5 @@ def parse_requested_sbom_report_response(response: Response) -> models.SbomRepor return models.RequestedSbomReportResultSchema().load(response.json()) @staticmethod - def parse_execution_status_response(response: Response) -> List[models.ReportExecutionSchema]: + def parse_execution_status_response(response: Response) -> list[models.ReportExecutionSchema]: return models.ReportExecutionSchema().load(response.json(), many=True) diff --git a/cycode/cyclient/scan_client.py b/cycode/cyclient/scan_client.py index 09908943..e0bf8131 100644 --- a/cycode/cyclient/scan_client.py +++ b/cycode/cyclient/scan_client.py @@ -1,6 +1,6 @@ import json from copy import deepcopy -from typing import TYPE_CHECKING, List, Set, Union +from typing import TYPE_CHECKING, Union from uuid import UUID from requests import Response @@ -135,7 +135,7 @@ def get_scan_details_path(self, scan_type: str, scan_id: str) -> str: return f'{self.get_scan_service_url_path(scan_type)}/{scan_id}' def get_scan_aggregation_report_url_path(self, aggregation_id: str, scan_type: str) -> str: - return f'{self.get_scan_service_url_path(scan_type)}' f'/reportUrlByAggregationId/{aggregation_id}' + return f'{self.get_scan_service_url_path(scan_type)}/reportUrlByAggregationId/{aggregation_id}' def get_scan_details(self, scan_type: str, scan_id: str) -> models.ScanDetailsResponse: path = self.get_scan_details_path(scan_type, scan_id) @@ -190,10 +190,10 @@ def _get_policy_type_by_scan_type(scan_type: str) -> str: return scan_type_to_policy_type[scan_type] @staticmethod - def parse_detection_rules_response(response: Response) -> List[models.DetectionRule]: + def parse_detection_rules_response(response: Response) -> list[models.DetectionRule]: return models.DetectionRuleSchema().load(response.json(), many=True) - def get_detection_rules(self, detection_rules_ids: Union[Set[str], List[str]]) -> List[models.DetectionRule]: + def get_detection_rules(self, detection_rules_ids: Union[set[str], list[str]]) -> list[models.DetectionRule]: response = self.scan_cycode_client.get( url_path=self.get_detection_rules_path(), params={'ids': detection_rules_ids}, @@ -208,7 +208,7 @@ def get_scan_detections_path(self) -> str: def get_scan_detections_list_path(self) -> str: return f'{self.get_scan_detections_path()}/detections' - def get_scan_raw_detections(self, scan_id: str) -> List[dict]: + def get_scan_raw_detections(self, scan_id: str) -> list[dict]: params = {'scan_id': scan_id} page_size = 200 diff --git a/cycode/logger.py b/cycode/logger.py index b63c796f..0ec6023f 100644 --- a/cycode/logger.py +++ b/cycode/logger.py @@ -1,6 +1,6 @@ import logging import sys -from typing import NamedTuple, Optional, Set, Union +from typing import NamedTuple, Optional, Union import click import typer @@ -42,7 +42,7 @@ class CreatedLogger(NamedTuple): control_level_in_runtime: bool -_CREATED_LOGGERS: Set[CreatedLogger] = set() +_CREATED_LOGGERS: set[CreatedLogger] = set() def get_logger_level() -> Optional[Union[int, str]]: diff --git a/poetry.lock b/poetry.lock index d0b6503d..65e6a971 100644 --- a/poetry.lock +++ b/poetry.lock @@ -50,14 +50,14 @@ chardet = ">=3.0.2" [[package]] name = "certifi" -version = "2024.8.30" +version = "2025.4.26" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" groups = ["main", "test"] files = [ - {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, - {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, + {file = "certifi-2025.4.26-py3-none-any.whl", hash = "sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3"}, + {file = "certifi-2025.4.26.tar.gz", hash = "sha256:0a816057ea3cdefcef70270d2c515e4506bbc954f417fa5ade2021213bb8f0c6"}, ] [[package]] @@ -74,129 +74,116 @@ files = [ [[package]] name = "charset-normalizer" -version = "3.4.0" +version = "3.4.1" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false -python-versions = ">=3.7.0" +python-versions = ">=3.7" groups = ["main", "test"] files = [ - {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-win32.whl", hash = "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-win32.whl", hash = "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-win32.whl", hash = "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-win32.whl", hash = "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-win32.whl", hash = "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-win32.whl", hash = "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-win32.whl", hash = "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca"}, - {file = "charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079"}, - {file = "charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:91b36a978b5ae0ee86c394f5a54d6ef44db1de0815eb43de826d41d21e4af3de"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7461baadb4dc00fd9e0acbe254e3d7d2112e7f92ced2adc96e54ef6501c5f176"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e218488cd232553829be0664c2292d3af2eeeb94b32bea483cf79ac6a694e037"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80ed5e856eb7f30115aaf94e4a08114ccc8813e6ed1b5efa74f9f82e8509858f"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b010a7a4fd316c3c484d482922d13044979e78d1861f0e0650423144c616a46a"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4532bff1b8421fd0a320463030c7520f56a79c9024a4e88f01c537316019005a"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d973f03c0cb71c5ed99037b870f2be986c3c05e63622c017ea9816881d2dd247"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3a3bd0dcd373514dcec91c411ddb9632c0d7d92aed7093b8c3bbb6d69ca74408"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:d9c3cdf5390dcd29aa8056d13e8e99526cda0305acc038b96b30352aff5ff2bb"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2bdfe3ac2e1bbe5b59a1a63721eb3b95fc9b6817ae4a46debbb4e11f6232428d"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:eab677309cdb30d047996b36d34caeda1dc91149e4fdca0b1a039b3f79d9a807"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-win32.whl", hash = "sha256:c0429126cf75e16c4f0ad00ee0eae4242dc652290f940152ca8c75c3a4b6ee8f"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:9f0b8b1c6d84c8034a44893aba5e767bf9c7a211e313a9605d9c617d7083829f"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8bfa33f4f2672964266e940dd22a195989ba31669bd84629f05fab3ef4e2d125"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28bf57629c75e810b6ae989f03c0828d64d6b26a5e205535585f96093e405ed1"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f08ff5e948271dc7e18a35641d2f11a4cd8dfd5634f55228b691e62b37125eb3"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:234ac59ea147c59ee4da87a0c0f098e9c8d169f4dc2a159ef720f1a61bbe27cd"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd4ec41f914fa74ad1b8304bbc634b3de73d2a0889bd32076342a573e0779e00"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eea6ee1db730b3483adf394ea72f808b6e18cf3cb6454b4d86e04fa8c4327a12"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c96836c97b1238e9c9e3fe90844c947d5afbf4f4c92762679acfe19927d81d77"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4d86f7aff21ee58f26dcf5ae81a9addbd914115cdebcbb2217e4f0ed8982e146"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:09b5e6733cbd160dcc09589227187e242a30a49ca5cefa5a7edd3f9d19ed53fd"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:5777ee0881f9499ed0f71cc82cf873d9a0ca8af166dfa0af8ec4e675b7df48e6"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:237bdbe6159cff53b4f24f397d43c6336c6b0b42affbe857970cefbb620911c8"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-win32.whl", hash = "sha256:8417cb1f36cc0bc7eaba8ccb0e04d55f0ee52df06df3ad55259b9a323555fc8b"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:d7f50a1f8c450f3925cb367d011448c39239bb3eb4117c36a6d354794de4ce76"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:73d94b58ec7fecbc7366247d3b0b10a21681004153238750bb67bd9012414545"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad3e487649f498dd991eeb901125411559b22e8d7ab25d3aeb1af367df5efd7"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c30197aa96e8eed02200a83fba2657b4c3acd0f0aa4bdc9f6c1af8e8962e0757"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2369eea1ee4a7610a860d88f268eb39b95cb588acd7235e02fd5a5601773d4fa"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffc9202a29ab3920fa812879e95a9e78b2465fd10be7fcbd042899695d75e616"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:804a4d582ba6e5b747c625bf1255e6b1507465494a40a2130978bda7b932c90b"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f55e69f030f7163dffe9fd0752b32f070566451afe180f99dbeeb81f511ad8d"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c4c3e6da02df6fa1410a7680bd3f63d4f710232d3139089536310d027950696a"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5df196eb874dae23dcfb968c83d4f8fdccb333330fe1fc278ac5ceeb101003a9"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e358e64305fe12299a08e08978f51fc21fac060dcfcddd95453eabe5b93ed0e1"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-win32.whl", hash = "sha256:9b23ca7ef998bc739bf6ffc077c2116917eabcc901f88da1b9856b210ef63f35"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:6ff8a4a60c227ad87030d76e99cd1698345d4491638dfa6673027c48b3cd395f"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:aabfa34badd18f1da5ec1bc2715cadc8dca465868a4e73a0173466b688f29dda"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22e14b5d70560b8dd51ec22863f370d1e595ac3d024cb8ad7d308b4cd95f8313"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8436c508b408b82d87dc5f62496973a1805cd46727c34440b0d29d8a2f50a6c9"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d074908e1aecee37a7635990b2c6d504cd4766c7bc9fc86d63f9c09af3fa11b"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:955f8851919303c92343d2f66165294848d57e9bba6cf6e3625485a70a038d11"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:44ecbf16649486d4aebafeaa7ec4c9fed8b88101f4dd612dcaf65d5e815f837f"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0924e81d3d5e70f8126529951dac65c1010cdf117bb75eb02dd12339b57749dd"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2967f74ad52c3b98de4c3b32e1a44e32975e008a9cd2a8cc8966d6a5218c5cb2"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c75cb2a3e389853835e84a2d8fb2b81a10645b503eca9bcb98df6b5a43eb8886"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:09b26ae6b1abf0d27570633b2b078a2a20419c99d66fb2823173d73f188ce601"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa88b843d6e211393a37219e6a1c1df99d35e8fd90446f1118f4216e307e48cd"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-win32.whl", hash = "sha256:eb8178fe3dba6450a3e024e95ac49ed3400e506fd4e9e5c32d30adda88cbd407"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:b1ac5992a838106edb89654e0aebfc24f5848ae2547d22c2c3f66454daa11971"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f30bf9fd9be89ecb2360c7d94a711f00c09b976258846efe40db3d05828e8089"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:97f68b8d6831127e4787ad15e6757232e14e12060bec17091b85eb1486b91d8d"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7974a0b5ecd505609e3b19742b60cee7aa2aa2fb3151bc917e6e2646d7667dcf"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc54db6c8593ef7d4b2a331b58653356cf04f67c960f584edb7c3d8c97e8f39e"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:311f30128d7d333eebd7896965bfcfbd0065f1716ec92bd5638d7748eb6f936a"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:7d053096f67cd1241601111b698f5cad775f97ab25d81567d3f59219b5f1adbd"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:807f52c1f798eef6cf26beb819eeb8819b1622ddfeef9d0977a8502d4db6d534"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:dccbe65bd2f7f7ec22c4ff99ed56faa1e9f785482b9bbd7c717e26fd723a1d1e"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:2fb9bd477fdea8684f78791a6de97a953c51831ee2981f8e4f583ff3b9d9687e"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:01732659ba9b5b873fc117534143e4feefecf3b2078b0a6a2e925271bb6f4cfa"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-win32.whl", hash = "sha256:7a4f97a081603d2050bfaffdefa5b02a9ec823f8348a572e39032caa8404a487"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7b1bef6280950ee6c177b326508f86cad7ad4dff12454483b51d8b7d673a2c5d"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ecddf25bee22fe4fe3737a399d0d177d72bc22be6913acfab364b40bce1ba83c"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c60ca7339acd497a55b0ea5d506b2a2612afb2826560416f6894e8b5770d4a9"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b7b2d86dd06bfc2ade3312a83a5c364c7ec2e3498f8734282c6c3d4b07b346b8"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd78cfcda14a1ef52584dbb008f7ac81c1328c0f58184bf9a84c49c605002da6"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e27f48bcd0957c6d4cb9d6fa6b61d192d0b13d5ef563e5f2ae35feafc0d179c"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01ad647cdd609225c5350561d084b42ddf732f4eeefe6e678765636791e78b9a"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:619a609aa74ae43d90ed2e89bdd784765de0a25ca761b93e196d938b8fd1dbbd"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:89149166622f4db9b4b6a449256291dc87a99ee53151c74cbd82a53c8c2f6ccd"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:7709f51f5f7c853f0fb938bcd3bc59cdfdc5203635ffd18bf354f6967ea0f824"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:345b0426edd4e18138d6528aed636de7a9ed169b4aaf9d61a8c19e39d26838ca"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0907f11d019260cdc3f94fbdb23ff9125f6b5d1039b76003b5b0ac9d6a6c9d5b"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-win32.whl", hash = "sha256:ea0d8d539afa5eb2728aa1932a988a9a7af94f18582ffae4bc10b3fbdad0626e"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:329ce159e82018d646c7ac45b01a430369d526569ec08516081727a20e9e4af4"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b97e690a2118911e39b4042088092771b4ae3fc3aa86518f84b8cf6888dbdb41"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78baa6d91634dfb69ec52a463534bc0df05dbd546209b79a3880a34487f4b84f"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a2bc9f351a75ef49d664206d51f8e5ede9da246602dc2d2726837620ea034b2"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75832c08354f595c760a804588b9357d34ec00ba1c940c15e31e96d902093770"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0af291f4fe114be0280cdd29d533696a77b5b49cfde5467176ecab32353395c4"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0167ddc8ab6508fe81860a57dd472b2ef4060e8d378f0cc555707126830f2537"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2a75d49014d118e4198bcee5ee0a6f25856b29b12dbf7cd012791f8a6cc5c496"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:363e2f92b0f0174b2f8238240a1a30142e3db7b957a5dd5689b0e75fb717cc78"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ab36c8eb7e454e34e60eb55ca5d241a5d18b2c6244f6827a30e451c42410b5f7"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4c0907b1928a36d5a998d72d64d8eaa7244989f7aaaf947500d3a800c83a3fd6"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:04432ad9479fa40ec0f387795ddad4437a2b50417c69fa275e212933519ff294"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-win32.whl", hash = "sha256:3bed14e9c89dcb10e8f3a29f9ccac4955aebe93c71ae803af79265c9ca5644c5"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:49402233c892a461407c512a19435d1ce275543138294f7ef013f0b63d5d3765"}, + {file = "charset_normalizer-3.4.1-py3-none-any.whl", hash = "sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85"}, + {file = "charset_normalizer-3.4.1.tar.gz", hash = "sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3"}, ] [[package]] name = "click" -version = "8.1.7" +version = "8.1.8" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, - {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, + {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, + {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, ] [package.dependencies] @@ -320,14 +307,14 @@ test = ["pytest (>=6)"] [[package]] name = "gitdb" -version = "4.0.11" +version = "4.0.12" description = "Git Object Database" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4"}, - {file = "gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b"}, + {file = "gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf"}, + {file = "gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571"}, ] [package.dependencies] @@ -335,21 +322,21 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.43" +version = "3.1.44" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff"}, - {file = "GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c"}, + {file = "GitPython-3.1.44-py3-none-any.whl", hash = "sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110"}, + {file = "gitpython-3.1.44.tar.gz", hash = "sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269"}, ] [package.dependencies] gitdb = ">=4.0.1,<5" [package.extras] -doc = ["sphinx (==4.3.2)", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "sphinxcontrib-applehelp (>=1.0.2,<=1.0.4)", "sphinxcontrib-devhelp (==1.0.2)", "sphinxcontrib-htmlhelp (>=2.0.0,<=2.0.1)", "sphinxcontrib-qthelp (==1.0.3)", "sphinxcontrib-serializinghtml (==1.1.5)"] +doc = ["sphinx (>=7.1.2,<7.2)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] [[package]] @@ -369,15 +356,15 @@ all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2 [[package]] name = "importlib-metadata" -version = "8.5.0" +version = "8.7.0" description = "Read metadata from Python packages" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["executable"] markers = "python_version < \"3.10\"" files = [ - {file = "importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b"}, - {file = "importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7"}, + {file = "importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd"}, + {file = "importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000"}, ] [package.dependencies] @@ -389,19 +376,19 @@ cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] enabler = ["pytest-enabler (>=2.2)"] perf = ["ipython"] -test = ["flufl.flake8", "importlib-resources (>=1.3) ; python_version < \"3.9\"", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] +test = ["flufl.flake8", "importlib_resources (>=1.3) ; python_version < \"3.9\"", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] type = ["pytest-mypy"] [[package]] name = "iniconfig" -version = "2.0.0" +version = "2.1.0" description = "brain-dead simple config-ini parsing" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" groups = ["test"] files = [ - {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, - {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, + {file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"}, + {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, ] [[package]] @@ -496,14 +483,14 @@ test = ["pytest (<5.4)", "pytest-cov"] [[package]] name = "packaging" -version = "24.2" +version = "25.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" groups = ["main", "executable", "test"] files = [ - {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, - {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, + {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, + {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, ] [[package]] @@ -548,26 +535,26 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "pyfakefs" -version = "5.7.2" +version = "5.7.4" description = "pyfakefs implements a fake file system that mocks the Python file system modules." optional = false python-versions = ">=3.7" groups = ["test"] files = [ - {file = "pyfakefs-5.7.2-py3-none-any.whl", hash = "sha256:e1527b0e8e4b33be52f0b024ca1deb269c73eecd68457c6b0bf608d6dab12ebd"}, - {file = "pyfakefs-5.7.2.tar.gz", hash = "sha256:40da84175c5af8d9c4f3b31800b8edc4af1e74a212671dd658b21cc881c60000"}, + {file = "pyfakefs-5.7.4-py3-none-any.whl", hash = "sha256:3e763d700b91c54ade6388be2cfa4e521abc00e34f7defb84ee511c73031f45f"}, + {file = "pyfakefs-5.7.4.tar.gz", hash = "sha256:4971e65cc80a93a1e6f1e3a4654909c0c493186539084dc9301da3d68c8878fe"}, ] [[package]] name = "pygments" -version = "2.18.0" +version = "2.19.1" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, - {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, + {file = "pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c"}, + {file = "pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f"}, ] [package.extras] @@ -610,32 +597,32 @@ hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] [[package]] name = "pyinstaller-hooks-contrib" -version = "2024.10" +version = "2025.3" description = "Community maintained hooks for PyInstaller" optional = false python-versions = ">=3.8" groups = ["executable"] markers = "python_version < \"3.13\"" files = [ - {file = "pyinstaller_hooks_contrib-2024.10-py3-none-any.whl", hash = "sha256:ad47db0e153683b4151e10d231cb91f2d93c85079e78d76d9e0f57ac6c8a5e10"}, - {file = "pyinstaller_hooks_contrib-2024.10.tar.gz", hash = "sha256:8a46655e5c5b0186b5e527399118a9b342f10513eb1425c483fa4f6d02e8800c"}, + {file = "pyinstaller_hooks_contrib-2025.3-py3-none-any.whl", hash = "sha256:70cba46b1a6b82ae9104f074c25926e31f3dde50ff217434d1d660355b949683"}, + {file = "pyinstaller_hooks_contrib-2025.3.tar.gz", hash = "sha256:af129da5cd6219669fbda360e295cc822abac55b7647d03fec63a8fcf0a608cf"}, ] [package.dependencies] -importlib-metadata = {version = ">=4.6", markers = "python_version < \"3.10\""} +importlib_metadata = {version = ">=4.6", markers = "python_version < \"3.10\""} packaging = ">=22.0" setuptools = ">=42.0.0" [[package]] name = "pyjwt" -version = "2.9.0" +version = "2.10.1" description = "JSON Web Token implementation in Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850"}, - {file = "pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"}, + {file = "PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb"}, + {file = "pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953"}, ] [package.extras] @@ -841,42 +828,42 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "ruff" -version = "0.6.9" +version = "0.11.7" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "ruff-0.6.9-py3-none-linux_armv6l.whl", hash = "sha256:064df58d84ccc0ac0fcd63bc3090b251d90e2a372558c0f057c3f75ed73e1ccd"}, - {file = "ruff-0.6.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:140d4b5c9f5fc7a7b074908a78ab8d384dd7f6510402267bc76c37195c02a7ec"}, - {file = "ruff-0.6.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:53fd8ca5e82bdee8da7f506d7b03a261f24cd43d090ea9db9a1dc59d9313914c"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:645d7d8761f915e48a00d4ecc3686969761df69fb561dd914a773c1a8266e14e"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eae02b700763e3847595b9d2891488989cac00214da7f845f4bcf2989007d577"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d5ccc9e58112441de8ad4b29dcb7a86dc25c5f770e3c06a9d57e0e5eba48829"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:417b81aa1c9b60b2f8edc463c58363075412866ae4e2b9ab0f690dc1e87ac1b5"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c866b631f5fbce896a74a6e4383407ba7507b815ccc52bcedabb6810fdb3ef7"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7b118afbb3202f5911486ad52da86d1d52305b59e7ef2031cea3425142b97d6f"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a67267654edc23c97335586774790cde402fb6bbdb3c2314f1fc087dee320bfa"}, - {file = "ruff-0.6.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3ef0cc774b00fec123f635ce5c547dac263f6ee9fb9cc83437c5904183b55ceb"}, - {file = "ruff-0.6.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:12edd2af0c60fa61ff31cefb90aef4288ac4d372b4962c2864aeea3a1a2460c0"}, - {file = "ruff-0.6.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:55bb01caeaf3a60b2b2bba07308a02fca6ab56233302406ed5245180a05c5625"}, - {file = "ruff-0.6.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:925d26471fa24b0ce5a6cdfab1bb526fb4159952385f386bdcc643813d472039"}, - {file = "ruff-0.6.9-py3-none-win32.whl", hash = "sha256:eb61ec9bdb2506cffd492e05ac40e5bc6284873aceb605503d8494180d6fc84d"}, - {file = "ruff-0.6.9-py3-none-win_amd64.whl", hash = "sha256:785d31851c1ae91f45b3d8fe23b8ae4b5170089021fbb42402d811135f0b7117"}, - {file = "ruff-0.6.9-py3-none-win_arm64.whl", hash = "sha256:a9641e31476d601f83cd602608739a0840e348bda93fec9f1ee816f8b6798b93"}, - {file = "ruff-0.6.9.tar.gz", hash = "sha256:b076ef717a8e5bc819514ee1d602bbdca5b4420ae13a9cf61a0c0a4f53a2baa2"}, + {file = "ruff-0.11.7-py3-none-linux_armv6l.whl", hash = "sha256:d29e909d9a8d02f928d72ab7837b5cbc450a5bdf578ab9ebee3263d0a525091c"}, + {file = "ruff-0.11.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:dd1fb86b168ae349fb01dd497d83537b2c5541fe0626e70c786427dd8363aaee"}, + {file = "ruff-0.11.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d3d7d2e140a6fbbc09033bce65bd7ea29d6a0adeb90b8430262fbacd58c38ada"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4809df77de390a1c2077d9b7945d82f44b95d19ceccf0c287c56e4dc9b91ca64"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f3a0c2e169e6b545f8e2dba185eabbd9db4f08880032e75aa0e285a6d3f48201"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49b888200a320dd96a68e86736cf531d6afba03e4f6cf098401406a257fcf3d6"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:2b19cdb9cf7dae00d5ee2e7c013540cdc3b31c4f281f1dacb5a799d610e90db4"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:64e0ee994c9e326b43539d133a36a455dbaab477bc84fe7bfbd528abe2f05c1e"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bad82052311479a5865f52c76ecee5d468a58ba44fb23ee15079f17dd4c8fd63"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7940665e74e7b65d427b82bffc1e46710ec7f30d58b4b2d5016e3f0321436502"}, + {file = "ruff-0.11.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:169027e31c52c0e36c44ae9a9c7db35e505fee0b39f8d9fca7274a6305295a92"}, + {file = "ruff-0.11.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:305b93f9798aee582e91e34437810439acb28b5fc1fee6b8205c78c806845a94"}, + {file = "ruff-0.11.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a681db041ef55550c371f9cd52a3cf17a0da4c75d6bd691092dfc38170ebc4b6"}, + {file = "ruff-0.11.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:07f1496ad00a4a139f4de220b0c97da6d4c85e0e4aa9b2624167b7d4d44fd6b6"}, + {file = "ruff-0.11.7-py3-none-win32.whl", hash = "sha256:f25dfb853ad217e6e5f1924ae8a5b3f6709051a13e9dad18690de6c8ff299e26"}, + {file = "ruff-0.11.7-py3-none-win_amd64.whl", hash = "sha256:0a931d85959ceb77e92aea4bbedfded0a31534ce191252721128f77e5ae1f98a"}, + {file = "ruff-0.11.7-py3-none-win_arm64.whl", hash = "sha256:778c1e5d6f9e91034142dfd06110534ca13220bfaad5c3735f6cb844654f6177"}, + {file = "ruff-0.11.7.tar.gz", hash = "sha256:655089ad3224070736dc32844fde783454f8558e71f501cb207485fe4eee23d4"}, ] [[package]] name = "sentry-sdk" -version = "2.19.2" +version = "2.27.0" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = ">=3.6" groups = ["main"] files = [ - {file = "sentry_sdk-2.19.2-py2.py3-none-any.whl", hash = "sha256:ebdc08228b4d131128e568d696c210d846e5b9d70aa0327dec6b1272d9d40b84"}, - {file = "sentry_sdk-2.19.2.tar.gz", hash = "sha256:467df6e126ba242d39952375dd816fbee0f217d119bf454a8ce74cf1e7909e8d"}, + {file = "sentry_sdk-2.27.0-py2.py3-none-any.whl", hash = "sha256:c58935bfff8af6a0856d37e8adebdbc7b3281c2b632ec823ef03cd108d216ff0"}, + {file = "sentry_sdk-2.27.0.tar.gz", hash = "sha256:90f4f883f9eff294aff59af3d58c2d1b64e3927b28d5ada2b9b41f5aeda47daf"}, ] [package.dependencies] @@ -920,29 +907,31 @@ sanic = ["sanic (>=0.8)"] sqlalchemy = ["sqlalchemy (>=1.2)"] starlette = ["starlette (>=0.19.1)"] starlite = ["starlite (>=1.48)"] +statsig = ["statsig (>=0.55.3)"] tornado = ["tornado (>=6)"] +unleash = ["UnleashClient (>=6.0.1)"] [[package]] name = "setuptools" -version = "75.3.0" +version = "80.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["executable"] markers = "python_version < \"3.13\"" files = [ - {file = "setuptools-75.3.0-py3-none-any.whl", hash = "sha256:f2504966861356aa38616760c0f66568e535562374995367b4e69c7143cf6bcd"}, - {file = "setuptools-75.3.0.tar.gz", hash = "sha256:fba5dd4d766e97be1b1681d98712680ae8f2f26d7881245f2ce9e40714f1a686"}, + {file = "setuptools-80.0.0-py3-none-any.whl", hash = "sha256:a38f898dcd6e5380f4da4381a87ec90bd0a7eec23d204a5552e80ee3cab6bd27"}, + {file = "setuptools-80.0.0.tar.gz", hash = "sha256:c40a5b3729d58dd749c0f08f1a07d134fb8a0a3d7f87dc33e7c5e1f762138650"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.5.2) ; sys_platform != \"cygwin\""] -core = ["importlib-metadata (>=6) ; python_version < \"3.10\"", "importlib-resources (>=5.10.2) ; python_version < \"3.9\"", "jaraco.collections", "jaraco.functools", "jaraco.text (>=3.7)", "more-itertools", "more-itertools (>=8.8)", "packaging", "packaging (>=24)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib-metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.12.*)", "pytest-mypy"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.14.*)", "pytest-mypy"] [[package]] name = "shellingham" @@ -970,14 +959,14 @@ files = [ [[package]] name = "smmap" -version = "5.0.1" +version = "5.0.2" description = "A pure Python implementation of a sliding window memory map manager" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"}, - {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, + {file = "smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e"}, + {file = "smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5"}, ] [[package]] @@ -1041,14 +1030,14 @@ files = [ [[package]] name = "typer" -version = "0.15.2" +version = "0.15.3" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "typer-0.15.2-py3-none-any.whl", hash = "sha256:46a499c6107d645a9c13f7ee46c5d5096cae6f5fc57dd11eccbbb9ae3e44ddfc"}, - {file = "typer-0.15.2.tar.gz", hash = "sha256:ab2fab47533a813c49fe1f16b1a370fd5819099c00b119e0633df65f22144ba5"}, + {file = "typer-0.15.3-py3-none-any.whl", hash = "sha256:c86a65ad77ca531f03de08d1b9cb67cd09ad02ddddf4b34745b5008f43b239bd"}, + {file = "typer-0.15.3.tar.gz", hash = "sha256:818873625d0569653438316567861899f7e9972f2e6e0c16dab608345ced713c"}, ] [package.dependencies] @@ -1071,26 +1060,26 @@ files = [ [[package]] name = "types-pyyaml" -version = "6.0.12.20240917" +version = "6.0.12.20250402" description = "Typing stubs for PyYAML" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["test"] files = [ - {file = "types-PyYAML-6.0.12.20240917.tar.gz", hash = "sha256:d1405a86f9576682234ef83bcb4e6fff7c9305c8b1fbad5e0bcd4f7dbdc9c587"}, - {file = "types_PyYAML-6.0.12.20240917-py3-none-any.whl", hash = "sha256:392b267f1c0fe6022952462bf5d6523f31e37f6cea49b14cee7ad634b6301570"}, + {file = "types_pyyaml-6.0.12.20250402-py3-none-any.whl", hash = "sha256:652348fa9e7a203d4b0d21066dfb00760d3cbd5a15ebb7cf8d33c88a49546681"}, + {file = "types_pyyaml-6.0.12.20250402.tar.gz", hash = "sha256:d7c13c3e6d335b6af4b0122a01ff1d270aba84ab96d1a1a1063ecba3e13ec075"}, ] [[package]] name = "typing-extensions" -version = "4.12.2" +version = "4.13.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, - {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, + {file = "typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c"}, + {file = "typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef"}, ] [[package]] @@ -1112,15 +1101,15 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "zipp" -version = "3.20.2" +version = "3.21.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["executable"] markers = "python_version < \"3.10\"" files = [ - {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, - {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, + {file = "zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931"}, + {file = "zipp-3.21.0.tar.gz", hash = "sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4"}, ] [package.extras] @@ -1134,4 +1123,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<3.14" -content-hash = "590be7f6a392d52a8d298596ef95e6ee664a8a3515530b01d727fe268e15fb0d" +content-hash = "14f258101aa534aadfc871aa5082ad773aa99873587c21c0598567435bfa5d9a" diff --git a/process_executable_file.py b/process_executable_file.py index ad4d702a..367bb18d 100755 --- a/process_executable_file.py +++ b/process_executable_file.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -""" -Used in the GitHub Actions workflow (build_executable.yml) to process the executable file. +"""Used in the GitHub Actions workflow (build_executable.yml) to process the executable file. + This script calculates hash and renames executable file depending on the OS, arch, and build mode. It also creates a file with the hash of the executable file. It uses SHA256 algorithm to calculate the hash. @@ -15,7 +15,7 @@ import shutil from pathlib import Path from string import Template -from typing import List, Tuple, Union +from typing import Union _ARCHIVE_FORMAT = 'zip' _HASH_FILE_EXT = '.sha256' @@ -27,7 +27,7 @@ _WINDOWS = 'windows' _WINDOWS_EXECUTABLE_SUFFIX = '.exe' -DirHashes = List[Tuple[str, str]] +DirHashes = list[tuple[str, str]] def get_hash_of_file(file_path: Union[str, Path]) -> str: @@ -35,7 +35,7 @@ def get_hash_of_file(file_path: Union[str, Path]) -> str: return hashlib.sha256(f.read()).hexdigest() -def get_hashes_of_many_files(root: str, file_paths: List[str]) -> DirHashes: +def get_hashes_of_many_files(root: str, file_paths: list[str]) -> DirHashes: hashes = [] for file_path in file_paths: diff --git a/pyproject.toml b/pyproject.toml index cde794b7..755d8207 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ sentry-sdk = ">=2.8.0,<3.0" pyjwt = ">=2.8.0,<3.0" rich = ">=13.9.4, <14" patch-ng = "1.18.1" -typer = "^0.15.2" +typer = "^0.15.3" tenacity = ">=9.0.0,<9.1.0" [tool.poetry.group.test.dependencies] @@ -56,7 +56,7 @@ pyinstaller = {version=">=5.13.2,<5.14.0", python=">=3.8,<3.13"} dunamai = ">=1.18.0,<1.22.0" [tool.poetry.group.dev.dependencies] -ruff = "0.6.9" +ruff = "0.11.7" [tool.pytest.ini_options] log_cli = true @@ -73,7 +73,7 @@ style = "pep440" [tool.ruff] line-length = 120 -target-version = "py38" +target-version = "py39" [tool.ruff.lint] extend-select = [ @@ -81,6 +81,7 @@ extend-select = [ "W", # pycodestyle warnings "F", # Pyflakes "I", # isort + "N", # pep8 naming "C90", # flake8-comprehensions "B", # flake8-bugbear "Q", # flake8-quotes @@ -100,19 +101,26 @@ extend-select = [ "RSE", "RUF", "SIM", + "T10", "T20", - "TCH", "TID", "YTT", + "LOG", "G", + "UP", + "DTZ", + "PYI", + "PT", + "SLOT", + "TC", ] ignore = [ "ANN002", # Missing type annotation for `*args` "ANN003", # Missing type annotation for `**kwargs` - "ANN101", # Missing type annotation for `self` in method - "ANN102", # Missing type annotation for `cls` in classmethod "ANN401", # Dynamically typed expressions (typing.Any) "ISC001", # Conflicts with ruff format + "S105", # False positives + "PT012", # `pytest.raises()` block should contain a single simple statement ] [tool.ruff.lint.flake8-quotes] diff --git a/tests/cli/commands/version/test_version_checker.py b/tests/cli/commands/version/test_version_checker.py index 926a21e8..14d6150e 100644 --- a/tests/cli/commands/version/test_version_checker.py +++ b/tests/cli/commands/version/test_version_checker.py @@ -71,7 +71,7 @@ def test_should_check_update_prerelease_daily(self, version_checker_cached: 'Ver assert version_checker_cached._should_check_update(is_prerelease=True) is True @pytest.mark.parametrize( - 'current_version, latest_version, expected_result', + ('current_version', 'latest_version', 'expected_result'), [ # Stable version comparisons ('1.2.3', '1.2.4', '1.2.4'), # Higher patch version diff --git a/tests/cli/exceptions/test_handle_scan_errors.py b/tests/cli/exceptions/test_handle_scan_errors.py index abd297db..ce72e9de 100644 --- a/tests/cli/exceptions/test_handle_scan_errors.py +++ b/tests/cli/exceptions/test_handle_scan_errors.py @@ -17,7 +17,7 @@ from _pytest.monkeypatch import MonkeyPatch -@pytest.fixture() +@pytest.fixture def ctx() -> typer.Context: ctx = typer.Context(click.Command('path'), obj={'verbose': False, 'output': OutputTypeOption.TEXT}) ctx.obj['console_printer'] = ConsolePrinter(ctx) @@ -25,7 +25,7 @@ def ctx() -> typer.Context: @pytest.mark.parametrize( - 'exception, expected_soft_fail', + ('exception', 'expected_soft_fail'), [ (custom_exceptions.RequestHttpError(400, 'msg', Response()), True), (custom_exceptions.ScanAsyncError('msg'), True), diff --git a/tests/cli/files_collector/test_walk_ignore.py b/tests/cli/files_collector/test_walk_ignore.py index b771cdf9..12b9d428 100644 --- a/tests/cli/files_collector/test_walk_ignore.py +++ b/tests/cli/files_collector/test_walk_ignore.py @@ -1,6 +1,6 @@ import os from os.path import normpath -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from cycode.cli.files_collector.walk_ignore import ( _collect_top_level_ignore_files, @@ -95,7 +95,7 @@ def test_collect_top_level_ignore_files(fs: 'FakeFilesystem') -> None: fs.create_file('/home/user/project/.gitignore', contents='*.pyc\n*.log') -def _collect_walk_ignore_files(path: str) -> List[str]: +def _collect_walk_ignore_files(path: str) -> list[str]: files = [] for root, _, filenames in walk_ignore(path): for filename in filenames: diff --git a/tests/cyclient/test_auth_client.py b/tests/cyclient/test_auth_client.py index 67147a6e..24d9b096 100644 --- a/tests/cyclient/test_auth_client.py +++ b/tests/cyclient/test_auth_client.py @@ -4,7 +4,7 @@ from requests import Timeout from cycode.cli.apps.auth.auth_manager import AuthManager -from cycode.cli.exceptions.custom_exceptions import CycodeError, RequestTimeout +from cycode.cli.exceptions.custom_exceptions import CycodeError, RequestTimeoutError from cycode.cyclient.auth_client import AuthClient from cycode.cyclient.models import ( ApiTokenGenerationPollingResponse, @@ -73,7 +73,7 @@ def test_start_session_timeout(client: AuthClient, start_url: str, code_challeng responses.add(responses.POST, start_url, body=timeout_error) - with pytest.raises(RequestTimeout): + with pytest.raises(RequestTimeoutError): client.start_session(code_challenge) diff --git a/tests/cyclient/test_scan_client.py b/tests/cyclient/test_scan_client.py index d81116fb..d6928118 100644 --- a/tests/cyclient/test_scan_client.py +++ b/tests/cyclient/test_scan_client.py @@ -1,5 +1,4 @@ import os -from typing import List, Tuple from uuid import uuid4 import pytest @@ -12,7 +11,7 @@ CycodeError, HttpUnauthorizedError, RequestConnectionError, - RequestTimeout, + RequestTimeoutError, ) from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip from cycode.cli.models import Document @@ -28,7 +27,7 @@ ) -def zip_scan_resources(scan_type: ScanTypeOption, scan_client: ScanClient) -> Tuple[str, InMemoryZip]: +def zip_scan_resources(scan_type: ScanTypeOption, scan_client: ScanClient) -> tuple[str, InMemoryZip]: url = get_zipped_file_scan_async_url(scan_type, scan_client) zip_file = get_test_zip_file(scan_type) @@ -37,11 +36,11 @@ def zip_scan_resources(scan_type: ScanTypeOption, scan_client: ScanClient) -> Tu def get_test_zip_file(scan_type: ScanTypeOption) -> InMemoryZip: # TODO(MarshalX): refactor scan_disk_files in code_scanner.py to reuse method here instead of this - test_documents: List[Document] = [] + test_documents: list[Document] = [] for root, _, files in os.walk(ZIP_CONTENT_PATH): for name in files: path = os.path.join(root, name) - with open(path, 'r', encoding='UTF-8') as f: + with open(path, encoding='UTF-8') as f: test_documents.append(Document(path, f.read(), is_git_diff_format=False)) from cycode.cli.files_collector.zip_documents import zip_documents @@ -132,7 +131,7 @@ def test_zipped_file_scan_async_timeout_error( responses.add(api_token_response) # mock token based client responses.add(method=responses.POST, url=url, body=timeout_error) - with pytest.raises(RequestTimeout): + with pytest.raises(RequestTimeoutError): scan_client.zipped_file_scan_async(zip_file=zip_file, scan_type=scan_type, scan_parameters={}) diff --git a/tests/test_performance_get_all_files.py b/tests/test_performance_get_all_files.py index 60155261..b0e8653d 100644 --- a/tests/test_performance_get_all_files.py +++ b/tests/test_performance_get_all_files.py @@ -3,17 +3,17 @@ import os import timeit from pathlib import Path -from typing import Dict, List, Tuple, Union +from typing import Union logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) -def filter_files(paths: List[Union[Path, str]]) -> List[str]: +def filter_files(paths: list[Union[Path, str]]) -> list[str]: return [str(path) for path in paths if os.path.isfile(path)] -def get_all_files_glob(path: Union[Path, str]) -> List[str]: +def get_all_files_glob(path: Union[Path, str]) -> list[str]: # DOESN'T RETURN HIDDEN FILES. CAN'T BE USED # and doesn't show the best performance if not str(path).endswith(os.sep): @@ -22,7 +22,7 @@ def get_all_files_glob(path: Union[Path, str]) -> List[str]: return filter_files(glob.glob(f'{path}**', recursive=True)) -def get_all_files_walk(path: str) -> List[str]: +def get_all_files_walk(path: str) -> list[str]: files = [] for root, _, filenames in os.walk(path): @@ -32,7 +32,7 @@ def get_all_files_walk(path: str) -> List[str]: return files -def get_all_files_listdir(path: str) -> List[str]: +def get_all_files_listdir(path: str) -> list[str]: files = [] def _(sub_path: str) -> None: @@ -50,12 +50,12 @@ def _(sub_path: str) -> None: return files -def get_all_files_rglob(path: str) -> List[str]: +def get_all_files_rglob(path: str) -> list[str]: return filter_files(list(Path(path).rglob(r'*'))) def test_get_all_files_performance(test_files_path: str) -> None: - results: Dict[str, Tuple[int, float]] = {} + results: dict[str, tuple[int, float]] = {} for func in { get_all_files_rglob, get_all_files_listdir, diff --git a/tests/user_settings/test_configuration_manager.py b/tests/user_settings/test_configuration_manager.py index 50251340..5aa7f6a8 100644 --- a/tests/user_settings/test_configuration_manager.py +++ b/tests/user_settings/test_configuration_manager.py @@ -1,6 +1,5 @@ from typing import TYPE_CHECKING, Optional - -from mock import Mock +from unittest.mock import Mock from cycode.cli.consts import DEFAULT_CYCODE_API_URL from cycode.cli.user_settings.configuration_manager import ConfigurationManager diff --git a/tests/utils/test_ignore_utils.py b/tests/utils/test_ignore_utils.py index 563c11a9..6988e1aa 100644 --- a/tests/utils/test_ignore_utils.py +++ b/tests/utils/test_ignore_utils.py @@ -87,9 +87,9 @@ def test_translate(self) -> None: for pattern, regex in TRANSLATE_TESTS: if re.escape(b'/') == b'/': regex = regex.replace(b'\\/', b'/') - assert ( - translate(pattern) == regex - ), f'orig pattern: {pattern!r}, regex: {translate(pattern)!r}, expected: {regex!r}' + assert translate(pattern) == regex, ( + f'orig pattern: {pattern!r}, regex: {translate(pattern)!r}, expected: {regex!r}' + ) def test_read_file(self) -> None: f = BytesIO( From f29a3825dbb0900e9aef2389c4da2757ef5890d1 Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Wed, 30 Apr 2025 16:47:50 +0200 Subject: [PATCH 15/19] Make changes in CLI v3.0.0 after feedback (part 2) (#303) --- README.md | 32 ++------- cycode/__main__.py | 4 ++ cycode/cli/__main__.py | 3 - cycode/cli/app.py | 47 +++++------- cycode/cli/apps/ai_remediation/__init__.py | 6 +- .../ai_remediation/ai_remediation_command.py | 2 +- cycode/cli/apps/auth/__init__.py | 7 +- cycode/cli/apps/auth/auth_common.py | 5 +- cycode/cli/apps/configure/__init__.py | 7 +- cycode/cli/apps/report/__init__.py | 2 +- .../cli/apps/report/sbom/path/path_command.py | 2 +- .../repository_url/repository_url_command.py | 2 +- cycode/cli/apps/scan/__init__.py | 7 +- cycode/cli/apps/scan/code_scanner.py | 11 ++- cycode/cli/apps/scan/scan_command.py | 72 ++++++++++++------- cycode/cli/apps/status/get_cli_status.py | 2 +- cycode/cli/printers/console_printer.py | 43 ++++++----- cycode/cli/printers/printer_base.py | 7 +- cycode/cli/printers/rich_printer.py | 2 + .../cli/printers/tables/sca_table_printer.py | 11 ++- cycode/cli/printers/tables/table_printer.py | 5 +- .../cli/printers/tables/table_printer_base.py | 36 +++------- cycode/cli/printers/text_printer.py | 11 ++- cycode/cli/printers/utils/__init__.py | 8 +++ .../cli/printers/utils/code_snippet_syntax.py | 36 +++++----- cycode/cli/printers/utils/detection_data.py | 22 +++++- cycode/cli/utils/get_api_client.py | 15 ++-- cycode/cli/utils/version_checker.py | 10 +-- cycode/cyclient/scan_client.py | 6 -- .../test_check_latest_version_on_close.py | 2 +- tests/test_code_scanner.py | 11 ++- 31 files changed, 217 insertions(+), 219 deletions(-) create mode 100644 cycode/__main__.py delete mode 100644 cycode/cli/__main__.py diff --git a/README.md b/README.md index fbe5c6a6..7966361b 100644 --- a/README.md +++ b/README.md @@ -21,10 +21,9 @@ This guide walks you through both installation and usage. 1. [Options](#options) 1. [Severity Threshold](#severity-option) 2. [Monitor](#monitor-option) - 3. [Report](#report-option) - 4. [Package Vulnerabilities](#package-vulnerabilities-option) - 5. [License Compliance](#license-compliance-option) - 6. [Lock Restore](#lock-restore-option) + 3. [Package Vulnerabilities](#package-vulnerabilities-option) + 4. [License Compliance](#license-compliance-option) + 5. [Lock Restore](#lock-restore-option) 2. [Repository Scan](#repository-scan) 1. [Branch Option](#branch-option) 3. [Path Scan](#path-scan) @@ -282,7 +281,7 @@ The following are the options and commands available with the Cycode CLI applica | [configure](#using-the-configure-command) | Initial command to configure your CLI client authentication. | | [ignore](#ignoring-scan-results) | Ignores a specific value, path or rule ID. | | [scan](#running-a-scan) | Scan the content for Secrets/IaC/SCA/SAST violations. You`ll need to specify which scan type to perform: commit-history/path/repository/etc. | -| [report](#report-command) | Generate report. You`ll need to specify which report type to perform. | +| [report](#report-command) | Generate report. You`ll need to specify which report type to perform as SBOM. | | status | Show the CLI status and exit. | # Scan Command @@ -301,7 +300,6 @@ The Cycode CLI application offers several types of scans so that you can choose | `--severity-threshold [INFO\|LOW\|MEDIUM\|HIGH\|CRITICAL]` | Show only violations at the specified level or higher. | | `--sca-scan` | Specify the SCA scan you wish to execute (`package-vulnerabilities`/`license-compliance`). The default is both. | | `--monitor` | When specified, the scan results will be recorded in the knowledge graph. Please note that when working in `monitor` mode, the knowledge graph will not be updated as a result of SCM events (Push, Repo creation). (Supported for SCA scan type only). | -| `--report` | When specified, a violations report will be generated. A URL link to the report will be printed as an output to the command execution. | | `--no-restore` | When specified, Cycode will not run restore command. Will scan direct dependencies ONLY! | | `--gradle-all-sub-projects` | When specified, Cycode will run gradle restore command for all sub projects. Should run from root project directory ONLY! | | `--help` | Show options for given command. | @@ -339,28 +337,6 @@ When using this option, the scan results from this scan will appear in the knowl > [!WARNING] > You must be an `owner` or an `admin` in Cycode to view the knowledge graph page. -#### Report Option - -> [!NOTE] -> This option is not available to IaC scans. - -To push scan results tied to the [SCA policies](https://docs.cycode.com/docs/sca-policies) found in the Repository scan to Cycode, add the argument `--report` to the scan command. - -`cycode scan -t sca --report repository ~/home/git/codebase` - -In the same way, you can push scan results of Secrets and SAST scans to Cycode by adding the `--report` option to the scan command. - -When using this option, the scan results from this scan will appear in the On-Demand Scans section of Cycode. To get to this page, click the link that appears after the printed results: - -> [!WARNING] -> You must be an `owner` or an `admin` in Cycode to view this page. - -![cli-report](https://raw.githubusercontent.com/cycodehq/cycode-cli/main/images/sca_report_url.png) - -The report page will look something like below: - -![](https://raw.githubusercontent.com/cycodehq/cycode-cli/main/images/scan_details.png) - #### Package Vulnerabilities Option > [!NOTE] diff --git a/cycode/__main__.py b/cycode/__main__.py new file mode 100644 index 00000000..7ad8ef7e --- /dev/null +++ b/cycode/__main__.py @@ -0,0 +1,4 @@ +from cycode.cli.consts import PROGRAM_NAME +from cycode.cli.main import app + +app(prog_name=PROGRAM_NAME) diff --git a/cycode/cli/__main__.py b/cycode/cli/__main__.py deleted file mode 100644 index dad7ac12..00000000 --- a/cycode/cli/__main__.py +++ /dev/null @@ -1,3 +0,0 @@ -from cycode.cli.main import app - -app() diff --git a/cycode/cli/app.py b/cycode/cli/app.py index b07b3221..6fd4f70d 100644 --- a/cycode/cli/app.py +++ b/cycode/cli/app.py @@ -1,14 +1,14 @@ import logging -from pathlib import Path from typing import Annotated, Optional import typer from typer import rich_utils +from typer._completion_classes import completion_init from typer.completion import install_callback, show_callback from cycode import __version__ from cycode.cli.apps import ai_remediation, auth, configure, ignore, report, scan, status -from cycode.cli.cli_types import ExportTypeOption, OutputTypeOption +from cycode.cli.cli_types import OutputTypeOption from cycode.cli.consts import CLI_CONTEXT_SETTINGS from cycode.cli.printers import ConsolePrinter from cycode.cli.user_settings.configuration_manager import ConfigurationManager @@ -24,14 +24,10 @@ # By default, it uses blue color which is too dark for some terminals rich_utils.RICH_HELP = "Try [cyan]'{command_path} {help_option}'[/] for help." +completion_init() # DO NOT TOUCH; this is required for the completion to work properly _cycode_cli_docs = 'https://github.com/cycodehq/cycode-cli/blob/main/README.md' -_cycode_cli_epilog = f"""[bold]Documentation[/] - - - -For more details and advanced usage, visit: [link={_cycode_cli_docs}]{_cycode_cli_docs}[/link] -""" +_cycode_cli_epilog = f'[bold]Documentation:[/] [link={_cycode_cli_docs}]{_cycode_cli_docs}[/link]' app = typer.Typer( pretty_exceptions_show_locals=False, @@ -64,13 +60,14 @@ def check_latest_version_on_close(ctx: typer.Context) -> None: def export_if_needed_on_close(ctx: typer.Context) -> None: + scan_finalized = ctx.obj.get('scan_finalized') printer = ctx.obj.get('console_printer') - if printer.is_recording: + if scan_finalized and printer.is_recording: printer.export() +_AUTH_RICH_HELP_PANEL = 'Authentication options' _COMPLETION_RICH_HELP_PANEL = 'Completion options' -_EXPORT_RICH_HELP_PANEL = 'Export options' @app.callback() @@ -90,25 +87,18 @@ def app_callback( Optional[str], typer.Option(hidden=True, help='Characteristic JSON object that lets servers identify the application.'), ] = None, - export_type: Annotated[ - ExportTypeOption, + client_secret: Annotated[ + Optional[str], typer.Option( - '--export-type', - case_sensitive=False, - help='Specify the export type. ' - 'HTML and SVG will export terminal output and rely on --output option. ' - 'JSON always exports JSON.', - rich_help_panel=_EXPORT_RICH_HELP_PANEL, + help='Specify a Cycode client secret for this specific scan execution.', + rich_help_panel=_AUTH_RICH_HELP_PANEL, ), - ] = ExportTypeOption.JSON, - export_file: Annotated[ - Optional[Path], + ] = None, + client_id: Annotated[ + Optional[str], typer.Option( - '--export-file', - help='Export file. Path to the file where the export will be saved. ', - dir_okay=False, - writable=True, - rich_help_panel=_EXPORT_RICH_HELP_PANEL, + help='Specify a Cycode client ID for this specific scan execution.', + rich_help_panel=_AUTH_RICH_HELP_PANEL, ), ] = None, _: Annotated[ @@ -150,10 +140,11 @@ def app_callback( if output == OutputTypeOption.JSON: no_progress_meter = True + ctx.obj['client_id'] = client_id + ctx.obj['client_secret'] = client_secret + ctx.obj['progress_bar'] = get_progress_bar(hidden=no_progress_meter, sections=SCAN_PROGRESS_BAR_SECTIONS) - ctx.obj['export_type'] = export_type - ctx.obj['export_file'] = export_file ctx.obj['console_printer'] = ConsolePrinter(ctx) ctx.call_on_close(lambda: export_if_needed_on_close(ctx)) diff --git a/cycode/cli/apps/ai_remediation/__init__.py b/cycode/cli/apps/ai_remediation/__init__.py index cd471a08..00d0c7c5 100644 --- a/cycode/cli/apps/ai_remediation/__init__.py +++ b/cycode/cli/apps/ai_remediation/__init__.py @@ -4,9 +4,9 @@ app = typer.Typer() -_ai_remediation_epilog = """ -Note: AI remediation suggestions are generated automatically and should be reviewed before applying. -""" +_ai_remediation_epilog = ( + 'Note: AI remediation suggestions are generated automatically and should be reviewed before applying.' +) app.command( name='ai-remediation', diff --git a/cycode/cli/apps/ai_remediation/ai_remediation_command.py b/cycode/cli/apps/ai_remediation/ai_remediation_command.py index ea5ef826..ab2eca5e 100644 --- a/cycode/cli/apps/ai_remediation/ai_remediation_command.py +++ b/cycode/cli/apps/ai_remediation/ai_remediation_command.py @@ -24,7 +24,7 @@ def ai_remediation_command( * `cycode ai-remediation `: View remediation guidance * `cycode ai-remediation --fix`: Apply suggested fixes """ - client = get_scan_cycode_client() + client = get_scan_cycode_client(ctx) try: remediation_markdown = client.get_ai_remediation(detection_id) diff --git a/cycode/cli/apps/auth/__init__.py b/cycode/cli/apps/auth/__init__.py index beecae38..f487e1bf 100644 --- a/cycode/cli/apps/auth/__init__.py +++ b/cycode/cli/apps/auth/__init__.py @@ -3,12 +3,7 @@ from cycode.cli.apps.auth.auth_command import auth_command _auth_command_docs = 'https://github.com/cycodehq/cycode-cli/blob/main/README.md#using-the-auth-command' -_auth_command_epilog = f"""[bold]Documentation[/] - - - -For more details and advanced usage, visit: [link={_auth_command_docs}]{_auth_command_docs}[/link] -""" +_auth_command_epilog = f'[bold]Documentation:[/] [link={_auth_command_docs}]{_auth_command_docs}[/link]' app = typer.Typer(no_args_is_help=False) app.command(name='auth', epilog=_auth_command_epilog, short_help='Authenticate your machine with Cycode.')(auth_command) diff --git a/cycode/cli/apps/auth/auth_common.py b/cycode/cli/apps/auth/auth_common.py index 52b7b6fa..96fec4cf 100644 --- a/cycode/cli/apps/auth/auth_common.py +++ b/cycode/cli/apps/auth/auth_common.py @@ -13,7 +13,10 @@ def get_authorization_info(ctx: 'Context') -> Optional[AuthInfo]: printer = ctx.obj.get('console_printer') - client_id, client_secret = CredentialsManager().get_credentials() + client_id, client_secret = ctx.obj.get('client_id'), ctx.obj.get('client_secret') + if not client_id or not client_secret: + client_id, client_secret = CredentialsManager().get_credentials() + if not client_id or not client_secret: return None diff --git a/cycode/cli/apps/configure/__init__.py b/cycode/cli/apps/configure/__init__.py index ce73c450..4944a3e3 100644 --- a/cycode/cli/apps/configure/__init__.py +++ b/cycode/cli/apps/configure/__init__.py @@ -3,12 +3,7 @@ from cycode.cli.apps.configure.configure_command import configure_command _configure_command_docs = 'https://github.com/cycodehq/cycode-cli/blob/main/README.md#using-the-configure-command' -_configure_command_epilog = f"""[bold]Documentation[/] - - - -For more details and advanced usage, visit: [link={_configure_command_docs}]{_configure_command_docs}[/link] -""" +_configure_command_epilog = f'[bold]Documentation:[/] [link={_configure_command_docs}]{_configure_command_docs}[/link]' app = typer.Typer(no_args_is_help=True) diff --git a/cycode/cli/apps/report/__init__.py b/cycode/cli/apps/report/__init__.py index 40cc696a..751157a4 100644 --- a/cycode/cli/apps/report/__init__.py +++ b/cycode/cli/apps/report/__init__.py @@ -4,5 +4,5 @@ from cycode.cli.apps.report.report_command import report_command app = typer.Typer(name='report', no_args_is_help=True) -app.callback(short_help='Generate report. You`ll need to specify which report type to perform.')(report_command) +app.callback(short_help='Generate report. You`ll need to specify which report type to perform as SBOM.')(report_command) app.add_typer(sbom.app) diff --git a/cycode/cli/apps/report/sbom/path/path_command.py b/cycode/cli/apps/report/sbom/path/path_command.py index 20e82848..9741aa73 100644 --- a/cycode/cli/apps/report/sbom/path/path_command.py +++ b/cycode/cli/apps/report/sbom/path/path_command.py @@ -24,7 +24,7 @@ def path_command( ) -> None: add_breadcrumb('path') - client = get_report_cycode_client() + client = get_report_cycode_client(ctx) report_parameters = ctx.obj['report_parameters'] output_format = report_parameters.output_format output_file = ctx.obj['output_file'] diff --git a/cycode/cli/apps/report/sbom/repository_url/repository_url_command.py b/cycode/cli/apps/report/sbom/repository_url/repository_url_command.py index 28be0114..9e2f4885 100644 --- a/cycode/cli/apps/report/sbom/repository_url/repository_url_command.py +++ b/cycode/cli/apps/report/sbom/repository_url/repository_url_command.py @@ -20,7 +20,7 @@ def repository_url_command( progress_bar.start() progress_bar.set_section_length(SbomReportProgressBarSection.PREPARE_LOCAL_FILES) - client = get_report_cycode_client() + client = get_report_cycode_client(ctx) report_parameters = ctx.obj['report_parameters'] output_file = ctx.obj['output_file'] output_format = report_parameters.output_format diff --git a/cycode/cli/apps/scan/__init__.py b/cycode/cli/apps/scan/__init__.py index ada2d105..b4d8ab79 100644 --- a/cycode/cli/apps/scan/__init__.py +++ b/cycode/cli/apps/scan/__init__.py @@ -10,12 +10,7 @@ app = typer.Typer(name='scan', no_args_is_help=True) _scan_command_docs = 'https://github.com/cycodehq/cycode-cli/blob/main/README.md#scan-command' -_scan_command_epilog = f"""[bold]Documentation[/] - - - -For more details and advanced usage, visit: [link={_scan_command_docs}]{_scan_command_docs}[/link] -""" +_scan_command_epilog = f'[bold]Documentation:[/] [link={_scan_command_docs}]{_scan_command_docs}[/link]' app.callback( short_help='Scan the content for Secrets, IaC, SCA, and SAST violations.', diff --git a/cycode/cli/apps/scan/code_scanner.py b/cycode/cli/apps/scan/code_scanner.py index 0209d9da..04aae2e3 100644 --- a/cycode/cli/apps/scan/code_scanner.py +++ b/cycode/cli/apps/scan/code_scanner.py @@ -323,7 +323,7 @@ def scan_documents( scan_batch_thread_func, scan_type, documents_to_scan, progress_bar=progress_bar ) - aggregation_report_url = _try_get_aggregation_report_url_if_needed(scan_parameters, ctx.obj['client'], scan_type) + aggregation_report_url = _try_get_aggregation_report_url(scan_parameters, ctx.obj['client'], scan_type) _set_aggregation_report_url(ctx, aggregation_report_url) progress_bar.set_section_length(ScanProgressBarSection.GENERATE_REPORT, 1) @@ -571,6 +571,7 @@ def print_results( ctx: typer.Context, local_scan_results: list[LocalScanResult], errors: Optional[dict[str, 'CliError']] = None ) -> None: printer = ctx.obj.get('console_printer') + printer.update_ctx(ctx) printer.print_scan_results(local_scan_results, errors) @@ -640,7 +641,6 @@ def parse_pre_receive_input() -> str: def _get_default_scan_parameters(ctx: typer.Context) -> dict: return { 'monitor': ctx.obj.get('monitor'), - 'report': ctx.obj.get('report'), 'package_vulnerabilities': ctx.obj.get('package-vulnerabilities'), 'license_compliance': ctx.obj.get('license-compliance'), 'command_type': ctx.info_name.replace('-', '_'), # save backward compatibility @@ -956,7 +956,7 @@ def _get_scan_result( did_detect=True, detections_per_file=_map_detections_per_file_and_commit_id(scan_type, scan_raw_detections), scan_id=scan_id, - report_url=_try_get_aggregation_report_url_if_needed(scan_parameters, cycode_client, scan_type), + report_url=_try_get_aggregation_report_url(scan_parameters, cycode_client, scan_type), ) @@ -972,12 +972,9 @@ def _set_aggregation_report_url(ctx: typer.Context, aggregation_report_url: Opti ctx.obj['aggregation_report_url'] = aggregation_report_url -def _try_get_aggregation_report_url_if_needed( +def _try_get_aggregation_report_url( scan_parameters: dict, cycode_client: 'ScanClient', scan_type: str ) -> Optional[str]: - if not scan_parameters.get('report', False): - return None - aggregation_id = scan_parameters.get('aggregation_id') if aggregation_id is None: return None diff --git a/cycode/cli/apps/scan/scan_command.py b/cycode/cli/apps/scan/scan_command.py index 38e4a610..2d323706 100644 --- a/cycode/cli/apps/scan/scan_command.py +++ b/cycode/cli/apps/scan/scan_command.py @@ -1,9 +1,10 @@ +from pathlib import Path from typing import Annotated, Optional import click import typer -from cycode.cli.cli_types import ScanTypeOption, ScaScanTypeOption, SeverityOption +from cycode.cli.cli_types import ExportTypeOption, ScanTypeOption, ScaScanTypeOption, SeverityOption from cycode.cli.consts import ( ISSUE_DETECTED_STATUS_CODE, NO_ISSUES_STATUS_CODE, @@ -12,8 +13,9 @@ from cycode.cli.utils.get_api_client import get_scan_cycode_client from cycode.cli.utils.sentry import add_breadcrumb -_AUTH_RICH_HELP_PANEL = 'Authentication options' +_EXPORT_RICH_HELP_PANEL = 'Export options' _SCA_RICH_HELP_PANEL = 'SCA options' +_SECRET_RICH_HELP_PANEL = 'Secret options' def scan_command( @@ -27,21 +29,6 @@ def scan_command( case_sensitive=False, ), ] = ScanTypeOption.SECRET, - client_secret: Annotated[ - Optional[str], - typer.Option( - help='Specify a Cycode client secret for this specific scan execution.', - rich_help_panel=_AUTH_RICH_HELP_PANEL, - ), - ] = None, - client_id: Annotated[ - Optional[str], - typer.Option( - help='Specify a Cycode client ID for this specific scan execution.', - rich_help_panel=_AUTH_RICH_HELP_PANEL, - ), - ] = None, - show_secret: Annotated[bool, typer.Option('--show-secret', help='Show Secrets in plain text.')] = False, soft_fail: Annotated[ bool, typer.Option('--soft-fail', help='Run the scan without failing; always return a non-error status code.') ] = False, @@ -58,13 +45,8 @@ def scan_command( '--sync', help='Run scan synchronously (INTERNAL FOR IDEs).', show_default='asynchronously', hidden=True ), ] = False, - report: Annotated[ - bool, - typer.Option( - '--report', - help='When specified, generates a violations report. ' - 'A link to the report will be displayed in the console output.', - ), + show_secret: Annotated[ + bool, typer.Option('--show-secret', help='Show Secrets in plain text.', rich_help_panel=_SECRET_RICH_HELP_PANEL) ] = False, sca_scan: Annotated[ list[ScaScanTypeOption], @@ -98,6 +80,27 @@ def scan_command( rich_help_panel=_SCA_RICH_HELP_PANEL, ), ] = False, + export_type: Annotated[ + ExportTypeOption, + typer.Option( + '--export-type', + case_sensitive=False, + help='Specify the export type. ' + 'HTML and SVG will export terminal output and rely on --output option. ' + 'JSON always exports JSON.', + rich_help_panel=_EXPORT_RICH_HELP_PANEL, + ), + ] = None, + export_file: Annotated[ + Optional[Path], + typer.Option( + '--export-file', + help='Export file. Path to the file where the export will be saved.', + dir_okay=False, + writable=True, + rich_help_panel=_EXPORT_RICH_HELP_PANEL, + ), + ] = None, ) -> None: """:mag: [bold cyan]Scan code for vulnerabilities (Secrets, IaC, SCA, SAST).[/] @@ -115,14 +118,28 @@ def scan_command( """ add_breadcrumb('scan') + if export_file and export_type is None: + raise typer.BadParameter( + 'Export type must be specified when --export-file is provided.', + param_hint='--export-type', + ) + if export_type and export_file is None: + raise typer.BadParameter( + 'Export file must be specified when --export-type is provided.', + param_hint='--export-file', + ) + ctx.obj['show_secret'] = show_secret ctx.obj['soft_fail'] = soft_fail - ctx.obj['client'] = get_scan_cycode_client(client_id, client_secret, not ctx.obj['show_secret']) + ctx.obj['client'] = get_scan_cycode_client(ctx) ctx.obj['scan_type'] = scan_type ctx.obj['sync'] = sync ctx.obj['severity_threshold'] = severity_threshold ctx.obj['monitor'] = monitor - ctx.obj['report'] = report + + if export_type and export_file: + console_printer = ctx.obj['console_printer'] + console_printer.enable_recording(export_type, export_file) _ = no_restore, gradle_all_sub_projects # they are actually used; via ctx.params @@ -136,7 +153,8 @@ def _sca_scan_to_context(ctx: typer.Context, sca_scan_user_selected: list[str]) @click.pass_context def scan_command_result_callback(ctx: click.Context, *_, **__) -> None: - add_breadcrumb('scan_finalize') + add_breadcrumb('scan_finalized') + ctx.obj['scan_finalized'] = True progress_bar = ctx.obj.get('progress_bar') if progress_bar: diff --git a/cycode/cli/apps/status/get_cli_status.py b/cycode/cli/apps/status/get_cli_status.py index 0a272c57..0cf6e8fd 100644 --- a/cycode/cli/apps/status/get_cli_status.py +++ b/cycode/cli/apps/status/get_cli_status.py @@ -22,7 +22,7 @@ def get_cli_status(ctx: 'Context') -> CliStatus: supported_modules_status = CliSupportedModulesStatus() if is_authenticated: try: - client = get_scan_cycode_client() + client = get_scan_cycode_client(ctx) supported_modules_preferences = client.get_supported_modules_preferences() supported_modules_status.secret_scanning = supported_modules_preferences.secret_scanning diff --git a/cycode/cli/printers/console_printer.py b/cycode/cli/printers/console_printer.py index f581c894..17c402ff 100644 --- a/cycode/cli/printers/console_printer.py +++ b/cycode/cli/printers/console_printer.py @@ -16,6 +16,8 @@ from cycode.cli.printers.text_printer import TextPrinter if TYPE_CHECKING: + from pathlib import Path + from cycode.cli.models import LocalScanResult from cycode.cli.printers.tables.table_printer_base import PrinterBase @@ -43,17 +45,9 @@ def __init__( self.console_err = console_err_override or console_err self.output_type = output_type_override or self.ctx.obj.get('output') - self.console_record = None - - self.export_type = self.ctx.obj.get('export_type') - self.export_file = self.ctx.obj.get('export_file') - if console_override is None and self.export_type and self.export_file: - self.console_record = ConsolePrinter( - ctx, - console_override=Console(record=True, file=io.StringIO()), - console_err_override=Console(stderr=True, record=True, file=io.StringIO()), - output_type_override='json' if self.export_type == 'json' else self.output_type, - ) + self.export_type: Optional[str] = None + self.export_file: Optional[Path] = None + self.console_record: Optional[ConsolePrinter] = None @property def scan_type(self) -> str: @@ -76,6 +70,21 @@ def printer(self) -> 'PrinterBase': return printer_class(self.ctx, self.console, self.console_err) + def update_ctx(self, ctx: 'typer.Context') -> None: + self.ctx = ctx + + def enable_recording(self, export_type: str, export_file: 'Path') -> None: + if self.console_record is None: + self.export_file = export_file + self.export_type = export_type + + self.console_record = ConsolePrinter( + self.ctx, + console_override=Console(record=True, file=io.StringIO()), + console_err_override=Console(stderr=True, record=True, file=io.StringIO()), + output_type_override='json' if self.export_type == 'json' else self.output_type, + ) + def print_scan_results( self, local_scan_results: list['LocalScanResult'], @@ -106,16 +115,18 @@ def export(self) -> None: if self.console_record is None: raise CycodeError('Console recording was not enabled. Cannot export.') - if not self.export_file.suffix: + export_file = self.export_file + if not export_file.suffix: # resolve file extension based on the export type if not provided in the file name - self.export_file = self.export_file.with_suffix(f'.{self.export_type.lower()}') + export_file = export_file.with_suffix(f'.{self.export_type.lower()}') + export_file = str(export_file) if self.export_type is ExportTypeOption.HTML: - self.console_record.console.save_html(self.export_file) + self.console_record.console.save_html(export_file) elif self.export_type is ExportTypeOption.SVG: - self.console_record.console.save_svg(self.export_file, title=consts.APP_NAME) + self.console_record.console.save_svg(export_file, title=consts.APP_NAME) elif self.export_type is ExportTypeOption.JSON: - with open(self.export_file, 'w', encoding='UTF-8') as f: + with open(export_file, 'w', encoding='UTF-8') as f: self.console_record.console.file.seek(0) f.write(self.console_record.console.file.read()) else: diff --git a/cycode/cli/printers/printer_base.py b/cycode/cli/printers/printer_base.py index 527cc31b..69596e2a 100644 --- a/cycode/cli/printers/printer_base.py +++ b/cycode/cli/printers/printer_base.py @@ -20,10 +20,10 @@ class PrinterBase(ABC): NO_DETECTIONS_MESSAGE = ( - '[green]Good job! No issues were found!!! :clapping_hands::clapping_hands::clapping_hands:[/]' + '[b green]Good job! No issues were found!!! :clapping_hands::clapping_hands::clapping_hands:[/]' ) FAILED_SCAN_MESSAGE = ( - '[red]Unfortunately, Cycode was unable to complete the full scan. ' + '[b red]Unfortunately, Cycode was unable to complete the full scan. ' 'Please note that not all results may be available:[/]' ) @@ -99,6 +99,7 @@ def print_scan_results_summary(self, local_scan_results: list['LocalScanResult'] detections_count += 1 severity_counts[SeverityOption(detection.severity)] += 1 + self.console.line() self.console.print(f'[bold]Cycode found {detections_count} violations[/]', end=': ') # Example of string: CRITICAL - 6 | HIGH - 0 | MEDIUM - 14 | LOW - 0 | INFO - 0 @@ -110,3 +111,5 @@ def print_scan_results_summary(self, local_scan_results: list['LocalScanResult'] self.console.print( SeverityOption.get_member_emoji(severity), severity, '-', severity_counts[severity], end=end ) + + self.console.line() diff --git a/cycode/cli/printers/rich_printer.py b/cycode/cli/printers/rich_printer.py index b2ed1a2e..755278d6 100644 --- a/cycode/cli/printers/rich_printer.py +++ b/cycode/cli/printers/rich_printer.py @@ -111,6 +111,8 @@ def _print_violation_card( detection, document, obfuscate=not self.show_secret, + lines_to_display_before=3, + lines_to_display_after=3, ), title=':computer: Code Snippet', ) diff --git a/cycode/cli/printers/tables/sca_table_printer.py b/cycode/cli/printers/tables/sca_table_printer.py index 0bf59a20..1bf358c8 100644 --- a/cycode/cli/printers/tables/sca_table_printer.py +++ b/cycode/cli/printers/tables/sca_table_printer.py @@ -7,6 +7,7 @@ from cycode.cli.printers.tables.table import Table from cycode.cli.printers.tables.table_models import ColumnInfoBuilder from cycode.cli.printers.tables.table_printer_base import TablePrinterBase +from cycode.cli.printers.utils import is_git_diff_based_scan from cycode.cli.printers.utils.detection_ordering.sca_ordering import sort_and_group_detections from cycode.cli.utils.string_utils import shortcut_dependency_paths @@ -31,23 +32,19 @@ class ScaTablePrinter(TablePrinterBase): def _print_results(self, local_scan_results: list['LocalScanResult']) -> None: - aggregation_report_url = self.ctx.obj.get('aggregation_report_url') detections_per_policy_id = self._extract_detections_per_policy_id(local_scan_results) for policy_id, detections in detections_per_policy_id.items(): table = self._get_table(policy_id) resulting_detections, group_separator_indexes = sort_and_group_detections(detections) for detection in resulting_detections: - self._enrich_table_with_values(policy_id, table, detection) + self._enrich_table_with_values(table, detection) table.set_group_separator_indexes(group_separator_indexes) self._print_summary_issues(len(detections), self._get_title(policy_id)) self._print_table(table) - self.print_scan_results_summary(local_scan_results) - self._print_report_urls(local_scan_results, aggregation_report_url) - @staticmethod def _get_title(policy_id: str) -> str: if policy_id == PACKAGE_VULNERABILITY_POLICY_ID: @@ -66,7 +63,7 @@ def _get_table(self, policy_id: str) -> Table: elif policy_id == LICENSE_COMPLIANCE_POLICY_ID: table.add_column(LICENSE_COLUMN) - if self._is_git_repository(): + if is_git_diff_based_scan(self.scan_type, self.command_scan_type): table.add_column(REPOSITORY_COLUMN) table.add_column(SEVERITY_COLUMN) @@ -80,7 +77,7 @@ def _get_table(self, policy_id: str) -> Table: return table @staticmethod - def _enrich_table_with_values(policy_id: str, table: Table, detection: Detection) -> None: + def _enrich_table_with_values(table: Table, detection: Detection) -> None: detection_details = detection.detection_details if detection.severity: diff --git a/cycode/cli/printers/tables/table_printer.py b/cycode/cli/printers/tables/table_printer.py index fe9f8dd5..6fc85a1b 100644 --- a/cycode/cli/printers/tables/table_printer.py +++ b/cycode/cli/printers/tables/table_printer.py @@ -6,6 +6,7 @@ from cycode.cli.printers.tables.table import Table from cycode.cli.printers.tables.table_models import ColumnInfoBuilder from cycode.cli.printers.tables.table_printer_base import TablePrinterBase +from cycode.cli.printers.utils import is_git_diff_based_scan from cycode.cli.printers.utils.detection_ordering.common_ordering import sort_and_group_detections_from_scan_result from cycode.cli.utils.string_utils import get_position_in_line, obfuscate_text @@ -37,8 +38,6 @@ def _print_results(self, local_scan_results: list['LocalScanResult']) -> None: table.set_group_separator_indexes(group_separator_indexes) self._print_table(table) - self.print_scan_results_summary(local_scan_results) - self._print_report_urls(local_scan_results, self.ctx.obj.get('aggregation_report_url')) def _get_table(self) -> Table: table = Table() @@ -49,7 +48,7 @@ def _get_table(self) -> Table: table.add_column(LINE_NUMBER_COLUMN) table.add_column(COLUMN_NUMBER_COLUMN) - if self._is_git_repository(): + if is_git_diff_based_scan(self.scan_type, self.command_scan_type): table.add_column(COMMIT_SHA_COLUMN) if self.scan_type == SECRET_SCAN_TYPE: diff --git a/cycode/cli/printers/tables/table_printer_base.py b/cycode/cli/printers/tables/table_printer_base.py index d7a2b502..8cb4cbda 100644 --- a/cycode/cli/printers/tables/table_printer_base.py +++ b/cycode/cli/printers/tables/table_printer_base.py @@ -11,11 +11,15 @@ class TablePrinterBase(PrinterBase, abc.ABC): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.text_printer = TextPrinter(self.ctx, self.console, self.console_err) + def print_result(self, result: CliResult) -> None: - TextPrinter(self.ctx, self.console, self.console_err).print_result(result) + self.text_printer.print_result(result) def print_error(self, error: CliError) -> None: - TextPrinter(self.ctx, self.console, self.console_err).print_error(error) + self.text_printer.print_error(error) def print_scan_results( self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None @@ -26,16 +30,8 @@ def print_scan_results( self._print_results(local_scan_results) - if not errors: - return - - self.console.print(self.FAILED_SCAN_MESSAGE) - for scan_id, error in errors.items(): - self.console.print(f'- {scan_id}: ', end='') - self.print_error(error) - - def _is_git_repository(self) -> bool: - return self.ctx.info_name in {'commit_history', 'pre_commit', 'pre_receive'} and 'remote_url' in self.ctx.obj + self.print_scan_results_summary(local_scan_results) + self.text_printer.print_report_urls_and_errors(local_scan_results, errors) @abc.abstractmethod def _print_results(self, local_scan_results: list['LocalScanResult']) -> None: @@ -44,19 +40,3 @@ def _print_results(self, local_scan_results: list['LocalScanResult']) -> None: def _print_table(self, table: 'Table') -> None: if table.get_rows(): self.console.print(table.get_table()) - - def _print_report_urls( - self, - local_scan_results: list['LocalScanResult'], - aggregation_report_url: Optional[str] = None, - ) -> None: - report_urls = [scan_result.report_url for scan_result in local_scan_results if scan_result.report_url] - if not report_urls and not aggregation_report_url: - return - if aggregation_report_url: - self.console.print(f'Report URL: {aggregation_report_url}') - return - - self.console.print('Report URLs:') - for report_url in report_urls: - self.console.print(f'- {report_url}') diff --git a/cycode/cli/printers/text_printer.py b/cycode/cli/printers/text_printer.py index 564456ae..05a360fd 100644 --- a/cycode/cli/printers/text_printer.py +++ b/cycode/cli/printers/text_printer.py @@ -98,10 +98,15 @@ def print_report_urls_and_errors( def print_report_urls(self, report_urls: list[str], aggregation_report_url: Optional[str] = None) -> None: if not report_urls and not aggregation_report_url: return - if aggregation_report_url: - self.console.print(f'Report URL: {aggregation_report_url}') + + # Prioritize aggregation report URL; if report urls is only one, use it instead + single_url = report_urls[0] if len(report_urls) == 1 else None + single_url = aggregation_report_url or single_url + if single_url: + self.console.print(f'[b]Report URL:[/] {single_url}') return - self.console.print('Report URLs:') + # If there are multiple report URLs, print them all + self.console.print('[b]Report URLs:[/]') for report_url in report_urls: self.console.print(f'- {report_url}') diff --git a/cycode/cli/printers/utils/__init__.py b/cycode/cli/printers/utils/__init__.py index e69de29b..e1676c35 100644 --- a/cycode/cli/printers/utils/__init__.py +++ b/cycode/cli/printers/utils/__init__.py @@ -0,0 +1,8 @@ +from cycode.cli import consts + + +def is_git_diff_based_scan(scan_type: str, command_scan_type: str) -> bool: + return ( + command_scan_type in consts.COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES + and scan_type in consts.COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES + ) diff --git a/cycode/cli/printers/utils/code_snippet_syntax.py b/cycode/cli/printers/utils/code_snippet_syntax.py index aae33872..12501544 100644 --- a/cycode/cli/printers/utils/code_snippet_syntax.py +++ b/cycode/cli/printers/utils/code_snippet_syntax.py @@ -1,10 +1,10 @@ -import math from typing import TYPE_CHECKING from rich.syntax import Syntax from cycode.cli import consts from cycode.cli.console import _SYNTAX_HIGHLIGHT_THEME +from cycode.cli.printers.utils import is_git_diff_based_scan from cycode.cli.utils.string_utils import get_position_in_line, obfuscate_text if TYPE_CHECKING: @@ -12,8 +12,8 @@ from cycode.cyclient.models import Detection -def _get_code_segment_start_line(detection_line: int, lines_to_display: int) -> int: - start_line = detection_line - math.ceil(lines_to_display / 2) +def _get_code_segment_start_line(detection_line: int, lines_to_display_before: int) -> int: + start_line = detection_line - lines_to_display_before return 0 if start_line < 0 else start_line @@ -26,17 +26,24 @@ def get_detection_line(scan_type: str, detection: 'Detection') -> int: def _get_code_snippet_syntax_from_file( - scan_type: str, detection: 'Detection', document: 'Document', lines_to_display: int, obfuscate: bool + scan_type: str, + detection: 'Detection', + document: 'Document', + lines_to_display_before: int, + lines_to_display_after: int, + obfuscate: bool, ) -> Syntax: detection_details = detection.detection_details detection_line = get_detection_line(scan_type, detection) - start_line_index = _get_code_segment_start_line(detection_line, lines_to_display) + start_line_index = _get_code_segment_start_line(detection_line, lines_to_display_before) detection_position = get_position_in_line(document.content, detection_details.get('start_position', -1)) violation_length = detection_details.get('length', -1) code_lines_to_render = [] document_content_lines = document.content.splitlines() - for line_index in range(lines_to_display): + total_lines_to_display = lines_to_display_before + 1 + lines_to_display_after + + for line_index in range(total_lines_to_display): current_line_index = start_line_index + line_index if current_line_index >= len(document_content_lines): break @@ -56,6 +63,7 @@ def _get_code_snippet_syntax_from_file( code=code_to_render, lexer=Syntax.guess_lexer(document.path, code=code_to_render), line_numbers=True, + word_wrap=True, dedent=True, tab_size=2, start_line=start_line_index + 1, @@ -91,23 +99,19 @@ def _get_code_snippet_syntax_from_git_diff( ) -def _is_git_diff_based_scan(scan_type: str, command_scan_type: str) -> bool: - return ( - command_scan_type in consts.COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES - and scan_type in consts.COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES - ) - - def get_code_snippet_syntax( scan_type: str, command_scan_type: str, detection: 'Detection', document: 'Document', - lines_to_display: int = 3, + lines_to_display_before: int = 1, + lines_to_display_after: int = 1, obfuscate: bool = True, ) -> Syntax: - if _is_git_diff_based_scan(scan_type, command_scan_type): + if is_git_diff_based_scan(scan_type, command_scan_type): # it will return syntax with just one line return _get_code_snippet_syntax_from_git_diff(scan_type, detection, document, obfuscate) - return _get_code_snippet_syntax_from_file(scan_type, detection, document, lines_to_display, obfuscate) + return _get_code_snippet_syntax_from_file( + scan_type, detection, document, lines_to_display_before, lines_to_display_after, obfuscate + ) diff --git a/cycode/cli/printers/utils/detection_data.py b/cycode/cli/printers/utils/detection_data.py index 358b4c63..989a6600 100644 --- a/cycode/cli/printers/utils/detection_data.py +++ b/cycode/cli/printers/utils/detection_data.py @@ -35,9 +35,21 @@ def get_cwe_cve_link(cwe_cve: Optional[str]) -> Optional[str]: return None +def clear_cwe_name(cwe: str) -> str: + """Clear CWE. + + Intput: CWE-532: Insertion of Sensitive Information into Log File + Output: CWE-532 + """ + if cwe.startswith('CWE'): + return cwe.split(':')[0] + + return cwe + + def get_detection_clickable_cwe_cve(scan_type: str, detection: 'Detection') -> str: def link(url: str, name: str) -> str: - return f'[link={url}]{name}[/]' + return f'[link={url}]{clear_cwe_name(name)}[/]' if scan_type == consts.SCA_SCAN_TYPE: cve = detection.detection_details.get('vulnerability_id') @@ -84,5 +96,13 @@ def get_detection_file_path(scan_type: str, detection: 'Detection') -> Path: folder_path = detection.detection_details.get('file_path', '') file_name = detection.detection_details.get('file_name', '') return Path.joinpath(Path(folder_path), Path(file_name)) + if scan_type == consts.SAST_SCAN_TYPE: + file_path = detection.detection_details.get('file_path', '') + + # fix the absolute path...BE returns string which does not start with / + if not file_path.startswith('/'): + file_path = f'/{file_path}' + + return Path(file_path) return Path(detection.detection_details.get('file_name', '')) diff --git a/cycode/cli/utils/get_api_client.py b/cycode/cli/utils/get_api_client.py index 91e8f0f7..110d528b 100644 --- a/cycode/cli/utils/get_api_client.py +++ b/cycode/cli/utils/get_api_client.py @@ -6,6 +6,8 @@ from cycode.cyclient.client_creator import create_report_client, create_scan_client if TYPE_CHECKING: + import typer + from cycode.cyclient.report_client import ReportClient from cycode.cyclient.scan_client import ScanClient @@ -23,15 +25,16 @@ def _get_cycode_client( return create_client_func(client_id, client_secret, hide_response_log) -def get_scan_cycode_client( - client_id: Optional[str] = None, client_secret: Optional[str] = None, hide_response_log: bool = True -) -> 'ScanClient': +def get_scan_cycode_client(ctx: 'typer.Context') -> 'ScanClient': + client_id = ctx.obj.get('client_id') + client_secret = ctx.obj.get('client_secret') + hide_response_log = not ctx.obj.get('show_secret', False) return _get_cycode_client(create_scan_client, client_id, client_secret, hide_response_log) -def get_report_cycode_client( - client_id: Optional[str] = None, client_secret: Optional[str] = None, hide_response_log: bool = True -) -> 'ReportClient': +def get_report_cycode_client(ctx: 'typer.Context', hide_response_log: bool = True) -> 'ReportClient': + client_id = ctx.obj.get('client_id') + client_secret = ctx.obj.get('client_secret') return _get_cycode_client(create_report_client, client_id, client_secret, hide_response_log) diff --git a/cycode/cli/utils/version_checker.py b/cycode/cli/utils/version_checker.py index 47da17c4..8fd1d005 100644 --- a/cycode/cli/utils/version_checker.py +++ b/cycode/cli/utils/version_checker.py @@ -86,7 +86,9 @@ def get_latest_version(self) -> Optional[str]: """ try: - response = self.get(f'{self.PYPI_PACKAGE_NAME}/json', timeout=self.PYPI_REQUEST_TIMEOUT) + response = self.get( + f'{self.PYPI_PACKAGE_NAME}/json', timeout=self.PYPI_REQUEST_TIMEOUT, hide_response_content_log=True + ) data = response.json() return data.get('info', {}).get('version') except Exception: @@ -203,10 +205,10 @@ def check_and_notify_update(self, current_version: str, use_cache: bool = True) should_update = bool(latest_version) if should_update: update_message = ( - '\nNew version of cycode available! ' - f'[yellow]{current_version}[/] → [bright_blue]{latest_version}[/]\n' + '\nNew release of Cycode CLI is available: ' + f'[red]{current_version}[/] -> [green]{latest_version}[/]\n' f'Changelog: [bright_blue]{self.GIT_CHANGELOG_URL_PREFIX}{latest_version}[/]\n' - f'Run [green]pip install --upgrade cycode[/] to update\n' + f'To update, run: [green]pip install --upgrade cycode[/]\n' ) console.print(update_message) diff --git a/cycode/cyclient/scan_client.py b/cycode/cyclient/scan_client.py index e0bf8131..bdbce37f 100644 --- a/cycode/cyclient/scan_client.py +++ b/cycode/cyclient/scan_client.py @@ -1,5 +1,4 @@ import json -from copy import deepcopy from typing import TYPE_CHECKING, Union from uuid import UUID @@ -74,11 +73,6 @@ def zipped_file_scan_sync( is_git_diff: bool = False, ) -> models.ScanResultsSyncFlow: files = {'file': ('multiple_files_scan.zip', zip_file.read())} - - scan_parameters = deepcopy(scan_parameters) # avoid mutating the original dict - if 'report' in scan_parameters: - del scan_parameters['report'] # BE raises validation error instead of ignoring it - response = self.scan_cycode_client.post( url_path=self.get_zipped_file_scan_sync_url_path(scan_type), data={ diff --git a/tests/cli/commands/test_check_latest_version_on_close.py b/tests/cli/commands/test_check_latest_version_on_close.py index b1f11e24..eccadf93 100644 --- a/tests/cli/commands/test_check_latest_version_on_close.py +++ b/tests/cli/commands/test_check_latest_version_on_close.py @@ -10,7 +10,7 @@ from tests.conftest import CLI_ENV_VARS _NEW_LATEST_VERSION = '999.0.0' # Simulate a newer version available -_UPDATE_MESSAGE_PART = 'new version of cycode available' +_UPDATE_MESSAGE_PART = 'new release of cycode cli is available' @patch.object(VersionChecker, 'check_for_update') diff --git a/tests/test_code_scanner.py b/tests/test_code_scanner.py index 9ef09123..9372ede0 100644 --- a/tests/test_code_scanner.py +++ b/tests/test_code_scanner.py @@ -6,7 +6,7 @@ from cycode.cli import consts from cycode.cli.apps.scan.code_scanner import ( - _try_get_aggregation_report_url_if_needed, + _try_get_aggregation_report_url, ) from cycode.cli.cli_types import ScanTypeOption from cycode.cli.files_collector.excluder import _is_relevant_file_to_scan @@ -29,7 +29,7 @@ def test_try_get_aggregation_report_url_if_no_report_command_needed_return_none( ) -> None: aggregation_id = uuid4().hex scan_parameter = {'aggregation_id': aggregation_id} - result = _try_get_aggregation_report_url_if_needed(scan_parameter, scan_client, scan_type) + result = _try_get_aggregation_report_url(scan_parameter, scan_client, scan_type) assert result is None @@ -37,8 +37,7 @@ def test_try_get_aggregation_report_url_if_no_report_command_needed_return_none( def test_try_get_aggregation_report_url_if_no_aggregation_id_needed_return_none( scan_type: ScanTypeOption, scan_client: ScanClient ) -> None: - scan_parameter = {'report': True} - result = _try_get_aggregation_report_url_if_needed(scan_parameter, scan_client, scan_type) + result = _try_get_aggregation_report_url({}, scan_client, scan_type) assert result is None @@ -48,12 +47,12 @@ def test_try_get_aggregation_report_url_if_needed_return_result( scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: aggregation_id = uuid4() - scan_parameter = {'report': True, 'aggregation_id': aggregation_id} + scan_parameter = {'aggregation_id': aggregation_id} url = get_scan_aggregation_report_url(aggregation_id, scan_client, scan_type) responses.add(api_token_response) # mock token based client responses.add(get_scan_aggregation_report_url_response(url, aggregation_id)) scan_aggregation_report_url_response = scan_client.get_scan_aggregation_report_url(str(aggregation_id), scan_type) - result = _try_get_aggregation_report_url_if_needed(scan_parameter, scan_client, scan_type) + result = _try_get_aggregation_report_url(scan_parameter, scan_client, scan_type) assert result == scan_aggregation_report_url_response.report_url From cdf571672f27e7f1cb282fe80c1be9f6aec7ca4e Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Tue, 6 May 2025 17:20:51 +0200 Subject: [PATCH 16/19] CM-48074 - Return report option with new name `--cycode-report` (#306) --- README.md | 27 ++++++++++++++++++++++++--- cycode/cli/apps/scan/code_scanner.py | 10 +++++++--- cycode/cli/apps/scan/scan_command.py | 8 ++++++++ cycode/cyclient/scan_client.py | 6 ++++++ tests/test_code_scanner.py | 11 ++++++----- 5 files changed, 51 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 7966361b..82575d6f 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,10 @@ This guide walks you through both installation and usage. 1. [Options](#options) 1. [Severity Threshold](#severity-option) 2. [Monitor](#monitor-option) - 3. [Package Vulnerabilities](#package-vulnerabilities-option) - 4. [License Compliance](#license-compliance-option) - 5. [Lock Restore](#lock-restore-option) + 3. [Cycode Report](#cycode-report-option) + 4. [Package Vulnerabilities](#package-vulnerabilities-option) + 5. [License Compliance](#license-compliance-option) + 6. [Lock Restore](#lock-restore-option) 2. [Repository Scan](#repository-scan) 1. [Branch Option](#branch-option) 3. [Path Scan](#path-scan) @@ -300,6 +301,7 @@ The Cycode CLI application offers several types of scans so that you can choose | `--severity-threshold [INFO\|LOW\|MEDIUM\|HIGH\|CRITICAL]` | Show only violations at the specified level or higher. | | `--sca-scan` | Specify the SCA scan you wish to execute (`package-vulnerabilities`/`license-compliance`). The default is both. | | `--monitor` | When specified, the scan results will be recorded in the knowledge graph. Please note that when working in `monitor` mode, the knowledge graph will not be updated as a result of SCM events (Push, Repo creation). (Supported for SCA scan type only). | +| `--cycode-report` | When specified, displays a link to the scan report in the Cycode platform in the console output. | | `--no-restore` | When specified, Cycode will not run restore command. Will scan direct dependencies ONLY! | | `--gradle-all-sub-projects` | When specified, Cycode will run gradle restore command for all sub projects. Should run from root project directory ONLY! | | `--help` | Show options for given command. | @@ -337,6 +339,25 @@ When using this option, the scan results from this scan will appear in the knowl > [!WARNING] > You must be an `owner` or an `admin` in Cycode to view the knowledge graph page. +#### Cycode Report Option + +For every scan performed using the Cycode CLI, a report is automatically generated and its results are sent to Cycode. These results are tied to the relevant policies (e.g., [SCA policies](https://docs.cycode.com/docs/sca-policies) for Repository scans) within the Cycode platform. + +To have the direct URL to this Cycode report printed in your CLI output after the scan completes, add the argument `--cycode-report` to your scan command. + +`cycode scan --cycode-report repository ~/home/git/codebase` + +All scan results from the CLI will appear in the CLI Logs section of Cycode. If you included the `--cycode-report` flag in your command, a direct link to the specific report will be displayed in your terminal following the scan results. + +> [!WARNING] +> You must be an `owner` or an `admin` in Cycode to view this page. + +![cli-report](https://raw.githubusercontent.com/cycodehq/cycode-cli/main/images/sca_report_url.png) + +The report page will look something like below: + +![](https://raw.githubusercontent.com/cycodehq/cycode-cli/main/images/scan_details.png) + #### Package Vulnerabilities Option > [!NOTE] diff --git a/cycode/cli/apps/scan/code_scanner.py b/cycode/cli/apps/scan/code_scanner.py index 04aae2e3..a40a066e 100644 --- a/cycode/cli/apps/scan/code_scanner.py +++ b/cycode/cli/apps/scan/code_scanner.py @@ -323,7 +323,7 @@ def scan_documents( scan_batch_thread_func, scan_type, documents_to_scan, progress_bar=progress_bar ) - aggregation_report_url = _try_get_aggregation_report_url(scan_parameters, ctx.obj['client'], scan_type) + aggregation_report_url = _try_get_aggregation_report_url_if_needed(scan_parameters, ctx.obj['client'], scan_type) _set_aggregation_report_url(ctx, aggregation_report_url) progress_bar.set_section_length(ScanProgressBarSection.GENERATE_REPORT, 1) @@ -641,6 +641,7 @@ def parse_pre_receive_input() -> str: def _get_default_scan_parameters(ctx: typer.Context) -> dict: return { 'monitor': ctx.obj.get('monitor'), + 'report': ctx.obj.get('report'), 'package_vulnerabilities': ctx.obj.get('package-vulnerabilities'), 'license_compliance': ctx.obj.get('license-compliance'), 'command_type': ctx.info_name.replace('-', '_'), # save backward compatibility @@ -956,7 +957,7 @@ def _get_scan_result( did_detect=True, detections_per_file=_map_detections_per_file_and_commit_id(scan_type, scan_raw_detections), scan_id=scan_id, - report_url=_try_get_aggregation_report_url(scan_parameters, cycode_client, scan_type), + report_url=_try_get_aggregation_report_url_if_needed(scan_parameters, cycode_client, scan_type), ) @@ -972,9 +973,12 @@ def _set_aggregation_report_url(ctx: typer.Context, aggregation_report_url: Opti ctx.obj['aggregation_report_url'] = aggregation_report_url -def _try_get_aggregation_report_url( +def _try_get_aggregation_report_url_if_needed( scan_parameters: dict, cycode_client: 'ScanClient', scan_type: str ) -> Optional[str]: + if not scan_parameters.get('report', False): + return None + aggregation_id = scan_parameters.get('aggregation_id') if aggregation_id is None: return None diff --git a/cycode/cli/apps/scan/scan_command.py b/cycode/cli/apps/scan/scan_command.py index 2d323706..a2ffb550 100644 --- a/cycode/cli/apps/scan/scan_command.py +++ b/cycode/cli/apps/scan/scan_command.py @@ -45,6 +45,13 @@ def scan_command( '--sync', help='Run scan synchronously (INTERNAL FOR IDEs).', show_default='asynchronously', hidden=True ), ] = False, + report: Annotated[ + bool, + typer.Option( + '--cycode-report', + help='When specified, displays a link to the scan report in the Cycode platform in the console output.', + ), + ] = False, show_secret: Annotated[ bool, typer.Option('--show-secret', help='Show Secrets in plain text.', rich_help_panel=_SECRET_RICH_HELP_PANEL) ] = False, @@ -136,6 +143,7 @@ def scan_command( ctx.obj['sync'] = sync ctx.obj['severity_threshold'] = severity_threshold ctx.obj['monitor'] = monitor + ctx.obj['report'] = report if export_type and export_file: console_printer = ctx.obj['console_printer'] diff --git a/cycode/cyclient/scan_client.py b/cycode/cyclient/scan_client.py index bdbce37f..e0bf8131 100644 --- a/cycode/cyclient/scan_client.py +++ b/cycode/cyclient/scan_client.py @@ -1,4 +1,5 @@ import json +from copy import deepcopy from typing import TYPE_CHECKING, Union from uuid import UUID @@ -73,6 +74,11 @@ def zipped_file_scan_sync( is_git_diff: bool = False, ) -> models.ScanResultsSyncFlow: files = {'file': ('multiple_files_scan.zip', zip_file.read())} + + scan_parameters = deepcopy(scan_parameters) # avoid mutating the original dict + if 'report' in scan_parameters: + del scan_parameters['report'] # BE raises validation error instead of ignoring it + response = self.scan_cycode_client.post( url_path=self.get_zipped_file_scan_sync_url_path(scan_type), data={ diff --git a/tests/test_code_scanner.py b/tests/test_code_scanner.py index 9372ede0..9ef09123 100644 --- a/tests/test_code_scanner.py +++ b/tests/test_code_scanner.py @@ -6,7 +6,7 @@ from cycode.cli import consts from cycode.cli.apps.scan.code_scanner import ( - _try_get_aggregation_report_url, + _try_get_aggregation_report_url_if_needed, ) from cycode.cli.cli_types import ScanTypeOption from cycode.cli.files_collector.excluder import _is_relevant_file_to_scan @@ -29,7 +29,7 @@ def test_try_get_aggregation_report_url_if_no_report_command_needed_return_none( ) -> None: aggregation_id = uuid4().hex scan_parameter = {'aggregation_id': aggregation_id} - result = _try_get_aggregation_report_url(scan_parameter, scan_client, scan_type) + result = _try_get_aggregation_report_url_if_needed(scan_parameter, scan_client, scan_type) assert result is None @@ -37,7 +37,8 @@ def test_try_get_aggregation_report_url_if_no_report_command_needed_return_none( def test_try_get_aggregation_report_url_if_no_aggregation_id_needed_return_none( scan_type: ScanTypeOption, scan_client: ScanClient ) -> None: - result = _try_get_aggregation_report_url({}, scan_client, scan_type) + scan_parameter = {'report': True} + result = _try_get_aggregation_report_url_if_needed(scan_parameter, scan_client, scan_type) assert result is None @@ -47,12 +48,12 @@ def test_try_get_aggregation_report_url_if_needed_return_result( scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: aggregation_id = uuid4() - scan_parameter = {'aggregation_id': aggregation_id} + scan_parameter = {'report': True, 'aggregation_id': aggregation_id} url = get_scan_aggregation_report_url(aggregation_id, scan_client, scan_type) responses.add(api_token_response) # mock token based client responses.add(get_scan_aggregation_report_url_response(url, aggregation_id)) scan_aggregation_report_url_response = scan_client.get_scan_aggregation_report_url(str(aggregation_id), scan_type) - result = _try_get_aggregation_report_url(scan_parameter, scan_client, scan_type) + result = _try_get_aggregation_report_url_if_needed(scan_parameter, scan_client, scan_type) assert result == scan_aggregation_report_url_response.report_url From 3575a218ab2033fd497d9bd803c8e22b1933f3ad Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Fri, 9 May 2025 10:43:34 +0200 Subject: [PATCH 17/19] CM-48211 - Update CODEOWNERS (#308) --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 32a2011c..aba89cba 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1 @@ -* @MarshalX @MichalBor @MaorDavidzon @artem-fedorov @elsapet @gotbadger @cfabianski +* @MarshalX @elsapet @gotbadger @cfabianski From 4e1f7e094548b2a68874ac1e52bfb131d497e712 Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Fri, 9 May 2025 10:44:17 +0200 Subject: [PATCH 18/19] CM-48075 - Update pre-commit hook to work with compact output (#307) --- .pre-commit-hooks.yaml | 4 +- README.md | 10 +-- cycode/cli/apps/scan/code_scanner.py | 29 +++--- .../scan/pre_commit/pre_commit_command.py | 6 +- .../scan/repository/repository_command.py | 9 +- cycode/cli/cli_types.py | 22 +++-- .../files_collector/repository_documents.py | 37 ++++---- .../files_collector/sca/sca_code_scanner.py | 18 ++-- cycode/cli/printers/console_printer.py | 3 +- cycode/cli/printers/rich_printer.py | 89 ++++++++++++------- cycode/cli/printers/text_printer.py | 27 +++++- cycode/cli/printers/utils/detection_data.py | 2 +- cycode/cyclient/models.py | 9 ++ 13 files changed, 172 insertions(+), 93 deletions(-) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 02a86db0..ab69bf3f 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -3,10 +3,10 @@ language: python language_version: python3 entry: cycode - args: [ '--no-progress-meter', 'scan', '--scan-type', 'secret', 'pre-commit' ] + args: [ '-o', 'text', '--no-progress-meter', 'scan', '-t', 'secret', 'pre-commit' ] - id: cycode-sca name: Cycode SCA pre-commit defender language: python language_version: python3 entry: cycode - args: [ '--no-progress-meter', 'scan', '--scan-type', 'sca', 'pre-commit' ] + args: [ '-o', 'text', '--no-progress-meter', 'scan', '-t', 'sca', 'pre-commit' ] diff --git a/README.md b/README.md index 82575d6f..13e23a6f 100644 --- a/README.md +++ b/README.md @@ -221,11 +221,11 @@ Perform the following steps to install the pre-commit hook: ```yaml repos: - repo: https://github.com/cycodehq/cycode-cli - rev: v2.3.0 + rev: v3.0.0 hooks: - id: cycode stages: - - commit + - pre-commit ``` 4. Modify the created file for your specific needs. Use hook ID `cycode` to enable scan for Secrets. Use hook ID `cycode-sca` to enable SCA scan. If you want to enable both, use this configuration: @@ -233,14 +233,14 @@ Perform the following steps to install the pre-commit hook: ```yaml repos: - repo: https://github.com/cycodehq/cycode-cli - rev: v2.3.0 + rev: v3.0.0 hooks: - id: cycode stages: - - commit + - pre-commit - id: cycode-sca stages: - - commit + - pre-commit ``` 5. Install Cycode’s hook: diff --git a/cycode/cli/apps/scan/code_scanner.py b/cycode/cli/apps/scan/code_scanner.py index a40a066e..c6337021 100644 --- a/cycode/cli/apps/scan/code_scanner.py +++ b/cycode/cli/apps/scan/code_scanner.py @@ -48,15 +48,17 @@ logger = get_logger('Code Scanner') -def scan_sca_pre_commit(ctx: typer.Context) -> None: +def scan_sca_pre_commit(ctx: typer.Context, repo_path: str) -> None: scan_type = ctx.obj['scan_type'] scan_parameters = get_scan_parameters(ctx) git_head_documents, pre_committed_documents = get_pre_commit_modified_documents( - ctx.obj['progress_bar'], ScanProgressBarSection.PREPARE_LOCAL_FILES + progress_bar=ctx.obj['progress_bar'], + progress_bar_section=ScanProgressBarSection.PREPARE_LOCAL_FILES, + repo_path=repo_path, ) git_head_documents = exclude_irrelevant_documents_to_scan(scan_type, git_head_documents) pre_committed_documents = exclude_irrelevant_documents_to_scan(scan_type, pre_committed_documents) - sca_code_scanner.perform_pre_hook_range_scan_actions(git_head_documents, pre_committed_documents) + sca_code_scanner.perform_pre_hook_range_scan_actions(repo_path, git_head_documents, pre_committed_documents) scan_commit_range_documents( ctx, git_head_documents, @@ -269,14 +271,13 @@ def scan_commit_range( commit_id = commit.hexsha commit_ids_to_scan.append(commit_id) parent = commit.parents[0] if commit.parents else git_proxy.get_null_tree() - diff = commit.diff(parent, create_patch=True, R=True) + diff_index = commit.diff(parent, create_patch=True, R=True) commit_documents_to_scan = [] - for blob in diff: - blob_path = get_path_by_os(os.path.join(path, get_diff_file_path(blob))) + for diff in diff_index: commit_documents_to_scan.append( Document( - path=blob_path, - content=blob.diff.decode('UTF-8', errors='replace'), + path=get_path_by_os(get_diff_file_path(diff)), + content=diff.diff.decode('UTF-8', errors='replace'), is_git_diff_format=True, unique_id=commit_id, ) @@ -413,10 +414,10 @@ def scan_commit_range_documents( _report_scan_status( cycode_client, scan_type, - local_scan_result.scan_id, + scan_id, scan_completed, - local_scan_result.relevant_detections_count, - local_scan_result.detections_count, + relevant_detections_count, + detections_count, len(to_documents_to_scan), zip_file_size, scan_command_type, @@ -658,7 +659,11 @@ def get_scan_parameters(ctx: typer.Context, paths: Optional[tuple[str, ...]] = N scan_parameters['paths'] = paths if len(paths) != 1: - # ignore remote url if multiple paths are provided + logger.debug('Multiple paths provided, going to ignore remote url') + return scan_parameters + + if not os.path.isdir(paths[0]): + logger.debug('Path is not a directory, going to ignore remote url') return scan_parameters remote_url = try_get_git_remote_url(paths[0]) diff --git a/cycode/cli/apps/scan/pre_commit/pre_commit_command.py b/cycode/cli/apps/scan/pre_commit/pre_commit_command.py index b919d659..40e6a8c1 100644 --- a/cycode/cli/apps/scan/pre_commit/pre_commit_command.py +++ b/cycode/cli/apps/scan/pre_commit/pre_commit_command.py @@ -27,14 +27,16 @@ def pre_commit_command( scan_type = ctx.obj['scan_type'] + repo_path = os.getcwd() # change locally for easy testing + progress_bar = ctx.obj['progress_bar'] progress_bar.start() if scan_type == consts.SCA_SCAN_TYPE: - scan_sca_pre_commit(ctx) + scan_sca_pre_commit(ctx, repo_path) return - diff_files = git_proxy.get_repo(os.getcwd()).index.diff('HEAD', create_patch=True, R=True) + diff_files = git_proxy.get_repo(repo_path).index.diff(consts.GIT_HEAD_COMMIT_REV, create_patch=True, R=True) progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, len(diff_files)) diff --git a/cycode/cli/apps/scan/repository/repository_command.py b/cycode/cli/apps/scan/repository/repository_command.py index 16ad8611..c96ca577 100644 --- a/cycode/cli/apps/scan/repository/repository_command.py +++ b/cycode/cli/apps/scan/repository/repository_command.py @@ -1,4 +1,3 @@ -import os from pathlib import Path from typing import Annotated, Optional @@ -44,16 +43,16 @@ def repository_command( progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, len(file_entries)) documents_to_scan = [] - for file in file_entries: + for blob in file_entries: # FIXME(MarshalX): probably file could be tree or submodule too. we expect blob only progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES) - absolute_path = get_path_by_os(os.path.join(path, file.path)) - file_path = file.path if monitor else absolute_path + absolute_path = get_path_by_os(blob.abspath) + file_path = get_path_by_os(blob.path) if monitor else absolute_path documents_to_scan.append( Document( file_path, - file.data_stream.read().decode('UTF-8', errors='replace'), + blob.data_stream.read().decode('UTF-8', errors='replace'), absolute_path=absolute_path, ) ) diff --git a/cycode/cli/cli_types.py b/cycode/cli/cli_types.py index 9b792a01..c2fa12a2 100644 --- a/cycode/cli/cli_types.py +++ b/cycode/cli/cli_types.py @@ -3,42 +3,50 @@ from cycode.cli import consts -class OutputTypeOption(str, Enum): +class StrEnum(str, Enum): + def __str__(self) -> str: + return self.value + + +class OutputTypeOption(StrEnum): RICH = 'rich' TEXT = 'text' JSON = 'json' TABLE = 'table' -class ExportTypeOption(str, Enum): +class ExportTypeOption(StrEnum): JSON = 'json' HTML = 'html' SVG = 'svg' -class ScanTypeOption(str, Enum): +class ScanTypeOption(StrEnum): SECRET = consts.SECRET_SCAN_TYPE SCA = consts.SCA_SCAN_TYPE IAC = consts.IAC_SCAN_TYPE SAST = consts.SAST_SCAN_TYPE + def __str__(self) -> str: + return self.value + -class ScaScanTypeOption(str, Enum): +class ScaScanTypeOption(StrEnum): PACKAGE_VULNERABILITIES = 'package-vulnerabilities' LICENSE_COMPLIANCE = 'license-compliance' -class SbomFormatOption(str, Enum): +class SbomFormatOption(StrEnum): SPDX_2_2 = 'spdx-2.2' SPDX_2_3 = 'spdx-2.3' CYCLONEDX_1_4 = 'cyclonedx-1.4' -class SbomOutputFormatOption(str, Enum): +class SbomOutputFormatOption(StrEnum): JSON = 'json' -class SeverityOption(str, Enum): +class SeverityOption(StrEnum): INFO = 'info' LOW = 'low' MEDIUM = 'medium' diff --git a/cycode/cli/files_collector/repository_documents.py b/cycode/cli/files_collector/repository_documents.py index b524ca4c..379346f8 100644 --- a/cycode/cli/files_collector/repository_documents.py +++ b/cycode/cli/files_collector/repository_documents.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Optional, Union from cycode.cli import consts -from cycode.cli.files_collector.sca import sca_code_scanner +from cycode.cli.files_collector.sca.sca_code_scanner import get_file_content_from_commit_diff from cycode.cli.models import Document from cycode.cli.utils.git_proxy import git_proxy from cycode.cli.utils.path_utils import get_file_content, get_path_by_os @@ -38,8 +38,14 @@ def parse_commit_range(commit_range: str, path: str) -> tuple[str, str]: return from_commit_rev, to_commit_rev -def get_diff_file_path(file: 'Diff') -> Optional[str]: - return file.b_path if file.b_path else file.a_path +def get_diff_file_path(file: 'Diff', relative: bool = False) -> Optional[str]: + if relative: + # relative to the repository root + return file.b_path if file.b_path else file.a_path + + if file.b_blob: + return file.b_blob.abspath + return file.a_blob.abspath def get_diff_file_content(file: 'Diff') -> str: @@ -47,21 +53,21 @@ def get_diff_file_content(file: 'Diff') -> str: def get_pre_commit_modified_documents( - progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection' + progress_bar: 'BaseProgressBar', + progress_bar_section: 'ProgressBarSection', + repo_path: str, ) -> tuple[list[Document], list[Document]]: git_head_documents = [] pre_committed_documents = [] - repo = git_proxy.get_repo(os.getcwd()) - diff_files = repo.index.diff(consts.GIT_HEAD_COMMIT_REV, create_patch=True, R=True) - progress_bar.set_section_length(progress_bar_section, len(diff_files)) - for file in diff_files: + repo = git_proxy.get_repo(repo_path) + diff_index = repo.index.diff(consts.GIT_HEAD_COMMIT_REV, create_patch=True, R=True) + progress_bar.set_section_length(progress_bar_section, len(diff_index)) + for diff in diff_index: progress_bar.update(progress_bar_section) - diff_file_path = get_diff_file_path(file) - file_path = get_path_by_os(diff_file_path) - - file_content = sca_code_scanner.get_file_content_from_commit(repo, consts.GIT_HEAD_COMMIT_REV, diff_file_path) + file_path = get_path_by_os(get_diff_file_path(diff)) + file_content = get_file_content_from_commit_diff(repo, consts.GIT_HEAD_COMMIT_REV, diff) if file_content is not None: git_head_documents.append(Document(file_path, file_content)) @@ -92,14 +98,13 @@ def get_commit_range_modified_documents( for blob in modified_files_diff: progress_bar.update(progress_bar_section) - diff_file_path = get_diff_file_path(blob) - file_path = get_path_by_os(diff_file_path) + file_path = get_path_by_os(get_diff_file_path(blob)) - file_content = sca_code_scanner.get_file_content_from_commit(repo, from_commit_rev, diff_file_path) + file_content = get_file_content_from_commit_diff(repo, from_commit_rev, blob) if file_content is not None: from_commit_documents.append(Document(file_path, file_content)) - file_content = sca_code_scanner.get_file_content_from_commit(repo, to_commit_rev, diff_file_path) + file_content = get_file_content_from_commit_diff(repo, to_commit_rev, blob) if file_content is not None: to_commit_documents.append(Document(file_path, file_content)) diff --git a/cycode/cli/files_collector/sca/sca_code_scanner.py b/cycode/cli/files_collector/sca/sca_code_scanner.py index e6ec0e9d..b9988122 100644 --- a/cycode/cli/files_collector/sca/sca_code_scanner.py +++ b/cycode/cli/files_collector/sca/sca_code_scanner.py @@ -1,4 +1,3 @@ -import os from typing import TYPE_CHECKING, Optional import typer @@ -18,7 +17,7 @@ from cycode.logger import get_logger if TYPE_CHECKING: - from git import Repo + from git import Diff, Repo BUILD_DEP_TREE_TIMEOUT = 180 @@ -39,9 +38,9 @@ def perform_pre_commit_range_scan_actions( def perform_pre_hook_range_scan_actions( - git_head_documents: list[Document], pre_committed_documents: list[Document] + repo_path: str, git_head_documents: list[Document], pre_committed_documents: list[Document] ) -> None: - repo = git_proxy.get_repo(os.getcwd()) + repo = git_proxy.get_repo(repo_path) add_ecosystem_related_files_if_exists(git_head_documents, repo, consts.GIT_HEAD_COMMIT_REV) add_ecosystem_related_files_if_exists(pre_committed_documents) @@ -69,7 +68,7 @@ def get_doc_ecosystem_related_project_files( file_to_search = join_paths(get_file_dir(doc.path), ecosystem_project_file) if not is_project_file_exists_in_documents(documents, file_to_search): if repo: - file_content = get_file_content_from_commit(repo, commit_rev, file_to_search) + file_content = get_file_content_from_commit_path(repo, commit_rev, file_to_search) else: file_content = get_file_content(file_to_search) @@ -151,13 +150,20 @@ def get_manifest_file_path(document: Document, is_monitor_action: bool, project_ return join_paths(project_path, document.path) if is_monitor_action else document.path -def get_file_content_from_commit(repo: 'Repo', commit: str, file_path: str) -> Optional[str]: +def get_file_content_from_commit_path(repo: 'Repo', commit: str, file_path: str) -> Optional[str]: try: return repo.git.show(f'{commit}:{file_path}') except git_proxy.get_git_command_error(): return None +def get_file_content_from_commit_diff(repo: 'Repo', commit: str, diff: 'Diff') -> Optional[str]: + from cycode.cli.files_collector.repository_documents import get_diff_file_path + + file_path = get_diff_file_path(diff, relative=True) + return get_file_content_from_commit_path(repo, commit, file_path) + + def perform_pre_scan_documents_actions( ctx: typer.Context, scan_type: str, documents_to_scan: list[Document], is_git_diff: bool = False ) -> None: diff --git a/cycode/cli/printers/console_printer.py b/cycode/cli/printers/console_printer.py index 17c402ff..50d48fd7 100644 --- a/cycode/cli/printers/console_printer.py +++ b/cycode/cli/printers/console_printer.py @@ -28,9 +28,8 @@ class ConsolePrinter: 'text': TextPrinter, 'json': JsonPrinter, 'table': TablePrinter, - # overrides + # overrides: 'table_sca': ScaTablePrinter, - 'text_sca': ScaTablePrinter, } def __init__( diff --git a/cycode/cli/printers/rich_printer.py b/cycode/cli/printers/rich_printer.py index 755278d6..7ee0f853 100644 --- a/cycode/cli/printers/rich_printer.py +++ b/cycode/cli/printers/rich_printer.py @@ -54,47 +54,69 @@ def _get_details_table(self, detection: 'Detection') -> Table: severity_icon = SeverityOption.get_member_emoji(severity.lower()) details_table.add_row('Severity', f'{severity_icon} {SeverityOption(severity).__rich__()}') - detection_details = detection.detection_details - path = str(get_detection_file_path(self.scan_type, detection)) shorten_path = f'...{path[-self.MAX_PATH_LENGTH :]}' if len(path) > self.MAX_PATH_LENGTH else path details_table.add_row('In file', f'[link=file://{path}]{shorten_path}[/]') - if self.scan_type == consts.SECRET_SCAN_TYPE: - details_table.add_row('Secret SHA', detection_details.get('sha512')) - elif self.scan_type == consts.SCA_SCAN_TYPE: - details_table.add_row('CVEs', get_detection_clickable_cwe_cve(self.scan_type, detection)) - details_table.add_row('Package', detection_details.get('package_name')) - details_table.add_row('Version', detection_details.get('package_version')) - - is_package_vulnerability = 'alert' in detection_details - if is_package_vulnerability: - details_table.add_row( - 'First patched version', detection_details['alert'].get('first_patched_version', 'Not fixed') - ) - - details_table.add_row('Dependency path', detection_details.get('dependency_paths', 'N/A')) - - if not is_package_vulnerability: - details_table.add_row('License', detection_details.get('license')) - elif self.scan_type == consts.IAC_SCAN_TYPE: - details_table.add_row('IaC Provider', detection_details.get('infra_provider')) - elif self.scan_type == consts.SAST_SCAN_TYPE: - details_table.add_row('CWE', get_detection_clickable_cwe_cve(self.scan_type, detection)) - details_table.add_row('Subcategory', detection_details.get('category')) - details_table.add_row('Language', ', '.join(detection_details.get('languages', []))) - - engine_id_to_display_name = { - '5db84696-88dc-11ec-a8a3-0242ac120002': 'Semgrep OSS (Orchestrated by Cycode)', - '560a0abd-d7da-4e6d-a3f1-0ed74895295c': 'Bearer (Powered by Cycode)', - } - engine_id = detection.detection_details.get('external_scanner_id') - details_table.add_row('Security Tool', engine_id_to_display_name.get(engine_id, 'N/A')) + self._add_scan_related_rows(details_table, detection) details_table.add_row('Rule ID', detection.detection_rule_id) return details_table + def _add_scan_related_rows(self, details_table: Table, detection: 'Detection') -> None: + scan_type_details_handlers = { + consts.SECRET_SCAN_TYPE: self.__add_secret_scan_related_rows, + consts.SCA_SCAN_TYPE: self.__add_sca_scan_related_rows, + consts.IAC_SCAN_TYPE: self.__add_iac_scan_related_rows, + consts.SAST_SCAN_TYPE: self.__add_sast_scan_related_rows, + } + + if self.scan_type not in scan_type_details_handlers: + raise ValueError(f'Unknown scan type: {self.scan_type}') + + scan_enricher_function = scan_type_details_handlers[self.scan_type] + scan_enricher_function(details_table, detection) + + @staticmethod + def __add_secret_scan_related_rows(details_table: Table, detection: 'Detection') -> None: + details_table.add_row('Secret SHA', detection.detection_details.get('sha512')) + + @staticmethod + def __add_sca_scan_related_rows(details_table: Table, detection: 'Detection') -> None: + detection_details = detection.detection_details + + details_table.add_row('CVEs', get_detection_clickable_cwe_cve(consts.SCA_SCAN_TYPE, detection)) + details_table.add_row('Package', detection_details.get('package_name')) + details_table.add_row('Version', detection_details.get('package_version')) + + if detection.has_alert: + patched_version = detection_details['alert'].get('patched_version') + details_table.add_row('First patched version', patched_version or 'Not fixed') + + dependency_path = detection_details.get('dependency_paths') + details_table.add_row('Dependency path', dependency_path or 'N/A') + + if not detection.has_alert: + details_table.add_row('License', detection_details.get('license')) + + @staticmethod + def __add_iac_scan_related_rows(details_table: Table, detection: 'Detection') -> None: + details_table.add_row('IaC Provider', detection.detection_details.get('infra_provider')) + + @staticmethod + def __add_sast_scan_related_rows(details_table: Table, detection: 'Detection') -> None: + details_table.add_row('CWE', get_detection_clickable_cwe_cve(consts.SAST_SCAN_TYPE, detection)) + details_table.add_row('Subcategory', detection.detection_details.get('category')) + details_table.add_row('Language', ', '.join(detection.detection_details.get('languages', []))) + + engine_id_to_display_name = { + '5db84696-88dc-11ec-a8a3-0242ac120002': 'Semgrep OSS (Orchestrated by Cycode)', + '560a0abd-d7da-4e6d-a3f1-0ed74895295c': 'Bearer (Powered by Cycode)', + } + engine_id = detection.detection_details.get('external_scanner_id') + details_table.add_row('Security Tool', engine_id_to_display_name.get(engine_id, 'N/A')) + def _print_violation_card( self, document: 'Document', detection: 'Detection', detection_number: int, detections_count: int ) -> None: @@ -117,8 +139,7 @@ def _print_violation_card( title=':computer: Code Snippet', ) - is_sca_package_vulnerability = self.scan_type == consts.SCA_SCAN_TYPE and 'alert' in detection.detection_details - if is_sca_package_vulnerability: + if detection.has_alert: summary = detection.detection_details['alert'].get('description') else: summary = detection.detection_details.get('description') or detection.message diff --git a/cycode/cli/printers/text_printer.py b/cycode/cli/printers/text_printer.py index 05a360fd..51da53c5 100644 --- a/cycode/cli/printers/text_printer.py +++ b/cycode/cli/printers/text_printer.py @@ -1,5 +1,6 @@ from typing import TYPE_CHECKING, Optional +from cycode.cli import consts from cycode.cli.cli_types import SeverityOption from cycode.cli.models import CliError, CliResult, Document from cycode.cli.printers.printer_base import PrinterBase @@ -66,10 +67,34 @@ def __print_detection_summary(self, detection: 'Detection', document_path: str) self.console.print( severity_icon, severity, - f'violation: [b bright_red]{title}[/]{detection_commit_id_message}\n' + f'violation: [b bright_red]{title}[/]{detection_commit_id_message}\n', + *self.__get_intermediate_summary_lines(detection), f'[dodger_blue1]File: {clickable_document_path}[/]', ) + def __get_intermediate_summary_lines(self, detection: 'Detection') -> list[str]: + intermediate_summary_lines = [] + + if self.scan_type == consts.SCA_SCAN_TYPE: + intermediate_summary_lines.extend(self.__get_sca_related_summary_lines(detection)) + + return intermediate_summary_lines + + @staticmethod + def __get_sca_related_summary_lines(detection: 'Detection') -> list[str]: + summary_lines = [] + + if detection.has_alert: + patched_version = detection.detection_details['alert'].get('first_patched_version') + patched_version = patched_version or 'Not fixed' + + summary_lines.append(f'First patched version: [cyan]{patched_version}[/]\n') + else: + package_license = detection.detection_details.get('license', 'N/A') + summary_lines.append(f'License: [cyan]{package_license}[/]\n') + + return summary_lines + def __print_detection_code_segment(self, detection: 'Detection', document: Document) -> None: self.console.print( get_code_snippet_syntax( diff --git a/cycode/cli/printers/utils/detection_data.py b/cycode/cli/printers/utils/detection_data.py index 989a6600..37bee310 100644 --- a/cycode/cli/printers/utils/detection_data.py +++ b/cycode/cli/printers/utils/detection_data.py @@ -83,7 +83,7 @@ def get_detection_title(scan_type: str, detection: 'Detection') -> str: elif scan_type == consts.SECRET_SCAN_TYPE: title = f'Hardcoded {detection.type} is used' - is_sca_package_vulnerability = scan_type == consts.SCA_SCAN_TYPE and 'alert' in detection.detection_details + is_sca_package_vulnerability = scan_type == consts.SCA_SCAN_TYPE and detection.has_alert if is_sca_package_vulnerability: title = detection.detection_details['alert'].get('summary', 'N/A') diff --git a/cycode/cyclient/models.py b/cycode/cyclient/models.py index 70e3e551..ed649644 100644 --- a/cycode/cyclient/models.py +++ b/cycode/cyclient/models.py @@ -33,6 +33,15 @@ def __repr__(self) -> str: f'detection_rule_id:{self.detection_rule_id}' ) + @property + def has_alert(self) -> bool: + """Check if the detection has an alert. + + For example, for SCA, it means that the detection is a package vulnerability. + Otherwise, it is a license. + """ + return 'alert' in self.detection_details + class DetectionSchema(Schema): class Meta: From b07c4332c6e2e30b11fce5dff9231cf6f4786bfd Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Wed, 14 May 2025 15:36:39 +0200 Subject: [PATCH 19/19] CM-48357 - Fix SCA restore error handling (#309) --- cycode/cli/apps/scan/code_scanner.py | 16 +++--- .../sca/base_restore_dependencies.py | 51 +++++++++---------- .../sca/go/restore_go_dependencies.py | 3 -- .../sca/maven/restore_gradle_dependencies.py | 11 ++-- .../sca/maven/restore_maven_dependencies.py | 26 +++++----- .../sca/npm/restore_npm_dependencies.py | 3 -- .../sca/nuget/restore_nuget_dependencies.py | 5 -- .../sca/ruby/restore_ruby_dependencies.py | 3 -- .../sca/sbt/restore_sbt_dependencies.py | 3 -- cycode/cli/utils/shell_executor.py | 8 ++- 10 files changed, 55 insertions(+), 74 deletions(-) diff --git a/cycode/cli/apps/scan/code_scanner.py b/cycode/cli/apps/scan/code_scanner.py index c6337021..e7dff93f 100644 --- a/cycode/cli/apps/scan/code_scanner.py +++ b/cycode/cli/apps/scan/code_scanner.py @@ -683,8 +683,8 @@ def try_get_git_remote_url(path: str) -> Optional[str]: remote_url = git_proxy.get_repo(path).remotes[0].config_reader.get('url') logger.debug('Found Git remote URL, %s', {'remote_url': remote_url, 'path': path}) return remote_url - except Exception as e: - logger.debug('Failed to get Git remote URL', exc_info=e) + except Exception: + logger.debug('Failed to get Git remote URL. Probably not a Git repository') return None @@ -706,7 +706,9 @@ def _get_plastic_repository_name(path: str) -> Optional[str]: f'--fieldseparator={consts.PLASTIC_VCS_DATA_SEPARATOR}', ] - status = shell(command=command, timeout=consts.PLASTIC_VSC_CLI_TIMEOUT, working_directory=path) + status = shell( + command=command, timeout=consts.PLASTIC_VSC_CLI_TIMEOUT, working_directory=path, silent_exc_info=True + ) if not status: logger.debug('Failed to get Plastic repository name (command failed)') return None @@ -717,8 +719,8 @@ def _get_plastic_repository_name(path: str) -> Optional[str]: return None return status_parts[2].strip() - except Exception as e: - logger.debug('Failed to get Plastic repository name', exc_info=e) + except Exception: + logger.debug('Failed to get Plastic repository name. Probably not a Plastic repository') return None @@ -738,7 +740,9 @@ def _get_plastic_repository_list(working_dir: Optional[str] = None) -> dict[str, try: command = ['cm', 'repo', 'ls', f'--format={{repname}}{consts.PLASTIC_VCS_DATA_SEPARATOR}{{repguid}}'] - status = shell(command=command, timeout=consts.PLASTIC_VSC_CLI_TIMEOUT, working_directory=working_dir) + status = shell( + command=command, timeout=consts.PLASTIC_VSC_CLI_TIMEOUT, working_directory=working_dir, silent_exc_info=True + ) if not status: logger.debug('Failed to get Plastic repository list (command failed)') return repo_name_to_guid diff --git a/cycode/cli/files_collector/sca/base_restore_dependencies.py b/cycode/cli/files_collector/sca/base_restore_dependencies.py index c4364c05..ea8a0bb7 100644 --- a/cycode/cli/files_collector/sca/base_restore_dependencies.py +++ b/cycode/cli/files_collector/sca/base_restore_dependencies.py @@ -1,9 +1,9 @@ +import os from abc import ABC, abstractmethod from typing import Optional import typer -from cycode.cli.logger import logger from cycode.cli.models import Document from cycode.cli.utils.path_utils import get_file_content, get_file_dir, get_path_from_context, join_paths from cycode.cli.utils.shell_executor import shell @@ -15,30 +15,27 @@ def build_dep_tree_path(path: str, generated_file_name: str) -> str: def execute_commands( commands: list[list[str]], - file_name: str, - command_timeout: int, - dependencies_file_name: Optional[str] = None, + timeout: int, + output_file_path: Optional[str] = None, working_directory: Optional[str] = None, ) -> Optional[str]: try: - all_dependencies = [] + outputs = [] - # Run all commands and collect outputs for command in commands: - dependencies = shell(command=command, timeout=command_timeout, working_directory=working_directory) - all_dependencies.append(dependencies) # Collect each command's output + command_output = shell(command=command, timeout=timeout, working_directory=working_directory) + if command_output: + outputs.append(command_output) - dependencies = '\n'.join(all_dependencies) + joined_output = '\n'.join(outputs) - # Write all collected outputs to the file if dependencies_file_name is provided - if dependencies_file_name: - with open(dependencies_file_name, 'w') as output_file: # Open once in 'w' mode to start fresh - output_file.writelines(dependencies) - except Exception as e: - logger.debug('Failed to restore dependencies via shell command, %s', {'filename': file_name}, exc_info=e) + if output_file_path: + with open(output_file_path, 'w', encoding='UTF-8') as output_file: + output_file.writelines(joined_output) + except Exception: return None - return dependencies + return joined_output class BaseRestoreDependencies(ABC): @@ -64,27 +61,25 @@ def try_restore_dependencies(self, document: Document) -> Optional[Document]: relative_restore_file_path = build_dep_tree_path(document.path, self.get_lock_file_name()) working_directory_path = self.get_working_directory(document) - if self.verify_restore_file_already_exist(restore_file_path): - restore_file_content = get_file_content(restore_file_path) - else: - output_file_path = restore_file_path if self.create_output_file_manually else None - execute_commands( + if not self.verify_restore_file_already_exist(restore_file_path): + output = execute_commands( self.get_commands(manifest_file_path), - manifest_file_path, self.command_timeout, - output_file_path, - working_directory_path, + output_file_path=restore_file_path if self.create_output_file_manually else None, + working_directory=working_directory_path, ) - restore_file_content = get_file_content(restore_file_path) + if output is None: # one of the commands failed + return None + restore_file_content = get_file_content(restore_file_path) return Document(relative_restore_file_path, restore_file_content, self.is_git_diff) def get_working_directory(self, document: Document) -> Optional[str]: return None - @abstractmethod - def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: - pass + @staticmethod + def verify_restore_file_already_exist(restore_file_path: str) -> bool: + return os.path.isfile(restore_file_path) @abstractmethod def is_project(self, document: Document) -> bool: diff --git a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py index 4f469896..6eb48a76 100644 --- a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py +++ b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py @@ -44,8 +44,5 @@ def get_commands(self, manifest_file_path: str) -> list[list[str]]: def get_lock_file_name(self) -> str: return GO_RESTORE_FILE_NAME - def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: - return os.path.isfile(restore_file_path) - def get_working_directory(self, document: Document) -> Optional[str]: return os.path.dirname(document.absolute_path) diff --git a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py index 89595e0e..777ae727 100644 --- a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py @@ -42,22 +42,19 @@ def get_commands(self, manifest_file_path: str) -> list[list[str]]: def get_lock_file_name(self) -> str: return BUILD_GRADLE_DEP_TREE_FILE_NAME - def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: - return os.path.isfile(restore_file_path) - def get_working_directory(self, document: Document) -> Optional[str]: return get_path_from_context(self.ctx) if self.is_gradle_sub_projects() else None def get_all_projects(self) -> set[str]: - projects_output = shell( + output = shell( command=BUILD_GRADLE_ALL_PROJECTS_COMMAND, timeout=BUILD_GRADLE_ALL_PROJECTS_TIMEOUT, working_directory=get_path_from_context(self.ctx), ) + if not output: + return set() - projects = re.findall(ALL_PROJECTS_REGEX, projects_output) - - return set(projects) + return set(re.findall(ALL_PROJECTS_REGEX, output)) def get_commands_for_sub_projects(self, manifest_file_path: str) -> list[list[str]]: project_name = os.path.basename(os.path.dirname(manifest_file_path)) diff --git a/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py index 1c3d860c..b9a2b1ed 100644 --- a/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py @@ -1,4 +1,3 @@ -import os from os import path from typing import Optional @@ -30,9 +29,6 @@ def get_commands(self, manifest_file_path: str) -> list[list[str]]: def get_lock_file_name(self) -> str: return join_paths('target', MAVEN_CYCLONE_DEP_TREE_FILE_NAME) - def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: - return os.path.isfile(restore_file_path) - def try_restore_dependencies(self, document: Document) -> Optional[Document]: restore_dependencies_document = super().try_restore_dependencies(document) manifest_file_path = self.get_manifest_file_path(document) @@ -51,8 +47,8 @@ def restore_from_secondary_command( self, document: Document, manifest_file_path: str, restore_dependencies_document: Optional[Document] ) -> Optional[Document]: # TODO(MarshalX): does it even work? Ignored restore_dependencies_document arg - secondary_restore_command = create_secondary_restore_command(manifest_file_path) - backup_restore_content = execute_commands(secondary_restore_command, manifest_file_path, self.command_timeout) + secondary_restore_command = create_secondary_restore_commands(manifest_file_path) + backup_restore_content = execute_commands(secondary_restore_command, self.command_timeout) restore_dependencies_document = Document( build_dep_tree_path(document.path, MAVEN_DEP_TREE_FILE_NAME), backup_restore_content, self.is_git_diff ) @@ -64,13 +60,15 @@ def restore_from_secondary_command( return restore_dependencies -def create_secondary_restore_command(manifest_file_path: str) -> list[str]: +def create_secondary_restore_commands(manifest_file_path: str) -> list[list[str]]: return [ - 'mvn', - 'dependency:tree', - '-B', - '-DoutputType=text', - '-f', - manifest_file_path, - f'-DoutputFile={MAVEN_DEP_TREE_FILE_NAME}', + [ + 'mvn', + 'dependency:tree', + '-B', + '-DoutputType=text', + '-f', + manifest_file_path, + f'-DoutputFile={MAVEN_DEP_TREE_FILE_NAME}', + ] ] diff --git a/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py b/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py index ed8e36c2..2563612f 100644 --- a/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py +++ b/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py @@ -33,9 +33,6 @@ def get_commands(self, manifest_file_path: str) -> list[list[str]]: def get_lock_file_name(self) -> str: return NPM_LOCK_FILE_NAME - def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: - return os.path.isfile(restore_file_path) - @staticmethod def prepare_manifest_file_path_for_command(manifest_file_path: str) -> str: return manifest_file_path.replace(os.sep + NPM_MANIFEST_FILE_NAME, '') diff --git a/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py b/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py index 3bd6627f..3035e206 100644 --- a/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py +++ b/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py @@ -1,5 +1,3 @@ -import os - import typer from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies @@ -21,6 +19,3 @@ def get_commands(self, manifest_file_path: str) -> list[list[str]]: def get_lock_file_name(self) -> str: return NUGET_LOCK_FILE_NAME - - def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: - return os.path.isfile(restore_file_path) diff --git a/cycode/cli/files_collector/sca/ruby/restore_ruby_dependencies.py b/cycode/cli/files_collector/sca/ruby/restore_ruby_dependencies.py index 4571b1c5..8c256f27 100644 --- a/cycode/cli/files_collector/sca/ruby/restore_ruby_dependencies.py +++ b/cycode/cli/files_collector/sca/ruby/restore_ruby_dependencies.py @@ -18,8 +18,5 @@ def get_commands(self, manifest_file_path: str) -> list[list[str]]: def get_lock_file_name(self) -> str: return RUBY_LOCK_FILE_NAME - def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: - return os.path.isfile(restore_file_path) - def get_working_directory(self, document: Document) -> Optional[str]: return os.path.dirname(document.absolute_path) diff --git a/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py b/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py index d7eeba3b..26a88646 100644 --- a/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py +++ b/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py @@ -18,8 +18,5 @@ def get_commands(self, manifest_file_path: str) -> list[list[str]]: def get_lock_file_name(self) -> str: return SBT_LOCK_FILE_NAME - def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: - return os.path.isfile(restore_file_path) - def get_working_directory(self, document: Document) -> Optional[str]: return os.path.dirname(document.absolute_path) diff --git a/cycode/cli/utils/shell_executor.py b/cycode/cli/utils/shell_executor.py index db0331da..2529890b 100644 --- a/cycode/cli/utils/shell_executor.py +++ b/cycode/cli/utils/shell_executor.py @@ -16,6 +16,7 @@ def shell( command: Union[str, list[str]], timeout: int = _SUBPROCESS_DEFAULT_TIMEOUT_SEC, working_directory: Optional[str] = None, + silent_exc_info: bool = False, ) -> Optional[str]: logger.debug('Executing shell command: %s', command) @@ -27,12 +28,15 @@ def shell( return result.stdout.decode('UTF-8').strip() except subprocess.CalledProcessError as e: - logger.debug('Error occurred while running shell command', exc_info=e) + if not silent_exc_info: + logger.debug('Error occurred while running shell command', exc_info=e) except subprocess.TimeoutExpired as e: logger.debug('Command timed out', exc_info=e) raise typer.Abort(f'Command "{command}" timed out') from e except Exception as e: - logger.debug('Unhandled exception occurred while running shell command', exc_info=e) + if not silent_exc_info: + logger.debug('Unhandled exception occurred while running shell command', exc_info=e) + raise click.ClickException(f'Unhandled exception: {e}') from e return None