diff --git a/.gitignore b/.gitignore index bd92954..fd83719 100644 --- a/.gitignore +++ b/.gitignore @@ -212,3 +212,4 @@ docs/* rules/* tests/* valkyrie/repositories/* +output.json \ No newline at end of file diff --git a/output.json b/output.json new file mode 100644 index 0000000..f005528 --- /dev/null +++ b/output.json @@ -0,0 +1,299 @@ +{ + "scan_id": "scan_2025-09-17T18:32:00.001935", + "status": "completed", + "timestamp": "2025-09-17T18:32:00.802998", + "scan_duration": 0.800964, + "summary": { + "total_findings": 0, + "critical": 0, + "high": 0, + "has_blocking_issues": false + }, + "findings": [], + "scanned_files": [ + ".venv/lib/python3.10/site-packages/pydantic/_internal/_decorators.py", + ".venv/lib/python3.10/site-packages/toml/encoder.py", + ".venv/lib/python3.10/site-packages/pydantic/_migration.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_utils.py", + "valkyrie/rules/repositories/local_repo.py", + ".venv/lib/python3.10/site-packages/typing_extensions.py", + ".venv/lib/python3.10/site-packages/yarl/_parse.py", + ".venv/lib/python3.10/site-packages/aiohttp/multipart.py", + "valkyrie/utils/__init__.py", + "output.json", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_config.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/errors.py", + ".venv/lib/python3.10/site-packages/pydantic/types.py", + ".venv/lib/python3.10/site-packages/yaml/loader.py", + ".venv/lib/python3.10/site-packages/aiohttp/web_response.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_known_annotated_metadata.py", + ".venv/lib/python3.10/site-packages/_yaml/__init__.py", + ".venv/lib/python3.10/site-packages/toml/decoder.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/parse.py", + ".venv/lib/python3.10/site-packages/attr/validators.py", + ".venv/lib/python3.10/site-packages/pydantic/deprecated/copy_internals.py", + ".venv/lib/python3.10/site-packages/attrs/validators.py", + ".venv/lib/python3.10/site-packages/idna/compat.py", + ".venv/lib/python3.10/site-packages/typing_inspection/typing_objects.py", + ".venv/lib/python3.10/site-packages/pydantic/error_wrappers.py", + ".venv/lib/python3.10/site-packages/__editable___valkyrie_0_1_0_finder.py", + ".venv/lib/python3.10/site-packages/aiohttp/web_exceptions.py", + ".venv/lib/python3.10/site-packages/aiohttp/connector.py", + ".venv/lib/python3.10/site-packages/aiohttp/cookiejar.py", + ".venv/lib/python3.10/site-packages/yarl/_quoting_py.py", + ".venv/lib/python3.10/site-packages/idna/idnadata.py", + "valkyrie/core/types.py", + "valkyrie/__main__.py", + ".venv/lib/python3.10/site-packages/aiohttp/formdata.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_repr.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/env_settings.py", + ".venv/lib/python3.10/site-packages/yaml/parser.py", + ".venv/lib/python3.10/site-packages/annotated_types/test_cases.py", + ".venv/lib/python3.10/site-packages/valkyrie-0.1.0.dist-info/direct_url.json", + ".venv/lib/python3.10/site-packages/pydantic/plugin/_loader.py", + ".venv/lib/python3.10/site-packages/aiohttp/_websocket/__init__.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_dataclasses.py", + ".venv/lib/python3.10/site-packages/pydantic/networks.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/annotated_types.py", + "valkyrie/cli.py", + ".venv/lib/python3.10/site-packages/aiohttp/tcp_helpers.py", + ".venv/lib/python3.10/site-packages/pydantic/mypy.py", + ".venv/lib/python3.10/site-packages/attrs/filters.py", + "valkyrie/core/formatters/base.py", + ".venv/lib/python3.10/site-packages/pydantic/json.py", + ".venv/lib/python3.10/site-packages/multidict/_multidict_py.py", + ".venv/lib/python3.10/site-packages/frozenlist/__init__.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/config.py", + ".venv/lib/python3.10/site-packages/aiohttp/web.py", + ".venv/lib/python3.10/site-packages/aiohttp/web_fileresponse.py", + ".venv/lib/python3.10/site-packages/aiosignal/__init__.py", + ".venv/lib/python3.10/site-packages/aiohttp/helpers.py", + ".venv/lib/python3.10/site-packages/attr/_compat.py", + ".venv/lib/python3.10/site-packages/aiohttp/client_proto.py", + ".venv/lib/python3.10/site-packages/aiohttp/typedefs.py", + ".venv/lib/python3.10/site-packages/yaml/reader.py", + "valkyrie/plugins/manager.py", + ".venv/lib/python3.10/site-packages/typing_inspection/introspection.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/typing.py", + ".venv/lib/python3.10/site-packages/aiohttp/compression_utils.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_model_construction.py", + ".venv/lib/python3.10/site-packages/pydantic/validate_call_decorator.py", + ".venv/lib/python3.10/site-packages/pydantic/errors.py", + ".venv/lib/python3.10/site-packages/pydantic/warnings.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/__init__.py", + ".venv/lib/python3.10/site-packages/yarl/_query.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py", + ".venv/lib/python3.10/site-packages/yamllib-0.0.1.dist-info/uv_build.json", + ".venv/lib/python3.10/site-packages/pydantic/v1/tools.py", + ".venv/lib/python3.10/site-packages/pydantic/parse.py", + ".venv/lib/python3.10/site-packages/propcache/_helpers.py", + ".venv/lib/python3.10/site-packages/attr/_version_info.py", + ".venv/lib/python3.10/site-packages/aiohttp/client_middleware_digest_auth.py", + ".venv/lib/python3.10/site-packages/idna/codec.py", + "valkyrie/rules/repositories/github_repo.py", + "valkyrie.yaml", + "valkyrie/core/formatters/sarif.py", + ".venv/lib/python3.10/site-packages/aiohttp/hdrs.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/__init__.py", + ".venv/lib/python3.10/site-packages/aiohttp/http.py", + ".venv/lib/python3.10/site-packages/pydantic/root_model.py", + ".venv/lib/python3.10/site-packages/aiohttp/_websocket/models.py", + ".venv/lib/python3.10/site-packages/pydantic/env_settings.py", + ".venv/lib/python3.10/site-packages/propcache/_helpers_py.py", + ".venv/lib/python3.10/site-packages/aiohappyeyeballs/__init__.py", + ".venv/lib/python3.10/site-packages/attrs/exceptions.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_decorators_v1.py", + ".venv/lib/python3.10/site-packages/pydantic/alias_generators.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/decorator.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/validators.py", + "valkyrie/rules/repositories/__init__.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/class_validators.py", + ".venv/lib/python3.10/site-packages/tomli/_parser.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/fields.py", + "valkyrie/plugins/vulnera/__init__.py", + ".venv/lib/python3.10/site-packages/yaml/dumper.py", + "valkyrie/core/formatters/json.py", + "valkyrie/plugins/vulnera/conf.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/generics.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/main.py", + ".venv/lib/python3.10/site-packages/aiohttp/payload_streamer.py", + ".venv/lib/python3.10/site-packages/aiohttp/client_ws.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_core_metadata.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_fields.py", + ".venv/lib/python3.10/site-packages/attr/filters.py", + ".venv/lib/python3.10/site-packages/pydantic/config.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/utils.py", + ".venv/lib/python3.10/site-packages/aiohttp/web_middlewares.py", + ".venv/lib/python3.10/site-packages/aiohttp/web_routedef.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_schema_generation_shared.py", + ".venv/lib/python3.10/site-packages/attr/_next_gen.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_discriminated_union.py", + ".venv/lib/python3.10/site-packages/aiohttp/web_server.py", + ".venv/lib/python3.10/site-packages/toml/ordered.py", + ".venv/lib/python3.10/site-packages/propcache/__init__.py", + ".venv/lib/python3.10/site-packages/annotated_types/__init__.py", + ".venv/lib/python3.10/site-packages/yaml/nodes.py", + ".venv/lib/python3.10/site-packages/aiohttp/client_exceptions.py", + ".venv/lib/python3.10/site-packages/yaml/composer.py", + ".venv/lib/python3.10/site-packages/aiohttp/http_exceptions.py", + ".venv/lib/python3.10/site-packages/aiohttp/_websocket/reader_py.py", + ".venv/bin/activate_this.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_schema_gather.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/color.py", + ".venv/lib/python3.10/site-packages/multidict/__init__.py", + ".venv/lib/python3.10/site-packages/aiohttp/payload.py", + ".venv/lib/python3.10/site-packages/aiohttp/test_utils.py", + ".venv/lib/python3.10/site-packages/idna/package_data.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_import_utils.py", + ".venv/lib/python3.10/site-packages/tomli/__init__.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_signature.py", + ".venv/lib/python3.10/site-packages/_virtualenv.py", + ".venv/lib/python3.10/site-packages/valkyrie-0.1.0.dist-info/uv_build.json", + "valkyrie/core/scanner.py", + ".venv/lib/python3.10/site-packages/aiohttp/web_log.py", + ".venv/lib/python3.10/site-packages/pydantic/__init__.py", + ".venv/lib/python3.10/site-packages/pydantic_core/core_schema.py", + ".venv/lib/python3.10/site-packages/pydantic/experimental/arguments_schema.py", + ".venv/lib/python3.10/site-packages/pydantic/tools.py", + ".venv/lib/python3.10/site-packages/pydantic/deprecated/json.py", + ".venv/lib/python3.10/site-packages/aiohttp/pytest_plugin.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/_hypothesis_plugin.py", + ".venv/lib/python3.10/site-packages/aiohttp/_websocket/helpers.py", + ".venv/lib/python3.10/site-packages/attrs/converters.py", + ".venv/lib/python3.10/site-packages/pydantic/json_schema.py", + ".venv/lib/python3.10/site-packages/attr/_funcs.py", + ".venv/lib/python3.10/site-packages/yaml/emitter.py", + ".venv/lib/python3.10/site-packages/yaml/scanner.py", + ".venv/lib/python3.10/site-packages/aiohttp/http_writer.py", + ".venv/lib/python3.10/site-packages/aiohttp/abc.py", + ".venv/lib/python3.10/site-packages/aiohttp/streams.py", + ".venv/lib/python3.10/site-packages/pydantic_core/__init__.py", + "valkyrie/core/formatters/html.py", + ".venv/lib/python3.10/site-packages/idna/core.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_docs_extraction.py", + ".venv/lib/python3.10/site-packages/attr/exceptions.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_generics.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_git.py", + ".venv/lib/python3.10/site-packages/tomli/_types.py", + "valkyrie/plugins/iamx/__init__.py", + ".venv/lib/python3.10/site-packages/toml/tz.py", + "valkyrie/plugins/iamx/conf.py", + "valkyrie/rules/base.py", + ".venv/lib/python3.10/site-packages/pydantic/annotated_handlers.py", + ".venv/lib/python3.10/site-packages/yaml/representer.py", + ".venv/lib/python3.10/site-packages/aiohttp/web_ws.py", + "valkyrie/rules/secrets/__init__.py", + "valkyrie/core/__init__.py", + "valkyrie/rules/secrets/conf.py", + ".venv/lib/python3.10/site-packages/pydantic/validators.py", + ".venv/lib/python3.10/site-packages/attrs/setters.py", + ".venv/lib/python3.10/site-packages/pydantic/decorator.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/version.py", + ".venv/lib/python3.10/site-packages/attr/_config.py", + ".venv/lib/python3.10/site-packages/pydantic/class_validators.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/datetime_parse.py", + ".venv/lib/python3.10/site-packages/yaml/tokens.py", + ".venv/lib/python3.10/site-packages/aiohttp/_cookie_helpers.py", + ".venv/lib/python3.10/site-packages/aiohttp/web_protocol.py", + ".venv/lib/python3.10/site-packages/aiohttp/_websocket/writer.py", + ".venv/lib/python3.10/site-packages/yarl/_quoters.py", + ".venv/lib/python3.10/site-packages/pydantic/fields.py", + ".venv/lib/python3.10/site-packages/idna/__init__.py", + ".venv/lib/python3.10/site-packages/pydantic/functional_validators.py", + ".venv/lib/python3.10/site-packages/pydantic/generics.py", + ".venv/lib/python3.10/site-packages/propcache/api.py", + ".venv/lib/python3.10/site-packages/yaml/__init__.py", + ".venv/lib/python3.10/site-packages/pydantic/main.py", + ".venv/lib/python3.10/site-packages/attr/_make.py", + ".venv/lib/python3.10/site-packages/typing_inspection/__init__.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_namespace_utils.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/schema.py", + ".venv/lib/python3.10/site-packages/toml/__init__.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_serializers.py", + ".venv/lib/python3.10/site-packages/pydantic/experimental/__init__.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/dataclasses.py", + ".venv/lib/python3.10/site-packages/attr/_cmp.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_internal_dataclass.py", + ".venv/lib/python3.10/site-packages/multidict/_abc.py", + "valkyrie/core/formatters/__init__.py", + ".venv/lib/python3.10/site-packages/pydantic/color.py", + ".venv/lib/python3.10/site-packages/aiohttp/_websocket/reader.py", + ".venv/lib/python3.10/site-packages/aiohappyeyeballs/_staggered.py", + ".venv/lib/python3.10/site-packages/pydantic/typing.py", + ".venv/lib/python3.10/site-packages/multidict/_compat.py", + ".venv/lib/python3.10/site-packages/aiohttp/http_parser.py", + ".venv/lib/python3.10/site-packages/aiohttp/client_middlewares.py", + ".venv/lib/python3.10/site-packages/pydantic/type_adapter.py", + ".venv/lib/python3.10/site-packages/pydantic/experimental/pipeline.py", + ".venv/lib/python3.10/site-packages/pydantic/plugin/__init__.py", + ".venv/lib/python3.10/site-packages/aiohttp/web_urldispatcher.py", + ".venv/lib/python3.10/site-packages/pydantic/deprecated/config.py", + "valkyrie/utils/context.py", + ".venv/lib/python3.10/site-packages/aiohttp/http_websocket.py", + ".venv/lib/python3.10/site-packages/aiohttp/_websocket/reader_c.py", + ".venv/lib/python3.10/site-packages/pydantic/aliases.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/types.py", + ".venv/lib/python3.10/site-packages/aiohappyeyeballs/utils.py", + ".venv/lib/python3.10/site-packages/attr/converters.py", + ".venv/lib/python3.10/site-packages/aiohttp/log.py", + "valkyrie/utils/logger.py", + ".venv/lib/python3.10/site-packages/async_timeout/__init__.py", + ".venv/lib/python3.10/site-packages/aiohttp/resolver.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_forward_ref.py", + ".venv/lib/python3.10/site-packages/yarl/_url.py", + ".venv/lib/python3.10/site-packages/aiohttp/__init__.py", + ".github/workflows/lint-format.yml", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_core_utils.py", + ".venv/lib/python3.10/site-packages/aiohttp/web_runner.py", + ".venv/lib/python3.10/site-packages/idna/intranges.py", + ".venv/lib/python3.10/site-packages/aiohttp/tracing.py", + "valkyrie/rules/repositories/base.py", + ".venv/lib/python3.10/site-packages/valkyrie-0.1.0.dist-info/uv_cache.json", + "valkyrie/plugins/base.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/error_wrappers.py", + ".venv/lib/python3.10/site-packages/aiohttp/web_app.py", + ".venv/lib/python3.10/site-packages/pydantic/version.py", + ".venv/lib/python3.10/site-packages/yaml/serializer.py", + ".venv/lib/python3.10/site-packages/aiohttp/worker.py", + ".venv/lib/python3.10/site-packages/pydantic/deprecated/__init__.py", + ".venv/lib/python3.10/site-packages/yaml/error.py", + ".venv/lib/python3.10/site-packages/pydantic/datetime_parse.py", + ".venv/lib/python3.10/site-packages/attr/setters.py", + "valkyrie/core/conf.py", + ".venv/lib/python3.10/site-packages/pydantic/deprecated/tools.py", + ".venv/lib/python3.10/site-packages/aiohappyeyeballs/types.py", + "valkyrie/plugins/vulnera/parser.py", + ".venv/lib/python3.10/site-packages/aiohappyeyeballs/impl.py", + ".venv/lib/python3.10/site-packages/yaml/events.py", + ".venv/lib/python3.10/site-packages/aiohttp/base_protocol.py", + ".venv/lib/python3.10/site-packages/yarl/_quoting.py", + ".venv/lib/python3.10/site-packages/yaml/resolver.py", + ".venv/lib/python3.10/site-packages/pydantic/functional_serializers.py", + ".venv/lib/python3.10/site-packages/pydantic/deprecated/parse.py", + ".venv/lib/python3.10/site-packages/aiohttp/client_reqrep.py", + ".venv/lib/python3.10/site-packages/yaml/constructor.py", + ".venv/lib/python3.10/site-packages/yaml/cyaml.py", + ".venv/lib/python3.10/site-packages/idna/uts46data.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_mock_val_ser.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_validate_call.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_validators.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/networks.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/mypy.py", + ".venv/lib/python3.10/site-packages/tomli/_re.py", + ".venv/lib/python3.10/site-packages/yarl/__init__.py", + ".venv/lib/python3.10/site-packages/pydantic/utils.py", + ".venv/lib/python3.10/site-packages/pydantic/v1/json.py", + ".venv/lib/python3.10/site-packages/aiohttp/web_request.py", + ".venv/lib/python3.10/site-packages/pydantic/schema.py", + ".venv/lib/python3.10/site-packages/pydantic/dataclasses.py", + ".venv/lib/python3.10/site-packages/pydantic/_internal/_typing_extra.py", + ".venv/lib/python3.10/site-packages/attr/__init__.py", + ".venv/lib/python3.10/site-packages/yarl/_path.py", + ".venv/lib/python3.10/site-packages/aiohttp/client.py", + ".venv/lib/python3.10/site-packages/pydantic/deprecated/decorator.py", + ".venv/lib/python3.10/site-packages/pydantic/plugin/_schema_validator.py", + ".venv/lib/python3.10/site-packages/attrs/__init__.py", + ".venv/lib/python3.10/site-packages/pydantic/deprecated/class_validators.py" + ], + "errors": [] +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 019ee38..29d6c6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,9 +12,9 @@ keywords = [ "pipelines", "iam", "secrets" ] dependencies = [ + "aiohttp>=3.12.15", "pydantic>=2.11.9", "toml>=0.10.2", - "tomli>=2.2.1", "yamllib>=0.0.1", ] @@ -32,6 +32,9 @@ dev = [ "mkdocs-static-i18n>=1.3.0", ] +[project.scripts] +valkyrie = "valkyrie.__main__:main" + [project.urls] # Homepage = "https://alldotpy.github.io/FletX" Repository = "https://github.com/AllDotPy/Valkyrie" diff --git a/valkyrie.yaml b/valkyrie.yaml new file mode 100644 index 0000000..3620707 --- /dev/null +++ b/valkyrie.yaml @@ -0,0 +1,152 @@ +# Core scanner settings +scanner: + # Target directory to scan (relative to config file) + target_path: "." + + # File inclusion patterns (glob patterns) + include_patterns: + - "**/*.py" + - "**/*.js" + - "**/*.ts" + - "**/*.java" + - "**/*.go" + - "**/*.rs" + - "**/*.yaml" + - "**/*.yml" + - "**/*.json" + - "**/*.tf" + - "**/*.hcl" + + # File exclusion patterns + exclude_patterns: + - "**/.git/**" + - "**/.vscode/**" + - "**/node_modules/**" + - "**/__pycache__/**" + - "**/target/**" + - "**/build/**" + - "**/dist/**" + - "**/*.min.js" + - "**/*.map" + - ".venv/**" + + # Maximum file size to scan (in bytes) + max_file_size: 10485760 # 10MB + + # Number of parallel scanning workers + parallel_workers: 4 + + # Minimum severity level to report + severity_threshold: "low" # critical, high, medium, low, info + + # Whether to fail the build on security findings + fail_on_findings: true + + # Scan only changed files in CI (if supported) + diff_only: false + +# Rule configuration +rules: + # Remote rule repository (GitHub, GitLab, etc.) + repository: + type: "github" + url: "AllDotPy/valkyrie-community-rules" + branch: "master" + token_env: "GITHUB_TOKEN" # Environment variable for authentication + + # Local rules directory + local_rules_dir: "./rules" + + # Rule filters (empty = all rules enabled) + include_rules: [] + + # Disabled rules + exclude_rules: [] + + # Custom rule categories to enable/disable + categories: + secrets: true + dependencies: true + iam_config: true + code_quality: false + infrastructure: true + +# Plugin configuration +plugins: + # Built-in plugins + secrets_detector: + enabled: true + config: + # Custom entropy threshold for secrets detection + entropy_threshold: 3.5 + # Additional secret patterns (regex) + custom_patterns: + - name: "Custom API Key" + pattern: "CUSTOM_[A-Z0-9]{32}" + keywords: ["custom", "api"] + + dependency_scanner: + enabled: true + config: + # Vulnerability database sources + databases: + - "osv.dev" + - "nvd.nist.gov" + # Cache duration for vulnerability data (hours) + cache_duration: 24 + # Check dev dependencies + include_dev_deps: true + + iam_scanner: + enabled: true + config: + # Cloud providers to check + providers: ["aws", "gcp", "azure"] + # Custom IAM policy patterns + custom_patterns: + - name: "Wildcard Principal" + pattern: '"Principal":\s*"\*"' + severity: "critical" + +# Output configuration +output: + # Default output format + format: "json" # json, sarif, html + + # Output file (optional) + file: "output.json" + + # Verbose logging + verbose: false + + # Include successful scans in output + include_success: false + +# CI/CD Integration settings +ci_integration: + # GitHub Actions + - github: + # Post results as check runs + create_check_runs: true + # Comment on Pull Requests + comment_on_pr: true + # Create annotations for findings + create_annotations: true + # Maximum annotations to create + max_annotations: 50 + + # GitLab CI + - gitlab: + # Post merge request notes + post_mr_notes: true + # Create commit statuses + create_commit_status: true + # Generate SAST reports + generate_sast_report: true + + # Generic CI settings + - generic: + # Exit codes + success_exit_code: 0 + findings_exit_code: 1 + failure_exit_code: 2 diff --git a/valkyrie/__main__.py b/valkyrie/__main__.py new file mode 100644 index 0000000..2289801 --- /dev/null +++ b/valkyrie/__main__.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- + +""" MAIN ENTRY POINT FOR Valkyrie CLI +This module serves as the main entry point for the Valkyrie CLI. It handles command-line +arguments, initializes the command registry, and executes the specified command. +""" + +import sys +import asyncio +from valkyrie.cli import ( + ValkyrieCLI +) + +import tracemalloc +tracemalloc.start() + +async def main(): + """ + Main function to handle command-line arguments and execute commands. + This function checks if any command is provided, retrieves the command class + from the command registry, and executes the command with the provided arguments. + If no command is provided, it lists all available commands. + """ + + cli = ValkyrieCLI() + exit_code = await cli.run() + sys.exit(exit_code) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/valkyrie/cli.py b/valkyrie/cli.py new file mode 100644 index 0000000..6149639 --- /dev/null +++ b/valkyrie/cli.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +""" +Valkyrie CLI - A command-line tool. +""" + +import argparse +from typing import List +from pathlib import Path + +import yaml + +from valkyrie.core import ( + ValkyrieConfig, ValkyrieScanner, + ScanStatus, get_formatter +) +from valkyrie.rules.repositories import ( + LocalRuleRepository, GitHubRuleRepository +) +from valkyrie.utils import ( + get_context, + get_logger +) + + +#### +## VALKYRIE CLI ENTRY POINT +##### +class ValkyrieCLI: + """Command line interface for Valkyrie scanner""" + + def __init__(self): + + self.logger = get_logger('Valkyrie.CLI') + self.context = get_context() + # self.ci_integrations = [ + # GitHubActionsIntegration(), + # GitLabCIIntegration() + # ] + # self.formatters = { + # "json": JSONFormatter(), + # "sarif": SARIFFormatter(), + # "html": HTMLFormatter() + # } + ... + + def create_parser(self) -> argparse.ArgumentParser: + """Create CLI argument parser""" + parser = argparse.ArgumentParser( + description="Valkyrie - Security scanner for CI/CD pipelines", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + valkyrie scan # Scan current directory + valkyrie scan --target # Target directoory to scan + valkyrie scan --config-file # Valyrie config file + """ + ) + + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + # Scan command + scan_parser = subparsers.add_parser("scan", help="Run security scan") + scan_parser.add_argument("target", nargs="?", default=".", help="Target path to scan") + scan_parser.add_argument("--config-file", "-f", nargs="?", default="valkyrie.yml", help="Valkyrie configuration file path") + # scan_parser.add_argument("--format", choices=["json", "sarif", "html"], default="json", help="Output format") + # scan_parser.add_argument("--output", "-o", type=Path, help="Output file path") + # scan_parser.add_argument("--severity", choices=["critical", "high", "medium", "low", "info"], default="low", help="Minimum severity level") + # scan_parser.add_argument("--rules-repo", default="valkyrie-community/rules", help="Rules repository") + # scan_parser.add_argument("--no-fail", action="store_true", help="Don't fail on findings") + # scan_parser.add_argument("--include", action="append", help="Include patterns") + # scan_parser.add_argument("--exclude", action="append", help="Exclude patterns") + # scan_parser.add_argument("--workers", type=int, default=4, help="Number of parallel workers") + # scan_parser.add_argument("--diff-only", action="store_true", help="Only scan changed files") + scan_parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output") + + # Version command + subparsers.add_parser("version", help="Show version information") + + return parser + + async def run(self, args: List[str] = None) -> int: + """Run CLI with given arguments""" + + parser = self.create_parser() + parsed_args = parser.parse_args(args) + + if parsed_args.command == "version": + print("Valkyrie Security Scanner v1.0.0") + return 0 + + elif parsed_args.command == "scan": + return await self._run_scan(parsed_args) + + else: + parser.print_help() + return 1 + + async def _run_scan(self, args) -> int: + """Execute security scan""" + + # Load Config file + file = Path(args.config_file) + + print(file) + + # Create configuration + config = ValkyrieConfig.from_json( + yaml.safe_load(file.read_text()) + ) + + # Add cmd args to the global context + self.context.set_data('cmd_args', args) + + # Detect and register configured plugins + # PluginManager.load_from_config(config.plugins) + + + # Detect CI environment + # active_ci = None + # for ci_integration in self.ci_integrations: + # if ci_integration.detect_environment(): + # active_ci = ci_integration + # break + + # if active_ci and args.verbose: + # print(f"Detected CI environment: {active_ci.__class__.__name__}") + + # # Filter files if diff-only mode + # if args.diff_only and active_ci: + # changed_files = active_ci.get_changed_files() + # if changed_files: + # # Convert to include patterns + # config.scanner.include_patterns = [str(f) for f in changed_files] + # if args.verbose: + # print(f"Scanning {len(changed_files)} changed files") + + # Set up scanner + + + # Init GH repository with config options + gh_repo = GitHubRuleRepository( + config = config.rules.repository + ) + # local_repo = LocalRuleRepository(Path("rules")) # In practice, use GitHub repo + scanner = ValkyrieScanner( + rule_repository = gh_repo, + config = config.scanner + ) + + # Run scan + if args.verbose: + print(f"Starting security scan of {config.scanner.target_path}") + + result = await scanner.scan() + + # Format results + formatter = get_formatter(config.output.format) + formatted_output = formatter().format(result) + + # Output results + if config.output.file: + Path(config.output.file).write_text(formatted_output, encoding='utf-8') + print(f"Results saved to: {args.output}") + else: + print(formatted_output) + + # Post to CI if in CI environment + # if active_ci: + # await active_ci.post_results(result, {}) + + # Determine exit code + if result.status == ScanStatus.FAILED: + return 2 # Scan failure + elif config.scanner.fail_on_findings and result.has_blocking_issues: + return 1 # Security issues found + else: + return 0 # Success + \ No newline at end of file diff --git a/valkyrie/core/__init__.py b/valkyrie/core/__init__.py new file mode 100644 index 0000000..034f430 --- /dev/null +++ b/valkyrie/core/__init__.py @@ -0,0 +1,37 @@ +from .formatters import ( + ResultFormatter, JSONFormatter, HTMLFormatter, + SARIFFormatter, get_formatter +) +from .conf import ( + LogLevel, LogFormat, LoggingConfig, + BaseConfigModel, PluginConfig, OutputConfig, + RuleRepositoryConfig, RuleCategoriesConfig, + RulesConfig, ScanConfig, ValkyrieConfig +) +from .types import ( + SeverityLevel, FindingCategory, ScanStatus, + FileLocation, SecurityFinding, ScanResult, + RuleMetadata, ScanRule, RuleRepository +) +from .scanner import ValkyrieScanner + + +__all__ = [ + # SCANNER + ValkyrieScanner, + + # SHARED TYPES + SeverityLevel, FindingCategory, ScanStatus, + FileLocation, SecurityFinding, ScanResult, + RuleMetadata, ScanRule, RuleRepository, + + # CONFIG MODELS + LogLevel, LogFormat, LoggingConfig, + BaseConfigModel, PluginConfig, OutputConfig, + RuleRepositoryConfig, RuleCategoriesConfig, + RulesConfig, ScanConfig, ValkyrieConfig, + + # FORMATTERS + ResultFormatter, JSONFormatter, HTMLFormatter, + SARIFFormatter, get_formatter, +] \ No newline at end of file diff --git a/valkyrie/core/conf.py b/valkyrie/core/conf.py index d4ee22d..07c712f 100644 --- a/valkyrie/core/conf.py +++ b/valkyrie/core/conf.py @@ -11,11 +11,12 @@ from pydantic import BaseModel, Field -from valkyrie.core.types import SeverityLevel +from .types import SeverityLevel #### GENERIC TYPES T = TypeVar('T', bound='BaseConfigModel') + #### ## LOG LEVELs ##### @@ -136,7 +137,7 @@ class RuleRepositoryConfig(BaseConfigModel): type: str = 'github' url: str = 'AllDotPt/valkyrie-community-rules' - branch: str = 'main' + branch: str = 'master' token_env: str = 'GITHUB_TOKEN' @@ -229,10 +230,12 @@ class ValkyrieConfig(BaseConfigModel): scanner: ScanConfig """Scanner engine configuration.""" - rules: Optional[RulesConfig] + rules: Optional[RulesConfig] = Field( + default_factory = lambda: RulesConfig() + ) """Rule repositories""" - plugins: List[PluginConfig] + plugins: Dict[str,Any] """Scanner Plugins to use.""" output: OutputConfig = Field( @@ -240,5 +243,7 @@ class ValkyrieConfig(BaseConfigModel): ) """Scan Result Outout format""" - ci_integration: List + ci_integration: List = Field( + default_factory= lambda: [] + ) """A list of ci integration (github action, gitlab ci, etc)""" diff --git a/valkyrie/core/scanner.py b/valkyrie/core/scanner.py index c815c2f..ef8219f 100644 --- a/valkyrie/core/scanner.py +++ b/valkyrie/core/scanner.py @@ -8,12 +8,13 @@ import logging from datetime import datetime - -from valkyrie.core.types import ( - RuleRepository, ScannerPlugin, ScanRule, - ScanConfig, SecurityFinding, ScanResult, +# from ..plugins.base import ScannerPlugin +from .types import ( + RuleRepository, ScanRule, + SecurityFinding, ScanResult, ScanStatus ) +from .conf import ScanConfig #### @@ -27,14 +28,16 @@ class ValkyrieScanner: def __init__( self, rule_repository: RuleRepository, + config: ScanConfig, logger: Optional[logging.Logger] = None ): + self.config = config self.rule_repository = rule_repository self.logger = logger or logging.getLogger(__name__) - self.plugins: Dict[str, ScannerPlugin] = {} + # self.plugins: Dict[str, ScannerPlugin] = {} self._rules_cache: Optional[List[ScanRule]] = None - async def register_plugin(self, plugin: ScannerPlugin) -> None: + async def register_plugin(self, plugin) -> None: """Register a scanner plugin""" # Initialize the plugin @@ -66,9 +69,9 @@ async def _load_all_rules(self) -> List[ScanRule]: rules = await self.rule_repository.load_rules() # Load from plugins - for plugin in self.plugins.values(): - plugin_rules = await plugin.get_rules() - rules.extend(plugin_rules) + # for plugin in self.plugins.values(): + # plugin_rules = await plugin.get_rules() + # rules.extend(plugin_rules) self._rules_cache = rules return rules @@ -138,7 +141,7 @@ async def _scan_file( return findings - async def scan(self, config: ScanConfig) -> ScanResult: + async def scan(self, config: Optional[ScanConfig] = None) -> ScanResult: """ Execute security scan based on configuration @@ -149,6 +152,8 @@ async def scan(self, config: ScanConfig) -> ScanResult: Complete scan results """ + # Detect config + config = config or self.config scan_id = f"scan_{datetime.now().isoformat()}" start_time = datetime.now() @@ -206,6 +211,8 @@ async def scan_with_semaphore(file_path: Path) -> List[SecurityFinding]: except Exception as e: self.logger.error(f"Scan failed: {e}") + import traceback + traceback.print_exc() return ScanResult( scan_id=scan_id, status=ScanStatus.FAILED, diff --git a/valkyrie/core/types.py b/valkyrie/core/types.py index 4fdd886..06be82b 100644 --- a/valkyrie/core/types.py +++ b/valkyrie/core/types.py @@ -193,8 +193,8 @@ def is_applicable(self, file_path: Path) -> bool: #### ## SCANNER PLUGINS BASE CLASS ##### -class ScannerPlugin(ABC): - """Abstract base class for scanner plugins""" +class Plugin(ABC): + """Abstract base class for all plugins""" @property @abstractmethod @@ -213,11 +213,6 @@ async def initialize(self, config: Dict[str, Any]) -> None: """Initialize the plugin with configuration""" pass - @abstractmethod - async def get_rules(self) -> List[ScanRule]: - """Return list of rules provided by this plugin""" - pass - @abstractmethod async def cleanup(self) -> None: """Cleanup plugin resources""" diff --git a/valkyrie/plugins/base.py b/valkyrie/plugins/base.py new file mode 100644 index 0000000..0abe6c3 --- /dev/null +++ b/valkyrie/plugins/base.py @@ -0,0 +1,140 @@ +""" +Valkyrie - Plugin module. +""" +from abc import abstractmethod +from pathlib import Path +from typing import ( + List, Dict, Any, Optional, + ClassVar, Type, Union, TypeVar +) + +from valkyrie.utils import ( + import_module_from, +) +from ..core.types import ( + ScanRule, + Plugin +) +from ..core.conf import PluginConfig + + +#### PLUGINS GENERIC TYPE +T = TypeVar('T', bound = 'PluginConfig') + +#### PLUGINS DIR +PLUGINS_DIR = 'valkyrie.plugins.{}' + + +#### +## SCANNER PLUGIN +#### +class ScannerPlugin(Plugin): + """Base class for all Valkyrie Scanner Plugins.""" + + config_class: Type[PluginConfig] = PluginConfig + """Plugin configuration model class.""" + + rule_config_class: Type[] + + category: + + def __init__( + self, + config: Optional[PluginConfig] = None + ): + """Initialize plugin.""" + + self.config = config + + @classmethod + def with_config(cls, config: Dict[str, Any]): + """Iniitiallize plugin with given config.""" + + return cls(cls.config_class.from_json(config)) + + @abstractmethod + async def get_rules(self) -> List[ScanRule]: + """Return list of rules provided by this plugin""" + pass + + +#### +## PLUGINS REGISTRY CLASS +##### +class PluginsRegistry: + """ + Registry for all valkyrie plugins. + This class is used to register and retrieve plugins based on their names. + """ + + _registry: ClassVar[Dict[str, Type[ScannerPlugin]]] = {} # type: ignore + + @classmethod + def register(cls, name: Optional[str] = None) -> None: # type: ignore + """Register a new plugin class.""" + + def wrapper(plugin: Type[ScannerPlugin]): + """Wrapper""" + + nonlocal name + name = name or plugin.name() + if name not in cls._registry.keys(): + cls._registry[name] = plugin + + return plugin + + return wrapper + + @classmethod + def get(cls, name: str) -> Type[ScannerPlugin]: # type: ignore + """Get a registered Plugin class by its name.""" + + if name not in cls._registry: + raise ValueError( + f"Invalid Adapter name: '{name}' not found." + ) + + return cls._registry[name] + + @classmethod + def all(cls) -> List[Type[ScannerPlugin]]: # type: ignore + """Get all registered Plugin classes.""" + return list(cls._registry.values()) + + @classmethod + def clear(cls) -> None: + """Clear the registry.""" + cls._registry.clear() + + @classmethod + def list(cls) -> List[str]: + """List all registered Plugins names.""" + return list(cls._registry.keys()) + + +#### UTILITIES +def register_plugin(name: Optional[str] = None): + """ + A Shirtcut to register valkyrie plugins. + + Usage: + ```python + @register_plugin() + class Mylugin(ScannerPlugin): + ... + + @property + def name(self): + return 'my-plugin' + ``` + """ + + return PluginsRegistry.register(name = name) + +#### IMPORRT PLUGIN +def import_plugin(name: Union[str,Path]): + """Load plugin from plugin dir""" + + return import_module_from( + PLUGINS_DIR.format(name) + ) \ No newline at end of file diff --git a/valkyrie/plugins/iamx/__init__.py b/valkyrie/plugins/iamx/__init__.py index b1a34db..4a43374 100644 --- a/valkyrie/plugins/iamx/__init__.py +++ b/valkyrie/plugins/iamx/__init__.py @@ -2,11 +2,23 @@ from pathlib import Path from typing import Dict, List, Any +<<<<<<< HEAD +from valkyrie.core import ( + BaseSecurityRule, ScanRule, + RuleMetadata, SecurityFinding, + FileLocation, SeverityLevel, FindingCategory, +) +from valkyrie.plugins.base import ( + ScannerPlugin, register_plugin +) + +======= from valkyrie.plugins import BaseSecurityRule from valkyrie.core.types import ( ScanRule, ScannerPlugin, RuleMetadata, SecurityFinding, FileLocation, SeverityLevel, FindingCategory, ) +>>>>>>> 46c4453be87e5209b03d8e2d67126063fb2c3b99 from .conf import RISKY_PATTERNS @@ -115,6 +127,10 @@ def _detect_cloud_provider(self, content: str) -> str: #### ## IAM CONFIGURATION PLUGIN ##### +<<<<<<< HEAD +@register_plugin() +======= +>>>>>>> 46c4453be87e5209b03d8e2d67126063fb2c3b99 class IAMPlugin(ScannerPlugin): """Plugin for IAM configuration scanning""" diff --git a/valkyrie/plugins/__init__.py b/valkyrie/plugins/manager.py similarity index 60% rename from valkyrie/plugins/__init__.py rename to valkyrie/plugins/manager.py index 45df10a..2474170 100644 --- a/valkyrie/plugins/__init__.py +++ b/valkyrie/plugins/manager.py @@ -1,46 +1,23 @@ """ -Valkyrie - Plugin module. +Valkyrie - Plugin Manager. """ -from pathlib import Path -from typing import List, Set, Dict, Any, Optional +from typing import ( + List, Set, Dict, Any, Optional, +) import logging +from valkyrie.utils import ( + get_logger, + get_context +) from valkyrie.core.types import ( - RuleMetadata, SecurityFinding, ScanRule, - ScannerPlugin + ScanRule, ) - -#### -## BASE CLASS FOR SECURITY RULE IMPLEMENTATION -##### -class BaseSecurityRule(ScanRule): - """Base implementation for security rules""" - - def __init__( - self, - metadata: RuleMetadata, - logger: Optional[logging.Logger] = None - ): - self._metadata = metadata - self.logger = logger or logging.getLogger(__name__) - - @property - def metadata(self) -> RuleMetadata: - return self._metadata - - def is_applicable(self, file_path: Path) -> bool: - """Default implementation - override in subclasses""" - return True - - async def scan( - self, - file_path: Path, - content: str - ) -> List[SecurityFinding]: - """Override in subclasses""" - - return [] +from .base import ( + ScannerPlugin, import_plugin, + PluginsRegistry +) #### @@ -55,7 +32,39 @@ def __init__( ): self.plugins: Dict[str, ScannerPlugin] = {} self.enabled_plugins: Set[str] = set() - self.logger = logger or logging.getLogger(__name__) + self.logger = logger or get_logger('Valkyrie.PluginManager') + + + @classmethod + async def load_from_config(cls, config: Dict[str,Any]): + """Detect and load plugins from config""" + + ctx = get_context() + cmd_args = ctx.get_data('cmd_arrgs') + + plugins = config + for plugin_name in plugins.keys(): + + try: + import_plugin(plugin_name) + # Log if verbose + if cmd_args.verbose: + cls.logger.info( + f'{plugin_name} plugin loaded...' + ) + + # Error importing plugin + except Exception as e: + if cmd_args.verbose: + cls.logger.warning( + f'Error loading {plugin_name} {e}' + ) + + # Use Regiistry to initialize plugins + for plugin in PluginsRegistry.all(): + await cls.register_plugin( + plugin.with_config(config.get(plugin.name)) + ) async def register_plugin( self, diff --git a/valkyrie/plugins/vulnera/__init__.py b/valkyrie/plugins/vulnera/__init__.py index f4619c3..bf9951e 100644 --- a/valkyrie/plugins/vulnera/__init__.py +++ b/valkyrie/plugins/vulnera/__init__.py @@ -2,12 +2,16 @@ from pathlib import Path from typing import Dict, List, Any -from valkyrie.plugins import BaseSecurityRule -from valkyrie.core.types import ( - ScanRule, ScannerPlugin, RuleMetadata, SecurityFinding, +from valkyrie.core import ( + BaseSecurityRule, ScanRule, + RuleMetadata, SecurityFinding, FileLocation, SeverityLevel, FindingCategory, ) +from valkyrie.plugins.base import ( + ScannerPlugin, register_plugin +) + from .conf import VulnerabilityInfo from .parser import parse_dependencies, is_supported @@ -115,6 +119,7 @@ def _is_version_affected( #### ## VULNERABILITY PLUGIN ##### +@register_plugin() class DependenciesPlugin(ScannerPlugin): """Plugin for dependency vulnerability scanning""" diff --git a/valkyrie/rules/base.py b/valkyrie/rules/base.py new file mode 100644 index 0000000..cd5ed43 --- /dev/null +++ b/valkyrie/rules/base.py @@ -0,0 +1,245 @@ +""" +Valkyrie - Rules module. +""" +import logging +from pathlib import Path +from abc import abstractmethod +from typing import ( + List, Optional, Dict, Any, ClassVar, + Type, Set, Union, TypeVar +) + +from pydantic import Field + +from valkyrie.core import ( + RuleMetadata, SecurityFinding, ScanRule, + BaseConfigModel, FindingCategory, SeverityLevel +) +from valkyrie.utils import ( + get_logger, +) + +T = TypeVar('T', bound = 'BaseSecurityRule') +V = TypeVar('V', bound = 'BaseRuleConfig') + +#### +## SECURITY RULE CONFIG +##### +class BaseRuleConfig(BaseConfigModel): + """Base class for all Secutiry rules configuration models.""" + + id: str + """Rulle identifyer""" + + name: str + """Rulee name""" + + description: Optional[str] = None + """Rulle Description (Optional)""" + + category: FindingCategory + """Rule Category""" + + severity: SeverityLevel = SeverityLevel.LOW + """Rule severity level""" + + author: Optional[str] = 'Valkyrie Community' + """Rule author""" + + version: Optional[str] = '1.0.0' + """Rule version""" + + tags: Set[str] = Field( + default_factory = lambda: set + ) + """Rule tags""" + + enabled: bool = True + """Is rule enabled / disabe""" + + file_extensions: Set[str] = Field( + default_factory = lambda: set + ) + """supported File extensions for the rule""" + + exclude_patterns: Set[str] = Field( + default_factory = lambda: set + ) + """Files to exclude""" + + remediation: Optional[str] = None + """Remediiation message""" + + references: Optional[List[str]] = None + """References""" + + +#### +## BASE CLASS FOR SECURITY RULE IMPLEMENTATION +##### +class BaseSecurityRule(ScanRule): + """Base implementation for security rules""" + + name: ClassVar[str] + description: str = 'Valkyrie Detection Rule' + + def __init__( + self, + config: V, + logger: Optional[logging.Logger] = None + ): + self.config: V = config + self._metadata = self.build_metadada() + + # Configure logger + self.logger = logger or get_logger( + f'Valkyrie.{self.__class__.__name__}' + ) + + @property + def metadata(self) -> RuleMetadata: + return self._metadata + + @classmethod + def from_config( + cls, + config: BaseRuleConfig + ) -> 'T': + """Load and return a SecretsDetection Rule instance from config""" + + ... + + @classmethod + def from_yaml( + cls, + file: Union[str, Path] + )-> 'T': + """Load Rule fom .yaml file""" + + ... + + @classmethod + def from_json( + cls, + json_content: Dict[str, Any] + )-> 'T': + """Load Rule fom json content (python dict)""" + + ... + + def build_metadada(self,config: V) -> RuleMetadata: + """Build rule metadata from config""" + + return RuleMetadata( + id = config.id or f"secrets-{id(self)}", + name = config.name or "Generic Secrets Detection", + description = ( + config.description + or self.description + ), + category = FindingCategory(config.category), + severity = SeverityLevel(self.config.severity), + author = config.author or "Valkyrie Core Team", + tags = set(config.tags) + ) + + def to_json(self) -> Dict[str, Any]: + """Return rule as Json object""" + + return self.config.to_json() + + def is_applicable(self, file_path: Path) -> bool: + """Default implementation - override in subclasses""" + return True + + async def scan( + self, + file_path: Path, + content: str + ) -> List[SecurityFinding]: + """Override in subclasses""" + + return [] + + @abstractmethod + async def initialize(self, config: Dict[str, Any]) -> None: + """Initialize the plugin with configuration""" + pass + + @abstractmethod + async def cleanup(self) -> None: + """Cleanup plugin resources""" + pass + + +#### +## RULES REGISTRY CLASS +##### +class RulesRegistry: + """ + Registry for all valkyrie Rules. + This class is used to register and retrieve Rules based on their categories. + """ + + _registry: ClassVar[Dict[str, Type[BaseSecurityRule]]] = {} # type: ignore + + @classmethod + def register(cls) -> None: # type: ignore + """Register a new rule class.""" + + def wrapper(rule_class: Type[BaseSecurityRule]): + """Wrapper""" + + name = rule_class.metadata.category + if name not in cls._registry.keys(): + cls._registry[name] = rule_class + + return rule_class + + return wrapper + + @classmethod + def get(cls, name: str) -> Type[BaseSecurityRule]: # type: ignore + """Get a registered Rule class by its name.""" + + if name not in cls._registry: + raise ValueError( + f"Invalid Adapter name: '{name}' not found." + ) + + return cls._registry[name] + + @classmethod + def all(cls) -> List[Type[BaseSecurityRule]]: # type: ignore + """Get all registered Rule classes.""" + return list(cls._registry.values()) + + @classmethod + def clear(cls) -> None: + """Clear the registry.""" + cls._registry.clear() + + @classmethod + def list(cls) -> List[str]: + """List all registered rules names.""" + return list(cls._registry.keys()) + + +#### UTILITIES +def register_rule(): + """ + A Shirtcut to register valkyrie plugins. + + Usage: + ```python + @register_rule() + class MylRule(BaseSecurityRule): + ... + + @property + def name(self): + return 'my-plugin' + ``` + """ + + return RulesRegistry.register() diff --git a/valkyrie/rules/repositories/__init__.py b/valkyrie/rules/repositories/__init__.py new file mode 100644 index 0000000..273bb36 --- /dev/null +++ b/valkyrie/rules/repositories/__init__.py @@ -0,0 +1,7 @@ +from .local_repo import LocalRuleRepository +from .github_repo import GitHubRuleRepository + +__all__ = [ + LocalRuleRepository, + GitHubRuleRepository +] \ No newline at end of file diff --git a/valkyrie/rules/repositories/base.py b/valkyrie/rules/repositories/base.py new file mode 100644 index 0000000..8e79340 --- /dev/null +++ b/valkyrie/rules/repositories/base.py @@ -0,0 +1,51 @@ +#### +## RULES REGISTRY CLASS +##### +class RulesRegistry: + """ + Registry for all valkyrie Rules. + This class is used to register and retrieve Rules based on their categories. + """ + + _registry: ClassVar[Dict[str, Type[BaseSecurityRule]]] = {} # type: ignore + + @classmethod + def register(cls) -> None: # type: ignore + """Register a new rule class.""" + + def wrapper(rule_class: Type[BaseSecurityRule]): + """Wrapper""" + + name = rule_class.metadata.category + if name not in cls._registry.keys(): + cls._registry[name] = rule_class + + return rule_class + + return wrapper + + @classmethod + def get(cls, name: str) -> Type[BaseSecurityRule]: # type: ignore + """Get a registered Rule class by its name.""" + + if name not in cls._registry: + raise ValueError( + f"Invalid Adapter name: '{name}' not found." + ) + + return cls._registry[name] + + @classmethod + def all(cls) -> List[Type[BaseSecurityRule]]: # type: ignore + """Get all registered Rule classes.""" + return list(cls._registry.values()) + + @classmethod + def clear(cls) -> None: + """Clear the registry.""" + cls._registry.clear() + + @classmethod + def list(cls) -> List[str]: + """List all registered rules names.""" + return list(cls._registry.keys()) diff --git a/valkyrie/rules/repositories/github_repo.py b/valkyrie/rules/repositories/github_repo.py new file mode 100644 index 0000000..57dcbe0 --- /dev/null +++ b/valkyrie/rules/repositories/github_repo.py @@ -0,0 +1,171 @@ +import os +import re +import hashlib +from pathlib import Path +from typing import Dict, List, Optional, Any + +import yaml +import aiohttp + +from valkyrie.core import ( + RuleRepository, SecurityFinding, + RuleMetadata, FileLocation, RuleRepositoryConfig +) +from valkyrie.rules.base import RulesRegistry, BaseSecurityRule + + +#### +## GITHUB RULE REPOSITORY +##### +class GitHubRuleRepository(RuleRepository): + """Rule repository backed by GitHub""" + + def __init__( + self, + config: RuleRepositoryConfig + ): + self.repository = config.url + self.branch = config.branch + self.token = os.getenv(config.token_env) + self.rules_cache: Dict[str, BaseSecurityRule] = {} + self.base_url = f"https://api.github.com/repos/{self.repository}" + + async def load_rules(self) -> List[BaseSecurityRule]: + """Load rules from GitHub repository""" + + async with aiohttp.ClientSession() as session: + headers = {} + if self.token: + headers["Authorization"] = f"token {self.token}" + + # Get repository contents + url = f"{self.base_url}/contents/rules?ref={self.branch}" + async with session.get(url, headers=headers) as response: + if response.status == 200: + contents = await response.json() + rules = [] + + for item in contents: + if item["type"] == "file" and item["name"].endswith(".yaml"): + rule = await self._load_rule_from_github(session, item["download_url"], headers) + if rule: + rules.append(rule) + + return rules + + return [] + + async def _load_rule_from_github( + self, + session: aiohttp.ClientSession, + download_url: str, + headers: Dict[str, str] + ) -> Optional[BaseSecurityRule]: + """Load individual rule from GitHub""" + + try: + async with session.get(download_url, headers=headers) as response: + if response.status == 200: + content = await response.text() + rule_data = yaml.safe_load(content) + return self._create_rule_from_yaml(rule_data) + except Exception as e: + # Log error but continue loading other rules + pass + + return None + + def _create_rule_from_yaml( + self, + rule_data: Dict[str, Any] + ) -> Optional[BaseSecurityRule]: + """Create rule instance from YAML data""" + + try: + # Create rule based on rule category + rule_class = RulesRegistry.get(rule_data.get('category')) + rule = rule_class.from_json(rule_data) + + return rule + + except Exception as e: + return None + + + def _create_generic_rule( + self, + metadata: RuleMetadata, + rule_data: Dict[str, Any] + ) -> BaseSecurityRule: + """Create generic rule from YAML""" + + class GenericRule(BaseSecurityRule): + """Generic Rule class""" + + def __init__( + self, metadata: RuleMetadata, patterns: List[Dict[str, Any]]): + super().__init__(metadata) + self.patterns = [re.compile(p["regex"], re.IGNORECASE) for p in patterns] + + def is_applicable(self, file_path: Path) -> bool: + extensions = rule_data.get("file_extensions", []) + if extensions: + return file_path.suffix.lower() in extensions + return True + + async def scan(self, file_path: Path, content: str) -> List[SecurityFinding]: + findings = [] + lines = content.split('\n') + + for line_num, line in enumerate(lines, 1): + for pattern in self.patterns: + matches = pattern.finditer(line) + for match in matches: + finding = SecurityFinding( + id = hashlib.md5(f"{file_path}:{line_num}:{self.metadata.id}".encode()).hexdigest(), + title = self.metadata.name, + description = self.metadata.description, + severity = self.metadata.severity, + category = self.metadata.category, + location = FileLocation( + file_path = file_path, + line_number = line_num, + column_start = match.start(), + column_end = match.end() + ), + rule_id = self.metadata.id, + confidence = 0.7, + metadata = {"line_content": line.strip()} + ) + findings.append(finding) + + return findings + + patterns = rule_data.get("patterns", []) + return GenericRule(metadata, patterns) + + async def get_rule(self, rule_id: str) -> Optional[BaseSecurityRule]: + """Get specific rule by ID""" + + if rule_id not in self.rules_cache: + rules = await self.load_rules() + for rule in rules: + self.rules_cache[rule.metadata.id] = rule + + return self.rules_cache.get(rule_id) + + async def add_rule(self, rule: BaseSecurityRule) -> None: + """Add new rule (would require GitHub API write access)""" + + # Implementation would create PR with new rule + raise NotImplementedError( + "Adding rules requires GitHub write access" + ) + + async def update_rule(self, rule: BaseSecurityRule) -> None: + """Update existing rule (would require GitHub API write access)""" + + # Implementation would create PR with rule update + raise NotImplementedError( + "Updating rules requires GitHub write access" + ) diff --git a/valkyrie/rules/repositories/local_repo.py b/valkyrie/rules/repositories/local_repo.py new file mode 100644 index 0000000..9082d85 --- /dev/null +++ b/valkyrie/rules/repositories/local_repo.py @@ -0,0 +1,73 @@ +import yaml +from pathlib import Path +from typing import Dict, List, Optional + +from valkyrie.core import ( + RuleRepository +) +from valkyrie.rules.base import RulesRegistry, BaseSecurityRule + + +#### +## LOCAL RULE REPOSITORY +##### +class LocalRuleRepository(RuleRepository): + """Rule repository backed by local files""" + + def __init__(self, rules_directory: Path): + self.rules_directory = rules_directory + self.rules_cache: Dict[str, BaseSecurityRule] = {} + + async def load_rules(self) -> List[BaseSecurityRule]: + """Load rules from local directory""" + + rules = [] + + if not self.rules_directory.exists(): + return rules + + for rule_file in self.rules_directory.glob("*.yaml"): + try: + content = rule_file.read_text(encoding='utf-8') + rule_data = yaml.safe_load(content) + + # Create rule based on rule category + rule_class = RulesRegistry.get(rule_data.get('category')) + rule = rule_class.from_json(rule_data) + + rules.append(rule) + self.rules_cache[rule.metadata.id] = rule + + except Exception as e: + # Log error but continue loading other rules + continue + + return rules + + async def get_rule( + self, + rule_id: str + ) -> Optional[BaseSecurityRule]: + """Get specific rule by ID""" + + if not self.rules_cache: + await self.load_rules() + return self.rules_cache.get(rule_id) + + async def add_rule(self, rule: BaseSecurityRule) -> None: + """Add new rule to local repository""" + + rule_file = self.rules_directory / f"{rule.metadata.id}.yaml" + + # Convert rule into JSON format + rule_data = rule.to_json() + + with open(rule_file, 'w', encoding='utf-8') as f: + yaml.dump(rule_data, f, default_flow_style=False) + + self.rules_cache[rule.metadata.id] = rule + + async def update_rule(self, rule: BaseSecurityRule) -> None: + """Update existing rule""" + await self.add_rule(rule) + \ No newline at end of file diff --git a/valkyrie/plugins/secrets/__init__.py b/valkyrie/rules/secrets/__init__.py similarity index 66% rename from valkyrie/plugins/secrets/__init__.py rename to valkyrie/rules/secrets/__init__.py index fb9e45b..2f356ac 100644 --- a/valkyrie/plugins/secrets/__init__.py +++ b/valkyrie/rules/secrets/__init__.py @@ -3,42 +3,74 @@ """ import hashlib from typing import ( - List, Dict, Any + List, Union, Dict, Any ) from pathlib import Path -from valkyrie.plugins import BaseSecurityRule -from valkyrie.core.types import ( - RuleMetadata, FindingCategory, SeverityLevel, - SecurityFinding, ScanRule, ScannerPlugin, +from valkyrie.core import ( + SecurityFinding, FileLocation ) +from valkyrie.utils import load_yaml +from valkyrie.rules.base import ( + BaseSecurityRule, register_rule, +) from .conf import ( - SECRETS_PATTERNS, SecretPattern + SecretPattern, + SecretsRuleConfig ) #### ## SECRET DETECTION RULE ##### +@register_rule() class SecretsDetectionRule(BaseSecurityRule): """Rule for detecting secrets and credentials""" + + description = "Detects API keys, tokens, passwords, and other secrets" + + def __init__( + self, + config: SecretsRuleConfig + ): + # Initialize c + super().__init__(config) + + # Define secret patterns + self.patterns = config.patterns + + @classmethod + def from_config( + cls, + config: SecretsRuleConfig + ) -> 'SecretsDetectionRule': + """Load and return a SecretsDetection Rule instance from config""" + + return cls(config) - def __init__(self): - metadata = RuleMetadata( - id = "secrets-001", - name = "Generic Secrets Detection", - description = "Detects API keys, tokens, passwords, and other secrets", - category = FindingCategory.SECRETS, - severity = SeverityLevel.CRITICAL, - author = "Valkyrie Core Team", - tags = {"secrets", "credentials", "api-keys"} + @classmethod + def from_yaml( + cls, + file: Union[str, Path] + )-> 'SecretsDetectionRule': + """Load Rule fom .Yaml file""" + + return cls.from_config( + SecretsRuleConfig.from_json(load_yaml(file)) ) - super().__init__(metadata) + + @classmethod + def from_json( + cls, + json_content: Dict[str, Any] + )-> 'SecretsDetectionRule': + """Load Rule fom json content (python dict)""" - # Define secret patterns - self.patterns = SECRETS_PATTERNS + return cls.from_config( + SecretsRuleConfig.from_json(json_content) + ) def _calculate_entropy(self, text: str) -> float: """Calculate Shannon entropy of text""" @@ -62,6 +94,7 @@ def _calculate_entropy(self, text: str) -> float: def is_applicable(self, file_path: Path) -> bool: """Check if file should be scanned for secrets""" + # Skip binary files and common non-text files skip_extensions = { '.jpg', '.jpeg', '.png', '.gif', @@ -73,7 +106,11 @@ def is_applicable(self, file_path: Path) -> bool: return True - async def scan(self, file_path: Path, content: str) -> List[SecurityFinding]: + async def scan( + self, + file_path: Path, + content: str + ) -> List[SecurityFinding]: """Scan for secrets in file content""" findings = [] @@ -112,7 +149,7 @@ async def scan(self, file_path: Path, content: str) -> List[SecurityFinding]: description = f"Found pattern matching {pattern.name} in {file_path}", severity = self.metadata.severity, category = self.metadata.category, - location =FileLocation( + location = FileLocation( file_path = file_path, line_number = line_num, column_start = match.start(), @@ -126,8 +163,11 @@ async def scan(self, file_path: Path, content: str) -> List[SecurityFinding]: "line_content": line.strip() }, remediation_advice = ( - f"Remove or secure the {pattern.name}. " - "Consider using environment variables or secure vault services." + self.config.remediation or + ( + f"Remove or secure the {pattern.name}. " + "Consider using environment variables or secure vault services." + ) ) ) findings.append(finding) @@ -158,25 +198,26 @@ def _calculate_confidence(self, line: str, pattern: SecretPattern) -> float: #### ## SECRETS PLUGIN ##### -class SecretsPlugin(ScannerPlugin): - """Plugin for secrets detection""" +# @register_plugin() +# class SecretsPlugin(ScannerPlugin): +# """Plugin for secrets detection""" - @property - def name(self) -> str: - return "secrets-detector" +# @property +# def name(self) -> str: +# return "secrets" - @property - def version(self) -> str: - return "1.0.0" +# @property +# def version(self) -> str: +# return "1.0.0" - async def initialize(self, config: Dict[str, Any]) -> None: - """Initialize secrets plugin""" - pass +# async def initialize(self, config: Dict[str, Any]) -> None: +# """Initialize secrets plugin""" +# pass - async def get_rules(self) -> List[ScanRule]: - """Return secrets detection rules""" - return [SecretsDetectionRule()] +# async def get_rules(self) -> List[ScanRule]: +# """Return secrets detection rules""" +# return [SecretsDetectionRule()] - async def cleanup(self) -> None: - """Cleanup plugin resources""" - pass +# async def cleanup(self) -> None: +# """Cleanup plugin resources""" +# pass diff --git a/valkyrie/plugins/secrets/conf.py b/valkyrie/rules/secrets/conf.py similarity index 75% rename from valkyrie/plugins/secrets/conf.py rename to valkyrie/rules/secrets/conf.py index 2cad3a3..6054bbc 100644 --- a/valkyrie/plugins/secrets/conf.py +++ b/valkyrie/rules/secrets/conf.py @@ -1,22 +1,34 @@ import re -from dataclasses import dataclass, field +from pydantic import Field from typing import ( - Set, Pattern + Set, Pattern, List ) +from valkyrie.core import BaseConfigModel +from valkyrie.rules.base import BaseRuleConfig + + +#### +## SECURITY RULE CONFIG +##### +class SecretsRuleConfig(BaseRuleConfig): + """Secrets Rule Confiiguration model.""" + + patterns: List['SecretPattern'] + #### ## SECRET PATTERN MODEL ##### -@dataclass -class SecretPattern: + +class SecretPattern(BaseConfigModel): """Pattern definition for secret detection""" name: str pattern: Pattern[str] entropy_threshold: float = 0.0 - keywords: Set[str] = field(default_factory=set) - file_extensions: Set[str] = field(default_factory=set) + keywords: Set[str] = Field(default_factory=set) + file_extensions: Set[str] = Field(default_factory=set) #### diff --git a/valkyrie/utils/__init__.py b/valkyrie/utils/__init__.py new file mode 100644 index 0000000..83e59c8 --- /dev/null +++ b/valkyrie/utils/__init__.py @@ -0,0 +1,47 @@ +import logging +from typing import Union, Any +from types import ModuleType +from pathlib import Path +from importlib import import_module + +import yaml + +from .context import ValkyrieContext + +__all__ = [ + ValkyrieContext, + 'import_module_from' +] + + +# VALLKYRIE LOGGER UTILITY +def get_logger(name: str) -> logging.Logger: + """Gets a logger from the global context""" + + base_logger = ValkyrieContext.get_data("logger") + if base_logger is None: + + # Fallback if the context is not initialized + logger = logging.getLogger(name) + logger.addHandler(logging.NullHandler()) + return logger + return base_logger.getChild(name) + +# GET CONTEXT +def get_context() -> ValkyrieContext: + """Return Valkyrie global context""" + + return ValkyrieContext + +# LOAD YAML FILE +def load_yaml(file: Union[str, Path]) -> Any: + """Load ad return a .yaml file content""" + + path = Path(file) + return yaml.safe_load(path.read_text(encoding='utf-8')) + +# IMPORT MODULE +def import_module_from(path: Union[str,Path]) ->'ModuleType': + """Import module using importlib""" + + return import_module(path) \ No newline at end of file diff --git a/valkyrie/utils/context.py b/valkyrie/utils/context.py new file mode 100644 index 0000000..e3c23f2 --- /dev/null +++ b/valkyrie/utils/context.py @@ -0,0 +1,68 @@ +""" +Global context of the application. + +This context holds shared state and configuration accessible throughout +the lifecycle of the operation, enabling consistent data management +and coordination between different components. +""" + +import threading +from typing import Optional, Dict, Any + +#### +## VALKYRIE CONTEXT +##### +class ValkyrieContext: + """ + Global context of the Valkyrie Scanner. + + Provides shared state and configuration accessible across all components + within a Scan Process lifecycle. + """ + + # _config: Optional[] = None + _data: Dict[str, Any] = {} + _debug: bool = False + _is_initialized: bool = False + _lock = threading.Lock() + + @classmethod + def initialize(cls, config, debug: bool = False): + """Initializes the global context""" + + with cls._lock: + cls._config = config + cls._debug = debug + cls._data = {} + cls._is_initialized = True + + @classmethod + def set_data(cls, key: str, value: Any): + """Stores data in the context""" + + with cls._lock: + cls._data[key] = value + + @classmethod + def get_data(cls, key: str, default: Any = None) -> Any: + """Retrieves data from the context""" + return cls._data.get(key, default) + + @classmethod + def remove_data(cls, key: str) -> bool: + """Removes data from the context""" + + if key in cls._data: + del cls._data[key] + return True + return False + + @classmethod + def clear_data(cls): + """Clears all data from the context""" + cls._data.clear() + + @classmethod + def is_debug(cls) -> bool: + """Returns whether debug mode is enabled""" + return cls._debug \ No newline at end of file