From d234d91decf1c45c04a83f77bd7c39528ed9da4d Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Sun, 22 Jun 2025 21:42:00 -0700 Subject: [PATCH 01/19] Generate megalinter workflows for all scan types --- include/singularity/application.hpp | 9 + include/singularity/workflow_generator.hpp | 144 ++++++ security_mappings.json | 25 +- src/CMakeLists.txt | 1 + src/application.cpp | 54 +- src/workflow_generator.cpp | 558 +++++++++++++++++++++ 6 files changed, 779 insertions(+), 12 deletions(-) create mode 100644 include/singularity/workflow_generator.hpp create mode 100644 src/workflow_generator.cpp diff --git a/include/singularity/application.hpp b/include/singularity/application.hpp index e5c7dd0..b3fe3bf 100644 --- a/include/singularity/application.hpp +++ b/include/singularity/application.hpp @@ -75,6 +75,13 @@ class Application { */ bool generate_security_report(const LanguageStats& stats); + /** + * @brief Generate workflow templates for the repository + * @param stats Language statistics from analysis + * @return True if workflow generation successful + */ + bool generate_workflow_templates(const LanguageStats& stats); + /** * @brief Progress callback function * @param percentage Progress percentage (0-100) @@ -85,7 +92,9 @@ class Application { std::string repo_url_; bool verbose_{false}; bool generate_report_{false}; + bool generate_workflows_{false}; std::string output_file_; + std::string output_dir_; }; } // namespace singularity diff --git a/include/singularity/workflow_generator.hpp b/include/singularity/workflow_generator.hpp new file mode 100644 index 0000000..ea560cc --- /dev/null +++ b/include/singularity/workflow_generator.hpp @@ -0,0 +1,144 @@ +#pragma once + +#include "singularity/language_stats.hpp" +#include +#include +#include +#include +#include +#include + +namespace singularity { + +/** + * @brief Class representing a MegaLinter workflow generator + */ +class WorkflowGenerator { +public: + /** + * @brief Get singleton instance + * @return Singleton instance + */ + static WorkflowGenerator& instance(); + + /** + * @brief Initialize the workflow generator + * @param security_mappings Path to security mappings JSON file + * @return True if initialization successful + */ + bool initialize(const std::string& security_mappings = "security_mappings.json"); + + /** + * @brief Generate MegaLinter workflow templates based on detected languages + * @param stats Language statistics from repository analysis + * @param output_dir Directory where workflow files should be written + * @return True if generation successful + */ + bool generate_workflows(const LanguageStats& stats, const std::string& output_dir = "output"); + + /** + * @brief Check if the workflow generator is initialized + * @return True if initialized + */ + bool is_initialized() const { return initialized_; } + +private: + WorkflowGenerator(); + WorkflowGenerator(const WorkflowGenerator&) = delete; + WorkflowGenerator& operator=(const WorkflowGenerator&) = delete; + + /** + * @brief Generate a GitHub Actions workflow file for MegaLinter + * @param scan_type The type of scan (e.g., "static_analysis") + * @param languages Vector of languages to include in the workflow + * @param output_dir Directory where workflow file should be written + * @return True if generation successful + */ + bool generate_megalinter_workflow( + const std::string& scan_type, + const std::vector& languages, + const std::string& output_dir); + + /** + * @brief Generate MegaLinter configuration file + * @param languages Vector of languages to include in configuration + * @param output_dir Directory where configuration file should be written + * @return True if generation successful + */ + bool generate_megalinter_config( + const std::vector& languages, + const std::string& output_dir); + + /** + * @brief Map Singularity scan types to MegaLinter linter groups + * @param scan_type The scan type from security_mappings.json + * @return Corresponding MegaLinter linter group + */ + std::string map_to_megalinter_group(const std::string& scan_type); + + /** + * @brief Map language name to MegaLinter language format + * @param language The language name (e.g., "C++") + * @return Language name in MegaLinter format (e.g., "cpp") + */ + std::string map_language_to_megalinter(const std::string& language); + + /** + * @brief Get appropriate MegaLinter linters for a language and scan type + * @param language The language name + * @param scan_type The scan type + * @return Vector of linter names + */ + std::vector get_linters_for_language( + const std::string& language, + const std::string& scan_type); + + /** + * @brief Get generic MegaLinter linters for a scan type (regardless of language) + * @param scan_type The scan type + * @return Vector of linter names + */ + std::vector get_generic_linters_for_scan_type( + const std::string& scan_type); + + /** + * @brief Add tool-specific configurations to the workflow file + * @param workflow_file Output stream for the workflow file + * @param language The language name + * @param scan_type The scan type + */ + void add_tool_configs( + std::ofstream& workflow_file, + const std::string& language, + const std::string& scan_type); + + nlohmann::json security_mappings_; + bool initialized_ = false; + + // Map tool names to MegaLinter linters + const std::unordered_map tool_to_linter = { + {"Cppcheck", "CPP_CPPCHECK"}, + {"CppLint", "CPP_CPPLINT"}, + {"clang-tidy", "CPP_CLANG_TIDY"}, + {"Clang Static Analyzer", "CPP_CLANG_TIDY"}, + {"FlawFinder", "CPP_FLAWFINDER"}, + {"Bandit", "PYTHON_BANDIT"}, + {"Pylint", "PYTHON_PYLINT"}, + {"Black", "PYTHON_BLACK"}, + {"Safety", "PYTHON_SAFETY"}, + {"Semgrep", "REPOSITORY_SEMGREP"}, + {"GitLeaks", "REPOSITORY_GITLEAKS"}, + {"Secretlint", "REPOSITORY_SECRETLINT"}, + {"OWASP Dependency-Check", "REPOSITORY_DEPENDENCY_CHECK"}, + {"Dependabot", "REPOSITORY_DEPENDABOT"}, + {"Fortify", "REPOSITORY_FORTIFY"}, + {"Detect-secrets", "REPOSITORY_SECRETLINT"}, + {"OWASP ZAP", "REPOSITORY_ZAP"}, + {"licensechecker", "REPOSITORY_LICENSECHECKER"}, + {"scancode-toolkit", "REPOSITORY_SCANCODE"}, + {"REUSE Tool", "REPOSITORY_REUSE"}, + {"Trivy", "REPOSITORY_TRIVY"} + }; +}; + +} // namespace singularity diff --git a/security_mappings.json b/security_mappings.json index 91c43a3..d480dcf 100644 --- a/security_mappings.json +++ b/security_mappings.json @@ -8,11 +8,13 @@ }, "Cppcheck": { "supported_languages": ["C", "C++"], - "configuration": "Enable all checks with focus on security-related issues." + "configuration": "Enable all checks with focus on security-related issues.", + "default_for_language": true }, "Bandit": { "supported_languages": ["Python"], - "configuration": "Focus on input validation, SQL injection, and command injection vulnerabilities." + "configuration": "Focus on input validation, SQL injection, and command injection vulnerabilities.", + "default_for_language": true }, "Pylint": { "supported_languages": ["Python"], @@ -36,7 +38,8 @@ }, "Dependabot": { "supported_languages": ["C", "C++", "Python"], - "configuration": "Configure with security advisories and version updates for all supported package ecosystems." + "configuration": "Configure with security advisories and version updates for all supported package ecosystems.", + "default_for_language": true } } }, @@ -48,7 +51,8 @@ }, "Semgrep": { "supported_languages": ["C", "C++", "Python"], - "configuration": "Use language-specific rule sets to detect security issues." + "configuration": "Use language-specific rule sets to detect security issues.", + "default_for_language": true }, "Fortify": { "supported_languages": ["C", "C++", "Python"], @@ -64,7 +68,8 @@ }, "clang-tidy": { "supported_languages": ["C", "C++"], - "configuration": "Configure with -checks=bugprone-*,cert-*,clang-analyzer-security.*,performance-*" + "configuration": "Configure with -checks=bugprone-*,cert-*,clang-analyzer-security.*,performance-*", + "default_for_language": true }, "CppLint": { "supported_languages": ["C++"], @@ -76,16 +81,13 @@ }, "Pylint security plugins": { "supported_languages": ["Python"], - "configuration": "Enable all security plugins in Pylint." + "configuration": "Enable all security plugins in Pylint.", + "default_for_language": true } } }, "code_quality": { "tools": { - "Cppcheck": { - "supported_languages": ["C", "C++"], - "configuration": "Enable all checks with focus on code quality issues." - }, "Radon": { "supported_languages": ["Python"], "configuration": "Compute code metrics and complexity." @@ -96,7 +98,8 @@ }, "Lizard": { "supported_languages": ["C", "C++", "Python"], - "configuration": "Analyze cyclomatic complexity in code base. Configure to fail builds with high complexity scores." + "configuration": "Analyze cyclomatic complexity in code base. Configure to fail builds with high complexity scores.", + "default_for_language": true }, "Complexity Checker": { "supported_languages": ["C", "C++"], diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4ef419a..712501b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,6 +4,7 @@ add_library(singularity_lib analyzers.cpp application.cpp security_recommender.cpp + workflow_generator.cpp ) target_include_directories(singularity_lib diff --git a/src/application.cpp b/src/application.cpp index b8fed36..b8fd2a0 100644 --- a/src/application.cpp +++ b/src/application.cpp @@ -1,6 +1,7 @@ #include "singularity/application.hpp" #include "singularity/repo_analyzer.hpp" #include "singularity/security_recommender.hpp" +#include "singularity/workflow_generator.hpp" #include // For std::exit #include #include @@ -78,6 +79,16 @@ bool Application::parse_args(int argc, char** argv) { else if (arg == "--verbose") { verbose_ = true; } + else if (arg == "--workflows") { + generate_workflows_ = true; + + // Check if the next argument is an output directory path + if (i + 1 < argc && argv[i+1][0] != '-') { + output_dir_ = argv[++i]; + } else { + output_dir_ = "output"; // Default output directory + } + } else { std::cerr << "Error: Unknown argument: " << arg << std::endl; print_usage(); @@ -108,7 +119,8 @@ void Application::print_usage() { << " -v, --version Show version and exit\n" << " -r, --repo Repository URL to analyze (currently only GitHub URLs are supported)\n" << " --verbose Show verbose output\n" - << " --security-report [file] Generate a security report, optionally write to file\n"; + << " --security-report [file] Generate a security report, optionally write to file\n" + << " --workflows [dir] Generate MegaLinter workflow templates in directory (default: output)\n"; } void Application::print_version() { @@ -153,6 +165,17 @@ bool Application::analyze_repo() { } } + // Generate workflow templates if requested + if (generate_workflows_) { + if (verbose_) { + std::cout << "Generating MegaLinter workflow templates...\n"; + } + + if (!generate_workflow_templates(stats)) { + return false; + } + } + return true; } @@ -201,4 +224,33 @@ bool Application::generate_security_report(const LanguageStats& stats) { } } +bool Application::generate_workflow_templates(const LanguageStats& stats) { + try { + // Get singleton instance and initialize it + WorkflowGenerator& generator = WorkflowGenerator::instance(); + + if (!generator.is_initialized()) { + if (!generator.initialize()) { + std::cerr << "Error: Could not initialize workflow generator" << std::endl; + return false; + } + } + + // Generate workflow files in the output directory + if (!generator.generate_workflows(stats, output_dir_)) { + std::cerr << "Error: Could not generate workflow templates" << std::endl; + return false; + } + + if (verbose_) { + std::cout << "Workflow templates generated in directory: " << output_dir_ << std::endl; + } + + return true; + } catch (const std::exception& e) { + std::cerr << "Error generating workflow templates: " << e.what() << std::endl; + return false; + } +} + } // namespace singularity diff --git a/src/workflow_generator.cpp b/src/workflow_generator.cpp new file mode 100644 index 0000000..70d3fef --- /dev/null +++ b/src/workflow_generator.cpp @@ -0,0 +1,558 @@ +/* + * Copyright 2025 Singularity Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "singularity/workflow_generator.hpp" + +using json = nlohmann::json; +namespace fs = std::filesystem; + +namespace singularity { + +// Singleton implementation +WorkflowGenerator& WorkflowGenerator::instance() { + static WorkflowGenerator instance; + return instance; +} + +WorkflowGenerator::WorkflowGenerator() { + // Private constructor for singleton +} + +bool WorkflowGenerator::initialize(const std::string& security_mappings) { + if (initialized_) { + return true; + } + + try { + // Try to open security mappings file + std::ifstream mappings_file; + std::vector possible_paths = { + security_mappings, + "../" + security_mappings, + "../../" + security_mappings, + "../../../" + security_mappings + }; + + for (const auto& path : possible_paths) { + mappings_file.open(path); + if (mappings_file.is_open()) { + break; + } + } + + if (!mappings_file.is_open()) { + throw std::runtime_error("Could not open " + security_mappings + " from any expected location"); + } + + // Parse JSON mappings + security_mappings_ = json::parse(mappings_file); + + initialized_ = true; + return true; + } + catch (const std::exception& e) { + std::cerr << "Error initializing workflow generator: " << e.what() << std::endl; + return false; + } +} + +bool WorkflowGenerator::generate_workflows(const LanguageStats& stats, const std::string& output_dir) { + if (!initialized_) { + std::cerr << "WorkflowGenerator not initialized! Call initialize() first." << std::endl; + return false; + } + + try { + // Create output directory if it doesn't exist + fs::create_directories(output_dir); + fs::create_directories(output_dir + "/workflows"); + fs::create_directories(output_dir + "/config"); + + // Get all detected languages + std::vector languages = stats.get_languages(); + + // Process each scan type from security_mappings.json + const json& scan_types = security_mappings_["scan_types"]; + + for (auto it = scan_types.begin(); it != scan_types.end(); ++it) { + const std::string& scan_type = it.key(); + + // Generate workflow file for this scan type + if (!generate_megalinter_workflow(scan_type, languages, output_dir)) { + std::cerr << "Failed to generate workflow for scan type: " << scan_type << std::endl; + continue; + } + } + + // Generate MegaLinter config file + if (!generate_megalinter_config(languages, output_dir)) { + std::cerr << "Failed to generate MegaLinter configuration" << std::endl; + return false; + } + + std::cout << "Successfully generated workflow templates in " << output_dir << std::endl; + return true; + } + catch (const std::exception& e) { + std::cerr << "Error generating workflows: " << e.what() << std::endl; + return false; + } +} + +bool WorkflowGenerator::generate_megalinter_workflow( + const std::string& scan_type, + const std::vector& languages, + const std::string& output_dir) { + + // Create the GitHub Actions workflow file + std::string workflow_filename = output_dir + "/workflows/megalinter-" + scan_type + ".yml"; + std::ofstream workflow_file(workflow_filename); + + if (!workflow_file.is_open()) { + std::cerr << "Could not open file for writing: " << workflow_filename << std::endl; + return false; + } + + // Map scan type to MegaLinter group + std::string megalinter_group = map_to_megalinter_group(scan_type); + + // Create workflow content + workflow_file << "---\n"; + workflow_file << "# MegaLinter GitHub Actions workflow for " << scan_type << " scans\n"; + workflow_file << "name: MegaLinter " << scan_type << "\n\n"; + workflow_file << "on:\n"; + workflow_file << " push:\n"; + workflow_file << " branches: [main, master]\n"; + workflow_file << " pull_request:\n"; + workflow_file << " branches: [main, master]\n\n"; + workflow_file << "jobs:\n"; + workflow_file << " megalinter-" << scan_type << ":\n"; + workflow_file << " name: MegaLinter " << scan_type << " Scan\n"; + workflow_file << " runs-on: ubuntu-latest\n"; + workflow_file << " steps:\n"; + workflow_file << " # Checkout code\n"; + workflow_file << " - name: Checkout Code\n"; + workflow_file << " uses: actions/checkout@v3\n"; + workflow_file << " with:\n"; + workflow_file << " fetch-depth: 0\n\n"; + workflow_file << " # MegaLinter\n"; + workflow_file << " - name: MegaLinter " << scan_type << " Scan\n"; + workflow_file << " id: ml\n"; + workflow_file << " uses: oxsecurity/megalinter@v8\n"; + workflow_file << " env:\n"; + workflow_file << " VALIDATE_ALL_CODEBASE: true\n"; + workflow_file << " GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n"; + workflow_file << " MEGALINTER_CONFIG: .github/config/.mega-linter.yml\n"; + + // Add comment about dynamic language detection + workflow_file << " # Dynamically enable linters based on detected languages: "; + for (const auto& language : languages) { + workflow_file << language << ", "; + } + workflow_file << "\n"; + + // Start building the ENABLE_LINTERS list + workflow_file << " ENABLE_LINTERS: >-\n"; + + // Get specific linters for each detected language and this scan type + std::set enabled_linters; // Use set to avoid duplicates + + for (const auto& language : languages) { + auto language_linters = get_linters_for_language(language, scan_type); + for (const auto& linter : language_linters) { + enabled_linters.insert(linter); + } + } + + // Write each linter on a separate line + for (const auto& linter : enabled_linters) { + workflow_file << " " << linter << ",\n"; + } + + // Add generic linters for this scan type regardless of language + auto generic_linters = get_generic_linters_for_scan_type(scan_type); + for (const auto& linter : generic_linters) { + workflow_file << " " << linter << ",\n"; + } + + // Add tool-specific configurations + workflow_file << "\n # Tool-specific configurations\n"; + for (const auto& language : languages) { + add_tool_configs(workflow_file, language, scan_type); + } + + // Continue with common configuration + workflow_file << "\n"; + workflow_file << " # Upload MegaLinter artifacts\n"; + workflow_file << " - name: Archive production artifacts\n"; + workflow_file << " uses: actions/upload-artifact@v3\n"; + workflow_file << " if: success() || failure()\n"; + workflow_file << " with:\n"; + workflow_file << " name: MegaLinter reports\n"; + workflow_file << " path: |\n"; + workflow_file << " megalinter-reports\n"; + workflow_file << " mega-linter.log\n\n"; + + workflow_file << " # Create pull request with fixes if applicable\n"; + workflow_file << " - name: Create Pull Request with applied fixes\n"; + workflow_file << " uses: peter-evans/create-pull-request@v4\n"; + workflow_file << " if: steps.ml.outputs.has_updated_sources == 1 && (env.APPLY_FIXES == 'all' || env.APPLY_FIXES == 'true')\n"; + workflow_file << " with:\n"; + workflow_file << " token: ${{ secrets.GITHUB_TOKEN }}\n"; + workflow_file << " commit-message: \"[MegaLinter] Apply " << scan_type << " fixes\"\n"; + workflow_file << " title: \"[MegaLinter] Apply " << scan_type << " fixes\"\n"; + workflow_file << " labels: bot\n"; + + workflow_file.close(); + std::cout << "Generated workflow file: " << workflow_filename << std::endl; + return true; +} + +bool WorkflowGenerator::generate_megalinter_config( + const std::vector& languages, + const std::string& output_dir) { + + // Create directory for config if it doesn't exist + fs::create_directories(output_dir + "/config"); + + // Create the MegaLinter config file + std::string config_filename = output_dir + "/config/.mega-linter.yml"; + std::ofstream config_file(config_filename); + + if (!config_file.is_open()) { + std::cerr << "Could not open file for writing: " << config_filename << std::endl; + return false; + } + + // Create config content + config_file << "---\n"; + config_file << "# MegaLinter Configuration file\n"; + config_file << "# Generated by Singularity\n\n"; + + config_file << "# Define global configuration\n"; + config_file << "APPLY_FIXES: all\n"; + config_file << "SHOW_ELAPSED_TIME: true\n"; + config_file << "FILEIO_REPORTER: false\n\n"; + + // Configure language-specific settings based on detected languages + config_file << "# Language specific configurations\n"; + + for (const auto& language : languages) { + std::string ml_language = map_language_to_megalinter(language); + if (!ml_language.empty()) { + config_file << "# " << language << " configuration\n"; + + // Add language-specific linters and their configurations + for (const auto& scan_type_entry : security_mappings_["scan_types"].items()) { + const std::string& scan_type = scan_type_entry.key(); + auto linters = get_linters_for_language(language, scan_type); + + // for (const auto& linter : linters) { + // config_file << linter << "_ARGUMENTS: \"\"\n"; + // config_file << linter << "_FILTER_REGEX_INCLUDE: \"\"\n"; + // config_file << linter << "_FILE_EXTENSIONS: \"\"\n\n"; + // } + } + } + } + + // Add configuration for general tools + config_file << "# General security scanning configuration\n"; + config_file << "REPOSITORY_GITLEAKS_CONFIG_FILE: .github/config/gitleaks.toml\n"; + config_file << "REPOSITORY_TRIVY_ARGUMENTS: --severity HIGH,CRITICAL\n\n"; + + config_file << "# Disable specific linters\n"; + config_file << "DISABLE:\n"; + config_file << " - COPYPASTE # Optional: disable copy-paste detection\n"; + config_file << " - SPELL # Optional: disable spell checking\n\n"; + + config_file << "# Example custom configuration for linters\n"; + config_file << "PYTHON_BANDIT_ARGUMENTS: -ll -ii\n"; + + config_file.close(); + std::cout << "Generated MegaLinter config file: " << config_filename << std::endl; + + return true; +} + +std::string WorkflowGenerator::map_to_megalinter_group(const std::string& scan_type) { + // Map scan types to MegaLinter linter groups + static const std::unordered_map scan_to_group = { + {"static_analysis", "SECURITY"}, + {"security_linting", "SECURITY"}, + {"sast", "SECURITY"}, + {"code_quality", "CODE_QUALITY"}, + {"cve_scanning", "REPOSITORY"}, + {"license_compliance", "LICENSE"} + }; + + auto it = scan_to_group.find(scan_type); + if (it != scan_to_group.end()) { + return it->second; + } + + // Default to empty if no mapping found + return ""; +} + +std::string WorkflowGenerator::map_language_to_megalinter(const std::string& language) { + // Map language names to MegaLinter language identifiers + static const std::unordered_map lang_map = { + {"C", "C"}, + {"C++", "CPP"}, + {"Python", "PYTHON"}, + {"JavaScript", "JAVASCRIPT"}, + {"TypeScript", "TYPESCRIPT"}, + {"Java", "JAVA"}, + {"Go", "GO"}, + {"Ruby", "RUBY"}, + {"PHP", "PHP"}, + {"C#", "CSHARP"}, + {"Shell", "BASH"}, + {"PowerShell", "POWERSHELL"}, + {"R", "R"}, + {"Kotlin", "KOTLIN"}, + {"Swift", "SWIFT"} + }; + + auto it = lang_map.find(language); + if (it != lang_map.end()) { + return it->second; + } + + // Default to empty if no mapping found + return ""; +} + +std::vector WorkflowGenerator::get_linters_for_language( + const std::string& language, + const std::string& scan_type) { + + std::vector linters; + + // Check if the scan_type exists in the security mappings + if (!security_mappings_["scan_types"].contains(scan_type)) { + return linters; + } + + // Get tools for this scan type from the security mappings + const auto& scan_type_tools = security_mappings_["scan_types"][scan_type]["tools"]; + + // Iterate through all tools for this scan type + for (auto& [tool_name, tool_info] : scan_type_tools.items()) { + // Check if this tool supports the requested language + if (tool_info.contains("supported_languages")) { + const auto& supported_langs = tool_info["supported_languages"]; + + // Check if language is supported by this tool + bool language_supported = false; + for (const auto& supported_lang : supported_langs) { + if (supported_lang.get() == language) { + language_supported = true; + break; + } + } + + // Only add the linter if: + // 1. Language is supported AND + // 2. "default_for_language" field exists and is set to true + if (language_supported) { + // Only add if the field exists and is true + if (tool_info.contains("default_for_language")) { + if (tool_info["default_for_language"].get()) { + auto it = tool_to_linter.find(tool_name); + if (it != tool_to_linter.end()) { + linters.push_back(it->second); + } + } + } + } + } + } + + return linters; +} + +std::vector WorkflowGenerator::get_generic_linters_for_scan_type(const std::string& scan_type) { + // Return generic linters that apply to any language for a given scan type + std::vector linters; + + // Map category names from security_mappings.json to scan_types + static const std::unordered_map category_to_scan_type = { + {"secret_detection", "security_linting"}, + {"dast", "security_linting"}, + {"license_compliance", "license_compliance"}, + {"runtime_protection", "static_analysis"}, + {"log_analysis", "code_quality"} + }; + + // Look for general recommendations that match this scan type + for (auto& [category, category_info] : security_mappings_["general_recommendations"].items()) { + // Skip if this category doesn't map to our scan type + auto it = category_to_scan_type.find(category); + if (it == category_to_scan_type.end() || it->second != scan_type) { + continue; + } + + // Check if this category has tools + if (category_info.contains("tools")) { + // Add tools for this category, but only if they're explicitly marked as default + for (auto& [tool_name, tool_info] : category_info["tools"].items()) { + // Check if this tool is marked as default + bool is_default = false; // Default to false if field doesn't exist + + // Only add if the field exists and is true + if (tool_info.contains("default_for_language")) { + is_default = tool_info["default_for_language"].get(); + } + + // Only add if it's explicitly marked as a default tool + if (is_default) { + auto tool_it = tool_to_linter.find(tool_name); + if (tool_it != tool_to_linter.end()) { + linters.push_back(tool_it->second); + } + } + } + } + } + + return linters; +} + +void WorkflowGenerator::add_tool_configs( + std::ofstream& workflow_file, + const std::string& language, + const std::string& scan_type) { + + // Map tool names to MegaLinter linter config argument names + static const std::unordered_map tool_to_config = { + {"Cppcheck", "CPP_CPPCHECK_ARGUMENTS"}, + {"CppLint", "CPP_CPPLINT_ARGUMENTS"}, + {"clang-tidy", "CPP_CLANG_TIDY_ARGUMENTS"}, + {"Clang Static Analyzer", "CPP_CLANG_TIDY_ARGUMENTS"}, + {"FlawFinder", "CPP_FLAWFINDER_ARGUMENTS"}, + {"Bandit", "PYTHON_BANDIT_ARGUMENTS"}, + {"Pylint", "PYTHON_PYLINT_ARGUMENTS"}, + {"Black", "PYTHON_BLACK_ARGUMENTS"}, + {"Safety", "PYTHON_SAFETY_ARGUMENTS"}, + {"GitLeaks", "REPOSITORY_GITLEAKS_ARGUMENTS"}, + {"Detect-secrets", "REPOSITORY_SECRETLINT_ARGUMENTS"}, + {"Semgrep", "REPOSITORY_SEMGREP_ARGUMENTS"}, + {"CodeQL", "REPOSITORY_CODEQL_ARGUMENTS"}, + {"OWASP Dependency-Check", "REPOSITORY_DEPENDENCY_CHECK_ARGUMENTS"}, + {"Trivy", "REPOSITORY_TRIVY_ARGUMENTS"} + }; + + // Check if the scan_type exists in the security mappings + if (!security_mappings_["scan_types"].contains(scan_type)) { + return; // No configuration available + } + + // Get tools for this scan type from the security mappings + const auto& scan_type_tools = security_mappings_["scan_types"][scan_type]["tools"]; + + // Iterate through all tools for this scan type + for (auto& [tool_name, tool_info] : scan_type_tools.items()) { + // Check if this tool supports the requested language + if (tool_info.contains("supported_languages") && tool_info.contains("configuration")) { + const auto& supported_langs = tool_info["supported_languages"]; + + // Check if language is supported by this tool + bool language_supported = false; + for (const auto& supported_lang : supported_langs) { + if (supported_lang.get() == language) { + language_supported = true; + break; + } + } + + // Only configure the tool if: + // 1. Language is supported AND + // 2. "default_for_language" field exists and is set to true + if (language_supported) { + // Check if this tool is explicitly marked as default for the language + bool is_default = false; // Default to false if field doesn't exist + + // Only add if the field exists and is true + if (tool_info.contains("default_for_language")) { + is_default = tool_info["default_for_language"].get(); + } + + // Only add config if it's explicitly marked as a default tool for this language + if (is_default) { + auto it = tool_to_config.find(tool_name); + if (it != tool_to_config.end()) { + // Extract the configuration string + const std::string& config_str = tool_info["configuration"].get(); + + // Generate a sensible argument string based on the configuration text + std::string args; + if (tool_name == "Bandit") { + args = "\"-ll -ii\""; + } else if (tool_name == "Pylint") { + args = "\"--max-line-length=100\""; + } else if (tool_name == "Black") { + args = "\"--line-length=100\""; + } else if (tool_name == "clang-tidy") { + args = "\"-checks=clang-analyzer-security.*,cert-*\""; + } else if (tool_name == "Cppcheck") { + args = "\"--enable=all --inconclusive\""; + } else if (tool_name == "Semgrep") { + args = "\"--config=p/security-audit\""; + } else if (tool_name == "GitLeaks") { + args = "\"--config-path=.github/config/gitleaks.toml\""; + } else if (tool_name == "Trivy") { + args = "\"--severity HIGH,CRITICAL\""; + } else { + args = "\"\""; // Default to empty arguments + } + + workflow_file << " " << it->second << ": " << args << "\n"; + } + } + } + } + + // Add generic tool configs from general_recommendations + if (security_mappings_.contains("general_recommendations")) { + if (scan_type == "security_linting" && + security_mappings_["general_recommendations"].contains("secret_detection")) { + // Add secret detection tools + workflow_file << " REPOSITORY_GITLEAKS_ARGUMENTS: \"--config-path=.github/config/gitleaks.toml\"\n"; + } else if (scan_type == "sast" && + security_mappings_["general_recommendations"].contains("secret_detection")) { + // Add SAST tools + workflow_file << " REPOSITORY_SEMGREP_ARGUMENTS: \"--config=p/security-audit\"\n"; + } else if (scan_type == "cve_scanning" && + (security_mappings_["general_recommendations"].contains("priority_order"))) { + // Add CVE scanning tools + workflow_file << " REPOSITORY_TRIVY_ARGUMENTS: \"--severity HIGH,CRITICAL\"\n"; + } + } +} + +} +} \ No newline at end of file From 0f357a7fce53dd9869e68b276b94640375323064 Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Mon, 23 Jun 2025 20:39:30 -0700 Subject: [PATCH 02/19] Enable lizard for cyclomatic complexity --- include/singularity/workflow_generator.hpp | 18 +- security_mappings.json | 2 +- src/workflow_generator.cpp | 305 +++++++++++++-------- 3 files changed, 210 insertions(+), 115 deletions(-) diff --git a/include/singularity/workflow_generator.hpp b/include/singularity/workflow_generator.hpp index ea560cc..cb4fef7 100644 --- a/include/singularity/workflow_generator.hpp +++ b/include/singularity/workflow_generator.hpp @@ -55,7 +55,7 @@ class WorkflowGenerator { * @return True if generation successful */ bool generate_megalinter_workflow( - const std::string& scan_type, + const std::string& scan_type, const std::vector& languages, const std::string& output_dir); @@ -69,6 +69,16 @@ class WorkflowGenerator { const std::vector& languages, const std::string& output_dir); + /** + * @brief Generate a GitHub Actions workflow file for Lizard complexity analysis + * @param languages Vector of languages to analyze + * @param output_dir Directory where workflow file should be written + * @return True if generation successful + */ + bool generate_lizard_workflow( + const std::vector& languages, + const std::string& output_dir); + /** * @brief Map Singularity scan types to MegaLinter linter groups * @param scan_type The scan type from security_mappings.json @@ -90,9 +100,9 @@ class WorkflowGenerator { * @return Vector of linter names */ std::vector get_linters_for_language( - const std::string& language, + const std::string& language, const std::string& scan_type); - + /** * @brief Get generic MegaLinter linters for a scan type (regardless of language) * @param scan_type The scan type @@ -100,7 +110,7 @@ class WorkflowGenerator { */ std::vector get_generic_linters_for_scan_type( const std::string& scan_type); - + /** * @brief Add tool-specific configurations to the workflow file * @param workflow_file Output stream for the workflow file diff --git a/security_mappings.json b/security_mappings.json index d480dcf..e376ad0 100644 --- a/security_mappings.json +++ b/security_mappings.json @@ -79,7 +79,7 @@ "supported_languages": ["Python"], "configuration": "Scan for common security issues in Python code." }, - "Pylint security plugins": { + "Pylint": { "supported_languages": ["Python"], "configuration": "Enable all security plugins in Pylint.", "default_for_language": true diff --git a/src/workflow_generator.cpp b/src/workflow_generator.cpp index 70d3fef..91932d9 100644 --- a/src/workflow_generator.cpp +++ b/src/workflow_generator.cpp @@ -20,7 +20,9 @@ #include #include #include +#include #include +#include #include #include "singularity/workflow_generator.hpp" @@ -99,6 +101,11 @@ bool WorkflowGenerator::generate_workflows(const LanguageStats& stats, const std for (auto it = scan_types.begin(); it != scan_types.end(); ++it) { const std::string& scan_type = it.key(); + if (scan_type == "code_quality") { + generate_lizard_workflow(languages, output_dir); + continue; + } + // Generate workflow file for this scan type if (!generate_megalinter_workflow(scan_type, languages, output_dir)) { std::cerr << "Failed to generate workflow for scan type: " << scan_type << std::endl; @@ -106,12 +113,6 @@ bool WorkflowGenerator::generate_workflows(const LanguageStats& stats, const std } } - // Generate MegaLinter config file - if (!generate_megalinter_config(languages, output_dir)) { - std::cerr << "Failed to generate MegaLinter configuration" << std::endl; - return false; - } - std::cout << "Successfully generated workflow templates in " << output_dir << std::endl; return true; } @@ -127,7 +128,7 @@ bool WorkflowGenerator::generate_megalinter_workflow( const std::string& output_dir) { // Create the GitHub Actions workflow file - std::string workflow_filename = output_dir + "/workflows/megalinter-" + scan_type + ".yml"; + std::string workflow_filename = output_dir + "/workflows/" + scan_type + ".yml"; std::ofstream workflow_file(workflow_filename); if (!workflow_file.is_open()) { @@ -230,73 +231,6 @@ bool WorkflowGenerator::generate_megalinter_workflow( return true; } -bool WorkflowGenerator::generate_megalinter_config( - const std::vector& languages, - const std::string& output_dir) { - - // Create directory for config if it doesn't exist - fs::create_directories(output_dir + "/config"); - - // Create the MegaLinter config file - std::string config_filename = output_dir + "/config/.mega-linter.yml"; - std::ofstream config_file(config_filename); - - if (!config_file.is_open()) { - std::cerr << "Could not open file for writing: " << config_filename << std::endl; - return false; - } - - // Create config content - config_file << "---\n"; - config_file << "# MegaLinter Configuration file\n"; - config_file << "# Generated by Singularity\n\n"; - - config_file << "# Define global configuration\n"; - config_file << "APPLY_FIXES: all\n"; - config_file << "SHOW_ELAPSED_TIME: true\n"; - config_file << "FILEIO_REPORTER: false\n\n"; - - // Configure language-specific settings based on detected languages - config_file << "# Language specific configurations\n"; - - for (const auto& language : languages) { - std::string ml_language = map_language_to_megalinter(language); - if (!ml_language.empty()) { - config_file << "# " << language << " configuration\n"; - - // Add language-specific linters and their configurations - for (const auto& scan_type_entry : security_mappings_["scan_types"].items()) { - const std::string& scan_type = scan_type_entry.key(); - auto linters = get_linters_for_language(language, scan_type); - - // for (const auto& linter : linters) { - // config_file << linter << "_ARGUMENTS: \"\"\n"; - // config_file << linter << "_FILTER_REGEX_INCLUDE: \"\"\n"; - // config_file << linter << "_FILE_EXTENSIONS: \"\"\n\n"; - // } - } - } - } - - // Add configuration for general tools - config_file << "# General security scanning configuration\n"; - config_file << "REPOSITORY_GITLEAKS_CONFIG_FILE: .github/config/gitleaks.toml\n"; - config_file << "REPOSITORY_TRIVY_ARGUMENTS: --severity HIGH,CRITICAL\n\n"; - - config_file << "# Disable specific linters\n"; - config_file << "DISABLE:\n"; - config_file << " - COPYPASTE # Optional: disable copy-paste detection\n"; - config_file << " - SPELL # Optional: disable spell checking\n\n"; - - config_file << "# Example custom configuration for linters\n"; - config_file << "PYTHON_BANDIT_ARGUMENTS: -ll -ii\n"; - - config_file.close(); - std::cout << "Generated MegaLinter config file: " << config_filename << std::endl; - - return true; -} - std::string WorkflowGenerator::map_to_megalinter_group(const std::string& scan_type) { // Map scan types to MegaLinter linter groups static const std::unordered_map scan_to_group = { @@ -508,51 +442,202 @@ void WorkflowGenerator::add_tool_configs( // Extract the configuration string const std::string& config_str = tool_info["configuration"].get(); - // Generate a sensible argument string based on the configuration text - std::string args; - if (tool_name == "Bandit") { - args = "\"-ll -ii\""; - } else if (tool_name == "Pylint") { - args = "\"--max-line-length=100\""; - } else if (tool_name == "Black") { - args = "\"--line-length=100\""; - } else if (tool_name == "clang-tidy") { - args = "\"-checks=clang-analyzer-security.*,cert-*\""; - } else if (tool_name == "Cppcheck") { - args = "\"--enable=all --inconclusive\""; - } else if (tool_name == "Semgrep") { - args = "\"--config=p/security-audit\""; - } else if (tool_name == "GitLeaks") { - args = "\"--config-path=.github/config/gitleaks.toml\""; - } else if (tool_name == "Trivy") { - args = "\"--severity HIGH,CRITICAL\""; - } else { - args = "\"\""; // Default to empty arguments + // Generate a sensible argument string based on the configuration text + std::string args; + if (tool_name == "Bandit") { + args = "\"-ll -ii\""; + } else if (tool_name == "Pylint") { + args = "\"--max-line-length=100\""; + } else if (tool_name == "Black") { + args = "\"--line-length=100\""; + } else if (tool_name == "clang-tidy") { + args = "\"-checks=clang-analyzer-security.*,cert-*\""; + } else if (tool_name == "Cppcheck") { + args = "\"--enable=all --inconclusive\""; + } else if (tool_name == "Semgrep") { + args = "\"--config=p/security-audit\""; + } else if (tool_name == "GitLeaks") { + args = "\"--config-path=.github/config/gitleaks.toml\""; + } else if (tool_name == "Trivy") { + args = "\"--severity HIGH,CRITICAL\""; + } else { + args = "\"\""; // Default to empty arguments + } + + workflow_file << " " << it->second << ": " << args << "\n"; } + } + } + } + } +} + +bool WorkflowGenerator::generate_lizard_workflow( + const std::vector& languages, + const std::string& output_dir) { + + // Determine which languages are applicable for Lizard analysis + std::vector lizard_languages; + std::unordered_map language_extensions; + + // Define mappings for supported languages to file extensions + language_extensions["C"] = "c"; + language_extensions["C++"] = "cpp,cc,cxx,h,hpp,hxx"; + language_extensions["Python"] = "py"; + + // Create a set of supported languages for Lizard + std::unordered_set supported_languages = {"C", "C++", "Python"}; - workflow_file << " " << it->second << ": " << args << "\n"; + // Filter languages that are supported by Lizard + for (const auto& language : languages) { + if (supported_languages.find(language) != supported_languages.end()) { + lizard_languages.push_back(language); + } + } + + // If no supported languages, skip Lizard workflow + if (lizard_languages.empty()) { + std::cout << "No languages supported by Lizard detected. Skipping Lizard workflow." << std::endl; + return true; + } + + // Create the GitHub Actions workflow file + std::string workflow_filename = output_dir + "/workflows/lizard-complexity.yml"; + std::ofstream workflow_file(workflow_filename); + + if (!workflow_file.is_open()) { + std::cerr << "Could not open file for writing: " << workflow_filename << std::endl; + return false; + } + + // Get Lizard configuration from security_mappings if available + int complexity_threshold = 15; // Default threshold + std::string lizard_args = ""; + + // Get the Lizard configuration from security_mappings.json + if (security_mappings_["scan_types"].contains("code_quality") && + security_mappings_["scan_types"]["code_quality"]["tools"].contains("Lizard")) { + + const auto& lizard_config = security_mappings_["scan_types"]["code_quality"]["tools"]["Lizard"]; + + // Extract the configuration string if available + if (lizard_config.contains("configuration")) { + std::string config_str = lizard_config["configuration"].get(); + + // Parse complexity threshold if it's in the configuration + // Look for numbers after "complexity" in the configuration string + std::size_t pos = config_str.find("complexity"); + if (pos != std::string::npos) { + // Extract the substring after "complexity" and search within it + std::string sub_str = config_str.substr(pos); + std::regex complexityRegex("\\d+"); + std::smatch match; + if (std::regex_search(sub_str, match, complexityRegex)) { + complexity_threshold = std::stoi(match.str()); } } } } - // Add generic tool configs from general_recommendations - if (security_mappings_.contains("general_recommendations")) { - if (scan_type == "security_linting" && - security_mappings_["general_recommendations"].contains("secret_detection")) { - // Add secret detection tools - workflow_file << " REPOSITORY_GITLEAKS_ARGUMENTS: \"--config-path=.github/config/gitleaks.toml\"\n"; - } else if (scan_type == "sast" && - security_mappings_["general_recommendations"].contains("secret_detection")) { - // Add SAST tools - workflow_file << " REPOSITORY_SEMGREP_ARGUMENTS: \"--config=p/security-audit\"\n"; - } else if (scan_type == "cve_scanning" && - (security_mappings_["general_recommendations"].contains("priority_order"))) { - // Add CVE scanning tools - workflow_file << " REPOSITORY_TRIVY_ARGUMENTS: \"--severity HIGH,CRITICAL\"\n"; + // Create workflow content + workflow_file << "---\n"; + workflow_file << "# Lizard code complexity analysis workflow\n"; + workflow_file << "name: Lizard Code Complexity Analysis\n\n"; + workflow_file << "on:\n"; + workflow_file << " push:\n"; + workflow_file << " branches: [main, master]\n"; + workflow_file << " pull_request:\n"; + workflow_file << " branches: [main, master]\n"; + workflow_file << " workflow_dispatch:\n"; + workflow_file << " # Allows manual triggering from GitHub UI\n"; + workflow_file << " inputs:\n"; + workflow_file << " complexity_threshold:\n"; + workflow_file << " description: 'Complexity threshold (default: " << complexity_threshold << ")'\n"; + workflow_file << " required: false\n"; + workflow_file << " default: '" << complexity_threshold << "'\n"; + workflow_file << " type: string\n"; + workflow_file << " schedule:\n"; + workflow_file << " # Run once per week on Sunday at 00:00 UTC\n"; + workflow_file << " - cron: '0 0 * * 0'\n\n"; + workflow_file << "jobs:\n"; + workflow_file << " lizard-analysis:\n"; + workflow_file << " name: Lizard Code Complexity Analysis\n"; + workflow_file << " runs-on: ubuntu-latest\n"; + workflow_file << " steps:\n"; + workflow_file << " # Checkout code\n"; + workflow_file << " - name: Checkout Code\n"; + workflow_file << " uses: actions/checkout@v3\n"; + workflow_file << " with:\n"; + workflow_file << " fetch-depth: 0\n\n"; + + workflow_file << " # Set up Python\n"; + workflow_file << " - name: Set up Python\n"; + workflow_file << " uses: actions/setup-python@v4\n"; + workflow_file << " with:\n"; + workflow_file << " python-version: '3.10'\n\n"; + + workflow_file << " # Install Lizard\n"; + workflow_file << " - name: Install Lizard\n"; + workflow_file << " run: pip install lizard\n\n"; + + // Prepare the file extensions for lizard command + std::string ext_args = ""; + for (const auto& language : lizard_languages) { + if (language_extensions.find(language) != language_extensions.end()) { + std::string exts = language_extensions[language]; + std::istringstream ss(exts); + std::string ext; + + while (std::getline(ss, ext, ',')) { + if (!ext_args.empty()) { + ext_args += " "; + } + ext_args += "--extension ." + ext; + } } } -} + // Add step to set complexity threshold based on input + workflow_file << " # Set complexity threshold\n"; + workflow_file << " - name: Set complexity threshold\n"; + workflow_file << " run: |\n"; + workflow_file << " if [[ \"${{ github.event_name }}\" == \"workflow_dispatch\" && \"${{ github.event.inputs.complexity_threshold }}\" != \"\" ]]; then\n"; + workflow_file << " echo \"COMPLEXITY_THRESHOLD=${{ github.event.inputs.complexity_threshold }}\" >> $GITHUB_ENV\n"; + workflow_file << " else\n"; + workflow_file << " echo \"COMPLEXITY_THRESHOLD=" << complexity_threshold << "\" >> $GITHUB_ENV\n"; + workflow_file << " fi\n\n"; + + // Run Lizard for text report + workflow_file << " # Run Lizard analysis with text output\n"; + workflow_file << " - name: Run Lizard Analysis (Text output)\n"; + workflow_file << " run: |\n"; + workflow_file << " mkdir -p lizard-reports\n"; + workflow_file << " echo \"Using complexity threshold: ${{ env.COMPLEXITY_THRESHOLD }}\"\n"; + workflow_file << " lizard --CCN ${{ env.COMPLEXITY_THRESHOLD }} " << ext_args << " --warnings_only > lizard-reports/complexity_report.txt\n"; + workflow_file << " echo \"Complexity issues found:\" $(grep -c \"has \\d\\+\" lizard-reports/complexity_report.txt || echo \"0\")\n\n"; + + // Upload Lizard reports as artifacts + workflow_file << " # Upload Lizard reports\n"; + workflow_file << " - name: Upload Lizard reports\n"; + workflow_file << " uses: actions/upload-artifact@v3\n"; + workflow_file << " with:\n"; + workflow_file << " name: Lizard Complexity Reports\n"; + workflow_file << " path: lizard-reports/\n\n"; + + workflow_file << " # Check if complexity thresholds were exceeded\n"; + workflow_file << " - name: Check complexity issues\n"; + workflow_file << " run: |\n"; + workflow_file << " if [[ $(grep -c \"has \\d\\+\" lizard-reports/complexity_report.txt || echo \"0\") -gt 0 ]]; then\n"; + workflow_file << " echo \"::warning::Found functions exceeding cyclomatic complexity threshold of ${{ env.COMPLEXITY_THRESHOLD }}\"\n"; + workflow_file << " cat lizard-reports/complexity_report.txt\n"; + workflow_file << " # Uncomment the next line to fail the build on complexity issues\n"; + workflow_file << " # exit 1\n"; + workflow_file << " else\n"; + workflow_file << " echo \"No complexity issues found.\"\n"; + workflow_file << " fi\n"; + + workflow_file.close(); + std::cout << "Generated Lizard workflow file: " << workflow_filename << std::endl; + return true; } } \ No newline at end of file From f58248e1d1a75a9868be4545718092ceea833d68 Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Mon, 23 Jun 2025 20:52:25 -0700 Subject: [PATCH 03/19] Remove configs from output --- src/workflow_generator.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/workflow_generator.cpp b/src/workflow_generator.cpp index 91932d9..17a8248 100644 --- a/src/workflow_generator.cpp +++ b/src/workflow_generator.cpp @@ -90,7 +90,6 @@ bool WorkflowGenerator::generate_workflows(const LanguageStats& stats, const std // Create output directory if it doesn't exist fs::create_directories(output_dir); fs::create_directories(output_dir + "/workflows"); - fs::create_directories(output_dir + "/config"); // Get all detected languages std::vector languages = stats.get_languages(); @@ -165,7 +164,6 @@ bool WorkflowGenerator::generate_megalinter_workflow( workflow_file << " env:\n"; workflow_file << " VALIDATE_ALL_CODEBASE: true\n"; workflow_file << " GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n"; - workflow_file << " MEGALINTER_CONFIG: .github/config/.mega-linter.yml\n"; // Add comment about dynamic language detection workflow_file << " # Dynamically enable linters based on detected languages: "; @@ -456,8 +454,6 @@ void WorkflowGenerator::add_tool_configs( args = "\"--enable=all --inconclusive\""; } else if (tool_name == "Semgrep") { args = "\"--config=p/security-audit\""; - } else if (tool_name == "GitLeaks") { - args = "\"--config-path=.github/config/gitleaks.toml\""; } else if (tool_name == "Trivy") { args = "\"--severity HIGH,CRITICAL\""; } else { From b1ffca9ae179373533638c584671088a3faafc12 Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Mon, 23 Jun 2025 21:03:04 -0700 Subject: [PATCH 04/19] cleanup --- include/singularity/workflow_generator.hpp | 7 ------ src/workflow_generator.cpp | 29 ---------------------- 2 files changed, 36 deletions(-) diff --git a/include/singularity/workflow_generator.hpp b/include/singularity/workflow_generator.hpp index cb4fef7..aaf841e 100644 --- a/include/singularity/workflow_generator.hpp +++ b/include/singularity/workflow_generator.hpp @@ -86,13 +86,6 @@ class WorkflowGenerator { */ std::string map_to_megalinter_group(const std::string& scan_type); - /** - * @brief Map language name to MegaLinter language format - * @param language The language name (e.g., "C++") - * @return Language name in MegaLinter format (e.g., "cpp") - */ - std::string map_language_to_megalinter(const std::string& language); - /** * @brief Get appropriate MegaLinter linters for a language and scan type * @param language The language name diff --git a/src/workflow_generator.cpp b/src/workflow_generator.cpp index 17a8248..0bb28b0 100644 --- a/src/workflow_generator.cpp +++ b/src/workflow_generator.cpp @@ -249,35 +249,6 @@ std::string WorkflowGenerator::map_to_megalinter_group(const std::string& scan_t return ""; } -std::string WorkflowGenerator::map_language_to_megalinter(const std::string& language) { - // Map language names to MegaLinter language identifiers - static const std::unordered_map lang_map = { - {"C", "C"}, - {"C++", "CPP"}, - {"Python", "PYTHON"}, - {"JavaScript", "JAVASCRIPT"}, - {"TypeScript", "TYPESCRIPT"}, - {"Java", "JAVA"}, - {"Go", "GO"}, - {"Ruby", "RUBY"}, - {"PHP", "PHP"}, - {"C#", "CSHARP"}, - {"Shell", "BASH"}, - {"PowerShell", "POWERSHELL"}, - {"R", "R"}, - {"Kotlin", "KOTLIN"}, - {"Swift", "SWIFT"} - }; - - auto it = lang_map.find(language); - if (it != lang_map.end()) { - return it->second; - } - - // Default to empty if no mapping found - return ""; -} - std::vector WorkflowGenerator::get_linters_for_language( const std::string& language, const std::string& scan_type) { From 6f37d209787c45b0c28ff17b51c65b0ca1d72bd9 Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Mon, 23 Jun 2025 21:10:11 -0700 Subject: [PATCH 05/19] New scans --- .github/workflows/cve_scanning.yml | 53 +++++++++++++++++ .github/workflows/lizard-complexity.yml | 77 +++++++++++++++++++++++++ .github/workflows/sast.yml | 55 ++++++++++++++++++ .github/workflows/security_linting.yml | 56 ++++++++++++++++++ .github/workflows/static_analysis.yml | 56 ++++++++++++++++++ 5 files changed, 297 insertions(+) create mode 100644 .github/workflows/cve_scanning.yml create mode 100644 .github/workflows/lizard-complexity.yml create mode 100644 .github/workflows/sast.yml create mode 100644 .github/workflows/security_linting.yml create mode 100644 .github/workflows/static_analysis.yml diff --git a/.github/workflows/cve_scanning.yml b/.github/workflows/cve_scanning.yml new file mode 100644 index 0000000..5f5a01e --- /dev/null +++ b/.github/workflows/cve_scanning.yml @@ -0,0 +1,53 @@ +--- +# MegaLinter GitHub Actions workflow for cve_scanning scans +name: MegaLinter cve_scanning + +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + +jobs: + megalinter-cve_scanning: + name: MegaLinter cve_scanning Scan + runs-on: ubuntu-latest + steps: + # Checkout code + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + # MegaLinter + - name: MegaLinter cve_scanning Scan + id: ml + uses: oxsecurity/megalinter@v8 + env: + VALIDATE_ALL_CODEBASE: true + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Dynamically enable linters based on detected languages: C++, Shell, CMake, Dockerfile, Python, + ENABLE_LINTERS: >- + REPOSITORY_DEPENDABOT, + + # Tool-specific configurations + + # Upload MegaLinter artifacts + - name: Archive production artifacts + uses: actions/upload-artifact@v3 + if: success() || failure() + with: + name: MegaLinter reports + path: | + megalinter-reports + mega-linter.log + + # Create pull request with fixes if applicable + - name: Create Pull Request with applied fixes + uses: peter-evans/create-pull-request@v4 + if: steps.ml.outputs.has_updated_sources == 1 && (env.APPLY_FIXES == 'all' || env.APPLY_FIXES == 'true') + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "[MegaLinter] Apply cve_scanning fixes" + title: "[MegaLinter] Apply cve_scanning fixes" + labels: bot diff --git a/.github/workflows/lizard-complexity.yml b/.github/workflows/lizard-complexity.yml new file mode 100644 index 0000000..fe8d6d5 --- /dev/null +++ b/.github/workflows/lizard-complexity.yml @@ -0,0 +1,77 @@ +--- +# Lizard code complexity analysis workflow +name: Lizard Code Complexity Analysis + +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + workflow_dispatch: + # Allows manual triggering from GitHub UI + inputs: + complexity_threshold: + description: 'Complexity threshold (default: 15)' + required: false + default: '15' + type: string + schedule: + # Run once per week on Sunday at 00:00 UTC + - cron: '0 0 * * 0' + +jobs: + lizard-analysis: + name: Lizard Code Complexity Analysis + runs-on: ubuntu-latest + steps: + # Checkout code + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + # Set up Python + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + # Install Lizard + - name: Install Lizard + run: pip install lizard + + # Set complexity threshold + - name: Set complexity threshold + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ github.event.inputs.complexity_threshold }}" != "" ]]; then + echo "COMPLEXITY_THRESHOLD=${{ github.event.inputs.complexity_threshold }}" >> $GITHUB_ENV + else + echo "COMPLEXITY_THRESHOLD=15" >> $GITHUB_ENV + fi + + # Run Lizard analysis with text output + - name: Run Lizard Analysis (Text output) + run: | + mkdir -p lizard-reports + echo "Using complexity threshold: ${{ env.COMPLEXITY_THRESHOLD }}" + lizard --CCN ${{ env.COMPLEXITY_THRESHOLD }} --extension .cpp --extension .cc --extension .cxx --extension .h --extension .hpp --extension .hxx --extension .py --warnings_only > lizard-reports/complexity_report.txt + echo "Complexity issues found:" $(grep -c "has \d\+" lizard-reports/complexity_report.txt || echo "0") + + # Upload Lizard reports + - name: Upload Lizard reports + uses: actions/upload-artifact@v3 + with: + name: Lizard Complexity Reports + path: lizard-reports/ + + # Check if complexity thresholds were exceeded + - name: Check complexity issues + run: | + if [[ $(grep -c "has \d\+" lizard-reports/complexity_report.txt || echo "0") -gt 0 ]]; then + echo "::warning::Found functions exceeding cyclomatic complexity threshold of ${{ env.COMPLEXITY_THRESHOLD }}" + cat lizard-reports/complexity_report.txt + # Uncomment the next line to fail the build on complexity issues + # exit 1 + else + echo "No complexity issues found." + fi diff --git a/.github/workflows/sast.yml b/.github/workflows/sast.yml new file mode 100644 index 0000000..dc7b02a --- /dev/null +++ b/.github/workflows/sast.yml @@ -0,0 +1,55 @@ +--- +# MegaLinter GitHub Actions workflow for sast scans +name: MegaLinter sast + +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + +jobs: + megalinter-sast: + name: MegaLinter sast Scan + runs-on: ubuntu-latest + steps: + # Checkout code + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + # MegaLinter + - name: MegaLinter sast Scan + id: ml + uses: oxsecurity/megalinter@v8 + env: + VALIDATE_ALL_CODEBASE: true + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Dynamically enable linters based on detected languages: C++, Shell, CMake, Dockerfile, Python, + ENABLE_LINTERS: >- + REPOSITORY_SEMGREP, + + # Tool-specific configurations + REPOSITORY_SEMGREP_ARGUMENTS: "--config=p/security-audit" + REPOSITORY_SEMGREP_ARGUMENTS: "--config=p/security-audit" + + # Upload MegaLinter artifacts + - name: Archive production artifacts + uses: actions/upload-artifact@v3 + if: success() || failure() + with: + name: MegaLinter reports + path: | + megalinter-reports + mega-linter.log + + # Create pull request with fixes if applicable + - name: Create Pull Request with applied fixes + uses: peter-evans/create-pull-request@v4 + if: steps.ml.outputs.has_updated_sources == 1 && (env.APPLY_FIXES == 'all' || env.APPLY_FIXES == 'true') + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "[MegaLinter] Apply sast fixes" + title: "[MegaLinter] Apply sast fixes" + labels: bot diff --git a/.github/workflows/security_linting.yml b/.github/workflows/security_linting.yml new file mode 100644 index 0000000..8225d79 --- /dev/null +++ b/.github/workflows/security_linting.yml @@ -0,0 +1,56 @@ +--- +# MegaLinter GitHub Actions workflow for security_linting scans +name: MegaLinter security_linting + +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + +jobs: + megalinter-security_linting: + name: MegaLinter security_linting Scan + runs-on: ubuntu-latest + steps: + # Checkout code + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + # MegaLinter + - name: MegaLinter security_linting Scan + id: ml + uses: oxsecurity/megalinter@v8 + env: + VALIDATE_ALL_CODEBASE: true + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Dynamically enable linters based on detected languages: C++, Shell, CMake, Dockerfile, Python, + ENABLE_LINTERS: >- + CPP_CLANG_TIDY, + PYTHON_PYLINT, + + # Tool-specific configurations + CPP_CLANG_TIDY_ARGUMENTS: "-checks=clang-analyzer-security.*,cert-*" + PYTHON_PYLINT_ARGUMENTS: "--max-line-length=100" + + # Upload MegaLinter artifacts + - name: Archive production artifacts + uses: actions/upload-artifact@v3 + if: success() || failure() + with: + name: MegaLinter reports + path: | + megalinter-reports + mega-linter.log + + # Create pull request with fixes if applicable + - name: Create Pull Request with applied fixes + uses: peter-evans/create-pull-request@v4 + if: steps.ml.outputs.has_updated_sources == 1 && (env.APPLY_FIXES == 'all' || env.APPLY_FIXES == 'true') + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "[MegaLinter] Apply security_linting fixes" + title: "[MegaLinter] Apply security_linting fixes" + labels: bot diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml new file mode 100644 index 0000000..039b81d --- /dev/null +++ b/.github/workflows/static_analysis.yml @@ -0,0 +1,56 @@ +--- +# MegaLinter GitHub Actions workflow for static_analysis scans +name: MegaLinter static_analysis + +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + +jobs: + megalinter-static_analysis: + name: MegaLinter static_analysis Scan + runs-on: ubuntu-latest + steps: + # Checkout code + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + # MegaLinter + - name: MegaLinter static_analysis Scan + id: ml + uses: oxsecurity/megalinter@v8 + env: + VALIDATE_ALL_CODEBASE: true + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Dynamically enable linters based on detected languages: C++, Shell, CMake, Dockerfile, Python, + ENABLE_LINTERS: >- + CPP_CPPCHECK, + PYTHON_BANDIT, + + # Tool-specific configurations + CPP_CPPCHECK_ARGUMENTS: "--enable=all --inconclusive" + PYTHON_BANDIT_ARGUMENTS: "-ll -ii" + + # Upload MegaLinter artifacts + - name: Archive production artifacts + uses: actions/upload-artifact@v3 + if: success() || failure() + with: + name: MegaLinter reports + path: | + megalinter-reports + mega-linter.log + + # Create pull request with fixes if applicable + - name: Create Pull Request with applied fixes + uses: peter-evans/create-pull-request@v4 + if: steps.ml.outputs.has_updated_sources == 1 && (env.APPLY_FIXES == 'all' || env.APPLY_FIXES == 'true') + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "[MegaLinter] Apply static_analysis fixes" + title: "[MegaLinter] Apply static_analysis fixes" + labels: bot From e018dae8e2508850ed08b936d76c16cf53f8e6f2 Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Mon, 23 Jun 2025 21:14:32 -0700 Subject: [PATCH 06/19] Update upload-artifact to v4 --- src/workflow_generator.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/workflow_generator.cpp b/src/workflow_generator.cpp index 0bb28b0..80d53d0 100644 --- a/src/workflow_generator.cpp +++ b/src/workflow_generator.cpp @@ -206,7 +206,7 @@ bool WorkflowGenerator::generate_megalinter_workflow( workflow_file << "\n"; workflow_file << " # Upload MegaLinter artifacts\n"; workflow_file << " - name: Archive production artifacts\n"; - workflow_file << " uses: actions/upload-artifact@v3\n"; + workflow_file << " uses: actions/upload-artifact@v4\n"; workflow_file << " if: success() || failure()\n"; workflow_file << " with:\n"; workflow_file << " name: MegaLinter reports\n"; @@ -586,7 +586,7 @@ bool WorkflowGenerator::generate_lizard_workflow( // Upload Lizard reports as artifacts workflow_file << " # Upload Lizard reports\n"; workflow_file << " - name: Upload Lizard reports\n"; - workflow_file << " uses: actions/upload-artifact@v3\n"; + workflow_file << " uses: actions/upload-artifact@v4\n"; workflow_file << " with:\n"; workflow_file << " name: Lizard Complexity Reports\n"; workflow_file << " path: lizard-reports/\n\n"; From dee112aaf2d143d0b3a7bc9121c574637d98b9ad Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Mon, 23 Jun 2025 21:15:57 -0700 Subject: [PATCH 07/19] Update generated workflows --- .github/workflows/cve_scanning.yml | 2 +- .github/workflows/lizard-complexity.yml | 2 +- .github/workflows/sast.yml | 2 +- .github/workflows/security_linting.yml | 2 +- .github/workflows/static_analysis.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cve_scanning.yml b/.github/workflows/cve_scanning.yml index 5f5a01e..8fe0ccf 100644 --- a/.github/workflows/cve_scanning.yml +++ b/.github/workflows/cve_scanning.yml @@ -34,7 +34,7 @@ jobs: # Upload MegaLinter artifacts - name: Archive production artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: success() || failure() with: name: MegaLinter reports diff --git a/.github/workflows/lizard-complexity.yml b/.github/workflows/lizard-complexity.yml index fe8d6d5..510fc67 100644 --- a/.github/workflows/lizard-complexity.yml +++ b/.github/workflows/lizard-complexity.yml @@ -59,7 +59,7 @@ jobs: # Upload Lizard reports - name: Upload Lizard reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: Lizard Complexity Reports path: lizard-reports/ diff --git a/.github/workflows/sast.yml b/.github/workflows/sast.yml index dc7b02a..70a80df 100644 --- a/.github/workflows/sast.yml +++ b/.github/workflows/sast.yml @@ -36,7 +36,7 @@ jobs: # Upload MegaLinter artifacts - name: Archive production artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: success() || failure() with: name: MegaLinter reports diff --git a/.github/workflows/security_linting.yml b/.github/workflows/security_linting.yml index 8225d79..069900c 100644 --- a/.github/workflows/security_linting.yml +++ b/.github/workflows/security_linting.yml @@ -37,7 +37,7 @@ jobs: # Upload MegaLinter artifacts - name: Archive production artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: success() || failure() with: name: MegaLinter reports diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml index 039b81d..5bf8924 100644 --- a/.github/workflows/static_analysis.yml +++ b/.github/workflows/static_analysis.yml @@ -37,7 +37,7 @@ jobs: # Upload MegaLinter artifacts - name: Archive production artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: success() || failure() with: name: MegaLinter reports From 021ad32fabeaf97036b564779e2bfff2802417d3 Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Mon, 23 Jun 2025 21:36:02 -0700 Subject: [PATCH 08/19] Fix workflow generation errors, remove cve scanning --- src/workflow_generator.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/workflow_generator.cpp b/src/workflow_generator.cpp index 80d53d0..813f701 100644 --- a/src/workflow_generator.cpp +++ b/src/workflow_generator.cpp @@ -105,6 +105,11 @@ bool WorkflowGenerator::generate_workflows(const LanguageStats& stats, const std continue; } + if (scan_type == "cve_scanning"){ + //generate_dependabot_workflow(languages, output_dir); + continue; + } + // Generate workflow file for this scan type if (!generate_megalinter_workflow(scan_type, languages, output_dir)) { std::cerr << "Failed to generate workflow for scan type: " << scan_type << std::endl; @@ -154,7 +159,7 @@ bool WorkflowGenerator::generate_megalinter_workflow( workflow_file << " steps:\n"; workflow_file << " # Checkout code\n"; workflow_file << " - name: Checkout Code\n"; - workflow_file << " uses: actions/checkout@v3\n"; + workflow_file << " uses: actions/checkout@v4\n"; workflow_file << " with:\n"; workflow_file << " fetch-depth: 0\n\n"; workflow_file << " # MegaLinter\n"; @@ -422,7 +427,7 @@ void WorkflowGenerator::add_tool_configs( } else if (tool_name == "clang-tidy") { args = "\"-checks=clang-analyzer-security.*,cert-*\""; } else if (tool_name == "Cppcheck") { - args = "\"--enable=all --inconclusive\""; + args = "\"--enable=all --inconclusive --suppress=missingIncludeSystem\""; } else if (tool_name == "Semgrep") { args = "\"--config=p/security-audit\""; } else if (tool_name == "Trivy") { @@ -533,7 +538,7 @@ bool WorkflowGenerator::generate_lizard_workflow( workflow_file << " steps:\n"; workflow_file << " # Checkout code\n"; workflow_file << " - name: Checkout Code\n"; - workflow_file << " uses: actions/checkout@v3\n"; + workflow_file << " uses: actions/checkout@v4\n"; workflow_file << " with:\n"; workflow_file << " fetch-depth: 0\n\n"; @@ -580,7 +585,7 @@ bool WorkflowGenerator::generate_lizard_workflow( workflow_file << " run: |\n"; workflow_file << " mkdir -p lizard-reports\n"; workflow_file << " echo \"Using complexity threshold: ${{ env.COMPLEXITY_THRESHOLD }}\"\n"; - workflow_file << " lizard --CCN ${{ env.COMPLEXITY_THRESHOLD }} " << ext_args << " --warnings_only > lizard-reports/complexity_report.txt\n"; + workflow_file << " python -m lizard --CCN ${{ env.COMPLEXITY_THRESHOLD }} " << ext_args << " --warnings_only > lizard-reports/complexity_report.txt\n"; workflow_file << " echo \"Complexity issues found:\" $(grep -c \"has \\d\\+\" lizard-reports/complexity_report.txt || echo \"0\")\n\n"; // Upload Lizard reports as artifacts From 6c608ae17d12c843f580e54cf26769fb98eb384a Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Mon, 23 Jun 2025 21:37:00 -0700 Subject: [PATCH 09/19] Remove cve scanning, update workflows --- .github/workflows/cve_scanning.yml | 53 ------------------------- .github/workflows/lizard-complexity.yml | 4 +- .github/workflows/sast.yml | 2 +- .github/workflows/security_linting.yml | 2 +- .github/workflows/static_analysis.yml | 4 +- 5 files changed, 6 insertions(+), 59 deletions(-) delete mode 100644 .github/workflows/cve_scanning.yml diff --git a/.github/workflows/cve_scanning.yml b/.github/workflows/cve_scanning.yml deleted file mode 100644 index 8fe0ccf..0000000 --- a/.github/workflows/cve_scanning.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -# MegaLinter GitHub Actions workflow for cve_scanning scans -name: MegaLinter cve_scanning - -on: - push: - branches: [main, master] - pull_request: - branches: [main, master] - -jobs: - megalinter-cve_scanning: - name: MegaLinter cve_scanning Scan - runs-on: ubuntu-latest - steps: - # Checkout code - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - # MegaLinter - - name: MegaLinter cve_scanning Scan - id: ml - uses: oxsecurity/megalinter@v8 - env: - VALIDATE_ALL_CODEBASE: true - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Dynamically enable linters based on detected languages: C++, Shell, CMake, Dockerfile, Python, - ENABLE_LINTERS: >- - REPOSITORY_DEPENDABOT, - - # Tool-specific configurations - - # Upload MegaLinter artifacts - - name: Archive production artifacts - uses: actions/upload-artifact@v4 - if: success() || failure() - with: - name: MegaLinter reports - path: | - megalinter-reports - mega-linter.log - - # Create pull request with fixes if applicable - - name: Create Pull Request with applied fixes - uses: peter-evans/create-pull-request@v4 - if: steps.ml.outputs.has_updated_sources == 1 && (env.APPLY_FIXES == 'all' || env.APPLY_FIXES == 'true') - with: - token: ${{ secrets.GITHUB_TOKEN }} - commit-message: "[MegaLinter] Apply cve_scanning fixes" - title: "[MegaLinter] Apply cve_scanning fixes" - labels: bot diff --git a/.github/workflows/lizard-complexity.yml b/.github/workflows/lizard-complexity.yml index 510fc67..970a12f 100644 --- a/.github/workflows/lizard-complexity.yml +++ b/.github/workflows/lizard-complexity.yml @@ -26,7 +26,7 @@ jobs: steps: # Checkout code - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -54,7 +54,7 @@ jobs: run: | mkdir -p lizard-reports echo "Using complexity threshold: ${{ env.COMPLEXITY_THRESHOLD }}" - lizard --CCN ${{ env.COMPLEXITY_THRESHOLD }} --extension .cpp --extension .cc --extension .cxx --extension .h --extension .hpp --extension .hxx --extension .py --warnings_only > lizard-reports/complexity_report.txt + python -m lizard --CCN ${{ env.COMPLEXITY_THRESHOLD }} --extension .cpp --extension .cc --extension .cxx --extension .h --extension .hpp --extension .hxx --extension .py --warnings_only > lizard-reports/complexity_report.txt echo "Complexity issues found:" $(grep -c "has \d\+" lizard-reports/complexity_report.txt || echo "0") # Upload Lizard reports diff --git a/.github/workflows/sast.yml b/.github/workflows/sast.yml index 70a80df..5333672 100644 --- a/.github/workflows/sast.yml +++ b/.github/workflows/sast.yml @@ -15,7 +15,7 @@ jobs: steps: # Checkout code - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/security_linting.yml b/.github/workflows/security_linting.yml index 069900c..de840e1 100644 --- a/.github/workflows/security_linting.yml +++ b/.github/workflows/security_linting.yml @@ -15,7 +15,7 @@ jobs: steps: # Checkout code - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml index 5bf8924..02503d4 100644 --- a/.github/workflows/static_analysis.yml +++ b/.github/workflows/static_analysis.yml @@ -15,7 +15,7 @@ jobs: steps: # Checkout code - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -32,7 +32,7 @@ jobs: PYTHON_BANDIT, # Tool-specific configurations - CPP_CPPCHECK_ARGUMENTS: "--enable=all --inconclusive" + CPP_CPPCHECK_ARGUMENTS: "--enable=all --inconclusive --suppress=missingIncludeSystem" PYTHON_BANDIT_ARGUMENTS: "-ll -ii" # Upload MegaLinter artifacts From 64c1ece39ac3e396c7cdcca00c703ff688dfde69 Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Mon, 23 Jun 2025 22:08:30 -0700 Subject: [PATCH 10/19] Fix sast workflow generation --- include/singularity/workflow_generator.hpp | 9 +++++ src/workflow_generator.cpp | 45 +--------------------- 2 files changed, 11 insertions(+), 43 deletions(-) diff --git a/include/singularity/workflow_generator.hpp b/include/singularity/workflow_generator.hpp index aaf841e..b97c75a 100644 --- a/include/singularity/workflow_generator.hpp +++ b/include/singularity/workflow_generator.hpp @@ -142,6 +142,15 @@ class WorkflowGenerator { {"REUSE Tool", "REPOSITORY_REUSE"}, {"Trivy", "REPOSITORY_TRIVY"} }; + + // Map tool names to MegaLinter linter config argument names + std::unordered_map tool_to_config = { + {"Cppcheck", "CPP_CPPCHECK_ARGUMENTS: \"--enable=all --inconclusive --suppress=missingIncludeSystem\""}, + {"clang-tidy", "CPP_CLANG_TIDY_ARGUMENTS: \"-checks=clang-analyzer-security.*,cert-*\""}, + {"Bandit", "PYTHON_BANDIT_ARGUMENTS: \"-ll -ii\""}, + {"Pylint", "PYTHON_PYLINT_ARGUMENTS: \"--max-line-length=100\""}, + {"Semgrep", "REPOSITORY_SEMGREP_ARGUMENTS: \"--config=p/security-audit\""} + }; }; } // namespace singularity diff --git a/src/workflow_generator.cpp b/src/workflow_generator.cpp index 813f701..f0bc4e5 100644 --- a/src/workflow_generator.cpp +++ b/src/workflow_generator.cpp @@ -355,25 +355,6 @@ void WorkflowGenerator::add_tool_configs( const std::string& language, const std::string& scan_type) { - // Map tool names to MegaLinter linter config argument names - static const std::unordered_map tool_to_config = { - {"Cppcheck", "CPP_CPPCHECK_ARGUMENTS"}, - {"CppLint", "CPP_CPPLINT_ARGUMENTS"}, - {"clang-tidy", "CPP_CLANG_TIDY_ARGUMENTS"}, - {"Clang Static Analyzer", "CPP_CLANG_TIDY_ARGUMENTS"}, - {"FlawFinder", "CPP_FLAWFINDER_ARGUMENTS"}, - {"Bandit", "PYTHON_BANDIT_ARGUMENTS"}, - {"Pylint", "PYTHON_PYLINT_ARGUMENTS"}, - {"Black", "PYTHON_BLACK_ARGUMENTS"}, - {"Safety", "PYTHON_SAFETY_ARGUMENTS"}, - {"GitLeaks", "REPOSITORY_GITLEAKS_ARGUMENTS"}, - {"Detect-secrets", "REPOSITORY_SECRETLINT_ARGUMENTS"}, - {"Semgrep", "REPOSITORY_SEMGREP_ARGUMENTS"}, - {"CodeQL", "REPOSITORY_CODEQL_ARGUMENTS"}, - {"OWASP Dependency-Check", "REPOSITORY_DEPENDENCY_CHECK_ARGUMENTS"}, - {"Trivy", "REPOSITORY_TRIVY_ARGUMENTS"} - }; - // Check if the scan_type exists in the security mappings if (!security_mappings_["scan_types"].contains(scan_type)) { return; // No configuration available @@ -413,30 +394,8 @@ void WorkflowGenerator::add_tool_configs( if (is_default) { auto it = tool_to_config.find(tool_name); if (it != tool_to_config.end()) { - // Extract the configuration string - const std::string& config_str = tool_info["configuration"].get(); - - // Generate a sensible argument string based on the configuration text - std::string args; - if (tool_name == "Bandit") { - args = "\"-ll -ii\""; - } else if (tool_name == "Pylint") { - args = "\"--max-line-length=100\""; - } else if (tool_name == "Black") { - args = "\"--line-length=100\""; - } else if (tool_name == "clang-tidy") { - args = "\"-checks=clang-analyzer-security.*,cert-*\""; - } else if (tool_name == "Cppcheck") { - args = "\"--enable=all --inconclusive --suppress=missingIncludeSystem\""; - } else if (tool_name == "Semgrep") { - args = "\"--config=p/security-audit\""; - } else if (tool_name == "Trivy") { - args = "\"--severity HIGH,CRITICAL\""; - } else { - args = "\"\""; // Default to empty arguments - } - - workflow_file << " " << it->second << ": " << args << "\n"; + workflow_file << " " << it->second << "\n"; + tool_to_config.erase(it); // Remove to avoid duplicates } } } From b67fb624896a9f18cf4d6a2b503e40f5eff0ad65 Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Mon, 23 Jun 2025 22:08:50 -0700 Subject: [PATCH 11/19] Update sast workflow --- .github/workflows/sast.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sast.yml b/.github/workflows/sast.yml index 5333672..2aa4b95 100644 --- a/.github/workflows/sast.yml +++ b/.github/workflows/sast.yml @@ -32,7 +32,6 @@ jobs: # Tool-specific configurations REPOSITORY_SEMGREP_ARGUMENTS: "--config=p/security-audit" - REPOSITORY_SEMGREP_ARGUMENTS: "--config=p/security-audit" # Upload MegaLinter artifacts - name: Archive production artifacts From c263e68d6e78a41c7d9b49ae06c5d81d9412d8c7 Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Mon, 23 Jun 2025 22:51:05 -0700 Subject: [PATCH 12/19] Attempt lizard and semgrep fix --- include/singularity/workflow_generator.hpp | 2 +- src/workflow_generator.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/singularity/workflow_generator.hpp b/include/singularity/workflow_generator.hpp index b97c75a..b0833ae 100644 --- a/include/singularity/workflow_generator.hpp +++ b/include/singularity/workflow_generator.hpp @@ -149,7 +149,7 @@ class WorkflowGenerator { {"clang-tidy", "CPP_CLANG_TIDY_ARGUMENTS: \"-checks=clang-analyzer-security.*,cert-*\""}, {"Bandit", "PYTHON_BANDIT_ARGUMENTS: \"-ll -ii\""}, {"Pylint", "PYTHON_PYLINT_ARGUMENTS: \"--max-line-length=100\""}, - {"Semgrep", "REPOSITORY_SEMGREP_ARGUMENTS: \"--config=p/security-audit\""} + {"Semgrep", "REPOSITORY_SEMGREP_ARGUMENTS: \"--config=p/security-audit\"\n REPOSITORY_SEMGREP_RULESETS_TYPE: security"} }; }; diff --git a/src/workflow_generator.cpp b/src/workflow_generator.cpp index f0bc4e5..9cc5649 100644 --- a/src/workflow_generator.cpp +++ b/src/workflow_generator.cpp @@ -509,7 +509,7 @@ bool WorkflowGenerator::generate_lizard_workflow( workflow_file << " # Install Lizard\n"; workflow_file << " - name: Install Lizard\n"; - workflow_file << " run: pip install lizard\n\n"; + workflow_file << " run: git clone --branch 1.17.31 https://github.com/terryyin/lizard.git\n\n"; // Prepare the file extensions for lizard command std::string ext_args = ""; @@ -544,7 +544,7 @@ bool WorkflowGenerator::generate_lizard_workflow( workflow_file << " run: |\n"; workflow_file << " mkdir -p lizard-reports\n"; workflow_file << " echo \"Using complexity threshold: ${{ env.COMPLEXITY_THRESHOLD }}\"\n"; - workflow_file << " python -m lizard --CCN ${{ env.COMPLEXITY_THRESHOLD }} " << ext_args << " --warnings_only > lizard-reports/complexity_report.txt\n"; + workflow_file << " python lizard/lizard.py --CCN ${{ env.COMPLEXITY_THRESHOLD }} " << ext_args << " --warnings_only > lizard-reports/complexity_report.txt\n"; workflow_file << " echo \"Complexity issues found:\" $(grep -c \"has \\d\\+\" lizard-reports/complexity_report.txt || echo \"0\")\n\n"; // Upload Lizard reports as artifacts From 9e1ddd2d52559186dcf418f232360c934abcc773 Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Mon, 23 Jun 2025 22:51:25 -0700 Subject: [PATCH 13/19] Update lizard and sast workflows --- .github/workflows/lizard-complexity.yml | 4 ++-- .github/workflows/sast.yml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lizard-complexity.yml b/.github/workflows/lizard-complexity.yml index 970a12f..eb5bb8b 100644 --- a/.github/workflows/lizard-complexity.yml +++ b/.github/workflows/lizard-complexity.yml @@ -38,7 +38,7 @@ jobs: # Install Lizard - name: Install Lizard - run: pip install lizard + run: git clone --branch 1.17.31 https://github.com/terryyin/lizard.git # Set complexity threshold - name: Set complexity threshold @@ -54,7 +54,7 @@ jobs: run: | mkdir -p lizard-reports echo "Using complexity threshold: ${{ env.COMPLEXITY_THRESHOLD }}" - python -m lizard --CCN ${{ env.COMPLEXITY_THRESHOLD }} --extension .cpp --extension .cc --extension .cxx --extension .h --extension .hpp --extension .hxx --extension .py --warnings_only > lizard-reports/complexity_report.txt + python lizard/lizard.py --CCN ${{ env.COMPLEXITY_THRESHOLD }} --extension .cpp --extension .cc --extension .cxx --extension .h --extension .hpp --extension .hxx --extension .py --warnings_only > lizard-reports/complexity_report.txt echo "Complexity issues found:" $(grep -c "has \d\+" lizard-reports/complexity_report.txt || echo "0") # Upload Lizard reports diff --git a/.github/workflows/sast.yml b/.github/workflows/sast.yml index 2aa4b95..7413cd1 100644 --- a/.github/workflows/sast.yml +++ b/.github/workflows/sast.yml @@ -32,6 +32,7 @@ jobs: # Tool-specific configurations REPOSITORY_SEMGREP_ARGUMENTS: "--config=p/security-audit" + REPOSITORY_SEMGREP_RULESETS_TYPE: security # Upload MegaLinter artifacts - name: Archive production artifacts From ddb0762fed0c3917992ad1b1e042e28e81f36713 Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Mon, 23 Jun 2025 23:04:26 -0700 Subject: [PATCH 14/19] Update workflows --- .github/workflows/lizard-complexity.yml | 7 +++-- .github/workflows/security_linting.yml | 39 ++++++++++++++++++------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/.github/workflows/lizard-complexity.yml b/.github/workflows/lizard-complexity.yml index eb5bb8b..81200c4 100644 --- a/.github/workflows/lizard-complexity.yml +++ b/.github/workflows/lizard-complexity.yml @@ -38,7 +38,10 @@ jobs: # Install Lizard - name: Install Lizard - run: git clone --branch 1.17.31 https://github.com/terryyin/lizard.git + run: | + git clone --branch 1.17.31 https://github.com/terryyin/lizard.git + cd lizard + python lizard.py --version # Set complexity threshold - name: Set complexity threshold @@ -54,7 +57,7 @@ jobs: run: | mkdir -p lizard-reports echo "Using complexity threshold: ${{ env.COMPLEXITY_THRESHOLD }}" - python lizard/lizard.py --CCN ${{ env.COMPLEXITY_THRESHOLD }} --extension .cpp --extension .cc --extension .cxx --extension .h --extension .hpp --extension .hxx --extension .py --warnings_only > lizard-reports/complexity_report.txt + python lizard.py --CCN ${{ env.COMPLEXITY_THRESHOLD }} --extension .cpp --extension .cc --extension .cxx --extension .h --extension .hpp --extension .hxx --extension .py --warnings_only > lizard-reports/complexity_report.txt echo "Complexity issues found:" $(grep -c "has \d\+" lizard-reports/complexity_report.txt || echo "0") # Upload Lizard reports diff --git a/.github/workflows/security_linting.yml b/.github/workflows/security_linting.yml index de840e1..a6604f4 100644 --- a/.github/workflows/security_linting.yml +++ b/.github/workflows/security_linting.yml @@ -19,7 +19,27 @@ jobs: with: fetch-depth: 0 - # MegaLinter + # Install clang-tidy + - name: Install clang-tidy + run: | + sudo apt-get update + sudo apt-get install -y clang-tidy + + # Run clang-tidy + - name: Run clang-tidy + run: | + mkdir -p clang-tidy-reports + echo "Running clang-tidy security checks..." + find src include -name "*.cpp" -o -name "*.hpp" -o -name "*.h" | xargs -I{} clang-tidy {} -checks=clang-analyzer-security.*,cert-*,security-* -- -I./include > clang-tidy-reports/report.txt || true + if [ -s clang-tidy-reports/report.txt ]; then + echo "::warning::clang-tidy found security issues:" + cat clang-tidy-reports/report.txt + echo "Full report saved to clang-tidy-reports/report.txt" + else + echo "No security issues found by clang-tidy" + fi + + # MegaLinter for Python - name: MegaLinter security_linting Scan id: ml uses: oxsecurity/megalinter@v8 @@ -28,29 +48,26 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Dynamically enable linters based on detected languages: C++, Shell, CMake, Dockerfile, Python, ENABLE_LINTERS: >- - CPP_CLANG_TIDY, PYTHON_PYLINT, # Tool-specific configurations - CPP_CLANG_TIDY_ARGUMENTS: "-checks=clang-analyzer-security.*,cert-*" - PYTHON_PYLINT_ARGUMENTS: "--max-line-length=100" + PYTHON_PYLINT_ARGUMENTS: "--max-line-length=100 --disable=C0111" - # Upload MegaLinter artifacts + # Upload artifacts - name: Archive production artifacts uses: actions/upload-artifact@v4 if: success() || failure() with: - name: MegaLinter reports + name: Security Reports path: | - megalinter-reports - mega-linter.log + clang-tidy-reports # Create pull request with fixes if applicable - name: Create Pull Request with applied fixes uses: peter-evans/create-pull-request@v4 - if: steps.ml.outputs.has_updated_sources == 1 && (env.APPLY_FIXES == 'all' || env.APPLY_FIXES == 'true') + if: steps.ml.outputs.has_updated_sources == 1 with: token: ${{ secrets.GITHUB_TOKEN }} - commit-message: "[MegaLinter] Apply security_linting fixes" - title: "[MegaLinter] Apply security_linting fixes" + commit-message: "Apply security_linting fixes" + title: "Apply security_linting fixes" labels: bot From 0cebf52883e7fbf684627af8d78d9311c72ca4b8 Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Mon, 23 Jun 2025 23:16:31 -0700 Subject: [PATCH 15/19] Fix clang-tidy syntax --- .clang-tidy | 1 - 1 file changed, 1 deletion(-) diff --git a/.clang-tidy b/.clang-tidy index 7ba0208..1122408 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -20,7 +20,6 @@ Checks: > WarningsAsErrors: '' HeaderFilterRegex: '.*' -AnalyzeTemporaryDtors: false FormatStyle: file CheckOptions: - key: readability-identifier-naming.ClassCase From 7b9e81d0bbc4e5a58be15cb9507d7e9f536d1354 Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Mon, 23 Jun 2025 23:25:10 -0700 Subject: [PATCH 16/19] Try lizard again --- .github/workflows/lizard-complexity.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lizard-complexity.yml b/.github/workflows/lizard-complexity.yml index 81200c4..bfadb47 100644 --- a/.github/workflows/lizard-complexity.yml +++ b/.github/workflows/lizard-complexity.yml @@ -40,8 +40,6 @@ jobs: - name: Install Lizard run: | git clone --branch 1.17.31 https://github.com/terryyin/lizard.git - cd lizard - python lizard.py --version # Set complexity threshold - name: Set complexity threshold @@ -57,7 +55,8 @@ jobs: run: | mkdir -p lizard-reports echo "Using complexity threshold: ${{ env.COMPLEXITY_THRESHOLD }}" - python lizard.py --CCN ${{ env.COMPLEXITY_THRESHOLD }} --extension .cpp --extension .cc --extension .cxx --extension .h --extension .hpp --extension .hxx --extension .py --warnings_only > lizard-reports/complexity_report.txt + cd lizard + python lizard.py --CCN ${{ env.COMPLEXITY_THRESHOLD }} --extension .cpp --extension .cc --extension .cxx --extension .h --extension .hpp --extension .hxx --extension .py --warnings_only > ../lizard-reports/complexity_report.txt echo "Complexity issues found:" $(grep -c "has \d\+" lizard-reports/complexity_report.txt || echo "0") # Upload Lizard reports From 92264e187471c050fa8ec09a2508880db31bfe06 Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Mon, 23 Jun 2025 23:33:16 -0700 Subject: [PATCH 17/19] Try pip for lizard again --- .github/workflows/lizard-complexity.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lizard-complexity.yml b/.github/workflows/lizard-complexity.yml index bfadb47..02e995d 100644 --- a/.github/workflows/lizard-complexity.yml +++ b/.github/workflows/lizard-complexity.yml @@ -38,8 +38,7 @@ jobs: # Install Lizard - name: Install Lizard - run: | - git clone --branch 1.17.31 https://github.com/terryyin/lizard.git + run: pip install lizard # Set complexity threshold - name: Set complexity threshold @@ -55,8 +54,7 @@ jobs: run: | mkdir -p lizard-reports echo "Using complexity threshold: ${{ env.COMPLEXITY_THRESHOLD }}" - cd lizard - python lizard.py --CCN ${{ env.COMPLEXITY_THRESHOLD }} --extension .cpp --extension .cc --extension .cxx --extension .h --extension .hpp --extension .hxx --extension .py --warnings_only > ../lizard-reports/complexity_report.txt + lizard --CCN ${{ env.COMPLEXITY_THRESHOLD }} > lizard-reports/complexity_report.txt echo "Complexity issues found:" $(grep -c "has \d\+" lizard-reports/complexity_report.txt || echo "0") # Upload Lizard reports From e54d9c2057b2ca4257afc5e02baee0a4ca57fb9c Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Mon, 23 Jun 2025 23:35:46 -0700 Subject: [PATCH 18/19] Try pip for lizard again2 --- .github/workflows/lizard-complexity.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lizard-complexity.yml b/.github/workflows/lizard-complexity.yml index 02e995d..4e77519 100644 --- a/.github/workflows/lizard-complexity.yml +++ b/.github/workflows/lizard-complexity.yml @@ -54,7 +54,7 @@ jobs: run: | mkdir -p lizard-reports echo "Using complexity threshold: ${{ env.COMPLEXITY_THRESHOLD }}" - lizard --CCN ${{ env.COMPLEXITY_THRESHOLD }} > lizard-reports/complexity_report.txt + lizard --CCN ${{ env.COMPLEXITY_THRESHOLD }} -o lizard-reports/complexity_report.txt echo "Complexity issues found:" $(grep -c "has \d\+" lizard-reports/complexity_report.txt || echo "0") # Upload Lizard reports From bf3bc69c3d103fac2e7d939525f1b43586f6eebc Mon Sep 17 00:00:00 2001 From: Viraj Malia Date: Mon, 23 Jun 2025 23:44:18 -0700 Subject: [PATCH 19/19] Try pip for lizard again3 --- .github/workflows/lizard-complexity.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lizard-complexity.yml b/.github/workflows/lizard-complexity.yml index 4e77519..2f5432b 100644 --- a/.github/workflows/lizard-complexity.yml +++ b/.github/workflows/lizard-complexity.yml @@ -54,8 +54,8 @@ jobs: run: | mkdir -p lizard-reports echo "Using complexity threshold: ${{ env.COMPLEXITY_THRESHOLD }}" - lizard --CCN ${{ env.COMPLEXITY_THRESHOLD }} -o lizard-reports/complexity_report.txt - echo "Complexity issues found:" $(grep -c "has \d\+" lizard-reports/complexity_report.txt || echo "0") + lizard --CCN ${{ env.COMPLEXITY_THRESHOLD }} -o lizard-reports/complexity_report.txt || true + tail -n 13 lizard-reports/complexity_report.txt || true # Upload Lizard reports - name: Upload Lizard reports