diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1ad88ccb68..86128d1eb9 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,6 +1,6 @@ version: 2 updates: - - package-ecosystem: "pip" + - package-ecosystem: "uv" directory: "/" schedule: interval: "weekly" diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5b486a8bb5..a34b3f0e4c 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -18,9 +18,9 @@ permissions: jobs: benchmark: runs-on: ubuntu-latest - + steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 # Need full history for branch comparison @@ -29,10 +29,11 @@ jobs: with: python-version: "3.11" + - name: Install uv + uses: astral-sh/setup-uv@v7 + - name: Install dependencies - run: | - pip install poetry - poetry install --with dev + run: uv sync --group dev - name: Install system dependencies run: | @@ -42,7 +43,7 @@ jobs: # Generate benchmark comparison report using our branch-based script - name: Generate benchmark comparison report run: | - poetry run python bbot/scripts/benchmark_report.py \ + uv run python bbot/scripts/benchmark_report.py \ --base ${{ github.base_ref }} \ --current ${{ github.head_ref }} \ --output benchmark_report.md \ @@ -51,7 +52,7 @@ jobs: # Upload benchmark results as artifacts - name: Upload benchmark results - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: benchmark-results path: | @@ -62,105 +63,89 @@ jobs: # Comment on PR with benchmark results - name: Comment benchmark results on PR - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | const fs = require('fs'); - - try { - const report = fs.readFileSync('benchmark_report.md', 'utf8'); - - // Find existing benchmark comment (with pagination) + + // Helper: find existing benchmark comments on this PR + async function findBenchmarkComments() { const comments = await github.rest.issues.listComments({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, - per_page: 100, // Get more comments per page + per_page: 100, }); - - // Debug: log all comments to see what we're working with + console.log(`Found ${comments.data.length} comments on this PR`); - comments.data.forEach((comment, index) => { - console.log(`Comment ${index}: user=${comment.user.login}, body preview="${comment.body.substring(0, 100)}..."`); - }); - - const existingComments = comments.data.filter(comment => + + const benchmarkComments = comments.data.filter(comment => comment.body.toLowerCase().includes('performance benchmark') && comment.user.login === 'github-actions[bot]' ); - - console.log(`Found ${existingComments.length} existing benchmark comments`); - - if (existingComments.length > 0) { - // Sort comments by creation date to find the most recent - const sortedComments = existingComments.sort((a, b) => + + console.log(`Found ${benchmarkComments.length} existing benchmark comments`); + return benchmarkComments; + } + + // Helper: post or update the benchmark comment + async function upsertComment(body) { + const existing = await findBenchmarkComments(); + + if (existing.length > 0) { + const sorted = existing.sort((a, b) => new Date(b.created_at) - new Date(a.created_at) ); - - const mostRecentComment = sortedComments[0]; - console.log(`Updating most recent benchmark comment: ${mostRecentComment.id} (created: ${mostRecentComment.created_at})`); - + await github.rest.issues.updateComment({ owner: context.repo.owner, repo: context.repo.repo, - comment_id: mostRecentComment.id, - body: report + comment_id: sorted[0].id, + body: body }); - console.log('Updated existing benchmark comment'); - - // Delete any older duplicate comments - if (existingComments.length > 1) { - console.log(`Deleting ${existingComments.length - 1} older duplicate comments`); - for (let i = 1; i < sortedComments.length; i++) { - const commentToDelete = sortedComments[i]; - console.log(`Attempting to delete comment ${commentToDelete.id} (created: ${commentToDelete.created_at})`); - - try { - await github.rest.issues.deleteComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: commentToDelete.id - }); - console.log(`Successfully deleted duplicate comment: ${commentToDelete.id}`); - } catch (error) { - console.error(`Failed to delete comment ${commentToDelete.id}: ${error.message}`); - console.error(`Error details:`, error); - } + console.log(`Updated benchmark comment: ${sorted[0].id}`); + + // Clean up older duplicates + for (let i = 1; i < sorted.length; i++) { + try { + await github.rest.issues.deleteComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: sorted[i].id + }); + console.log(`Deleted duplicate comment: ${sorted[i].id}`); + } catch (e) { + console.error(`Failed to delete comment ${sorted[i].id}: ${e.message}`); } } } else { - // Create new comment await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, - body: report + body: body }); console.log('Created new benchmark comment'); } - } catch (error) { - console.error('Failed to post benchmark results:', error); - - // Post a fallback comment - const fallbackMessage = [ - '## Performance Benchmark Report', - '', - '> ⚠️ **Failed to generate detailed benchmark comparison**', - '> ', - '> The benchmark comparison failed to run. This might be because:', - '> - Benchmark tests don\'t exist on the base branch yet', - '> - Dependencies are missing', - '> - Test execution failed', - '> ', - '> Please check the [workflow logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.', - '> ', - '> 📁 Benchmark artifacts may be available for download from the workflow run.' - ].join('\\n'); - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: fallbackMessage - }); - } \ No newline at end of file + } + + let report; + try { + report = fs.readFileSync('benchmark_report.md', 'utf8'); + } catch (e) { + console.error('Failed to read benchmark report:', e.message); + report = `## Performance Benchmark Report + + > **Failed to generate detailed benchmark comparison** + > + > The benchmark comparison failed to run. This might be because: + > - Benchmark tests don't exist on the base branch yet + > - Dependencies are missing + > - Test execution failed + > + > Please check the [workflow logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. + > + > Benchmark artifacts may be available for download from the workflow run.`; + } + + await upsertComment(report); diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index 089210cefe..bc28e132f0 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -11,5 +11,5 @@ permissions: jobs: cla: - uses: blacklanternsecurity/.github/.github/workflows/cla-reusable.yml@main + uses: blacklanternsecurity/CLA/.github/workflows/cla-reusable.yml@main secrets: inherit diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 3b4d8e2bb7..7117a72e89 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -55,7 +55,7 @@ jobs: # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 # Add any setup steps before running the `github/codeql-action/init` action. # This includes steps like installing compilers or runtimes (`actions/setup-node` diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 72b73dcacc..66ca3f03a4 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -14,14 +14,14 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "kalilinux/kali-rolling", "parrotsec/security"] + os: ["ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "kalilinux/kali-rolling"] steps: - - uses: actions/checkout@v6 - - name: Install Python and Poetry + - uses: actions/checkout@v7 + - name: Install Python and uv run: | if [ -f /etc/os-release ]; then . /etc/os-release - if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ] || [ "$ID" = "kali" ] || [ "$ID" = "parrotsec" ]; then + if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ] || [ "$ID" = "kali" ]; then export DEBIAN_FRONTEND=noninteractive apt-get update apt-get -y install curl git bash build-essential docker.io libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev @@ -38,35 +38,23 @@ jobs: fi fi - # Re-run the script with bash - exec bash -c " - curl https://pyenv.run | bash - export PATH=\"$HOME/.pyenv/bin:\$PATH\" - export PATH=\"$HOME/.local/bin:\$PATH\" - eval \"\$(pyenv init --path)\" - eval \"\$(pyenv init -)\" - eval \"\$(pyenv virtualenv-init -)\" - pyenv install 3.11 - pyenv global 3.11 - pyenv rehash - python3.11 -m pip install --user pipx - python3.11 -m pipx ensurepath - pipx install poetry - " + # Install uv (standalone binary, no Python prerequisite) + curl -LsSf https://astral.sh/uv/install.sh | sh - name: Set OS Environment Variable run: echo "OS_NAME=${{ matrix.os }}" | sed 's|[:/]|_|g' >> $GITHUB_ENV - name: Run tests run: | - export PATH="$HOME/.local/bin:$PATH" - export PATH="$HOME/.pyenv/bin:$PATH" - export PATH="$HOME/.pyenv/shims:$PATH" + export PATH="/github/home/.local/bin:$PATH" + # Fix HOME to match euid (root) so rustup/cargo don't refuse to build + export HOME="$(getent passwd $(whoami) | cut -d: -f6)" export BBOT_DISTRO_TESTS=true - poetry env use python3.11 - poetry install - poetry run pytest --reruns 2 --exitfirst -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO . + uv python install 3.12 + uv python pin 3.12 + uv sync --group dev + uv run pytest --reruns 2 --exitfirst -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO . - name: Upload Debug Logs if: always() - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: pytest-debug-logs-${{ env.OS_NAME }} path: pytest_debug.log diff --git a/.github/workflows/docs_updater.yml b/.github/workflows/docs_updater.yml index 94d5222aca..4ad688e9d2 100644 --- a/.github/workflows/docs_updater.yml +++ b/.github/workflows/docs_updater.yml @@ -9,7 +9,7 @@ jobs: update_docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} ref: dev # Checkout the dev branch @@ -17,13 +17,13 @@ jobs: uses: actions/setup-python@v6 with: python-version: "3.x" + - name: Install uv + uses: astral-sh/setup-uv@v7 - name: Install dependencies - run: | - pip install poetry - poetry install + run: uv sync --frozen --group dev - name: Generate docs run: | - poetry run bbot/scripts/docs.py + uv run --no-sync bbot/scripts/docs.py - name: Create or Update Pull Request uses: peter-evans/create-pull-request@v8 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1db07ac6b8..f5a828e7f2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,34 +17,36 @@ jobs: # if one python version fails, let the others finish fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Set Python Version Environment Variable run: echo "PYTHON_VERSION=${{ matrix.python-version }}" | sed 's|[:/]|_|g' >> $GITHUB_ENV + - name: Install uv + uses: astral-sh/setup-uv@v7 - name: Install dependencies - run: | - pip install poetry - poetry install + run: uv sync --group dev - name: Lint run: | - poetry run ruff check - poetry run ruff format --check + uv run ruff check + uv run ruff format --check - name: Run tests + env: + BBOT_IO_API_KEY: ${{ secrets.BBOT_IO_API_KEY }} run: | - poetry run pytest -vv --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . + uv run pytest -vv --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . - name: Upload Debug Logs if: always() - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: pytest-debug-logs-${{ env.PYTHON_VERSION }} path: pytest_debug.log - name: Upload Code Coverage - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./cov.xml @@ -55,13 +57,13 @@ jobs: if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable') continue-on-error: true steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to Docker Hub - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} @@ -69,23 +71,47 @@ jobs: uses: actions/setup-python@v6 with: python-version: "3.x" - - name: Install dependencies + - name: Install uv + uses: astral-sh/setup-uv@v7 + - name: Calculate version + id: calc_version run: | - python -m pip install --upgrade pip - pip install poetry build - poetry self add "poetry-dynamic-versioning[plugin]" + # Source of truth for base version is pyproject.toml + BASE_VERSION=$(python3 -c " + import re + text = open('pyproject.toml').read() + print(re.search(r'^version\s*=\s*\"([^\"]+)\"', text, re.MULTILINE).group(1)) + ") + + if [[ "${{ github.ref }}" == "refs/heads/stable" ]]; then + VERSION="$BASE_VERSION" + elif [[ "${{ github.ref }}" == "refs/heads/dev" ]]; then + # Dev: base version + commit distance from stable as rc (e.g., 3.0.0.123rc) + STABLE_HEAD=$(git rev-parse origin/stable 2>/dev/null || git rev-list --max-parents=0 HEAD) + DISTANCE=$(git rev-list ${STABLE_HEAD}..HEAD --count) + VERSION="${BASE_VERSION}.${DISTANCE}rc" + fi + + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + echo "Calculated version: $VERSION" + + # For dev builds, write the rc version to pyproject.toml for hatchling + # Stable builds use pyproject.toml as-is + if [[ "${{ github.ref }}" == "refs/heads/dev" ]]; then + sed -i "s/^version = .*/version = \"$VERSION\"/" pyproject.toml + fi - name: Build Pypi package if: github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev' - run: python -m build + run: uv build - name: Publish Pypi package if: github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev' - uses: pypa/gh-action-pypi-publish@release/v1.13 + uses: pypa/gh-action-pypi-publish@release/v1.14 with: password: ${{ secrets.PYPI_API_TOKEN }} - name: Get BBOT version id: version run: | - FULL_VERSION=$(poetry version | cut -d' ' -f2) + FULL_VERSION="${{ steps.calc_version.outputs.VERSION }}" echo "BBOT_VERSION=$FULL_VERSION" >> $GITHUB_OUTPUT # Extract major.minor (e.g., 2.7 from 2.7.1) MAJOR_MINOR=$(echo "$FULL_VERSION" | cut -d'.' -f1-2) @@ -95,48 +121,44 @@ jobs: echo "BBOT_VERSION_MAJOR=$MAJOR" >> $GITHUB_OUTPUT - name: Publish to Docker Hub (dev) if: github.event_name == 'push' && github.ref == 'refs/heads/dev' - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: push: true context: . tags: | - blacklanternsecurity/bbot:latest blacklanternsecurity/bbot:dev blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION }} - blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR_MINOR }} - blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR }} - name: Publish to Docker Hub (stable) if: github.event_name == 'push' && github.ref == 'refs/heads/stable' - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: push: true context: . tags: | + blacklanternsecurity/bbot:latest blacklanternsecurity/bbot:stable blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION }} blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR_MINOR }} blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR }} - name: Publish Full Docker Image to Docker Hub (dev) if: github.event_name == 'push' && github.ref == 'refs/heads/dev' - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: push: true file: Dockerfile.full context: . tags: | - blacklanternsecurity/bbot:latest-full blacklanternsecurity/bbot:dev-full blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION }}-full - blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR_MINOR }}-full - blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR }}-full - name: Publish Full Docker Image to Docker Hub (stable) if: github.event_name == 'push' && github.ref == 'refs/heads/stable' - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: push: true file: Dockerfile.full context: . tags: | + blacklanternsecurity/bbot:latest-full blacklanternsecurity/bbot:stable-full blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION }}-full blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR_MINOR }}-full @@ -177,27 +199,49 @@ jobs: done outputs: BBOT_VERSION: ${{ steps.version.outputs.BBOT_VERSION }} + tag_commit: + needs: publish_code + runs-on: ubuntu-latest + if: github.event_name == 'push' && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev') + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Configure git + run: | + git config --local user.email "info@blacklanternsecurity.com" + git config --local user.name "GitHub Actions" + - name: Tag commit + run: | + VERSION="v${{ needs.publish_code.outputs.BBOT_VERSION }}" + if [[ "${{ github.ref }}" == "refs/heads/stable" ]]; then + git tag -a "$VERSION" -m "Stable Release $VERSION" + else + git tag -a "$VERSION" -m "Dev Release $VERSION" + fi + git push origin "$VERSION" + publish_docs: runs-on: ubuntu-latest if: github.event_name == 'push' && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev') steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} - uses: actions/setup-python@v6 with: python-version: "3.11" - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV - - uses: actions/cache@v5 + - uses: actions/cache@v6 with: key: mkdocs-material-${{ env.cache_id }} path: .cache restore-keys: | mkdocs-material- + - name: Install uv + uses: astral-sh/setup-uv@v7 - name: Install dependencies - run: | - pip install poetry - poetry install --only=docs + run: uv sync --frozen --group docs - name: Configure Git run: | git config user.name github-actions @@ -211,35 +255,12 @@ jobs: - name: Generate docs (stable branch) if: github.ref == 'refs/heads/stable' run: | - poetry run mike deploy Stable + uv run --no-sync mike deploy Stable - name: Generate docs (dev branch) if: github.ref == 'refs/heads/dev' run: | - poetry run mike deploy Dev + uv run --no-sync mike deploy Dev - name: Publish docs run: | git switch gh-pages git push - # tag_commit: - # needs: publish_code - # runs-on: ubuntu-latest - # if: github.event_name == 'push' && github.ref == 'refs/heads/stable' - # steps: - # - uses: actions/checkout@v6 - # with: - # ref: ${{ github.head_ref }} - # fetch-depth: 0 # Fetch all history for all tags and branches - # - name: Configure git - # run: | - # git config --local user.email "info@blacklanternsecurity.com" - # git config --local user.name "GitHub Actions" - # - name: Tag commit - # run: | - # VERSION="${{ needs.publish_code.outputs.BBOT_VERSION }}" - # if [[ "${{ github.ref }}" == "refs/heads/dev" ]]; then - # TAG_MESSAGE="Dev Release $VERSION" - # elif [[ "${{ github.ref }}" == "refs/heads/stable" ]]; then - # TAG_MESSAGE="Stable Release $VERSION" - # fi - # git tag -a $VERSION -m "$TAG_MESSAGE" - # git push origin --tags diff --git a/.github/workflows/version_updater.yml b/.github/workflows/version_updater.yml index e3d6237d2c..446788c328 100644 --- a/.github/workflows/version_updater.yml +++ b/.github/workflows/version_updater.yml @@ -9,7 +9,7 @@ jobs: update-nuclei-version: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: ref: dev fetch-depth: 0 @@ -36,12 +36,12 @@ jobs: - name: Get current version id: get-current-version run: | - version=$(grep -m 1 -oP '(?<=version": ")[^"]*' bbot/modules/nuclei.py) + version=$(grep -m 1 -oP '(?<=Field\(")[^"]*' bbot/modules/nuclei.py) echo "current_version=$version" >> $GITHUB_ENV - name: Update version id: update-version if: env.latest_version != env.current_version - run: "sed -i '0,/\"version\": \".*\",/ s/\"version\": \".*\",/\"version\": \"${{ env.latest_version }}\",/g' bbot/modules/nuclei.py" + run: "sed -i '0,/Field(\"[^\"]*\",/ s/Field(\"[^\"]*\",/Field(\"${{ env.latest_version }}\",/' bbot/modules/nuclei.py" - name: Create pull request to update the version if: steps.update-version.outcome == 'success' uses: peter-evans/create-pull-request@v8 @@ -61,7 +61,7 @@ jobs: update-trufflehog-version: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: ref: dev fetch-depth: 0 @@ -88,12 +88,12 @@ jobs: - name: Get current version id: get-current-version run: | - version=$(grep -m 1 -oP '(?<=version": ")[^"]*' bbot/modules/trufflehog.py) + version=$(grep -m 1 -oP '(?<=Field\(")[^"]*' bbot/modules/trufflehog.py) echo "current_version=$version" >> $GITHUB_ENV - name: Update version id: update-version if: env.latest_version != env.current_version - run: "sed -i '0,/\"version\": \".*\",/ s/\"version\": \".*\",/\"version\": \"${{ env.latest_version }}\",/g' bbot/modules/trufflehog.py" + run: "sed -i '0,/Field(\"[^\"]*\",/ s/Field(\"[^\"]*\",/Field(\"${{ env.latest_version }}\",/' bbot/modules/trufflehog.py" - name: Create pull request to update the version if: steps.update-version.outcome == 'success' uses: peter-evans/create-pull-request@v8 diff --git a/.gitignore b/.gitignore index 0c6b86a341..84360a1600 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ __pycache__/ .coverage* /data/ /neo4j/ +.DS_Store +pytest_debug.log diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..c86e5cf4e1 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,951 @@ +# BBOT Developer Guide + +## Core Principles + +### Modularity Principle +When writing a BBOT module, make sure all module-specific code lives in the module itself. Don't hard-code module-specific things in core or in helpers. + +### DRY Principle +Don't Repeat Yourself -- and interpret this broadly. If two pieces of code aren't identical but follow a similar enough pattern that they could be generalized, they should be. Extract shared logic into a common abstraction rather than duplicating the pattern. Usually this means creating a shared helper, or a shared module template in `bbot/modules/templates`. When you notice structural similarity, unify it. + +### Engineering Principle +Every system that is implemented must be implemented properly. No hacks, no hardcoding, no shortcuts. If we implement one of something, we build a proper system for it. It's okay to take a step back from the current task, in order to do things right. This relates directly to the Modularity Principle above. + +### Testing Principle +BBOT has extremely thorough tests, including **one or more individual tests for each module, with no exceptions**. This is critical to maintaining stability in a recursive tool, which by its nature flirts with race conditions and infinite loops. If you add a module, you write a test. If you change a module, you make sure its test still passes. + +--- + +## Tooling + +- **Package manager**: [uv](https://docs.astral.sh/uv/) +- **Linter/formatter**: [ruff](https://docs.astral.sh/ruff/) (pinned to 0.15.10) +- **Test framework**: [pytest](https://docs.pytest.org/) with pytest-asyncio +- **Python**: 3.10 - 3.14 + +--- + +## Dev Environment Setup + +```bash +# 1. Fork and clone +git clone git@github.com:/bbot.git +cd bbot + +# 2. Switch to dev branch, then create a feature branch +git checkout dev +git checkout -b my-feature + +# 3. Install uv (if you haven't already) +curl -LsSf https://astral.sh/uv/install.sh | sh + +# 4. Install all dependencies (including dev) +uv sync --group dev + +# 5. Install pre-commit hooks (ruff, file checks, etc.) +uv run pre-commit install + +# 6. Activate the virtualenv +source .venv/bin/activate + +# 7. Verify +bbot --help +``` + +### Running Tests + +```bash +# Run the full suite +./bbot/test/run_tests.sh + +# Run specific module tests +./bbot/test/run_tests.sh robots,sslcert + +# Run a single test file directly +pytest bbot/test/test_step_2/module_tests/test_module_robots.py -x -vv +``` + +### Linting + +```bash +ruff check # lint +ruff format # auto-format +ruff format --check # verify formatting without changes +``` + +### Git Workflow + +- `stable` - production releases +- `dev` - active development, **almost all PRs should target this branch** +- Feature branches should be created from `dev` + +--- + +## Architecture Overview + +### How a Scan Works + +BBOT is an async, recursive OSINT tool. A scan starts with **seed events** (targets) and passes them through a pipeline of **modules**. Each module watches for specific event types, processes them, and may emit new events, which feed back into the pipeline. This continues until no module has anything left to do. + +``` + Seeds (targets) + | + v + +--------------+ + | ScanIngress | dedup, blacklist, scope check + +--------------+ + | + v + +--------------+ + | Intercept | dns, cloud tagging + | Modules | (modify/tag events before distribution) + +--------------+ + | + v + +--------------+ + | ScanEgress | scope filtering, graph management + +--------------+ + | + v + +-----------+-----------+ + | | | + Module A Module B Module C ... + | | | + +-----------+-----------+ + | + v + Output Modules (json, csv, neo4j, ...) +``` + +### Events + +Events are the currency of BBOT. Every piece of data -- a hostname, IP, URL, open port, finding -- is an event. Events have: + +- **type**: `DNS_NAME`, `IP_ADDRESS`, `URL`, `OPEN_TCP_PORT`, `HTTP_RESPONSE`, `FINDING`, `EMAIL_ADDRESS`, etc. +- **data**: the actual data (a string, dict, etc.) +- **parent**: the event that led to this one (forming a discovery chain) +- **scope_distance**: how many hops from the original target (0 = in-scope) +- **tags**: metadata like `in-scope`, `affiliate`, `cloud-azure`, `open-port`, etc. +- **module**: which module discovered it + +### Scope Distance + +Scope distance tracks how far an event is from the original target: +- `0` = explicitly in-scope (matches target or discovered in-scope) +- `1` = one hop away (e.g. a hostname found in an SSL cert of an in-scope host) +- `2+` = further away + +The scan's `scope.search_distance` (default 0) controls how far modules are allowed to look. A module's `scope_distance_modifier` adjusts this per-module. + +### Helpers + +BBOT has a helper for almost everything. **Please use them.** They're accessible via `self.helpers` inside any module. + +Key helpers: + +| Helper | What it does | +|--------|-------------| +| `self.helpers.request(url)` | Make an HTTP request (with retries, SSL handling, etc.) | +| `self.helpers.blasthttp` | Shared blasthttp client (rate-limited via `web.http_rate_limit` config) | +| `self.helpers.resolve(host)` | DNS resolution | +| `self.helpers.is_ip(s)` | Check if string is an IP | +| `self.helpers.is_dns_name(s)` | Check if string is a hostname | +| `self.helpers.split_domain(host)` | Split into subdomain + root domain | +| `self.helpers.domain_parents(domain)` | Get all parent domains | +| `self.helpers.make_netloc(host, port)` | Format `host:port` (handles IPv6) | +| `self.helpers.parent_domain(domain)` | Get immediate parent domain | +| `self.helpers.beautifulsoup(html, parser)` | Parse HTML | +| `self.helpers.validators.validate_host(h)` | Validate and normalize a hostname | +| `self.helpers.tempfile(data, pipe=False)` | Create a temp file with content | +| `self.helpers.run(command)` | Run a shell command | +| `self.helpers.run_live(command)` | Run a shell command, stream output | +| `self.helpers.as_completed(tasks)` | Async iteration of completed tasks | +| `self.helpers.wordlist(url_or_path)` | Download/cache a wordlist | +| `self.helpers.rand_string(n)` | Random string of length n | +| `self.helpers.regexes.email_regex` | Pre-compiled email regex | +| `self.helpers.add_get_params(url, params)` | Add query params to a URL | +| `self.helpers.quote(s)` | URL-encode a string | +| `self.helpers.make_ip_type(s)` | Convert string to `ipaddress` object | +| `self.helpers.parse_port_string(s)` | Parse port range string (e.g. `"80,443,8000-9000"`) | +| `self.helpers.top_tcp_ports(n)` | Get top N TCP ports | + +There are hundreds more in `bbot/core/helpers/misc.py`. Browse them before writing utility code yourself. + +--- + +## Writing a Module + +### Quick Start + +1. Create `bbot/modules/my_module.py` +2. Define a class that inherits from `BaseModule` +3. Set `watched_events`, `produced_events`, `flags`, and `meta` +4. Implement `handle_event()` +5. Create `bbot/test/test_step_2/module_tests/test_module_my_module.py` + +Here's a minimal module: + +```python +from bbot.modules.base import BaseModule + + +class my_module(BaseModule): + watched_events = ["DNS_NAME"] + produced_events = ["EMAIL_ADDRESS"] + flags = ["passive", "email-enum"] + meta = { + "description": "Query example.com for email addresses", + "created_date": "2025-01-01", + "author": "@you", + } + + async def handle_event(self, event): + url = f"https://api.example.com/lookup?domain={event.data}" + r = await self.helpers.request(url) + if r and r.status_code == 200: + for email in r.json().get("emails", []): + await self.emit_event( + email, + "EMAIL_ADDRESS", + parent=event, + context=f"{{module}} queried example.com and found {{event.type}}: {{event.data}}", + ) +``` + +And its test: + +```python +from .base import ModuleTestBase + + +class TestMyModule(ModuleTestBase): + async def setup_after_prep(self, module_test): + module_test.blasthttp_mock.add_response( + url="https://api.example.com/lookup?domain=blacklanternsecurity.com", + json={"emails": ["info@blacklanternsecurity.com"]}, + ) + + def check(self, module_test, events): + assert any( + e.data == "info@blacklanternsecurity.com" and e.type == "EMAIL_ADDRESS" + for e in events + ), "Failed to find email" +``` + +### Module Lifecycle + +``` +setup() --> handle_event() (called many times) --> finish() --> report() --> cleanup() +``` + +1. **`setup()`** - one-time initialization (validate config, download data, check API keys) +2. **`handle_event(event)`** - called for each matching event +3. **`finish()`** - called when the scan is finishing; can still emit events +4. **`report()`** - called once after finish; for summary output +5. **`cleanup()`** - called last; close files, delete temp data; **cannot** emit events + +--- + +### Module Attributes Reference + +#### Event Configuration + +##### `watched_events` (list) +Event types this module wants to process. The module's `handle_event()` is only called for these types. + +```python +# sslcert.py - watches for open ports to grab SSL certs from +watched_events = ["OPEN_TCP_PORT"] + +# newsletters.py - watches HTTP responses to scan HTML +watched_events = ["HTTP_RESPONSE"] + +# json.py (output module) - watches everything +watched_events = ["*"] +``` + +##### `produced_events` (list) +Event types this module may emit. Used for dependency resolution and documentation. + +```python +# sslcert.py - can discover hostnames and emails from certificates +produced_events = ["DNS_NAME", "EMAIL_ADDRESS"] + +# portscan.py - finds open ports +produced_events = ["OPEN_TCP_PORT"] +``` + +##### `flags` (list) +Tags that describe the module's behavior. Must include at least one activity flag (`passive` or `active`). Must also include `safe`, `loud`, or `invasive` (or a combination of `loud` and `invasive`). + +Common flags: +- `passive` / `active` - whether the module touches the target directly +- `safe` - non-intrusive and non-destructive +- `loud` - generates a large amount of network traffic +- `invasive` - intrusive or potentially destructive +- `subdomain-enum` - participates in subdomain enumeration +- `web` - basic web scanning +- `email-enum` - email discovery + +```python +# crt.py - queries a third-party API, never touches the target +flags = ["subdomain-enum", "passive"] + +# sslcert.py - connects directly to target ports +flags = ["affiliates", "subdomain-enum", "email-enum", "active", "web"] +``` + +##### `meta` (dict) +Module metadata. Must include `description`, `created_date`, and `author`. Set `auth_required: True` if the module needs an API key. + +```python +meta = { + "description": "Query crt.sh (certificate transparency) for subdomains", + "created_date": "2022-05-13", + "author": "@TheTechromancer", +} + +# For API-key modules: +meta = {"description": "Query API for subdomains", "auth_required": True} +``` + +--- + +#### Options + +##### `options` / `options_desc` (dict) +User-configurable settings. Access them via `self.config.get("option_name")`. + +```python +# robots.py - configurable parsing options +options = {"include_sitemap": False, "include_allow": True, "include_disallow": True} +options_desc = { + "include_sitemap": "Include 'sitemap' entries", + "include_allow": "Include 'Allow' Entries", + "include_disallow": "Include 'Disallow' Entries", +} + +# In handle_event(): +if self.config.get("include_sitemap") is True: + ... +``` + +```python +# sslcert.py - timeout and behavior options +options = {"timeout": 5.0, "skip_non_ssl": True} +options_desc = {"timeout": "Socket connect timeout in seconds", "skip_non_ssl": "Don't try common non-SSL ports"} +``` + +--- + +#### Scope & Filtering + +##### `scope_distance_modifier` (int or None) -- default: `0` +Controls which events the module accepts based on how far they are from the target. + +- `0` (default) - accept events up to the scan's configured search distance +- `1` - accept events up to search distance + 1 +- `None` - accept all events regardless of distance + +```python +# sslcert.py - looks one hop beyond normal scope, because certificate names +# found on in-scope hosts often reveal related infrastructure +scope_distance_modifier = 1 +``` + +##### `in_scope_only` (bool) -- default: `False` +Only accept events that are explicitly in-scope (distance == 0). More restrictive than `scope_distance_modifier = 0`. + +```python +# robots.py - only fetch robots.txt for in-scope hosts +in_scope_only = True +``` + +##### `target_only` (bool) -- default: `False` +Only accept the initial target/seed events. Useful for modules that should only run once against the original targets. + +##### `accept_seeds` (bool) -- default: `True` for passive, `False` for active +Whether to process seed events (the initial targets provided to the scan). + +##### `accept_url_special` (bool) -- default: `False` +Whether to accept "special" URLs (e.g. JavaScript files) that are not normally distributed to web modules. + +```python +# http.py - needs to process all URLs including special ones +accept_url_special = True +``` + +--- + +#### Deduplication + +##### `accept_dupes` (bool) -- default: `False` +Whether to accept the same event more than once. Most modules should leave this `False`. + +```python +# Output modules set this True because they need to see every event +accept_dupes = True +``` + +##### `suppress_dupes` (bool) -- default: `True` +Whether to suppress duplicate *outgoing* events. Prevents the same event from being emitted twice. + +##### `per_host_only` (bool) -- default: `False` +Only process one event per unique host. After processing `1.2.3.4`, skip any future events for `1.2.3.4`. + +##### `per_hostport_only` (bool) -- default: `False` +Only process one event per unique host:port combination. + +```python +# robots.py - only fetch robots.txt once per host:port +per_hostport_only = True +``` + +##### `per_domain_only` (bool) -- default: `False` +Only process one event per unique root domain. After processing `www.example.com`, skip `api.example.com`. + +```python +# emailformat.py - one API query per domain is enough +per_domain_only = True +``` + +##### `_incoming_dedup_hash(self, event)` -- override for custom dedup +Override this to define custom deduplication logic. Return a hash (int) or `(hash, reason_string)`. + +```python +# securitytxt.py - dedupe by parent domain so we only check security.txt once +# per parent domain, not once per subdomain +def _incoming_dedup_hash(self, event): + parent_domain = self.helpers.parent_domain(event.data) + return hash(parent_domain), "already processed parent domain" +``` + +```python +# subdomain_enum.py template - dedupe by highest or lowest parent domain +def _incoming_dedup_hash(self, event): + return hash(self.make_query(event)), f"dedup_strategy={self.dedup_strategy}" +``` + +--- + +#### Concurrency & Batching + +##### `_module_threads` (int) -- default: `1` +How many `handle_event()` calls can run concurrently. Increase this for I/O-bound modules. + +```python +# sslcert.py - connects to many hosts in parallel +_module_threads = 25 +``` + +##### `_batch_size` (int) -- default: `1` +When > 1, events are collected into batches and passed to `handle_batch(*events)` instead of `handle_event()`. Useful for tools that work better with bulk input. + +```python +# portscan.py - masscan is most efficient with all targets at once +batch_size = 1000000 + +async def handle_batch(self, *events): + targets, correlator = await self.make_targets(events, self.syn_scanned) + async for ip, port, parent_event in self.masscan(targets, correlator): + await self.emit_open_port(ip, port, parent_event) +``` + +##### `_shuffle_incoming_queue` (bool) -- default: `True` +Whether to randomize the order of incoming events. Set to `False` when order matters. + +```python +# portscan.py - processes all events together, order doesn't matter but +# we disable shuffle because batch_size is huge +_shuffle_incoming_queue = False +``` + +--- + +#### Dependencies + +##### `deps_pip` (list) +Python packages to install. + +```python +# sslcert.py +deps_pip = ["pyOpenSSL~=25.3.0"] +``` + +##### `deps_apt` (list) +System packages to install. + +```python +# sslcert.py +deps_apt = ["openssl"] +``` + +##### `deps_modules` (list) +Other BBOT modules that must be enabled for this module to work. + +##### `deps_shell` (list) +Shell commands to run for installation (uses Ansible's `shell` module). + +##### `deps_ansible` (list) +Ansible tasks for complex dependency installation (downloading binaries, etc.). + +```python +# fingerprintx.py - downloads a Go binary +deps_ansible = [ + { + "name": "Download fingerprintx", + "unarchive": { + "src": "https://github.com/.../fingerprintx_{version}_{platform}_{arch}.tar.gz", + "include": "fingerprintx", + "dest": "#{BBOT_TOOLS}", + "remote_src": True, + }, + }, +] +``` + +--- + +#### Priority & Queue + +##### `_priority` (int) -- default: `3` +Module priority from 1 (highest) to 5 (lowest). Lower-priority modules get events first. + +```python +# sslcert.py - runs early because other modules depend on the hostnames it discovers +_priority = 2 +``` + +##### `_qsize` (int) -- default: `1000` +Outgoing event queue size. A smaller queue creates backpressure that helps with rate limiting. + +```python +# subdomain_enum.py template - small queue to combat API rate limiting +_qsize = 10 +``` + +##### `_preserve_graph` (bool) -- default: `False` +Accept duplicate events that are needed for complete event chain construction. Only used by output modules. + +```python +# json.py - needs complete event chains for accurate output +_preserve_graph = True +``` + +##### `_stats_exclude` (bool) -- default: `False` +Exclude this module from scan statistics. Used by output and report modules. + +##### `_disable_auto_module_deps` (bool) -- default: `False` +Prevent BBOT from automatically enabling dependency modules. For example, if your module watches `URL` events, BBOT normally auto-enables `http`. Set this to `True` to prevent that. + +--- + +### Key Methods + +#### `handle_event(self, event)` -- the core method + +Called once for each matching event. This is where your module does its work. + +```python +# robots.py - fetch and parse robots.txt +async def handle_event(self, event): + host = f"{event.parsed_url.scheme}://{event.parsed_url.netloc}/" + url = f"{host}robots.txt" + result = await self.helpers.request(url) + if result: + body = result.text + if body: + for line in body.split("\n"): + if line.startswith("Disallow:"): + path = line.split(": ", 1)[1].lstrip("/") + await self.emit_event( + f"{host}{path}", + "URL_UNVERIFIED", + parent=event, + tags=["spider-danger"], + ) +``` + +#### `handle_batch(self, *events)` -- bulk processing + +Used when `_batch_size > 1`. Receives multiple events at once. + +```python +# portscan.py - bulk port scanning with masscan +async def handle_batch(self, *events): + targets, correlator = await self.make_targets(events, self.syn_scanned) + async for ip, port, parent_event in self.masscan(targets, correlator): + await self.emit_open_port(ip, port, parent_event) +``` + +#### `filter_event(self, event)` -- custom event filtering + +Called before `handle_event()`. Return `True` to accept, `False` to reject, or `(False, "reason")` to reject with a logged reason. + +```python +# sslcert.py - skip ports that don't typically use SSL +async def filter_event(self, event): + if self.skip_non_ssl and event.port in self.non_ssl_ports: + return False, f"Port {event.port} doesn't typically use SSL" + return True +``` + +```python +# subdomain_enum.py template - reject wildcards and cloud resources +async def filter_event(self, event): + query = self.make_query(event) + is_wildcard = await self._is_wildcard(query) + if self.reject_wildcards and is_wildcard: + return False, "Event is a wildcard domain" + return True, "" +``` + +#### `setup(self)` -- one-time initialization + +Return values: +- `True` -- success +- `(True, "message")` -- success with message +- `None` or `(None, "message")` -- **soft fail**: module is disabled, scan continues +- `False` or `(False, "message")` -- **hard fail**: scan aborts + +```python +# portscan.py - validates config, checks masscan, checks IPv6 support +async def setup(self): + self.top_ports = self.config.get("top_ports", 100) + self.rate = self.config.get("rate", 300) + self.ports = self.config.get("ports", "") + if self.ports: + try: + self.helpers.parse_port_string(self.ports) + except ValueError as e: + return False, f"Error parsing ports '{self.ports}': {e}" + # ... + return True +``` + +```python +# subdomain_enum_apikey template - soft-fail if API key is missing +async def setup(self): + await super().setup() + return await self.require_api_key() + # Returns (None, "No API key set") if missing, disabling the module +``` + +#### `finish(self)` -- called when scan is finishing + +Can still emit events. May be called multiple times if new activity is detected. + +#### `report(self)` -- summary output + +Called once after `finish()`. Use for generating tables or summary data. + +```python +# asn.py - output ASN statistics +async def report(self): + self.log_table(table_data, headers=["ASN", "Subnet", "Count"], table_name="asns") +``` + +#### `cleanup(self)` -- resource cleanup + +Called once at the very end. Close files, delete temp files. **Cannot emit events.** + +```python +# json.py +async def cleanup(self): + if getattr(self, "_file", None) is not None: + with suppress(Exception): + self.file.close() +``` + +```python +# portscan.py +async def cleanup(self): + with suppress(Exception): + self.exclude_file.unlink() +``` + +--- + +### Emitting Events + +#### `emit_event(data, event_type, parent, **kwargs)` + +Creates and queues an event for processing by other modules. + +```python +# Simple string event +await self.emit_event("sub.example.com", "DNS_NAME", parent=event) + +# With context (used for discovery chain documentation) +await self.emit_event( + "sub.example.com", + "DNS_NAME", + parent=event, + context=f"{{module}} queried crt.sh and found {{event.type}}: {{event.data}}", +) + +# With tags +await self.emit_event(url, "URL_UNVERIFIED", parent=event, tags=["spider-danger"]) + +# FINDING event (dict data) +await self.emit_event( + { + "host": str(event.host), + "description": "Found something interesting", + "url": event.data["url"], + "severity": "HIGH", + }, + "FINDING", + parent=event, +) +``` + +#### `make_event(data, event_type, parent, **kwargs)` + +Creates an event without emitting it. Useful when you need to inspect or modify it first. + +```python +ssl_event = self.make_event(hostname, "DNS_NAME", parent=event, raise_error=True) +if ssl_event: + await self.emit_event(ssl_event, tags=["affiliate"]) +``` + +--- + +### API Helpers + +#### `api_request(url, **kwargs)` + +HTTP request with automatic retry, rate-limit handling (429), API key cycling, and failure tracking. After too many failures, the module enters error state. + +```python +r = await self.api_request("https://api.example.com/search?q=test") +if r and r.status_code == 200: + data = r.json() +``` + +#### `require_api_key()` + +Validates that an API key is configured. Call in `setup()`. + +```python +async def setup(self): + return await self.require_api_key() +``` + +#### `api_page_iter(url, page_size=100, **kwargs)` + +Async generator for paginated API results. URL can contain `{page}`, `{page_size}`, and `{offset}` placeholders. + +```python +async for page in self.api_page_iter( + "https://api.example.com/search?q=test&page={page}&limit={page_size}" +): + if not page.get("results"): + break + for result in page["results"]: + await self.emit_event(result["hostname"], "DNS_NAME", parent=event) +``` + +--- + +### Running External Processes + +```python +# Run a command and get the result +result = await self.run_process(["nmap", "-p", "22,80", target]) +if result.returncode == 0: + output = result.stdout + +# Stream output line-by-line (for long-running tools) +async for line in self.run_process_live(["masscan", "-oJ", "-", ...]): + data = json.loads(line) +``` + +--- + +### Logging + +```python +self.debug("Low-level detail") # only visible with -d flag +self.verbose("Useful but not critical") # visible with -v flag +self.info("Standard info") +self.success("Something good happened") # green +self.warning("Something concerning") # orange +self.error("Something failed") # red + +# "Huge" variants: entire line in bold color +self.hugesuccess("Major discovery!") +self.hugewarning("Major concern!") +``` + +--- + +### Templates + +For common patterns, inherit from a template instead of `BaseModule` directly. Templates live in `bbot/modules/templates/`: + +- **`subdomain_enum`** - passive subdomain enumeration via free API. Handles dedup, wildcard rejection, query building. +- **`subdomain_enum_apikey`** - same as above but requires an API key. +- **`shodan`** - Shodan API integration. +- **`github`** - GitHub API integration. +- **`censys`** - Censys API integration. +- **`bucket`** - Cloud storage bucket enumeration. +- **`webhook`** - Webhook output. + +Example: `crt.py` inherits from `subdomain_enum` and only needs to override the request/parse logic: + +```python +from bbot.modules.templates.subdomain_enum import subdomain_enum + + +class crt(subdomain_enum): + flags = ["subdomain-enum", "passive"] + watched_events = ["DNS_NAME"] + produced_events = ["DNS_NAME"] + meta = { + "description": "Query crt.sh (certificate transparency) for subdomains", + "created_date": "2022-05-13", + "author": "@TheTechromancer", + } + base_url = "https://crt.sh" + + async def request_url(self, query): + params = {"q": f"%.{query}", "output": "json"} + url = self.helpers.add_get_params(self.base_url, params).geturl() + return await self.api_request(url, timeout=self.http_timeout + 30) + + async def parse_results(self, r, query): + results = set() + for cert_info in r.json(): + domain = cert_info.get("name_value") + if domain: + for d in domain.splitlines(): + results.add(d.lower()) + return results +``` + +--- + +### Module Types + +#### Scan Modules (default) +Normal modules that watch events and produce new ones. This is what you'll write 95% of the time. + +#### Output Modules +Inherit from `BaseOutputModule`. Receive all events and write them somewhere. + +```python +from bbot.modules.output.base import BaseOutputModule + +class my_output(BaseOutputModule): + watched_events = ["*"] + meta = {"description": "Custom output"} + _preserve_graph = True # maintain complete event chains + + async def handle_event(self, event): + # write event to file, database, API, etc. + ... +``` + +Output modules automatically get: +- `accept_dupes = True` +- `scope_distance_modifier = None` (see all events) +- `_stats_exclude = True` + +#### Internal Modules +Inherit from `BaseInternalModule`. System-level modules that aren't exposed to users. + +#### Intercept Modules +Inherit from `BaseInterceptModule`. Special high-priority modules that can modify or reject events before they reach normal modules. Used for DNS resolution, cloud detection, etc. You probably don't need to write one. + +--- + +### Writing Tests + +Every module needs a test in `bbot/test/test_step_2/module_tests/`. The test file must be named `test_module_.py`. + +Test classes inherit from `ModuleTestBase` and follow this pattern: + +```python +from .base import ModuleTestBase + + +class TestMyModule(ModuleTestBase): + # Optional: override targets (default: ["blacklanternsecurity.com"]) + targets = ["http://127.0.0.1:8888"] + + # Optional: override which modules are enabled + modules_overrides = ["http", "my_module"] + + # Optional: override config + config_overrides = {"modules": {"my_module": {"some_option": True}}} + + async def setup_before_prep(self, module_test): + """Called BEFORE the scan is prepared. Set up HTTP mocks here.""" + pass + + async def setup_after_prep(self, module_test): + """Called AFTER the scan is prepared. Modify modules, add mocks here.""" + # Mock an HTTP response + module_test.blasthttp_mock.add_response( + url="https://api.example.com/lookup?domain=blacklanternsecurity.com", + json={"results": ["sub.blacklanternsecurity.com"]}, + ) + + # Mock DNS + await module_test.mock_dns({ + "blacklanternsecurity.com": {"A": ["127.0.0.88"]}, + }) + + # Mock an HTTP server response + module_test.set_expect_requests( + expect_args={"method": "GET", "uri": "/robots.txt"}, + respond_args={"response_data": "Disallow: /secret/"}, + ) + + def check(self, module_test, events): + """Verify the scan produced the expected events.""" + assert any( + e.data == "sub.blacklanternsecurity.com" and e.type == "DNS_NAME" + for e in events + ), "Failed to find subdomain" +``` + +The test lifecycle runs: +1. `setup_before_prep()` - set up mocks +2. Scan `_prep()` - loads modules, config +3. `setup_after_prep()` - modify scan state +4. Scan runs and collects events +5. `check()` - your assertions + +### Test Utilities + +- **`module_test.blasthttp_mock`** - mock HTTP responses +- **`module_test.httpserver`** - real HTTP server on port 8888 +- **`module_test.httpserver_ssl`** - real HTTPS server on port 9999 +- **`module_test.mock_dns(data)`** - mock DNS responses +- **`module_test.mock_interactsh(name)`** - mock out-of-band interactions +- **`module_test.module`** - reference to the module instance being tested +- **`module_test.scan`** - reference to the Scanner instance + +Real example -- `test_module_robots.py`: + +```python +class TestRobots(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["http", "robots"] + config_overrides = {"modules": {"robots": {"include_sitemap": True}}} + + async def setup_after_prep(self, module_test): + robots = "Allow: /allow/\nDisallow: /disallow/\nSitemap: http://127.0.0.1:8888/sitemap.txt" + module_test.set_expect_requests( + expect_args={"method": "GET", "uri": "/robots.txt"}, + respond_args={"response_data": robots}, + ) + + def check(self, module_test, events): + assert any(e.data == "http://127.0.0.1:8888/allow/" for e in events) + assert any(e.data == "http://127.0.0.1:8888/disallow/" for e in events) + assert any(e.data == "http://127.0.0.1:8888/sitemap.txt" for e in events) +``` diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000..ec23a777d5 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +Treat @AGENTS.md the same way you'd treat CLAUDE.md. \ No newline at end of file diff --git a/CONTRIBUTIONS.md b/CONTRIBUTIONS.md new file mode 100644 index 0000000000..b7155f5318 --- /dev/null +++ b/CONTRIBUTIONS.md @@ -0,0 +1,5 @@ +# Contributing to BBOT + +See our full contribution guide at: + +**https://www.blacklanternsecurity.com/bbot/Stable/contribution/** diff --git a/README.md b/README.md index 0070ae52f2..192ddbedf2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![bbot_banner](https://github.com/user-attachments/assets/f02804ce-9478-4f1e-ac4d-9cf5620a3214)](https://github.com/blacklanternsecurity/bbot) -[![Python Version](https://img.shields.io/badge/python-3.10+-FF8400)](https://www.python.org) [![License](https://img.shields.io/badge/license-AGPLv3-FF8400.svg)](https://github.com/blacklanternsecurity/bbot/blob/dev/LICENSE) [![DEF CON Recon Village 2024](https://img.shields.io/badge/DEF%20CON%20Demo%20Labs-2023-FF8400.svg)](https://www.reconvillage.org/talks) [![PyPi Downloads](https://static.pepy.tech/personalized-badge/bbot?right_color=orange&left_color=grey)](https://pepy.tech/project/bbot) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![Tests](https://github.com/blacklanternsecurity/bbot/actions/workflows/tests.yml/badge.svg?branch=stable)](https://github.com/blacklanternsecurity/bbot/actions?query=workflow%3A"tests") [![Codecov](https://codecov.io/gh/blacklanternsecurity/bbot/branch/dev/graph/badge.svg?token=IR5AZBDM5K)](https://codecov.io/gh/blacklanternsecurity/bbot) [![Discord](https://img.shields.io/discord/859164869970362439)](https://discord.com/invite/PZqkgxu5SA) +[![Python Version](https://img.shields.io/badge/python-3.10+-FF8400)](https://www.python.org) [![License](https://img.shields.io/badge/license-AGPLv3-FF8400.svg)](https://github.com/blacklanternsecurity/bbot/blob/dev/LICENSE) [![PyPi Downloads](https://static.pepy.tech/personalized-badge/bbot?right_color=orange&left_color=grey)](https://pepy.tech/project/bbot) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![Tests](https://github.com/blacklanternsecurity/bbot/actions/workflows/tests.yml/badge.svg?branch=stable)](https://github.com/blacklanternsecurity/bbot/actions?query=workflow%3A"tests") [![Codecov](https://codecov.io/gh/blacklanternsecurity/bbot/branch/dev/graph/badge.svg?token=IR5AZBDM5K)](https://codecov.io/gh/blacklanternsecurity/bbot) [![Discord](https://img.shields.io/discord/859164869970362439)](https://discord.com/invite/PZqkgxu5SA) ### **BEE·bot** is a multipurpose scanner inspired by [Spiderfoot](https://github.com/smicallef/spiderfoot), built to automate your **Recon**, **Bug Bounties**, and **ASM**! @@ -20,6 +20,10 @@ pipx install --pip-args '\--pre' bbot _For more installation methods, including [Docker](https://hub.docker.com/r/blacklanternsecurity/bbot), see [Getting Started](https://www.blacklanternsecurity.com/bbot/Stable/)_ +> **Upgrading from 2.x?** BBOT 3.0 contains breaking changes to the CLI, presets, modules, events, and Python API. See the [2.x → 3.0 Migration Guide](https://www.blacklanternsecurity.com/bbot/Stable/migration/3.0_breaking_changes/) ([source](docs/migration/3.0_breaking_changes.md)) before upgrading. + +> **Speed tip:** BBOT's DNS resolver ([blastdns](https://github.com/blacklanternsecurity/blastdns)) spins up multiple threads per resolver in `/etc/resolv.conf`. Adding more unfiltered resolvers dramatically speeds up scans. See the [sample resolv.conf](docs/data/resolv-sample.conf) and [Tips and Tricks](https://www.blacklanternsecurity.com/bbot/Stable/scanning/tips_and_tricks/#speed-up-scans-with-more-dns-resolvers) for details. + ## Example Commands ### 1) Subdomain Finder @@ -69,7 +73,7 @@ config: -BBOT consistently finds 20-50% more subdomains than other tools. The bigger the domain, the bigger the difference. To learn how this is possible, see [How It Works](https://www.blacklanternsecurity.com/bbot/Dev/how_it_works/). +BBOT consistently finds 20-50% more subdomains than other tools. The bigger the domain, the bigger the difference. To learn how this is possible, see [How It Works](https://www.blacklanternsecurity.com/bbot/Stable/how_it_works/). ![subdomain-stats-ebay](https://github.com/blacklanternsecurity/bbot/assets/20261699/de3e7f21-6f52-4ac4-8eab-367296cd385f) @@ -89,7 +93,7 @@ bbot -t evilcorp.com -p spider description: Recursive web spider modules: - - httpx + - http blacklist: # Prevent spider from invalidating sessions by logging out @@ -144,16 +148,16 @@ output_modules: ```bash # run a light web scan against www.evilcorp.com -bbot -t www.evilcorp.com -p web-basic +bbot -t www.evilcorp.com -p web # run a heavy web scan against www.evilcorp.com -bbot -t www.evilcorp.com -p web-thorough +bbot -t www.evilcorp.com -p web-heavy ``` - +
-web-basic.yml +web.yml ```yaml description: Quick web scan @@ -162,43 +166,43 @@ include: - iis-shortnames flags: - - web-basic + - web ```
- + - +
-web-thorough.yml +web-heavy.yml ```yaml description: Aggressive web scan include: - # include the web-basic preset - - web-basic + # include the web preset + - web flags: - - web-thorough + - web-heavy ```
- + ### 5) Everything Everywhere All at Once ```bash # everything everywhere all at once -bbot -t evilcorp.com -p kitchen-sink --allow-deadly +bbot -t evilcorp.com -p kitchen-sink # roughly equivalent to: -bbot -t evilcorp.com -p subdomain-enum cloud-enum code-enum email-enum spider web-basic paramminer dirbust-light web-screenshots --allow-deadly +bbot -t evilcorp.com -p subdomain-enum cloud-enum code-enum email-enum spider web paramminer dirbust-light web-screenshots ``` @@ -215,16 +219,26 @@ include: - code-enum - email-enum - spider - - web-basic + - web - paramminer - dirbust-light - web-screenshots - - baddns-intense + - baddns-heavy config: modules: baddns: enable_references: True + dnsbrute: + recursive_mutations: true + dnscommonsrv: + recursive_mutations: true + webbrute: + avoid_wafs: False + wayback: + urls: True + parameters: True + archive: True ``` @@ -382,8 +396,11 @@ For details, see [Configuration](https://www.blacklanternsecurity.com/bbot/Stabl - **Modules** - [List of Modules](https://www.blacklanternsecurity.com/bbot/Stable/modules/list_of_modules) - [Nuclei](https://www.blacklanternsecurity.com/bbot/Stable/modules/nuclei) + - [Wayback](https://www.blacklanternsecurity.com/bbot/Stable/modules/wayback) - [Custom YARA Rules](https://www.blacklanternsecurity.com/bbot/Stable/modules/custom_yara_rules) - [Lightfuzz](https://www.blacklanternsecurity.com/bbot/Stable/modules/lightfuzz) + - **Migration** + - [2.x → 3.0 Breaking Changes](https://www.blacklanternsecurity.com/bbot/Stable/migration/3.0_breaking_changes) - **Misc** - [Contribution](https://www.blacklanternsecurity.com/bbot/Stable/contribution) - [Release History](https://www.blacklanternsecurity.com/bbot/Stable/release_history) @@ -391,8 +408,10 @@ For details, see [Configuration](https://www.blacklanternsecurity.com/bbot/Stabl - **Developer Manual** - [Development Overview](https://www.blacklanternsecurity.com/bbot/Stable/dev/) - [Setting Up a Dev Environment](https://www.blacklanternsecurity.com/bbot/Stable/dev/dev_environment) + - [Core Dependencies](https://www.blacklanternsecurity.com/bbot/Stable/dev/core_dependencies) - [BBOT Internal Architecture](https://www.blacklanternsecurity.com/bbot/Stable/dev/architecture) - [How to Write a BBOT Module](https://www.blacklanternsecurity.com/bbot/Stable/dev/module_howto) + - [Validating & Inspecting Presets](https://www.blacklanternsecurity.com/bbot/Stable/dev/preset_validation) - [Unit Tests](https://www.blacklanternsecurity.com/bbot/Stable/dev/tests) - [Discord Bot Example](https://www.blacklanternsecurity.com/bbot/Stable/dev/discord_bot) - **Code Reference** @@ -402,7 +421,6 @@ For details, see [Configuration](https://www.blacklanternsecurity.com/bbot/Stabl - [Target](https://www.blacklanternsecurity.com/bbot/Stable/dev/target) - [BaseModule](https://www.blacklanternsecurity.com/bbot/Stable/dev/basemodule) - [BBOTCore](https://www.blacklanternsecurity.com/bbot/Stable/dev/core) - - [Engine](https://www.blacklanternsecurity.com/bbot/Stable/dev/engine) - **Helpers** - [Overview](https://www.blacklanternsecurity.com/bbot/Stable/dev/helpers/) - [Command](https://www.blacklanternsecurity.com/bbot/Stable/dev/helpers/command) diff --git a/bbot/__init__.py b/bbot/__init__.py index 914c45ff4b..1a7242313c 100644 --- a/bbot/__init__.py +++ b/bbot/__init__.py @@ -1,6 +1,6 @@ -# version placeholder (replaced by poetry-dynamic-versioning) -__version__ = "v0.0.0" +from importlib.metadata import version, PackageNotFoundError -from .scanner import Scanner, Preset - -__all__ = ["Scanner", "Preset"] +try: + __version__ = version("bbot") +except PackageNotFoundError: + __version__ = "0.0.0" diff --git a/bbot/cli.py b/bbot/cli.py index 6f88447e1a..72a457d2b7 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -1,26 +1,33 @@ #!/usr/bin/env python3 import io +import os import sys import logging import multiprocessing from bbot.errors import * from bbot import __version__ from bbot.logger import log_to_stderr -from bbot.core.helpers.misc import chain_lists, rm_rf +from bbot.core.helpers.misc import chain_lists if multiprocessing.current_process().name == "MainProcess": + # the --no-color flag is parsed later, so honor it (and the NO_COLOR env) before any color is printed + no_color = "--no-color" in sys.argv or bool(os.environ.get("NO_COLOR", "")) + if no_color: + os.environ["NO_COLOR"] = "1" silent = "-s" in sys.argv or "--silent" in sys.argv if not silent: - ascii_art = rf"""  ______  _____ ____ _______ - | ___ \| __ \ / __ \__ __| - | |___) | |__) | | | | | | - | ___ <| __ <| | | | | | - | |___) | |__) | |__| | | | - |______/|_____/ \____/ |_| - BIGHUGE BLS OSINT TOOL {__version__} + o = "" if no_color else "\033[1;38;5;208m" + e = "" if no_color else "\033[0m" + ascii_art = rf""" {o} ______ {e} _____ ____ _______ + {o}| ___ \{e}| __ \ / __ \__ __| + {o}| |___) {e}| |__) | | | | | | + {o}| ___ <{e}| __ <| | | | | | + {o}| |___) {e}| |__) | |__| | | | + {o}|______/{e}|_____/ \____/ |_| + {o}BIGHUGE{e} BLS OSINT TOOL {__version__} www.blacklanternsecurity.com/bbot """ @@ -56,6 +63,17 @@ async def _main(): return # ensure arguments (-c config options etc.) are valid options = preset.args.parsed + # apply CLI log level options (e.g. --debug/--verbose/--silent) to the + # global core logger even for CLI-only commands (like --install-all-deps) + # that don't construct a full Scanner. + preset.apply_log_level(apply_core=True) + + # which generated config files the user is regenerating this run + reset_labels = [ + label + for label, requested in (("config", options.reset_config), ("secrets", options.reset_secrets)) + if requested + ] # print help if no arguments if len(sys.argv) == 1: @@ -69,6 +87,41 @@ async def _main(): sys.exit(0) return + # --reset-config / --reset-secrets + if reset_labels: + reset_paths = [ + spec["path"] + for spec in preset.module_loader._generated_config_files() + if spec["label"] in reset_labels + ] + log.hugewarning( + "Regenerating from current defaults. Any settings you have customized " + "(uncommented) in these files WILL BE WIPED OUT:" + ) + for p in reset_paths: + log.warning(f" {p}") + log.warning("A backup of each existing file will be saved with a .bak extension.") + try: + stdin_is_tty = sys.stdin.isatty() + except (ValueError, io.UnsupportedOperation): + stdin_is_tty = False + if not options.yes: + if not stdin_is_tty: + log.error("Refusing to reset config without confirmation; re-run with --yes to proceed.") + sys.exit(1) + return + answer = input("Continue? [y/N] ").strip().lower() + if answer not in ("y", "yes"): + log.info("Aborted. No changes made.") + sys.exit(0) + return + backups = preset.module_loader.reset_config_files(reset_labels) + log.success("Regenerated config files from current defaults.") + for b in backups: + log.info(f"Backup saved: {b}") + sys.exit(0) + return + # --list-presets if options.list_presets: print("") @@ -90,7 +143,9 @@ async def _main(): preset._default_output_modules = options.output_modules preset._default_internal_modules = [] - preset.bake() + # Bake a temporary copy of the preset so that flags correctly enable their associated modules before listing them + preset.validate() + preset = preset.bake() # --list-modules if options.list_modules: @@ -145,58 +200,82 @@ async def _main(): return try: - scan = Scanner(preset=preset) - except (PresetAbortError, ValidationError) as e: - log.warning(str(e)) + preset.validate() + except ValidationError as e: + log.error(str(e)) + # if a bad option actually lives in one of the user's generated + # config files (vs, say, a -c CLI typo), point them at the matching + # reset flag -- validate each file's own contents to be sure + import yaml + from bbot.scanner.preset.validate import validate_preset + + for spec in preset.module_loader._generated_config_files(): + if not spec["path"].exists(): + continue + try: + file_config = yaml.safe_load(spec["path"].read_text()) or {} + except yaml.YAMLError: + continue + if isinstance(file_config, dict) and validate_preset( + {"config": file_config}, module_loader=preset.module_loader + ): + log.warning( + f"Some options in {spec['path']} are not recognized. They may be left over from an " + f"older version of BBOT. You have the option of regenerating from current defaults " + f"with: bbot {spec['reset_flag']}" + ) return - - deadly_modules = [ - m for m in scan.preset.scan_modules if "deadly" in preset.preloaded_module(m).get("flags", []) - ] - if deadly_modules and not options.allow_deadly: - log.hugewarning(f"You enabled the following deadly modules: {','.join(deadly_modules)}") - log.hugewarning("Deadly modules are highly intrusive") - log.hugewarning("Please specify --allow-deadly to continue") - return False - - # --current-preset - if options.current_preset: - print(scan.preset.to_yaml()) + baked_preset = preset.bake() + + # --current-preset / --current-preset-full + if options.current_preset or options.current_preset_full: + # Ensure we always have a human-friendly description. Prefer an + # explicit scan_name if present, otherwise fall back to the + # preset name (e.g. "bbot_cli_main"). + if not baked_preset.description: + if baked_preset.scan_name: + baked_preset.description = str(baked_preset.scan_name) + elif baked_preset.name: + baked_preset.description = str(baked_preset.name) + if options.current_preset_full: + print(baked_preset.to_yaml(full_config=True)) + else: + print(baked_preset.to_yaml()) sys.exit(0) return - # --current-preset-full - if options.current_preset_full: - print(scan.preset.to_yaml(full_config=True)) - sys.exit(0) + try: + scan = Scanner(preset=baked_preset) + except (PresetAbortError, ValidationError) as e: + log.warning(str(e)) return # --install-all-deps if options.install_all_deps: + # create a throwaway Scanner solely so that Preset.bake(scan) can perform find_and_replace() on all module configs so that placeholders like "#{BBOT_TOOLS}" are resolved before running Ansible tasks. + from bbot.scanner import Scanner as _ScannerForDeps + preloaded_modules = preset.module_loader.preloaded() - scan_modules = [k for k, v in preloaded_modules.items() if str(v.get("type", "")) == "scan"] - output_modules = [k for k, v in preloaded_modules.items() if str(v.get("type", "")) == "output"] - log.verbose("Creating dummy scan with all modules + output modules for deps installation") - dummy_scan = Scanner(preset=preset, modules=scan_modules, output_modules=output_modules) - dummy_scan.helpers.depsinstaller.force_deps = True + modules_for_deps = [ + k for k, v in preloaded_modules.items() if str(v.get("type", "")) in ("scan", "output") + ] + + # dummy scan used only for environment preparation + dummy_scan = _ScannerForDeps(preset=preset) + + helper = dummy_scan.helpers log.info("Installing module dependencies") - await dummy_scan.load_modules() - log.verbose("Running module setups") - succeeded, hard_failed, soft_failed = await dummy_scan.setup_modules(deps_only=True) - # remove any leftovers from the dummy scan - rm_rf(dummy_scan.home, ignore_errors=True) - rm_rf(dummy_scan.temp_dir, ignore_errors=True) + succeeded, failed = await helper.depsinstaller.install(*modules_for_deps) if succeeded: log.success( f"Successfully installed dependencies for {len(succeeded):,} modules: {','.join(succeeded)}" ) - if soft_failed or hard_failed: - failed = soft_failed + hard_failed + if failed: log.warning(f"Failed to install dependencies for {len(failed):,} modules: {', '.join(failed)}") return False return True - scan_name = str(scan.name) + await scan._prep() log.verbose("") log.verbose("### MODULES ENABLED ###") @@ -205,22 +284,56 @@ async def _main(): log.verbose(row) scan.helpers.word_cloud.load() - await scan._prep() + + scan_name = str(scan.name) if not options.dry_run: log.trace(f"Command: {' '.join(sys.argv)}") - if sys.stdin.isatty(): + # In some environments (e.g. tests) stdin may be closed or not support isatty(). Treat those cases as non-interactive. + try: + stdin_is_tty = sys.stdin.isatty() + except (ValueError, io.UnsupportedOperation): + stdin_is_tty = False + + if stdin_is_tty: # warn if any targets belong directly to a cloud provider if not scan.preset.strict_scope: + from cloudcheck import CloudCheckError + for event in scan.target.seeds.event_seeds: if event.type == "DNS_NAME": - cloudcheck_result = await scan.helpers.cloudcheck.lookup(event.host) + # a cloudcheck failure here (e.g. signatures couldn't be + # fetched) shouldn't abort the scan — this is only a + # pre-scan heads-up, so warn and move on + try: + cloudcheck_result = await scan.helpers.cloudcheck.lookup(event.host) + except CloudCheckError as e: + scan.warning(f"Unable to check whether {event.host} is a cloud domain: {e}") + cloudcheck_result = None if cloudcheck_result: scan.hugewarning( f'YOUR TARGET CONTAINS A CLOUD DOMAIN: "{event.host}". You\'re in for a wild ride!' ) + # warn about loud/invasive modules + loud_modules = [] + invasive_modules = [] + for m in scan.preset.scan_modules: + flags = scan.preset.preloaded_module(m).get("flags", []) + if "loud" in flags: + loud_modules.append(m) + if "invasive" in flags: + invasive_modules.append(m) + if loud_modules: + log.hugewarning( + f"LOUD modules enabled: {','.join(loud_modules)}. These generate a lot of traffic. To exclude, use -ef loud" + ) + if invasive_modules: + log.hugewarning( + f"INVASIVE modules enabled: {','.join(invasive_modules)}. These may be intrusive or destructive. To exclude, use -ef invasive" + ) + if not options.yes: log.hugesuccess(f"Scan ready. Press enter to execute {scan.name}") input() @@ -309,6 +422,8 @@ def main(): import traceback from bbot.core import CORE + log = logging.getLogger("bbot.cli") + global scan_name try: asyncio.run(_main()) @@ -319,10 +434,18 @@ def main(): msg = "Interrupted" if scan_name: msg = f"You killed {scan_name}" + log.warning(msg) + log.trace(traceback.format_exc()) log_to_stderr(msg, level="WARNING") if CORE.logger.log_level <= logging.DEBUG: log_to_stderr(traceback.format_exc(), level="DEBUG") exit(1) + except Exception as e: + log.error(f"Unhandled exception: {e}") + log.trace(traceback.format_exc()) + log_to_stderr(f"Unhandled exception: {e}", level="CRITICAL") + log_to_stderr(traceback.format_exc(), level="DEBUG") + exit(1) if __name__ == "__main__": diff --git a/bbot/constants.py b/bbot/constants.py new file mode 100644 index 0000000000..364aead2ff --- /dev/null +++ b/bbot/constants.py @@ -0,0 +1,72 @@ +SCAN_STATUS_QUEUED = 0 +SCAN_STATUS_NOT_STARTED = 1 +SCAN_STATUS_STARTING = 2 +SCAN_STATUS_RUNNING = 3 +SCAN_STATUS_FINISHING = 4 +SCAN_STATUS_ABORTING = 5 +SCAN_STATUS_FINISHED = 6 +SCAN_STATUS_FAILED = 7 +SCAN_STATUS_ABORTED = 8 + + +SCAN_STATUSES = { + "QUEUED": SCAN_STATUS_QUEUED, + "NOT_STARTED": SCAN_STATUS_NOT_STARTED, + "STARTING": SCAN_STATUS_STARTING, + "RUNNING": SCAN_STATUS_RUNNING, + "FINISHING": SCAN_STATUS_FINISHING, + "ABORTING": SCAN_STATUS_ABORTING, + "FINISHED": SCAN_STATUS_FINISHED, + "FAILED": SCAN_STATUS_FAILED, + "ABORTED": SCAN_STATUS_ABORTED, +} + +SCAN_STATUS_CODES = {v: k for k, v in SCAN_STATUSES.items()} + + +def is_valid_scan_status(status): + """ + Check if a status is a valid scan status + """ + return status in SCAN_STATUSES + + +def is_valid_scan_status_code(status): + """ + Check if a status is a valid scan status code + """ + return status in SCAN_STATUS_CODES + + +def get_scan_status_name(status): + """ + Convert a numeric scan status code to a string status name + """ + try: + if isinstance(status, str): + if not is_valid_scan_status(status): + raise ValueError(f"Invalid scan status: {status}") + return status + elif isinstance(status, int): + return SCAN_STATUS_CODES[status] + else: + raise ValueError(f"Invalid scan status: {status} (must be int or str)") + except KeyError: + raise ValueError(f"Invalid scan status: {status}") + + +def get_scan_status_code(status): + """ + Convert a scan status string to a numeric status code + """ + try: + if isinstance(status, int): + if not is_valid_scan_status_code(status): + raise ValueError(f"Invalid scan status code: {status}") + return status + elif isinstance(status, str): + return SCAN_STATUSES[status] + else: + raise ValueError(f"Invalid scan status: {status} (must be int or str)") + except KeyError: + raise ValueError(f"Invalid scan status: {status}") diff --git a/bbot/core/config/files.py b/bbot/core/config/files.py index 2be7bbaa1a..bf04591132 100644 --- a/bbot/core/config/files.py +++ b/bbot/core/config/files.py @@ -1,41 +1,82 @@ +import os import sys +import yaml +import atexit +import shutil +import tempfile from pathlib import Path -from omegaconf import OmegaConf +from .merge import deep_merge from ...logger import log_to_stderr from ...errors import ConfigLoadError bbot_code_dir = Path(__file__).parent.parent.parent +# cached per-process so every BBOTConfigFiles in a run resolves to the same dir +_test_config_dir = None + + +def isolated_test_config_dir(): + """A throwaway config dir for tests, so we never read or write the user's + real ~/.config/bbot. It's created fresh per run (so previous or concurrent + runs can't interfere) and shared across the run's processes via an env var + so spawned children resolve to the same dir.""" + global _test_config_dir + if _test_config_dir is None: + env_dir = os.environ.get("BBOT_TEST_CONFIG_DIR") + if env_dir: + _test_config_dir = Path(env_dir) + else: + _test_config_dir = Path(tempfile.mkdtemp(prefix="bbot_test_config_")) + os.environ["BBOT_TEST_CONFIG_DIR"] = str(_test_config_dir) + atexit.register(lambda: shutil.rmtree(_test_config_dir, ignore_errors=True)) + return _test_config_dir + class BBOTConfigFiles: - config_dir = (Path.home() / ".config" / "bbot").resolve() defaults_filename = (bbot_code_dir / "defaults.yml").resolve() - config_filename = (config_dir / "bbot.yml").resolve() - secrets_filename = (config_dir / "secrets.yml").resolve() def __init__(self, core): self.core = core + if os.environ.get("BBOT_TESTING", "") == "True": + base_dir = isolated_test_config_dir() + else: + base_dir = Path.home() / ".config" / "bbot" + self.config_dir = base_dir.resolve() + self.config_filename = (self.config_dir / "bbot.yml").resolve() + self.secrets_filename = (self.config_dir / "secrets.yml").resolve() - def _get_config(self, filename, name="config"): + def _get_config(self, filename, name="config") -> dict: filename = Path(filename).resolve() + if not filename.exists(): + return {} try: - conf = OmegaConf.load(str(filename)) + with open(filename) as f: + conf = yaml.safe_load(f) or {} + if not isinstance(conf, dict): + raise ConfigLoadError( + f"Error parsing config at {filename}: expected a YAML mapping at the top level, " + f"got {type(conf).__name__}" + ) cli_silent = any(x in sys.argv for x in ("-s", "--silent")) if __name__ == "__main__" and not cli_silent: log_to_stderr(f"Loaded {name} from {filename}") return conf + except ConfigLoadError: + raise + except yaml.YAMLError as e: + raise ConfigLoadError( + f"YAML syntax error in {filename}:\n\n{e}\n\nPlease check the file for indentation or formatting errors." + ) except Exception as e: - if filename.exists(): - raise ConfigLoadError(f"Error parsing config at {filename}:\n\n{e}") - return OmegaConf.create() + raise ConfigLoadError(f"Error parsing config at {filename}:\n\n{e}") - def get_custom_config(self): - return OmegaConf.merge( + def get_custom_config(self) -> dict: + return deep_merge( self._get_config(self.config_filename, name="config"), self._get_config(self.secrets_filename, name="secrets"), ) - def get_default_config(self): + def get_default_config(self) -> dict: return self._get_config(self.defaults_filename, name="defaults") diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py index c5773a3a0c..dda240a8f6 100644 --- a/bbot/core/config/logger.py +++ b/bbot/core/config/logger.py @@ -78,6 +78,11 @@ def __init__(self, core): self.log_level = logging.INFO def cleanup_logging(self): + # Stop queue listener first (drains queue and stops monitor thread) + if self.listener is not None: + with suppress(Exception): + self.listener.stop() + # Close the queue handler with suppress(Exception): self.queue_handler.close() @@ -91,10 +96,6 @@ def cleanup_logging(self): with suppress(Exception): handler.close() - # Stop queue listener - with suppress(Exception): - self.listener.stop() - def setup_queue_handler(self, logging_queue=None, log_level=logging.DEBUG): if logging_queue is None: logging_queue = self.queue diff --git a/bbot/core/config/merge.py b/bbot/core/config/merge.py new file mode 100644 index 0000000000..a59964995e --- /dev/null +++ b/bbot/core/config/merge.py @@ -0,0 +1,94 @@ +""" +Deep-merge helpers replacing omegaconf's merge semantics. + +`deep_merge(a, b)` returns a new dict that is `a` with `b` merged in: nested +dicts are merged recursively, leaf values (and lists) from `b` replace those in +`a`. This matches `OmegaConf.merge(a, b)` for BBOT's preset layering use case. +""" + +from __future__ import annotations + +from copy import deepcopy +from typing import Any + + +def deep_merge(base: dict[str, Any] | None, *updates: dict[str, Any] | None) -> dict[str, Any]: + """ + Deep-merge one or more update dicts into a copy of `base`. Last wins on + leaf conflicts; lists are replaced wholesale (not concatenated). + + The returned dict shares no mutable state with the inputs — nested dicts, + lists, and other mutable values are deep-copied as they're carried over. + """ + result: dict[str, Any] = deepcopy(base) if base else {} + for update in updates: + if not update: + continue + for k, v in update.items(): + if k in result and isinstance(result[k], dict) and isinstance(v, dict): + result[k] = deep_merge(result[k], v) + else: + result[k] = deepcopy(v) + return result + + +def dotted_get(data: dict[str, Any], path: str, default: Any = None) -> Any: + """ + Look up a dotted path in a nested dict. + + Note: keys containing literal `.` are not addressable (no escape syntax). + + >>> dotted_get({"a": {"b": {"c": 1}}}, "a.b.c") + 1 + >>> dotted_get({"a": 1}, "a.b.c", default="x") + 'x' + """ + cursor: Any = data + for part in path.split("."): + if not isinstance(cursor, dict) or part not in cursor: + return default + cursor = cursor[part] + return cursor + + +def dotted_set(data: dict[str, Any], path: str, value: Any) -> None: + """ + Set a dotted path in a nested dict, creating intermediate dicts as needed. + + Non-dict intermediates are silently replaced. This is intentional — + callers (CLI parsing) feed the result through pydantic validation, which + surfaces any resulting type mismatch. + + >>> d = {} + >>> dotted_set(d, "a.b.c", 1) + >>> d + {'a': {'b': {'c': 1}}} + """ + parts = path.split(".") + cursor = data + for part in parts[:-1]: + if part not in cursor or not isinstance(cursor[part], dict): + cursor[part] = {} + cursor = cursor[part] + cursor[parts[-1]] = value + + +def iter_dotted_paths(data: dict[str, Any], prefix: str = "") -> list[str]: + """ + Return every dotted leaf path in a nested dict. Empty dicts are treated + as leaves (so they round-trip through dotted_get/dotted_set). + + >>> iter_dotted_paths({"a": 1, "b": {"c": 2}}) + ['a', 'b.c'] + """ + paths: list[str] = [] + for k, v in data.items(): + path = f"{prefix}.{k}" if prefix else k + if isinstance(v, dict) and v: + paths.extend(iter_dotted_paths(v, path)) + else: + paths.append(path) + return paths + + +__all__ = ["deep_merge", "dotted_get", "dotted_set", "iter_dotted_paths"] diff --git a/bbot/core/config/models.py b/bbot/core/config/models.py new file mode 100644 index 0000000000..3f3e85105d --- /dev/null +++ b/bbot/core/config/models.py @@ -0,0 +1,488 @@ +""" +Pydantic schema for BBOT's global config and preset files. + +These models describe the *shape* of valid BBOT configuration — field names +and their expected types — so that `validate_preset()` can catch typos +(`scpoe:`, `http_timoeut:`) and type errors at the boundary. + +Defaults live in `bbot/defaults.yml` — the single source of truth. This file +intentionally does **not** repeat those values; every field is optional, and +an absent field passes validation. At runtime, `BBOTConfigFiles` loads the +merged dict straight from YAML, and these models only ever validate shape. +""" + +from __future__ import annotations + +import os +from typing import Annotated, Any, Literal, Optional + +from pydantic import BaseModel, BeforeValidator, ConfigDict, field_validator +from pydantic import Field as _PydanticField +from pydantic_core import PydanticUndefined + +from bbot.core.helpers.validators import validate_fqdn_or_ip + + +STRICT = ConfigDict(extra="forbid") + + +def _normalize_upper(v): + """Uppercase a string so severity/confidence options are case-insensitive.""" + return v.upper() if isinstance(v, str) else v + + +# Single source of truth for severity/confidence option types (used by the baddns family). +# The BeforeValidator normalizes case at validation time so e.g. "low" validates as "LOW". +SeverityLiteral = Annotated[Literal["INFO", "LOW", "MEDIUM", "HIGH", "CRITICAL"], BeforeValidator(_normalize_upper)] +ConfidenceLiteral = Annotated[ + Literal["UNKNOWN", "LOW", "MEDIUM", "HIGH", "CONFIRMED"], BeforeValidator(_normalize_upper) +] + + +def Field(default=PydanticUndefined, *, sensitive: bool = False, mandatory: bool = False, **kwargs): + """ + Drop-in replacement for `pydantic.Field` that records two BBOT-specific + flags as field metadata: + + - `sensitive=True`: value should be redacted when serializing configs + (api keys, passwords, http cookies, …). + - `mandatory=True`: option must be supplied for the module to function; + drives the "Needs API Key" column in `bbot -l` and the + `BaseModule.auth_required` property. + + Both flags are stashed under `json_schema_extra` so pydantic preserves + them on `FieldInfo.json_schema_extra` (and in any generated JSON schema) + without affecting validation. All other arguments pass through unchanged. + """ + extra = dict(kwargs.pop("json_schema_extra", None) or {}) + if sensitive: + extra["sensitive"] = True + if mandatory: + extra["mandatory"] = True + if extra: + kwargs["json_schema_extra"] = extra + return _PydanticField(default, **kwargs) + + +def field_flags(field) -> dict: + """Return the BBOT flags dict for a pydantic FieldInfo (empty if none).""" + extra = getattr(field, "json_schema_extra", None) + return dict(extra) if isinstance(extra, dict) else {} + + +def is_sensitive(field) -> bool: + return bool(field_flags(field).get("sensitive")) + + +def is_mandatory(field) -> bool: + return bool(field_flags(field).get("mandatory")) + + +def _unwrap_optional(annotation): + """Strip a single `Optional[X]` / `Union[X, None]` wrapper, return X. Pass-through otherwise.""" + import typing + + origin = typing.get_origin(annotation) + if origin is typing.Union: + args = [a for a in typing.get_args(annotation) if a is not type(None)] + if len(args) == 1: + return args[0] + return annotation + + +def _resolve_field(model, key): + """Resolve `key` against `model.model_fields`, honoring `Field(alias=...)`.""" + fields = getattr(model, "model_fields", None) + if not fields: + return None + if key in fields: + return fields[key] + for f in fields.values(): + if getattr(f, "alias", None) == key: + return f + return None + + +def _field_submodel(field): + """If `field`'s annotation is a `BaseModel` subclass, return it; else None.""" + if field is None: + return None + ann = _unwrap_optional(field.annotation) + if isinstance(ann, type) and issubclass(ann, BaseModel): + return ann + return None + + +def _yaml_scalar(value): + """yaml.safe_load a raw string, but never coerce to date/time. Non-strings pass through.""" + import datetime as _dt + import yaml as _yaml + + if not isinstance(value, str): + return value + if value == "": + return "" + try: + parsed = _yaml.safe_load(value) + except _yaml.YAMLError: + return value + return value if isinstance(parsed, (_dt.date, _dt.time)) else parsed + + +def coerce_value(value, adapter): + """Coerce one config value toward its declared type via a pydantic TypeAdapter. + + `value` may be a raw CLI string OR an already-parsed YAML value. `adapter` is + the field's TypeAdapter (from the config type index), or None when the field is + unknown -- then fall back to YAML scalar parsing; unknown keys are still caught + by validation. Returns `value` unchanged if it can't be validated as the declared + type, so the schema pass reports the real error instead of coercion hiding it. + """ + from pydantic import ValidationError + + # pydantic won't coerce os.PathLike -> str, so do it up front. + if isinstance(value, os.PathLike): + value = str(value) + + if adapter is None: + return _yaml_scalar(value) if isinstance(value, str) else value + + if isinstance(value, str): + parsed = _yaml_scalar(value) + # Keep the raw string for scalar fields (lossless: "1.10", "0755", dates, + # bad YAML); only the parsed form can satisfy a list/dict-typed field. + primary = parsed if isinstance(parsed, (list, dict)) else value + fallback = parsed if primary is value else value + for candidate in (primary, fallback): + try: + return adapter.validate_python(candidate) + except ValidationError: + pass + return value + + try: + return adapter.validate_python(value) + except ValidationError: + return value + + +def coerce_config(config, index, prefix=""): + """Walk a config dict and coerce each leaf toward its declared type.""" + out = {} + for k, v in config.items(): + dotted = f"{prefix}.{k}" if prefix else k + if isinstance(v, dict): + out[k] = coerce_config(v, index, dotted) + else: + out[k] = coerce_value(v, index.get(dotted)) + return out + + +def partition_sensitive_config(config, model, *, keep_sensitive: bool): + """ + Walk `config` (a dict) alongside the pydantic `model`, returning a copy + that either drops or extracts every `sensitive=True` field. + + - `keep_sensitive=False` -> caller wants the public, non-secret view (used + by `BBOTCore.no_secrets_config`). + - `keep_sensitive=True` -> caller wants only the secrets (used by + `BBOTCore.secrets_only_config` to materialize `~/.config/bbot/secrets.yml`). + + Unknown keys (no matching field in the schema) pass through unchanged when + redacting and are dropped when extracting secrets-only. + """ + import copy as _copy + + if not isinstance(config, dict) or model is None: + return _copy.deepcopy(config) if keep_sensitive is False else {} + + out: dict = {} + for key, val in config.items(): + field = _resolve_field(model, key) + sub_model = _field_submodel(field) + + if sub_model is not None and isinstance(val, dict): + child = partition_sensitive_config(val, sub_model, keep_sensitive=keep_sensitive) + # When redacting, preserve the parent key even if its body was + # entirely sensitive — matches the prior `clean_dict` behavior. + # When extracting secrets-only, drop empty branches so the result + # is just the secrets that exist. + if keep_sensitive: + if child: + out[key] = child + else: + out[key] = child + continue + + if field is None: + # No matching schema field — pass through unchanged when redacting, + # drop when extracting secrets-only. + if not keep_sensitive: + out[key] = _copy.deepcopy(val) + continue + + sensitive = is_sensitive(field) + if keep_sensitive: + if sensitive: + out[key] = _copy.deepcopy(val) + else: + if not sensitive: + out[key] = _copy.deepcopy(val) + return out + + +class ScopeConfig(BaseModel): + model_config = STRICT + + strict: Optional[bool] = None + report_distance: Optional[int] = None + search_distance: Optional[int] = None + + +class DnsConfig(BaseModel): + model_config = STRICT + + disable: Optional[bool] = None + minimal: Optional[bool] = None + threads: Optional[int] = None + cache_size: Optional[int] = None + brute_threads: Optional[int] = None + brute_nameservers: Optional[str] = None + search_distance: Optional[int] = None + runaway_limit: Optional[int] = None + timeout: Optional[int] = None + retries: Optional[int] = None + wildcard_disable: Optional[bool] = None + wildcard_ignore: Optional[list[str]] = None + wildcard_tests: Optional[int] = None + abort_threshold: Optional[int] = None + filter_ptrs: Optional[bool] = None + debug: Optional[bool] = None + omit_queries: Optional[list[str]] = None + + +class BodySpillConfig(BaseModel): + """`web.body_spill` — keeps large HTTP_RESPONSE bodies off the Python heap.""" + + model_config = STRICT + + enabled: Optional[bool] = None + cache_mb: Optional[int] = None + compress: Optional[bool] = None + + +class WebConfig(BaseModel): + model_config = STRICT + + http_proxy: Optional[str] = None + http_proxy_exclude: Optional[list[str]] = None + user_agent: Optional[str] = None + user_agent_suffix: Optional[str] = None + spider_distance: Optional[int] = None + spider_depth: Optional[int] = None + spider_links_per_page: Optional[int] = None + http_timeout: Optional[int] = None + http_timeout_infrastructure: Optional[int] = None + http_headers: Optional[dict[str, str]] = None + http_cookies: Optional[dict[str, str]] = Field(default=None, sensitive=True) + api_retries: Optional[int] = None + http_retries: Optional[int] = None + http_rate_limit: Optional[int] = None + body_spill: Optional[BodySpillConfig] = None + # The `429_*` keys start with a digit, so we expose them via aliases. + sleep_interval_429: Optional[int] = Field(default=None, alias="429_sleep_interval") + max_sleep_interval_429: Optional[int] = Field(default=None, alias="429_max_sleep_interval") + debug: Optional[bool] = None + http_max_redirects: Optional[int] = None + ssl_verify_target: Optional[bool] = None + ssl_verify_infrastructure: Optional[bool] = None + + +class EngineConfig(BaseModel): + model_config = STRICT + + debug: Optional[bool] = None + + +class DepsToolConfig(BaseModel): + """Per-tool dep config (e.g. `deps.ffuf.version`).""" + + model_config = STRICT + + version: Optional[str] = None + + +class DepsConfig(BaseModel): + model_config = STRICT + + behavior: Optional[Literal["abort_on_failure", "retry_failed", "ignore_failed", "disable", "force_install"]] = None + ffuf: Optional[DepsToolConfig] = None + + +class BaseModuleConfig(BaseModel): + """ + Shared base for every module's `class Config(BaseModuleConfig)`. + + Declares the three universal module options that are applied to every + module regardless of declaration. The actual default values live in + `bbot/defaults.yml`; this class only validates shape. + """ + + model_config = STRICT + + batch_size: Optional[int] = Field( + default=None, + description="The number of events to process in a single batch (only applies to batch modules)", + ) + module_threads: Optional[int] = Field( + default=None, + description="How many event handlers to run in parallel", + ) + module_timeout: Optional[int] = Field( + default=None, + description="Max time in seconds to spend handling each event or batch of events", + ) + + +class BBOTConfig(BaseModel): + """ + Root BBOT config schema. Unknown top-level keys are rejected so that + typos like `scpoe:` or `moudules:` become loud errors instead of silent + no-ops. + + This is a validation schema only -- it has no default values. The real + defaults live in `bbot/defaults.yml`. + """ + + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, + ) + + # Basic options + home: Optional[str] = None + keep_scans: Optional[int] = None + status_frequency: Optional[int] = None + redact_secrets: Optional[bool] = None + file_blobs: Optional[bool] = None + folder_blobs: Optional[bool] = None + max_mem_percent: Optional[int] = None + + # Nested sections + scope: Optional[ScopeConfig] = None + dns: Optional[DnsConfig] = None + web: Optional[WebConfig] = None + engine: Optional[EngineConfig] = None + deps: Optional[DepsConfig] = None + + # Module loader paths + module_dirs: Optional[list[str]] = None + + # Module runtime + module_handle_event_timeout: Optional[int] = None + module_handle_batch_timeout: Optional[int] = None + + # Internal module toggles (hardcoded because they're first-class scan + # pipeline features; the set changes rarely) + speculate: Optional[bool] = None + excavate: Optional[bool] = None + aggregate: Optional[bool] = None + dnsresolve: Optional[bool] = None + cloudcheck: Optional[bool] = None + unarchive: Optional[bool] = None + python: Optional[bool] = None + + # URL handling + url_querystring_remove: Optional[bool] = None + url_querystring_collapse: Optional[bool] = None + url_extension_blacklist: Optional[list[str]] = None + url_extension_special: Optional[list[str]] = None + url_extension_static: Optional[list[str]] = None + + # Parameter handling + parameter_blacklist: Optional[list[str]] = None + parameter_blacklist_prefixes: Optional[list[str]] = None + + # Event output filter + omit_event_types: Optional[list[str]] = None + + # Interactsh + interactsh_server: Optional[str] = None + interactsh_token: Optional[str] = Field(default=None, sensitive=True) + interactsh_disable: Optional[bool] = None + + # bbot.io API key (used by the asn helper) + bbot_io_api_key: Optional[str] = Field(default=None, sensitive=True) + + _validate_interactsh_server = field_validator("interactsh_server")(validate_fqdn_or_ip) + + # Per-module configs — validated separately, per-module, against each + # module's own `class Config(BaseModuleConfig)`. + modules: Optional[dict[str, dict[str, Any]]] = None + + +class PresetSchema(BaseModel): + """ + Schema for the top-level keys in a preset YAML file. Catches typos like + `modlues:` or `flgas:` at load time. + + `target`/`targets` and `include`/`presets` are aliases; both are accepted. + The `config` key is validated separately as `BBOTConfig`. + """ + + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, + ) + + target: Optional[list[str]] = None + targets: Optional[list[str]] = None + seeds: Optional[list[str]] = None + blacklist: Optional[list[str]] = None + + modules: Optional[list[str]] = None + output_modules: Optional[list[str]] = None + exclude_modules: Optional[list[str]] = None + exclude_output_modules: Optional[list[str]] = None + flags: Optional[list[str]] = None + require_flags: Optional[list[str]] = None + exclude_flags: Optional[list[str]] = None + + config: Optional[dict[str, Any]] = None + module_dirs: Optional[list[str]] = None + + include: Optional[list[str]] = None + presets: Optional[list[str]] = None + + scan_name: Optional[str] = None + output_dir: Optional[str] = None + name: Optional[str] = None + description: Optional[str] = None + + conditions: Optional[list[str]] = None + + verbose: Optional[bool] = None + debug: Optional[bool] = None + silent: Optional[bool] = None + + +__all__ = [ + "BBOTConfig", + "BaseModuleConfig", + "ConfidenceLiteral", + "DepsConfig", + "DepsToolConfig", + "DnsConfig", + "EngineConfig", + "Field", + "PresetSchema", + "ScopeConfig", + "SeverityLiteral", + "WebConfig", + "coerce_config", + "coerce_value", + "field_flags", + "is_mandatory", + "is_sensitive", + "partition_sensitive_config", +] diff --git a/bbot/core/core.py b/bbot/core/core.py index 5814052771..983d489d0f 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -1,15 +1,14 @@ import os import logging -from copy import copy +from copy import copy, deepcopy from pathlib import Path -from contextlib import suppress -from omegaconf import OmegaConf from bbot.errors import BBOTError +from .config.merge import deep_merge from .multiprocess import SHARED_INTERPRETER_STATE -DEFAULT_CONFIG = None +DEFAULT_CONFIG: dict | None = None class BBOTCore: @@ -26,20 +25,21 @@ class BBOTCore: - load quickly """ - # used for filtering out sensitive config values - secrets_strings = ["api_key", "username", "password", "token", "secret", "_id"] - # don't filter/remove entries under this key - secrets_exclude_keys = ["modules"] - def __init__(self): self._logger = None self._files_config = None - self._config = None - self._custom_config = None + self._config: dict | None = None + self._custom_config: dict | None = None # bare minimum == logging - self.logger + try: + self.logger + except BBOTError as e: + import sys + + print(f"\n[CRITICAL] {e}\n", file=sys.stderr) + sys.exit(1) self.log = logging.getLogger("bbot.core") self._prep_multiprocessing() @@ -85,22 +85,20 @@ def scans_dir(self): return self.home / "scans" @property - def config(self): + def config(self) -> dict: """ - .config is just .default_config + .custom_config merged together + .config is just .default_config + .custom_config merged together. - any new values should be added to custom_config. + Any new values should be added to custom_config. """ if self._config is None: - self._config = OmegaConf.merge(self.default_config, self.custom_config) - # set read-only flag (change .custom_config instead) - OmegaConf.set_readonly(self._config, True) + self._config = deep_merge(self.default_config, self.custom_config) return self._config @property - def default_config(self): + def default_config(self) -> dict: """ - The default BBOT config (from `defaults.yml`). Read-only. + The default BBOT config (from `defaults.yml`). """ global DEFAULT_CONFIG if DEFAULT_CONFIG is None: @@ -111,16 +109,14 @@ def default_config(self): return DEFAULT_CONFIG @default_config.setter - def default_config(self, value): + def default_config(self, value: dict): # we temporarily clear out the config so it can be refreshed if/when default_config changes global DEFAULT_CONFIG self._config = None - DEFAULT_CONFIG = value - # set read-only flag (change .custom_config instead) - OmegaConf.set_readonly(DEFAULT_CONFIG, True) + DEFAULT_CONFIG = dict(value) if value else {} @property - def custom_config(self): + def custom_config(self) -> dict: """ Custom BBOT config (from `~/.config/bbot/bbot.yml`) """ @@ -131,59 +127,59 @@ def custom_config(self): return self._custom_config @custom_config.setter - def custom_config(self, value): - # we temporarily clear out the config so it can be refreshed if/when custom_config changes + def custom_config(self, value: dict): self._config = None - # ensure the modules key is always a dictionary - modules_entry = value.get("modules", None) - if modules_entry is not None and not OmegaConf.is_dict(modules_entry): - value["modules"] = {} - self._custom_config = value + self._custom_config = dict(value) if value else {} def no_secrets_config(self, config): - from .helpers.misc import clean_dict + """Return a copy of `config` with every `sensitive=True` field removed. - with suppress(ValueError): - config = OmegaConf.to_object(config) + Sensitivity is read from the per-field `json_schema_extra["sensitive"]` + flag declared on `BBOTConfig` (and each module's `class Config`). + Module-level redaction uses the composite schema built lazily by + `MODULE_LOADER.config_schema`; if a key isn't covered by any schema + (e.g. an unknown module), it passes through unchanged. + """ + from .config.models import partition_sensitive_config - return clean_dict( - config, - *self.secrets_strings, - fuzzy=True, - exclude_keys=self.secrets_exclude_keys, - ) + return partition_sensitive_config(config, self._config_schema(), keep_sensitive=False) def secrets_only_config(self, config): - from .helpers.misc import filter_dict + """Return a copy of `config` containing only `sensitive=True` fields. + + Inverse of `no_secrets_config()`. Useful for splitting a merged config + into a public `bbot.yml` and a private `secrets.yml`. + """ + from .config.models import partition_sensitive_config - with suppress(ValueError): - config = OmegaConf.to_object(config) + return partition_sensitive_config(config, self._config_schema(), keep_sensitive=True) - return filter_dict( - config, - *self.secrets_strings, - fuzzy=True, - exclude_keys=self.secrets_exclude_keys, - ) + def _config_schema(self): + """Resolve the runtime BBOTConfig schema (with per-module configs).""" + try: + from bbot.core.modules import MODULE_LOADER + + return MODULE_LOADER.config_schema + except Exception: + from .config.models import BBOTConfig + + return BBOTConfig def merge_custom(self, config): - """ - Merge a config into the custom config. - """ - self.custom_config = OmegaConf.merge(self.custom_config, OmegaConf.create(config)) + """Merge a config dict into the custom config.""" + self.custom_config = deep_merge(self.custom_config, dict(config) if config else {}) def merge_default(self, config): - """ - Merge a config into the default config. - """ - self.default_config = OmegaConf.merge(self.default_config, OmegaConf.create(config)) + """Merge a config dict into the default config.""" + self.default_config = deep_merge(self.default_config, dict(config) if config else {}) def copy(self): """ Return a semi-shallow copy of self. (`custom_config` is copied, but `default_config` stays the same) """ core_copy = copy(self) - core_copy._custom_config = self._custom_config.copy() + core_copy._custom_config = deepcopy(self._custom_config) if self._custom_config else {} + core_copy._config = None return core_copy @property diff --git a/bbot/core/engine.py b/bbot/core/engine.py deleted file mode 100644 index d7c821a333..0000000000 --- a/bbot/core/engine.py +++ /dev/null @@ -1,687 +0,0 @@ -import os -import sys -import zmq -import pickle -import asyncio -import inspect -import logging -import tempfile -import traceback -import contextlib -import contextvars -import zmq.asyncio -import multiprocessing -from pathlib import Path -from concurrent.futures import CancelledError -from contextlib import asynccontextmanager, suppress - -from bbot.core import CORE -from bbot.errors import BBOTEngineError -from bbot.core.helpers.async_helpers import get_event_loop -from bbot.core.multiprocess import SHARED_INTERPRETER_STATE -from bbot.core.helpers.misc import rand_string, in_exception_chain - - -error_sentinel = object() - - -class EngineBase: - """ - Base Engine class for Server and Client. - - An Engine is a simple and lightweight RPC implementation that allows offloading async tasks - to a separate process. It leverages ZeroMQ in a ROUTER-DEALER configuration. - - BBOT makes use of this by spawning a dedicated engine for DNS and HTTP tasks. - This offloads I/O and helps free up the main event loop for other tasks. - - To use Engine, you must subclass both EngineClient and EngineServer. - - See the respective EngineClient and EngineServer classes for usage examples. - """ - - ERROR_CLASS = BBOTEngineError - - def __init__(self, debug=False): - self._shutdown_status = False - self.log = logging.getLogger(f"bbot.core.{self.__class__.__name__.lower()}") - self._engine_debug = debug - - def pickle(self, obj): - try: - return pickle.dumps(obj) - except Exception as e: - self.log.error(f"Error serializing object: {obj}: {e}") - self.log.trace(traceback.format_exc()) - return error_sentinel - - def unpickle(self, binary): - try: - return pickle.loads(binary) - except Exception as e: - self.log.error(f"Error deserializing binary: {e}") - self.log.trace(f"Offending binary: {binary}") - self.log.trace(traceback.format_exc()) - return error_sentinel - - async def _infinite_retry(self, callback, *args, **kwargs): - interval = kwargs.pop("_interval", 300) - context = kwargs.pop("_context", "") - # default overall timeout of 10 minutes (300 second interval * 2 iterations) - max_retries = kwargs.pop("_max_retries", 1) - if not context: - context = f"{callback.__name__}({args}, {kwargs})" - retries = 0 - while not self._shutdown_status: - try: - return await asyncio.wait_for(callback(*args, **kwargs), timeout=interval) - except (TimeoutError, asyncio.exceptions.TimeoutError): - self.log.debug(f"{self.name}: Timeout after {interval:,} seconds {context}, retrying...") - retries += 1 - if max_retries is not None and retries > max_retries: - raise TimeoutError(f"Timed out after {(max_retries + 1) * interval:,} seconds {context}") - - def engine_debug(self, *args, **kwargs): - if self._engine_debug: - self.log.trace(*args, **kwargs) - - -class EngineClient(EngineBase): - """ - The client portion of BBOT's RPC Engine. - - To create an engine, you must create a subclass of this class and also - define methods for each of your desired functions. - - Note that this only supports async functions. If you need to offload a synchronous function to another CPU, use BBOT's multiprocessing pool instead. - - Any CPU or I/O intense logic should be implemented in the EngineServer. - - These functions are typically stubs whose only job is to forward the arguments to the server. - - Functions with the same names should be defined on the EngineServer. - - The EngineClient must specify its associated server class via the `SERVER_CLASS` variable. - - Depending on whether your function is a generator, you will use either `run_and_return()`, or `run_and_yield`. - - Examples: - >>> from bbot.core.engine import EngineClient - >>> - >>> class MyClient(EngineClient): - >>> SERVER_CLASS = MyServer - >>> - >>> async def my_function(self, **kwargs) - >>> return await self.run_and_return("my_function", **kwargs) - >>> - >>> async def my_generator(self, **kwargs): - >>> async for _ in self.run_and_yield("my_generator", **kwargs): - >>> yield _ - """ - - SERVER_CLASS = None - - def __init__(self, debug=False, **kwargs): - self.name = f"EngineClient {self.__class__.__name__}" - super().__init__(debug=debug) - self.process = None - if self.SERVER_CLASS is None: - raise ValueError(f"Must set EngineClient SERVER_CLASS, {self.SERVER_CLASS}") - self.CMDS = dict(self.SERVER_CLASS.CMDS) - for k, v in list(self.CMDS.items()): - self.CMDS[v] = k - self.socket_address = f"zmq_{rand_string(8)}.sock" - self.socket_path = Path(tempfile.gettempdir()) / self.socket_address - self.server_kwargs = kwargs.pop("server_kwargs", {}) - self._server_process = None - self.context = zmq.asyncio.Context() - self.context.setsockopt(zmq.LINGER, 0) - self.sockets = set() - - def check_error(self, message): - if isinstance(message, dict) and len(message) == 1 and "_e" in message: - self.engine_debug(f"{self.name}: got error message: {message}") - error, trace = message["_e"] - error = self.ERROR_CLASS(error) - error.engine_traceback = trace - self.engine_debug(f"{self.name}: raising {error.__class__.__name__}") - raise error - return False - - async def run_and_return(self, command, *args, **kwargs): - fn_str = f"{command}({args}, {kwargs})" - self.engine_debug(f"{self.name}: executing run-and-return {fn_str}") - if self._shutdown_status and not command == "_shutdown": - self.log.verbose(f"{self.name} has been shut down and is not accepting new tasks") - return - async with self.new_socket() as socket: - try: - message = self.make_message(command, args=args, kwargs=kwargs) - if message is error_sentinel: - return - await socket.send(message) - binary = await self._infinite_retry(socket.recv, _context=f"waiting for return value from {fn_str}") - except BaseException: - try: - await self.send_cancel_message(socket, fn_str) - except Exception: - self.log.debug(f"{self.name}: {fn_str} failed to send cancel message after exception") - self.log.trace(traceback.format_exc()) - raise - # self.log.debug(f"{self.name}.{command}({kwargs}) got binary: {binary}") - message = self.unpickle(binary) - self.engine_debug(f"{self.name}: {fn_str} got return value: {message}") - # error handling - if self.check_error(message): - return - return message - - async def run_and_yield(self, command, *args, **kwargs): - fn_str = f"{command}({args}, {kwargs})" - self.engine_debug(f"{self.name}: executing run-and-yield {fn_str}") - if self._shutdown_status: - self.log.verbose("Engine has been shut down and is not accepting new tasks") - return - message = self.make_message(command, args=args, kwargs=kwargs) - if message is error_sentinel: - return - async with self.new_socket() as socket: - # TODO: synchronize server-side generator by limiting qsize - # socket.setsockopt(zmq.RCVHWM, 1) - # socket.setsockopt(zmq.SNDHWM, 1) - await socket.send(message) - while 1: - try: - binary = await self._infinite_retry( - socket.recv, _context=f"waiting for new iteration from {fn_str}" - ) - # self.log.debug(f"{self.name}.{command}({kwargs}) got binary: {binary}") - message = self.unpickle(binary) - self.engine_debug(f"{self.name}: {fn_str} got iteration: {message}") - # error handling - if self.check_error(message) or self.check_stop(message): - break - yield message - except (StopAsyncIteration, GeneratorExit) as e: - exc_name = e.__class__.__name__ - self.engine_debug(f"{self.name}.{command} got {exc_name}") - try: - await self.send_cancel_message(socket, fn_str) - except Exception: - self.engine_debug(f"{self.name}.{command} failed to send cancel message after {exc_name}") - self.log.trace(traceback.format_exc()) - break - - async def send_cancel_message(self, socket, context): - """ - Send a cancel message and wait for confirmation from the server - """ - # -1 == special "cancel" signal - message = pickle.dumps({"c": -1}) - await self._infinite_retry(socket.send, message) - while 1: - response = await self._infinite_retry( - socket.recv, _context=f"waiting for CANCEL_OK from {context}", _max_retries=4 - ) - response = pickle.loads(response) - if isinstance(response, dict): - response = response.get("m", "") - if response == "CANCEL_OK": - break - - async def send_shutdown_message(self): - async with self.new_socket() as socket: - # -99 == special shutdown message - message = pickle.dumps({"c": -99}) - with suppress(TimeoutError, asyncio.exceptions.TimeoutError): - await asyncio.wait_for(socket.send(message), 0.5) - with suppress(TimeoutError, asyncio.exceptions.TimeoutError): - while 1: - response = await asyncio.wait_for(socket.recv(), 0.5) - response = pickle.loads(response) - if isinstance(response, dict): - response = response.get("m", "") - if response == "SHUTDOWN_OK": - break - - def check_stop(self, message): - if isinstance(message, dict) and len(message) == 1 and "_s" in message: - return True - return False - - def make_message(self, command, args=None, kwargs=None): - try: - cmd_id = self.CMDS[command] - except KeyError: - raise KeyError(f'Command "{command}" not found. Available commands: {",".join(self.available_commands)}') - message = {"c": cmd_id} - if args: - message["a"] = args - if kwargs: - message["k"] = kwargs - return pickle.dumps(message) - - @property - def available_commands(self): - return [s for s in self.CMDS if isinstance(s, str)] - - def start_server(self): - process_name = multiprocessing.current_process().name - if SHARED_INTERPRETER_STATE.is_scan_process: - kwargs = dict(self.server_kwargs) - # if we're in tests, we use a single event loop to avoid weird race conditions - # this allows us to more easily mock http, etc. - if os.environ.get("BBOT_TESTING", "") == "True": - kwargs["_loop"] = get_event_loop() - kwargs["debug"] = self._engine_debug - self.process = CORE.create_process( - target=self.server_process, - args=( - self.SERVER_CLASS, - self.socket_path, - ), - kwargs=kwargs, - custom_name=f"BBOT {self.__class__.__name__}", - ) - self.process.start() - return self.process - else: - raise BBOTEngineError( - f"Tried to start server from process {process_name}. Did you forget \"if __name__ == '__main__'?\"" - ) - - @staticmethod - def server_process(server_class, socket_path, **kwargs): - try: - loop = kwargs.pop("_loop", None) - engine_server = server_class(socket_path, **kwargs) - if loop is not None: - future = asyncio.run_coroutine_threadsafe(engine_server.worker(), loop) - future.result() - else: - asyncio.run(engine_server.worker()) - except (asyncio.CancelledError, KeyboardInterrupt, CancelledError): - return - except Exception: - import traceback - - log = logging.getLogger("bbot.core.engine.server") - log.critical(f"Unhandled error in {server_class.__name__} server process: {traceback.format_exc()}") - - @asynccontextmanager - async def new_socket(self): - if self._server_process is None: - self._server_process = self.start_server() - while not self.socket_path.exists(): - self.engine_debug(f"{self.name}: waiting for server process to start...") - await asyncio.sleep(0.1) - socket = self.context.socket(zmq.DEALER) - socket.setsockopt(zmq.LINGER, 0) # Discard pending messages immediately disconnect() or close() - socket.setsockopt(zmq.SNDHWM, 0) # Unlimited send buffer - socket.setsockopt(zmq.RCVHWM, 0) # Unlimited receive buffer - socket.connect(f"ipc://{self.socket_path}") - self.sockets.add(socket) - try: - yield socket - finally: - self.sockets.remove(socket) - with suppress(Exception): - socket.close() - - async def shutdown(self): - if not self._shutdown_status: - self._shutdown_status = True - self.log.verbose(f"{self.name}: shutting down...") - # send shutdown signal - await self.send_shutdown_message() - # then terminate context - try: - self.context.destroy(linger=0) - except Exception: - print(traceback.format_exc(), file=sys.stderr) - try: - self.context.term() - except Exception: - print(traceback.format_exc(), file=sys.stderr) - # delete socket file on exit - self.socket_path.unlink(missing_ok=True) - - -class EngineServer(EngineBase): - """ - The server portion of BBOT's RPC Engine. - - Methods defined here must match the methods in your EngineClient. - - To use the functions, you must create mappings for them in the CMDS attribute, as shown below. - - Examples: - >>> from bbot.core.engine import EngineServer - >>> - >>> class MyServer(EngineServer): - >>> CMDS = { - >>> 0: "my_function", - >>> 1: "my_generator", - >>> } - >>> - >>> def my_function(self, arg1=None): - >>> await asyncio.sleep(1) - >>> return str(arg1) - >>> - >>> def my_generator(self): - >>> for i in range(10): - >>> await asyncio.sleep(1) - >>> yield i - """ - - CMDS = {} - - def __init__(self, socket_path, debug=False): - self.name = f"EngineServer {self.__class__.__name__}" - super().__init__(debug=debug) - self.engine_debug(f"{self.name}: finished setup 1 (_debug={self._engine_debug})") - self.socket_path = socket_path - self.client_id_var = contextvars.ContextVar("client_id", default=None) - # task <--> client id mapping - self.tasks = {} - # child tasks spawned by main tasks - self.child_tasks = {} - self.engine_debug(f"{self.name}: finished setup 2 (_debug={self._engine_debug})") - if self.socket_path is not None: - # create ZeroMQ context - self.context = zmq.asyncio.Context() - # ROUTER socket can handle multiple concurrent requests - self.socket = self.context.socket(zmq.ROUTER) - self.socket.setsockopt(zmq.LINGER, 0) # Discard pending messages immediately disconnect() or close() - self.socket.setsockopt(zmq.SNDHWM, 0) # Unlimited send buffer - self.socket.setsockopt(zmq.RCVHWM, 0) # Unlimited receive buffer - # create socket file - self.socket.bind(f"ipc://{self.socket_path}") - self.engine_debug(f"{self.name}: finished setup 3 (_debug={self._engine_debug})") - - @contextlib.contextmanager - def client_id_context(self, value): - token = self.client_id_var.set(value) - try: - yield - finally: - self.client_id_var.reset(token) - - async def run_and_return(self, client_id, command_fn, *args, **kwargs): - fn_str = f"{command_fn.__name__}({args}, {kwargs})" - self.engine_debug(fn_str) - with self.client_id_context(client_id): - try: - self.engine_debug(f"{self.name}: starting run-and-return {fn_str}") - try: - result = await command_fn(*args, **kwargs) - except BaseException as e: - if in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)): - log_fn = self.log.debug - else: - log_fn = self.log.error - error = f"{self.name}: error in {fn_str}: {e}" - trace = traceback.format_exc() - log_fn(error) - self.log.trace(trace) - result = {"_e": (error, trace)} - finally: - self.tasks.pop(client_id, None) - self.engine_debug(f"{self.name}: sending response to {fn_str}: {result}") - await self.send_socket_multipart(client_id, result) - except BaseException as e: - self.log.critical( - f"Unhandled exception in {self.name}.run_and_return({client_id}, {command_fn}, {args}, {kwargs}): {e}" - ) - self.log.critical(traceback.format_exc()) - finally: - self.engine_debug(f"{self.name} finished run-and-return {fn_str}") - - async def run_and_yield(self, client_id, command_fn, *args, **kwargs): - fn_str = f"{command_fn.__name__}({args}, {kwargs})" - with self.client_id_context(client_id): - try: - self.engine_debug(f"{self.name}: starting run-and-yield {fn_str}") - try: - async for _ in command_fn(*args, **kwargs): - self.engine_debug(f"{self.name}: sending iteration for {fn_str}: {_}") - await self.send_socket_multipart(client_id, _) - except BaseException as e: - if in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)): - log_fn = self.log.debug - else: - log_fn = self.log.error - error = f"{self.name}: error in {fn_str}: {e}" - trace = traceback.format_exc() - log_fn(error) - self.log.trace(trace) - result = {"_e": (error, trace)} - await self.send_socket_multipart(client_id, result) - finally: - self.engine_debug(f"{self.name}: reached end of run-and-yield iteration for {fn_str}") - # _s == special signal that means StopIteration - await self.send_socket_multipart(client_id, {"_s": None}) - self.tasks.pop(client_id, None) - except BaseException as e: - self.log.critical( - f"Unhandled exception in {self.name}.run_and_yield({client_id}, {command_fn}, {args}, {kwargs}): {e}" - ) - self.log.critical(traceback.format_exc()) - finally: - self.engine_debug(f"{self.name}: finished run-and-yield {fn_str}") - - async def send_socket_multipart(self, client_id, message): - try: - message = pickle.dumps(message) - await self._infinite_retry(self.socket.send_multipart, [client_id, message]) - except Exception as e: - self.log.verbose(f"{self.name}: error sending ZMQ message: {e}") - self.log.trace(traceback.format_exc()) - - def check_error(self, message): - if message is error_sentinel: - return True - - async def worker(self): - self.engine_debug(f"{self.name}: starting worker") - try: - while 1: - client_id, binary = await self.socket.recv_multipart() - message = self.unpickle(binary) - self.engine_debug(f"{self.name} got message: {message}") - if self.check_error(message): - continue - - cmd = message.get("c", None) - if not isinstance(cmd, int): - self.log.warning(f"{self.name}: no command sent in message: {message}") - continue - - # -1 == cancel task - if cmd == -1: - self.engine_debug(f"{self.name} got cancel signal") - await self.send_socket_multipart(client_id, {"m": "CANCEL_OK"}) - await self.cancel_task(client_id) - continue - - # -99 == shutdown task - if cmd == -99: - self.log.verbose(f"{self.name} got shutdown signal") - await self.send_socket_multipart(client_id, {"m": "SHUTDOWN_OK"}) - await self._shutdown() - return - - args = message.get("a", ()) - if not isinstance(args, tuple): - self.log.warning(f"{self.name}: received invalid args of type {type(args)}, should be tuple") - continue - kwargs = message.get("k", {}) - if not isinstance(kwargs, dict): - self.log.warning(f"{self.name}: received invalid kwargs of type {type(kwargs)}, should be dict") - continue - - command_name = self.CMDS[cmd] - command_fn = getattr(self, command_name, None) - - if command_fn is None: - self.log.warning(f'{self.name} has no function named "{command_fn}"') - continue - - if inspect.isasyncgenfunction(command_fn): - self.engine_debug(f"{self.name}: creating run-and-yield coroutine for {command_name}()") - coroutine = self.run_and_yield(client_id, command_fn, *args, **kwargs) - else: - self.engine_debug(f"{self.name}: creating run-and-return coroutine for {command_name}()") - coroutine = self.run_and_return(client_id, command_fn, *args, **kwargs) - - self.engine_debug(f"{self.name}: creating task for {command_name}() coroutine") - task = asyncio.create_task(coroutine) - self.tasks[client_id] = task, command_fn, args, kwargs - self.engine_debug(f"{self.name}: finished creating task for {command_name}() coroutine") - except BaseException as e: - await self._shutdown() - if not in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)): - self.log.error(f"{self.name}: error in EngineServer worker: {e}") - self.log.trace(traceback.format_exc()) - finally: - self.engine_debug(f"{self.name}: finished worker()") - - async def _shutdown(self): - if not self._shutdown_status: - self.log.verbose(f"{self.name}: shutting down...") - self._shutdown_status = True - await self.cancel_all_tasks() - context = getattr(self, "context", None) - if context is not None: - try: - context.destroy(linger=0) - except Exception: - self.log.trace(traceback.format_exc()) - try: - context.term() - except Exception: - self.log.trace(traceback.format_exc()) - self.log.verbose(f"{self.name}: finished shutting down") - - async def task_pool(self, fn, args_kwargs, threads=10, timeout=300, global_kwargs=None): - if global_kwargs is None: - global_kwargs = {} - - tasks = {} - args_kwargs = list(args_kwargs) - - def new_task(): - if args_kwargs: - kwargs = {} - tracker = None - args = args_kwargs.pop(0) - if isinstance(args, (list, tuple)): - # you can specify a custom tracker value if you want - # this helps with correlating results - with suppress(ValueError): - args, kwargs, tracker = args - # or you can just specify args/kwargs - with suppress(ValueError): - args, kwargs = args - - if not isinstance(kwargs, dict): - raise ValueError(f"kwargs must be dict (got: {kwargs})") - if not isinstance(args, (list, tuple)): - args = [args] - - task = self.new_child_task(fn(*args, **kwargs, **global_kwargs)) - tasks[task] = (args, kwargs, tracker) - - for _ in range(threads): # Start initial batch of tasks - new_task() - - while tasks: # While there are tasks pending - # Wait for the first task to complete - finished = await self.finished_tasks(tasks, timeout=timeout) - for task in finished: - result = task.result() - (args, kwargs, tracker) = tasks.pop(task) - yield (args, kwargs, tracker), result - new_task() - - def new_child_task(self, coro): - """ - Create a new asyncio task, making sure to track it based on the client id. - - This allows the task to be automatically cancelled if its parent is cancelled. - """ - client_id = self.client_id_var.get() - task = asyncio.create_task(coro) - - if client_id: - - def remove_task(t): - tasks = self.child_tasks.get(client_id, set()) - tasks.discard(t) - if not tasks: - self.child_tasks.pop(client_id, None) - - task.add_done_callback(remove_task) - - try: - self.child_tasks[client_id].add(task) - except KeyError: - self.child_tasks[client_id] = {task} - - return task - - async def finished_tasks(self, tasks, timeout=None): - """ - Given a list of asyncio tasks, return the ones that are finished with an optional timeout - """ - if tasks: - try: - done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED, timeout=timeout) - return done - except BaseException as e: - if isinstance(e, (TimeoutError, asyncio.exceptions.TimeoutError)): - self.log.warning(f"{self.name}: Timeout after {timeout:,} seconds in finished_tasks({tasks})") - for task in list(tasks): - task.cancel() - self._await_cancelled_task(task) - else: - if not in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)): - self.log.error(f"{self.name}: Unhandled exception in finished_tasks({tasks}): {e}") - self.log.trace(traceback.format_exc()) - raise - return set() - - async def cancel_task(self, client_id): - parent_task = self.tasks.pop(client_id, None) - if parent_task is None: - return - parent_task, _cmd, _args, _kwargs = parent_task - self.engine_debug(f"{self.name}: Cancelling client id {client_id} (task: {parent_task})") - parent_task.cancel() - child_tasks = self.child_tasks.pop(client_id, set()) - if child_tasks: - self.engine_debug(f"{self.name}: Cancelling {len(child_tasks):,} child tasks for client id {client_id}") - for child_task in child_tasks: - child_task.cancel() - - for task in [parent_task] + list(child_tasks): - await self._await_cancelled_task(task) - - async def _await_cancelled_task(self, task): - try: - await asyncio.wait_for(task, timeout=10) - except (TimeoutError, asyncio.exceptions.TimeoutError): - self.log.trace(f"{self.name}: Timeout cancelling task: {task}") - return - except (KeyboardInterrupt, asyncio.CancelledError): - return - except BaseException as e: - self.log.error(f"Unhandled error in {task.get_coro().__name__}(): {e}") - self.log.trace(traceback.format_exc()) - - async def cancel_all_tasks(self): - for client_id in list(self.tasks): - await self.cancel_task(client_id) - for client_id, tasks in self.child_tasks.items(): - for task in list(tasks): - await self._await_cancelled_task(task) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 00a03aeed7..215c9f61f9 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1,5 +1,6 @@ import io import re +import sys import uuid import json import base64 @@ -11,7 +12,9 @@ from pathlib import Path from typing import Optional +from zoneinfo import ZoneInfo from copy import copy, deepcopy +from types import MappingProxyType from contextlib import suppress from radixtarget import RadixTarget from pydantic import BaseModel, field_validator @@ -21,6 +24,7 @@ from bbot.errors import * from .helpers import EventSeed from bbot.core.helpers import ( + bytes_to_human, extract_words, is_domain, is_subdomain, @@ -32,6 +36,7 @@ domain_stem, make_netloc, make_ip_type, + cached_ip_address, recursive_decode, sha1, smart_decode, @@ -40,12 +45,24 @@ validators, get_file_extension, ) +from bbot.models.helpers import utc_datetime_validator from bbot.core.helpers.web.envelopes import BaseEnvelope log = logging.getLogger("bbot.core.event") +# Shared empty defaults for lazy-init slots. Returned from property accessors +# when the underlying slot is None — saves ~560 bytes per event compared to +# allocating real empty containers (set/dict) at __init__ time. Mutating +# helpers (add_tag, etc.) replace the slot with a real container before +# mutating, so callers never see the singletons in a mutable position. +# `_resolved_hosts` has no in-place mutation API — see the resolved_hosts +# property setter for the assign-once-then-frozen contract. +_EMPTY_FROZENSET: "frozenset[str]" = frozenset() +_EMPTY_DICT = MappingProxyType({}) + + class BaseEvent: """ Represents a piece of data discovered during a BBOT scan. @@ -96,9 +113,10 @@ class BaseEvent: "timestamp": 1688526222.723366, "resolved_hosts": ["185.199.108.153"], "parent": "OPEN_TCP_PORT:cf7e6a937b161217eaed99f0c566eae045d094c7", - "tags": ["in-scope", "distance-0", "dir", "ip-185-199-108-153", "status-301", "http-title-301-moved-permanently"], - "module": "httpx", - "module_sequence": "httpx" + "tags": ["in-scope", "distance-0", "dir", "status-301"], + "http_title": "301 Moved Permanently", + "module": "http", + "module_sequence": "http" } ``` """ @@ -106,9 +124,10 @@ class BaseEvent: # Always emit this event type even if it's not in scope _always_emit = False # Always emit events with these tags even if they're not in scope - _always_emit_tags = ["affiliate", "target"] + + _always_emit_tags = ["affiliate", "seed"] # Bypass scope checking and dns resolution, distribute immediately to modules - # This is useful for "end-of-line" events like FINDING and VULNERABILITY + # This is useful for "end-of-line" events like FINDING _quick_emit = False # Data validation, if data is a dictionary _data_validator = None @@ -119,6 +138,10 @@ class BaseEvent: # Don't allow duplicates to occur within a parent chain # In other words, don't emit the event if the same one already exists in its discovery context _suppress_chain_dupes = False + # Shared compiled regex for discovery context formatting (class-level to avoid per-instance overhead) + _discovery_context_regex = re.compile(r"\{(?:event|module)[^}]*\}") + # Stats class for the status line — override in subclasses for custom formatting + _stats_class = None # using __slots__ dramatically reduces memory usage in large scans __slots__ = [ @@ -147,23 +170,22 @@ class BaseEvent: "_graph_important", "_resolved_hosts", "_discovery_context", - "_discovery_context_regex", "_stats_recorded", "_internal", - "_confidence", "_dummy", "_module", - # DNS-related attributes - "dns_children", - "raw_dns_records", + # DNS-related attributes — backing slots; public access via the + # ``dns_children`` / ``raw_dns_records`` properties so a None + # underlying slot transparently reads as an empty dict (lazy-init + # saves ~128 bytes per event). + "_dns_children", + "_raw_dns_records", "dns_resolve_distance", - # Web-related attributes - "web_spider_distance", - "parsed_url", - "url_extension", - "num_redirects", - # File-related attributes - "_data_path", + # Host metadata (cloud providers, ASN, whois, etc.) + "_host_metadata", + "_cloudcheck_done", + # Memory management + "_module_consumers", # Public attributes "module", "scan", @@ -179,7 +201,6 @@ def __init__( module=None, scan=None, tags=None, - confidence=100, timestamp=None, _dummy=False, _internal=None, @@ -197,7 +218,6 @@ def __init__( module (str, optional): Module that discovered the event. Defaults to None. scan (Scan, optional): BBOT Scan object. Required unless _dummy is True. Defaults to None. tags (list of str, optional): Descriptive tags for the event. Defaults to None. - confidence (int, optional): Confidence level for the event, on a scale of 1-100. Defaults to 100. timestamp (datetime, optional): Time of event discovery. Defaults to current UTC time. _dummy (bool, optional): If True, disables certain data validations. Defaults to False. _internal (Any, optional): If specified, makes the event internal. Defaults to None. @@ -210,7 +230,10 @@ def __init__( self._hash = None self._data = None self.__host = None - self._tags = set() + # Lazy-init: replaced with a real set/dict on first mutation. + # Reading via the property returns a shared empty frozenset/dict + # so callers never see None. + self._tags = None self._port = None self._omit = False self.__words = None @@ -222,12 +245,11 @@ def __init__( self._scope_distance = None self._module_priority = None self._graph_important = False - self._resolved_hosts = set() - self.dns_children = {} - self.raw_dns_records = {} + self._resolved_hosts = None + self._dns_children = None + self._raw_dns_records = None self._discovery_context = "" - self._discovery_context_regex = re.compile(r"\{(?:event|module)[^}]*\}") - self.web_spider_distance = 0 + self._module_consumers = 0 # for creating one-off events without enforcing parent requirement self._dummy = _dummy @@ -245,7 +267,6 @@ def __init__( except AttributeError: self.timestamp = datetime.datetime.utcnow() - self.confidence = int(confidence) self._internal = False # self.scan holds the instantiated scan object (for helpers, etc.) @@ -257,7 +278,8 @@ def __init__( self.data = self._sanitize_data(data) except Exception as e: log.trace(traceback.format_exc()) - raise ValidationError(f'Error sanitizing event data "{data}" for type "{self.type}": {e}') + data_preview = str(data)[:200] + "..." if len(str(data)) > 200 else str(data) + raise ValidationError(f'Error sanitizing event data "{data_preview}" for type "{self.type}": {e}') if not self.data: raise ValidationError(f'Invalid event data "{data}" for type "{self.type}"') @@ -286,33 +308,50 @@ def data(self): return self._data @property - def confidence(self): - return self._confidence - - @confidence.setter - def confidence(self, confidence): - self._confidence = min(100, max(1, int(confidence))) + def resolved_hosts(self): + if is_ip(self.host): + return frozenset({self.host}) + return self._resolved_hosts if self._resolved_hosts is not None else _EMPTY_FROZENSET + + @resolved_hosts.setter + def resolved_hosts(self, value): + """Assign ``_resolved_hosts`` as a frozenset (or None for unset). + + Resolved hosts are naturally immutable: there is no in-place mutation + API. Callers (typically ``dnsresolve``) build the set locally and then + assign it once via this setter, after which the slot holds a + ``frozenset`` that downstream consumers can read but cannot modify. + """ + if value is None: + self._resolved_hosts = None + elif isinstance(value, frozenset): + self._resolved_hosts = value + else: + self._resolved_hosts = frozenset(value) @property - def cumulative_confidence(self): - """ - Considers the confidence of parent events. This is useful for filtering out speculative/unreliable events. - - E.g. an event with a confidence of 50 whose parent is also 50 would have a cumulative confidence of 25. - - A confidence of 100 will reset the cumulative confidence to 100. - """ - if self._confidence == 100 or self.parent is None or self.parent is self: - return self._confidence - return int(self._confidence * self.parent.cumulative_confidence / 100) + def dns_children(self): + return self._dns_children if self._dns_children is not None else _EMPTY_DICT @property - def resolved_hosts(self): - if is_ip(self.host): - return { - self.host, - } - return self._resolved_hosts + def raw_dns_records(self): + return self._raw_dns_records if self._raw_dns_records is not None else _EMPTY_DICT + + def add_dns_child(self, rdtype, host): + """Record a DNS child host under ``rdtype``, lazy-allocating dict + child set.""" + if self._dns_children is None: + self._dns_children = {} + existing = self._dns_children.get(rdtype) + if existing is None: + self._dns_children[rdtype] = {host} + else: + existing.add(host) + + def set_raw_dns_record(self, rdtype, answers): + """Store the raw DNS answer list for ``rdtype``, lazy-allocating the dict.""" + if self._raw_dns_records is None: + self._raw_dns_records = {} + self._raw_dns_records[rdtype] = answers @data.setter def data(self, data): @@ -323,6 +362,18 @@ def data(self, data): self._port = None self._data = data + @property + def host_metadata(self): + try: + return self._host_metadata + except AttributeError: + self._host_metadata = {} + return self._host_metadata + + @host_metadata.setter + def host_metadata(self, value): + self._host_metadata = value + @property def internal(self): return self._internal @@ -383,6 +434,13 @@ def host_original(self): return self.host return self._host_original + @property + def url(self): + parsed_url = getattr(self, "parsed_url", None) + if parsed_url is not None: + return parsed_url.geturl() + return "" + @property def host_filterable(self): """ @@ -401,6 +459,8 @@ def host_filterable(self): @property def port(self): self.host + if self._port: + return self._port if getattr(self, "parsed_url", None): if self.parsed_url.port is not None: return self.parsed_url.port @@ -408,7 +468,6 @@ def port(self): return 443 elif self.parsed_url.scheme == "http": return 80 - return self._port @property def netloc(self): @@ -474,7 +533,7 @@ def _words(self): @property def tags(self): - return self._tags + return self._tags if self._tags is not None else _EMPTY_FROZENSET @tags.setter def tags(self, tags): @@ -485,15 +544,19 @@ def tags(self, tags): self.add_tag(tag) def add_tag(self, tag): - self._tags.add(tagify(tag)) + if self._tags is None: + self._tags = set() + self._tags.add(sys.intern(tagify(tag))) def add_tags(self, tags): for tag in set(tags): self.add_tag(tag) def remove_tag(self, tag): + if not self._tags: + return with suppress(KeyError): - self._tags.remove(tagify(tag)) + self._tags.remove(sys.intern(tagify(tag))) @property def always_emit(self): @@ -618,12 +681,15 @@ def parent(self, parent): new_scope_distance += 1 self.scope_distance = new_scope_distance # inherit certain tags + # inherit seed tag from DNS_NAME_UNRESOLVED -> DNS_NAME only + if "seed" in parent.tags and parent.type == "DNS_NAME_UNRESOLVED" and self.type == "DNS_NAME": + self.add_tag("seed") if hosts_are_same: # inherit web spider distance from parent self.web_spider_distance = getattr(parent, "web_spider_distance", 0) event_has_url = getattr(self, "parsed_url", None) is not None for t in parent.tags: - if t in ("affiliate",): + if t in ("affiliate", "from-wayback", "from-lightfuzz"): self.add_tag(t) elif t.startswith("mutation-"): self.add_tag(t) @@ -652,6 +718,26 @@ def parent_uuid(self): return parent_uuid return self._parent_uuid + @property + def archive_url(self): + """Traverse the parent chain to find the nearest archive_url. + + The 'from-wayback' tag signals that this event descends from archived content. + The actual archive URL is stored only in the data dict of the originating + wayback HTTP_RESPONSE; this property walks upward to find it. + """ + if "from-wayback" not in self.tags: + return None + event = self + while event is not None: + if isinstance(event.data, dict) and "archive_url" in event.data: + return event.data["archive_url"] + parent = getattr(event, "parent", None) + if parent is None or parent is event: + break + event = parent + return None + @property def validators(self): """ @@ -692,6 +778,20 @@ def get_parents(self, omit=False, include_self=False): e = parent return parents + def _minimize(self): + """ + Called when a module is done processing this event. + + Decrements the consumer count. When no modules are left waiting to + process this event, heavy payload data is stripped to free memory. + """ + self._module_consumers = max(0, self._module_consumers - 1) + if self._module_consumers <= 0: + # release container slots; lazy-init pattern means None == empty + self._dns_children = None + self._raw_dns_records = None + self._resolved_hosts = None + def clone(self): # Create a shallow copy of the event first cloned_event = copy(self) @@ -813,11 +913,11 @@ def __contains__(self, other): return True # hostnames and IPs radixtarget = RadixTarget() - radixtarget.insert(self.host) - return bool(radixtarget.search(other_event.host)) + radixtarget.insert(str(self.host)) + return bool(radixtarget.search(str(other_event.host))) return False - def json(self, mode="json", siem_friendly=False): + def json(self, mode="json"): """ Serializes the event object to a JSON-compatible dictionary. @@ -826,7 +926,6 @@ def json(self, mode="json", siem_friendly=False): Parameters: mode (str): Specifies the data serialization mode. Default is "json". Other options include "graph", "human", and "id". - siem_friendly (bool): Whether to format the JSON in a way that's friendly to SIEM ingestion by Elastic, Splunk, etc. This ensures the value of "data" is always the same type (a dictionary). Returns: dict: JSON-serializable dictionary representation of the event object. @@ -843,10 +942,12 @@ def json(self, mode="json", siem_friendly=False): data = data_attr else: data = smart_decode(self.data) - if siem_friendly: - j["data"] = {self.type: data} - else: + if isinstance(data, str): j["data"] = data + elif isinstance(data, dict): + j["data_json"] = data + else: + raise ValueError(f"Invalid data type: {type(data)}") # host, dns children if self.host: j["host"] = str(self.host) @@ -864,7 +965,7 @@ def json(self, mode="json", siem_friendly=False): if self.scan: j["scan"] = self.scan.id # timestamp - j["timestamp"] = self.timestamp.isoformat() + j["timestamp"] = utc_datetime_validator(self.timestamp).timestamp() # parent event parent_id = self.parent_id if parent_id: @@ -873,8 +974,7 @@ def json(self, mode="json", siem_friendly=False): if parent_uuid: j["parent_uuid"] = parent_uuid # tags - if self.tags: - j.update({"tags": list(self.tags)}) + j.update({"tags": sorted(self.tags)}) # parent module if self.module: j.update({"module": str(self.module)}) @@ -886,6 +986,10 @@ def json(self, mode="json", siem_friendly=False): j["discovery_path"] = self.discovery_path j["parent_chain"] = self.parent_chain + # host metadata (cloud providers, ASN, etc.) + host_metadata = getattr(self, "host_metadata", None) + if host_metadata: + j["host_metadata"] = host_metadata # parameter envelopes parameter_envelopes = getattr(self, "envelopes", None) if parameter_envelopes is not None: @@ -1055,10 +1159,20 @@ def sanitize_data(self, data): class DictEvent(BaseEvent): + __slots__ = ["url_extension"] + def sanitize_data(self, data): url = data.get("url", "") if url: self.parsed_url = self.validators.validate_url_parsed(url) + # extract url_extension from any dict event with a URL + url_path = self.parsed_url.path + if url_path: + parsed_path_lower = str(url_path).lower() + extension = get_file_extension(parsed_path_lower) + if extension: + self.url_extension = extension + self.add_tag(f"extension-{extension}") return data def _data_load(self, data): @@ -1073,13 +1187,13 @@ def _host(self): return make_ip_type(self.data["host"]) else: parsed = getattr(self, "parsed_url", None) - if parsed is not None: + if parsed is not None and parsed.hostname: return make_ip_type(parsed.hostname) class ClosestHostEvent(DictHostEvent): # if a host/path/url isn't specified, this event type grabs it from the closest parent - # inherited by FINDING and VULNERABILITY + # inherited by FINDING def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if not self.host: @@ -1094,9 +1208,10 @@ def __init__(self, *args, **kwargs): parent_path = parent.data.get("path", None) if parent_path is not None: self.data["path"] = parent_path - # inherit closest host + # inherit closest host+port if parent.host: self.data["host"] = str(parent.host) + self._port = parent.port # we do this to refresh the hash self.data = self.data break @@ -1105,8 +1220,13 @@ def __init__(self, *args, **kwargs): raise ValueError(f"No host was found in event parents: {self.get_parents()}. Host must be specified!") -class DictPathEvent(DictEvent): +class DictPathEvent(DictHostEvent): + __slots__ = [ + "_data_path", + ] + def sanitize_data(self, data): + data = super().sanitize_data(data) new_data = dict(data) new_data["path"] = str(new_data["path"]) file_blobs = getattr(self.scan, "_file_blobs", False) @@ -1145,6 +1265,24 @@ class ASN(DictEvent): _always_emit = True _quick_emit = True + def sanitize_data(self, data): + # accept bare int (from make_event(12345, "ASN")) or dict (from JSON round-trip) + if isinstance(data, int): + data = {"asn": data} + if not isinstance(data, dict) or "asn" not in data: + raise ValidationError(f"Invalid ASN data (expected dict with 'asn' key): {data}") + data["asn"] = int(data["asn"]) + return data + + def _data_id(self): + return str(self.data["asn"]) + + def _pretty_string(self): + return str(self.data["asn"]) + + def _data_human(self): + return f"AS{self.data['asn']}" + class CODE_REPOSITORY(DictHostEvent): _always_emit = True @@ -1156,11 +1294,14 @@ class _data_validator(BaseModel): def _pretty_string(self): return self.data["url"] + def _data_human(self): + return self.data["url"] + class IP_ADDRESS(BaseEvent): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - ip = ipaddress.ip_address(self.data) + ip = cached_ip_address(self.data) self.add_tag(f"ipv{ip.version}") if ip.is_private: self.add_tag("private-ip") @@ -1170,7 +1311,7 @@ def sanitize_data(self, data): return validators.validate_host(data) def _host(self): - return ipaddress.ip_address(self.data) + return cached_ip_address(self.data) class DnsEvent(BaseEvent): @@ -1234,6 +1375,9 @@ def _words(self): class OPEN_TCP_PORT(BaseEvent): + # we generally don't care about open ports on affiliates + _always_emit_tags = ["seed"] + def sanitize_data(self, data): return validators.validate_open_port(data) @@ -1251,19 +1395,31 @@ class OPEN_UDP_PORT(OPEN_TCP_PORT): pass -class URL_UNVERIFIED(BaseEvent): +class URL_UNVERIFIED(DictHostEvent): _status_code_regex = re.compile(r"^status-(\d{1,3})$") + __slots__ = [ + "web_spider_distance", + "num_redirects", + ] + def __init__(self, *args, **kwargs): + self.web_spider_distance = 0 super().__init__(*args, **kwargs) self.num_redirects = getattr(self.parent, "num_redirects", 0) + def _data_load(self, data): + # accept a bare URL string and wrap it into a dict + if isinstance(data, str): + return {"url": data} + return data + def _data_id(self): - data = super()._data_id() + url = self.url # remove the querystring for URL/URL_UNVERIFIED events, because we will conditionally add it back in (based on settings) if self.__class__.__name__.startswith("URL") and self.scan is not None: - prefix = data.split("?")[0] + prefix = url.split("?")[0] # consider spider-danger tag when deduping if "spider-danger" in self.tags: @@ -1279,11 +1435,13 @@ def _data_id(self): cleaned_query = "&".join( f"{key}={','.join(sorted(values))}" for key, values in sorted(query_dict.items()) ) - data = f"{prefix}:{self.parsed_url.scheme}:{self.parsed_url.netloc}:{self.parsed_url.path}:{cleaned_query}" - return data + url = f"{prefix}:{self.parsed_url.scheme}:{self.parsed_url.netloc}:{self.parsed_url.path}:{cleaned_query}" + return url def sanitize_data(self, data): - self.parsed_url = self.validators.validate_url_parsed(data) + url = data.get("url", "") + self.parsed_url = self.validators.validate_url_parsed(url) + data["url"] = self.parsed_url.geturl() # special handling of URL extensions if self.parsed_url is not None: @@ -1301,9 +1459,28 @@ def sanitize_data(self, data): else: self.add_tag("endpoint") - data = self.parsed_url.geturl() return data + @property + def pretty_string(self): + return self.url + + def _data_human(self): + parts = [] + status = self.http_status + if status: + parts.append(f"[{status}]") + parts.append(self.url) + if status and str(status).startswith("3"): + location = self.data.get("redirect_location", "") + if location: + parts.append(f"-> {location}") + else: + title = self.http_title + if title: + parts.append(f"- [{title}]") + return " ".join(parts) + def add_tag(self, tag): self_url = getattr(self, "parsed_url", "") self_host = getattr(self, "host", "") @@ -1351,12 +1528,31 @@ def _host(self): @property def http_status(self): + status_code = self.data.get("status_code", 0) + if status_code: + return int(status_code) for t in self.tags: match = self._status_code_regex.match(t) if match: return int(match.groups()[0]) return 0 + @property + def http_title(self): + return self.data.get("http_title", "") + + @http_title.setter + def http_title(self, value): + self.data["http_title"] = value + + @property + def redirect_location(self): + return self.data.get("redirect_location", "") + + @redirect_location.setter + def redirect_location(self, value): + self.data["redirect_location"] = value + class URL(URL_UNVERIFIED): def __init__(self, *args, **kwargs): @@ -1367,17 +1563,8 @@ def __init__(self, *args, **kwargs): 'Must specify HTTP status tag for URL event, e.g. "status-200". Use URL_UNVERIFIED if the URL is unvisited.' ) - @property - def resolved_hosts(self): - # TODO: remove this when we rip out httpx - return {".".join(i.split("-")[1:]) for i in self.tags if i.startswith("ip-")} - @property - def pretty_string(self): - return self.data - - -class STORAGE_BUCKET(DictEvent, URL_UNVERIFIED): +class STORAGE_BUCKET(URL_UNVERIFIED): _always_emit = True _suppress_chain_dupes = True @@ -1391,6 +1578,9 @@ def sanitize_data(self, data): data["name"] = data["name"].lower() return data + def _data_human(self): + return f"{self.data['name']} ({self.data['url']})" + def _words(self): return self.data["name"] @@ -1400,6 +1590,18 @@ class URL_HINT(URL_UNVERIFIED): class WEB_PARAMETER(DictHostEvent): + __slots__ = [ + "envelopes", + ] + + def _minimize(self): + super()._minimize() + if self._module_consumers <= 0: + self._data.pop("original_value", None) + self._data.pop("additional_params", None) + self._data.pop("assigned_cookies", None) + self._data.pop("same_param_values", None) + @property def children(self): # if we have any subparams, raise a new WEB_PARAMETER for each one @@ -1421,6 +1623,7 @@ def children(self): return children def sanitize_data(self, data): + data = super().sanitize_data(data) original_value = data.get("original_value", None) if original_value is not None: try: @@ -1454,6 +1657,29 @@ def _outgoing_dedup_hash(self, event): def _url(self): return self.data["url"] + def _data_human(self): + param_type = self.data.get("type", "") + name = self.data.get("name", "") + original_value = self.data.get("original_value", "") + url = self.data.get("url", "") + description = self.data.get("description", "") + additional_params = self.data.get("additional_params", {}) + parts = [] + if param_type: + parts.append(f"[{param_type}]") + if original_value: + parts.append(f"{name}={original_value}") + else: + parts.append(name) + if description: + parts.append(f"- {description}") + if additional_params: + param_names = ", ".join(sorted(additional_params.keys())) + parts.append(f"Additional Params: [{param_names}]") + if url: + parts.append(f"({url})") + return " ".join(parts) + def __str__(self): max_event_len = 200 d = str(self.data) @@ -1473,7 +1699,7 @@ def _words(self): return extract_words(self.host_stem) -class HTTP_RESPONSE(URL_UNVERIFIED, DictEvent): +class HTTP_RESPONSE(URL_UNVERIFIED): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # count number of consecutive redirects @@ -1481,6 +1707,45 @@ def __init__(self, *args, **kwargs): if str(self.http_status).startswith("3"): self.num_redirects += 1 + # Spill body to disk if a per-scan store is available. The body + # is removed from `_data` so JSON / human renderers don't see it; + # readers use the `.body` property which lazy-loads from the store. + # When spill is disabled (no store on the scan), body stays in + # `_data` and `.body` falls back to reading it from there. + store = getattr(getattr(self, "scan", None), "body_spill_store", None) + if store is not None and isinstance(self._data, dict) and "body" in self._data: + body = self._data.pop("body") + if isinstance(body, str): + body_bytes = body.encode("utf-8", errors="replace") + elif isinstance(body, (bytes, bytearray, memoryview)): + body_bytes = bytes(body) + else: + body_bytes = str(body).encode("utf-8", errors="replace") + if body_bytes: + store.write(str(self._uuid), body_bytes) + + @property + def body(self): + """ + The HTTP response body as a string. + + With spill enabled, the body lives in the per-scan ``BodySpillStore`` + (LRU cache + disk file). Cache hits are instant; misses re-read + from disk. After ``_minimize()`` fires the body is gone and this + property returns ``""``. + + With spill disabled, falls back to ``self._data["body"]``. + """ + store = getattr(getattr(self, "scan", None), "body_spill_store", None) + if store is None: + if isinstance(self._data, dict): + return self._data.get("body", "") or "" + return "" + body_bytes = store.read(str(self._uuid)) + if not body_bytes: + return "" + return body_bytes.decode("utf-8", errors="replace") + def _data_id(self): return self.data["method"] + "|" + self.data["url"] @@ -1518,14 +1783,47 @@ def _words(self): def _pretty_string(self): return f"{self.data['hash']['header_mmh3']}:{self.data['hash']['body_mmh3']}" + def _data_human(self): + parts = [] + status = self.http_status + if status: + parts.append(f"[{status}]") + method = self.data.get("method", "") + if method: + parts.append(method) + parts.append(self.data.get("url", "")) + if status and str(status).startswith("3"): + location = self.redirect_location + if location: + parts.append(f"-> {location}") + else: + title = self.http_title + if title: + parts.append(f"- [{title}]") + content_length = self.data.get("content_length", 0) + if content_length: + parts.append(f"({bytes_to_human(content_length)})") + return " ".join(parts) + + def _minimize(self): + super()._minimize() + if self._module_consumers <= 0: + store = getattr(getattr(self, "scan", None), "body_spill_store", None) + if store is not None: + store.evict_and_delete(str(self._uuid)) + # `body` may still be in _data if spill was disabled at creation + self._data.pop("body", None) + self._data.pop("raw_header", None) + self._data.pop("header", None) + self._data.pop("cert_info", None) + @property def raw_response(self): """ Formats the status code, headers, and body into a single string formatted as an HTTP/1.1 response. """ raw_header = self.data.get("raw_header", "") - body = self.data.get("body", "") - return f"{raw_header}{body}" + return f"{raw_header}{self.body}" @property def http_status(self): @@ -1556,49 +1854,126 @@ def redirect_location(self): return location -class VULNERABILITY(ClosestHostEvent): +class FINDING(ClosestHostEvent): _always_emit = True _quick_emit = True - severity_colors = { - "CRITICAL": "🟪", - "HIGH": "🟥", - "MEDIUM": "🟧", + + class _stats_class: + _severity_order = ["CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO"] + + def __init__(self): + self.count = 0 + self.severities = {} + + def increment(self, event): + self.count += 1 + sev = event.data.get("severity", "UNKNOWN") + try: + self.severities[sev] += 1 + except KeyError: + self.severities[sev] = 1 + + def format(self, event_type): + if not self.severities: + return f"{event_type}: {self.count}" + parts = [] + for sev in self._severity_order: + n = self.severities.get(sev, 0) + if n: + parts.append(f"{n} {sev}") + for sev, n in sorted(self.severities.items()): + if sev not in self._severity_order and n: + parts.append(f"{n} {sev}") + return f"{event_type}: {self.count} ({', '.join(parts)})" + + # Standard finding color palette. Single source of truth for every output module. + # Severity selects the hue (blue, yellow, orange, red, purple); confidence dims it. + severity_colors_rgb = { + "INFO": (113, 161, 255), + "LOW": (255, 215, 0), + "MEDIUM": (255, 135, 0), + "HIGH": (255, 0, 0), + "CRITICAL": (207, 0, 255), + } + confidence_brightness = { + "CONFIRMED": 1.00, + "HIGH": 0.88, + "MEDIUM": 0.77, + "LOW": 0.66, + "UNKNOWN": 0.55, + } + + # emoji rendering of the same palette, for chat-based output (Slack, Discord, Teams) + severity_colors_emoji = { + "INFO": "🟦", "LOW": "🟨", - "UNKNOWN": "⬜", + "MEDIUM": "🟧", + "HIGH": "🟥", + "CRITICAL": "🟪", + } + confidence_colors_emoji = { + "CONFIRMED": "🟣", + "HIGH": "🔴", + "MEDIUM": "🟠", + "LOW": "🟡", + "UNKNOWN": "⚪", + } + + # Adaptive Card style names for Microsoft Teams output + severity_card_colors = { + "CRITICAL": "Attention", + "HIGH": "Attention", + "MEDIUM": "Warning", + "LOW": "Good", + "INFO": "Accent", } def sanitize_data(self, data): - self.add_tag(data["severity"].lower()) + data = super().sanitize_data(data) + self.add_tag(f"severity-{data['severity'].lower()}") + self.add_tag(f"confidence-{data['confidence'].lower()}") return data class _data_validator(BaseModel): host: Optional[str] = None severity: str + name: str description: str + confidence: str url: Optional[str] = None + full_url: Optional[str] = None path: Optional[str] = None + cves: Optional[list[str]] = None + archive_url: Optional[str] = None _validate_url = field_validator("url")(validators.validate_url) _validate_host = field_validator("host")(validators.validate_host) _validate_severity = field_validator("severity")(validators.validate_severity) + _validate_confidence = field_validator("confidence")(validators.validate_confidence) def _pretty_string(self): - return f"[{self.data['severity']}] {self.data['description']}" - + severity = self.data["severity"] + confidence = self.data["confidence"] + description = self.data["description"] -class FINDING(ClosestHostEvent): - _always_emit = True - _quick_emit = True - - class _data_validator(BaseModel): - host: Optional[str] = None - description: str - url: Optional[str] = None - path: Optional[str] = None - _validate_url = field_validator("url")(validators.validate_url) - _validate_host = field_validator("host")(validators.validate_host) + # Add bold formatting for CONFIRMED confidence + if confidence == "CONFIRMED": + confidence_str = f"[\033[1m{confidence}\033[0m]" + else: + confidence_str = f"[{confidence}]" + return f"Severity: [{severity}] Confidence: {confidence_str} {description}" - def _pretty_string(self): - return self.data["description"] + def _data_human(self): + parts = [] + parts.append(f"Severity: [{self.data['severity']}]") + parts.append(f"Confidence: [{self.data['confidence']}]") + parts.append(self.data["description"]) + url = self.data.get("url", "") + if url and url not in self.data["description"]: + parts.append(f"({url})") + cves = self.data.get("cves", []) + if cves: + parts.append(f"[{', '.join(cves)}]") + return " ".join(parts) class TECHNOLOGY(DictHostEvent): @@ -1609,6 +1984,11 @@ class _data_validator(BaseModel): _validate_url = field_validator("url")(validators.validate_url) _validate_host = field_validator("host")(validators.validate_host) + def _sanitize_data(self, data): + data = super()._sanitize_data(data) + data["technology"] = data["technology"].lower() + return data + def _data_id(self): # dedupe by host+port+tech tech = self.data.get("technology", "") @@ -1617,17 +1997,41 @@ def _data_id(self): def _pretty_string(self): return self.data["technology"] + def _data_human(self): + tech = self.data["technology"] + url = self.data.get("url", "") + if url: + return f"{tech} ({url})" + return tech + -class VHOST(DictHostEvent): +class VIRTUAL_HOST(DictHostEvent): class _data_validator(BaseModel): host: str - vhost: str + virtual_host: str url: Optional[str] = None + description: Optional[str] = None + ip: Optional[str] = None _validate_url = field_validator("url")(validators.validate_url) _validate_host = field_validator("host")(validators.validate_host) + def _data_id(self): + virtual_host = self.data.get("virtual_host", "") + return f"{self.host}:{virtual_host}" + def _pretty_string(self): - return self.data["vhost"] + virtual_host = self.data.get("virtual_host", "") + url = self.data.get("url", "") + description = self.data.get("description", "") + parts = [virtual_host] + if url: + parts.append(f"({url})") + if description: + parts.append(description) + return " ".join(parts) + + def _data_human(self): + return self._pretty_string() class PROTOCOL(DictHostEvent): @@ -1651,11 +2055,35 @@ def port(self): def _pretty_string(self): return self.data["protocol"] + def _data_human(self): + protocol = self.data["protocol"] + port = self.data.get("port") + banner = self.data.get("banner", "") + if port: + result = f"{protocol}/{port}" + else: + result = protocol + if banner: + result += f" - {banner}" + return result + class GEOLOCATION(BaseEvent): _always_emit = True _quick_emit = True + def _data_human(self): + country = self.data.get("country_name", "") + region = self.data.get("region_name", "") + city = self.data.get("city_name", "") or self.data.get("city", "") + lat = self.data.get("latitude", "") + lon = self.data.get("longitude", "") + location_parts = [p for p in (city, region, country) if p] + result = ", ".join(location_parts) + if lat and lon: + result += f" ({lat}, {lon})" + return result if result else super()._data_human() + class PASSWORD(BaseEvent): _always_emit = True @@ -1677,16 +2105,50 @@ class SOCIAL(DictHostEvent): _quick_emit = True _scope_distance_increment_same_host = True + def _data_human(self): + platform = self.data.get("platform", "") + profile_name = self.data.get("profile_name", "") + url = self.data.get("url", "") + parts = [] + if platform: + parts.append(f"{platform}:") + parts.append(profile_name) + if url: + parts.append(f"({url})") + return " ".join(parts) + -class WEBSCREENSHOT(DictPathEvent, DictHostEvent): +class WEBSCREENSHOT(DictPathEvent): _always_emit = True _quick_emit = True + def _data_human(self): + return f"{self.data.get('url', '')} Saved to: {self.data.get('path', '')}" + class AZURE_TENANT(DictEvent): _always_emit = True _quick_emit = True + def _data_human(self): + max_domains = 20 + tenant_names = self.data.get("tenant-names", []) + tenant_id = self.data.get("tenant-id", "") + domains = self.data.get("domains", []) + parts = [] + if tenant_names: + parts.append(", ".join(tenant_names)) + if tenant_id: + parts.append(f"({tenant_id})") + if domains: + if len(domains) <= max_domains: + parts.append(f"- {', '.join(domains)}") + else: + shown = ", ".join(domains[:max_domains]) + hidden = len(domains) - max_domains + parts.append(f"- {shown} (hiding {hidden} additional domains)") + return " ".join(parts) if parts else super()._data_human() + class WAF(DictHostEvent): _always_emit = True @@ -1703,8 +2165,25 @@ class _data_validator(BaseModel): def _pretty_string(self): return self.data["waf"] + def _data_human(self): + waf = self.data["waf"] + info = self.data.get("info", "") + url = self.data.get("url", "") + if info: + waf += f" - {info}" + if url: + waf += f" ({url})" + return waf + class FILESYSTEM(DictPathEvent): + def _data_human(self): + path = self.data.get("path", "") + description = self.data.get("magic_description", "") + if description: + return f"{path} ({description})" + return path + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if self._data_path.is_file(): @@ -1731,13 +2210,20 @@ def __init__(self, *args, **kwargs): class RAW_DNS_RECORD(DictHostEvent, DnsEvent): # don't emit raw DNS records for affiliates - _always_emit_tags = ["target"] + _always_emit_tags = ["seed"] + + def _data_human(self): + rdtype = self.data.get("type", "") + host = self.data.get("host", "") + answer = self.data.get("answer", "") + return f"{rdtype} {host} -> {answer}" class MOBILE_APP(DictEvent): _always_emit = True def _sanitize_data(self, data): + data = super()._sanitize_data(data) if isinstance(data, str): data = {"url": data} if "url" not in data: @@ -1761,6 +2247,13 @@ def _sanitize_data(self, data): def _pretty_string(self): return self.data["url"] + def _data_human(self): + app_id = self.data.get("id", "") + url = self.data.get("url", "") + if app_id and url: + return f"{app_id} ({url})" + return url or app_id + def update_event( event, @@ -1819,7 +2312,6 @@ def make_event( module=None, scan=None, tags=None, - confidence=100, dummy=False, internal=None, ): @@ -1838,7 +2330,6 @@ def make_event( scan (Scan, optional): BBOT Scan object associated with the event. scans (List[Scan], optional): Multiple BBOT Scan objects, primarily used for unserialization. tags (Union[str, List[str]], optional): Descriptive tags for the event, as a list or a single string. - confidence (int, optional): Confidence level for the event, on a scale of 1-100. Defaults to 100. dummy (bool, optional): Disables data validations if set to True. Defaults to False. internal (Any, optional): Makes the event internal if set to True. Defaults to None. @@ -1872,7 +2363,7 @@ def make_event( if not dummy: log.debug(f'Autodetected event type "{event_type}" based on data: "{data}"') - event_type = str(event_type).strip().upper() + event_type = sys.intern(str(event_type).strip().upper()) # Catch these common whoopsies if event_type in ("DNS_NAME", "IP_ADDRESS"): @@ -1885,7 +2376,8 @@ def make_event( data = validators.validate_host(data) except Exception as e: log.trace(traceback.format_exc()) - raise ValidationError(f'Error sanitizing event data "{data}" for type "{event_type}": {e}') + data_preview = str(data)[:200] + "..." if len(str(data)) > 200 else str(data) + raise ValidationError(f'Error sanitizing event data "{data_preview}" for type "{event_type}": {e}') data_is_ip = is_ip(data) if event_type == "DNS_NAME" and data_is_ip: event_type = "IP_ADDRESS" @@ -1912,13 +2404,12 @@ def make_event( module=module, scan=scan, tags=tags, - confidence=confidence, _dummy=dummy, _internal=internal, ) -def event_from_json(j, siem_friendly=False): +def event_from_json(j): """ Creates an event object from a JSON dictionary. @@ -1945,14 +2436,15 @@ def event_from_json(j, siem_friendly=False): kwargs = { "event_type": event_type, "tags": j.get("tags", []), - "confidence": j.get("confidence", 100), "context": j.get("discovery_context", None), "dummy": True, } - if siem_friendly: - data = j["data"][event_type] - else: - data = j["data"] + data = j.get("data_json", None) + if data is None: + data = j.get("data", None) + if data is None: + json_pretty = json.dumps(j, indent=2) + raise ValueError(f"data or data_json must be provided. JSON: {json_pretty}") kwargs["data"] = data event = make_event(**kwargs) event_uuid = j.get("uuid", None) @@ -1960,8 +2452,20 @@ def event_from_json(j, siem_friendly=False): event._uuid = uuid.UUID(event_uuid.split(":")[-1]) resolved_hosts = j.get("resolved_hosts", []) - event._resolved_hosts = set(resolved_hosts) - event.timestamp = datetime.datetime.fromisoformat(j["timestamp"]) + event._resolved_hosts = frozenset(resolved_hosts) if resolved_hosts else None + + http_title = j.get("http_title", "") + if http_title: + try: + event.http_title = http_title + except AttributeError: + pass + + # accept both isoformat and unix timestamp + try: + event.timestamp = datetime.datetime.fromtimestamp(j["timestamp"], ZoneInfo("UTC")) + except Exception: + event.timestamp = datetime.datetime.fromisoformat(j["timestamp"]) event.scope_distance = j["scope_distance"] parent_id = j.get("parent", None) if parent_id is not None: diff --git a/bbot/core/event/helpers.py b/bbot/core/event/helpers.py index 524eccbcd8..be85faef55 100644 --- a/bbot/core/event/helpers.py +++ b/bbot/core/event/helpers.py @@ -1,14 +1,27 @@ -import ipaddress import regex as re from functools import cached_property from bbot.errors import ValidationError from bbot.core.helpers import validators -from bbot.core.helpers.misc import split_host_port, make_ip_type +from bbot.core.helpers.misc import split_host_port, make_ip_type, cached_ip_address, cached_ip_network from bbot.core.helpers import regexes, smart_decode, smart_encode_punycode bbot_event_seeds = {} +# Pre-compute sorted event classes for performance +# This is computed once when the module is loaded instead of on every EventSeed() call +def _get_sorted_event_classes(): + """ + Sort event classes by priority (higher priority first). + This ensures specific patterns like ASN:12345 are checked before broad patterns like hostname:port. + """ + return sorted(bbot_event_seeds.items(), key=lambda x: getattr(x[1], "priority", 5), reverse=True) + + +# This will be populated after all event seed classes are registered +_sorted_event_classes = None + + """ An "Event Seed" is a lightweight event containing only the minimum logic required to: - parse input to determine the event type + data @@ -18,6 +31,19 @@ It's useful for quickly parsing target lists without the cpu+memory overhead of creating full-fledged BBOT events Not every type of BBOT event needs to be represented here. Only ones that are meant to be targets. + +PRIORITY SYSTEM: +Event seeds support a priority system to control the order in which regex patterns are checked. +This prevents conflicts where one event type's regex might incorrectly match another type's input. + +Priority values: +- Higher numbers = checked first +- Default priority = 5 +- Range: 1-10 + +To set priority on an event seed class: + class MyEventSeed(BaseEventSeed): + priority = 8 # Higher than default, will be checked before most others """ @@ -27,17 +53,25 @@ class EventSeedRegistry(type): """ def __new__(mcs, name, bases, attrs): - global bbot_event_seeds + global bbot_event_seeds, _sorted_event_classes cls = super().__new__(mcs, name, bases, attrs) # Don't register the base EventSeed class if name != "BaseEventSeed": bbot_event_seeds[cls.__name__] = cls + # Recompute sorted classes whenever a new event seed is registered + _sorted_event_classes = _get_sorted_event_classes() return cls def EventSeed(input): input = smart_encode_punycode(smart_decode(input).strip()) - for _, event_class in bbot_event_seeds.items(): + + # Use pre-computed sorted event classes for better performance + global _sorted_event_classes + if _sorted_event_classes is None: + _sorted_event_classes = _get_sorted_event_classes() + + for _, event_class in _sorted_event_classes: if hasattr(event_class, "precheck"): if event_class.precheck(input): return event_class(input) @@ -53,6 +87,7 @@ def EventSeed(input): class BaseEventSeed(metaclass=EventSeedRegistry): regexes = [] _target_type = "TARGET" + priority = 5 # Default priority for event seed matching (1-10, higher = checked first) __slots__ = ["data", "host", "port", "input"] @@ -76,6 +111,9 @@ def _sanitize_and_extract_host(self, data): """ return data, None, None + async def _generate_children(self, helpers): + return [] + def _override_input(self, input): return self.data @@ -83,6 +121,12 @@ def _override_input(self, input): def type(self): return self.__class__.__name__ + @property + def url(self): + if self.type == "URL_UNVERIFIED": + return self.data + return "" + @cached_property def _hash(self): return hash(self.input) @@ -106,13 +150,13 @@ class IP_ADDRESS(BaseEventSeed): @staticmethod def precheck(data): try: - return ipaddress.ip_address(data) + return cached_ip_address(data) except ValueError: return False @staticmethod def _sanitize_and_extract_host(data): - validated = ipaddress.ip_address(data) + validated = cached_ip_address(data) return str(validated), validated, None @@ -131,18 +175,19 @@ class IP_RANGE(BaseEventSeed): @staticmethod def precheck(data): try: - return ipaddress.ip_network(str(data), strict=False) + return cached_ip_network(str(data)) except ValueError: return False @staticmethod def _sanitize_and_extract_host(data): - validated = ipaddress.ip_network(str(data), strict=False) + validated = cached_ip_network(str(data)) return str(validated), validated, None class OPEN_TCP_PORT(BaseEventSeed): regexes = regexes.event_type_regexes["OPEN_TCP_PORT"] + priority = 1 # Low priority: broad hostname:port pattern should be checked after specific patterns @staticmethod def _sanitize_and_extract_host(data): @@ -236,3 +281,25 @@ def _override_input(self, input): @staticmethod def handle_match(match): return match.group(1) + + +class ASN(BaseEventSeed): + regexes = (re.compile(r"^(?:ASN|AS):?(\d+)$", re.I),) # adjust regex to match ASN:17178 AS17178 + priority = 10 # High priority + + def _override_input(self, input): + return f"ASN:{self.data}" + + # ASNs are essentially a superset of IP_RANGES. This resolves the ASN to its + # subnets via the shared ASN helper and emits each CIDR as a child seed, + # which is later resolved to an IP_RANGE seed and added to the target. + async def _generate_children(self, helpers): + asn_data = await helpers.asn.asn_to_subnets(self.data) + subnets = asn_data.get("subnets") or [] + if isinstance(subnets, str): + subnets = [subnets] + return list(subnets) + + @staticmethod + def handle_match(match): + return match.group(1) diff --git a/bbot/core/event/spill.py b/bbot/core/event/spill.py new file mode 100644 index 0000000000..f5ac39bb0d --- /dev/null +++ b/bbot/core/event/spill.py @@ -0,0 +1,225 @@ +""" +Per-scan disk-spill store for HTTP_RESPONSE bodies. + +Bodies are the dominant memory tenant in long-running scans (in our +benchmarks, mid-scan body bytes peak at ~640 MB on a wide-and-busy +workload). Holding them in event objects until ``_minimize()`` fires +keeps Python's working set above what the actual processing pipeline +needs. + +This store keeps bodies on disk under ``${scan.temp_dir}/bodies/`` and +serves reads from a bounded LRU cache. The cache absorbs the +processing-window working set; reads only touch disk when the working +set exceeds the cache. + +Design (locked-in): + + - **Trigger**: every body always spills (never threshold-based). + - **Storage**: file-per-event, ``${scan.temp_dir}/bodies/{uuid}.body[.zst]``. + - **Compression**: zstd level 1 (configurable on/off). + - **Cache**: bounded by total bytes (default 512 MB), keyed by event + UUID, value is decompressed body bytes. + - **Eviction**: biggest-first, FIFO tie-break. (Pure LRU is wrong for + BBOT's pipeline — "least recently used" tends to be the next event + to be processed, not a candidate for eviction.) + - **Reads**: synchronous. Cache hit is instant; misses do a small + blocking read from page cache (typically microseconds). + - **Writes**: synchronous to OS page cache (a few hundred microseconds + for a typical body; OS handles actual disk flush async). + - **Lifecycle**: file + cache entry are deleted when the event's + ``_minimize()`` fires. Scan-end ``rm_rf`` of ``temp_dir`` mops up + anything that escaped. +""" + +import logging +from pathlib import Path +from typing import Optional + +import zstandard as zstd + +log = logging.getLogger("bbot.spill") + + +class BodySpillStore: + """ + LRU + on-disk store for HTTP response bodies. + + Bodies are always written to disk on insert. The LRU is a working-set + cache — hits are fast, misses re-read from disk. Eviction is + biggest-first with FIFO tie-break, which respects BBOT's roughly-FIFO + pipeline order (the oldest cache entry is typically the next event + to be processed, so the smaller-newer entries are safer to evict). + + Thread-safety: not thread-safe. Designed for single-event-loop use + inside a Scanner. Calls cross the event loop boundary only via + cooperative ``await``, but I/O itself is synchronous (page cache). + + Hit/miss accounting is exposed via ``stats()`` for benchmark and + operational visibility. + """ + + # Module-internal sentinel for evicted-but-not-deleted entries during + # iteration. Currently unused — left here for future async-write work. + _UNSET = object() + + def __init__( + self, + base_dir: Path, + cache_bytes: int = 512 * 1024 * 1024, + compress: bool = True, + compress_level: int = 1, + ): + self.base_dir = Path(base_dir) + self.base_dir.mkdir(parents=True, exist_ok=True) + self.cache_bytes = int(cache_bytes) + self.compress = bool(compress) + # Compressors are cheap to construct but reusable. + self._cctx = zstd.ZstdCompressor(level=compress_level) if compress else None + self._dctx = zstd.ZstdDecompressor() if compress else None + + # Cache value: dict with bytes + insertion sequence (for FIFO tie-break). + # {uuid: {"body": bytes, "seq": int}} + self._cache: dict[str, dict] = {} + self._cache_total_bytes = 0 + self._seq = 0 # monotonic insertion counter + + # Stats + self._hits = 0 + self._misses = 0 + self._writes = 0 + self._evictions = 0 + + # ── Public API ──────────────────────────────────────────────────── + + def write(self, event_uuid: str, body: bytes) -> None: + """ + Spill a body to disk and seed the cache. The body remains + immediately readable (cache hit) until evicted; thereafter, + reads pull from disk. + + ``body`` must be ``bytes``. Callers passing ``str`` should + ``.encode("utf-8", errors="replace")`` first. + """ + if not isinstance(body, (bytes, bytearray, memoryview)): + raise TypeError(f"body must be bytes-like, got {type(body).__name__}") + body_bytes = bytes(body) + + path = self._path_for(event_uuid) + if self.compress: + on_disk = self._cctx.compress(body_bytes) + else: + on_disk = body_bytes + + # Synchronous write to page cache. Fast (memcpy), the OS does + # the real disk flush asynchronously. + path.write_bytes(on_disk) + self._writes += 1 + + self._insert_cache(event_uuid, body_bytes) + + def read(self, event_uuid: str) -> Optional[bytes]: + """ + Return the body for ``event_uuid``, or ``None`` if neither + cache nor disk has it. + + Cache hit: instant. Cache miss + file present: small blocking + disk read (page-cached after first hit). + """ + entry = self._cache.get(event_uuid) + if entry is not None: + self._hits += 1 + return entry["body"] + + # Miss — try disk. + path = self._path_for(event_uuid) + if not path.exists(): + self._misses += 1 + return None + + on_disk = path.read_bytes() + body_bytes = self._dctx.decompress(on_disk) if self.compress else on_disk + + self._misses += 1 + self._insert_cache(event_uuid, body_bytes) + return body_bytes + + def evict_and_delete(self, event_uuid: str) -> None: + """ + Drop ``event_uuid`` from the cache and delete the file. + + Called from ``HTTP_RESPONSE._minimize()`` when the event is no + longer needed by any module. + """ + entry = self._cache.pop(event_uuid, None) + if entry is not None: + self._cache_total_bytes -= len(entry["body"]) + + path = self._path_for(event_uuid) + try: + path.unlink() + except FileNotFoundError: + pass + except Exception as e: # pragma: no cover — defensive + log.debug(f"failed to unlink spill file {path}: {e}") + + def stats(self) -> dict: + """Hit/miss/eviction counters and current cache fill.""" + total = self._hits + self._misses + hit_rate = (self._hits / total) if total else 0.0 + return { + "hits": self._hits, + "misses": self._misses, + "writes": self._writes, + "evictions": self._evictions, + "hit_rate": round(hit_rate, 4), + "cache_entries": len(self._cache), + "cache_bytes": self._cache_total_bytes, + "cache_bytes_limit": self.cache_bytes, + } + + # ── Internal ────────────────────────────────────────────────────── + + def _path_for(self, event_uuid: str) -> Path: + suffix = ".body.zst" if self.compress else ".body" + return self.base_dir / f"{event_uuid}{suffix}" + + def _insert_cache(self, event_uuid: str, body_bytes: bytes) -> None: + """Insert into cache, evicting biggest-first / oldest-first as needed.""" + body_size = len(body_bytes) + + # Replace existing entry if present (keeps semantics simple if a + # body is rewritten — currently never happens, but cheap insurance). + existing = self._cache.pop(event_uuid, None) + if existing is not None: + self._cache_total_bytes -= len(existing["body"]) + + # If a single body exceeds the entire cache budget, don't try + # to cache it — disk reads will be the only path. + if body_size > self.cache_bytes: + return + + # Evict until there's room. + while self._cache_total_bytes + body_size > self.cache_bytes and self._cache: + self._evict_one() + + self._seq += 1 + self._cache[event_uuid] = {"body": body_bytes, "seq": self._seq} + self._cache_total_bytes += body_size + + def _evict_one(self) -> None: + """Pick a victim by ``(-size, seq)`` — biggest-first, FIFO tie-break.""" + # Single-pass min: maximize size, minimize seq among ties. + victim_uuid = None + victim_size = -1 + victim_seq = float("inf") + for u, entry in self._cache.items(): + sz = len(entry["body"]) + if sz > victim_size or (sz == victim_size and entry["seq"] < victim_seq): + victim_uuid = u + victim_size = sz + victim_seq = entry["seq"] + if victim_uuid is None: # pragma: no cover — empty cache, defensive + return + entry = self._cache.pop(victim_uuid) + self._cache_total_bytes -= len(entry["body"]) + self._evictions += 1 diff --git a/bbot/core/flags.py b/bbot/core/flags.py index 3391b48635..26ee48f840 100644 --- a/bbot/core/flags.py +++ b/bbot/core/flags.py @@ -1,25 +1,25 @@ flag_descriptions = { "active": "Makes active connections to target systems", "affiliates": "Discovers affiliated hostnames/domains", - "aggressive": "Generates a large amount of network traffic", "baddns": "Runs all modules from the DNS auditing tool BadDNS", "cloud-enum": "Enumerates cloud resources", "code-enum": "Find public code repositories and search them for secrets etc.", - "deadly": "Highly aggressive", "download": "Modules that download files, apps, or repositories", "email-enum": "Enumerates email addresses", "iis-shortnames": "Scans for IIS Shortname vulnerability", + "invasive": "Intrusive or potentially destructive", + "loud": "Generates a large amount of network traffic", "passive": "Never connects to target systems", + "safe": "Non-intrusive and non-destructive", "portscan": "Discovers open ports", "report": "Generates a report at the end of the scan", - "safe": "Non-intrusive, safe to run", "service-enum": "Identifies protocols running on open ports", "slow": "May take a long time to complete", "social-enum": "Enumerates social media", "subdomain-enum": "Enumerates subdomains", "subdomain-hijack": "Detects hijackable subdomains", - "web-basic": "Basic, non-intrusive web scan functionality", + "web": "Non-intrusive web scan functionality", + "web-heavy": "More advanced web scanning functionality", "web-paramminer": "Discovers HTTP parameters through brute-force", "web-screenshots": "Takes screenshots of web pages", - "web-thorough": "More advanced web scanning functionality", } diff --git a/bbot/core/helpers/asn.py b/bbot/core/helpers/asn.py new file mode 100644 index 0000000000..30de61554d --- /dev/null +++ b/bbot/core/helpers/asn.py @@ -0,0 +1,118 @@ +import logging + +from bbot.errors import ASNResolutionError +from bbot.core.helpers.async_helpers import NamedLock + +log = logging.getLogger("bbot.core.helpers.asn") + + +class ASNHelper: + """Thin wrapper around the asndb library for ASN lookups. + + Delegates all HTTP, caching, and retry logic to the asndb library. + Normalizes response dicts to the BBOT-internal format with keys: + asn, subnets, name, description, country + """ + + UNKNOWN_ASN = { + "asn": 0, + "subnets": [], + "name": "Unknown", + "description": "Unknown ASN", + "country": "Unknown", + } + + FAILURE_THRESHOLD = 5 + MAX_RETRIES = 3 + RETRY_DELAY = 3 + + def __init__(self, parent_helper): + self.parent_helper = parent_helper + self._client = None + self._consecutive_failures = 0 + self._circuit_broken = False + # serialize asndb requests so concurrent callers don't stampede the rate-limited API + self._request_lock = NamedLock() + + @property + def client(self): + if self._client is None: + from asndb import ASNDB + + ssl_verify = self.parent_helper.web_config.get("ssl_verify_infrastructure", True) + api_key = self.parent_helper.config.get("bbot_io_api_key", None) + self._client = ASNDB(bbot_io_api_key=api_key, verify=ssl_verify) + return self._client + + def _normalize(self, response): + """Convert asndb response dict to BBOT internal format.""" + if response is None or response.get("asn", 0) == 0: + return self.UNKNOWN_ASN + return { + "asn": int(response.get("asn", 0)), + "subnets": response.get("subnets", []), + "name": response.get("asn_name") or response.get("name") or "", + "description": response.get("org") or response.get("description") or "", + "country": response.get("country") or "", + } + + def _record_failure(self): + self._consecutive_failures += 1 + if self._consecutive_failures >= self.FAILURE_THRESHOLD and not self._circuit_broken: + self._circuit_broken = True + log.critical( + "ASN lookups disabled for the rest of this scan after %d consecutive failures. " + "The bbot.io ASN API could not be reached (this may be due to regional network restrictions). " + "ASN enrichment data will not be available.", + self.FAILURE_THRESHOLD, + ) + + async def ip_to_subnets(self, ip): + """Return ASN info for an IP address.""" + if self._circuit_broken: + return self.UNKNOWN_ASN + try: + async with self._request_lock.lock("asndb"): + response = await self.client.lookup_ip(str(ip), include_subnets=True) + except Exception as e: + log.warning(f"ASN lookup failed for IP {ip}: {e}") + self._record_failure() + return self.UNKNOWN_ASN + self._consecutive_failures = 0 + return self._normalize(response) + + async def asn_to_subnets(self, asn): + """Resolve an ASN number to its subnets, retrying on transient failure. + + Used by ASN-as-target seed expansion, which cannot degrade gracefully: + without the ASN's subnets there is nothing to scan. Raises + ASNResolutionError if the ASN can't be resolved after MAX_RETRIES so the + scan aborts with guidance instead of silently scanning nothing. + """ + asn = int(str(asn).lower().lstrip("as")) + last_error = None + for attempt in range(1, self.MAX_RETRIES + 1): + try: + async with self._request_lock.lock("asndb"): + response = await self.client.lookup_asn(asn, include_subnets=True) + return self._normalize(response) + except Exception as e: + last_error = e + log.warning(f"ASN resolution attempt {attempt}/{self.MAX_RETRIES} failed for AS{asn}: {e}") + if attempt < self.MAX_RETRIES: + await self.parent_helper.sleep(self.RETRY_DELAY) + raise ASNResolutionError(f"AS{asn}: {last_error}") + + async def cleanup(self): + """Clean up the asndb client.""" + if self._client is not None: + try: + await self._client.cleanup() + except Exception: + pass + self._client = None + # Reset the asndb global singleton so the next ASNDB() call + # creates a fresh client instead of returning the closed one + import asndb.asndb + + asndb.asndb.asndb_client = None diff --git a/bbot/core/helpers/async_helpers.py b/bbot/core/helpers/async_helpers.py index a9d8adc6fe..4620dc8345 100644 --- a/bbot/core/helpers/async_helpers.py +++ b/bbot/core/helpers/async_helpers.py @@ -51,24 +51,20 @@ async def lock(self, name): class TaskCounter: def __init__(self): self.tasks = {} - self._lock = None @property def value(self): return sum([t.n for t in self.tasks.values()]) - @property - def lock(self): - if self._lock is None: - self._lock = asyncio.Lock() - return self._lock - def count(self, task_name, n=1, asyncio_task=None, _log=True): if callable(task_name): task_name = f"{task_name.__qualname__}()" return self.Task(self, task_name, n=n, _log=_log, asyncio_task=asyncio_task) class Task: + # asyncio is single-threaded, so the dict mutations below are safe without a lock. + # Keeping __aenter__/__aexit__ free of awaits means `async with counter.count(...)` resolves + # without yielding to the event loop — cheap enough for hot paths like precheck/postcheck. def __init__(self, manager, task_name, n=1, _log=True, asyncio_task=None): self.manager = manager self.task_name = task_name @@ -80,18 +76,12 @@ def __init__(self, manager, task_name, n=1, _log=True, asyncio_task=None): async def __aenter__(self): self.task_id = uuid.uuid4() - # if self.log: - # log.trace(f"Starting task {self.task_name} ({self.task_id})") - async with self.manager.lock: - self.start_time = time.time() - self.manager.tasks[self.task_id] = self + self.start_time = time.time() + self.manager.tasks[self.task_id] = self return self async def __aexit__(self, exc_type, exc_val, exc_tb): - async with self.manager.lock: - self.manager.tasks.pop(self.task_id, None) - # if self.log: - # log.trace(f"Finished task {self.task_name} ({self.task_id})") + self.manager.tasks.pop(self.task_id, None) @property def asyncio_task(self): @@ -133,6 +123,25 @@ def async_to_sync_gen(async_gen): yield loop.run_until_complete(async_gen.__anext__()) except StopAsyncIteration: pass + finally: + # Explicitly close the async generator so its finally block + # (e.g. Scanner.async_start's cleanup) runs while the loop + # is still alive, not deferred to interpreter shutdown. + with suppress(BaseException): + loop.run_until_complete(async_gen.aclose()) + # Cancel any remaining pending tasks to prevent + # "Task was destroyed but it is pending!" warnings + pending = [t for t in asyncio.all_tasks(loop) if not t.done()] + for t in pending: + t.cancel() + if pending: + with suppress(BaseException): + loop.run_until_complete( + asyncio.wait_for( + asyncio.gather(*pending, return_exceptions=True), + timeout=5, + ) + ) def async_cachedmethod(cache, key=keys.hashkey): diff --git a/bbot/core/helpers/command.py b/bbot/core/helpers/command.py index 7da96bbd38..173e100698 100644 --- a/bbot/core/helpers/command.py +++ b/bbot/core/helpers/command.py @@ -1,6 +1,7 @@ import os import asyncio import logging +import contextlib import traceback from signal import SIGINT from subprocess import CompletedProcess, CalledProcessError, SubprocessError @@ -157,7 +158,18 @@ async def run_live(self, *command, check=False, text=True, idle_timeout=None, ** command_str = " ".join(command) log.warning(f"Stderr for run_live({command_str}):\n\t{stderr}") finally: - proc_tracker.remove(proc) + proc_tracker.discard(proc) + # Kill the subprocess if it's still running (e.g. generator was cancelled/closed) + if proc.returncode is None: + with contextlib.suppress(Exception): + proc.terminate() + try: + await asyncio.wait_for(proc.wait(), timeout=5) + except (asyncio.TimeoutError, Exception): + with contextlib.suppress(Exception): + proc.kill() + if input_task is not None: + input_task.cancel() async def _spawn_proc(self, *command, **kwargs): @@ -270,7 +282,7 @@ def _prepare_command_kwargs(self, command, kwargs): >>> _prepare_command_kwargs(['ls', '-l'], {'sudo': True}) (['sudo', '-E', '-A', 'LD_LIBRARY_PATH=...', 'PATH=...', 'ls', '-l'], {'limit': 104857600, 'stdout': -1, 'stderr': -1, 'env': environ(...)}) """ - # limit = 100MB (this is needed for cases like httpx that are sending large JSON blobs over stdout) + # limit = 100MB (this is needed for cases that are sending large JSON blobs over stdout) if "limit" not in kwargs: kwargs["limit"] = 1024 * 1024 * 100 if "stdout" not in kwargs: diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index 1348ed077c..baba35d562 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -202,7 +202,7 @@ async def install(self, *modules): log.debug(f'Setup succeeded for module "{m}"') succeeded.append(m) else: - log.warning(f'Setup failed for module "{m}"') + log.error(f'Setup failed for module "{m}"') failed.append(m) else: if success or self.deps_behavior == "ignore_failed": @@ -211,7 +211,7 @@ async def install(self, *modules): ) succeeded.append(m) else: - log.warning( + log.error( f'Skipping dependency install for module "{m}" because it failed previously (--retry-deps to retry or --ignore-failed-deps to ignore)' ) failed.append(m) @@ -288,7 +288,7 @@ async def pip_install(self, packages, constraints=None): log.info(message) return True except CalledProcessError as err: - log.warning(f"Failed to install pip packages {packages_str} (return code {err.returncode}): {err.stderr}") + log.error(f"Failed to install pip packages {packages_str} (return code {err.returncode}): {err.stderr}") return False def apt_install(self, packages): @@ -300,11 +300,9 @@ def apt_install(self, packages): if success: log.info(f'Successfully installed OS packages "{",".join(sorted(packages))}"') else: - log.warning( - f"Failed to install OS packages ({err}). Recommend installing the following packages manually:" - ) + log.error(f"Failed to install OS packages ({err}). Recommend installing the following packages manually:") for p in packages: - log.warning(f" - {p}") + log.error(f" - {p}") return success def _make_apt_ansible_args(self, packages): @@ -341,7 +339,7 @@ def shell(self, module, commands): if success: log.info(f"Successfully ran {len(commands):,} shell commands") else: - log.warning("Failed to run shell dependencies") + log.error("Failed to run shell dependencies") return success def tasks(self, module, tasks): @@ -350,7 +348,7 @@ def tasks(self, module, tasks): if success: log.info(f"Successfully ran {len(tasks):,} Ansible tasks for {module}") else: - log.warning(f"Failed to run Ansible tasks for {module}") + log.error(f"Failed to run Ansible tasks for {module}") return success def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None): @@ -426,8 +424,10 @@ def ensure_root(self, message=""): with self.ensure_root_lock: # first check if the environment variable is set _sudo_password = os.environ.get("BBOT_SUDO_PASS", None) - if _sudo_password is not None or os.geteuid() == 0 or can_sudo_without_password(): - # if we're already root or we can sudo without a password, there's no need to prompt + if _sudo_password is not None: + self._sudo_password = _sudo_password + return + if os.geteuid() == 0 or can_sudo_without_password(): return if message: @@ -435,13 +435,34 @@ def ensure_root(self, message=""): while not self._sudo_password: # sleep for a split second to flush previous log messages sleep(0.1) - _sudo_password = getpass.getpass(prompt="[USER] Please enter sudo password: ") + try: + _sudo_password = getpass.getpass(prompt="[USER] Please enter sudo password: ") + except OSError: + log.error("Unable to read sudo password (no TTY). Set BBOT_SUDO_PASS env var.") + return if self.parent_helper.verify_sudo_password(_sudo_password): log.success("Authentication successful") self._sudo_password = _sudo_password else: log.warning("Incorrect password") + def _core_dep_satisfied(self, command): + """Check if a core dependency is satisfied. + + For normal binary deps, check if the command exists on PATH. + For special entries like openssl_dev_headers, use a custom check. + """ + if command == "openssl_dev_headers": + # check for openssl headers by looking for the pkg-config file or header + return any( + Path(p).exists() + for p in [ + "/usr/include/openssl/ssl.h", + "/usr/local/include/openssl/ssl.h", + ] + ) or bool(self.parent_helper.which("openssl")) + return bool(self.parent_helper.which(command)) + async def install_core_deps(self): # skip if we've already successfully installed core deps for this definition core_deps_hash = str(mmh3.hash(orjson.dumps(self.CORE_DEPS, option=orjson.OPT_SORT_KEYS))) @@ -453,18 +474,25 @@ async def install_core_deps(self): to_install = set() to_install_friendly = set() playbook = [] - self._install_sudo_askpass() - # ensure tldextract data is cached - self.parent_helper.tldextract("evilcorp.co.uk") - # install any missing commands + # check which commands are missing for command, package_name_or_playbook in self.CORE_DEPS.items(): - if not self.parent_helper.which(command): - to_install_friendly.add(command) - if isinstance(package_name_or_playbook, str): - to_install.add(package_name_or_playbook) - else: - playbook.extend(package_name_or_playbook) - # install ansible community.general collection + if self._core_dep_satisfied(command): + continue + to_install_friendly.add(command) + if isinstance(package_name_or_playbook, str): + to_install.add(package_name_or_playbook) + else: + playbook.extend(package_name_or_playbook) + # construct ansible playbook + if to_install: + playbook.append( + { + "name": "Install Core BBOT Dependencies", + "package": {"name": list(to_install), "state": "present"}, + "become": True, + } + ) + # install ansible community.general collection if needed overall_success = True if not self.setup_status.get("ansible:community.general", False): log.info("Installing Ansible Community General Collection") @@ -478,17 +506,12 @@ async def install_core_deps(self): f"Failed to install Ansible Community.General Collection (return code {err.returncode}): {err.stderr}" ) overall_success = False - # construct ansible playbook - if to_install: - playbook.append( - { - "name": "Install Core BBOT Dependencies", - "package": {"name": list(to_install), "state": "present"}, - "become": True, - } - ) - # run playbook + # only run ansible if there's actually something to install if playbook: + self._install_sudo_askpass() + # ensure tldextract data is cached + self.parent_helper.tldextract("evilcorp.co.uk") + # run playbook log.info(f"Installing core BBOT dependencies: {','.join(sorted(to_install_friendly))}") self.ensure_root() success, _ = self.ansible_run(tasks=playbook) diff --git a/bbot/core/helpers/depsinstaller/sudo_askpass.py b/bbot/core/helpers/depsinstaller/sudo_askpass.py index ccd8cd01b6..69454bb631 100644 --- a/bbot/core/helpers/depsinstaller/sudo_askpass.py +++ b/bbot/core/helpers/depsinstaller/sudo_askpass.py @@ -33,7 +33,7 @@ def main(): decrypted_password = decrypt_password(encrypted_password, key) print(decrypted_password, end="") except Exception as e: - print(f'Error decrypting password "{encrypted_password}": {str(e)}', file=sys.stderr) + print(f"Error decrypting sudo password: {str(e)}", file=sys.stderr) sys.exit(1) diff --git a/bbot/core/helpers/diff.py b/bbot/core/helpers/diff.py index 64c1b1e6a5..126f122e68 100644 --- a/bbot/core/helpers/diff.py +++ b/bbot/core/helpers/diff.py @@ -8,6 +8,67 @@ log = logging.getLogger("bbot.core.helpers.diff") +class _BaselineSnapshot: + """Lightweight stand-in for a blasthttp Response held by HttpCompare. + + Stores only the fields external code accesses (status_code, headers, + text, content) and optionally spills the body to the scan's + BodySpillStore so the raw bytes don't pin Python heap memory for the + lifetime of the HttpCompare instance. + + The blasthttp Headers object is kept by reference (it survives + Response GC independently) to preserve case-insensitive lookups and + duplicate-header semantics that DeepDiff relies on. + """ + + __slots__ = ("status_code", "headers", "_text", "_spill_key", "_spill_store") + + def __init__(self, response, spill_store=None): + self.status_code = response.status_code + self.headers = response.headers + if spill_store is not None: + body_bytes = response.body_bytes or b"" + self._spill_key = f"baseline-{id(self):x}" + spill_store.write(self._spill_key, body_bytes) + self._spill_store = spill_store + self._text = None + else: + self._text = response.text + self._spill_key = None + self._spill_store = None + + @property + def text(self): + if self._text is not None: + return self._text + if self._spill_store is not None: + body = self._spill_store.read(self._spill_key) + if body is not None: + return body.decode("utf-8", errors="replace") + return "" + + @property + def content(self): + if self._spill_store is not None: + return self._spill_store.read(self._spill_key) or b"" + if self._text is not None: + return self._text.encode("utf-8", errors="replace") + return b"" + + def _cleanup(self): + if self._spill_store is not None and self._spill_key is not None: + self._spill_store.evict_and_delete(self._spill_key) + self._spill_key = None + + def __del__(self): + # Reclaim the spilled body when the snapshot is GC'd (HttpCompare + # instances are mostly short-lived locals, so this fires promptly). + try: + self._cleanup() + except Exception: + pass + + class HttpCompare: def __init__( self, @@ -21,9 +82,14 @@ def __init__( headers=None, cookies=None, timeout=10, + on_baseline_ready=None, + baseline_url_2=None, ): self.parent_helper = parent_helper self.baseline_url = baseline_url + # When set, the second baseline sample uses this URL instead of self.baseline_url, + # so the auto-filter captures inter-URL variation (used by wildcard detection). + self.baseline_url_2 = baseline_url_2 self.include_cache_buster = include_cache_buster self.method = method self.data = data @@ -33,6 +99,8 @@ def __init__( self.headers = headers self.cookies = cookies self.timeout = 10 + # Optional async callback fired once with baseline_1 after the baseline is established. + self.on_baseline_ready = on_baseline_ready @staticmethod def merge_dictionaries(headers1, headers2): @@ -67,7 +135,8 @@ async def _baseline(self): if self.include_cache_buster: get_params.update(self.gen_cache_buster()) - url_2 = self.parent_helper.add_get_params(self.baseline_url, get_params).geturl() + second_target = self.baseline_url_2 if self.baseline_url_2 else self.baseline_url + url_2 = self.parent_helper.add_get_params(second_target, get_params).geturl() baseline_2 = await self.parent_helper.request( url_2, headers=self.merge_dictionaries( @@ -84,11 +153,12 @@ async def _baseline(self): timeout=self.timeout, ) - self.baseline = baseline_1 if baseline_1 is None or baseline_2 is None: log.debug("HTTP error while establishing baseline, aborting") + baseline_1_repr = f"HTTP {baseline_1.status_code}" if baseline_1 is not None else "None" + baseline_2_repr = f"HTTP {baseline_2.status_code}" if baseline_2 is not None else "None" raise HttpCompareError( - f"Can't get baseline from source URL: {url_1}:{baseline_1} / {url_2}:{baseline_2}" + f"Can't get baseline from source URL: {url_1} ({baseline_1_repr}) / {url_2} ({baseline_2_repr})" ) if baseline_1.status_code != baseline_2.status_code: log.debug("Status code not stable during baseline, aborting") @@ -98,7 +168,6 @@ async def _baseline(self): baseline_1_json = xmltodict.parse(baseline_1.text) baseline_2_json = xmltodict.parse(baseline_2.text) except ExpatError: - log.debug(f"Can't HTML parse for {self.baseline_url}. Switching to text parsing as a backup") baseline_1_json = baseline_1.text.split("\n") baseline_2_json = baseline_2.text.split("\n") @@ -118,6 +187,7 @@ async def _baseline(self): "date", "last-modified", "content-length", + "connection", "ETag", "X-Pad", "X-Backside-Transport", @@ -129,6 +199,19 @@ async def _baseline(self): self.baseline_ignore_headers += [x.lower() for x in dynamic_headers] self._baselined = True + if self.on_baseline_ready is not None: + try: + await self.on_baseline_ready(baseline_1) + except Exception as e: + log.debug(f"on_baseline_ready callback raised: {e}") + + # Replace the heavy blasthttp Response with a lightweight + # snapshot. If the scan has a body_spill_store the body bytes + # are written to disk and served from the LRU cache on demand. + scan = getattr(self.parent_helper, "scan", None) + store = getattr(scan, "body_spill_store", None) + self.baseline = _BaselineSnapshot(baseline_1, spill_store=store) + def gen_cache_buster(self): return {self.parent_helper.rand_string(6): "1"} @@ -139,7 +222,6 @@ def compare_headers(self, headers_1, headers_2): for header, value in list(headers.items()): if header.lower() in self.baseline_ignore_headers: with suppress(KeyError): - log.debug(f'found ignored header "{header}" in headers_{i + 1} and removed') del headers[header] ddiff = DeepDiff(headers_1, headers_2, ignore_order=True, view="tree", threshold_to_diff_deeper=0) @@ -148,7 +230,7 @@ def compare_headers(self, headers_1, headers_2): for x in list(ddiff[k]): try: header_value = str(x).split("'")[1] - except KeyError: + except (KeyError, IndexError): continue differing_headers.append(header_value) return differing_headers @@ -233,35 +315,37 @@ async def compare( if item in subject_response.text: reflection = True break + diff_reasons = await self.parent_helper.run_in_executor_cpu( + self._compare_sync, + subject_response, + subject, + ) + + if not diff_reasons: + return (True, [], reflection, subject_response) + else: + return (False, diff_reasons, reflection, subject_response) + + def _compare_sync(self, subject_response, subject): + """CPU-bound comparison work offloaded from the event loop.""" try: subject_json = xmltodict.parse(subject_response.text) - except ExpatError: - log.debug(f"Can't HTML parse for {subject.split('?')[0]}. Switching to text parsing as a backup") subject_json = subject_response.text.split("\n") diff_reasons = [] if self.baseline.status_code != subject_response.status_code: - log.debug( - f"status code was different [{str(self.baseline.status_code)}] -> [{str(subject_response.status_code)}], no match" - ) diff_reasons.append("code") different_headers = self.compare_headers(self.baseline.headers, subject_response.headers) if different_headers: - log.debug("headers were different, no match") diff_reasons.append("header") if self.compare_body(self.baseline_json, subject_json) is False: - log.debug("difference in HTML body, no match") - diff_reasons.append("body") - if not diff_reasons: - return (True, [], reflection, subject_response) - else: - return (False, diff_reasons, reflection, subject_response) + return diff_reasons async def canary_check(self, url, mode, rounds=3): """ diff --git a/bbot/core/helpers/dns/brute.py b/bbot/core/helpers/dns/brute.py index 71424c5a87..d243073a18 100644 --- a/bbot/core/helpers/dns/brute.py +++ b/bbot/core/helpers/dns/brute.py @@ -49,7 +49,7 @@ async def dnsbrute(self, module, domain, subdomains, type=None): wildcard_domains = await self.parent_helper.dns.is_wildcard_domain(domain, (type, "CNAME")) wildcard_rdtypes = set() - for domain, rdtypes in wildcard_domains.items(): + for wildcard_domain, rdtypes in wildcard_domains.items(): wildcard_rdtypes.update(rdtypes) if wildcard_domains: self.log.hugewarning( @@ -57,8 +57,8 @@ async def dnsbrute(self, module, domain, subdomains, type=None): ) return [] - canaries = self.gen_random_subdomains(self.num_canaries) - canaries_list = list(canaries) + canaries_list = list(self.gen_random_subdomains(self.num_canaries)) + canary_set = set(canaries_list) canaries_pre = canaries_list[: int(self.num_canaries / 2)] canaries_post = canaries_list[int(self.num_canaries / 2) :] # sandwich subdomains between canaries @@ -67,8 +67,8 @@ async def dnsbrute(self, module, domain, subdomains, type=None): results = [] canaries_triggered = [] async for hostname, ip, rdtype in self._massdns(module, domain, subdomains, rdtype=type): - sub = hostname.split(domain)[0] - if sub in canaries: + sub = hostname.split(domain)[0].rstrip(".") + if sub in canary_set: canaries_triggered.append(sub) else: results.append(hostname) diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py index f064edad3b..c9b59e6cc3 100644 --- a/bbot/core/helpers/dns/dns.py +++ b/bbot/core/helpers/dns/dns.py @@ -1,199 +1,469 @@ -import dns +import asyncio import logging -import dns.exception -import dns.asyncresolver -from cachetools import LFUCache +import time +from contextlib import suppress + +from cachetools import LFUCache, LRUCache from radixtarget import RadixTarget -from bbot.errors import DNSError -from bbot.core.engine import EngineClient -from bbot.core.helpers.async_helpers import async_cachedmethod -from ..misc import clean_dns_record, is_ip, is_domain, is_dns_name +from blastdns import Client, ClientConfig, DNSError, DNSResult, MockClient, get_system_resolvers +from blastdns.exceptions import BlastDNSError -from .engine import DNSEngine +from bbot.core.helpers.async_helpers import NamedLock, async_cachedmethod +from .helpers import all_rdtypes, extract_targets, record_to_text +from ..misc import clean_dns_record, domain_parents, is_dns_name, is_domain, is_ip, parent_domain, rand_string log = logging.getLogger("bbot.core.helpers.dns") -class DNSHelper(EngineClient): - SERVER_CLASS = DNSEngine - ERROR_CLASS = DNSError - +class DNSHelper: """Helper class for DNS-related operations within BBOT. - This class provides mechanisms for host resolution, wildcard domain detection, event tagging, and more. - It centralizes all DNS-related activities in BBOT, offering both synchronous and asynchronous methods - for DNS resolution, as well as various utilities for batch resolution and DNS query filtering. + Wraps the blastdns ``Client`` (a Rust-backed async DNS engine) and adds the + BBOT-specific concerns that live above raw resolution: wildcard detection, + per-zone error tracking, connectivity checks, and ``dns_omit_queries`` + filtering. Attributes: parent_helper: A reference to the instantiated `ConfigAwareHelper` (typically `scan.helpers`). - resolver (BBOTAsyncResolver): An asynchronous DNS resolver tailored for BBOT with rate-limiting capabilities. - timeout (int): The timeout value for DNS queries. Defaults to 5 seconds. - retries (int): The number of retries for failed DNS queries. Defaults to 1. - abort_threshold (int): The threshold for aborting after consecutive failed queries. Defaults to 50. - runaway_limit (int): Maximum allowed distance for consecutive DNS resolutions. Defaults to 5. - all_rdtypes (list): A list of DNS record types to be considered during operations. - wildcard_ignore (tuple): Domains to be ignored during wildcard detection. - wildcard_tests (int): Number of tests to be run for wildcard detection. Defaults to 5. - _wildcard_cache (dict): Cache for wildcard detection results. - _dns_cache (LRUCache): Cache for DNS resolution results, limited in size. - resolver_file (Path): File containing system's current resolver nameservers. - - Args: - parent_helper: The parent helper object with configuration details and utilities. - - Raises: - DNSError: If an issue arises when creating the BBOTAsyncResolver instance. - - Examples: - >>> dns_helper = DNSHelper(parent_config) - >>> resolved_host = dns_helper.resolver.resolve("example.com") + blastdns (blastdns.Client): The underlying Rust DNS client. + timeout (int): Per-query timeout in seconds. Defaults to 5. + retries (int): Number of retries for failed DNS queries. Defaults to 5. + abort_threshold (int): Consecutive failed queries per parent before aborting. Defaults to 50. + wildcard_ignore (RadixTarget): Domains to skip during wildcard detection. + wildcard_tests (int): Random subdomains generated per wildcard check. Defaults to 5. + resolver_file (Path): File containing the system's resolver IPs (for tools that need it). """ def __init__(self, parent_helper): + self.log = log self.parent_helper = parent_helper self.config = self.parent_helper.config self.dns_config = self.config.get("dns", {}) - engine_debug = self.config.get("engine", {}).get("debug", False) - super().__init__(server_kwargs={"config": self.config}, debug=engine_debug) - # resolver + # config self.timeout = self.dns_config.get("timeout", 5) - self.resolver = dns.asyncresolver.Resolver() - self.resolver.rotate = True - self.resolver.timeout = self.timeout - self.resolver.lifetime = self.timeout - + self.retries = self.dns_config.get("retries", 5) + self.threads = self.dns_config.get("threads", 5) + self.cache_size = self.dns_config.get("cache_size", 10000) + self.abort_threshold = self.dns_config.get("abort_threshold", 50) + # how many consecutive DNS resolution hops we allow before tagging an event as runaway self.runaway_limit = self.dns_config.get("runaway_limit", 5) + # blastdns client + self.system_resolvers = get_system_resolvers() + self.log.debug( + f"Starting BlastDNS client with {self.threads} threads per resolver, " + f"{self.retries} retries, {self.cache_size} cache size, " + f"and {self.timeout} second timeout" + ) + self.blastdns = Client( + self.system_resolvers, + ClientConfig( + request_timeout_ms=self.timeout * 1000, + max_retries=self.retries, + threads_per_resolver=self.threads, + cache_capacity=self.cache_size, + ), + ) + + # parse dns.omit_queries (e.g. "A:internal.bad.com") into {rdtype: {host, ...}} + self.dns_omit_queries = {} + for entry in self.dns_config.get("omit_queries", None) or []: + parts = entry.split(":") + if len(parts) == 2: + rdtype, host = parts + self.dns_omit_queries.setdefault(rdtype.upper(), set()).add(host.lower()) + # wildcard handling self.wildcard_disable = self.dns_config.get("wildcard_disable", False) + self.wildcard_tests = self.dns_config.get("wildcard_tests", 5) self.wildcard_ignore = RadixTarget() for d in self.dns_config.get("wildcard_ignore", []): self.wildcard_ignore.insert(d) + self._wildcard_cache = LRUCache(maxsize=10000) + self._wildcard_lock = NamedLock() + + # error tracking + connectivity + self._errors = LRUCache(maxsize=10000) + self._dns_warnings = LRUCache(maxsize=10000) + self._dns_connectivity_lock = None + self._last_dns_success = None + self._last_connectivity_warning = time.time() # copy the system's current resolvers to a text file for tool use - self.system_resolvers = dns.resolver.Resolver().nameservers - # TODO: DNS server speed test (start in background task) self.resolver_file = self.parent_helper.tempfile(self.system_resolvers, pipe=False) # brute force helper self._brute = None + # method-level dedup caches for is_wildcard / is_wildcard_domain self._is_wildcard_cache = LFUCache(maxsize=1000) self._is_wildcard_domain_cache = LFUCache(maxsize=1000) - async def resolve(self, query, **kwargs): - return await self.run_and_return("resolve", query=query, **kwargs) + # ------------------------------------------------------------------ + # Resolution -- thin pass-throughs to blastdns. Anything more complex + # belongs in a caller that knows the exact shape it wants. + # ------------------------------------------------------------------ - async def resolve_raw(self, query, **kwargs): - return await self.run_and_return("resolve_raw", query=query, **kwargs) + async def resolve(self, query, rdtype="A"): + """Resolve to a set of rdata strings (e.g. IPs). - async def resolve_batch(self, queries, **kwargs): - agen = self.run_and_yield("resolve_batch", queries=queries, **kwargs) - while 1: - try: - yield await agen.__anext__() - except (StopAsyncIteration, GeneratorExit): - await agen.aclose() - break + Returns an empty set on DNS failure (timeout, SERVFAIL, etc). + """ + try: + return set(await self.blastdns.resolve(query, rdtype)) + except BlastDNSError as e: + self.log.debug(f"DNS error resolving {query}/{rdtype}: {e}") + return set() - async def resolve_raw_batch(self, queries): - agen = self.run_and_yield("resolve_raw_batch", queries=queries) - while 1: - try: - yield await agen.__anext__() - except (StopAsyncIteration, GeneratorExit): - await agen.aclose() - break + async def resolve_full(self, query, rdtype="A"): + """Return blastdns ``DNSResult`` (full response with Record objects). - @property - def brute(self): - if self._brute is None: - from .brute import DNSBrute + Returns an empty-answer DNSResult on DNS failure so callers can + unconditionally iterate ``.response.answers`` without try/except. + """ + try: + return await self.blastdns.resolve_full(query, rdtype) + except BlastDNSError as e: + self.log.debug(f"DNS error resolving {query}/{rdtype}: {e}") + return self._empty_result(query) + + async def resolve_multi_full(self, query, rdtypes): + """Resolve many rdtypes for one host concurrently in Rust. + + Skips rdtypes listed in ``dns_omit_queries`` and rdtypes whose parent + zone has exceeded ``abort_threshold`` consecutive errors. + Returns ``dict[rdtype, DNSResult | DNSError]``. + """ + rdtypes = [r for r in rdtypes if not self._is_omitted(query, r)] + filtered = [] + for r in rdtypes: + if not await self._is_aborted(query, r): + filtered.append(r) + rdtypes = filtered + if not rdtypes: + return {} + results = await self.blastdns.resolve_multi_full(query, rdtypes) + # Track per-zone errors so we can circuit-break dead zones + for rdtype, response in results.items(): + if isinstance(response, DNSError): + self.record_dns_error(query, rdtype) + elif isinstance(response, DNSResult) and response.response.answers: + self.reset_dns_errors(query, rdtype) + return results + + async def resolve_batch_full(self, hosts, rdtype="A", skip_empty=False, skip_errors=False): + """Resolve many hosts for one rdtype concurrently in Rust. + + Yields ``(host, DNSResult | DNSError)``. + """ + async for host, result in self.blastdns.resolve_batch_full( + hosts, rdtype, skip_empty=skip_empty, skip_errors=skip_errors + ): + yield host, result + + def _is_omitted(self, query, rdtype): + omit_hosts = self.dns_omit_queries.get(rdtype.upper()) + if not omit_hosts: + return False + q = str(query).lower() + return any(q == h or q.endswith(f".{h}") for h in omit_hosts) - self._brute = DNSBrute(self.parent_helper) - return self._brute + # ------------------------------------------------------------------ + # Wildcard detection + # ------------------------------------------------------------------ @async_cachedmethod( lambda self: self._is_wildcard_cache, - key=lambda query, rdtypes, raw_dns_records: (query, tuple(sorted(rdtypes)), bool(raw_dns_records)), + key=lambda query, rdtypes, raw_dns_records=None: (query, tuple(sorted(rdtypes)), bool(raw_dns_records)), ) async def is_wildcard(self, query, rdtypes, raw_dns_records=None): """ - Use this method to check whether a *host* is a wildcard entry - - This can reliably tell the difference between a valid DNS record and a wildcard within a wildcard domain. - - If you want to know whether a domain is using wildcard DNS, use `is_wildcard_domain()` instead. + Check whether ``query`` is a wildcard hit within a wildcard domain. Args: - query (str): The hostname to check for a wildcard entry. - ips (list, optional): List of IPs to compare against, typically obtained from a previous DNS resolution of the query. - rdtype (str, optional): The DNS record type (e.g., "A", "AAAA") to consider during the check. + query (str): The hostname to check. + rdtypes (list): DNS record types to consider. + raw_dns_records (dict, optional): ``{rdtype: [Record, ...]}`` already + resolved for this query. If omitted, the records are fetched. Returns: - dict: A dictionary indicating if the query is a wildcard for each checked DNS record type. - Keys are DNS record types like "A", "AAAA", etc. - Values are tuples where the first element is a boolean indicating if the query is a wildcard, - and the second element is the wildcard parent if it's a wildcard. - - Raises: - ValueError: If only one of `ips` or `rdtype` is specified or if no valid IPs are specified. - - Examples: - >>> is_wildcard("www.github.io") - {"A": (True, "github.io"), "AAAA": (True, "github.io")} - - >>> is_wildcard("www.evilcorp.com", ips=["93.184.216.34"], rdtype="A") - {"A": (False, "evilcorp.com")} - - Note: - `is_wildcard` can be True, False, or None (indicating that wildcard detection was inconclusive) + dict: ``{rdtype: (is_wildcard, parent)}`` for each rdtype that resolved. + ``is_wildcard`` may be ``True``, ``False``, ``None``, ``"POSSIBLE"``, or ``"ERROR"``. """ query = self._wildcard_prevalidation(query) if not query: return {} - # skip check if the query is a domain + # skip check if the query is itself a domain if is_domain(query): return {} - return await self.run_and_return("is_wildcard", query=query, rdtypes=rdtypes, raw_dns_records=raw_dns_records) + if isinstance(rdtypes, str): + rdtypes = [rdtypes] + + result = {} + + # if the work of resolving hasn't been done yet, do it + if raw_dns_records is None: + raw_dns_records = {} + multi = await self.resolve_multi_full(query, list(rdtypes)) + for rdtype, response in multi.items(): + if isinstance(response, DNSResult) and response.response.answers: + raw_dns_records[rdtype] = response.response.answers + elif isinstance(response, DNSError): + self.log.debug(f"Failed to resolve {query} ({rdtype}) during wildcard detection: {response.error}") + result[rdtype] = ("ERROR", query) + + # build the baseline (the IPs/hosts we actually got back for this query) + baseline = {} + baseline_raw = {} + for rdtype, answers in raw_dns_records.items(): + for answer in answers: + text_answer = record_to_text(answer) + baseline_raw.setdefault(rdtype, set()).add(text_answer) + for _, host in extract_targets(answer): + baseline.setdefault(rdtype, set()).add(host) + + if not raw_dns_records: + return result + + rdtypes_to_check = set(raw_dns_records) + + # walk parent domains shortest-first, comparing baseline against any wildcard pool + parents = list(domain_parents(query)) + for parent in parents[::-1]: + wildcard_results = await self.is_wildcard_domain(parent, rdtypes_to_check) + + for rdtype in list(baseline_raw): + if rdtype in result: + continue + + _baseline = baseline.get(rdtype, set()) + _baseline_raw = baseline_raw.get(rdtype, set()) + + wildcard_rdtypes = wildcard_results.get(parent, {}) + wildcards = wildcard_rdtypes.get(rdtype) + if wildcards is None: + continue + wildcards, wildcard_raw = wildcards + + if wildcard_raw: + rdtypes_to_check.discard(rdtype) + is_wc = any(r in wildcards for r in _baseline) + is_wc_raw = any(r in wildcard_raw for r in _baseline_raw) + if is_wc or is_wc_raw: + result[rdtype] = (True, parent) + else: + result[rdtype] = ("POSSIBLE", parent) + + for rdtype, answers in baseline_raw.items(): + if answers and rdtype not in result: + result[rdtype] = (False, query) + + return result @async_cachedmethod( - lambda self: self._is_wildcard_domain_cache, key=lambda domain, rdtypes: (domain, tuple(sorted(rdtypes))) + lambda self: self._is_wildcard_domain_cache, + key=lambda domain, rdtypes: (domain, tuple(sorted(rdtypes))), ) async def is_wildcard_domain(self, domain, rdtypes): + """For each parent of ``domain``, return the wildcard pool per rdtype. + + Returns ``{parent: {rdtype: (hosts_set, raw_text_set)}}``. + """ domain = self._wildcard_prevalidation(domain) if not domain: return {} - return await self.run_and_return("is_wildcard_domain", domain=domain, rdtypes=rdtypes) + if isinstance(rdtypes, str): + rdtypes = [rdtypes] + rdtypes = set(rdtypes) + + wildcard_results = {} + # walk parents from shortest (root) to longest, narrowing rdtypes as we find wildcards + for host in list(domain_parents(domain, include_self=True))[::-1]: + host_results = {} + # check each rdtype concurrently for this parent + tasks = [self._is_wildcard_zone(host, rdtype) for rdtype in list(rdtypes)] + if not tasks: + break + for rdtype, (results, results_raw) in zip(list(rdtypes), await asyncio.gather(*tasks)): + if results_raw: + rdtypes.discard(rdtype) + host_results[rdtype] = (results, results_raw) + if host_results: + wildcard_results[host] = host_results + + return wildcard_results + + async def _is_wildcard_zone(self, host, rdtype): + """Test one (host, rdtype) for wildcard configuration. Cached per-pair.""" + rdtype = rdtype.upper() + host_hash = hash((host, rdtype)) + + async with self._wildcard_lock.lock(host_hash): + try: + cached = self._wildcard_cache[host_hash] + self.log.debug(f"Got {host}:{rdtype} from wildcard cache") + return cached + except KeyError: + pass + + self.log.debug(f"Checking if {host}:{rdtype} is a wildcard") + results = set() + results_raw = set() + + rand_hosts = [f"{rand_string(digits=False, length=10)}.{host}" for _ in range(self.wildcard_tests)] + async for _, response in self.resolve_batch_full(rand_hosts, rdtype): + if not isinstance(response, DNSResult): + continue + for answer in response.response.answers: + results_raw.add(record_to_text(answer)) + for _, t in extract_targets(answer): + results.add(t) + + if results: + self.log.info(f"Encountered domain with wildcard DNS ({rdtype}): *.{host}") + else: + self.log.debug(f"Finished checking {host}:{rdtype}, it is not a wildcard") + + self._wildcard_cache[host_hash] = (results, results_raw) + return results, results_raw def _wildcard_prevalidation(self, host): if self.wildcard_disable: return False host = clean_dns_record(host) - # skip check if it's an IP or a plain hostname if is_ip(host) or "." not in host: return False - - # skip if query isn't a dns name if not is_dns_name(host): return False - # skip check if the query's parent domain is excluded in the config wildcard_ignore = self.wildcard_ignore.search(host) if wildcard_ignore: - log.debug(f"Skipping wildcard detection on {host} because {wildcard_ignore} is excluded in the config") + self.log.debug( + f"Skipping wildcard detection on {host} because {wildcard_ignore} is excluded in the config" + ) return False return host - async def _mock_dns(self, mock_data, custom_lookup_fn=None): - from .mock import MockResolver + # ------------------------------------------------------------------ + # Error tracking + connectivity + # ------------------------------------------------------------------ + + async def _is_aborted(self, query, rdtype): + """Check if queries for this parent zone + rdtype have been circuit-broken. + + Only triggers on sustained timeouts (DNSError), not instant failures + like NXDOMAIN or SERVFAIL. When the threshold is hit, verifies + network connectivity first — if the network is down, clears error + counters instead of aborting (the zone might be fine). + """ + parent = parent_domain(str(query)) + parent_hash = hash((parent, rdtype)) + error_count = self._errors.get(parent_hash, 0) + if error_count >= self.abort_threshold: + # before aborting, make sure our network is actually up + connectivity = await self._connectivity_check() + if not connectivity: + # network is down — don't blame the zone + self._errors.clear() + return False + if parent_hash not in self._dns_warnings: + self.log.info( + f'Aborting {rdtype} queries to "{parent}" — ' + f"{error_count} consecutive errors exceeded threshold ({self.abort_threshold})" + ) + self._dns_warnings[parent_hash] = True + return True + return False + + def record_dns_error(self, query, rdtype): + """Bump the error counter for ``query``'s parent zone. Returns the new count.""" + parent_hash = hash((parent_domain(str(query)), rdtype)) + self._errors[parent_hash] = self._errors.get(parent_hash, 0) + 1 + return self._errors[parent_hash] + + def reset_dns_errors(self, query, rdtype): + parent_hash = hash((parent_domain(str(query)), rdtype)) + if parent_hash in self._errors: + self._errors[parent_hash] = 0 + + @property + def dns_connectivity_lock(self): + if self._dns_connectivity_lock is None: + self._dns_connectivity_lock = asyncio.Lock() + return self._dns_connectivity_lock + + async def _connectivity_check(self, interval=5): + """Confirm the network can reach DNS. Cached for ``interval`` seconds.""" + if self._last_dns_success is not None and time.time() - self._last_dns_success < interval: + return True + + async with self.dns_connectivity_lock: + with suppress(Exception): + answers = await self.blastdns.resolve("www.google.com", "A") + if answers: + self._last_dns_success = time.time() + return True + + if time.time() - self._last_connectivity_warning > interval: + self.log.error("DNS queries are failing, please check your internet connection") + self._last_connectivity_warning = time.time() + self._errors.clear() + return False + + # ------------------------------------------------------------------ + # Brute / mock helpers + # ------------------------------------------------------------------ + + @property + def brute(self): + if self._brute is None: + from .brute import DNSBrute + + self._brute = DNSBrute(self.parent_helper) + return self._brute - self.resolver = MockResolver(mock_data, custom_lookup_fn=custom_lookup_fn) - await self.run_and_return("_mock_dns", mock_data=mock_data, custom_lookup_fn=custom_lookup_fn) + async def _mock_dns(self, mock_data): + """Swap the underlying client for a ``MockClient`` configured with ``mock_data``.""" + mock_client = MockClient() + mock_client.mock_dns(mock_data) + self.blastdns = mock_client + + @staticmethod + def _empty_result(host=""): + """Build a minimal ``DNSResult`` with no answers, for use as a safe fallback.""" + from blastdns.models import Header, Response + + header = Header( + id=0, + message_type="Response", + op_code="Query", + authoritative=False, + truncation=False, + recursion_desired=True, + recursion_available=True, + authentic_data=False, + checking_disabled=False, + response_code="NoError", + query_count=0, + answer_count=0, + name_server_count=0, + additional_count=0, + ) + return DNSResult( + host=host, response=Response(header=header, queries=[], answers=[], name_servers=[], additionals=[]) + ) + + async def shutdown(self): + """No-op kept for API compatibility -- blastdns runs in-process, nothing to tear down.""" + return None + + +# Re-export for convenience +__all__ = ["DNSHelper", "all_rdtypes", "extract_targets", "record_to_text"] diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py deleted file mode 100644 index 1c98f9a9e3..0000000000 --- a/bbot/core/helpers/dns/engine.py +++ /dev/null @@ -1,663 +0,0 @@ -import os -import dns -import time -import asyncio -import logging -import traceback -from cachetools import LRUCache -from contextlib import suppress - -from bbot.core.engine import EngineServer -from bbot.core.helpers.async_helpers import NamedLock -from bbot.core.helpers.dns.helpers import extract_targets -from bbot.core.helpers.misc import ( - is_ip, - rand_string, - parent_domain, - domain_parents, -) - - -log = logging.getLogger("bbot.core.helpers.dns.engine.server") - -all_rdtypes = ["A", "AAAA", "SRV", "MX", "NS", "SOA", "CNAME", "TXT"] - - -class DNSEngine(EngineServer): - CMDS = { - 0: "resolve", - 1: "resolve_raw", - 2: "resolve_batch", - 3: "resolve_raw_batch", - 4: "is_wildcard", - 5: "is_wildcard_domain", - 99: "_mock_dns", - } - - def __init__(self, socket_path, config={}, debug=False): - super().__init__(socket_path, debug=debug) - - self.config = config - self.dns_config = self.config.get("dns", {}) - # config values - self.timeout = self.dns_config.get("timeout", 5) - self.retries = self.dns_config.get("retries", 1) - self.abort_threshold = self.dns_config.get("abort_threshold", 50) - - # resolver - self.resolver = dns.asyncresolver.Resolver() - self.resolver.rotate = True - self.resolver.timeout = self.timeout - self.resolver.lifetime = self.timeout - - # skip certain queries - dns_omit_queries = self.dns_config.get("omit_queries", None) - if not dns_omit_queries: - dns_omit_queries = [] - self.dns_omit_queries = {} - for d in dns_omit_queries: - d = d.split(":") - if len(d) == 2: - rdtype, query = d - rdtype = rdtype.upper() - query = query.lower() - try: - self.dns_omit_queries[rdtype].add(query) - except KeyError: - self.dns_omit_queries[rdtype] = {query} - - # wildcard handling - self.wildcard_ignore = self.dns_config.get("wildcard_ignore", None) - if not self.wildcard_ignore: - self.wildcard_ignore = [] - self.wildcard_ignore = tuple([str(d).strip().lower() for d in self.wildcard_ignore]) - self.wildcard_tests = self.dns_config.get("wildcard_tests", 5) - self._wildcard_cache = {} - # since wildcard detection takes some time, This is to prevent multiple - # modules from kicking off wildcard detection for the same domain at the same time - self._wildcard_lock = NamedLock() - - self._dns_connectivity_lock = None - self._last_dns_success = None - self._last_connectivity_warning = time.time() - # keeps track of warnings issued for wildcard detection to prevent duplicate warnings - self._dns_warnings = set() - self._errors = {} - self._debug = self.dns_config.get("debug", False) - self._dns_cache = LRUCache(maxsize=10000) - - async def resolve(self, query, **kwargs): - """Resolve DNS names and IP addresses to their corresponding results. - - This is a high-level function that can translate a given domain name to its associated IP addresses - or an IP address to its corresponding domain names. It's structured for ease of use within modules - and will abstract away most of the complexity of DNS resolution, returning a simple set of results. - - Args: - query (str): The domain name or IP address to resolve. - **kwargs: Additional arguments to be passed to the resolution process. - - Returns: - set: A set containing resolved domain names or IP addresses. - - Examples: - >>> results = await resolve("1.2.3.4") - {"evilcorp.com"} - - >>> results = await resolve("evilcorp.com") - {"1.2.3.4", "dead::beef"} - """ - results = set() - try: - answers, errors = await self.resolve_raw(query, **kwargs) - for answer in answers: - for _, host in extract_targets(answer): - results.add(host) - except BaseException: - self.log.trace(f"Caught exception in resolve({query}, {kwargs}):") - self.log.trace(traceback.format_exc()) - raise - - self.debug(f"Results for {query} with kwargs={kwargs}: {results}") - return results - - async def resolve_raw(self, query, **kwargs): - """Resolves the given query to its associated DNS records. - - This function is a foundational method for DNS resolution in this class. It understands both IP addresses and - hostnames and returns their associated records in a raw format provided by the dnspython library. - - Args: - query (str): The IP address or hostname to resolve. - type (str or list[str], optional): Specifies the DNS record type(s) to fetch. Can be a single type like 'A' - or a list like ['A', 'AAAA']. If set to 'any', 'all', or '*', it fetches all supported types. If not - specified, the function defaults to fetching 'A' and 'AAAA' records. - **kwargs: Additional arguments that might be passed to the resolver. - - Returns: - tuple: A tuple containing two lists: - - list: A list of tuples where each tuple consists of a record type string (like 'A') and the associated - raw dnspython answer. - - list: A list of tuples where each tuple consists of a record type string and the associated error if - there was an issue fetching the record. - - Examples: - >>> await resolve_raw("8.8.8.8") - ([('PTR', )], []) - - >>> await resolve_raw("dns.google") - (, []) - """ - # DNS over TCP is more reliable - # But setting this breaks DNS resolution on Ubuntu because systemd-resolve doesn't support TCP - # kwargs["tcp"] = True - try: - query = str(query).strip() - kwargs.pop("rdtype", None) - rdtype = kwargs.pop("type", "A") - if is_ip(query): - return await self._resolve_ip(query, **kwargs) - else: - return await self._resolve_hostname(query, rdtype=rdtype, **kwargs) - except BaseException: - self.log.trace(f"Caught exception in resolve_raw({query}, {kwargs}):") - self.log.trace(traceback.format_exc()) - raise - - async def _resolve_hostname(self, query, **kwargs): - """Translate a hostname into its corresponding IP addresses. - - This is the foundational function for converting a domain name into its associated IP addresses. It's designed - for internal use within the class and handles retries, caching, and a variety of error/timeout scenarios. - It also respects certain configurations that might ask to skip certain types of queries. Results are returned - in the default dnspython answer object format. - - Args: - query (str): The hostname to resolve. - rdtype (str, optional): The type of DNS record to query (e.g., 'A', 'AAAA'). Defaults to 'A'. - retries (int, optional): The number of times to retry on failure. Defaults to class-wide `retries`. - use_cache (bool, optional): Whether to check the cache before trying a fresh resolution. Defaults to True. - **kwargs: Additional arguments that might be passed to the resolver. - - Returns: - tuple: A tuple containing: - - list: A list of resolved IP addresses. - - list: A list of errors encountered during the resolution process. - - Examples: - >>> results, errors = await _resolve_hostname("google.com") - (, []) - """ - self.debug(f"Resolving {query} with kwargs={kwargs}") - results = [] - errors = [] - rdtype = kwargs.get("rdtype", "A") - - # skip certain queries if requested - if rdtype in self.dns_omit_queries: - if any(h == query or query.endswith(f".{h}") for h in self.dns_omit_queries[rdtype]): - self.debug(f"Skipping {rdtype}:{query} because it's omitted in the config") - return results, errors - - parent = parent_domain(query) - retries = kwargs.pop("retries", self.retries) - use_cache = kwargs.pop("use_cache", True) - tries_left = int(retries) + 1 - parent_hash = hash((parent, rdtype)) - dns_cache_hash = hash((query, rdtype)) - while tries_left > 0: - try: - if use_cache: - results = self._dns_cache.get(dns_cache_hash, []) - if not results: - error_count = self._errors.get(parent_hash, 0) - if error_count >= self.abort_threshold: - connectivity = await self._connectivity_check() - if connectivity: - self.log.verbose( - f'Aborting query "{query}" because failed {rdtype} queries for "{parent}" ({error_count:,}) exceeded abort threshold ({self.abort_threshold:,})' - ) - if parent_hash not in self._dns_warnings: - self.log.verbose( - f'Aborting future {rdtype} queries to "{parent}" because error count ({error_count:,}) exceeded abort threshold ({self.abort_threshold:,})' - ) - self._dns_warnings.add(parent_hash) - return results, errors - results = await self._catch(self.resolver.resolve, query, **kwargs) - if use_cache: - self._dns_cache[dns_cache_hash] = results - if parent_hash in self._errors: - self._errors[parent_hash] = 0 - break - except ( - dns.resolver.NoNameservers, - dns.exception.Timeout, - dns.resolver.LifetimeTimeout, - TimeoutError, - asyncio.exceptions.TimeoutError, - ) as e: - try: - self._errors[parent_hash] += 1 - except KeyError: - self._errors[parent_hash] = 1 - errors.append(e) - # don't retry if we get a SERVFAIL - if isinstance(e, dns.resolver.NoNameservers): - break - tries_left -= 1 - err_msg = ( - f'DNS error or timeout for {rdtype} query "{query}" ({self._errors[parent_hash]:,} so far): {e}' - ) - if tries_left > 0: - retry_num = (retries + 1) - tries_left - self.debug(err_msg) - self.debug(f"Retry (#{retry_num}) resolving {query} with kwargs={kwargs}") - else: - self.log.verbose(err_msg) - - if results: - self._last_dns_success = time.time() - self.debug(f"Answers for {query} with kwargs={kwargs}: {list(results)}") - - if errors: - self.debug(f"Errors for {query} with kwargs={kwargs}: {errors}") - - return results, errors - - async def _resolve_ip(self, query, **kwargs): - """Translate an IP address into a corresponding DNS name. - - This is the most basic function that will convert an IP address into its associated domain name. It handles - retries, caching, and multiple types of timeout/error scenarios internally. The function is intended for - internal use and should not be directly called by modules without understanding its intricacies. - - Args: - query (str): The IP address to be reverse-resolved. - retries (int, optional): The number of times to retry on failure. Defaults to 0. - use_cache (bool, optional): Whether to check the cache for the result before attempting resolution. Defaults to True. - **kwargs: Additional arguments to be passed to the resolution process. - - Returns: - tuple: A tuple containing: - - list: A list of resolved domain names (in default dnspython answer format). - - list: A list of errors encountered during resolution. - - Examples: - >>> results, errors = await _resolve_ip("8.8.8.8") - (, []) - """ - self.debug(f"Reverse-resolving {query} with kwargs={kwargs}") - retries = kwargs.pop("retries", 0) - use_cache = kwargs.pop("use_cache", True) - tries_left = int(retries) + 1 - results = [] - errors = [] - dns_cache_hash = hash((query, "PTR")) - while tries_left > 0: - try: - if use_cache: - results = self._dns_cache.get(dns_cache_hash, []) - if not results: - results = await self._catch(self.resolver.resolve_address, query, **kwargs) - if use_cache: - self._dns_cache[dns_cache_hash] = results - break - except ( - dns.resolver.NoNameservers, - dns.exception.Timeout, - dns.resolver.LifetimeTimeout, - TimeoutError, - asyncio.exceptions.TimeoutError, - ) as e: - errors.append(e) - # don't retry if we get a SERVFAIL - if isinstance(e, dns.resolver.NoNameservers): - self.debug(f"{e} (query={query}, kwargs={kwargs})") - break - else: - tries_left -= 1 - if tries_left > 0: - retry_num = (retries + 2) - tries_left - self.debug(f"Retrying (#{retry_num}) {query} with kwargs={kwargs}") - - if results: - self._last_dns_success = time.time() - - return results, errors - - async def resolve_batch(self, queries, threads=10, **kwargs): - """ - A helper to execute a bunch of DNS requests. - - Args: - queries (list): List of queries to resolve. - **kwargs: Additional keyword arguments to pass to `resolve()`. - - Yields: - tuple: A tuple containing the original query and its resolved value. - - Examples: - >>> import asyncio - >>> async def example_usage(): - ... async for result in resolve_batch(['www.evilcorp.com', 'evilcorp.com']): - ... print(result) - ('www.evilcorp.com', {'1.1.1.1'}) - ('evilcorp.com', {'2.2.2.2'}) - """ - async for (args, _, _), responses in self.task_pool( - self.resolve, args_kwargs=queries, threads=threads, global_kwargs=kwargs - ): - yield args[0], responses - - async def resolve_raw_batch(self, queries, threads=10, **kwargs): - queries_kwargs = [[q[0], {"type": q[1]}] for q in queries] - async for (args, kwargs, _), (answers, errors) in self.task_pool( - self.resolve_raw, args_kwargs=queries_kwargs, threads=threads, global_kwargs=kwargs - ): - query = args[0] - rdtype = kwargs["type"] - yield ((query, rdtype), (answers, errors)) - - async def _catch(self, callback, *args, **kwargs): - """ - Asynchronously catches exceptions thrown during DNS resolution and logs them. - - This method wraps around a given asynchronous callback function to handle different - types of DNS exceptions and general exceptions. It logs the exceptions for debugging - and, in some cases, re-raises them. - - Args: - callback (callable): The asynchronous function to be executed. - *args: Positional arguments to pass to the callback. - **kwargs: Keyword arguments to pass to the callback. - - Returns: - Any: The return value of the callback function, or an empty list if an exception is caught. - - Raises: - dns.resolver.NoNameservers: When no nameservers could be reached. - """ - try: - return await callback(*args, **kwargs) - except dns.resolver.NoNameservers: - raise - except (dns.exception.Timeout, dns.resolver.LifetimeTimeout, TimeoutError): - self.log.debug(f"DNS query with args={args}, kwargs={kwargs} timed out after {self.timeout} seconds") - raise - except dns.exception.DNSException as e: - self.debug(f"{e} (args={args}, kwargs={kwargs})") - except Exception as e: - self.log.warning(f"Error in {callback.__qualname__}() with args={args}, kwargs={kwargs}: {e}") - self.log.trace(traceback.format_exc()) - return [] - - async def is_wildcard(self, query, rdtypes, raw_dns_records=None): - """ - Use this method to check whether a *host* is a wildcard entry - - This can reliably tell the difference between a valid DNS record and a wildcard within a wildcard domain. - - It works by making a bunch of random DNS queries to the parent domain, compiling a list of wildcard IPs, - then comparing those to the IPs of the host in question. If the host's IP matches the wildcard ones, it's a wildcard. - - If you want to know whether a domain is using wildcard DNS, use `is_wildcard_domain()` instead. - - Args: - query (str): The hostname to check for a wildcard entry. - rdtypes (list): The DNS record type (e.g., "A", "AAAA") to consider during the check. - raw_dns_records (dict, optional): Dictionary of {rdtype: [answer1, answer2, ...], ...} containing raw dnspython answers for the query. - - Returns: - dict: A dictionary indicating if the query is a wildcard for each checked DNS record type. - Keys are DNS record types like "A", "AAAA", etc. - Values are tuples where the first element is a boolean indicating if the query is a wildcard, - and the second element is the wildcard parent if it's a wildcard. - - Examples: - >>> is_wildcard("www.github.io", rdtypes=["A", "AAAA", "MX"]) - {"A": (True, "github.io"), "AAAA": (True, "github.io"), "MX": (False, "github.io")} - - >>> is_wildcard("www.evilcorp.com", rdtypes=["A"]) - {"A": (False, "evilcorp.com")} - - Note: - `is_wildcard` can be True, False, or None (indicating that wildcard detection was inconclusive) - """ - if isinstance(rdtypes, str): - rdtypes = [rdtypes] - - result = {} - - # if the work of resolving hasn't been done yet, do it - if raw_dns_records is None: - raw_dns_records = {} - queries = [(query, rdtype) for rdtype in rdtypes] - async for (_, rdtype), (answers, errors) in self.resolve_raw_batch(queries): - if answers: - for answer in answers: - try: - raw_dns_records[rdtype].add(answer) - except KeyError: - raw_dns_records[rdtype] = {answer} - else: - if errors: - self.debug(f"Failed to resolve {query} ({rdtype}) during wildcard detection") - result[rdtype] = ("ERROR", query) - - # clean + process the raw records into a baseline - baseline = {} - baseline_raw = {} - for rdtype, answers in raw_dns_records.items(): - for answer in answers: - text_answer = answer.to_text() - try: - baseline_raw[rdtype].add(text_answer) - except KeyError: - baseline_raw[rdtype] = {text_answer} - for _, host in extract_targets(answer): - try: - baseline[rdtype].add(host) - except KeyError: - baseline[rdtype] = {host} - - # if it's unresolved, it's a big nope - if not raw_dns_records: - return result - - # once we've resolved the base query and have IP addresses to work with - # we can compare the IPs to the ones we have on file for wildcards - - # only bother to check the rdypes that actually resolve - rdtypes_to_check = set(raw_dns_records) - - # for every parent domain, starting with the shortest - parents = list(domain_parents(query)) - for parent in parents[::-1]: - # check if the parent domain is set up with wildcards - wildcard_results = await self.is_wildcard_domain(parent, rdtypes_to_check) - - # for every rdtype - for rdtype in list(baseline_raw): - # skip if we already found a wildcard for this rdtype - if rdtype in result: - continue - - # get our baseline IPs from above - _baseline = baseline.get(rdtype, set()) - _baseline_raw = baseline_raw.get(rdtype, set()) - - wildcard_rdtypes = wildcard_results.get(parent, {}) - wildcards = wildcard_rdtypes.get(rdtype, None) - if wildcards is None: - continue - wildcards, wildcard_raw = wildcards - - if wildcard_raw: - # skip this rdtype from now on - rdtypes_to_check.remove(rdtype) - - # check if any of our baseline IPs are in the wildcard results - is_wildcard = any(r in wildcards for r in _baseline) - is_wildcard_raw = any(r in wildcard_raw for r in _baseline_raw) - - # if there are any matches, we have a wildcard - if is_wildcard or is_wildcard_raw: - result[rdtype] = (True, parent) - else: - # otherwise, it's still suspicious, because we had random stuff resolve at this level - result[rdtype] = ("POSSIBLE", parent) - - # any rdtype that wasn't a wildcard, mark it as False - for rdtype, answers in baseline_raw.items(): - if answers and rdtype not in result: - result[rdtype] = (False, query) - - return result - - async def is_wildcard_domain(self, domain, rdtypes): - """ - Check whether a given host or its children make use of wildcard DNS entries. Wildcard DNS can have - various implications, particularly in subdomain enumeration and subdomain takeovers. - - Args: - domain (str): The domain to check for wildcard DNS entries. - rdtypes (list): Which DNS record types to check. - - Returns: - dict: A dictionary where the keys are the parent domains that have wildcard DNS entries, - and the values are another dictionary of DNS record types ("A", "AAAA", etc.) mapped to - sets of their resolved IP addresses. - - Examples: - >>> is_wildcard_domain("github.io") - {"github.io": {"A": {"1.2.3.4"}, "AAAA": {"dead::beef"}}} - - >>> is_wildcard_domain("example.com") - {} - """ - if isinstance(rdtypes, str): - rdtypes = [rdtypes] - rdtypes = set(rdtypes) - - wildcard_results = {} - # make a list of its parents - parents = list(domain_parents(domain, include_self=True)) - # and check each of them, beginning with the highest parent (i.e. the root domain) - for i, host in enumerate(parents[::-1]): - host_results = {} - queries = [((host, rdtype), {}) for rdtype in rdtypes] - async for ((_, rdtype), _, _), (results, results_raw) in self.task_pool( - self._is_wildcard_zone, args_kwargs=queries - ): - # if we hit a wildcard, we can skip this rdtype from now on - if results_raw: - rdtypes.remove(rdtype) - host_results[rdtype] = results, results_raw - - if host_results: - wildcard_results[host] = host_results - - return wildcard_results - - async def _is_wildcard_zone(self, host, rdtype): - """ - Check whether a specific DNS zone+rdtype has a wildcard configuration - """ - rdtype = rdtype.upper() - - # have we checked this host before? - host_hash = hash((host, rdtype)) - async with self._wildcard_lock.lock(host_hash): - # if we've seen this host before - try: - wildcard_results, wildcard_results_raw = self._wildcard_cache[host_hash] - self.debug(f"Got {host}:{rdtype} from cache") - except KeyError: - wildcard_results = set() - wildcard_results_raw = set() - self.debug(f"Checking if {host}:{rdtype} is a wildcard") - - # determine if this is a wildcard domain - # resolve a bunch of random subdomains of the same parent - rand_queries = [] - for _ in range(self.wildcard_tests): - rand_query = f"{rand_string(digits=False, length=10)}.{host}" - rand_queries.append((rand_query, rdtype)) - - async for (query, rdtype), (answers, errors) in self.resolve_raw_batch(rand_queries, use_cache=False): - for answer in answers: - # consider both the raw record - wildcard_results_raw.add(answer.to_text()) - # and all the extracted hosts - for _, t in extract_targets(answer): - wildcard_results.add(t) - - if wildcard_results: - self.log.info(f"Encountered domain with wildcard DNS ({rdtype}): *.{host}") - else: - self.debug(f"Finished checking {host}:{rdtype}, it is not a wildcard") - self._wildcard_cache[host_hash] = wildcard_results, wildcard_results_raw - - return wildcard_results, wildcard_results_raw - - async def _is_wildcard(self, query, rdtypes, dns_children): - if isinstance(rdtypes, str): - rdtypes = [rdtypes] - - @property - def dns_connectivity_lock(self): - if self._dns_connectivity_lock is None: - self._dns_connectivity_lock = asyncio.Lock() - return self._dns_connectivity_lock - - async def _connectivity_check(self, interval=5): - """ - Periodically checks for an active internet connection by attempting DNS resolution. - - Args: - interval (int, optional): The time interval, in seconds, at which to perform the check. - Defaults to 5 seconds. - - Returns: - bool: True if there is an active internet connection, False otherwise. - - Examples: - >>> await _connectivity_check() - True - """ - if self._last_dns_success is not None: - if time.time() - self._last_dns_success < interval: - return True - dns_server_working = [] - async with self.dns_connectivity_lock: - with suppress(Exception): - dns_server_working = await self._catch(self.resolver.resolve, "www.google.com", rdtype="A") - if dns_server_working: - self._last_dns_success = time.time() - return True - if time.time() - self._last_connectivity_warning > interval: - self.log.warning("DNS queries are failing, please check your internet connection") - self._last_connectivity_warning = time.time() - self._errors.clear() - return False - - def debug(self, *args, **kwargs): - if self._debug: - self.log.trace(*args, **kwargs) - - @property - def in_tests(self): - return os.getenv("BBOT_TESTING", "") == "True" - - async def _mock_dns(self, mock_data, custom_lookup_fn=None): - from .mock import MockResolver - - def deserialize_function(func_source): - assert self.in_tests, "Can only mock when BBOT_TESTING=True" - if func_source is None: - return None - namespace = {} - exec(func_source, {}, namespace) - return namespace["custom_lookup"] - - self.resolver = MockResolver(mock_data, custom_lookup_fn=deserialize_function(custom_lookup_fn)) diff --git a/bbot/core/helpers/dns/helpers.py b/bbot/core/helpers/dns/helpers.py index 340af5a425..d741ea3f36 100644 --- a/bbot/core/helpers/dns/helpers.py +++ b/bbot/core/helpers/dns/helpers.py @@ -1,11 +1,83 @@ import logging -from bbot.core.helpers.regexes import dns_name_extraction_regex -from bbot.core.helpers.misc import clean_dns_record, smart_decode +from bbot.core.helpers.regexes import ( + dns_name_extraction_regex, + ip_range_regexes, + ipv4_regex, + ipv6_regex, + spf_ip_mechanism_regex, +) +from bbot.core.helpers.misc import clean_dns_record log = logging.getLogger("bbot.core.helpers.dns") +# Default rdtypes BBOT cares about during recursive resolution +all_rdtypes = ["A", "AAAA", "SRV", "MX", "NS", "SOA", "CNAME", "TXT"] + + +def extract_targets(record): + """Hostnames/IPs worth following from a blastdns ``Record``. + + For structured rdata (A/AAAA/CNAME/NS/PTR/MX/SOA/SRV/etc), blastdns has + already extracted the embedded names in Rust -- we just hand those back. + + For TXT records we additionally apply a hostname regex to the text content, + since SPF / DKIM / similar TXT payloads commonly embed hostnames. SPF records + (v=spf1) also get the IPs and CIDR ranges from their ip4:/ip6: mechanisms + extracted; SPF macros are skipped since they are evaluation-time templates, + not literal targets. That regex extraction is BBOT-specific and stays here. + """ + results = set() + for rdtype, host in record.extract_targets(): + cleaned = clean_dns_record(host) + if cleaned: + results.add((rdtype, cleaned)) + + # TXT: pull additional targets out of the free-form text content + is_txt = "TXT" in record.rdata + if is_txt: + text = record.to_text() + if "v=spf1" in text.lower(): + # SPF (RFC 7208): ip4:/ip6: mechanisms embed IPs and CIDR ranges + for token in text.split(): + # macros (e.g. "exists:%{i}.evilcorp.com") are evaluation-time templates + if "%" in token: + continue + mechanism = spf_ip_mechanism_regex.match(token) + if mechanism: + value = token[mechanism.end() :] + if ( + any(r.fullmatch(value) for r in ip_range_regexes) + or ipv4_regex.fullmatch(value) + or ipv6_regex.fullmatch(value) + ): + results.add(("TXT", value)) + continue + # hostnames, e.g. "include:cloudprovider.com", "redirect=_spf.example.com" + for match in dns_name_extraction_regex.finditer(token): + cleaned = clean_dns_record(token[match.start() : match.end()]) + if cleaned: + results.add(("TXT", cleaned)) + else: + # non-SPF TXT (DKIM, verification strings, etc.): hostnames only + for match in dns_name_extraction_regex.finditer(text): + cleaned = clean_dns_record(text[match.start() : match.end()]) + if cleaned: + results.add(("TXT", cleaned)) + + return results + + +def record_to_text(record): + """Presentation-format text for a blastdns ``Record``. + + Equivalent to dnspython's ``answer.to_text()``. blastdns produces this on + the Rust side via hickory's ``Display`` impl, so this is a thin pass-through. + """ + return record.to_text() + + # the following are the result of a 1-day internet survey to find the top SRV records # the scan resulted in 36,282 SRV records. the count for each one is shown. common_srvs = [ @@ -154,61 +226,6 @@ ] -def extract_targets(record): - """ - Extracts hostnames or IP addresses from a given DNS record. - - This method reads the DNS record's type and based on that, extracts the target - hostnames or IP addresses it points to. The type of DNS record - (e.g., "A", "MX", "CNAME", etc.) determines which fields are used for extraction. - - Args: - record (dns.rdata.Rdata): The DNS record to extract information from. - - Returns: - set: A set of tuples, each containing the DNS record type and the extracted value. - - Examples: - >>> from dns.rrset import from_text - >>> record = from_text('www.example.com', 3600, 'IN', 'A', '192.0.2.1') - >>> extract_targets(record[0]) - {('A', '192.0.2.1')} - - >>> record = from_text('example.com', 3600, 'IN', 'MX', '10 mail.example.com.') - >>> extract_targets(record[0]) - {('MX', 'mail.example.com')} - - """ - results = set() - - def add_result(rdtype, _record): - cleaned = clean_dns_record(_record) - if cleaned: - results.add((rdtype, cleaned)) - - rdtype = str(record.rdtype.name).upper() - if rdtype in ("A", "AAAA", "NS", "CNAME", "PTR"): - add_result(rdtype, record) - elif rdtype == "SOA": - add_result(rdtype, record.mname) - elif rdtype == "MX": - add_result(rdtype, record.exchange) - elif rdtype == "SRV": - add_result(rdtype, record.target) - elif rdtype == "TXT": - for s in record.strings: - s = smart_decode(s) - for match in dns_name_extraction_regex.finditer(s): - start, end = match.span() - host = s[start:end] - add_result(rdtype, host) - elif rdtype == "NSEC": - add_result(rdtype, record.next) - else: - log.warning(f'Unknown DNS record type "{rdtype}"') - return results - - def service_record(host, rdtype=None): """ Indicates that the provided host name and optional rdtype is an SRV or related service record. diff --git a/bbot/core/helpers/dns/mock.py b/bbot/core/helpers/dns/mock.py deleted file mode 100644 index 3f6fd83ea5..0000000000 --- a/bbot/core/helpers/dns/mock.py +++ /dev/null @@ -1,74 +0,0 @@ -import dns -import logging - -log = logging.getLogger("bbot.core.helpers.dns.mock") - - -class MockResolver: - def __init__(self, mock_data=None, custom_lookup_fn=None): - self.mock_data = mock_data if mock_data else {} - self._custom_lookup_fn = custom_lookup_fn - self.nameservers = ["127.0.0.1"] - - async def resolve_address(self, ipaddr, *args, **kwargs): - modified_kwargs = {} - modified_kwargs.update(kwargs) - modified_kwargs["rdtype"] = "PTR" - return await self.resolve(str(dns.reversename.from_address(ipaddr)), *args, **modified_kwargs) - - def _lookup(self, query, rdtype): - query = query.strip(".") - ret = [] - if self._custom_lookup_fn is not None: - answers = self._custom_lookup_fn(query, rdtype) - if answers is not None: - ret.extend(list(answers)) - answers = self.mock_data.get(query, {}).get(rdtype, []) - if answers: - ret.extend(list(answers)) - if not ret: - raise dns.resolver.NXDOMAIN(f"No answer found for {query} {rdtype}") - return ret - - def create_dns_response(self, query_name, answers, rdtype): - query_name = query_name.strip(".") - message_text = f"""id 1234 -opcode QUERY -rcode NOERROR -flags QR AA RD -;QUESTION -{query_name}. IN {rdtype} -;ANSWER""" - for answer in answers: - if answer == "": - answer = '""' - message_text += f"\n{query_name}. 1 IN {rdtype} {answer}" - - message_text += "\n;AUTHORITY\n;ADDITIONAL\n" - message = dns.message.from_text(message_text) - # log.verbose(message_text) - return message - - async def resolve(self, query_name, rdtype=None): - if rdtype is None: - rdtype = "A" - elif isinstance(rdtype, str): - rdtype = rdtype.upper() - else: - rdtype = str(rdtype.name).upper() - - domain_name = dns.name.from_text(query_name) - rdtype_obj = dns.rdatatype.from_text(rdtype) - - if "_NXDOMAIN" in self.mock_data and query_name in self.mock_data["_NXDOMAIN"]: - # Simulate the NXDOMAIN exception - raise dns.resolver.NXDOMAIN - - try: - answers = self._lookup(query_name, rdtype) - log.verbose(f"Answers for {query_name}:{rdtype}: {answers}") - response = self.create_dns_response(query_name, answers, rdtype) - answer = dns.resolver.Answer(domain_name, rdtype_obj, dns.rdataclass.IN, response) - return answer - except dns.resolver.NXDOMAIN: - return [] diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py index 1f5762214e..f3d8e8064f 100644 --- a/bbot/core/helpers/helper.py +++ b/bbot/core/helpers/helper.py @@ -1,11 +1,17 @@ import os +import sys +import signal +import ctypes +import ctypes.util +import asyncio import logging from pathlib import Path import multiprocessing as mp from functools import partial -from concurrent.futures import ProcessPoolExecutor +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor from . import misc +from .asn import ASNHelper from .dns import DNSHelper from .web import WebHelper from .diff import HttpCompare @@ -13,6 +19,7 @@ from .wordcloud import WordCloud from .interactsh import Interactsh from .yara_helper import YaraHelper +from .simhash import SimHashHelper from .depsinstaller import DepsInstaller from .async_helpers import get_event_loop @@ -20,6 +27,24 @@ log = logging.getLogger("bbot.core.helpers") +_PR_SET_PDEATHSIG = 1 + + +def _pool_worker_init(): + """Set PR_SET_PDEATHSIG so pool workers die when the parent process dies. + + Prevents zombie worker accumulation after OOM kills, SIGKILL, etc. + Uses SIGKILL because ProcessPoolExecutor's `except BaseException` catches + SIGTERM's SystemExit, keeping workers alive until the broken pipe surfaces. + + prctl is Linux-specific, so this is a no-op elsewhere (the symbol is absent + on other platforms and would otherwise raise, breaking the whole pool). + """ + if not sys.platform.startswith("linux"): + return + libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True) + libc.prctl(_PR_SET_PDEATHSIG, signal.SIGKILL, 0, 0, 0) + class ConfigAwareHelper: """ @@ -73,23 +98,24 @@ def __init__(self, preset): self._loop = None - # multiprocessing thread pool + # multiprocessing process pool start_method = mp.get_start_method() if start_method != "spawn": self.warning(f"Multiprocessing spawn method is set to {start_method}.") - - # we spawn 1 fewer processes than cores - # this helps to avoid locking up the system or competing with the main python process for cpu time - num_processes = max(1, mp.cpu_count() - 1) - self.process_pool = ProcessPoolExecutor(max_workers=num_processes) + self.process_pool = self._create_process_pool() + self._pool_reset_lock = asyncio.Lock() self._cloud = None + self._blasthttp_client = None self.re = RegexHelper(self) self.yara = YaraHelper(self) + self.simhash = SimHashHelper() self._dns = None self._web = None + self._asn = None self._cloudcheck = None + self._asn = None self.config_aware_validators = self.validators.Validators(self) self.depsinstaller = DepsInstaller(self) self.word_cloud = WordCloud(self) @@ -107,12 +133,30 @@ def web(self): self._web = WebHelper(self) return self._web + @property + def asn(self): + if self._asn is None: + self._asn = ASNHelper(self) + return self._asn + + @property + def blasthttp(self): + if self._blasthttp_client is None: + import blasthttp as _blasthttp + + self._blasthttp_client = _blasthttp.BlastHTTP() + rate_limit = self.web_config.get("http_rate_limit", 0) + if rate_limit: + self._blasthttp_client.set_rate_limit(rate_limit) + return self._blasthttp_client + @property def cloudcheck(self): if self._cloudcheck is None: from cloudcheck import CloudCheck - self._cloudcheck = CloudCheck() + ssl_verify = self.web_config.get("ssl_verify_infrastructure", True) + self._cloudcheck = CloudCheck(verify_ssl=ssl_verify) return self._cloudcheck def bloom_filter(self, size): @@ -134,6 +178,7 @@ def http_compare( data=None, json=None, timeout=10, + on_baseline_ready=None, ): return HttpCompare( url, @@ -146,6 +191,7 @@ def http_compare( method=method, data=data, json=json, + on_baseline_ready=on_baseline_ready, ) def temp_filename(self, extension=None): @@ -185,30 +231,96 @@ def loop(self): """ if self._loop is None: self._loop = get_event_loop() + # only current caller is wafw00f (sync requests library) + self._io_executor = ThreadPoolExecutor(max_workers=max(8, (os.cpu_count() or 1) + 4)) + self._cpu_executor = ThreadPoolExecutor(max_workers=max(8, os.cpu_count() or 4)) + self._loop.set_default_executor(self._io_executor) return self._loop - def run_in_executor(self, callback, *args, **kwargs): + @staticmethod + def _create_process_pool(): + # we spawn 1 fewer processes than cores + # this helps to avoid locking up the system or competing with the main python process for cpu time + num_processes = max(1, mp.cpu_count() - 1) + pool_kwargs = {"max_workers": num_processes, "initializer": _pool_worker_init} + # max_tasks_per_child replaces workers after N tasks, preventing memory leaks + # and reducing the chance of a degraded worker process causing hangs + if sys.version_info >= (3, 11): + pool_kwargs["max_tasks_per_child"] = 25 + return ProcessPoolExecutor(**pool_kwargs) + + def run_in_executor_io(self, callback, *args, **kwargs): """ Run a synchronous task in the event loop's default thread pool executor Examples: Execute callback: - >>> result = await self.helpers.run_in_executor(callback_fn, arg1, arg2) + >>> result = await self.helpers.run_in_executor_io(callback_fn, arg1, arg2) + """ + callback = partial(callback, **kwargs) + return self.loop.run_in_executor(self._io_executor, callback, *args) + + def run_in_executor_cpu(self, callback, *args, **kwargs): + """ + Run short CPU-bound work that releases the GIL in a dedicated thread pool, + separate from I/O so it never queues behind long-running network calls. + + Examples: + Execute callback: + >>> result = await self.helpers.run_in_executor_cpu(callback_fn, arg1, arg2) """ callback = partial(callback, **kwargs) - return self.loop.run_in_executor(None, callback, *args) + return self.loop.run_in_executor(self._cpu_executor, callback, *args) - def run_in_executor_mp(self, callback, *args, **kwargs): + async def run_in_executor_mp(self, callback, *args, **kwargs): """ - Same as run_in_executor() except with a process pool executor - Use only in cases where callback is CPU-bound + Same as run_in_executor_io() except with a process pool executor. + Use only in cases where callback is CPU-bound. + + Includes a timeout (default 300s) to prevent indefinite hangs if a child process dies or the pool enters a broken state. + On timeout, the entire pool is terminated and replaced so that stuck workers cannot accumulate and starve the scan. + + Pass ``_timeout=seconds`` to override the default timeout. Examples: Execute callback: >>> result = await self.helpers.run_in_executor_mp(callback_fn, arg1, arg2) """ + timeout = kwargs.pop("_timeout", 300) callback = partial(callback, **kwargs) - return self.loop.run_in_executor(self.process_pool, callback, *args) + future = self.loop.run_in_executor(self.process_pool, callback, *args) + try: + return await asyncio.wait_for(future, timeout=timeout) + except asyncio.TimeoutError: + log.warning(f"Process pool task timed out after {timeout}s, killing stuck workers and replacing pool") + await self._reset_process_pool() + raise + + async def _reset_process_pool(self): + """Terminate all workers in the current process pool and replace it. + + This is the nuclear option — every in-flight task on the old pool will fail with BrokenProcessPool. + We accept that trade-off because a timeout means something is genuinely broken, and leaving the stuck worker alive would permanently consume a pool slot. + + # TODO: Python 3.14 adds ProcessPoolExecutor.terminate_workers() + # and kill_workers() (https://github.com/python/cpython/pull/130849). + # Once we drop 3.13 support we can replace the _processes access + # with those official methods. + """ + async with self._pool_reset_lock: + old_pool = self.process_pool + self.process_pool = self._create_process_pool() + # snapshot workers before shutdown (shutdown sets _processes = None) + workers = list((old_pool._processes or {}).values()) + # terminate workers before shutdown so stuck ones don't block + for proc in workers: + if proc.is_alive(): + proc.terminate() + old_pool.shutdown(wait=False, cancel_futures=True) + # escalate to SIGKILL for anything that ignored SIGTERM + for proc in workers: + if proc.is_alive(): + proc.kill() @property def in_tests(self): diff --git a/bbot/core/helpers/interactsh.py b/bbot/core/helpers/interactsh.py index 5c2eb0e526..e178c6db8a 100644 --- a/bbot/core/helpers/interactsh.py +++ b/bbot/core/helpers/interactsh.py @@ -140,7 +140,11 @@ async def register(self, callback=None): "correlation-id": self.correlation_id, } r = await self.parent_helper.request( - f"https://{server}/register", headers=headers, json=data, method="POST" + f"https://{server}/register", + headers=headers, + json=data, + method="POST", + ssl_verify=self.parent_helper.web.ssl_verify_infrastructure, ) if r is None: continue @@ -190,11 +194,19 @@ async def deregister(self): data = {"secret-key": self.secret, "correlation-id": self.correlation_id} r = await self.parent_helper.request( - f"https://{self.server}/deregister", headers=headers, json=data, method="POST" + f"https://{self.server}/deregister", + headers=headers, + json=data, + method="POST", + ssl_verify=self.parent_helper.web.ssl_verify_infrastructure, ) if self._poll_task is not None: self._poll_task.cancel() + try: + await self._poll_task + except (asyncio.CancelledError, Exception): + pass if "success" not in getattr(r, "text", ""): raise InteractshError(f"Failed to de-register with interactsh server {self.server}") @@ -237,6 +249,7 @@ async def poll(self): f"https://{self.server}/poll?id={self.correlation_id}&secret={self.secret}", headers=headers, timeout=15, + ssl_verify=self.parent_helper.web.ssl_verify_infrastructure, ) if r is None: raise InteractshError("Error polling interact.sh: No response from server") @@ -274,21 +287,41 @@ async def poll_loop(self, callback): return await self._poll_loop(callback) async def _poll_loop(self, callback): + consecutive_failures = 0 + max_failures = 5 + log.debug(f"Starting interactsh poll loop (interval={self.poll_interval}s, server={self.server})") while 1: if self.parent_helper.scan.stopping: - await asyncio.sleep(1) - continue + log.debug("Stopping interactsh poll loop (scan is stopping)") + break data_list = [] try: data_list = await self.poll() + consecutive_failures = 0 except InteractshError as e: - log.warning(e) + consecutive_failures += 1 + if consecutive_failures == 1: + log.warning(e) + elif consecutive_failures >= max_failures: + log.error(f"Interactsh poll failed {max_failures} consecutive times, giving up: {e}") + break + else: + log.debug(f"Interactsh poll failure #{consecutive_failures}: {e}") log.trace(traceback.format_exc()) + backoff = min(self.poll_interval * (2 ** (consecutive_failures - 1)), 300) + log.debug(f"Interactsh backing off for {backoff}s after failure #{consecutive_failures}") + await asyncio.sleep(backoff) + continue if not data_list: + log.debug(f"Interactsh poll returned no interactions, sleeping {self.poll_interval}s") await asyncio.sleep(self.poll_interval) continue + log.debug(f"Interactsh poll returned {len(data_list)} interaction(s)") for data in data_list: if data: + protocol = data.get("protocol", "unknown") + full_id = data.get("full-id", "unknown") + log.debug(f"Interactsh interaction: protocol={protocol}, full-id={full_id}") await self.parent_helper.execute_sync_or_async(callback, data) def _decrypt(self, aes_key, data): diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index ad96cdb374..cbdfd4482e 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -7,13 +7,16 @@ import string import asyncio import logging +import functools import ipaddress import regex as re import subprocess as sp + from pathlib import Path from contextlib import suppress from unidecode import unidecode # noqa F401 +from typing import Iterable, Awaitable, Optional from asyncio import create_task, gather, sleep, wait_for # noqa from urllib.parse import urlparse, quote, unquote, urlunparse, urljoin # noqa F401 @@ -615,11 +618,11 @@ def is_ip(d, version=None, include_network=False): """ ip = None try: - ip = ipaddress.ip_address(d) + ip = cached_ip_address(d) except Exception: if include_network: try: - ip = ipaddress.ip_network(d, strict=False) + ip = cached_ip_network(d) except Exception: pass if ip is not None and (version is None or ip.version == version): @@ -655,6 +658,39 @@ def is_ip_type(i, network=None): return ipaddress._IPAddressBase in i.__class__.__mro__ +# Valid characters in any IP address or CIDR string: hex digits, dot, colon, +# slash (prefix length), and percent (IPv6 zone ID). A typical hostname like +# "google.com" fails this check at its first letter. +_IP_CHARS = frozenset("0123456789abcdefABCDEF.:/%") + + +def _looks_like_ip(s): + """Cheap reject for strings that obviously aren't IPs/CIDRs.""" + for c in s: + if c not in _IP_CHARS: + return False + return True + + +@functools.lru_cache(maxsize=16384) +def cached_ip_address(s): + """ + Cached wrapper around `ipaddress.ip_address()`. Same semantics: returns an + IPv4Address/IPv6Address, or raises ValueError. Exceptions are not cached. + """ + return ipaddress.ip_address(s) + + +@functools.lru_cache(maxsize=16384) +def cached_ip_network(s): + """ + Cached wrapper around `ipaddress.ip_network(strict=False)`. Same semantics. + Exceptions are not cached. + """ + return ipaddress.ip_network(s, strict=False) + + +@functools.lru_cache(maxsize=16384) def make_ip_type(s): """ Convert a string to its corresponding IP address or network type. @@ -680,12 +716,20 @@ def make_ip_type(s): """ if not s: raise ValueError(f'Invalid hostname: "{s}"') - # IP address - with suppress(Exception): + # Pass through anything that isn't a string (e.g. an already-parsed IP object). + if not isinstance(s, str): + return s + # Skip the expensive ipaddress parse+raise path for obvious non-IPs. + if not _looks_like_ip(s): + return s + try: return ipaddress.ip_address(s) - # IP network - with suppress(Exception): + except ValueError: + pass + try: return ipaddress.ip_network(s, strict=False) + except ValueError: + pass return s @@ -835,7 +879,7 @@ def rand_string(length=10, digits=True, numeric_only=False): return "".join(random.choice(pool) for _ in range(length)) -def truncate_string(s: str, n: int) -> str: +def truncate_string(s: str, n: int = 200) -> str: if not isinstance(s, str): raise ValueError(f"Expected string, got {type(s)}") if len(s) > n: @@ -1116,6 +1160,32 @@ def str_or_file(s): yield s +_comment_re = re.compile(r"\s#") + + +def strip_comments(line): + """Strip #-style comments from a line. + + Handles full-line comments (``# ...``) and inline comments (``target # ...``). + The ``#`` must be preceded by whitespace to count as an inline comment, + so URL fragments like ``http://example.com/page#section`` are preserved. + + Examples: + >>> strip_comments("evilcorp.com # main domain") + 'evilcorp.com' + >>> strip_comments("# full line comment") + '' + >>> strip_comments("http://example.com/page#section") + 'http://example.com/page#section' + """ + if line.lstrip().startswith("#"): + return "" + m = _comment_re.search(line) + if m: + return line[: m.start()] + return line + + split_regex = re.compile(r"[\s,]") @@ -1126,6 +1196,7 @@ def chain_lists( remove_blank=True, validate=False, validate_chars='<>:"/\\|?*)', + _strip_comments=False, ): """Chains together list elements, allowing for entries separated by commas. @@ -1141,6 +1212,7 @@ def chain_lists( remove_blank (bool, optional): Whether to remove blank entries from the list. Defaults to True. validate (bool, optional): Whether to perform validation for undesirable characters. Defaults to False. validate_chars (str, optional): When performing validation, what additional set of characters to block (blocks non-printable ascii automatically). Defaults to '<>:"/\\|?*)' + _strip_comments (bool, optional): Whether to strip ``#``-style comments from entries and file lines. Defaults to False. Returns: list: The list of chained elements. @@ -1159,6 +1231,8 @@ def chain_lists( l = [l] final_list = {} for entry in l: + if _strip_comments: + entry = strip_comments(entry) for s in split_regex.split(entry): f = s.strip() if validate: @@ -1170,6 +1244,8 @@ def chain_lists( new_msg = str(msg).format(filename=f_path) log.info(new_msg) for line in str_or_file(f): + if _strip_comments: + line = strip_comments(line) final_list[line] = None else: final_list[f] = None @@ -1659,7 +1735,7 @@ def rm_rf(f, ignore_errors=False): f (str or Path): The directory path to delete. Examples: - >>> rm_rf("/tmp/httpx98323849") + >>> rm_rf("/tmp/bbot98323849") """ import shutil @@ -2525,27 +2601,16 @@ def weighted_shuffle(items, weights): ['banana', 'apple', 'cherry'] Note: - The sum of all weights does not have to be 1. They will be normalized internally. + Weights must be positive. The sum does not need to be 1. """ - # Create a list of tuples where each tuple is (item, weight) - pool = list(zip(items, weights)) - - shuffled_items = [] - - # While there are still items to be chosen... - while pool: - # Normalize weights - total = sum(weight for item, weight in pool) - weights = [weight / total for item, weight in pool] - - # Choose an index based on weight - chosen_index = random.choices(range(len(pool)), weights=weights, k=1)[0] - - # Add the chosen item to the shuffled list - chosen_item, chosen_weight = pool.pop(chosen_index) - shuffled_items.append(chosen_item) - - return shuffled_items + # Give each item a random number, but skewed by its weight + # (heavier weight → number closer to 1). Sort by that number, + # highest first. Items with bigger weights are more likely to + # end up near the top, but every item still has a chance. + rand = random.random + keyed = [(rand() ** (1.0 / w), item) for item, w in zip(items, weights)] + keyed.sort(key=lambda x: x[0], reverse=True) + return [item for _, item in keyed] def parse_port_string(port_string): @@ -2595,30 +2660,106 @@ def parse_port_string(port_string): return ports -async def as_completed(coros): +async def as_completed( + coroutines: Iterable[Awaitable], + max_concurrent: Optional[int] = 20, +): """ - Async generator that yields completed Tasks as they are completed. + Yield completed coroutines as they finish with optional concurrency limiting. + All coroutines are scheduled as tasks internally for execution. - Args: - coros (iterable): An iterable of coroutine objects or asyncio Tasks. + Guarantees cleanup: + - If the consumer breaks early or an internal cancellation is detected, all remaining + tasks are cancelled and awaited (with return_exceptions=True) to avoid + "Task exception was never retrieved" warnings. + """ + it = iter(coroutines) - Yields: - asyncio.Task: A Task object that has completed its execution. + running: set[asyncio.Task] = set() + limit = max_concurrent or float("inf") - Examples: - >>> async def main(): - ... async for task in as_completed([coro1(), coro2(), coro3()]): - ... result = task.result() - ... print(f'Task completed with result: {result}') + async def _cancel_and_drain_remaining(): + if not running: + return + for t in running: + t.cancel() + try: + await asyncio.gather(*running, return_exceptions=True) + finally: + running.clear() - >>> asyncio.run(main()) + # Prime the running set up to the concurrency limit (or all, if unlimited) + try: + while len(running) < limit: + coro = next(it) + running.add(asyncio.create_task(coro)) + except StopIteration: + pass + + # Dedup state for repeated error messages + _last_err = {"msg": None, "count": 0} + + try: + # Drain: yield completed tasks, backfill from the iterator as slots free up + while running: + done, running = await asyncio.wait(running, return_when=asyncio.FIRST_COMPLETED) + for task in done: + # Immediately backfill one slot per completed task, if more work remains + try: + coro = next(it) + running.add(asyncio.create_task(coro)) + except StopIteration: + pass + + # If task raised, handle cancellation gracefully and dedupe noisy repeats + if task.exception() is not None: + e = task.exception() + if in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)): + # Quietly stop if we're being cancelled + log.info("as_completed: cancellation detected; exiting early") + await _cancel_and_drain_remaining() + return + # Build a concise message + msg = f"as_completed yielded exception: {e}" + if msg == _last_err["msg"]: + _last_err["count"] += 1 + if _last_err["count"] <= 3: + log.warning(msg) + elif _last_err["count"] % 10 == 0: + log.warning(f"{msg} (repeated {_last_err['count']}x)") + else: + log.debug(msg) + else: + _last_err["msg"] = msg + _last_err["count"] = 1 + log.warning(msg) + yield task + finally: + # If the consumer breaks early or an error bubbles, ensure we don't leak tasks + await _cancel_and_drain_remaining() + + +def get_waf_strings(): + """ + Returns a list of common WAF (Web Application Firewall) detection strings. + + Returns: + list: List of WAF detection strings + + Examples: + >>> waf_strings = get_waf_strings() + >>> "The requested URL was rejected" in waf_strings + True """ - tasks = {coro if isinstance(coro, asyncio.Task) else asyncio.create_task(coro): coro for coro in coros} - while tasks: - done, _ = await asyncio.wait(tasks.keys(), return_when=asyncio.FIRST_COMPLETED) - for task in done: - tasks.pop(task) - yield task + return [ + "The requested URL was rejected", + "This content has been blocked", + "The URL you requested has been blocked", + "Request unsuccessful. Incapsula incident", + "Access Denied - Sucuri Website Firewall", + "Attention Required! | Cloudflare", + "Microsoft-Azure-Application-Gateway", + ] def clean_dns_record(record): @@ -2638,14 +2779,12 @@ def clean_dns_record(record): >>> clean_dns_record('www.evilcorp.com.') 'www.evilcorp.com' - >>> from dns.rrset import from_text - >>> record = from_text('www.evilcorp.com', 3600, 'IN', 'A', '1.2.3.4')[0] - >>> clean_dns_record(record) - '1.2.3.4' + >>> clean_dns_record('*.evilcorp.com.') + 'evilcorp.com' """ if not isinstance(record, str): record = str(record.to_text()) - return str(record).rstrip(".").lower() + return str(record).strip("'\"").strip("*.").lower() def truncate_filename(file_path, max_length=255): @@ -2684,36 +2823,23 @@ def truncate_filename(file_path, max_length=255): def get_keys_in_dot_syntax(config): - """Retrieve all keys in an OmegaConf configuration in dot notation. - - This function converts an OmegaConf configuration into a list of keys - represented in dot notation. + """Retrieve all leaf keys in a nested dict in dot notation. Args: - config (DictConfig): The OmegaConf configuration object. + config (dict): A nested dict. Returns: - List[str]: A list of keys in dot notation. + List[str]: A list of leaf keys in dot notation. Examples: - >>> config = OmegaConf.create({ - ... "web": { - ... "test": True - ... }, - ... "db": { - ... "host": "localhost", - ... "port": 5432 - ... } - ... }) - >>> get_keys_in_dot_syntax(config) + >>> get_keys_in_dot_syntax({"web": {"test": True}, "db": {"host": "localhost", "port": 5432}}) ['web.test', 'db.host', 'db.port'] """ - from omegaconf import OmegaConf - - container = OmegaConf.to_container(config, resolve=True) keys = [] def recursive_keys(d, parent_key=""): + if not isinstance(d, dict): + return for k, v in d.items(): full_key = f"{parent_key}.{k}" if parent_key else k if isinstance(v, dict): @@ -2721,7 +2847,7 @@ def recursive_keys(d, parent_key=""): else: keys.append(full_key) - recursive_keys(container) + recursive_keys(config) return keys diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py index 9c0da33607..13d43c007b 100644 --- a/bbot/core/helpers/names_generator.py +++ b/bbot/core/helpers/names_generator.py @@ -17,6 +17,9 @@ "autistic", "awkward", "baby", + "bamboozled", + "based", + "befuddled", "begrudged", "benevolent", "bewildered", @@ -24,24 +27,39 @@ "black", "blazed", "bloodshot", + "bodacious", + "bonkers", + "bootleg", + "bricked", "brown", + "bussin", + "caffeinated", + "cantankerous", + "chaotic", "cheeky", "childish", "chiseled", + "chonky", + "clapped", "cold", "condescending", "considerate", "constipated", "contentious", + "cooked", "corrupted", "cosmic", + "cracked", "crafty", + "cranked", "crazed", "creamy", "crispy", "crumbly", + "crusty", "cryptic", "cuddly", + "cursed", "cute", "dark", "dastardly", @@ -55,6 +73,7 @@ "depressed", "deranged", "derogatory", + "derpy", "despicable", "devilish", "devious", @@ -62,6 +81,7 @@ "diabolical", "difficult", "dilapidated", + "discombobulated", "dismal", "distilled", "disturbed", @@ -82,10 +102,12 @@ "expired", "exquisite", "extreme", + "feral", "fermented", "ferocious", "fiendish", "fierce", + "flabbergasted", "flamboyant", "fleecy", "flirtatious", @@ -98,10 +120,15 @@ "fuzzy", "gentle", "giddy", + "glitched", "glowering", "glutinous", + "goated", + "gobsmacked", "golden", + "goofy", "gothic", + "greasy", "grievous", "gummy", "hallucinogenic", @@ -136,11 +163,16 @@ "intoxicated", "inventive", "irritable", + "jacked", + "janky", + "juiced", + "lackadaisical", "large", "liquid", "loveable", "lovely", "lucid", + "lumpy", "malevolent", "malfunctioning", "malicious", @@ -158,17 +190,21 @@ "mysterious", "nascent", "naughty", + "nautical", "nefarious", "negligent", + "nerfed", "neurotic", "nihilistic", "normal", "overattached", + "overclocked", "overcompensating", "overenthusiastic", "overmedicated", "overwhelming", "overzealous", + "pacific", "paranoid", "pasty", "peckish", @@ -201,6 +237,7 @@ "rapid_unscheduled", "raving", "reckless", + "recursive", "reductive", "ripped", "ruthless", @@ -210,6 +247,7 @@ "savvy", "scheming", "schizophrenic", + "scuffed", "secretive", "sedated", "senile", @@ -218,13 +256,16 @@ "sinful", "sinister", "slippery", + "sloppy", "sly", "sneaky", "soft", + "soggy", "sophisticated", "spasmodic", "spicy", "spiteful", + "spooky", "squishy", "steamy", "sticky", @@ -241,6 +282,7 @@ "sunburned", "super", "surreal", + "sus", "suspicious", "sweet", "swole", @@ -253,6 +295,8 @@ "ticklish", "tiny", "tricky", + "turbo", + "turnt", "twitchy", "ugly", "unabated", @@ -285,8 +329,11 @@ "wild", "wispy", "witty", + "wonky", "woolly", + "yeeted", "zesty", + "zooted", ] names = [ @@ -308,6 +355,7 @@ "amir", "amy", "andrea", + "andres", "andrew", "angela", "ann", @@ -342,12 +390,15 @@ "brandon", "brandybuck", "brenda", + "brendan", "brian", "brianna", "brittany", "bruce", "bryan", + "buhner", "caitlyn", + "cal", "caleb", "cameron", "carl", @@ -394,6 +445,7 @@ "diana", "diane", "dobby", + "dominic", "donald", "donna", "dooku", @@ -432,6 +484,7 @@ "evan", "evelyn", "faramir", + "felix", "florence", "fox", "frances", @@ -458,6 +511,7 @@ "gollum", "grace", "gregory", + "griffey", "gus", "hagrid", "hank", @@ -472,6 +526,7 @@ "homer", "howard", "hunter", + "ichiro", "irene", "isaac", "isabella", @@ -515,6 +570,7 @@ "judy", "julia", "julie", + "julio", "justin", "karen", "katherine", @@ -547,6 +603,7 @@ "logan", "lois", "lori", + "lou", "louis", "louise", "lucius", @@ -578,6 +635,7 @@ "mildred", "milhouse", "monica", + "moose", "nancy", "natalie", "nathan", @@ -663,6 +721,7 @@ "theoden", "theon", "theresa", + "thetechromancer", "thomas", "tiffany", "timothy", @@ -694,6 +753,8 @@ "wendy", "william", "willie", + "wilson", + "woo", "worf", "wormtongue", "xavier", diff --git a/bbot/core/helpers/regex.py b/bbot/core/helpers/regex.py index 72c4029cb7..7a6ec0b9cc 100644 --- a/bbot/core/helpers/regex.py +++ b/bbot/core/helpers/regex.py @@ -29,19 +29,19 @@ def compile(self, *args, **kwargs): async def search(self, compiled_regex, *args, **kwargs): self.ensure_compiled_regex(compiled_regex) - return await self.parent_helper.run_in_executor(compiled_regex.search, *args, **kwargs) + return await self.parent_helper.run_in_executor_cpu(compiled_regex.search, *args, **kwargs) async def match(self, compiled_regex, *args, **kwargs): self.ensure_compiled_regex(compiled_regex) - return await self.parent_helper.run_in_executor(compiled_regex.match, *args, **kwargs) + return await self.parent_helper.run_in_executor_cpu(compiled_regex.match, *args, **kwargs) async def sub(self, compiled_regex, *args, **kwargs): self.ensure_compiled_regex(compiled_regex) - return await self.parent_helper.run_in_executor(compiled_regex.sub, *args, **kwargs) + return await self.parent_helper.run_in_executor_cpu(compiled_regex.sub, *args, **kwargs) async def findall(self, compiled_regex, *args, **kwargs): self.ensure_compiled_regex(compiled_regex) - return await self.parent_helper.run_in_executor(compiled_regex.findall, *args, **kwargs) + return await self.parent_helper.run_in_executor_cpu(compiled_regex.findall, *args, **kwargs) async def findall_multi(self, compiled_regexes, *args, threads=10, **kwargs): """ @@ -55,7 +55,7 @@ async def findall_multi(self, compiled_regexes, *args, threads=10, **kwargs): tasks = {} def new_task(regex_name, r): - task = self.parent_helper.run_in_executor(r.findall, *args, **kwargs) + task = self.parent_helper.run_in_executor_cpu(r.findall, *args, **kwargs) tasks[task] = regex_name compiled_regexes = dict(compiled_regexes) @@ -77,7 +77,7 @@ def new_task(regex_name, r): async def finditer(self, compiled_regex, *args, **kwargs): self.ensure_compiled_regex(compiled_regex) - return await self.parent_helper.run_in_executor(self._finditer, compiled_regex, *args, **kwargs) + return await self.parent_helper.run_in_executor_cpu(self._finditer, compiled_regex, *args, **kwargs) async def finditer_multi(self, compiled_regexes, *args, **kwargs): """ @@ -85,7 +85,7 @@ async def finditer_multi(self, compiled_regexes, *args, **kwargs): """ for r in compiled_regexes: self.ensure_compiled_regex(r) - return await self.parent_helper.run_in_executor(self._finditer_multi, compiled_regexes, *args, **kwargs) + return await self.parent_helper.run_in_executor_cpu(self._finditer_multi, compiled_regexes, *args, **kwargs) def _finditer_multi(self, compiled_regexes, *args, **kwargs): matches = [] @@ -98,16 +98,16 @@ def _finditer(self, compiled_regex, *args, **kwargs): return list(compiled_regex.finditer(*args, **kwargs)) async def extract_params_html(self, *args, **kwargs): - return await self.parent_helper.run_in_executor(misc.extract_params_html, *args, **kwargs) + return await self.parent_helper.run_in_executor_cpu(misc.extract_params_html, *args, **kwargs) async def extract_emails(self, *args, **kwargs): - return await self.parent_helper.run_in_executor(misc.extract_emails, *args, **kwargs) + return await self.parent_helper.run_in_executor_cpu(misc.extract_emails, *args, **kwargs) async def search_dict_values(self, *args, **kwargs): def _search_dict_values(*_args, **_kwargs): return list(misc.search_dict_values(*_args, **_kwargs)) - return await self.parent_helper.run_in_executor(_search_dict_values, *args, **kwargs) + return await self.parent_helper.run_in_executor_cpu(_search_dict_values, *args, **kwargs) async def recursive_decode(self, *args, **kwargs): - return await self.parent_helper.run_in_executor(misc.recursive_decode, *args, **kwargs) + return await self.parent_helper.run_in_executor_cpu(misc.recursive_decode, *args, **kwargs) diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py index f50ea2519c..15226ed9d3 100644 --- a/bbot/core/helpers/regexes.py +++ b/bbot/core/helpers/regexes.py @@ -53,6 +53,9 @@ ) ip_range_regexes = [re.compile(r, re.I) for r in _ip_range_regexes] +# SPF qualifier + ip4:/ip6: mechanism prefix, e.g. the "ip4:" in "ip4:1.2.3.0/24" (RFC 7208) +spf_ip_mechanism_regex = re.compile(r"^[+\-~?]?ip[46]:", re.I) + # all dns names including IP addresses and bare hostnames (e.g. "localhost") _dns_name_regex = r"(?:\w(?:[\w-]{0,100}\w)?\.?)+(?:[xX][nN]--)?[^\W_]{1,63}\.?" # dns names with periods (e.g. "www.example.com") @@ -157,7 +160,14 @@ re.DOTALL, ) post_form_regex_noaction = re.compile( - r"]*(?:\baction=[\"']?([^\s\"'<>]+)[\"']?)?[^>]*\bmethod=[\"']?[pP][oO][sS][tT][\"']?[^>]*>([\s\S]*?)<\/form>", + # Negative lookahead rejects forms that already carry action= (those are + # handled by post_form_regex / post_form_regex2 with the real action URL). + # Without it, the action group's `?` makes this regex over-eagerly match + # forms with action, capturing '' and emitting a phantom WEB_PARAMETER + # whose url falls back to event.url — racing the legitimate emission. + # The empty `()` keeps findall's tuple shape (form_action, form_content) + # consistent with the other form regexes consumed by GetForm.extract(). + r"]*\baction=)()[^>]*\bmethod=[\"']?[pP][oO][sS][tT][\"']?[^>]*>([\s\S]*?)<\/form>", re.DOTALL, ) generic_form_regex = re.compile( @@ -166,9 +176,15 @@ ) select_tag_regex = re.compile( - r"]+?name=[\"\']?([_\-\.\w]+)[\"\']?[^>]*>(?:\s*]*?value=[\"\']?([_\.\-\w]*)[\"\']?[^>]*>)?", + r"]+?\bname=[\"\']?([_\-\.\w]+)[\"\']?[^>]*>((?:\s*]*>(?:[^<]*(?:)?)?\s*)*)", re.IGNORECASE | re.DOTALL, ) +option_tag_regex = re.compile(r"]*)>", re.IGNORECASE) +option_value_regex = re.compile( + r"""\bvalue\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>'"]+))""", + re.IGNORECASE, +) +option_selected_regex = re.compile(r"(?]*?\sname=[\"\']?([\-\._=+\/\w]+)[\"\']?[^>]*?\svalue=[\"\']?([:%\-\._=+\/\w]*)[\"\']?[^>]*?>" diff --git a/bbot/core/helpers/simhash.py b/bbot/core/helpers/simhash.py new file mode 100644 index 0000000000..e602c2923e --- /dev/null +++ b/bbot/core/helpers/simhash.py @@ -0,0 +1,116 @@ +import xxhash +import re + +_non_word_re = re.compile(r"[^\w]+") + + +class SimHashHelper: + def __init__(self, bits=64): + self.bits = bits + + @staticmethod + def compute_simhash(text, bits=64, truncate=True, normalization_filter=None): + """ + Static method for computing a SimHash fingerprint. + + Designed to be called via run_in_executor_cpu(): the work is short and the + input is truncated to ~3KB inside the helper, so a thread pool avoids the + pickle/spawn overhead of a process pool. + + Args: + text (str): The text to hash + bits (int): Number of bits for the hash. Defaults to 64. + truncate (bool): Whether to truncate large text for performance. Defaults to True. + normalization_filter (str): Text to remove for normalization. Defaults to None. + + Returns: + int: The SimHash fingerprint + """ + helper = SimHashHelper(bits=bits) + return helper.hash(text, truncate=truncate, normalization_filter=normalization_filter) + + def _truncate_content(self, content): + """ + Truncate large content for similarity comparison to improve performance. + + Truncation rules: + - If content <= 3072 bytes: return as-is + - If content > 3072 bytes: return first 2048 bytes + last 1024 bytes + """ + content_length = len(content) + + # No truncation needed for smaller content + if content_length <= 3072: + return content + + # Truncate: first 2048 + last 1024 bytes + first_part = content[:2048] + last_part = content[-1024:] + + return first_part + last_part + + def _normalize_text(self, text, normalization_filter): + """ + Normalize text by removing the normalization filter from the text. + """ + return text.replace(normalization_filter, "") + + def _get_features(self, text): + """Extract 3-character shingles as features""" + width = 3 + text = text.lower() + # Remove non-word characters + text = _non_word_re.sub("", text) + # Create 3-character shingles + return [text[i : i + width] for i in range(max(len(text) - width + 1, 1))] + + def _hash_feature(self, feature): + """Return a hash of a feature using xxHash""" + return xxhash.xxh64(feature.encode("utf-8")).intdigest() + + def hash(self, text, truncate=True, normalization_filter=None): + """ + Generate a SimHash fingerprint for the given text. + + Args: + text (str): The text to hash + truncate (bool): Whether to truncate large text for performance. Defaults to True. + When enabled, text larger than 4KB is truncated to first 2KB + last 1KB for comparison. + + Returns: + int: The SimHash fingerprint + """ + # Apply truncation if enabled + if truncate: + text = self._truncate_content(text) + + if normalization_filter: + text = self._normalize_text(text, normalization_filter) + + vector = [0] * self.bits + features = self._get_features(text) + + for feature in features: + hv = self._hash_feature(feature) + for i in range(self.bits): + bit = (hv >> i) & 1 + vector[i] += 1 if bit else -1 + + # Final fingerprint + fingerprint = 0 + for i, val in enumerate(vector): + if val >= 0: + fingerprint |= 1 << i + return fingerprint + + def similarity(self, hash1, hash2): + """ + Compute similarity between two SimHashes as a value between 0.0 and 1.0. + """ + # Hamming distance: count of differing bits + diff = (hash1 ^ hash2).bit_count() + return 1.0 - (diff / self.bits) + + +# Module-level alias for the static method to enable clean imports +compute_simhash = SimHashHelper.compute_simhash diff --git a/bbot/core/helpers/validators.py b/bbot/core/helpers/validators.py index 97a39fae3c..577f124756 100644 --- a/bbot/core/helpers/validators.py +++ b/bbot/core/helpers/validators.py @@ -7,7 +7,7 @@ from bbot.core.helpers import regexes from bbot.errors import ValidationError from bbot.core.helpers.url import parse_url, hash_url -from bbot.core.helpers.misc import smart_encode_punycode, split_host_port, make_netloc, is_ip +from bbot.core.helpers.misc import smart_encode_punycode, split_host_port, make_netloc, is_dns_name, is_ip log = logging.getLogger("bbot.core.helpers.validators") @@ -129,14 +129,60 @@ def validate_host(host: Union[str, ipaddress.IPv4Address, ipaddress.IPv6Address] raise ValidationError(f'Invalid hostname: "{host}"') +def validate_fqdn_or_ip(host): + """Strict FQDN-or-IP check. Accepts an IPv4/IPv6 address or a hostname + containing at least one dot. Rejects single-label values (e.g. + `localhost`, or a domain typed without its TLD), so domain-shaped + config fields fail at preset-load time instead of surfacing as runtime + errors deep in a scan. + + `None` and the empty string pass through to support optional config + fields with `null` defaults. + + Unlike `validate_host`, this function does not normalize the input + (no port-stripping, lowercasing, or punycode conversion); it returns + the value as supplied. Use it as a pydantic `field_validator` on + schema fields that must be an FQDN or IP and nothing else. + + Examples: + >>> validate_fqdn_or_ip("example.com") + 'example.com' + >>> validate_fqdn_or_ip("192.168.1.1") + '192.168.1.1' + >>> validate_fqdn_or_ip("localhost") + ValueError: not a valid FQDN or IP address: 'localhost' + """ + if host is None or host == "": + return host + if is_ip(host): + return host + if isinstance(host, str) and "." in host and is_dns_name(host): + return host + raise ValueError(f"not a valid FQDN or IP address: {host!r}") + + +FINDING_SEVERITY_LEVELS = ("INFO", "LOW", "MEDIUM", "HIGH", "CRITICAL") + + @validator def validate_severity(severity: str): severity = str(severity).strip().upper() - if severity not in ("UNKNOWN", "INFO", "LOW", "MEDIUM", "HIGH", "CRITICAL"): + if severity not in FINDING_SEVERITY_LEVELS: raise ValueError(f"Invalid severity: {severity}") return severity +FINDING_CONFIDENCE_LEVELS = ("UNKNOWN", "LOW", "MEDIUM", "HIGH", "CONFIRMED") + + +@validator +def validate_confidence(confidence: str): + confidence = str(confidence).strip().upper() + if confidence not in FINDING_CONFIDENCE_LEVELS: + raise ValueError(f"Invalid confidence: {confidence}") + return confidence + + @validator def validate_email(email: str): email = smart_encode_punycode(str(email).strip().lower()) diff --git a/bbot/core/helpers/web/client.py b/bbot/core/helpers/web/client.py deleted file mode 100644 index b76e6058ee..0000000000 --- a/bbot/core/helpers/web/client.py +++ /dev/null @@ -1,119 +0,0 @@ -import httpx -import logging -from httpx._models import Cookies - -log = logging.getLogger("bbot.core.helpers.web.client") - - -class DummyCookies(Cookies): - def extract_cookies(self, *args, **kwargs): - pass - - -class BBOTAsyncClient(httpx.AsyncClient): - """ - A subclass of httpx.AsyncClient tailored with BBOT-specific configurations and functionalities. - This class provides rate limiting, logging, configurable timeouts, user-agent customization, custom - headers, and proxy settings. Additionally, it allows the disabling of cookies, making it suitable - for use across an entire scan. - - Attributes: - _bbot_scan (object): BBOT scan object containing configuration details. - _persist_cookies (bool): Flag to determine whether cookies should be persisted across requests. - - Examples: - >>> async with BBOTAsyncClient(_bbot_scan=bbot_scan_object) as client: - >>> response = await client.request("GET", "https://example.com") - >>> print(response.status_code) - 200 - """ - - @classmethod - def from_config(cls, config, target, *args, **kwargs): - kwargs["_config"] = config - kwargs["_target"] = target - web_config = config.get("web", {}) - retries = kwargs.pop("retries", web_config.get("http_retries", 1)) - ssl_verify = web_config.get("ssl_verify", False) - if ssl_verify is False: - from .ssl_context import ssl_context_noverify - - ssl_verify = ssl_context_noverify - kwargs["transport"] = httpx.AsyncHTTPTransport(retries=retries, verify=ssl_verify) - kwargs["verify"] = ssl_verify - return cls(*args, **kwargs) - - def __init__(self, *args, **kwargs): - self._config = kwargs.pop("_config") - self._target = kwargs.pop("_target") - - self._web_config = self._config.get("web", {}) - http_debug = self._web_config.get("debug", None) - if http_debug: - log.trace(f"Creating AsyncClient: {args}, {kwargs}") - - self._persist_cookies = kwargs.pop("persist_cookies", False) - - # timeout - http_timeout = self._web_config.get("http_timeout", 20) - if "timeout" not in kwargs: - kwargs["timeout"] = http_timeout - - # headers - headers = kwargs.get("headers", None) - if headers is None: - headers = {} - - # cookies - cookies = kwargs.get("cookies", None) - if cookies is None: - cookies = {} - - # user agent - user_agent = self._web_config.get("user_agent", "BBOT") - if "User-Agent" not in headers: - headers["User-Agent"] = user_agent - kwargs["headers"] = headers - kwargs["cookies"] = cookies - # proxy - proxies = self._web_config.get("http_proxy", None) - kwargs["proxy"] = proxies - - log.verbose(f"Creating httpx.AsyncClient({args}, {kwargs})") - super().__init__(*args, **kwargs) - if not self._persist_cookies: - self._cookies = DummyCookies() - - def build_request(self, *args, **kwargs): - if args: - url = args[0] - kwargs["url"] = url - url = kwargs["url"] - - target_in_scope = self._target.in_scope(str(url)) - - if target_in_scope: - if not kwargs.get("cookies", None): - kwargs["cookies"] = {} - for ck, cv in self._web_config.get("http_cookies", {}).items(): - if ck not in kwargs["cookies"]: - kwargs["cookies"][ck] = cv - - request = super().build_request(**kwargs) - - if target_in_scope: - for hk, hv in self._web_config.get("http_headers", {}).items(): - hv = str(hv) - # don't clobber headers - if hk not in request.headers: - request.headers[hk] = hv - return request - - def _merge_cookies(self, cookies): - if self._persist_cookies: - return super()._merge_cookies(cookies) - return cookies - - @property - def retries(self): - return self._transport._pool._retries diff --git a/bbot/core/helpers/web/engine.py b/bbot/core/helpers/web/engine.py deleted file mode 100644 index 1c3ecc0f52..0000000000 --- a/bbot/core/helpers/web/engine.py +++ /dev/null @@ -1,240 +0,0 @@ -import ssl -import anyio -import httpx -import asyncio -import logging -import traceback -from socksio.exceptions import SOCKSError -from contextlib import asynccontextmanager - -from bbot.core.engine import EngineServer -from bbot.core.helpers.misc import bytes_to_human, human_to_bytes, get_exception_chain, truncate_string - -log = logging.getLogger("bbot.core.helpers.web.engine") - - -class HTTPEngine(EngineServer): - CMDS = { - 0: "request", - 1: "request_batch", - 2: "request_custom_batch", - 3: "download", - } - - client_only_options = ( - "retries", - "max_redirects", - ) - - def __init__(self, socket_path, target, config={}, debug=False): - super().__init__(socket_path, debug=debug) - self.target = target - self.config = config - self.web_config = self.config.get("web", {}) - self.http_debug = self.web_config.get("debug", False) - self._ssl_context_noverify = None - self.web_clients = {} - self.web_client = self.AsyncClient(persist_cookies=False) - - def AsyncClient(self, *args, **kwargs): - # cache by retries to prevent unwanted accumulation of clients - # (they are not garbage-collected) - retries = kwargs.get("retries", 1) - try: - return self.web_clients[retries] - except KeyError: - from .client import BBOTAsyncClient - - client = BBOTAsyncClient.from_config(self.config, self.target, *args, **kwargs) - self.web_clients[client.retries] = client - return client - - async def request(self, *args, **kwargs): - raise_error = kwargs.pop("raise_error", False) - # TODO: use this - cache_for = kwargs.pop("cache_for", None) # noqa - - client = kwargs.get("client", self.web_client) - - # allow vs follow, httpx why?? - allow_redirects = kwargs.pop("allow_redirects", None) - if allow_redirects is not None and "follow_redirects" not in kwargs: - kwargs["follow_redirects"] = allow_redirects - - # in case of URL only, assume GET request - if len(args) == 1: - kwargs["url"] = args[0] - args = [] - - url = kwargs.get("url", "") - - if not args and "method" not in kwargs: - kwargs["method"] = "GET" - - client_kwargs = {} - for k in list(kwargs): - if k in self.client_only_options: - v = kwargs.pop(k) - client_kwargs[k] = v - - if client_kwargs: - client = self.AsyncClient(**client_kwargs) - - try: - async with self._acatch(url, raise_error): - if self.http_debug: - log.trace(f"Web request: {str(args)}, {str(kwargs)}") - response = await client.request(*args, **kwargs) - if self.http_debug: - log.trace( - f"Web response from {url}: {response} (Length: {len(response.content)}) headers: {response.headers}" - ) - return response - except httpx.HTTPError as e: - if raise_error: - _response = getattr(e, "response", None) - return {"_request_error": str(e), "_response": _response} - - async def request_batch(self, urls, threads=10, **kwargs): - async for (args, _, _), response in self.task_pool( - self.request, args_kwargs=urls, threads=threads, global_kwargs=kwargs - ): - yield args[0], response - - async def request_custom_batch(self, urls_and_kwargs, threads=10, **kwargs): - async for (args, kwargs, tracker), response in self.task_pool( - self.request, args_kwargs=urls_and_kwargs, threads=threads, global_kwargs=kwargs - ): - yield args[0], kwargs, tracker, response - - async def download(self, url, **kwargs): - warn = kwargs.pop("warn", True) - raise_error = kwargs.pop("raise_error", False) - filename = kwargs.pop("filename") - try: - result = await self.stream_request(url, **kwargs) - if result is None: - raise httpx.HTTPError(f"No response from {url}") - content, response = result - log.debug(f"Download result: HTTP {response.status_code}") - response.raise_for_status() - with open(filename, "wb") as f: - f.write(content) - return filename - except httpx.HTTPError as e: - log_fn = log.verbose - if warn: - log_fn = log.warning - log_fn(f"Failed to download {url}: {e}") - if raise_error: - _response = getattr(e, "response", None) - return {"_download_error": str(e), "_response": _response} - - async def stream_request(self, url, **kwargs): - follow_redirects = kwargs.pop("follow_redirects", True) - max_size = kwargs.pop("max_size", None) - raise_error = kwargs.pop("raise_error", False) - if max_size is not None: - max_size = human_to_bytes(max_size) - kwargs["follow_redirects"] = follow_redirects - if "method" not in kwargs: - kwargs["method"] = "GET" - try: - total_size = 0 - chunk_size = 8192 - chunks = [] - - async with self._acatch(url, raise_error=True), self.web_client.stream(url=url, **kwargs) as response: - agen = response.aiter_bytes(chunk_size=chunk_size) - async for chunk in agen: - _chunk_size = len(chunk) - if max_size is not None and total_size + _chunk_size > max_size: - log.verbose( - f"Size of response from {url} exceeds {bytes_to_human(max_size)}, file will be truncated" - ) - await agen.aclose() - break - total_size += _chunk_size - chunks.append(chunk) - return b"".join(chunks), response - except httpx.HTTPError as e: - self.log.debug(f"Error requesting {url}: {e}") - if raise_error: - raise - - def ssl_context_noverify(self): - if self._ssl_context_noverify is None: - ssl_context = ssl.create_default_context() - ssl_context.check_hostname = False - ssl_context.verify_mode = ssl.CERT_NONE - ssl_context.options &= ~ssl.OP_NO_SSLv2 & ~ssl.OP_NO_SSLv3 - ssl_context.set_ciphers("ALL:@SECLEVEL=0") - ssl_context.options |= 0x4 # Add the OP_LEGACY_SERVER_CONNECT option - self._ssl_context_noverify = ssl_context - return self._ssl_context_noverify - - @asynccontextmanager - async def _acatch(self, url, raise_error): - """ - Asynchronous context manager to handle various httpx errors during a request. - - Yields: - None - - Note: - This function is internal and should generally not be used directly. - `url`, `args`, `kwargs`, and `raise_error` should be in the same context as this function. - """ - try: - yield - except httpx.TimeoutException: - if raise_error: - raise - else: - log.verbose(f"HTTP timeout to URL: {url}") - except httpx.ConnectError: - if raise_error: - raise - else: - log.debug(f"HTTP connect failed to URL: {url}") - except httpx.HTTPError as e: - if raise_error: - raise - else: - log.trace(f"Error with request to URL: {url}: {e}") - log.trace(traceback.format_exc()) - except httpx.InvalidURL as e: - if raise_error: - raise - else: - log.warning( - f"Invalid URL (possibly due to dangerous redirect) on request to : {url}: {truncate_string(str(e), 200)}" - ) - log.trace(traceback.format_exc()) - except ssl.SSLError as e: - msg = f"SSL error with request to URL: {url}: {e}" - if raise_error: - raise httpx.RequestError(msg) - else: - log.trace(msg) - log.trace(traceback.format_exc()) - except anyio.EndOfStream as e: - msg = f"AnyIO error with request to URL: {url}: {e}" - if raise_error: - raise httpx.RequestError(msg) - else: - log.trace(msg) - log.trace(traceback.format_exc()) - except SOCKSError as e: - msg = f"SOCKS error with request to URL: {url}: {e}" - if raise_error: - raise httpx.RequestError(msg) - else: - log.trace(msg) - log.trace(traceback.format_exc()) - except BaseException as e: - # don't log if the error is the result of an intentional cancellation - if not any(isinstance(_e, asyncio.exceptions.CancelledError) for _e in get_exception_chain(e)): - log.trace(f"Unhandled exception with request to URL: {url}: {e}") - log.trace(traceback.format_exc()) - raise diff --git a/bbot/core/helpers/web/response_event.py b/bbot/core/helpers/web/response_event.py new file mode 100644 index 0000000000..16dafd4695 --- /dev/null +++ b/bbot/core/helpers/web/response_event.py @@ -0,0 +1,87 @@ +"""Build HTTP_RESPONSE event data dicts from blasthttp.Response objects. + +Materializes hash and cert_info eagerly, so only call when about to emit an +HTTP_RESPONSE event (otherwise blasthttp.Response stays lazy). +""" + +import re +from urllib.parse import urlparse + + +_TITLE_REGEX = re.compile(r"]*>(.*?)", re.IGNORECASE | re.DOTALL) + + +def response_to_event_dict(response, url_input, method="GET"): + """Convert a blasthttp.Response into the dict format HTTP_RESPONSE events expect. + + Args: + response: a blasthttp.Response (native 0.5+ object with .hash, .cert_info) + url_input: the original scan-target identifier (host:port or full URL) + method: HTTP method used to fetch the response + + Returns: + dict with keys url, input, status_code, method, path, host, raw_header, + header, content_type, content_length, title, body, location, hash. Adds + cert_info when the response carried a TLS certificate. + """ + parsed = urlparse(response.url) + path = parsed.path or "/" + + status_line = f"HTTP/1.1 {response.status} \r\n" + raw_header = f"{status_line}{response.raw_headers}\r\n\r\n" + + header_dict = {} + for k, v in response.headers.items(): + key = k.lower().replace("-", "_") + if key in header_dict: + header_dict[key] += f", {v}" + else: + header_dict[key] = v + + content_type = header_dict.get("content_type", "") + content_length = int(header_dict.get("content_length", len(response.body_bytes))) + location = header_dict.get("location", "") + + body = response.body + title = "" + title_match = _TITLE_REGEX.search(body) + if title_match: + title = title_match.group(1).strip() + + j = { + "url": response.url, + "input": url_input, + "status_code": response.status, + "method": method, + "path": path, + "host": parsed.hostname or "", + "raw_header": raw_header, + "header": header_dict, + "content_type": content_type, + "content_length": content_length, + "title": title, + "body": body, + "location": location, + "hash": { + "body_md5": response.hash.body_md5, + "body_mmh3": response.hash.body_mmh3, + "body_sha256": response.hash.body_sha256, + "header_md5": response.hash.header_md5, + "header_mmh3": response.hash.header_mmh3, + "header_sha256": response.hash.header_sha256, + }, + } + + ci = response.cert_info + if ci is not None: + j["cert_info"] = { + "common_name": ci.common_name, + "sans": ci.sans, + "emails": ci.emails, + "issuer": ci.issuer, + "not_before": ci.not_before, + "not_after": ci.not_after, + "fingerprint_sha256": ci.fingerprint_sha256, + } + + return j diff --git a/bbot/core/helpers/web/web.py b/bbot/core/helpers/web/web.py index 60ff35dd59..72ffe66e5b 100644 --- a/bbot/core/helpers/web/web.py +++ b/bbot/core/helpers/web/web.py @@ -1,16 +1,23 @@ +import json +import asyncio import logging +import traceback import warnings from pathlib import Path -from bs4 import BeautifulSoup - -from bbot.core.engine import EngineClient -from bbot.core.helpers.misc import truncate_filename -from bbot.errors import WordlistError, CurlError, WebError +from urllib.parse import urlencode, urlparse, urlunparse, parse_qs +from bs4 import BeautifulSoup from bs4 import MarkupResemblesLocatorWarning from bs4.builder import XMLParsedAsHTMLWarning -from .engine import HTTPEngine +from blasthttp import HTTPStatusError + +from bbot.core.helpers.misc import truncate_filename, bytes_to_human, get_exception_chain +from cachetools import LRUCache + +from bbot.core.helpers.async_helpers import NamedLock +from bbot.core.helpers.diff import HttpCompare +from bbot.errors import HttpCompareError, WordlistError, WebError warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning) @@ -18,21 +25,36 @@ log = logging.getLogger("bbot.core.helpers.web") -class WebHelper(EngineClient): - SERVER_CLASS = HTTPEngine - ERROR_CLASS = WebError +async def iter_batch_results(stream): + """ + Yield individual ``BatchResult`` objects from a ``request_batch_stream`` iterator. + + The native blasthttp 0.4.0 iterator yields lists of ``BatchResult`` (drained in + chunks of 1000 / 200ms to amortize the Python↔Rust boundary). A future Python + wrapper is expected to unwrap these into individual items. This helper handles + both shapes so callers can write a single ``async for`` loop. + """ + async for item in stream: + if isinstance(item, list): + for r in item: + yield r + else: + yield item + +class WebHelper: """ - Main utility class for managing HTTP operations in BBOT. It serves as a wrapper around the BBOTAsyncClient, - which itself is a subclass of httpx.AsyncClient. The class provides functionalities to make HTTP requests, - download files, and handle cached wordlists. + Main utility class for managing HTTP operations in BBOT. Uses blasthttp (Rust) as the + HTTP engine for all requests, downloads, and wordlist retrieval. + + All requests go through the shared blasthttp client on the parent helper, + which supports global rate limiting via ``web.http_rate_limit``. Attributes: parent_helper (object): The parent helper object containing scan configurations. http_debug (bool): Flag to indicate whether HTTP debugging is enabled. - ssl_verify (bool): Flag to indicate whether SSL verification is enabled. - web_client (BBOTAsyncClient): An instance of BBOTAsyncClient for making HTTP requests. - client_only_options (tuple): A tuple of options only applicable to the web client. + ssl_verify_target (bool): Whether to verify SSL for target-directed traffic (default False). + ssl_verify_infrastructure (bool): Whether to verify SSL for non-target traffic (default True). Examples: Basic web request: @@ -52,27 +74,167 @@ def __init__(self, parent_helper): self.web_config = self.config.get("web", {}) self.web_spider_depth = self.web_config.get("spider_depth", 1) self.web_spider_distance = self.web_config.get("spider_distance", 0) - self.web_clients = {} self.target = self.preset.target - self.ssl_verify = self.config.get("ssl_verify", False) - engine_debug = self.config.get("engine", {}).get("debug", False) - super().__init__( - server_kwargs={"config": self.config, "target": self.parent_helper.preset.target}, - debug=engine_debug, - ) + self.http_debug = self.web_config.get("debug", False) + self.ssl_verify_target = self.web_config.get("ssl_verify_target", False) + self.ssl_verify_infrastructure = self.web_config.get("ssl_verify_infrastructure", True) + # Pre-compute config values for request preprocessing + self._http_timeout = self.web_config.get("http_timeout", 10) + self._http_retries = self.web_config.get("http_retries", 1) + self._http_proxy = self.web_config.get("http_proxy", None) + self._http_proxy_exclude = self.web_config.get("http_proxy_exclude", []) or [] + ua = self.web_config.get("user_agent", "BBOT") + ua_suffix = self.web_config.get("user_agent_suffix") or "" + self._user_agent = f"{ua} {ua_suffix}".strip() + self._custom_headers = self.web_config.get("http_headers", {}) + self._custom_cookies = self.web_config.get("http_cookies", {}) + self._wildcard_cache = LRUCache(maxsize=50000) + self._wildcard_locks = NamedLock(max_size=50000) + + @property + def client(self): + """The shared rate-limited blasthttp client for target-directed traffic.""" + return self.parent_helper.blasthttp + + def _build_blasthttp_kwargs(self, url, **kwargs): + """ + Translate request kwargs into blasthttp.request() kwargs. - def AsyncClient(self, *args, **kwargs): - # cache by retries to prevent unwanted accumulation of clients - # (they are not garbage-collected) - retries = kwargs.get("retries", 1) - try: - return self.web_clients[retries] - except KeyError: - from .client import BBOTAsyncClient + Handles: method, headers, body/data/json, timeout, follow_redirects, + max_redirects, proxy, retries, params, cookies, auth. - client = BBOTAsyncClient.from_config(self.config, self.target, *args, persist_cookies=False, **kwargs) - self.web_clients[client.retries] = client - return client + Returns (url, method, blast_kwargs) — url may be modified if params were appended. + """ + method = kwargs.pop("method", "GET") + headers = kwargs.pop("headers", None) or {} + body = kwargs.pop("body", None) + data = kwargs.pop("data", None) + files = kwargs.pop("files", None) + json_body = kwargs.pop("json", None) + + body_sources = [ + name + for name, val in (("body", body), ("data", data), ("json", json_body), ("files", files)) + if val is not None + ] + if len(body_sources) > 1: + raise ValueError( + f"request() got conflicting body kwargs {body_sources}; pass at most one of body, data, json, files" + ) + timeout = kwargs.pop("timeout", self._http_timeout) + follow_redirects = kwargs.pop("follow_redirects", None) + max_redirects = kwargs.pop("max_redirects", None) + proxy = kwargs.pop("proxy", self._http_proxy) + no_proxy = kwargs.pop("no_proxy", self._http_proxy_exclude) + retries = kwargs.pop("retries", self._http_retries) + params = kwargs.pop("params", None) + cookies = kwargs.pop("cookies", None) + auth = kwargs.pop("auth", None) + ssl_verify = kwargs.pop("ssl_verify", None) + max_body_size = kwargs.pop("max_body_size", None) + request_target = kwargs.pop("request_target", None) + resolve_ip = kwargs.pop("resolve_ip", None) + ignore_bbot_global_settings = kwargs.pop("ignore_bbot_global_settings", False) + + # -- URL params -- + if params: + parsed = urlparse(url) + existing = parse_qs(parsed.query, keep_blank_values=True) + if isinstance(params, dict): + existing.update(params) + new_query = urlencode(existing, doseq=True) + url = urlunparse(parsed._replace(query=new_query)) + + # -- Headers as list of tuples -- + header_list = [] + + if not ignore_bbot_global_settings: + # User-Agent (can be overridden by caller) + if "User-Agent" not in headers: + header_list.append(("User-Agent", self._user_agent)) + + # Scan-level custom headers (only for in-scope URLs) + if self.target.in_target(url): + for hk, hv in self._custom_headers.items(): + if hk not in headers: + header_list.append((hk, str(hv))) + + # Scan-level custom cookies (merge with caller cookies) + if self._custom_cookies: + if cookies is None: + cookies = {} + for ck, cv in self._custom_cookies.items(): + if ck not in cookies: + cookies[ck] = cv + + # Caller-supplied headers + for hk, hv in headers.items(): + if isinstance(hv, list): + for v in hv: + header_list.append((hk, str(v))) + else: + header_list.append((hk, str(hv))) + + # -- JSON body -- + if json_body is not None: + body = json.dumps(json_body) + # Only set Content-Type if not already provided + if not any(k.lower() == "content-type" for k, _ in header_list): + header_list.append(("Content-Type", "application/json")) + + # -- Form data -- + if data is not None and body is None: + if isinstance(data, dict): + body = urlencode(data) + if not any(k.lower() == "content-type" for k, _ in header_list): + header_list.append(("Content-Type", "application/x-www-form-urlencoded")) + elif isinstance(data, (str, bytes)): + body = str(data) if isinstance(data, bytes) else data + + # -- Cookies -- + if cookies: + cookie_str = "; ".join(f"{ck}={cv}" for ck, cv in cookies.items()) + header_list.append(("Cookie", cookie_str)) + + # -- Basic auth -- + if auth: + import base64 + + user, passwd = auth + cred = base64.b64encode(f"{user}:{passwd}".encode()).decode() + header_list.append(("Authorization", f"Basic {cred}")) + + blast_kwargs = { + "method": method, + "headers": header_list, + "timeout": int(timeout) if timeout else self._http_timeout, + "verify_certs": bool(ssl_verify if ssl_verify is not None else self.ssl_verify_target), + "retries": int(retries), + } + + if body is not None: + blast_kwargs["body"] = body if isinstance(body, (bytes, bytearray)) else str(body) + if files is not None: + blast_kwargs["files"] = files + if follow_redirects is not None: + blast_kwargs["follow_redirects"] = follow_redirects + if max_redirects is not None: + blast_kwargs["max_redirects"] = int(max_redirects) + if proxy: + blast_kwargs["proxy"] = proxy + # no_proxy lists hosts that bypass the proxy; it only has an effect + # alongside a proxy (blasthttp errors if it's set without one), so + # only forward it when a proxy is actually in play. + if no_proxy: + blast_kwargs["no_proxy"] = list(no_proxy) + if max_body_size is not None: + blast_kwargs["max_body_size"] = int(max_body_size) + if request_target is not None: + blast_kwargs["request_target"] = request_target + if resolve_ip is not None: + blast_kwargs["resolve_ip"] = resolve_ip + + return url, method, blast_kwargs async def request(self, *args, **kwargs): """ @@ -92,23 +254,23 @@ async def request(self, *args, **kwargs): cookies (dict, optional): Dictionary or CookieJar object containing cookies. json (Any, optional): A JSON serializable Python object to send in the body. data (dict, optional): Dictionary, list of tuples, or bytes to send in the body. - files (dict, optional): Dictionary of 'name': file-like-objects for multipart encoding upload. + body (str, optional): Raw string body to send (not URL-encoded). auth (tuple, optional): Auth tuple to enable Basic/Digest/Custom HTTP auth. timeout (float, optional): The maximum time to wait for the request to complete. proxy (str, optional): HTTP proxy URL. allow_redirects (bool, optional): Enables or disables redirection. Defaults to None. - stream (bool, optional): Enables or disables response streaming. raise_error (bool, optional): Whether to raise exceptions for HTTP connect, timeout errors. Defaults to False. - client (httpx.AsyncClient, optional): A specific httpx.AsyncClient to use for the request. Defaults to self.web_client. - cache_for (int, optional): Time in seconds to cache the request. Not used currently. Defaults to None. + ssl_verify (bool, optional): Override SSL certificate verification for this request. + Defaults to ssl_verify_target for target traffic; pass ssl_verify_infrastructure for API/infra calls. + request_target (str, optional): Override the HTTP request-line target. + resolve_ip (str, optional): Connect TCP to this IP instead of DNS resolution. + ignore_bbot_global_settings (bool, optional): Skip User-Agent/header/cookie merging. Raises: - httpx.TimeoutException: If the request times out. - httpx.ConnectError: If the connection fails. - httpx.RequestError: For other request-related errors. + WebError: If raise_error is True and the request fails. Returns: - httpx.Response or None: The HTTP response object returned by the httpx library. + Response or None: The HTTP response object. Examples: >>> response = await self.helpers.request("https://www.evilcorp.com") @@ -118,68 +280,151 @@ async def request(self, *args, **kwargs): Note: If the web request fails, it will return None unless `raise_error` is `True`. """ - raise_error = kwargs.get("raise_error", False) - result = await self.run_and_return("request", *args, **kwargs) - if isinstance(result, dict) and "_request_error" in result: + raise_error = kwargs.pop("raise_error", False) + kwargs.pop("cache_for", None) + kwargs.pop("client", None) + kwargs.pop("stream", None) + + # allow vs follow + allow_redirects = kwargs.pop("allow_redirects", None) + if allow_redirects is not None and "follow_redirects" not in kwargs: + kwargs["follow_redirects"] = allow_redirects + + # In case of URL only as positional arg + if len(args) == 1: + kwargs["url"] = args[0] + args = () + + url = kwargs.pop("url", "") + + if not url: if raise_error: - error_msg = result["_request_error"] - response = result["_response"] - error = self.ERROR_CLASS(error_msg) - error.response = response + error = WebError("No URL provided") raise error - return result + return None - async def request_batch(self, urls, *args, **kwargs): - """ - Given a list of URLs, request them in parallel and yield responses as they come in. + if "method" not in kwargs: + kwargs["method"] = "GET" - Args: - urls (list[str]): List of URLs to visit - *args: Positional arguments to pass through to httpx - **kwargs: Keyword arguments to pass through to httpx + # Translate kwargs to blasthttp format + url, method, blast_kwargs = self._build_blasthttp_kwargs(url, **kwargs) - Examples: - >>> async for url, response in self.helpers.request_batch(urls, headers={"X-Test": "Test"}): - >>> if response is not None and response.status_code == 200: - >>> self.hugesuccess(response) - """ - agen = self.run_and_yield("request_batch", urls, *args, **kwargs) - while 1: - try: - yield await agen.__anext__() - except (StopAsyncIteration, GeneratorExit): - await agen.aclose() - break + try: + if self.http_debug: + log.trace(f"blasthttp request: {method} {url}") + + # blasthttp returns a native coroutine via pyo3-async-runtimes + response = await self.client.request(url, **blast_kwargs) + + if self.http_debug: + log.trace( + f"blasthttp response from {url}: {response.status_code} " + f"(Length: {len(response.content)}) headers: {response.headers}" + ) + return response - async def request_custom_batch(self, urls_and_kwargs): + except RuntimeError as e: + error_msg = str(e) + if raise_error: + error = WebError(error_msg) + raise error + # Classify error for appropriate log level + lower = error_msg.lower() + if "timeout" in lower: + attempts = blast_kwargs.get("retries", 0) + 1 + log.verbose(f"HTTP timeout to URL: {url} (after {attempts} attempt(s))") + elif "connect" in lower or "connection" in lower: + log.debug(f"HTTP connect failed to URL: {url}") + else: + log.trace(f"blasthttp error for {url}: {error_msg}") + except BaseException as e: + if not any(isinstance(_e, asyncio.exceptions.CancelledError) for _e in get_exception_chain(e)): + log.trace(f"Unhandled exception with request to URL: {url}: {e}") + log.trace(traceback.format_exc()) + raise + + async def request_batch_stream(self, urls, threads=10, **kwargs): """ - Make web requests in parallel with custom options for each request. Yield responses as they come in. + Request multiple URLs in parallel via blasthttp's native Rust batch engine, + yielding each response as soon as it completes (completion order, not input + order). - Similar to `request_batch` except it allows individual arguments for each URL. + Applies the same header/cookie/proxy/timeout logic as ``request()`` — each + entry is translated into a ``blasthttp.BatchConfig`` and dispatched through + ``blasthttp.request_batch_stream``. A slow request no longer blocks faster + peers behind it, and Python work overlaps with in-flight HTTP I/O. + + Each entry in ``urls`` can be: + - A plain URL string (uses shared ``**kwargs`` for all requests) + - A ``(url, per_request_kwargs)`` tuple for per-request options + - A ``(url, per_request_kwargs, tracker)`` tuple to attach arbitrary + tracking data that is yielded alongside the response + + Yields: + When entries are plain strings: ``(url, response)`` + When any entry includes a tracker: ``(url, response, tracker)`` Args: - urls_and_kwargs (list[tuple]): List of tuples in the format: (url, kwargs, custom_tracker) - where custom_tracker is an optional value for your own internal use. You may use it to - help correlate requests, etc. + urls: URLs to visit — strings or ``(url, kwargs[, tracker])`` tuples. + threads (int): Concurrency passed to blasthttp. Defaults to 10. + **kwargs: Default keyword arguments (same as ``request()``). + Overridden by per-request kwargs when entries are tuples. Examples: - >>> urls_and_kwargs = [ - >>> ("http://evilcorp.com/1", {"method": "GET"}, "request-1"), - >>> ("http://evilcorp.com/2", {"method": "POST"}, "request-2"), - >>> ] - >>> async for url, kwargs, custom_tracker, response in self.helpers.request_custom_batch( - >>> urls_and_kwargs - >>> ): - >>> if response is not None and response.status_code == 200: - >>> self.hugesuccess(response) + Simple (shared kwargs):: + + async for url, response in self.helpers.request_batch_stream(urls, headers={"X-Test": "Test"}): + ... + + Per-request kwargs with tracker:: + + reqs = [("http://example.com", {"method": "POST"}, "my-tracker")] + async for url, response, tracker in self.helpers.request_batch_stream(reqs): + ... """ - agen = self.run_and_yield("request_custom_batch", urls_and_kwargs) - while 1: - try: - yield await agen.__anext__() - except (StopAsyncIteration, GeneratorExit): - await agen.aclose() - break + import blasthttp + + # Parse entries into uniform (url, req_kwargs, tracker) tuples + entries = [] + has_tracker = False + for entry in urls: + if isinstance(entry, str): + entries.append((entry, kwargs, None)) + elif isinstance(entry, tuple): + url = entry[0] + req_kwargs = entry[1] if len(entry) > 1 and isinstance(entry[1], dict) else kwargs + tracker = entry[2] if len(entry) > 2 else None + if tracker is not None: + has_tracker = True + entries.append((url, req_kwargs, tracker)) + else: + entries.append((str(entry), kwargs, None)) + + if not entries: + return + + # Build BatchConfig objects using the same logic as request(). + # Map each config URL back to a queue of trackers so we can correlate + # completion-order results to original entries even when multiple entries + # share a URL. + from collections import deque + + configs = [] + trackers_by_url = {} + for url, req_kwargs, tracker in entries: + url, method, blast_kwargs = self._build_blasthttp_kwargs(url, **req_kwargs) + config = blasthttp.BatchConfig(url, **blast_kwargs) + configs.append(config) + trackers_by_url.setdefault(config.url, deque()).append(tracker) + + async for br in iter_batch_results(self.client.request_batch_stream(configs, concurrency=threads)): + response = br.response # blasthttp.Response or None + if has_tracker: + queue = trackers_by_url.get(br.url) + tracker = queue.popleft() if queue else None + yield br.url, response, tracker + else: + yield br.url, response async def download(self, url, **kwargs): """ @@ -196,7 +441,7 @@ async def download(self, url, **kwargs): A negative value disables caching. Defaults to -1. method (str, optional): The HTTP method to use for the request, defaults to 'GET'. raise_error (bool, optional): Whether to raise exceptions for HTTP connect, timeout errors. Defaults to False. - **kwargs: Additional keyword arguments to pass to the httpx request. + **kwargs: Additional keyword arguments to pass to request(). Returns: Path or None: The full path of the downloaded file as a Path object if successful, otherwise None. @@ -205,29 +450,67 @@ async def download(self, url, **kwargs): >>> filepath = await self.helpers.download("https://www.evilcorp.com/passwords.docx", cache_hrs=24) """ success = False + warn = kwargs.pop("warn", True) raise_error = kwargs.get("raise_error", False) filename = kwargs.pop("filename", self.parent_helper.cache_filename(url)) filename = truncate_filename(Path(filename).resolve()) - kwargs["filename"] = filename max_size = kwargs.pop("max_size", None) if max_size is not None: max_size = self.parent_helper.human_to_bytes(max_size) - kwargs["max_size"] = max_size cache_hrs = float(kwargs.pop("cache_hrs", -1)) + if cache_hrs > 0 and self.parent_helper.is_cached(url): log.debug(f"{url} is cached at {self.parent_helper.cache_filename(url)}") success = True else: - result = await self.run_and_return("download", url, **kwargs) - if isinstance(result, dict) and "_download_error" in result: + try: + kwargs["follow_redirects"] = kwargs.pop("follow_redirects", True) + if "method" not in kwargs: + kwargs["method"] = "GET" + if "ssl_verify" not in kwargs: + kwargs["ssl_verify"] = self.ssl_verify_infrastructure + kwargs["raise_error"] = True + # Use a longer timeout for downloads (default 5 minutes) + if "timeout" not in kwargs: + kwargs["timeout"] = 300 + # Raise the body size limit for downloads + if "max_body_size" not in kwargs: + if max_size is not None: + kwargs["max_body_size"] = max_size + else: + kwargs["max_body_size"] = 500 * 1024 * 1024 # 500MB default + + response = await self.request(url, **kwargs) + + if response is None: + raise HTTPStatusError(f"No response from {url}") + + log.debug(f"Download result: HTTP {response.status_code}") + response.raise_for_status() + + content = response.content + # Truncate if max_size specified + if max_size is not None: + if len(content) > max_size: + log.verbose( + f"Size of response from {url} exceeds {bytes_to_human(max_size)}, file will be truncated" + ) + content = content[:max_size] + + with open(filename, "wb") as f: + f.write(content) + success = True + + except (HTTPStatusError, WebError, RuntimeError) as e: + log_fn = log.verbose + if warn: + log_fn = log.warning + log_fn(f"Failed to download {url}: {e}") if raise_error: - error_msg = result["_download_error"] - response = result["_response"] - error = self.ERROR_CLASS(error_msg) - error.response = response + _response = getattr(e, "response", None) + error = WebError(str(e)) + error.response = _response raise error - elif result: - success = True if success: return filename @@ -238,8 +521,12 @@ async def wordlist(self, path, lines=None, zip=False, zip_filename=None, **kwarg Allows for optional line-based truncation and caching. Returns the full path of the wordlist file or a truncated version of it. + Also accepts a list of paths/URLs, in which case all wordlists are fetched and merged + into a single deduplicated file before being returned. + Args: - path (str): The local or remote path of the wordlist. + path (str | list): The local or remote path of the wordlist, or a list of paths/URLs + to merge into a single deduplicated wordlist. lines (int, optional): Number of lines to read from the wordlist. If specified, will return a truncated wordlist with this many lines. zip (bool, optional): Whether to unzip the file after downloading. Defaults to False. @@ -261,175 +548,64 @@ async def wordlist(self, path, lines=None, zip=False, zip_filename=None, **kwarg Fetching and truncating to the first 100 lines >>> wordlist_path = await self.helpers.wordlist("/root/rockyou.txt", lines=100) + + Merging multiple wordlists into one + >>> wordlist_path = await self.helpers.wordlist(["/custom.txt", "https://example.com/wordlist.txt"]) """ import zipfile if not path: raise WordlistError(f"Invalid wordlist: {path}") - if "cache_hrs" not in kwargs: - # 4320 hrs = 180 days = 6 months - kwargs["cache_hrs"] = 4320 - if self.parent_helper.is_url(path): - filename = await self.download(str(path), **kwargs) - if filename is None: - raise WordlistError(f"Unable to retrieve wordlist from {path}") - else: - filename = Path(path).resolve() - if not filename.is_file(): - raise WordlistError(f"Unable to find wordlist at {path}") - if zip: - if not zip_filename: - raise WordlistError("zip_filename must be specified when zip is True") - try: - with zipfile.ZipFile(filename, "r") as zip_ref: - if zip_filename not in zip_ref.namelist(): - raise WordlistError(f"File {zip_filename} not found in the zip archive {filename}") - zip_ref.extract(zip_filename, filename.parent) - filename = filename.parent / zip_filename - except Exception as e: - raise WordlistError(f"Error unzipping file {filename}: {e}") + # Handle list of wordlists - fetch each and merge into a single order-preserving deduplicated file, + # then fall through to the unified truncation logic below + if not isinstance(path, (str, Path)): + paths = list(path) + all_words = [] + for p in paths: + f = await self.wordlist(p, **kwargs) + all_words.extend(self.parent_helper.read_file(f)) + cache_key = "merged_wordlist:" + ":".join(sorted(str(p) for p in paths)) + filename = self.parent_helper.cache_filename(cache_key) + with open(filename, "w") as f: + for word in dict.fromkeys(all_words): + f.write(f"{word}\n") + else: + if "cache_hrs" not in kwargs: + # 4320 hrs = 180 days = 6 months + kwargs["cache_hrs"] = 4320 + if self.parent_helper.is_url(path): + filename = await self.download(str(path), **kwargs) + if filename is None: + raise WordlistError(f"Unable to retrieve wordlist from {path}") + else: + filename = Path(path).resolve() + if not filename.is_file(): + raise WordlistError(f"Unable to find wordlist at {path}") + + if zip: + if not zip_filename: + raise WordlistError("zip_filename must be specified when zip is True") + try: + with zipfile.ZipFile(filename, "r") as zip_ref: + if zip_filename not in zip_ref.namelist(): + raise WordlistError(f"File {zip_filename} not found in the zip archive {filename}") + zip_ref.extract(zip_filename, filename.parent) + filename = filename.parent / zip_filename + except Exception as e: + raise WordlistError(f"Error unzipping file {filename}: {e}") if lines is None: return filename - else: - lines = int(lines) - with open(filename) as f: - read_lines = f.readlines() - cache_key = f"{filename}:{lines}" - truncated_filename = self.parent_helper.cache_filename(cache_key) - with open(truncated_filename, "w") as f: - for line in read_lines[:lines]: - f.write(line) - return truncated_filename - - async def curl(self, *args, **kwargs): - """ - An asynchronous function that runs a cURL command with specified arguments and options. - - This function constructs and executes a cURL command based on the provided parameters. - It offers support for various cURL options such as headers, post data, and cookies. - - Args: - *args: Variable length argument list for positional arguments. Unused in this function. - url (str): The URL for the cURL request. Mandatory. - raw_path (bool, optional): If True, activates '--path-as-is' in cURL. Defaults to False. - headers (dict, optional): A dictionary of HTTP headers to include in the request. - ignore_bbot_global_settings (bool, optional): If True, ignores the global settings of BBOT. Defaults to False. - post_data (dict, optional): A dictionary containing data to be sent in the request body. - method (str, optional): The HTTP method to use for the request (e.g., 'GET', 'POST'). - cookies (dict, optional): A dictionary of cookies to include in the request. - path_override (str, optional): Overrides the request-target to use in the HTTP request line. - head_mode (bool, optional): If True, includes '-I' to fetch headers only. Defaults to None. - raw_body (str, optional): Raw string to be sent in the body of the request. - **kwargs: Arbitrary keyword arguments that will be forwarded to the HTTP request function. - - Returns: - str: The output of the cURL command. - - Raises: - CurlError: If 'url' is not supplied. - - Examples: - >>> output = await curl(url="https://example.com", headers={"X-Header": "Wat"}) - >>> print(output) - """ - url = kwargs.get("url", "") - - if not url: - raise CurlError("No URL supplied to CURL helper") - - curl_command = ["curl", url, "-s"] - - raw_path = kwargs.get("raw_path", False) - if raw_path: - curl_command.append("--path-as-is") - - # respect global ssl verify settings - if self.ssl_verify is not True: - curl_command.append("-k") - - headers = kwargs.get("headers", {}) - cookies = kwargs.get("cookies", {}) - - ignore_bbot_global_settings = kwargs.get("ignore_bbot_global_settings", False) - - if ignore_bbot_global_settings: - http_timeout = 20 # setting 20 as a worse-case setting - log.debug("ignore_bbot_global_settings enabled. Global settings will not be applied") - else: - http_timeout = self.parent_helper.web_config.get("http_timeout", 20) - user_agent = self.parent_helper.web_config.get("user_agent", "BBOT") - - if "User-Agent" not in headers: - headers["User-Agent"] = user_agent - - # only add custom headers / cookies if the URL is in-scope - if self.parent_helper.preset.in_scope(url): - for hk, hv in self.web_config.get("http_headers", {}).items(): - # Only add the header if it doesn't already exist in the headers dictionary - if hk not in headers: - headers[hk] = hv - - for ck, cv in self.web_config.get("http_cookies", {}).items(): - # don't clobber cookies - if ck not in cookies: - cookies[ck] = cv - - # add the timeout - if "timeout" not in kwargs: - timeout = http_timeout - - curl_command.append("-m") - curl_command.append(str(timeout)) - - for k, v in headers.items(): - if isinstance(v, list): - for x in v: - curl_command.append("-H") - curl_command.append(f"{k}: {x}") - - else: - curl_command.append("-H") - curl_command.append(f"{k}: {v}") - - post_data = kwargs.get("post_data", {}) - if len(post_data.items()) > 0: - curl_command.append("-d") - post_data_str = "" - for k, v in post_data.items(): - post_data_str += f"&{k}={v}" - curl_command.append(post_data_str.lstrip("&")) - - method = kwargs.get("method", "") - if method: - curl_command.append("-X") - curl_command.append(method) - - cookies = kwargs.get("cookies", "") - if cookies: - curl_command.append("-b") - cookies_str = "" - for k, v in cookies.items(): - cookies_str += f"{k}={v}; " - curl_command.append(f"{cookies_str.rstrip(' ')}") - - path_override = kwargs.get("path_override", None) - if path_override: - curl_command.append("--request-target") - curl_command.append(f"{path_override}") - - head_mode = kwargs.get("head_mode", None) - if head_mode: - curl_command.append("-I") - - raw_body = kwargs.get("raw_body", None) - if raw_body: - curl_command.append("-d") - curl_command.append(raw_body) - log.verbose(f"Running curl command: {curl_command}") - output = (await self.parent_helper.run(curl_command)).stdout - return output + lines = int(lines) + with open(filename) as f: + read_lines = f.readlines() + cache_key = f"{filename}:{lines}" + truncated_filename = self.parent_helper.cache_filename(cache_key) + with open(truncated_filename, "w") as f: + for line in read_lines[:lines]: + f.write(line) + return truncated_filename def beautifulsoup( self, @@ -474,13 +650,16 @@ def beautifulsoup( - Write tests for this function Examples: - >>> soup = self.helpers.beautifulsoup(event.data["body"], "html.parser") + >>> soup = self.helpers.beautifulsoup(event.body, "html.parser") Perform an html parse of the 'markup' argument and return a soup instance >>> email_type = soup.find(type="email") Searches the soup instance for all occurrences of the passed in argument """ try: + # If a response object is passed, extract the text + if hasattr(markup, "text") and not isinstance(markup, (str, bytes)): + markup = markup.text soup = BeautifulSoup( markup, features, builder, parse_only, from_encoding, exclude_encodings, element_classes, **kwargs ) @@ -489,9 +668,71 @@ def beautifulsoup( log.debug(f"Error parsing beautifulsoup: {e}") return False + async def is_http_wildcard_host(self, scheme, host, port): + """Detect whether a host returns the same response regardless of URL path. + + Probes two random paths and the root URL via HttpCompare. Cached per + (scheme, host, port); 3 HTTP requests on first call, instant thereafter. + + Returns: + HttpCompare -- host is a wildcard responder (cached baseline). + False -- host distinguishes responses by path. + None -- probe failed after retry; treat as unknown. + """ + key = (scheme, host, port) + if key in self._wildcard_cache: + return self._wildcard_cache[key] + async with self._wildcard_locks.lock(key): + if key in self._wildcard_cache: + return self._wildcard_cache[key] + result = await self._probe_wildcard_host(scheme, host, port) + if result == "retry": + log.debug(f"is_http_wildcard_host: first probe failed for {host}:{port}; retrying once") + result = await self._probe_wildcard_host(scheme, host, port) + if result == "retry": + log.debug(f"is_http_wildcard_host: retry also failed for {host}:{port}; caching as unknown") + self._wildcard_cache[key] = None + return None + self._wildcard_cache[key] = result + return result + + async def _probe_wildcard_host(self, scheme, host, port): + """Single probe attempt. Returns HttpCompare (wildcard), False (not wildcard), or "retry".""" + baseline_url_1 = ( + f"{scheme}://{host}:{port}/{self.parent_helper.rand_string(12)}/{self.parent_helper.rand_string(8)}" + ) + baseline_url_2 = ( + f"{scheme}://{host}:{port}/{self.parent_helper.rand_string(12)}/{self.parent_helper.rand_string(8)}" + ) + compare = HttpCompare( + baseline_url_1, + self.parent_helper, + allow_redirects=False, + timeout=10, + baseline_url_2=baseline_url_2, + ) + try: + await compare._baseline() + except HttpCompareError as e: + log.debug(f"is_http_wildcard_host: baseline failed for {host}:{port}: {e}") + return "retry" + root_url = f"{scheme}://{host}:{port}/" + try: + root_match, root_reasons, _, _ = await compare.compare(root_url) + except HttpCompareError as e: + log.debug(f"is_http_wildcard_host: root probe failed for {host}:{port}: {e}") + return "retry" + if not root_match: + log.debug( + f"is_http_wildcard_host: {host}:{port} root distinct from random-path baseline ({root_reasons}); not a wildcard" + ) + return False + log.verbose(f"is_http_wildcard_host: {scheme}://{host}:{port} is an HTTP wildcard responder") + return compare + def response_to_json(self, response): """ - Convert web response to JSON object, similar to the output of `httpx -irr -json` + Convert web response to JSON object, to a JSON-serializable dict. """ if response is None: diff --git a/bbot/core/helpers/yara_helper.py b/bbot/core/helpers/yara_helper.py index 7f9428b55b..7ee37451c0 100644 --- a/bbot/core/helpers/yara_helper.py +++ b/bbot/core/helpers/yara_helper.py @@ -40,7 +40,7 @@ async def match(self, compiled_rules, text): Given a compiled YARA rule and a body of text, return a list of strings that match the rule """ matched_strings = [] - matches = await self.parent_helper.run_in_executor(compiled_rules.match, data=text) + matches = await self.parent_helper.run_in_executor_cpu(compiled_rules.match, data=text) if matches: for match in matches: for string_match in match.strings: diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 2360129457..cee81ef6f2 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -1,15 +1,18 @@ +import os import re import ast import sys +import stat +import yaml import atexit +import shutil import pickle import logging +import tempfile import importlib -import omegaconf import traceback from copy import copy from pathlib import Path -from omegaconf import OmegaConf from contextlib import suppress from bbot.core import CORE @@ -40,6 +43,229 @@ def find_class(self, module, name): bbot_code_dir = Path(__file__).parent.parent +# Bump when the preloader's output schema or validation rules change. Folded +# into the per-module cache_key so stale entries from older bbot versions get +# rebuilt instead of silently bypassing new checks. +PRELOAD_CACHE_VERSION = 3 + + +_UNEVALUATED = object() + + +def _eval_ast_default(node): + """ + Extract a literal default value from an AST node. Returns _UNEVALUATED if + the node can't be resolved statically (e.g. `default_factory=lambda: ...` + or a non-constant expression). Preload uses this for display-time defaults + only; actual validation happens at bake time against the real pydantic + Config class. + """ + if node is None: + return _UNEVALUATED + try: + return ast.literal_eval(node) + except (ValueError, TypeError, SyntaxError): + # Recognize a few common default_factory values. + if isinstance(node, ast.Name): + return {"list": [], "dict": {}, "set": set(), "tuple": (), "str": "", "int": 0, "float": 0.0}.get( + node.id, _UNEVALUATED + ) + return _UNEVALUATED + + +def _exec_config_class(source: str, module_name: str): + """ + Execute a `class Config(BaseModuleConfig):` snippet in a controlled + namespace and return the resulting class. The namespace provides exactly + what a Config block is allowed to reference: the typing primitives, + pydantic's `Field` factory and validator decorators, and `BaseModuleConfig`. + + This replaces parsing annotations as strings: pydantic handles every + valid type expression (`Optional[str]`, `Literal["a", "b"]`, + `list[Union[int, str]]`, …) without any hand-rolled resolver. + """ + from typing import Annotated, Any, Dict, List, Literal, Optional, Set, Tuple, Union + from pydantic import AfterValidator, BeforeValidator, field_validator, model_validator + from bbot.core.config.models import BaseModuleConfig, ConfidenceLiteral, Field, SeverityLiteral + + namespace: dict = { + "Annotated": Annotated, + "Any": Any, + "Dict": Dict, + "List": List, + "Literal": Literal, + "Optional": Optional, + "Set": Set, + "Tuple": Tuple, + "Union": Union, + "Field": Field, + "field_validator": field_validator, + "model_validator": model_validator, + "BeforeValidator": BeforeValidator, + "AfterValidator": AfterValidator, + "BaseModuleConfig": BaseModuleConfig, + "SeverityLiteral": SeverityLiteral, + "ConfidenceLiteral": ConfidenceLiteral, + } + try: + exec(source, namespace) + except Exception as e: + raise BBOTError( + f'module "{module_name}" has an invalid Config class ({type(e).__name__}: {e}). ' + "Config blocks may only reference: Optional, Union, Literal, Any, List, Dict, Tuple, Set, " + "Field, field_validator, model_validator, BeforeValidator, AfterValidator, BaseModuleConfig, " + "and Python builtins." + ) from e + cfg = namespace.get("Config") + if cfg is None: + raise BBOTError(f'module "{module_name}": Config snippet did not define a class named "Config"') + return cfg + + +def _build_validation_schema(preloaded: dict): + """ + Build the composite preset validation schema. + + Structure: + FullPresetSchema + ├── (all PresetSchema fields — target, modules, flags, …) + └── config: FullBBOTConfig + ├── (all BBOTConfig fields — scope, dns, web, …) + └── modules: ModulesSchema + ├── nuclei: NucleiModuleConfig + ├── httpx: HttpxModuleConfig + ├── sslcert: SslcertModuleConfig + └── … one field per known module + + A single `FullPresetSchema.model_validate(preset_dict)` call then catches + every class of error in one pass: + - top-level preset typos (extra='forbid' on PresetSchema) + - global config typos / wrong types (extra='forbid' on BBOTConfig) + - unknown module names (extra='forbid' on ModulesSchema) + - wrong module option names / wrong types (extra='forbid' per module) + """ + import warnings + from typing import Optional + from pydantic import ConfigDict, Field, create_model + from bbot.core.config.models import BaseModuleConfig, BBOTConfig, PresetSchema + + module_fields = {} + for name, data in preloaded.items(): + source = data.get("config_source") + if not source: + # Module declares no Config — only the universal options apply. + field_type = BaseModuleConfig + else: + try: + field_type = _exec_config_class(source, name) + except BBOTError as e: + # The Config references a name not available in the isolated exec + # namespace (a module-level constant, imported type, Enum, …); it's + # valid at real import time. Don't reject the module — accept any + # config for it (its own options just aren't strictly validated here). + log.debug(f"{e} -- accepting any config for module '{name}'") + field_type = dict + module_fields[name] = (Optional[field_type], Field(default=None)) + + # Some module names (e.g. `json`) shadow BaseModel's deprecated method + # names and trigger a UserWarning. The field still validates correctly; + # silence the noise. + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=r'Field name ".+" in ".+" shadows an attribute in parent "BaseModel"', + category=UserWarning, + ) + ModulesSchema = create_model( + "ModulesSchema", + __config__=ConfigDict(extra="forbid"), + **module_fields, + ) + FullBBOTConfig = create_model( + "FullBBOTConfig", + __base__=BBOTConfig, + modules=(Optional[ModulesSchema], Field(default=None)), + ) + FullPresetSchema = create_model( + "FullPresetSchema", + __base__=PresetSchema, + config=(Optional[FullBBOTConfig], Field(default=None)), + ) + return FullPresetSchema + + +def _extract_pydantic_config(config_class: ast.ClassDef) -> tuple[dict, dict, set, set, dict]: + """ + Walk a `class Config(BaseModuleConfig):` block and extract + `(defaults, descriptions, sensitive, mandatory, types)` -- cheap metadata + used for `bbot -l`, type-directed config coercion, and similar listing + paths, without importing or exec-ing anything. + + `types` maps each field name to its raw annotation string (e.g. + `"bool"`, `"Union[str, list[str]]"`, `"Literal['manual', 'severe']"`). + The actual typed pydantic class is built later via `_exec_config_class` + on the captured source text. + """ + defaults: dict = {} + descriptions: dict = {} + sensitive: set = set() + mandatory: set = set() + types: dict = {} + for node in config_class.body: + # `model_config = ConfigDict(...)` etc. are plain assigns, not typed -- skip. + if not isinstance(node, ast.AnnAssign) or not isinstance(node.target, ast.Name): + continue + name = node.target.id + if name.startswith("_"): + continue + + types[name] = ast.unparse(node.annotation) + default = _UNEVALUATED + description = "" + is_sensitive = False + is_mandatory = False + + value = node.value + if isinstance(value, ast.Call) and isinstance(value.func, ast.Name) and value.func.id == "Field": + # Field(default, description="...", default_factory=..., sensitive=..., mandatory=..., ...) + # - first positional arg is the default, if given + if value.args: + default = _eval_ast_default(value.args[0]) + for kw in value.keywords: + if kw.arg == "default": + default = _eval_ast_default(kw.value) + elif kw.arg == "default_factory": + default = _eval_ast_default(kw.value) + elif kw.arg == "description": + with suppress(ValueError, TypeError, SyntaxError): + description = ast.literal_eval(kw.value) + elif kw.arg == "sensitive": + with suppress(ValueError, TypeError, SyntaxError): + is_sensitive = bool(ast.literal_eval(kw.value)) + elif kw.arg == "mandatory": + with suppress(ValueError, TypeError, SyntaxError): + is_mandatory = bool(ast.literal_eval(kw.value)) + elif kw.arg == "json_schema_extra": + with suppress(ValueError, TypeError, SyntaxError): + extra = ast.literal_eval(kw.value) + if isinstance(extra, dict): + is_sensitive = is_sensitive or bool(extra.get("sensitive")) + is_mandatory = is_mandatory or bool(extra.get("mandatory")) + elif value is not None: + default = _eval_ast_default(value) + + if default is _UNEVALUATED: + # couldn't statically determine; fall back to None so listing still works + default = None + defaults[name] = default + descriptions[name] = description + if is_sensitive: + sensitive.add(name) + if is_mandatory: + mandatory.add(name) + return defaults, descriptions, sensitive, mandatory, types + + class ModuleLoader: """ Main class responsible for preloading BBOT modules. @@ -54,7 +280,7 @@ class ModuleLoader: module_dir_regex = re.compile(r"^[a-z][a-z0-9_]*$") # if a module consumes these event types, automatically assume these dependencies - default_module_deps = {"HTTP_RESPONSE": "httpx", "URL": "httpx", "SOCIAL": "social"} + default_module_deps = {"HTTP_RESPONSE": "http", "URL": "http", "SOCIAL": "social"} def __init__(self): self.core = CORE @@ -70,6 +296,10 @@ def __init__(self): self.internal_module_choices = set() self._preload_cache = None + # Composite preset-validation schema, built from preloaded modules. + # Invalidated whenever a new module is preloaded. + self._validation_schema = None + self._config_type_index = None self._module_dirs = set() self._module_dirs_preloaded = set() @@ -151,7 +381,7 @@ def preload(self, module_dirs=None): module_file = module_file.resolve() # try to load from cache - module_cache_key = (str(module_file), tuple(module_file.stat())) + module_cache_key = (PRELOAD_CACHE_VERSION, str(module_file), tuple(module_file.stat())) preloaded = self.preload_cache.get(module_name, {}) cache_key = preloaded.get("cache_key", ()) if preloaded and module_cache_key == cache_key: @@ -184,6 +414,12 @@ def preload(self, module_dirs=None): preloaded["namespace"] = namespace preloaded["cache_key"] = module_cache_key + except BBOTError as e: + # Intentional, user-facing errors raised from preload_module + # (e.g. the pre-3.0 options-dict migration message). Skip the + # traceback so the message reads cleanly. + log_to_stderr(str(e), level="CRITICAL") + sys.exit(1) except Exception: log_to_stderr(f"Error preloading {module_file}\n\n{traceback.format_exc()}", level="CRITICAL") log_to_stderr(f"Error in {module_file.name}", level="CRITICAL") @@ -202,18 +438,18 @@ def preload(self, module_dirs=None): self.flag_choices.update(set(flags)) self.__preloaded[module_name] = preloaded - config = OmegaConf.create(preloaded.get("config", {})) - self._configs[module_name] = config + self._configs[module_name] = dict(preloaded.get("config", {})) self._module_dirs_preloaded.add(module_dir) # update default config with module defaults - module_config = omegaconf.OmegaConf.create( - { - "modules": self.configs(), - } - ) - self.core.merge_default(module_config) + self.core.merge_default({"modules": self.configs()}) + + # invalidate the composite validation schema; it'll rebuild lazily + # on next access now that the set of modules has changed + if new_modules: + self._validation_schema = None + self._config_type_index = None return new_modules @@ -260,12 +496,94 @@ def preloaded(self, type=None): return preloaded def configs(self, type=None): - configs = {} if type is not None: - configs = {k: v for k, v in self._configs.items() if self.check_type(k, type)} - else: - configs = dict(self._configs) - return OmegaConf.create(configs) + return {k: dict(v) for k, v in self._configs.items() if self.check_type(k, type)} + return {k: dict(v) for k, v in self._configs.items()} + + @property + def validation_schema(self): + """ + The composite pydantic schema for validating a full preset dict. + + Built lazily from preloaded module metadata and cached. Rebuilt when + `preload()` discovers new modules (e.g. after `add_module_dir`). + """ + if self._validation_schema is None: + self._validation_schema = _build_validation_schema(self._preloaded) + return self._validation_schema + + @property + def config_schema(self): + """ + The runtime `BBOTConfig` schema with per-module configs grafted in. + + This is `validation_schema.config` (i.e. `FullBBOTConfig`) and is the + right model to walk a config dict against — used by + `BBOTCore.no_secrets_config()` and `BBOTCore.secrets_only_config()` to + partition sensitive fields. + """ + from bbot.core.config.models import _unwrap_optional + + field = self.validation_schema.model_fields.get("config") + if field is None: + from bbot.core.config.models import BBOTConfig + + return BBOTConfig + return _unwrap_optional(field.annotation) + + @property + def config_type_index(self): + """{dotted_config_path: TypeAdapter} for every known config option. + + Built by walking the materialized config schema (`config_schema`, i.e. + global config + every module's exec'd Config), so a single pydantic + TypeAdapter per leaf field drives coercion -- no hand-rolled type-name + parsing. Adapters are memoized by annotation (many fields share e.g. + `Optional[str]`). Nested models are recursed into via `_field_submodel`, + so `modules..