Dev-> 3.0 sync

.github/workflows/benchmark.yml

@@ -63,7 +63,7 @@ jobs:
 
     # Comment on PR with benchmark results
     - name: Comment benchmark results on PR
-      uses: actions/github-script@v8
+      uses: actions/github-script@v9
       with:
         script: |
           const fs = require('fs');

.github/workflows/cla.yml

@@ -0,0 +1,97 @@
+name: "CLA Assistant"
+on:
+  issue_comment:
+    types: [created]
+  pull_request_target:
+    types: [opened, closed, synchronize]
+
+permissions:
+  pull-requests: write
+  statuses: write
+
+jobs:
+  CLAAssistant:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Generate token from GitHub App
+        id: app-token
+        uses: actions/create-github-app-token@v3
+        with:
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          owner: blacklanternsecurity
+
+      - name: Check all committers against org and allowlist
+        id: cla-check
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+        run: |
+          if [ "${{ github.event_name }}" = "pull_request_target" ]; then
+            PR_NUM="${{ github.event.pull_request.number }}"
+          else
+            PR_NUM="${{ github.event.issue.number }}"
+          fi
+
+          COMMITTERS=$(gh api "repos/${{ github.repository }}/pulls/$PR_NUM/commits" --paginate --jq '.[].author.login' | sort -u)
+          ALL_EXEMPT=true
+
+          for LOGIN in $COMMITTERS; do
+            # treat commits with no associated GitHub login as non-exempt
+            if [ -z "$LOGIN" ] || [ "$LOGIN" = "null" ]; then
+              echo "Unknown committer (no GitHub login) — not exempt"
+              ALL_EXEMPT=false
+              break
+            fi
+
+            EXEMPT=false
+
+            # check if account type is Bot (GitHub App accounts)
+            AUTHOR_TYPE=$(gh api "users/${LOGIN}" --jq '.type' 2>/dev/null || echo "Unknown")
+            if [ "$AUTHOR_TYPE" = "Bot" ]; then
+              echo "$LOGIN is a Bot account — exempt"
+              EXEMPT=true
+            fi
+
+            # check org membership
+            if [ "$EXEMPT" = "false" ]; then
+              if gh api "orgs/blacklanternsecurity/members/$LOGIN" > /dev/null 2>&1; then
+                echo "$LOGIN is an org member — exempt"
+                EXEMPT=true
+              fi
+            fi
+
+            if [ "$EXEMPT" = "false" ]; then
+              echo "$LOGIN is not exempt — CLA required"
+              ALL_EXEMPT=false
+              break
+            fi
+          done
+
+          echo "all_exempt=$ALL_EXEMPT" >> "$GITHUB_OUTPUT"
+
+      - name: Skip CLA when all committers are exempt
+        if: steps.cla-check.outputs.all_exempt == 'true' && github.event_name == 'pull_request_target'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api --method POST "repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }}" \
+            -f state=success \
+            -f context="CLAAssistant" \
+            -f description="CLA check skipped — all committers are org members or bots"
+
+      - name: "CLA Assistant"
+        if: |
+          (steps.cla-check.outputs.all_exempt != 'true') &&
+          ((github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target')
+        uses: contributor-assistant/github-action@v2.6.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PERSONAL_ACCESS_TOKEN: ${{ steps.app-token.outputs.token }}
+        with:
+          path-to-signatures: "signatures/version1/cla.json"
+          path-to-document: "https://github.com/blacklanternsecurity/CLA/blob/main/ICLA.md"
+          branch: "main"
+          allowlist: "dependabot[bot],github-actions[bot],renovate[bot]"
+          remote-organization-name: "blacklanternsecurity"
+          remote-repository-name: "CLA"
+          lock-pullrequest-aftermerge: "false"

.github/workflows/distro_tests.yml

@@ -23,14 +23,6 @@ jobs:
             . /etc/os-release
             if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ] || [ "$ID" = "kali" ] || [ "$ID" = "parrotsec" ]; then
               export DEBIAN_FRONTEND=noninteractive
-              # Pin Parrot to MIT mirror to avoid stale third-party mirrors (e.g. mirror.clarkson.edu)
-              # Note: parrotsec/security image reports ID=debian, so we detect via parrot.list
-              if [ -f /etc/apt/sources.list.d/parrot.list ]; then
-                rm -f /etc/apt/sources.list /etc/apt/sources.list.old /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources
-                echo "deb https://mirrors.mit.edu/parrot/ echo main contrib non-free non-free-firmware" > /etc/apt/sources.list.d/parrot.list
-                echo "deb https://mirrors.mit.edu/parrot/ echo-security main contrib non-free non-free-firmware" >> /etc/apt/sources.list.d/parrot.list
-                echo "deb https://mirrors.mit.edu/parrot/ echo-backports main contrib non-free non-free-firmware" >> /etc/apt/sources.list.d/parrot.list
-              fi
               apt-get update
               apt-get -y install curl git bash build-essential docker.io libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev
             elif [ "$ID" = "alpine" ]; then

.github/workflows/docs_updater.yml

@@ -20,10 +20,10 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v7
       - name: Install dependencies
-        run: uv sync --group dev
+        run: uv sync --frozen --group dev
       - name: Generate docs
         run: |
-          uv run bbot/scripts/docs.py
+          uv run --no-sync bbot/scripts/docs.py
       - name: Create or Update Pull Request
         uses: peter-evans/create-pull-request@v8
         with:

.github/workflows/tests.yml

@@ -46,7 +46,7 @@ jobs:
           name: pytest-debug-logs-${{ env.PYTHON_VERSION }}
           path: pytest_debug.log
       - name: Upload Code Coverage
-        uses: codecov/codecov-action@v5
+        uses: codecov/codecov-action@v6
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./cov.xml
@@ -61,9 +61,9 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -76,30 +76,36 @@ jobs:
       - name: Calculate version
         id: calc_version
         run: |
-          # Get base version from latest stable tag (exclude rc tags, strip 'v' prefix)
-          LATEST_STABLE_TAG=$(git describe --tags --abbrev=0 --exclude="*rc*")
-          BASE_VERSION=$(echo "$LATEST_STABLE_TAG" | sed 's/^v//')
+          # Source of truth for base version is pyproject.toml
+          BASE_VERSION=$(python3 -c "
+          import re
+          text = open('pyproject.toml').read()
+          print(re.search(r'^version\s*=\s*\"([^\"]+)\"', text, re.MULTILINE).group(1))
+          ")
 
           if [[ "${{ github.ref }}" == "refs/heads/stable" ]]; then
-            # Stable: clean version from tag
             VERSION="$BASE_VERSION"
           elif [[ "${{ github.ref }}" == "refs/heads/dev" ]]; then
-            # Dev: version.distancerc (e.g., 3.0.0.123rc)
-            DISTANCE=$(git rev-list ${LATEST_STABLE_TAG}..HEAD --count)
+            # Dev: base version + commit distance from stable as rc (e.g., 3.0.0.123rc)
+            STABLE_HEAD=$(git rev-parse origin/stable 2>/dev/null || git rev-list --max-parents=0 HEAD)
+            DISTANCE=$(git rev-list ${STABLE_HEAD}..HEAD --count)
             VERSION="${BASE_VERSION}.${DISTANCE}rc"
           fi
 
           echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
           echo "Calculated version: $VERSION"
 
-          # Write version to file for hatchling to pick up
-          echo "__version__ = \"$VERSION\"" > bbot/_version.py
+          # For dev builds, write the rc version to pyproject.toml for hatchling
+          # Stable builds use pyproject.toml as-is
+          if [[ "${{ github.ref }}" == "refs/heads/dev" ]]; then
+            sed -i "s/^version = .*/version = \"$VERSION\"/" pyproject.toml
+          fi
       - name: Build Pypi package
         if: github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev'
         run: uv build
       - name: Publish Pypi package
         if: github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev'
-        uses: pypa/gh-action-pypi-publish@release/v1.13
+        uses: pypa/gh-action-pypi-publish@release/v1.14
         with:
           password: ${{ secrets.PYPI_API_TOKEN }}
       - name: Get BBOT version
@@ -115,18 +121,16 @@ jobs:
           echo "BBOT_VERSION_MAJOR=$MAJOR" >> $GITHUB_OUTPUT
       - name: Publish to Docker Hub (dev)
         if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           push: true
           context: .
           tags: |
             blacklanternsecurity/bbot:dev
             blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION }}
-            blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR_MINOR }}
-            blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR }}
       - name: Publish to Docker Hub (stable)
         if: github.event_name == 'push' && github.ref == 'refs/heads/stable'
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           push: true
           context: .
@@ -138,19 +142,17 @@ jobs:
             blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR }}
       - name: Publish Full Docker Image to Docker Hub (dev)
         if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           push: true
           file: Dockerfile.full
           context: .
           tags: |
             blacklanternsecurity/bbot:dev-full
             blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION }}-full
-            blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR_MINOR }}-full
-            blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR }}-full
       - name: Publish Full Docker Image to Docker Hub (stable)
         if: github.event_name == 'push' && github.ref == 'refs/heads/stable'
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           push: true
           file: Dockerfile.full
@@ -239,7 +241,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v7
       - name: Install dependencies
-        run: uv sync --only-group docs
+        run: uv sync --frozen --group docs
       - name: Configure Git
         run: |
           git config user.name github-actions
@@ -253,11 +255,11 @@ jobs:
       - name: Generate docs (stable branch)
         if: github.ref == 'refs/heads/stable'
         run: |
-          uv run mike deploy Stable
+          uv run --no-sync mike deploy Stable
       - name: Generate docs (dev branch)
         if: github.ref == 'refs/heads/dev'
         run: |
-          uv run mike deploy Dev
+          uv run --no-sync mike deploy Dev
       - name: Publish docs
         run: |
           git switch gh-pages

.gitignore

@@ -3,3 +3,4 @@ __pycache__/
 /data/
 /neo4j/
 .DS_Store
+pytest_debug.log

AGENTS.md

@@ -19,7 +19,7 @@ BBOT has extremely thorough tests, including **one or more individual tests for
 ## Tooling
 
 - **Package manager**: [uv](https://docs.astral.sh/uv/)
-- **Linter/formatter**: [ruff](https://docs.astral.sh/ruff/) (pinned to 0.15.2)
+- **Linter/formatter**: [ruff](https://docs.astral.sh/ruff/) (pinned to 0.15.10)
 - **Test framework**: [pytest](https://docs.pytest.org/) with pytest-asyncio
 - **Python**: 3.10 - 3.14
 
@@ -146,6 +146,7 @@ Key helpers:
 | Helper | What it does |
 |--------|-------------|
 | `self.helpers.request(url)` | Make an HTTP request (with retries, SSL handling, etc.) |
+| `self.helpers.blasthttp` | Shared blasthttp client (rate-limited via `web.http_rate_limit` config) |
 | `self.helpers.resolve(host)` | DNS resolution |
 | `self.helpers.is_ip(s)` | Check if string is an IP |
 | `self.helpers.is_dns_name(s)` | Check if string is a hostname |
@@ -219,7 +220,7 @@ from .base import ModuleTestBase
 
 class TestMyModule(ModuleTestBase):
     async def setup_after_prep(self, module_test):
-        module_test.httpx_mock.add_response(
+        module_test.blasthttp_mock.add_response(
             url="https://api.example.com/lookup?domain=blacklanternsecurity.com",
             json={"emails": ["info@blacklanternsecurity.com"]},
         )
@@ -370,7 +371,7 @@ Whether to process seed events (the initial targets provided to the scan).
 Whether to accept "special" URLs (e.g. JavaScript files) that are not normally distributed to web modules.
 
 ```python
-# httpx.py - needs to process all URLs including special ones
+# http.py - needs to process all URLs including special ones
 accept_url_special = True
 ```
 
@@ -535,7 +536,7 @@ _preserve_graph = True
 Exclude this module from scan statistics. Used by output and report modules.
 
 ##### `_disable_auto_module_deps` (bool) -- default: `False`
-Prevent BBOT from automatically enabling dependency modules. For example, if your module watches `URL` events, BBOT normally auto-enables `httpx`. Set this to `True` to prevent that.
+Prevent BBOT from automatically enabling dependency modules. For example, if your module watches `URL` events, BBOT normally auto-enables `http`. Set this to `True` to prevent that.
 
 ---
 
@@ -875,7 +876,7 @@ class TestMyModule(ModuleTestBase):
     targets = ["http://127.0.0.1:8888"]
 
     # Optional: override which modules are enabled
-    modules_overrides = ["httpx", "my_module"]
+    modules_overrides = ["http", "my_module"]
 
     # Optional: override config
     config_overrides = {"modules": {"my_module": {"some_option": True}}}
@@ -887,7 +888,7 @@ class TestMyModule(ModuleTestBase):
     async def setup_after_prep(self, module_test):
         """Called AFTER the scan is prepared. Modify modules, add mocks here."""
         # Mock an HTTP response
-        module_test.httpx_mock.add_response(
+        module_test.blasthttp_mock.add_response(
             url="https://api.example.com/lookup?domain=blacklanternsecurity.com",
             json={"results": ["sub.blacklanternsecurity.com"]},
         )
@@ -920,7 +921,7 @@ The test lifecycle runs:
 
 ### Test Utilities
 
-- **`module_test.httpx_mock`** - mock HTTP responses (from pytest-httpx)
+- **`module_test.blasthttp_mock`** - mock HTTP responses 
 - **`module_test.httpserver`** - real HTTP server on port 8888
 - **`module_test.httpserver_ssl`** - real HTTPS server on port 9999
 - **`module_test.mock_dns(data)`** - mock DNS responses
@@ -933,7 +934,7 @@ Real example -- `test_module_robots.py`:
 ```python
 class TestRobots(ModuleTestBase):
     targets = ["http://127.0.0.1:8888"]
-    modules_overrides = ["httpx", "robots"]
+    modules_overrides = ["http", "robots"]
     config_overrides = {"modules": {"robots": {"include_sitemap": True}}}
 
     async def setup_after_prep(self, module_test):

README.md

@@ -20,6 +20,8 @@ pipx install --pip-args '\--pre' bbot
 
 _For more installation methods, including [Docker](https://hub.docker.com/r/blacklanternsecurity/bbot), see [Getting Started](https://www.blacklanternsecurity.com/bbot/Stable/)_
 
+> **Speed tip:** BBOT's DNS engine spins up ten workers per resolver in `/etc/resolv.conf`. Adding more unfiltered resolvers dramatically speeds up scans. See the [sample resolv.conf](docs/data/resolv-sample.conf) and [Tips and Tricks](https://www.blacklanternsecurity.com/bbot/Stable/scanning/tips_and_tricks/#speed-up-scans-with-more-dns-resolvers) for details.
+
 ## Example Commands
 
 ### 1) Subdomain Finder

bbot/__init__.py

@@ -1,4 +1,6 @@
+from importlib.metadata import version, PackageNotFoundError
+
 try:
-    from bbot._version import __version__
-except ImportError:
+    __version__ = version("bbot")
+except PackageNotFoundError:
     __version__ = "0.0.0"