Merge pull request #12 from ArchiveBox/chrome-remote-providers

Allow Chrome to support remote browser providers and one-chrome-per-snapshot mode

Author: pirate

.github/workflows/test-parallel.yml

@@ -74,6 +74,7 @@ jobs:
     needs: discover-tests
     env:
       ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      ABX_PYTEST_TEE_SUBPROCESS_OUTPUT: "1"
 
     strategy:
       fail-fast: false
@@ -135,7 +136,7 @@ jobs:
 
       - name: Run test - ${{ matrix.test.name }}
         run: |
-          uv run pytest -xvs "${{ matrix.test.path }}" --basetemp="$RUNNER_TEMP/pytest-out"
+          uv run pytest -xvs -rA "${{ matrix.test.path }}" --basetemp="$RUNNER_TEMP/pytest-out"
         env:
           TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
           CHROME_SANDBOX: "false"

abx_plugins/plugins/__init__.py

@@ -26,11 +26,7 @@ const {
 } = require('../base/utils.js');
 ensureNodeModuleResolution(module);
 const puppeteer = require('puppeteer-core');
-const {
-    waitForChromeSession,
-    connectToPage,
-    waitForPageLoaded,
-} = require('../chrome/chrome_utils.js');
+const { connectToPage } = require('../chrome/chrome_utils.js');
 
 // Extractor metadata
 const PLUGIN_NAME = 'accessibility';
@@ -52,18 +48,15 @@ async function extractAccessibility(url, timeoutMs) {
     let browser = null;
 
     try {
-        if (!(await waitForChromeSession(CHROME_SESSION_DIR, Math.min(timeoutMs, 1000), true))) {
-            return { success: false, error: 'No Chrome session found (chrome plugin must run first)' };
-        }
-
         const connection = await connectToPage({
             chromeSessionDir: CHROME_SESSION_DIR,
             timeoutMs,
+            waitForNavigationComplete: true,
+            postLoadDelayMs: 200,
             puppeteer,
         });
         browser = connection.browser;
         const page = connection.page;
-        await waitForPageLoaded(CHROME_SESSION_DIR, timeoutMs * 4, 200);
 
         // Get accessibility snapshot
         const accessibilityTree = await page.accessibility.snapshot({ interestingOnly: true });

abx_plugins/plugins/accessibility/on_Snapshot__39_accessibility.js

@@ -78,9 +78,7 @@ def test_accessibility_extracts_page_outline(self, chrome_test_url):
 
                 # Run accessibility hook with the active Chrome session
                 result = subprocess.run(
-                    [
-                        "node",
-                        str(ACCESSIBILITY_HOOK),
+                    [str(ACCESSIBILITY_HOOK),
                         f"--url={test_url}",
                         f"--snapshot-id={snapshot_id}",
                     ],
@@ -133,9 +131,7 @@ def test_accessibility_disabled_skips(self, chrome_test_url):
         env["ACCESSIBILITY_ENABLED"] = "False"
 
         result = subprocess.run(
-            [
-                "node",
-                str(ACCESSIBILITY_HOOK),
+            [str(ACCESSIBILITY_HOOK),
                 f"--url={test_url}",
                 f"--snapshot-id={snapshot_id}",
             ],
@@ -158,7 +154,7 @@ def test_accessibility_missing_url_argument(self):
         snapshot_id = "test-missing-url"
 
         result = subprocess.run(
-            ["node", str(ACCESSIBILITY_HOOK), f"--snapshot-id={snapshot_id}"],
+            [str(ACCESSIBILITY_HOOK), f"--snapshot-id={snapshot_id}"],
             cwd=str(self.temp_dir),
             capture_output=True,
             text=True,
@@ -174,7 +170,7 @@ def test_accessibility_missing_snapshot_id_argument(self, chrome_test_url):
         test_url = chrome_test_url
 
         result = subprocess.run(
-            ["node", str(ACCESSIBILITY_HOOK), f"--url={test_url}"],
+            [str(ACCESSIBILITY_HOOK), f"--url={test_url}"],
             cwd=str(self.temp_dir),
             capture_output=True,
             text=True,
@@ -191,9 +187,7 @@ def test_accessibility_with_no_chrome_session(self, chrome_test_url):
         snapshot_id = "test-no-chrome"
 
         result = subprocess.run(
-            [
-                "node",
-                str(ACCESSIBILITY_HOOK),
+            [str(ACCESSIBILITY_HOOK),
                 f"--url={test_url}",
                 f"--snapshot-id={snapshot_id}",
             ],

abx_plugins/plugins/accessibility/tests/__init__.py

@@ -2,7 +2,7 @@
 # /// script
 # requires-python = ">=3.12"
 # dependencies = [
-#   "click",
+#   "rich-click",
 #   "abx-pkg",
 # ]
 # ///

abx_plugins/plugins/accessibility/tests/test_accessibility.py

@@ -52,9 +52,7 @@ def test_hook_script_exists(self):
     def test_hook_skips_when_apt_not_allowed(self):
         """Hook should skip when apt not in allowed binproviders."""
         result = subprocess.run(
-            [
-                sys.executable,
-                str(INSTALL_HOOK),
+            [str(INSTALL_HOOK),
                 "--name=wget",
                 "--binary-id=test-uuid",
                 "--machine-id=test-machine",
@@ -74,9 +72,7 @@ def test_hook_detects_apt(self):
         """Hook should detect apt binary when available."""
         assert apt_available(), "apt not installed"
         result = subprocess.run(
-            [
-                sys.executable,
-                str(INSTALL_HOOK),
+            [str(INSTALL_HOOK),
                 "--name=nonexistent-pkg-xyz123",
                 "--binary-id=test-uuid",
                 "--machine-id=test-machine",
@@ -94,9 +90,7 @@ def test_hook_handles_overrides(self):
         overrides = json.dumps({"apt": {"install_args": ["custom-package-name"]}})
 
         result = subprocess.run(
-            [
-                sys.executable,
-                str(INSTALL_HOOK),
+            [str(INSTALL_HOOK),
                 "--name=test-pkg",
                 "--binary-id=test-uuid",
                 "--machine-id=test-machine",
@@ -120,9 +114,7 @@ def test_detect_existing_binary(self):
         assert apt_available(), "apt not installed"
         # Check for a binary that's almost certainly installed (like 'ls' or 'bash')
         result = subprocess.run(
-            [
-                sys.executable,
-                str(INSTALL_HOOK),
+            [str(INSTALL_HOOK),
                 "--name=bash",
                 "--binary-id=test-uuid",
                 "--machine-id=test-machine",

abx_plugins/plugins/apt/on_Binary__13_apt_install.py

@@ -36,9 +36,7 @@ def test_submits_to_archivedotorg():
         env["ARCHIVEDOTORG_TIMEOUT"] = "45"
 
         result = subprocess.run(
-            [
-                sys.executable,
-                str(ARCHIVEDOTORG_HOOK),
+            [str(ARCHIVEDOTORG_HOOK),
                 "--url",
                 TEST_URL,
                 "--snapshot-id",
@@ -77,9 +75,7 @@ def test_config_save_archivedotorg_false_skips():
         env["ARCHIVEDOTORG_ENABLED"] = "False"
 
         result = subprocess.run(
-            [
-                sys.executable,
-                str(ARCHIVEDOTORG_HOOK),
+            [str(ARCHIVEDOTORG_HOOK),
                 "--url",
                 TEST_URL,
                 "--snapshot-id",
@@ -115,9 +111,7 @@ def test_handles_timeout():
         env["TIMEOUT"] = "1"
 
         result = subprocess.run(
-            [
-                sys.executable,
-                str(ARCHIVEDOTORG_HOOK),
+            [str(ARCHIVEDOTORG_HOOK),
                 "--url",
                 TEST_URL,
                 "--snapshot-id",