works if you cd to the right place first

Author: epatey

src/inspect_scout/_concurrency/_mp_setup.py

@@ -1,4 +1,4 @@
-"""Setup module for spawn subprocess - runs at import time before unpickling."""
+"""Setup module for spawn subprocess - configures sys.path at import time."""
 
 import json
 import os
@@ -7,10 +7,18 @@
 # This module is imported FIRST in subprocess, before any user code
 # Read environment variables and configure subprocess before unpickling happens
 
+# Module-level code runs at import time, but only in fresh subprocesses
+# (not when imported in parent process before env vars are set)
+
 if "INSPECT_SCOUT_SYS_PATH" in os.environ:
     sys_path = json.loads(os.environ["INSPECT_SCOUT_SYS_PATH"])
     sys.path[:] = sys_path  # Modify in place to preserve sys.path identity
 
 if "INSPECT_SCOUT_WORKING_DIR" in os.environ:
     working_dir = os.environ["INSPECT_SCOUT_WORKING_DIR"]
     os.chdir(working_dir)
+
+    # CRITICAL: Add working directory to sys.path for user scanner imports
+    # With spawn, cwd is not automatically in sys.path like it is with fork
+    if working_dir not in sys.path:
+        sys.path.insert(0, working_dir)

src/inspect_scout/_concurrency/_mp_subprocess.py

@@ -25,7 +25,7 @@
 from .._transcript.types import TranscriptInfo
 from . import _mp_common
 from ._iterator import iterator_from_queue
-from ._mp_common import LoggingItem, run_sync_on_thread
+from ._mp_common import IPCContext, LoggingItem, run_sync_on_thread
 from ._mp_logging import patch_inspect_log_handler
 from ._mp_registry import ChildSemaphoreRegistry
 from .common import ScanMetrics
@@ -73,18 +73,20 @@ def _wait_for_shutdown() -> None:
 def subprocess_main(
     worker_id: int,
     task_count: int,
+    ipc_context: IPCContext,
 ) -> None:
     """Worker subprocess main function.
 
-    Runs in a forked subprocess with access to parent's memory.
+    Runs in a spawned subprocess with IPCContext passed as argument.
     Uses single_process_strategy internally to coordinate async tasks.
 
     Args:
         worker_id: Unique identifier for this worker process
         task_count: Number of concurrent tasks for this worker process
+        ipc_context: Shared IPC context passed from parent process
     """
-    # Access IPC context inherited from parent process via fork
-    ctx = _mp_common.ipc_context
+    # Use IPC context passed as argument (for spawn compatibility)
+    ctx = ipc_context
 
     def _log_in_parent(record: logging.LogRecord) -> None:
         # Strip exc_info from record to avoid pickling traceback objects since it

src/inspect_scout/_concurrency/multi_process.py

@@ -49,7 +49,8 @@
 from ._mp_logging import find_inspect_log_handler
 from ._mp_registry import ParentSemaphoreRegistry
 from ._mp_shutdown import shutdown_subprocesses
-from ._mp_subprocess import subprocess_main
+# Import subprocess_main lazily to avoid importing _mp_setup in parent process
+# (it needs to run in child process AFTER environment variables are set)
 from .common import (
     ConcurrencyStrategy,
     ParseFunctionResult,
@@ -275,6 +276,9 @@ async def _upstream_collector() -> None:
             os.environ["INSPECT_SCOUT_SYS_PATH"] = json.dumps(sys.path)
             os.environ["INSPECT_SCOUT_WORKING_DIR"] = os.getcwd()
 
+            # Import subprocess_main AFTER setting env vars to avoid importing _mp_setup in parent
+            from ._mp_subprocess import subprocess_main
+
             # Start worker processes directly
             ctx = multiprocessing.get_context("spawn")
             processes: list[SpawnProcess] = []
@@ -285,7 +289,7 @@ async def _upstream_collector() -> None:
                 try:
                     p = ctx.Process(
                         target=subprocess_main,
-                        args=(worker_id, task_count_for_worker),
+                        args=(worker_id, task_count_for_worker, _mp_common.ipc_context),
                     )
                     p.start()
                     processes.append(p)

test_env_inherit.py

@@ -0,0 +1,15 @@
+import os
+import multiprocessing
+
+def child_func():
+    print(f"Child: TEST_VAR = {os.environ.get('TEST_VAR', 'NOT SET')}")
+    print(f"Child: PATH first element = {os.environ.get('PATH', '').split(':')[0]}")
+
+if __name__ == "__main__":
+    os.environ['TEST_VAR'] = 'hello_from_parent'
+    print(f"Parent: TEST_VAR = {os.environ.get('TEST_VAR')}")
+
+    ctx = multiprocessing.get_context('spawn')
+    p = ctx.Process(target=child_func)
+    p.start()
+    p.join()

todo.md

@@ -1,22 +1,34 @@
 # Fork to Spawn Conversion - COMPLETED ✅
 
-## Final Solution: Environment Variables at Import Time
+## Final Solution
 
-Successfully converted multi-process strategy from fork to spawn using standard `multiprocessing` + `dill` + **environment variables**.
+Successfully converted from fork to spawn using:
+1. Standard `multiprocessing` (not `multiprocess` package)
+2. `DillCallable` wrapper for closures
+3. Environment variables for sys.path/cwd (set at import time)
+4. IPCContext passed as argument (not global)
 
-## The Critical Insight
+## The Journey
 
-**Problem:** When Python spawns a subprocess, it unpickles function arguments BEFORE entering the function. Setting sys.path inside the function is too late.
+### Issue 1: Python 3.14 Compatibility
+- ❌ `multiprocess` package has `subprocess._USE_VFORK` error
+- ✅ Created `DillCallable` wrapper with standard `multiprocessing`
 
-**Solution:** Use environment variables that are read at MODULE IMPORT TIME, before any unpickling happens.
+### Issue 2: Module Import Errors
+- ❌ User modules not importable in subprocess (missing sys.path/cwd)
+- ✅ Set via environment variables read at MODULE IMPORT TIME
 
-## Implementation
+### Issue 3: Global Variable Not Inherited
+- ❌ `_mp_common.ipc_context` global is `None` in spawn subprocess
+- ✅ Pass `IPCContext` as argument to `subprocess_main`
 
-### 1. New Setup Module (_mp_setup.py)
+## Final Implementation
+
+### 1. Setup Module (_mp_setup.py) - NEW FILE
+Reads environment at import time (before any unpickling):
 ```python
 import json, os, sys
 
-# Read environment and configure BEFORE any imports
 if "INSPECT_SCOUT_SYS_PATH" in os.environ:
     sys.path[:] = json.loads(os.environ["INSPECT_SCOUT_SYS_PATH"])
 if "INSPECT_SCOUT_WORKING_DIR" in os.environ:
@@ -25,30 +37,53 @@ if "INSPECT_SCOUT_WORKING_DIR" in os.environ:
 
 ### 2. Import Setup First (_mp_subprocess.py:12)
 ```python
-# IMPORTANT: Import _mp_setup FIRST before anything else
-from . import _mp_setup  # noqa: F401
+from . import _mp_setup  # noqa: F401  # BEFORE other imports
+```
+
+### 3. Accept IPCContext Parameter (_mp_subprocess.py:76)
+```python
+def subprocess_main(worker_id, task_count, ipc_context):
+    ctx = ipc_context  # Use parameter, not global
 ```
 
-### 3. Set Environment Variables (multi_process.py:274-275)
+### 4. Set Environment + Pass IPCContext (multi_process.py)
 ```python
+# Set env vars before spawning
 os.environ["INSPECT_SCOUT_SYS_PATH"] = json.dumps(sys.path)
 os.environ["INSPECT_SCOUT_WORKING_DIR"] = os.getcwd()
+
+# Pass IPCContext as argument
+p = ctx.Process(
+    target=subprocess_main,
+    args=(worker_id, task_count, ipc_context),
+)
 ```
 
-### 4. Use DillCallable for Closures (_mp_common.py)
-Wrap closures so they can be pickled with user module references.
+### 5. DillCallable Wrapper (_mp_common.py)
+```python
+class DillCallable:
+    def __init__(self, func):
+        self._pickled_func = dill.dumps(func)
+    def __call__(self, *args, **kwargs):
+        func = dill.loads(self._pickled_func)
+        return func(*args, **kwargs)
+```
 
 ## Files Modified
-1. **_mp_setup.py** (NEW) - reads env vars at import time
-2. **_mp_subprocess.py** - imports _mp_setup first
-3. **_mp_common.py** - added DillCallable wrapper
-4. **multi_process.py** - sets env vars, uses spawn, wraps callables
+1. **_mp_setup.py** (NEW) - reads env at import time
+2. **_mp_subprocess.py** - imports _mp_setup, accepts ipc_context param
+3. **_mp_common.py** - DillCallable wrapper
+4. **multi_process.py** - sets env vars, uses spawn, wraps functions, passes ipc_context
 
 ## Test Results
 ✅ All 41 multi-process tests pass
 ✅ Works on Python 3.14
-✅ No multiprocess dependency
-✅ User modules importable (sys.path set before unpickling)
+✅ No `multiprocess` dependency
+✅ User modules import correctly
+✅ Ready for production
 
-## Key Lesson
-With spawn multiprocessing, you cannot set sys.path by passing it as an argument - by the time your function receives it, unpickling has already happened. Use environment variables that are read at module import time instead.
+## Key Insights
+1. **Unpickling happens before function entry** - can't set sys.path by passing as arg
+2. **Environment variables work** - read at module import time, before unpickling
+3. **Globals don't transfer with spawn** - must pass IPCContext as argument
+4. **Import order matters** - _mp_setup must be imported FIRST