Fix multiprocessing logging with non-fork start methods

.github/workflows/tests.yml

@@ -23,6 +23,9 @@ jobs:
 
   test:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.13", "3.14"]
     steps:
       - uses: actions/checkout@v6
 
@@ -36,6 +39,8 @@ jobs:
 
       - name: Install uv
         uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ matrix.python-version }}
 
       - name: Install dependencies
         run: uv sync --dev

man_spider/lib/logger.py

@@ -3,7 +3,6 @@
 from sys import stdout
 from pathlib import Path
 from datetime import datetime
-from multiprocessing import Queue
 from logging.handlers import QueueHandler, QueueListener
 
 
@@ -74,22 +73,29 @@ def _monitor(self):
             pass
 
 
-### LOG TO STDERR ###
+### LOG TO CONSOLE ###
 
 console = logging.StreamHandler(stdout)
 # tell the handler to use this format
 console.setFormatter(ColoredFormatter("%(levelname)s %(message)s"))
 
 ### LOG TO FILE ###
 
-log_queue = Queue()
-listener = CustomQueueListener(log_queue, console)
-sender = QueueHandler(log_queue)
-logging.getLogger("manspider").handlers = [sender]
-
 logdir = Path.home() / ".manspider" / "logs"
 logdir.mkdir(parents=True, exist_ok=True)
 logfile = f"manspider_{datetime.now().strftime('%m-%d-%Y')}.log"
 handler = logging.FileHandler(str(logdir / logfile))
 handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
-logging.getLogger("manspider").addHandler(handler)
+
+
+def configure_logging(log_queue=None, level=None):
+    """Configure process-local handlers, optionally forwarding console records through a shared queue."""
+
+    console_handler = QueueHandler(log_queue) if log_queue is not None else console
+    manspider_log = logging.getLogger("manspider")
+    manspider_log.handlers = [console_handler, handler]
+    if level is not None:
+        manspider_log.setLevel(level)
+
+
+configure_logging()

man_spider/lib/spider.py

@@ -1,23 +1,34 @@
 import re
 import queue
+import logging
 from time import sleep
 import multiprocessing
 from pathlib import Path
 
 from man_spider.lib.spiderling import *
 from man_spider.lib.parser import FileParser
+from man_spider.lib.logger import configure_logging
 
 # set up logging
 log = logging.getLogger("manspider")
 
 
+def run_spiderling(spiderling_cls, target, parent):
+    """Configure child-process logging before starting a spiderling."""
+
+    configure_logging(parent.log_queue, level=parent.log_level)
+    spiderling_cls(target, parent)
+
+
 class MANSPIDER:
-    def __init__(self, options):
+    def __init__(self, options, log_queue=None):
 
         self.targets = options.targets
         self.threads = options.threads
         self.maxdepth = options.maxdepth
         self.quiet = options.quiet
+        self.log_queue = log_queue
+        self.log_level = logging.DEBUG if options.verbose else logging.INFO
 
         self.username = options.username
         self.password = options.password
@@ -83,6 +94,14 @@ def __init__(self, options):
         if self.modified_before:
             log.info(f"Filtering files modified before: {self.modified_before.strftime('%Y-%m-%d')}")
 
+    def __getstate__(self):
+        """Exclude parent-only runtime objects when serializing a spiderling's configuration."""
+
+        state = self.__dict__.copy()
+        state["spiderling_pool"] = [None] * self.threads
+        state["smb_client_cache"] = {}
+        return state
+
     def start(self):
 
         for target in self.targets:
@@ -93,7 +112,7 @@ def start(self):
                         if process is None or not process.is_alive():
                             # start spiderling
                             self.spiderling_pool[i] = multiprocessing.Process(
-                                target=Spiderling, args=(target, self), daemon=False
+                                target=run_spiderling, args=(Spiderling, target, self), daemon=False
                             )
                             self.spiderling_pool[i].start()
                             # success, break out of infinite loop

man_spider/lib/spiderling.py

@@ -11,11 +11,47 @@
 from man_spider.lib.util import *
 from man_spider.lib.errors import *
 from man_spider.lib.processpool import *
+from man_spider.lib.logger import configure_logging
 
 
 log = logging.getLogger("manspider.spiderling")
 
 
+def save_file_to_loot(remote_file, loot_dir):
+    allowed_chars = string.ascii_lowercase + string.ascii_uppercase + string.digits + "._ "
+    loot_filename = str(remote_file).replace("\\", "_")
+    loot_filename = "".join([c for c in loot_filename if c in allowed_chars])
+    loot_dest = loot_dir / loot_filename
+    try:
+        move(str(remote_file.tmp_filename), str(loot_dest))
+    except Exception:
+        log.warning(f"Error saving {remote_file}")
+
+
+def parse_file_worker(file, parser, no_download, loot_dir, log_queue, log_level):
+    """Parse one file without serializing the live Spiderling or SMB connection."""
+
+    configure_logging(log_queue, level=log_level)
+    try:
+        if type(file) == RemoteFile:
+            matches = parser.parse_file(str(file.tmp_filename), pretty_filename=str(file))
+            if matches and not no_download:
+                save_file_to_loot(file, loot_dir)
+            else:
+                file.tmp_filename.unlink()
+        else:
+            log.debug(f"Found file: {file}")
+            matches = parser.parse_file(file, file)
+        return matches
+    except Exception as e:
+        if log.level <= logging.DEBUG:
+            log.error(format_exc())
+        else:
+            log.error(f"Error parsing file {file}: {e}")
+    except KeyboardInterrupt:
+        log.critical("File parsing interrupted")
+
+
 class SpiderlingMessage:
     """
     Message which gets sent back to the parent through parent_queue
@@ -123,7 +159,17 @@ def go(self):
                         self.parser_process.join()
                     except AttributeError:
                         pass
-                    self.parser_process = multiprocessing.Process(target=self.parse_file, args=(file,))
+                    self.parser_process = multiprocessing.Process(
+                        target=parse_file_worker,
+                        args=(
+                            file,
+                            self.parent.parser,
+                            self.parent.no_download,
+                            self.parent.loot_dir,
+                            self.parent.log_queue,
+                            self.parent.log_level,
+                        ),
+                    )
                     self.parser_process.start()
 
                 # otherwise, just save it
@@ -175,30 +221,17 @@ def files(self):
     def parse_file(self, file):
         """
         Simple wrapper around self.parent.parser.parse_file()
-        For sole purpose of threading
+        For sole purpose of multiprocessing
         """
 
-        try:
-            if type(file) == RemoteFile:
-                matches = self.parent.parser.parse_file(str(file.tmp_filename), pretty_filename=str(file))
-                if matches and not self.parent.no_download:
-                    self.save_file(file)
-                else:
-                    file.tmp_filename.unlink()
-
-            else:
-                log.debug(f"Found file: {file}")
-                self.parent.parser.parse_file(file, file)
-
-        # log all exceptions
-        except Exception as e:
-            if log.level <= logging.DEBUG:
-                log.error(format_exc())
-            else:
-                log.error(f"Error parsing file {file}: {e}")
-
-        except KeyboardInterrupt:
-            log.critical("File parsing interrupted")
+        return parse_file_worker(
+            file,
+            self.parent.parser,
+            self.parent.no_download,
+            self.parent.loot_dir,
+            self.parent.log_queue,
+            self.parent.log_level,
+        )
 
     @property
     def shares(self):
@@ -447,25 +480,25 @@ def message_parent(self, message_type, content=""):
     def parse_local_files(self, files):
 
         with ProcessPool(self.parent.threads) as pool:
-            for r in pool.map(self.parse_file, files):
+            for r in pool.map(
+                parse_file_worker,
+                files,
+                args=(
+                    self.parent.parser,
+                    self.parent.no_download,
+                    self.parent.loot_dir,
+                    self.parent.log_queue,
+                    self.parent.log_level,
+                ),
+            ):
                 pass
 
     def save_file(self, remote_file):
         """
         Moves a file from temp storage into the loot directory
         """
 
-        allowed_chars = string.ascii_lowercase + string.ascii_uppercase + string.digits + "._ "
-
-        # replace backslashes with underscores to preserve directory names
-        loot_filename = str(remote_file).replace("\\", "_")
-        # remove weird characters
-        loot_filename = "".join([c for c in loot_filename if c in allowed_chars])
-        loot_dest = self.parent.loot_dir / loot_filename
-        try:
-            move(str(remote_file.tmp_filename), str(loot_dest))
-        except Exception:
-            log.warning(f"Error saving {remote_file}")
+        save_file_to_loot(remote_file, self.parent.loot_dir)
 
     def get_file(self, remote_file):
         """

man_spider/manspider.py

@@ -17,7 +17,10 @@
 log.setLevel(logging.INFO)
 
 
-def go(options):
+def go(options, log_queue=None):
+
+    log_level = logging.DEBUG if options.verbose else logging.INFO
+    configure_logging(log_queue, level=log_level)
 
     log.info("MANSPIDER command executed: " + " ".join(sys.argv))
 
@@ -42,7 +45,7 @@ def go(options):
         log.info(f"Skipping files larger than {bytes_to_human(options.max_filesize)}")
         log.info(f"Using {options.threads:,} threads")
 
-        manspider = MANSPIDER(options)
+        manspider = MANSPIDER(options, log_queue=log_queue)
         manspider.start()
 
     except KeyboardInterrupt:
@@ -81,6 +84,9 @@ def load_content_wordlist(filepath, options):
 def main():
 
     interrupted = False
+    listener = None
+    log_queue = None
+    p = None
 
     examples = """
 
@@ -289,8 +295,11 @@ def main():
         [[targets.add(t) for t in g] for g in options.targets]
         options.targets = list(targets)
 
-        p = multiprocessing.Process(target=go, args=(options,), daemon=False)
+        ctx = multiprocessing.get_context()
+        log_queue = ctx.Queue()
+        p = ctx.Process(target=go, args=(options, log_queue), daemon=False)
         p.start()
+        listener = CustomQueueListener(log_queue, console)
         listener.start()
 
     except argparse.ArgumentError as e:
@@ -316,12 +325,20 @@ def main():
         sleep(1)
         try:
             # wait for go to finish
-            p.join()
+            if p is not None:
+                p.join()
         except:
             pass
         try:
             # stop the log listener
-            listener.stop()
+            if listener is not None:
+                listener.stop()
+        except:
+            pass
+        try:
+            if log_queue is not None:
+                log_queue.close()
+                log_queue.join_thread()
         except:
             pass