Fix macOS multiprocessing compatibility

man_spider/lib/logger.py

@@ -85,7 +85,7 @@ def _monitor(self):
 log_queue = Queue()
 listener = CustomQueueListener(log_queue, console)
 sender = QueueHandler(log_queue)
-logging.getLogger("manspider").handlers = [sender]
+logging.getLogger("manspider").handlers = [console]
 
 logdir = Path.home() / ".manspider" / "logs"
 logdir.mkdir(parents=True, exist_ok=True)

man_spider/lib/spider.py

@@ -2,6 +2,7 @@
 import queue
 from time import sleep
 import multiprocessing
+import threading
 from pathlib import Path
 
 from man_spider.lib.spiderling import *
@@ -54,7 +55,7 @@ def __init__(self, options):
         self.failed_logons = 0
 
         self.spiderling_pool = [None] * self.threads
-        self.spiderling_queue = multiprocessing.Manager().Queue()
+        self.spiderling_queue = queue.Queue()
 
         # prevents needing to continually instantiate new SMBClients
         # {target: SMBClient() ...}
@@ -92,7 +93,7 @@ def start(self):
                         # if there's room in the pool
                         if process is None or not process.is_alive():
                             # start spiderling
-                            self.spiderling_pool[i] = multiprocessing.Process(
+                            self.spiderling_pool[i] = threading.Thread(
                                 target=Spiderling, args=(target, self), daemon=False
                             )
                             self.spiderling_pool[i].start()

man_spider/lib/spiderling.py

@@ -2,6 +2,7 @@
 import logging
 import pathlib
 import multiprocessing
+import threading
 from shutil import move
 from traceback import format_exc
 from datetime import datetime
@@ -123,7 +124,7 @@ def go(self):
                         self.parser_process.join()
                     except AttributeError:
                         pass
-                    self.parser_process = multiprocessing.Process(target=self.parse_file, args=(file,))
+                    self.parser_process = threading.Thread(target=self.parse_file, args=(file,), daemon=True)
                     self.parser_process.start()
 
                 # otherwise, just save it
@@ -506,4 +507,4 @@ def date_match(self, file_time):
                 log.debug(f"{self.target}: File too new: {file_date.strftime('%Y-%m-%d')}")
                 return False
 
-        return True
+        return True

man_spider/manspider.py

@@ -7,6 +7,7 @@
 import traceback
 from time import sleep
 import multiprocessing
+import threading
 from datetime import datetime
 
 from man_spider.lib import *
@@ -289,9 +290,9 @@ def main():
         [[targets.add(t) for t in g] for g in options.targets]
         options.targets = list(targets)
 
-        p = multiprocessing.Process(target=go, args=(options,), daemon=False)
-        p.start()
         listener.start()
+        p = threading.Thread(target=go, args=(options,), daemon=False)
+        p.start()
 
     except argparse.ArgumentError as e:
         syntax_error = True