Add optional delay between job processing

vcstool/commands/command.py

@@ -59,6 +59,7 @@ def add_common_arguments(
     group.add_argument(
         '-w', '--workers', type=check_greater_zero, metavar='N',
         default=default_workers, help='Number of parallel worker threads')
+    group.add_argument("-d", "--delay", type=float, default=0.0, help="Delay between job processing to prevent DOS")
     group.add_argument(
         '--repos', action='store_true', default=False,
         help='List repositories which the command operates on')
@@ -93,7 +94,7 @@ def simple_main(parser, command_class, args=None):
         output_repositories(clients)
     jobs = generate_jobs(clients, command)
     results = execute_jobs(
-        jobs, show_progress=True, number_of_workers=args.workers,
+        jobs, show_progress=True, number_of_workers=args.workers, delay_time=args.delay,
         debug_jobs=args.debug)
 
     output_results(results, hide_empty=args.hide_empty)

vcstool/commands/custom.py

@@ -83,7 +83,7 @@ def main(args=None, stdout=None, stderr=None):
         output_repositories(clients)
     jobs = generate_jobs(clients, command)
     results = execute_jobs(
-        jobs, show_progress=True, number_of_workers=args.workers,
+        jobs, show_progress=True, number_of_workers=args.workers, delay_time=args.delay,
         debug_jobs=args.debug)
 
     output_results(results, hide_empty=args.hide_empty)

vcstool/commands/export.py

@@ -95,7 +95,7 @@ def main(args=None, stdout=None, stderr=None):
     if command.output_repos:
         output_repositories(clients)
     jobs = generate_jobs(clients, command)
-    results = execute_jobs(jobs, number_of_workers=args.workers)
+    results = execute_jobs(jobs, number_of_workers=args.workers, delay_time=args.delay)
 
     # check if at least one repo was found in the client directory
     basename = None

vcstool/commands/import_.py

@@ -250,7 +250,7 @@ def main(args=None, stdout=None, stderr=None):
 
     results = execute_jobs(
         jobs, show_progress=True, number_of_workers=workers,
-        debug_jobs=args.debug)
+        debug_jobs=args.debug, delay_time=args.delay)
     output_results(results)
 
     any_error = any(r['returncode'] for r in results)

vcstool/commands/validate.py

@@ -80,7 +80,7 @@ def main(args=None, stdout=None, stderr=None):
 
     results = execute_jobs(
         jobs, show_progress=True, number_of_workers=args.workers,
-        debug_jobs=args.debug)
+        debug_jobs=args.debug, delay_time=args.delay)
 
     output_results(results, hide_empty=args.hide_empty)
 

vcstool/executor.py

@@ -3,6 +3,7 @@
 from queue import Empty, Queue
 import sys
 import threading
+from time import sleep
 import traceback
 
 logger = logging.getLogger(__name__)
@@ -79,7 +80,7 @@ def get_ready_job(jobs):
 
 
 def execute_jobs(
-    jobs, show_progress=False, number_of_workers=10, debug_jobs=False
+    jobs, show_progress=False, number_of_workers=10, debug_jobs=False, delay_time=0.0,
 ):
     global windows_force_posix
     from vcstool.streams import stdout
@@ -97,7 +98,7 @@ def execute_jobs(
     # create worker threads
     workers = []
     for _ in range(min(number_of_workers, len(jobs))):
-        worker = Worker(job_queue, result_queue)
+        worker = Worker(job_queue, result_queue, delay_time)
         workers.append(worker)
 
     # fill job_queue with jobs for each worker
@@ -157,12 +158,13 @@ def execute_jobs(
 
 class Worker(threading.Thread):
 
-    def __init__(self, job_queue, result_queue):
+    def __init__(self, job_queue, result_queue, delay_time=0.0):
         super(Worker, self).__init__()
         self.daemon = True
         self.done = False
         self.job_queue = job_queue
         self.result_queue = result_queue
+        self.delay_time = delay_time
 
     def run(self):
         # process all incoming jobs
@@ -174,6 +176,8 @@ def run(self):
                 result = self.process_job(job)
                 # send result
                 self.result_queue.put((job, result))
+                if self.delay_time > 0.0:
+                    sleep(self.delay_time)
             except Empty:
                 pass