diff --git a/vcstool/commands/command.py b/vcstool/commands/command.py index 77659891..87c7480d 100644 --- a/vcstool/commands/command.py +++ b/vcstool/commands/command.py @@ -59,6 +59,7 @@ def add_common_arguments( group.add_argument( '-w', '--workers', type=check_greater_zero, metavar='N', default=default_workers, help='Number of parallel worker threads') + group.add_argument("-d", "--delay", type=float, default=0.0, help="Delay between job processing to prevent DOS") group.add_argument( '--repos', action='store_true', default=False, help='List repositories which the command operates on') @@ -93,7 +94,7 @@ def simple_main(parser, command_class, args=None): output_repositories(clients) jobs = generate_jobs(clients, command) results = execute_jobs( - jobs, show_progress=True, number_of_workers=args.workers, + jobs, show_progress=True, number_of_workers=args.workers, delay_time=args.delay, debug_jobs=args.debug) output_results(results, hide_empty=args.hide_empty) diff --git a/vcstool/commands/custom.py b/vcstool/commands/custom.py index fdaa9a69..b8257111 100644 --- a/vcstool/commands/custom.py +++ b/vcstool/commands/custom.py @@ -83,7 +83,7 @@ def main(args=None, stdout=None, stderr=None): output_repositories(clients) jobs = generate_jobs(clients, command) results = execute_jobs( - jobs, show_progress=True, number_of_workers=args.workers, + jobs, show_progress=True, number_of_workers=args.workers, delay_time=args.delay, debug_jobs=args.debug) output_results(results, hide_empty=args.hide_empty) diff --git a/vcstool/commands/export.py b/vcstool/commands/export.py index 6140c206..618da2ab 100644 --- a/vcstool/commands/export.py +++ b/vcstool/commands/export.py @@ -95,7 +95,7 @@ def main(args=None, stdout=None, stderr=None): if command.output_repos: output_repositories(clients) jobs = generate_jobs(clients, command) - results = execute_jobs(jobs, number_of_workers=args.workers) + results = execute_jobs(jobs, number_of_workers=args.workers, delay_time=args.delay) # check if at least one repo was found in the client directory basename = None diff --git a/vcstool/commands/import_.py b/vcstool/commands/import_.py index 55b3e184..4ba9d4af 100644 --- a/vcstool/commands/import_.py +++ b/vcstool/commands/import_.py @@ -250,7 +250,7 @@ def main(args=None, stdout=None, stderr=None): results = execute_jobs( jobs, show_progress=True, number_of_workers=workers, - debug_jobs=args.debug) + debug_jobs=args.debug, delay_time=args.delay) output_results(results) any_error = any(r['returncode'] for r in results) diff --git a/vcstool/commands/validate.py b/vcstool/commands/validate.py index d62f113c..1c63536f 100644 --- a/vcstool/commands/validate.py +++ b/vcstool/commands/validate.py @@ -80,7 +80,7 @@ def main(args=None, stdout=None, stderr=None): results = execute_jobs( jobs, show_progress=True, number_of_workers=args.workers, - debug_jobs=args.debug) + debug_jobs=args.debug, delay_time=args.delay) output_results(results, hide_empty=args.hide_empty) diff --git a/vcstool/executor.py b/vcstool/executor.py index e0de9213..2672a383 100644 --- a/vcstool/executor.py +++ b/vcstool/executor.py @@ -3,6 +3,7 @@ from queue import Empty, Queue import sys import threading +from time import sleep import traceback logger = logging.getLogger(__name__) @@ -79,7 +80,7 @@ def get_ready_job(jobs): def execute_jobs( - jobs, show_progress=False, number_of_workers=10, debug_jobs=False + jobs, show_progress=False, number_of_workers=10, debug_jobs=False, delay_time=0.0, ): global windows_force_posix from vcstool.streams import stdout @@ -97,7 +98,7 @@ def execute_jobs( # create worker threads workers = [] for _ in range(min(number_of_workers, len(jobs))): - worker = Worker(job_queue, result_queue) + worker = Worker(job_queue, result_queue, delay_time) workers.append(worker) # fill job_queue with jobs for each worker @@ -157,12 +158,13 @@ def execute_jobs( class Worker(threading.Thread): - def __init__(self, job_queue, result_queue): + def __init__(self, job_queue, result_queue, delay_time=0.0): super(Worker, self).__init__() self.daemon = True self.done = False self.job_queue = job_queue self.result_queue = result_queue + self.delay_time = delay_time def run(self): # process all incoming jobs @@ -174,6 +176,8 @@ def run(self): result = self.process_job(job) # send result self.result_queue.put((job, result)) + if self.delay_time > 0.0: + sleep(self.delay_time) except Empty: pass