Skip to content

Commit 93a51d7

Browse files
committed
1 parent 3595189 commit 93a51d7

File tree

1 file changed

+13
-9
lines changed

1 file changed

+13
-9
lines changed

core/importers/importer.py

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -375,15 +375,7 @@ def traverse_dependencies(self, package_file, path, resource_types, dependencies
375375

376376
def prepare_tasks(self, resource_types, packages, resources):
377377
tasks = []
378-
# Count all items to determine batch size
379-
all_count = 0
380-
for resource, item in resources.items():
381-
for filepath, count in item.items():
382-
all_count += count
383-
if all_count > 50000:
384-
task_batch_size = (all_count / 1000)
385-
else:
386-
task_batch_size = self.MIN_BATCH_SIZE
378+
task_batch_size = self.calculate_batch_size(resources)
387379

388380
# Import in groups in order. Resources within groups are imported in parallel.
389381
for package in packages:
@@ -427,6 +419,18 @@ def prepare_tasks(self, resource_types, packages, resources):
427419
tasks.append(groups)
428420
return tasks
429421

422+
def calculate_batch_size(self, resources):
423+
# Count all items to determine batch size
424+
all_count = 0
425+
for _, item in resources.items():
426+
for _, count in item.items():
427+
all_count += count
428+
if all_count > 50000:
429+
task_batch_size = all_count / 1000
430+
else:
431+
task_batch_size = self.MIN_BATCH_SIZE
432+
return task_batch_size
433+
430434
def schedule_tasks(self, tasks):
431435
chained_tasks = chain()
432436
for task in tasks:

0 commit comments

Comments
 (0)