Skip to content

[FIX] recompute_fields: get ids in batches #303

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[FIX] recompute_fields: get ids in batches #303

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 19 additions & 7 deletions src/util/orm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
from contextlib import contextmanager
from functools import wraps
from itertools import chain
from operator import itemgetter
from textwrap import dedent

try:
Expand Down Expand Up @@ -43,7 +42,7 @@
from .exceptions import MigrationError
from .helpers import table_of_model
from .misc import chunks, log_progress, version_between, version_gte
from .pg import column_exists, get_columns
from .pg import column_exists, format_query, get_columns, named_cursor

# python3 shims
try:
Expand Down Expand Up @@ -281,18 +280,31 @@ def recompute_fields(cr, model, fields, ids=None, logger=_logger, chunk_size=256
assert strategy in {"flush", "commit", "auto"}
Model = env(cr)[model] if isinstance(model, basestring) else model
model = Model._name

def get_ids():
if ids is not None:
for id_ in ids:
yield id_
else:
with named_cursor(cr, itersize=2**20) as ncr:
ncr.execute(format_query(cr, "SELECT id FROM {t} ORDER BY id", t=table_of_model(cr, model)))
for (id_,) in ncr:
yield id_

if ids is None:
cr.execute('SELECT id FROM "%s"' % table_of_model(cr, model))
ids = tuple(map(itemgetter(0), cr.fetchall()))
cr.execute(format_query(cr, "SELECT COUNT(id) FROM {t}", t=table_of_model(cr, model)))
(count,) = cr.fetchone()
else:
count = len(ids)

if strategy == "auto":
big_table = len(ids) > BIG_TABLE_THRESHOLD
big_table = count > BIG_TABLE_THRESHOLD
any_tracked_field = any(getattr(Model._fields[f], _TRACKING_ATTR, False) for f in fields)
strategy = "commit" if big_table and any_tracked_field else "flush"

size = (len(ids) + chunk_size - 1) / chunk_size
size = (count + chunk_size - 1) / chunk_size
qual = "%s %d-bucket" % (model, chunk_size) if chunk_size != 1 else model
for subids in log_progress(chunks(ids, chunk_size, list), logger, qualifier=qual, size=size):
for subids in log_progress(chunks(get_ids(), chunk_size, list), logger, qualifier=qual, size=size):
records = Model.browse(subids)
for field_name in fields:
field = records._fields[field_name]
Expand Down