[FIX] recompute_fields: get ids in batches

If no ids are given to recompute, the util will fetch
all ids in the target table and then recompute in chunks.
Fetching all the ids itself can cause a memory error
if the table is too large.
Using a named cursor with a limit of 1M records to fetch
can eliminate this possibility.

Author: jjmaksoud

src/util/orm.py

@@ -43,7 +43,7 @@
 from .exceptions import MigrationError
 from .helpers import table_of_model
 from .misc import chunks, log_progress, version_between, version_gte
-from .pg import column_exists, get_columns
+from .pg import column_exists, format_query, get_columns, named_cursor
 
 # python3 shims
 try:
@@ -281,35 +281,47 @@ def recompute_fields(cr, model, fields, ids=None, logger=_logger, chunk_size=256
     assert strategy in {"flush", "commit", "auto"}
     Model = env(cr)[model] if isinstance(model, basestring) else model
     model = Model._name
-    if ids is None:
-        cr.execute('SELECT id FROM "%s"' % table_of_model(cr, model))
-        ids = tuple(map(itemgetter(0), cr.fetchall()))
-
-    if strategy == "auto":
-        big_table = len(ids) > BIG_TABLE_THRESHOLD
-        any_tracked_field = any(getattr(Model._fields[f], _TRACKING_ATTR, False) for f in fields)
-        strategy = "commit" if big_table and any_tracked_field else "flush"
-
-    size = (len(ids) + chunk_size - 1) / chunk_size
-    qual = "%s %d-bucket" % (model, chunk_size) if chunk_size != 1 else model
-    for subids in log_progress(chunks(ids, chunk_size, list), logger, qualifier=qual, size=size):
-        records = Model.browse(subids)
-        for field_name in fields:
-            field = records._fields[field_name]
-            if hasattr(records, "_recompute_todo"):
-                # < 13.0
-                records._recompute_todo(field)
-            else:
-                Model.env.add_to_compute(field, records)
 
-        recompute(records)
-        # trigger dependent fields recomputation
-        records.modified(fields)
-        if strategy == "commit":
-            cr.commit()
-        else:
-            flush(records)
-        invalidate(records)
+    def get_record_ids():
+        if ids:
+            yield ids
+            return
+        MAX_SIZE = 1000000
+        ncr = named_cursor(cr, MAX_SIZE)
+        ncr.execute(format_query(cr, "SELECT id FROM {t}", t=table_of_model(cr, model)))
+        res = ncr.fetchmany(MAX_SIZE)
+        while res:
+            yield tuple(map(itemgetter(0), res))
+            res = ncr.fetchmany(MAX_SIZE)
+        ncr.close()
+
+    for _ids in get_record_ids():
+        # will be checked with the first batch of ids only
+        if strategy == "auto":
+            big_table = len(_ids) > BIG_TABLE_THRESHOLD
+            any_tracked_field = any(getattr(Model._fields[f], _TRACKING_ATTR, False) for f in fields)
+            strategy = "commit" if big_table and any_tracked_field else "flush"
+
+        size = (len(_ids) + chunk_size - 1) / chunk_size
+        qual = "%s %d-bucket" % (model, chunk_size) if chunk_size != 1 else model
+        for subids in log_progress(chunks(_ids, chunk_size, list), logger, qualifier=qual, size=size):
+            records = Model.browse(subids)
+            for field_name in fields:
+                field = records._fields[field_name]
+                if hasattr(records, "_recompute_todo"):
+                    # < 13.0
+                    records._recompute_todo(field)
+                else:
+                    Model.env.add_to_compute(field, records)
+
+            recompute(records)
+            # trigger dependent fields recomputation
+            records.modified(fields)
+            if strategy == "commit":
+                cr.commit()
+            else:
+                flush(records)
+            invalidate(records)
 
 
 class iter_browse(object):