[IMP] autovacuum_message_attachment: search/unlink optimization

Author: divad1196

autovacuum_message_attachment/models/autovacuum_mixin.py

@@ -10,18 +10,36 @@
 
 _logger = logging.getLogger(__name__)
 
+# Number of records deleted per unlink() call. Bounds the id list, the SQL
+# DELETE size and the per-commit work, independently of how many records the
+# rule retrieves (rule.batch_size).
+UNLINK_BATCH_SIZE = 5000
+
 
 class AutovacuumMixin(models.AbstractModel):
     _name = "autovacuum.mixin"
     _description = "Mixin used to delete messages or attachments"
 
-    def batch_unlink(self):
+    def batch_unlink(self, batch_size=0):
+        # batch_size <= 0 => delete everything in a single unlink (no chunking).
         with Registry(self.env.cr.dbname).cursor() as new_cr:
+            if batch_size == -1:
+                batch_size = len(self)
+            if not batch_size or batch_size < 0:
+                batch_size = new_cr.IN_MAX
+            elif batch_size > new_cr.IN_MAX:
+                # Adapt cursor IN_MAX to batch_size if needed:
+                # unlink silently chunk using IN_MAX (1000-rows)
+                # Assigning on the instance shadows the class attribute
+                # for this cursor only, not the class
+                # (Cursor.IN_MAX stays 1000)
+                new_cr.IN_MAX = batch_size
             new_env = api.Environment(new_cr, self.env.uid, self.env.context)
             try:
-                while self:
-                    batch_delete = self[0:1000]
-                    self -= batch_delete
+                remaining = self
+                while remaining:
+                    batch_delete = remaining[0:batch_size]
+                    remaining -= batch_delete
                     # do not attach new env to self because it may be
                     # huge, and the cache is cleaned after each unlink
                     # so we do not want to much record is the env in
@@ -38,7 +56,13 @@ def autovacuum(self, ttype="message"):
         rules = self.env["vacuum.rule"].search([("ttype", "=", ttype)])
         for rule in rules:
             records = rule._search_autovacuum_records()
-            records.batch_unlink()
+            _logger.info(
+                "Autovacuum rule %s: %s %s record(s) matched, deleting...",
+                rule.name,
+                len(records),
+                self._name,
+            )
+            records.batch_unlink(UNLINK_BATCH_SIZE)
 
     def _get_autovacuum_domain(self, rule):
         return []
@@ -95,21 +119,28 @@ def _get_autovacuum_records_model(self, rule):
                     self._prefix_domain_fields(autovacuum_relation, mixin_domain),
                 ]
             )
-            records = self.env[rule.model_id.model].search(
-                record_domain,
-                # Thanks to the optimization with _prefix_domain_fields,
-                # We now have at least 1 mixin-record per related-record
-                # => We can also limit this search here
-                #   and this won't affect the final search
-                limit=limit,
-            )
+            related_model = self.env[rule.model_id.model]
+            # Optimizations
+            # 1. We have at least 1 mixin-record per related-record
+            #    (see _prefix_domain_fields)
+            # => we can limit this search and get a free optimization.
+            # Reason: len(models) <= len(models mails)
+
+            # 2. The Query is injected directly into the domain as a SQL subquery
+            # (res_id IN (SELECT ...)) instead of materialising ids in Python.
+            # _search:
+            # "No default order is applied when
+            #   the method is invoked without parameter ``order``."
+            related_query = related_model._search(record_domain, limit=limit)
             mixin_domain = expression.AND(
-                [mixin_domain, [("res_id", "in", records.ids)]]
+                [mixin_domain, [("res_id", "in", related_query)]]
             )
-        return self.search(
-            mixin_domain,
-            limit=limit,
-        )
+        # Use _search (not search): deletion order is irrelevant and _search
+        # applies no ORDER BY. The model default (mail.message._order =
+        # 'id desc') combined with LIMIT would force Postgres to sort the whole
+        # matching set on every batch (~18s for the 15M stock.picking backlog);
+        # without it the LIMIT short-circuits the index scan (~70ms).
+        return self.browse(self._search(mixin_domain, limit=limit))
 
     # Retro-compatibility
     _get_autovacuum_records = _get_autovacuum_records_model