fix(backend): disable metrics for mutation locks

Granular locking produces more cardinality for lock names thus bloating
prometheus metrics leading to high memory usage.
Completely disable metrics for these locks as they would be useless for
the moment.

Signed-off-by: Fatih Acar <[email protected]>

Author: fatih-acar

backend/infrahub/core/node/create.py

@@ -212,7 +212,7 @@ async def _persist(current_db: InfrahubDatabase) -> Node:
         )
 
     obj: Node
-    async with InfrahubMultiLock(lock_registry=lock.registry, locks=lock_names):
+    async with InfrahubMultiLock(lock_registry=lock.registry, locks=lock_names, metrics=False):
         if db.is_transaction:
             obj = await _persist(db)
         else:

backend/infrahub/core/node/save.py

@@ -51,7 +51,7 @@ async def _persist() -> None:
         await node.save(db=db, fields=fields_to_save)
 
     if manage_lock:
-        async with InfrahubMultiLock(lock_registry=lock.registry, locks=locks):
+        async with InfrahubMultiLock(lock_registry=lock.registry, locks=locks, metrics=False):
             await _persist()
     else:
         await _persist()

backend/infrahub/graphql/mutations/ipam.py

@@ -223,7 +223,7 @@ async def mutate_update(
 
         namespace_lock_names = cls._get_lock_names(namespace_id)
         async with InfrahubMultiLock(lock_registry=lock.registry, locks=namespace_lock_names):
-            async with InfrahubMultiLock(lock_registry=lock.registry, locks=lock_names):
+            async with InfrahubMultiLock(lock_registry=lock.registry, locks=lock_names, metrics=False):
                 async with db.start_transaction() as dbt:
                     reconciled_address = await cls._mutate_update_object_and_reconcile(
                         info=info,
@@ -400,7 +400,7 @@ async def mutate_update(
 
         namespace_lock_names = cls._get_lock_names(namespace_id)
         async with InfrahubMultiLock(lock_registry=lock.registry, locks=namespace_lock_names):
-            async with InfrahubMultiLock(lock_registry=lock.registry, locks=lock_names):
+            async with InfrahubMultiLock(lock_registry=lock.registry, locks=lock_names, metrics=False):
                 async with db.start_transaction() as dbt:
                     reconciled_prefix = await cls._mutate_update_object_and_reconcile(
                         info=info,

backend/infrahub/graphql/mutations/main.py

@@ -248,7 +248,7 @@ async def _mutate(current_db: InfrahubDatabase) -> tuple[Node, Self]:
             result = await cls.mutate_update_to_graphql(db=current_db, info=info, obj=updated_obj)
             return updated_obj, result
 
-        async with InfrahubMultiLock(lock_registry=lock.registry, locks=lock_names):
+        async with InfrahubMultiLock(lock_registry=lock.registry, locks=lock_names, metrics=False):
             if db.is_transaction:
                 return await _mutate(db)
 

backend/infrahub/lock.py

@@ -50,9 +50,12 @@
 class InfrahubMultiLock:
     """Context manager to allow multiple locks to be reserved together"""
 
-    def __init__(self, lock_registry: InfrahubLockRegistry, locks: list[str] | None = None) -> None:
+    def __init__(
+        self, lock_registry: InfrahubLockRegistry, locks: list[str] | None = None, metrics: bool = True
+    ) -> None:
         self.registry = lock_registry
         self.locks = locks or []
+        self.metrics = metrics
 
     async def __aenter__(self):
         await self.acquire()
@@ -67,11 +70,11 @@ async def __aexit__(
 
     async def acquire(self) -> None:
         for lock in self.locks:
-            await self.registry.get(name=lock).acquire()
+            await self.registry.get(name=lock, metrics=self.metrics).acquire()
 
     async def release(self) -> None:
         for lock in reversed(self.locks):
-            await self.registry.get(name=lock).release()
+            await self.registry.get(name=lock, metrics=self.metrics).release()
 
 
 class NATSLock:
@@ -123,6 +126,7 @@ def __init__(
         connection: redis.Redis | InfrahubServices | None = None,
         local: bool | None = None,
         in_multi: bool = False,
+        metrics: bool = True,
     ) -> None:
         self.use_local: bool = local
         self.local: LocalLock = None
@@ -134,6 +138,7 @@ def __init__(
         self.acquire_time: int | None = None
         self.event = asyncio.Event()
         self._recursion_var: ContextVar[int | None] = ContextVar(f"infrahub_lock_recursion_{self.name}", default=None)
+        self.metrics = metrics
 
         if not self.connection or (self.use_local is None and name.startswith("local.")):
             self.use_local = True
@@ -162,11 +167,17 @@ async def acquire(self) -> None:
             self._recursion_var.set(depth + 1)
             return
 
-        with LOCK_ACQUIRE_TIME_METRICS.labels(self.name, self.lock_type).time():
-            if not self.use_local:
-                await self.remote.acquire(token=f"{current_timestamp()}::{WORKER_IDENTITY}")
-            else:
-                await self.local.acquire()
+        if self.metrics:
+            with LOCK_ACQUIRE_TIME_METRICS.labels(self.name, self.lock_type).time():
+                if not self.use_local:
+                    await self.remote.acquire(token=f"{current_timestamp()}::{WORKER_IDENTITY}")
+                else:
+                    await self.local.acquire()
+        elif not self.use_local:
+            await self.remote.acquire(token=f"{current_timestamp()}::{WORKER_IDENTITY}")
+        else:
+            await self.local.acquire()
+
         self.acquire_time = time.time_ns()
         self.event.clear()
         self._recursion_var.set(1)
@@ -182,7 +193,8 @@ async def release(self) -> None:
 
         if self.acquire_time is not None:
             duration_ns = time.time_ns() - self.acquire_time
-            LOCK_RESERVE_TIME_METRICS.labels(self.name, self.lock_type).observe(duration_ns / 1000000000)
+            if self.metrics:
+                LOCK_RESERVE_TIME_METRICS.labels(self.name, self.lock_type).observe(duration_ns / 1000000000)
             self.acquire_time = None
 
         if not self.use_local:
@@ -252,11 +264,18 @@ def get_existing(
         return self.locks[lock_name]
 
     def get(
-        self, name: str, namespace: str | None = None, local: bool | None = None, in_multi: bool = False
+        self,
+        name: str,
+        namespace: str | None = None,
+        local: bool | None = None,
+        in_multi: bool = False,
+        metrics: bool = True,
     ) -> InfrahubLock:
         lock_name = self._generate_name(name=name, namespace=namespace, local=local)
         if lock_name not in self.locks:
-            self.locks[lock_name] = InfrahubLock(name=lock_name, connection=self.connection, in_multi=in_multi)
+            self.locks[lock_name] = InfrahubLock(
+                name=lock_name, connection=self.connection, in_multi=in_multi, metrics=metrics
+            )
         return self.locks[lock_name]
 
     def local_schema_lock(self) -> LocalLock: