The LRU cache capacity can be configured via an env variable

Signed-off-by: noemotiovon <[email protected]>

Author: noemotiovon

docs/backend/CANN.md

@@ -314,3 +314,7 @@ Converting the matmul weight format from ND to NZ to improve performance. Enable
 ### GGML_CANN_ACL_GRAPH
 
 Operators are executed using ACL graph execution, rather than in op-by-op (eager) mode. Enabled by default.
+
+### GGML_CANN_GRAPH_CACHE_CAPACITY
+
+Maximum number of compiled CANN graphs kept in the LRU cache, default is 12. When the number of cached graphs exceeds this capacity, the least recently used graph will be evicted.

ggml/src/ggml-cann/common.h

@@ -368,12 +368,21 @@ struct ggml_cann_graph {
  * move existing graphs to the front (most recently used), and clear the cache.
  */
 struct ggml_cann_graph_lru_cache {
-    size_t capacity = 12;  /**< Maximum number of graphs in the cache. */
+    size_t capacity;  /**< Maximum number of graphs in the cache. */
 
     std::list<std::shared_ptr<ggml_cann_graph>> cache_list; /**< List storing cached graphs. */
 
     std::shared_ptr<ggml_cann_graph> matched_graph = nullptr; /**< Pointer to a recently matched graph. */
 
+    ggml_cann_graph_lru_cache() {
+        std::string env_val = get_env("GGML_CANN_GRAPH_CACHE_CAPACITY").value_or("12");
+        try {
+            capacity = std::stoul(env_val);
+        } catch (...) {
+            capacity = 12; // fallback to default if invalid
+        }
+    }
+
     /**
      * @brief Push a new graph to the front of the cache.
      * If the cache exceeds capacity, the least recently used graph is removed.