[NPU] Disable idle pruning if property is set (#33364)

### Details:
 - *Disable idle pruning if property is set*

### Tickets:
 - *CVS-178801*

---------

Signed-off-by: Bogdan Pereanu <bogdan.pereanu@intel.com>
Signed-off-by: alexandruenache1111 <alexandru.enache@intel.com>
Co-authored-by: alexandruenache1111 <alexandru.enache@intel.com>

Author: pereanub

docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.rst

@@ -152,6 +152,7 @@ offer a limited set of supported OpenVINO features.
          ov::intel_npu::bypass_umd_caching
          ov::intel_npu::defer_weights_load
          ov::intel_npu::run_inferences_sequentially
+         ov::intel_npu::disable_idle_memory_prunning
 
    .. tab-item:: Read-only properties
 

src/bindings/python/src/pyopenvino/core/properties/properties.cpp

@@ -351,4 +351,5 @@ void regmodule_properties(py::module m) {
     wrap_property_RW(m_intel_npu, ov::intel_npu::qdq_optimization, "qdq_optimization");
     wrap_property_RW(m_intel_npu, ov::intel_npu::qdq_optimization_aggressive, "qdq_optimization_aggressive");
     wrap_property_RW(m_intel_npu, ov::intel_npu::run_inferences_sequentially, "run_inferences_sequentially");
+    wrap_property_RW(m_intel_npu, ov::intel_npu::disable_idle_memory_prunning, "disable_idle_memory_prunning");
 }

src/bindings/python/tests/test_runtime/test_properties.py

@@ -496,6 +496,11 @@ def test_properties_ro(ov_property_ro, expected_value):
             "NPU_QDQ_OPTIMIZATION_AGGRESSIVE",
             ((True, True),),
         ),
+        (
+            intel_npu.disable_idle_memory_prunning,
+            "NPU_DISABLE_IDLE_MEMORY_PRUNING",
+            ((True, True),),
+        ),
         (props.enable_weightless, "ENABLE_WEIGHTLESS", ((True, True), (False, False))),
     ],
 )

src/inference/include/openvino/runtime/intel_npu/properties.hpp

@@ -148,5 +148,13 @@ static constexpr ov::Property<bool> defer_weights_load{"NPU_DEFER_WEIGHTS_LOAD"}
  */
 static constexpr ov::Property<bool> run_inferences_sequentially{"NPU_RUN_INFERENCES_SEQUENTIALLY"};
 
+/**
+ * @brief [Only for NPU Plugin]
+ * Type: boolean, default is false.
+ * This option allows to disable pruning of memory during idle time.
+ * @ingroup ov_runtime_npu_prop_cpp_api
+ */
+static constexpr ov::Property<bool> disable_idle_memory_prunning{"NPU_DISABLE_IDLE_MEMORY_PRUNING"};
+
 }  // namespace intel_npu
 }  // namespace ov

src/plugins/intel_npu/README.md

@@ -207,6 +207,7 @@ The following properties are supported (may differ based on current system confi
 | `ov::intel_npu::bypass_umd_caching`/</br>`NPU_BYPASS_UMD_CACHING` | RW | Bypass the caching of compiled models in UMD. | `YES`/ `NO`| `NO` |
 | `ov::intel_npu::defer_weights_load`/</br>`NPU_DEFER_WEIGHTS_LOAD` | RW | Delay loading the weights until inference is created. | `YES`/ `NO`| `NO` |
 | `ov::intel_npu::run_inferences_sequentially`/</br>`NPU_RUN_INFERENCES_SEQUENTIALLY` | RW | Run inferences in async mode sequentially in the order in which they are started to optimize host scheduling. | `YES`/ `NO`| `NO` |
+| `ov::intel_npu::disable_idle_memory_prunning`/</br>`NPU_DISABLE_IDLE_MEMORY_PRUNING` | RW | Enable/Disable pruning of memory during idle time. | `YES` / `NO` | `NO` |
 <br>
 
 ### Compiled_model properties VS Plugin properties

src/plugins/intel_npu/src/al/include/intel_npu/config/options.hpp

@@ -1491,4 +1491,22 @@ struct ENABLE_STRIDES_FOR final : OptionBase<ENABLE_STRIDES_FOR, std::string> {
     }
 };
 
+struct DISABLE_IDLE_MEMORY_PRUNING final : OptionBase<DISABLE_IDLE_MEMORY_PRUNING, bool> {
+    static std::string_view key() {
+        return ov::intel_npu::disable_idle_memory_prunning.name();
+    }
+
+    static bool defaultValue() {
+        return false;
+    }
+
+    static OptionMode mode() {
+        return OptionMode::RunTime;
+    }
+
+    static bool isPublic() {
+        return true;
+    }
+};
+
 }  // namespace intel_npu

src/plugins/intel_npu/src/backend/include/zero_backend.hpp

@@ -28,10 +28,11 @@ class ZeroEngineBackend final : public IEngineBackend {
 
     bool isCommandQueueExtSupported() const override;
     bool isLUIDExtSupported() const override;
+    bool isContextExtSupported() const override;
 
     void* getContext() const override;
 
-    void updateInfo(const Config& config) override;
+    void updateInfo(const ov::AnyMap& properties) override;
 
     const std::shared_ptr<ZeroInitStructsHolder> getInitStructs() const override;
 

src/plugins/intel_npu/src/backend/include/zero_device.hpp

@@ -34,9 +34,8 @@ class ZeroDevice : public IDevice {
 
     std::shared_ptr<SyncInferRequest> createInferRequest(const std::shared_ptr<const ICompiledModel>& compiledModel,
                                                          const Config& config) override;
-    void updateInfo(const Config& config) override {
-        log.setLevel(config.get<LOG_LEVEL>());
-    }
+
+    void updateInfo(const ov::AnyMap& properties) override;
 
     ZeroDevice& operator=(const ZeroDevice&) = delete;
     ZeroDevice(const ZeroDevice&) = delete;
@@ -46,18 +45,15 @@ class ZeroDevice : public IDevice {
 private:
     const std::shared_ptr<ZeroInitStructsHolder> _initStructs;
 
-    ze_device_properties_t device_properties = {};
-
-    ze_pci_ext_properties_t pci_properties = {};
-
-    ze_device_luid_ext_properties_t device_luid = {};
-
-    std::map<ov::element::Type, float> device_gops = {{ov::element::f32, 0.f},
-                                                      {ov::element::f16, 0.f},
-                                                      {ov::element::bf16, 0.f},
-                                                      {ov::element::u8, 0.f},
-                                                      {ov::element::i8, 0.f}};
+    ze_device_properties_t _device_properties = {};
+    ze_pci_ext_properties_t _pci_properties = {};
+    ze_device_luid_ext_properties_t _device_luid = {};
+    std::map<ov::element::Type, float> _device_gops = {{ov::element::f32, 0.f},
+                                                       {ov::element::f16, 0.f},
+                                                       {ov::element::bf16, 0.f},
+                                                       {ov::element::u8, 0.f},
+                                                       {ov::element::i8, 0.f}};
 
-    Logger log;
+    Logger _log;
 };
 }  // namespace intel_npu

src/plugins/intel_npu/src/backend/src/zero_backend.cpp

@@ -37,6 +37,10 @@ bool ZeroEngineBackend::isLUIDExtSupported() const {
     return _initStruct->isExtensionSupported(std::string(ZE_DEVICE_LUID_EXT_NAME), ZE_MAKE_VERSION(1, 0));
 }
 
+bool ZeroEngineBackend::isContextExtSupported() const {
+    return _initStruct->isExtensionSupported(std::string(ZE_CONTEXT_NPU_EXT_NAME), ZE_MAKE_VERSION(1, 0));
+}
+
 const std::shared_ptr<IDevice> ZeroEngineBackend::getDevice() const {
     if (_devices.empty()) {
         _logger.debug("ZeroEngineBackend - getDevice() returning empty list");
@@ -130,11 +134,22 @@ void* ZeroEngineBackend::getContext() const {
     return _initStruct->getContext();
 }
 
-void ZeroEngineBackend::updateInfo(const Config& config) {
-    _logger.setLevel(config.get<LOG_LEVEL>());
+void ZeroEngineBackend::updateInfo(const ov::AnyMap& properties) {
+    if (properties.count(ov::log::level.name()) != 0) {
+        _logger.setLevel(properties.at(ov::log::level.name()).as<ov::log::Level>());
+    }
+
+    if (properties.count(ov::intel_npu::disable_idle_memory_prunning.name()) != 0) {
+        if (properties.at(ov::intel_npu::disable_idle_memory_prunning.name()).as<bool>()) {
+            _initStruct->clearContextOptions(ZE_NPU_CONTEXT_OPTION_IDLE_OPTIMIZATIONS);
+        } else {
+            _initStruct->setContextOptions(ZE_NPU_CONTEXT_OPTION_IDLE_OPTIMIZATIONS);
+        }
+    }
+
     if (_devices.size() > 0) {
         for (auto& dev : _devices) {
-            dev.second->updateInfo(config);
+            dev.second->updateInfo(properties);
         }
     }
 }

src/plugins/intel_npu/src/backend/src/zero_device.cpp

@@ -13,35 +13,35 @@ using namespace intel_npu;
 
 ZeroDevice::ZeroDevice(const std::shared_ptr<ZeroInitStructsHolder>& initStructs)
     : _initStructs(initStructs),
-      log("ZeroDevice", Logger::global().level()) {
-    log.debug("ZeroDevice::ZeroDevice init");
-    device_properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES;
+      _log("ZeroDevice", Logger::global().level()) {
+    _log.debug("ZeroDevice::ZeroDevice init");
+    _device_properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES;
 
     // Get LUID info, if supported
     if (_initStructs->isExtensionSupported(std::string(ZE_DEVICE_LUID_EXT_NAME), ZE_MAKE_VERSION(1, 0))) {
-        device_luid.stype = ZE_STRUCTURE_TYPE_DEVICE_LUID_EXT_PROPERTIES;
-        device_properties.pNext = &device_luid;
+        _device_luid.stype = ZE_STRUCTURE_TYPE_DEVICE_LUID_EXT_PROPERTIES;
+        _device_properties.pNext = &_device_luid;
     }
     THROW_ON_FAIL_FOR_LEVELZERO("zeDeviceGetProperties",
-                                zeDeviceGetProperties(_initStructs->getDevice(), &device_properties));
+                                zeDeviceGetProperties(_initStructs->getDevice(), &_device_properties));
 
     // Query PCI information
     // Older drivers do not have this implementend. Linux driver returns NOT_IMPLEMENTED, while windows driver returns
     // zero values. If this is detected, we populate only device with ID from device_properties for backwards
     // compatibility. For any other error, we just fall-back to device ID to assure backwards compatibilty with even
     // older drivers
-    pci_properties.stype = ZE_STRUCTURE_TYPE_PCI_EXT_PROPERTIES;
-    ze_result_t retpci = zeDevicePciGetPropertiesExt(_initStructs->getDevice(), &pci_properties);
+    _pci_properties.stype = ZE_STRUCTURE_TYPE_PCI_EXT_PROPERTIES;
+    ze_result_t retpci = zeDevicePciGetPropertiesExt(_initStructs->getDevice(), &_pci_properties);
     if (ZE_RESULT_SUCCESS == retpci) {
         // windows driver specific backwards compatibility
-        if (pci_properties.address.device == 0) {
-            log.warning("PCI information not available in driver. Falling back to deviceId");
-            pci_properties.address.device = device_properties.deviceId;
+        if (_pci_properties.address.device == 0) {
+            _log.warning("PCI information not available in driver. Falling back to deviceId");
+            _pci_properties.address.device = _device_properties.deviceId;
         }
     } else {
         // general backwards compatibility
-        log.warning("PCI information not available in driver. Falling back to deviceId");
-        pci_properties.address.device = device_properties.deviceId;
+        _log.warning("PCI information not available in driver. Falling back to deviceId");
+        _pci_properties.address.device = _device_properties.deviceId;
     }
 
     /// Calculate and store device GOPS with formula: frequency * number of tiles * ops per tile
@@ -53,12 +53,12 @@ ZeroDevice::ZeroDevice(const std::shared_ptr<ZeroInitStructsHolder>& initStructs
     gops_support_drv_version = 1715354569;  /// Linux driver version which supports Gops calculations
 #endif                                // _WIN32 || __CYGWIN__
     if (_initStructs->getDriverVersion() >= gops_support_drv_version) {
-        float gops = (device_properties.coreClockRate / powf(1000, 3)) * device_properties.numSlices *
-                     device_properties.physicalEUSimdWidth;
-        device_gops[ov::element::f32] = 0;
-        device_gops[ov::element::u8] = gops;
-        device_gops[ov::element::i8] = gops;
-        device_gops[ov::element::f16] = 0.5f * gops;
+        float gops = (_device_properties.coreClockRate / powf(1000, 3)) * _device_properties.numSlices *
+                     _device_properties.physicalEUSimdWidth;
+        _device_gops[ov::element::f32] = 0;
+        _device_gops[ov::element::u8] = gops;
+        _device_gops[ov::element::i8] = gops;
+        _device_gops[ov::element::f16] = 0.5f * gops;
     }
 }
 
@@ -69,7 +69,7 @@ std::string ZeroDevice::getName() const {
 #define NPU_4000_DEVICE_ID   0x643E
 
     std::string name;
-    switch (device_properties.deviceId) {
+    switch (_device_properties.deviceId) {
     case NPU_3720_P_DEVICE_ID:
     case NPU_3720_S_DEVICE_ID:
         name = ov::intel_npu::Platform::NPU3720;
@@ -85,15 +85,15 @@ std::string ZeroDevice::getName() const {
 }
 
 std::string ZeroDevice::getFullDeviceName() const {
-    return device_properties.name;
+    return _device_properties.name;
 }
 
 IDevice::Uuid ZeroDevice::getUuid() const {
     Uuid uuid{};
-    static_assert(sizeof(device_properties.uuid.id) == uuid.uuid.size(),
+    static_assert(sizeof(_device_properties.uuid.id) == uuid.uuid.size(),
                   "ze_device_uuid_t::id size doesn't match intel_npu::Uuid::uuid size");
 
-    std::copy(std::begin(device_properties.uuid.id), std::end(device_properties.uuid.id), std::begin(uuid.uuid));
+    std::copy(std::begin(_device_properties.uuid.id), std::end(_device_properties.uuid.id), std::begin(uuid.uuid));
 
     return uuid;
 }
@@ -103,17 +103,17 @@ ov::device::LUID ZeroDevice::getLUID() const {
     // incompatibility check
     static_assert(ZE_MAX_DEVICE_LUID_SIZE_EXT == ov::device::LUID::MAX_LUID_SIZE, "LUID size mismatch");
     for (int i = 0; i < ZE_MAX_DEVICE_LUID_SIZE_EXT; i++) {
-        luidstruct.luid[i] = device_luid.luid.id[i];
+        luidstruct.luid[i] = _device_luid.luid.id[i];
     }
     return luidstruct;
 }
 
 uint32_t ZeroDevice::getSubDevId() const {
-    return device_properties.subdeviceId;
+    return _device_properties.subdeviceId;
 }
 
 uint32_t ZeroDevice::getMaxNumSlices() const {
-    return device_properties.numSlices;
+    return _device_properties.numSlices;
 }
 
 uint64_t ZeroDevice::getAllocMemSize() const {
@@ -155,14 +155,14 @@ uint64_t ZeroDevice::getTotalMemSize() const {
 }
 
 ov::device::PCIInfo ZeroDevice::getPciInfo() const {
-    return ov::device::PCIInfo{pci_properties.address.domain,
-                               pci_properties.address.bus,
-                               pci_properties.address.device,
-                               pci_properties.address.function};
+    return ov::device::PCIInfo{_pci_properties.address.domain,
+                               _pci_properties.address.bus,
+                               _pci_properties.address.device,
+                               _pci_properties.address.function};
 }
 
 std::map<ov::element::Type, float> ZeroDevice::getGops() const {
-    return device_gops;
+    return _device_gops;
 }
 
 ov::device::Type ZeroDevice::getDeviceType() const {
@@ -174,3 +174,9 @@ std::shared_ptr<SyncInferRequest> ZeroDevice::createInferRequest(
     const Config& config) {
     return std::make_shared<ZeroInferRequest>(_initStructs, compiledModel, config);
 }
+
+void ZeroDevice::updateInfo(const ov::AnyMap& properties) {
+    if (properties.count(ov::log::level.name()) != 0) {
+        _log.setLevel(properties.at(ov::log::level.name()).as<ov::log::Level>());
+    }
+}