From df06d8a21fca7eaa273c8235f1d7db0e632461de Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Fri, 18 Jul 2025 14:37:46 +0530 Subject: [PATCH 1/2] Prevent infinite autoscaling --- .../cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java | 2 ++ .../network/as/dao/AutoScaleVmGroupVmMapDaoImpl.java | 9 +++++++++ .../cloud/configuration/ConfigurationManagerImpl.java | 2 ++ .../main/java/com/cloud/network/as/AutoScaleManager.java | 6 ++++++ .../java/com/cloud/network/as/AutoScaleManagerImpl.java | 8 +++++++- 5 files changed, 26 insertions(+), 1 deletion(-) diff --git a/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java b/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java index 4b25c63403e2..409042e7e33c 100644 --- a/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java +++ b/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java @@ -35,4 +35,6 @@ public interface AutoScaleVmGroupVmMapDao extends GenericDao= 0; } + + @Override + public int getErroredInstanceCount(long vmGroupId) { + SearchCriteria sc = CountBy.create(); + sc.setParameters("vmGroupId", vmGroupId); + sc.setJoinParameters("vmSearch", "states", State.Error); + final List results = customSearch(sc, null); + return results.get(0); + } } diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 4c03122e4637..b10b69503405 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -47,6 +47,7 @@ import com.cloud.hypervisor.HypervisorGuru; +import com.cloud.network.as.AutoScaleManager; import com.cloud.user.AccountManagerImpl; import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.host.HostTagVO; @@ -570,6 +571,7 @@ protected void populateConfigValuesForValidationSet() { configValuesForValidation.add(UserDataManager.VM_USERDATA_MAX_LENGTH_STRING); configValuesForValidation.add(UnmanagedVMsManager.RemoteKvmInstanceDisksCopyTimeout.key()); configValuesForValidation.add(UnmanagedVMsManager.ConvertVmwareInstanceToKvmTimeout.key()); + configValuesForValidation.add(AutoScaleManager.AutoScaleErroredInstanceThreshold.key()); } private void weightBasedParametersForValidation() { diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManager.java b/server/src/main/java/com/cloud/network/as/AutoScaleManager.java index cf6aab6a7bb5..26598879a51a 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManager.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManager.java @@ -38,6 +38,12 @@ public interface AutoScaleManager extends AutoScaleService { "The Number of worker threads to scan the autoscale vm groups.", false); + ConfigKey AutoScaleErroredInstanceThreshold = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Integer.class, + "autoscale.errored.instance.threshold", + "10", + "The number of Error Instances allowed in autoscale vm groups for scale up.", + true); + void checkAutoScaleUser(Long autoscaleUserId, long accountId); boolean deleteAutoScaleVmGroupsByAccount(Long accountId); diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 5bbd85c2415d..e30b6736f576 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -1716,6 +1716,11 @@ private boolean checkConditionUp(AutoScaleVmGroupVO asGroup, Integer numVm) { s_logger.warn("number of VM will greater than the maximum in this group if scaling up, so do nothing more"); return false; } + int erroredInstanceCount = autoScaleVmGroupVmMapDao.getErroredInstanceCount(asGroup.getId()); + if (erroredInstanceCount > AutoScaleManager.AutoScaleErroredInstanceThreshold.value()) { + s_logger.warn("number of Errored Instances are greater than the threshold in this group for scaling up, so do nothing more"); + return false; + } return true; } @@ -2202,7 +2207,8 @@ public ConfigKey[] getConfigKeys() { return new ConfigKey[] { AutoScaleStatsInterval, AutoScaleStatsCleanupDelay, - AutoScaleStatsWorker + AutoScaleStatsWorker, + AutoScaleErroredInstanceThreshold }; } From 49f898cce836bcf951d16d191b51ddd95a69aa27 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Fri, 25 Jul 2025 12:45:18 +0530 Subject: [PATCH 2/2] Update server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java --- .../main/java/com/cloud/network/as/AutoScaleManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index e30b6736f576..8e6b7222da27 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -1718,7 +1718,7 @@ private boolean checkConditionUp(AutoScaleVmGroupVO asGroup, Integer numVm) { } int erroredInstanceCount = autoScaleVmGroupVmMapDao.getErroredInstanceCount(asGroup.getId()); if (erroredInstanceCount > AutoScaleManager.AutoScaleErroredInstanceThreshold.value()) { - s_logger.warn("number of Errored Instances are greater than the threshold in this group for scaling up, so do nothing more"); + s_logger.warn("Number of Errored Instances are greater than the threshold in this group for scaling up, so do nothing more"); return false; } return true;