From 6fb7373ebdec093cb5527ccef771258ceb0edca4 Mon Sep 17 00:00:00 2001 From: Prashant Sankhla Date: Tue, 22 Jul 2025 11:06:48 +0530 Subject: [PATCH] Automlx Metrics Updates --- .../run-forecast-explainer-tests.yml | 2 +- .github/workflows/run-forecast-unit-tests.yml | 2 +- .../operator/lowcode/anomaly/model/automlx.py | 4 +-- ads/opctl/operator/lowcode/forecast/const.py | 2 +- .../lowcode/forecast/model/automlx.py | 36 +++++++++++-------- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/.github/workflows/run-forecast-explainer-tests.yml b/.github/workflows/run-forecast-explainer-tests.yml index ecd1f048f..fc03691b0 100644 --- a/.github/workflows/run-forecast-explainer-tests.yml +++ b/.github/workflows/run-forecast-explainer-tests.yml @@ -54,6 +54,6 @@ jobs: $CONDA/bin/conda init source /home/runner/.bashrc pip install -r test-requirements-operators.txt - pip install "oracle-automlx[forecasting]>=25.1.1" + pip install "oracle-automlx[forecasting]>=25.3.0" pip install pandas>=2.2.0 python -m pytest -v -p no:warnings --durations=5 tests/operators/forecast/test_explainers.py diff --git a/.github/workflows/run-forecast-unit-tests.yml b/.github/workflows/run-forecast-unit-tests.yml index b5b830cda..4a12deb51 100644 --- a/.github/workflows/run-forecast-unit-tests.yml +++ b/.github/workflows/run-forecast-unit-tests.yml @@ -56,6 +56,6 @@ jobs: $CONDA/bin/conda init source /home/runner/.bashrc pip install -r test-requirements-operators.txt - pip install "oracle-automlx[forecasting]>=25.1.1" + pip install "oracle-automlx[forecasting]>=25.3.0" pip install pandas>=2.2.0 python -m pytest -v -p no:warnings --durations=5 tests/operators/forecast --ignore=tests/operators/forecast/test_explainers.py diff --git a/ads/opctl/operator/lowcode/anomaly/model/automlx.py b/ads/opctl/operator/lowcode/anomaly/model/automlx.py index 059545cf8..3a170f242 100644 --- a/ads/opctl/operator/lowcode/anomaly/model/automlx.py +++ b/ads/opctl/operator/lowcode/anomaly/model/automlx.py @@ -24,8 +24,8 @@ class AutoMLXOperatorModel(AnomalyOperatorBaseModel): @runtime_dependency( module="automlx", err_msg=( - "Please run `pip3 install oracle-automlx>=23.4.1` and " - "`pip3 install oracle-automlx[classic]>=23.4.1` " + "Please run `pip3 install oracle-automlx>=25.3.0` and " + "`pip3 install oracle-automlx[classic]>=25.3.0` " "to install the required dependencies for automlx." ), ) diff --git a/ads/opctl/operator/lowcode/forecast/const.py b/ads/opctl/operator/lowcode/forecast/const.py index 36adfa694..88eb05b22 100644 --- a/ads/opctl/operator/lowcode/forecast/const.py +++ b/ads/opctl/operator/lowcode/forecast/const.py @@ -76,7 +76,7 @@ class ForecastOutputColumns(ExtendedEnum): AUTOMLX_METRIC_MAP = { "smape": "neg_sym_mean_abs_percent_error", - "mape": "neg_sym_mean_abs_percent_error", + "mape": "neg_mean_abs_percent_error", "mase": "neg_mean_abs_scaled_error", "mae": "neg_mean_absolute_error", "mse": "neg_mean_squared_error", diff --git a/ads/opctl/operator/lowcode/forecast/model/automlx.py b/ads/opctl/operator/lowcode/forecast/model/automlx.py index 0517e1b2a..cc8f20bc8 100644 --- a/ads/opctl/operator/lowcode/forecast/model/automlx.py +++ b/ads/opctl/operator/lowcode/forecast/model/automlx.py @@ -28,7 +28,9 @@ logging.getLogger("report_creator").setLevel(logging.WARNING) AUTOMLX_N_ALGOS_TUNED = 4 -AUTOMLX_DEFAULT_SCORE_METRIC = "neg_sym_mean_abs_percent_error" +AUTOMLX_DEFAULT_SCORE_METRIC = ['neg_sym_mean_abs_percent_error', + 'neg_mean_abs_percent_error', + 'neg_root_mean_squared_error'] class AutoMLXOperatorModel(ForecastOperatorBaseModel): @@ -45,10 +47,13 @@ def set_kwargs(self): model_kwargs_cleaned["n_algos_tuned"] = model_kwargs_cleaned.get( "n_algos_tuned", AUTOMLX_N_ALGOS_TUNED ) - model_kwargs_cleaned["score_metric"] = AUTOMLX_METRIC_MAP.get( - self.spec.metric, - model_kwargs_cleaned.get("score_metric", AUTOMLX_DEFAULT_SCORE_METRIC), - ) + metric_to_optimize = AUTOMLX_METRIC_MAP.get(self.spec.metric) + model_kwargs_cleaned["score_metric"] = AUTOMLX_DEFAULT_SCORE_METRIC + # The first score metric in the list will be the one for which the pipeline optimizes + if metric_to_optimize is not None: + model_kwargs_cleaned["score_metric"].remove(metric_to_optimize) + model_kwargs_cleaned["score_metric"].insert(0, metric_to_optimize) + model_kwargs_cleaned.pop("task", None) time_budget = model_kwargs_cleaned.pop("time_budget", -1) model_kwargs_cleaned["preprocessing"] = ( @@ -70,7 +75,7 @@ def preprocess(self, data, series_id): # TODO: re-use self.le for explanations @runtime_dependency( module="automlx", err_msg=( - "Please run `pip3 install oracle-automlx[forecasting]>=25.1.1` " + "Please run `pip3 install oracle-automlx[forecasting]>=25.3.0` " "to install the required dependencies for automlx." ), ) @@ -163,7 +168,7 @@ def _build_model(self) -> pd.DataFrame: self.models[s_id] = {} self.models[s_id]["model"] = model self.models[s_id]["le"] = self.le[s_id] - self.models[s_id]["score"] = self.get_validation_score_and_metric(model) + self.models[s_id]["score"] = self.get_all_metrics(model) # In case of Naive model, model.forecast function call does not return confidence intervals. if f"{target}_ci_upper" not in summary_frame: @@ -518,26 +523,27 @@ def explain_model(self): ) logger.debug(f"Full Traceback: {traceback.format_exc()}") - def get_validation_score_and_metric(self, model): + def get_all_metrics(self, model): trials = model.completed_trials_summary_ model_params = model.selected_model_params_ if len(trials) > 0: - score_col = [col for col in trials.columns if "Score" in col][0] - validation_score = trials[trials.Hyperparameters == model_params][ - score_col + all_metrics = trials[trials.Hyperparameters == model_params][ + "All Metrics" ].iloc[0] else: - validation_score = 0 - return -1 * validation_score + all_metrics = {} + reverse_map = {v: k for k, v in AUTOMLX_METRIC_MAP.items()} + all_metrics = {reverse_map[key]: -1 * value for key, value in all_metrics.items() if key in reverse_map} + return all_metrics def generate_train_metrics(self) -> pd.DataFrame: """ - Generate Training Metrics when fitted data is not available. + Generate Training Metrics for Automlx """ total_metrics = pd.DataFrame() for s_id in self.forecast_output.list_series_ids(): try: - metrics = {self.spec.metric.upper(): self.models[s_id]["score"]} + metrics = self.models[s_id]["score"] metrics_df = pd.DataFrame.from_dict( metrics, orient="index", columns=[s_id] )