ADULT: Change objective from max to min problem --> New Container Version 0.0.2

Author: PhMueller

hpobench/benchmarks/mo/adult_benchmark.py

@@ -1,6 +1,8 @@
 """
 Changelog:
 ==========
+0.0.2:
+* Change the objective value from accuracy to misclassification rate. (1 - accuracy)
 
 0.0.1:
 * First implementation of the Multi-Objective Fair Adult Benchmark.
@@ -127,7 +129,7 @@ def get_meta_information() -> Dict:
     @staticmethod
     def get_objective_names() -> List[str]:
         """Get a list of objectives evaluated in the objective_function. """
-        return ['accuracy', 'DSP', 'DEO', 'DFP']
+        return ['misclassification_rate', 'DSP', 'DEO', 'DFP']
 
     @AbstractMultiObjectiveBenchmark.check_parameters
     def objective_function(self, configuration: Union[CS.Configuration, Dict],
@@ -165,7 +167,7 @@ def objective_function(self, configuration: Union[CS.Configuration, Dict],
         -------
         Dict -
             function_value : Dict - validation metrics after training on train
-                accuracy: float
+                misclassification_rate: float: 1 - validation accuracy
                 DSO: float
                 DEO: float
                 DFP: float
@@ -247,7 +249,7 @@ def objective_function(self, configuration: Union[CS.Configuration, Dict],
 
         elapsed_time = time.time() - ts_start
 
-        return {'function_value': {'accuracy': float(val_accuracy),
+        return {'function_value': {'misclassification_rate': 1 - float(val_accuracy),
                                    'DSO': float(val_statistical_disparity),
                                    'DEO': float(val_unequal_opportunity),
                                    'DFP': float(val_unequalized_odds)
@@ -310,7 +312,7 @@ def objective_function_test(self, configuration: Union[CS.Configuration, Dict],
         -------
         Dict -
             function_value : Dict - test metrics reported after training on (train+valid)
-                accuracy: float
+                misclassification_rate: float: 1 - test accuracy
                 DSO: float
                 DEO: float
                 DFP: float
@@ -381,7 +383,7 @@ def objective_function_test(self, configuration: Union[CS.Configuration, Dict],
         logger.debug(f"config:{configuration}, test_score: {test_accuracy}, train score:{train_accuracy},"
                      f"dsp:{test_statistical_disparity}, deo :{test_unequal_opportunity}, dfp :{test_unequalized_odds}")
 
-        return {'function_value': {'accuracy': float(test_accuracy),
+        return {'function_value': {'misclassification_rate': 1 - float(test_accuracy),
                                    'DSO': float(test_statistical_disparity),
                                    'DEO': float(test_unequal_opportunity),
                                    'DFP': float(test_unequalized_odds)

hpobench/container/benchmarks/mo/adult_benchmark.py

@@ -8,5 +8,5 @@ class AdultBenchmark(AbstractBenchmarkClient):
     def __init__(self, **kwargs):
         kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'AdultBenchmark')
         kwargs['container_name'] = kwargs.get('container_name', 'fair_adult')
-        kwargs['latest'] = kwargs.get('container_tag', '0.0.1')
+        kwargs['latest'] = kwargs.get('container_tag', '0.0.2')
         super(AdultBenchmark, self).__init__(**kwargs)

tests/test_adult.py

@@ -28,10 +28,10 @@ def test_adult_benchmark():
     result_2 = benchmark.objective_function(test_config, rng=1, fidelity={'budget': 3})
 
     assert result_1['info']['valid_accuracy'] == pytest.approx(0.7539, rel=0.001)
-    assert result_1['info']['valid_accuracy'] == result_1['function_value']['accuracy']
+    assert 1 - result_1['info']['valid_accuracy'] == result_1['function_value']['misclassification_rate']
     assert result_1['info']['train_accuracy'] == pytest.approx(0.76145, rel=0.001)
     assert result_1['info']['train_accuracy'] == result_2['info']['train_accuracy']
 
     result_1 = benchmark.objective_function_test(test_config, rng=1, fidelity={'budget': 3})
-    assert result_1['function_value']['accuracy'] == pytest.approx(0.76377, rel=0.001)
-    assert result_1['function_value']['accuracy'] == result_1['info']['test_accuracy']
+    assert 1 - result_1['function_value']['misclassification_rate'] == pytest.approx(0.76377, rel=0.001)
+    assert 1 - result_1['function_value']['misclassification_rate'] == result_1['info']['test_accuracy']