Update python requirement

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "modelbench"
-requires-python = ">=3.10, !=3.12.5, <3.13"
+requires-python = ">=3.12,!=3.12.5,<4"
 
 [tool.poetry]
 name = "modelbench"
@@ -49,7 +49,7 @@ packages = [
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.10,!=3.12.5,<3.13"
+python = ">=3.12,!=3.12.5,<4"
 jq = "^1.6.0"
 click = "^8.3.0"
 casefy = "^1.0.0"

src/modelgauge/config.py

@@ -7,13 +7,16 @@
 import tomli
 
 from modelgauge import config_templates
+from modelgauge.log_config import get_logger
 from modelgauge.secret_values import MissingSecretValues, RawSecrets, SecretDescription
 
 DEFAULT_CONFIG_DIR = "config"
 DEFAULT_SECRETS = "secrets.toml"
 SECRETS_PATH = os.path.join(DEFAULT_CONFIG_DIR, DEFAULT_SECRETS)
 CONFIG_TEMPLATES = [DEFAULT_SECRETS]
 
+logger = get_logger(__name__)
+
 
 def find_config_dir(path: str = ".") -> str:
     """Search up the tree for the config directory."""
@@ -47,9 +50,13 @@ def write_default_config(parent_dir: str = "."):
 
 def load_secrets_from_config(path: str = ".") -> RawSecrets:
     """Load the toml file and verify it is shaped as expected."""
-    secrets_path = os.path.join(find_config_dir(path), DEFAULT_SECRETS)
-    with open(secrets_path, "rb") as f:
-        data = tomli.load(f)
+    try:
+        secrets_path = os.path.join(find_config_dir(path), DEFAULT_SECRETS)
+        with open(secrets_path, "rb") as f:
+            data = tomli.load(f)
+    except FileNotFoundError as exc:
+        logger.warning("Could not find secrets file", exc_info=exc)
+        data = {}
     for values in data.values():
         # Verify the config is shaped as expected.
         assert isinstance(values, Mapping), "All keys should be in a [scope]."

src/modelgauge/ensemble_annotator.py

@@ -6,22 +6,23 @@
 from modelgauge.config import load_secrets_from_config, raise_if_missing_from_config
 from modelgauge.ensemble_strategies import ENSEMBLE_STRATEGIES
 from modelgauge.prompt import ChatPrompt, TextPrompt
-from modelgauge.secret_values import MissingSecretValues
+from modelgauge.secret_values import MissingSecretValues, RawSecrets
 from modelgauge.sut import SUTResponse
 
 
 class EnsembleAnnotator(Annotator):
     """Defines an ensemble; responds like an annotator."""
 
-    def __init__(self, uid, annotators: list[str], ensemble_strategy: str):
+    def __init__(self, uid, annotators: list[str], ensemble_strategy: str, secrets: RawSecrets | None = None):
         super().__init__(uid)
-        self.annotators = self._make_annotators(annotators)
+        self.annotators = self._make_annotators(annotators, secrets)
         if ensemble_strategy not in ENSEMBLE_STRATEGIES:
             raise ValueError(f"Ensemble strategy {ensemble_strategy} not recognized.")
         self.ensemble_strategy = ENSEMBLE_STRATEGIES[ensemble_strategy]
 
-    def _make_annotators(self, annotator_uids: list[str]) -> dict[str, Annotator]:
-        secrets = load_secrets_from_config()
+    def _make_annotators(self, annotator_uids: list[str], secrets: RawSecrets | None) -> dict[str, Annotator]:
+        if secrets is None:
+            secrets = load_secrets_from_config()
         missing_secrets: list[MissingSecretValues] = []
         for annotator_uid in annotator_uids:
             missing_secrets.extend(ANNOTATORS.get_missing_dependencies(annotator_uid, secrets=secrets))

src/modelgauge/private_benchmark_uids.py

@@ -0,0 +1,2 @@
+SAFETY_ANNOTATOR_V1_1_UID = "safety_v1.1"
+SECURITY_ANNOTATOR_V0_5_UID = "security_v0.5"

src/modelgauge/private_ensemble_annotator_uids.py

@@ -1,3 +1,5 @@
+# TODO: remove this file once modelbench-private is removed
+
 try:
     from modelgauge.annotators.official import (
         SAFETY_ANNOTATOR_V1_0_UID,

src/modelgauge/suts/huggingface_chat_completion.py

@@ -88,6 +88,7 @@ def evaluate(self, request: HuggingFaceChatCompletionRequest) -> HuggingFaceChat
         except HTTPError as http_error:
             if http_error.response.status_code >= 500 or http_error.response.status_code == 429:
                 raise TransientHttpError from http_error
+            raise
 
         # Convert to cacheable pydantic object.
         return HuggingFaceChatCompletionOutput(

tests/modelgauge_tests/test_config.py

@@ -86,8 +86,8 @@ def test_load_secrets_from_config_no_file(tmpdir):
     config_dir = tmpdir.join("config")
     os.makedirs(config_dir)
 
-    with pytest.raises(FileNotFoundError):
-        load_secrets_from_config(tmpdir)
+    secrets = load_secrets_from_config(tmpdir)
+    assert secrets == {}
 
 
 def test_load_secrets_from_config_bad_format(tmpdir):