Airflow update fix

doc/requirements.txt

@@ -1,7 +1,7 @@
 sphinx==7.2.6
 sphinx-rtd-theme==3.0.0
 docutils>=0.18.1,<0.21
-packaging>=23.0,<25
+packaging>=24.0,<25
 jinja2==3.1.6
 schema==0.7.5
 accelerate>=0.24.1,<=0.27.0

pyproject.toml

@@ -37,11 +37,11 @@ dependencies = [
   "docker",
   "fastapi",
   "google-pasta",
-  "importlib-metadata>=1.4.0,<7.0",
+  "importlib-metadata>=1.4.0,<9.0",
   "jsonschema",
   "numpy==1.26.4",
   "omegaconf>=2.2,<3",
-  "packaging>=23.0,<25",
+  "packaging>=24.0,<26",
   "pandas",
   "pathos",
   "platformdirs",

requirements/constraints.txt

@@ -0,0 +1,22 @@
+# Constraints file to resolve dependency conflicts
+# This forces specific versions to be used regardless of what other packages require
+
+# Core conflicts
+importlib-metadata>=1.4.0,<9.0
+rich>=13.7.0
+Flask-Limiter==3.5.0
+
+# Additional constraints to ensure compatibility
+sagemaker-core>=1.0.17
+apache-airflow==3.0.3
+apache-airflow-providers-amazon==7.2.1
+
+# Sphinx dependencies
+sphinx==7.2.6
+sphinx-rtd-theme==3.0.0
+docutils>=0.18.1,<0.21
+packaging>=24.0,<26
+jinja2==3.1.6
+schema==0.7.5
+accelerate>=0.24.1,<=0.27.0
+graphene<4.0

requirements/extras/test_requirements.txt

@@ -14,9 +14,11 @@ awslogs==0.14.0
 black==24.3.0
 stopit==1.1.2
 # Update tox.ini to have correct version of airflow constraints file
-apache-airflow==2.10.4
+apache-airflow==3.0.3
 apache-airflow-providers-amazon==7.2.1
-Flask-Limiter==3.11
+Flask-Limiter==3.5.0
+rich>=13.7.0
+importlib-metadata>=1.4.0,<9.0
 attrs>=24,<26
 fabric==3.2.2
 requests==2.32.2

src/sagemaker/serve/utils/conda_in_process.yml

@@ -15,8 +15,8 @@ dependencies:
     - numpy==1.26.4
     - protobuf>=3.12,<5.0
     - smdebug_rulesconfig==1.0.1
-    - importlib-metadata>=1.4.0,<7.0
-    - packaging>=23.0,<25
+    - importlib-metadata>=1.4.0,<9.0
+    - packaging>=24.0,<26
     - pandas
     - pathos
     - schema
@@ -53,7 +53,7 @@ dependencies:
             - h11>=0.14.0
             - huggingface-hub>=0.13.4
             - idna>=3.4
-            - importlib-metadata>=4.13.0
+            - importlib-metadata>=1.4.0,<9.0
             - jinja2>=3.1.2
             - jmespath>=1.0.1
             - jsonschema>=4.17.3
@@ -85,7 +85,8 @@ dependencies:
             - pyyaml>=6.0.1
             - regex>=2023.3.23
             - requests>=2.28.2
-            - rich>=13.3.4
+            - rich==13.7.0
+            - safetensors>=0.4.0
             - rsa>=4.7.2
             - s3transfer>=0.6.0
             - sagemaker>=2.148.0

src/sagemaker/serve/utils/in_process_requirements.txt

@@ -21,7 +21,7 @@ gunicorn>=20.1.0
 h11>=0.14.0
 huggingface-hub>=0.13.4
 idna>=3.4
-importlib-metadata>=4.13.0
+importlib-metadata>=1.4.0,<9.0
 jinja2>=3.1.2
 jmespath>=1.0.1
 jsonschema>=4.17.3
@@ -31,7 +31,7 @@ mdurl>=0.1.2
 mpmath>=1.3.0
 multiprocess>=0.70.14
 networkx>=3.1
-packaging>=23.1
+packaging>=24.0,<26
 pandas>=1.5.3
 pathos>=0.3.0
 pillow>=9.5.0
@@ -53,7 +53,7 @@ pytz-deprecation-shim>=0.1.0.post0
 pyyaml>=6.0.1
 regex>=2023.3.23
 requests>=2.28.2
-rich>=13.3.4
+rich==13.7.0
 rsa>=4.7.2
 s3transfer>=0.6.0
 sagemaker>=2.148.0

tox.ini

@@ -82,15 +82,17 @@ passenv =
 # Can be used to specify which tests to run, e.g.: tox -- -s
 commands =
     python -c "import os; os.system('install-custom-pkgs --install-boto-wheels')"
-    pip install 'apache-airflow==2.10.4' --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.4/constraints-3.9.txt"
+    pip install 'apache-airflow==3.0.3' 'Flask-Limiter==3.5.0' 'rich==13.7.0' --constraint {toxinidir}/requirements/constraints.txt
     pip install 'torch==2.3.1+cpu' -f 'https://download.pytorch.org/whl/torch_stable.html'
     pip install 'torchvision==0.18.1+cpu' -f 'https://download.pytorch.org/whl/torch_stable.html'
     pip install 'dill>=0.3.9'
     pip install 'altair>=5.3' # needed for amtviz
     pip install -U "sagemaker-core" # needed to keep sagemaker-core up to date
 
     pytest {posargs}
-deps = .[test]
+deps =
+    -c {toxinidir}/requirements/constraints.txt
+    .[test]
 depends =
     {py39,py310,py311,py312}: clean
 
@@ -139,12 +141,22 @@ commands =
 
 [testenv:sphinx]
 pip_version = pip==24.3
+skip_install = true
 changedir = doc
 # pip install requirements.txt is separate as RTD does it in separate steps
 # having the requirements.txt installed in deps above results in Double Requirement exception
 # https://github.com/pypa/pip/issues/988
+deps =
+    sphinx==7.2.6
+    sphinx-rtd-theme==3.0.0
+    docutils>=0.18.1,<0.21
+    packaging>=23.0,<26
+    jinja2==3.1.6
+    schema==0.7.5
+    accelerate>=0.24.1,<=0.27.0
+    graphene<4.0
 commands =
-    pip install --exists-action=w -r requirements.txt
+    pip install -e {toxinidir}
     sphinx-build -T -b html -d _build/doctrees-readthedocs -D language=en . _build/html
 
 [testenv:doc8]
@@ -199,6 +211,8 @@ commands =
 
 [testenv:collect-tests]
 # this needs to succeed for tests to display in some IDEs
-deps = .[test]
+skip_install = true
+deps =
+    pytest==6.2.5
 commands =
     pytest --collect-only