Modernize dependencies and CI infrastructure + minor refactor (#218)

satishlokkoju · web-flow · commit 131fc2aa366b · 2025-11-04T18:38:17.000+01:00
Modernize dependencies and CI infrastructure

- Add Python 3.12 support, drop Python 3.6/3.7
- Update dependencies: numpy &lt;3, pyarrow &gt;=16.0.0, fire &lt;0.7.0
- Upgrade GitHub Actions: ubuntu-22.04, actions v2→v4
- Sync pyspark version in Makefile with requirements-test.txt
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -10,13 +10,13 @@ on:
 
 jobs:
   lint:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: '3.10'
       - name: Install
         run: |
           python3 -m venv .env
@@ -28,15 +28,21 @@ jobs:
           source .env/bin/activate
           make lint
   tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
+      fail-fast: false
       matrix:
-        python-version: [3.8, 3.9, '3.10', 3.11]
+        python-version: ['3.10', 3.11, 3.12]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
+    - name: Set up Java
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin'
+        java-version: '17'
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install
diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml
@@ -11,15 +11,15 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: [3.9]
+        python-version: ['3.10']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install pandoc
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -9,14 +9,19 @@ on:
     - master
 jobs:
   build-pex:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ['3.10']
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
+    - name: Set up Java
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin'
+        java-version: '17'
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
@@ -27,28 +32,33 @@ jobs:
       run: |
         make build-pex
         mv autofaiss.pex autofaiss-${{ matrix.python-version }}.pex
-    - uses: actions/upload-artifact@v2
+    - uses: actions/upload-artifact@v4
       with:
         name: autofaiss_pex
         path: autofaiss-${{ matrix.python-version }}.pex
   deploy:
     needs: build-pex
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
+    - name: Set up Java
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin'
+        java-version: '17'
     - uses: actions-ecosystem/action-regex-match@v2
       id: regex-match
       with:
         text: ${{ github.event.head_commit.message }}
         regex: '^Release ([^ ]+)'
     - name: Download pex
-      uses: actions/download-artifact@v2
+      uses: actions/download-artifact@v4
       with:
         name: autofaiss_pex
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
-        python-version: '3.8'
+        python-version: '3.10'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
@@ -58,7 +68,7 @@ jobs:
       uses: softprops/action-gh-release@v1
       with:
         files: |
-          autofaiss-3.8.pex
+          autofaiss-3.10.pex
         tag_name: ${{ steps.regex-match.outputs.group1 }}
     - name: Build and publish
       if: ${{ steps.regex-match.outputs.match != '' && github.event_name != 'pull_request' }}
diff --git a/.gitignore b/.gitignore
@@ -5,7 +5,8 @@
 .coverage*
 *.npy
 *.index
-
+*.index*
+*.json*
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
diff --git a/.nojekyll b/.nojekyll
diff --git a/Makefile b/Makefile
@@ -30,7 +30,7 @@ build-dist: ## [Continuous integration] Build package for pypi
 build-pex:
 	python3 -m venv .pexing
 	. .pexing/bin/activate && python -m pip install -U pip && python -m pip install pex
-	. .pexing/bin/activate && python -m pex setuptools pyspark==3.2.1 s3fs>=2022.1.0 . -o autofaiss.pex -v
+	. .pexing/bin/activate && python -m pex setuptools "pyspark>=4.0.1,<5.0.0" s3fs>=2022.1.0 "numpy>=1.19.5,<2" . -o autofaiss.pex -v
 	rm -rf .pexing
 
 .PHONY: help
diff --git a/docs/.gitignore b/docs/.gitignore
diff --git a/docs/Makefile b/docs/Makefile
diff --git a/docs/conf.py b/docs/conf.py
diff --git a/docs/getting_started/quantization.rst b/docs/getting_started/quantization.rst
diff --git a/mypy.ini b/mypy.ini
@@ -1,5 +1,5 @@
 # Global options:
 
 [mypy]
-python_version = 3.8
+python_version = 3.10
 ignore_missing_imports = True
diff --git a/requirements-test.txt b/requirements-test.txt
@@ -4,5 +4,4 @@ pylint==3.0.3
 pytest-cov==4.1.0
 pytest-xdist==3.5.0
 pytest==8.0.1
-pyspark==3.2.2; python_version < "3.11"
-pyspark<3.6.0; python_version >= "3.11"
+pyspark>=4.0.1,<5.0.0
diff --git a/requirements.txt b/requirements.txt
@@ -1,10 +1,8 @@
-dataclasses>=0.6,<1.0.0; python_version < "3.7"
-fire>=0.4.0,<0.6.0
+fire>=0.4.0,<0.7.0
 numpy>=1.19.5,<2
 pandas>=1.1.5,<3
-pyarrow>=6.0.1,<16
+pyarrow>=6.0.1,<30
 tqdm>=4.62.3,<5
-faiss-cpu<1.7.3; python_version < "3.7"
-faiss-cpu>=1,<2; python_version >= "3.7"
+faiss-cpu>=1,<2
 fsspec>=2022.1.0
-embedding_reader>=1.5.1,<2
+embedding_reader>=1.8.0,<2
diff --git a/setup.py b/setup.py
@@ -33,14 +33,12 @@ def _read_reqs(relpath):
             "License :: OSI Approved :: Apache Software License",
             "Operating System :: OS Independent",
             "Programming Language :: Python :: 3",
-            "Programming Language :: Python :: 3.6",
-            "Programming Language :: Python :: 3.7",
-            "Programming Language :: Python :: 3.8",
-            "Programming Language :: Python :: 3.9",
             "Programming Language :: Python :: 3.10",
             "Programming Language :: Python :: 3.11",
+            "Programming Language :: Python :: 3.12",
             "Intended Audience :: Developers",
         ],
+        python_requires=">=3.10",
         long_description=long_description,
         long_description_content_type="text/markdown",
         description=long_description.split("\n")[0],
diff --git a/tests/unit/test_quantize.py b/tests/unit/test_quantize.py
@@ -386,14 +386,15 @@ def test_index_correctness_in_distributed_mode_with_multiple_indices(tmpdir):
         file_format="parquet",
         temporary_indices_folder=temporary_indices_folder,
         max_index_memory_usage="2GB",
-        current_memory_available="500MB",
+        current_memory_available="4GB",
         embedding_column_name="embedding",
         index_key="IVF1,Flat",
         should_be_memory_mappable=True,
         ids_path=ids_path,
         nb_indices_to_keep=2,
         save_on_disk=True,
         id_columns=["id"],
+        index_param="nprobe=1",
     )
     index_paths = sorted(index_path2_metric_infos.keys())
     K, NB_QUERIES = 5, 1
@@ -427,11 +428,12 @@ def test_index_correctness_in_distributed_mode_with_multiple_indices(tmpdir):
         file_format="npy",
         temporary_indices_folder=temporary_indices_folder,
         max_index_memory_usage="2GB",
-        current_memory_available="500MB",
+        current_memory_available="4GB",
         embedding_column_name="embedding",
         index_key="IVF1,Flat",
         should_be_memory_mappable=True,
         nb_indices_to_keep=2,
+        index_param="nprobe=1",
     )
 
     ground_truth_index = faiss.index_factory(dim, "IVF1,Flat", faiss.METRIC_INNER_PRODUCT)
