Merge pull request #856 from davidhewitt/aarch64-manylinux

wangxiaoying · web-flow · commit e687b6095113 · 2025-10-21T10:24:35.000-07:00
build aarch64 with manylinux2_28
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -8,11 +8,17 @@ on:
 
 jobs:
   linux:
-    runs-on: ubuntu-latest
-    container: quay.io/pypa/manylinux_2_28_x86_64
+    runs-on: ${{ matrix.platform.runs-on }}
+    container: quay.io/pypa/manylinux_2_28_${{ matrix.platform.arch }}
     strategy:
       matrix:
-        python-version: [[310, "3.10"], [311, "3.11"], [312, "3.12"], [313, "3.13"]]
+        platform:
+          [
+            { "arch": "x86_64", "runs-on": "ubuntu-latest" },
+            { "arch": "aarch64", "runs-on": "ubuntu-22.04-arm" },
+          ]
+        python-version:
+          [[310, "3.10"], [311, "3.11"], [312, "3.12"], [313, "3.13"]]
     steps:
       - uses: actions/checkout@v5
 
@@ -66,80 +72,17 @@ jobs:
 
       - uses: actions/upload-artifact@v4
         with:
-          name: "ubuntu-latest-${{ matrix.python-version[1] }}"
-          path: connectorx-python/target/wheels/*.whl
-
-  linux-aarch:
-    runs-on: ubuntu-22.04-arm
-    strategy:
-      matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13"]
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v5
-
-      # Check architecture of the target machine
-      - name: Check Target Architecture
-        run: |
-          echo "Architecture: $(uname -m)"
-          if [[ $(uname -m) != "aarch64" ]]; then
-            echo "Error: This workflow requires ARM architecture (aarch64)."
-            exit 1
-          fi
-
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Setup Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          components: rustfmt
-          target: aarch64-unknown-linux-gnu
-          default: true
-
-      - name: Setup Just
-        uses: extractions/setup-just@v3
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Install Tools
-        run: |
-          sudo apt-get update -y
-          sudo apt-get install -y mysql-client libpq-dev freetds-dev krb5-locales clang libkrb5-dev build-essential gcc musl-tools libsasl2-modules-gssapi-mit libgssapi-krb5-2 krb5-user
-      - name: Setup Poetry
-        uses: Gr1N/setup-poetry@v9
-
-      - name: Setup Project
-        run: |
-          just bootstrap-python
-          python -m pip install --upgrade pip
-          pip install maturin
-
-      - name: Build Wheel (Native)
-        run: |
-          maturin build -m connectorx-python/Cargo.toml --target aarch64-unknown-linux-gnu -i python${{ matrix.python-version }} --release
-        env:
-          SQLITE3_STATIC: 1
-          KRB5_INCLUDE_DIR: /usr/include
-          KRB5_LIB_DIR: /usr/lib/aarch64-linux-gnu
-
-      - name: Upload Artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: "linux-arm-${{ matrix.python-version }}"
+          name: "linux-${{matrix.platform.arch}}-${{ matrix.python-version[1] }}"
           path: connectorx-python/target/wheels/*.whl
 
   win-and-mac:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: ["windows-latest", "macos-13"]
+        os: ["windows-latest", "macos-15-intel"]
         python-version: ["3.10", "3.11", "3.12", "3.13"]
         include:
-          - os: "macos-13"
+          - os: "macos-15-intel"
             features: "--features integrated-auth-gssapi"
     steps:
       - uses: actions/checkout@v5
@@ -236,30 +179,22 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.10", "3.11", "3.12", "3.13"]
-        os: [macos-13, ubuntu-latest, windows-latest]
-    steps:
-      - uses: actions/checkout@v5
-
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          architecture: x64
-
-      - uses: actions/download-artifact@v5
-        with:
-          name: "${{ matrix.os }}-${{ matrix.python-version }}"
-
-      - run: |
-          pip install *.whl
-          python -c "import connectorx"
-
-  verify-apple-arm:
-    runs-on: ${{ matrix.os }}
-    needs: [apple-arm]
-    strategy:
-      matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13"]
-        os: [macos-latest]
+        os:
+          [
+            macos-15-intel,
+            ubuntu-latest,
+            windows-latest,
+            ubuntu-22.04-arm,
+            macos-latest,
+          ]
+        include:
+          # override artifact-prefix for linux variants
+          - os: "ubuntu-latest"
+            artifact-prefix: "linux-x86_64"
+          - os: "ubuntu-22.04-arm"
+            artifact-prefix: "linux-aarch64"
+          - os: "macos-latest"
+            artifact-prefix: "macos-arm"
     steps:
       - uses: actions/checkout@v5
 
@@ -269,50 +204,15 @@ jobs:
 
       - uses: actions/download-artifact@v5
         with:
-          name: "macos-arm-${{ matrix.python-version }}"
+          name: "${{ matrix.artifact-prefix || matrix.os }}-${{ matrix.python-version }}"
 
       - run: |
           pip install *.whl
           python -c "import connectorx"
 
-  verify-linux-arm:
-    runs-on: ubuntu-22.04-arm
-    needs: [linux-aarch]
-    strategy:
-      matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13"]
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v5
-
-      # Check architecture of the target machine
-      - name: Check Target Architecture
-        run: |
-          echo "Architecture: $(uname -m)"
-          if [[ $(uname -m) != "aarch64" ]]; then
-            echo "Error: This workflow requires ARM architecture (aarch64)."
-            exit 1
-          fi
-
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Download Artifact
-        uses: actions/download-artifact@v5
-        with:
-          name: "linux-arm-${{ matrix.python-version }}"
-
-      - name: Install and Test
-        run: |
-          python -m pip install --upgrade pip
-          pip install *.whl
-          python -c 'import connectorx'
-
   upload:
     runs-on: ubuntu-latest
-    needs: [verify, verify-apple-arm, verify-linux-arm]
+    needs: [verify]
     steps:
       - name: Download all artifacts
         uses: actions/download-artifact@v5
diff --git a/README.md b/README.md
@@ -29,7 +29,7 @@ cx.read_sql("postgresql://username:password@server:port/database", "SELECT * FRO
 
 The function will partition the query by **evenly** splitting the specified column to the amount of partitions.
 ConnectorX will assign one thread for each partition to load and write data in parallel.
-Currently, we support partitioning on **numerical** columns (**cannot contain NULL**) for **SPJA** queries. 
+Currently, we support partitioning on **numerical** columns (**cannot contain NULL**) for **SPJA** queries.
 
 **Experimental: We are now providing federated query support, you can write a single query to join tables from two or more databases!**
 ```python
@@ -48,8 +48,6 @@ Check out more detailed usage and examples [here](https://sfu-db.github.io/conne
 pip install connectorx
 ```
 
-_For AArch64 or ARM64 Linux users, `connectorx==0.4.3 & above` is only available for distributions using `glibc 2.35` and above. Specifically, the re-release for this architecture was tested on Ubuntu 22.04. For older distributions, the latest version available is `connectorx==0.2.3` due to dependency limitations._
-
 Check out [here](https://sfu-db.github.io/connector-x/install.html#build-from-source-code) to see how to build python wheel from source.
 
 # Performance
@@ -86,7 +84,7 @@ is not specified, the count query will be `SELECT COUNT(*) FROM (SELECT * FROM l
 Finally, ConnectorX will use the schema info as well as the count info to allocate memory and download data by executing the queries normally.
 
 Once the downloading begins, there will be one thread for each partition so that the data are downloaded in parallel at the partition level. The thread will issue the query of the corresponding
-partition to the database and then write the returned data to the destination row-wise or column-wise (depends on the database) in a streaming fashion. 
+partition to the database and then write the returned data to the destination row-wise or column-wise (depends on the database) in a streaming fashion.
 
 
 # Supported Sources & Destinations
diff --git a/docs/install.md b/docs/install.md
@@ -10,10 +10,6 @@ The easiest way to install ConnectorX is using pip, with the following command:
 pip install connectorx
 ```
 
-```{note}
-For AArch64 or ARM64 Linux users, `connectorx==0.4.3 & above` is only available for distributions using `glibc 2.35` and above. Specifically, the re-release for this architecture was tested on Ubuntu 22.04. For older distributions, the latest version available is `connectorx==0.2.3` due to dependency limitations.
-```
-
 ### Build from source code
 
 * Step 0: Install tools.
@@ -47,4 +43,3 @@ just build-python-wheel
 NOTES:
 * `OPENSSL_NO_VENDOR=1` might required to compile for windows users.
 * Dynamic library is required for the python installation. (e.g. If you are using `pyenv`, use command `PYTHON_CONFIGURE_OPTS=“--enable-shared” pyenv install {version}` to install python since dylib is not enabled by default.)
-
