Merge pull request #2024 from apache/build_release_artifact_ci_job

Add new CI job which builds the release artifact, remove MANIFEST.in

Author: Kami

.github/workflows/main.yml

@@ -170,6 +170,83 @@ jobs:
         run: |
           tox -e black-check,isort-check,pyupgrade,checks,import-timings,lint,pylint,mypy
 
+  build_test_release_artifact:
+    name: Build and Test Release Artifact
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python_version: [3.9]
+
+    steps:
+      - uses: actions/checkout@master
+        with:
+          fetch-depth: 1
+
+      - name: Use Python ${{ matrix.python_version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python_version }}
+
+      - name: Cache Python Dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-lint.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install Python Dependencies
+        run: |
+          pip install -r requirements-ci.txt
+          pip install "build==1.2.2"
+
+      - name: Build Release Artifact
+        run: |
+          pip list installed
+          python -m build -vv
+
+      - name: Set Environment
+        run: |
+          export PYTHONPATH=.
+          export VERSION=$(python -c "import libcloud ; print(libcloud.__version__)")
+          echo "VERSION=${VERSION}" >> "$GITHUB_ENV"
+
+      - name: Verify Tarball Release Artifact
+        run: |
+          # Verify tarball file exists
+          export TARBALL_FILENAME="apache_libcloud-${VERSION}.tar.gz"
+
+          ls -la "dist/${TARBALL_FILENAME}"
+
+          cd dist/
+
+          # Unpack tarball and verify + run the tests
+          tar -xzvf "${TARBALL_FILENAME}"
+
+          cd "apache_libcloud-${VERSION}/" 
+          tox -c tox.ini -epy3.9
+
+      - name: Verify Wheel Release Artifact
+        run: |
+          # Verify wheel file exists
+          export WHEEL_FILENAME="apache_libcloud-${VERSION}-py3-none-any.whl"
+
+          ls -la "dist/${WHEEL_FILENAME}"
+
+          cd dist/
+
+          # Unpack wheel and verify + run tests
+          unzip "${WHEEL_FILENAME}" -d "wheel"
+          cd wheel
+
+          # Since wheel doesn't include those files, we need to manually copy them over from
+          # repo root so we can run the tests
+          cp ../../tox.ini .
+          cp ../../requirements-tests.txt .
+          cp ../../libcloud/test/secrets.py-dist libcloud/test/secrets.py-dist
+          tox -c tox.ini -epy3.9
+
   build_test_docker_image:
     name: Build and Verify Docker Image
     runs-on: ubuntu-latest

.gitignore

@@ -15,6 +15,7 @@ coverage.xml
 .idea
 dist/*apache-libcloud*
 dist/*apache_libcloud*
+dist/wheel
 docs/apidocs/*
 _build/
 apache_libcloud.egg-info/

CHANGES.rst

@@ -7,6 +7,12 @@ Changes in Apache Libcloud 3.9.0
 Common
 ~~~~~~
 
+- Unused ``setup.py`` file has been removed. The project has switched
+  to ``pyproject.toml`` a while ago and unused file has been removed to
+  reduce potential confusion.
+  (#2024)
+  [Tomaz Muraus - @Kami]
+
 - Indicate we also support Python 3.12 (non beta) and Python 3.13.
   (#2050)
   [Tomaz Muraus - @Kami]
@@ -36,6 +42,16 @@ Common
   (#1940)
   [@munahaf on behalf of OpenRefactory and Open Source Security Foundation]
 
+- Update versions of build and packaging tools required to build the package.
+
+  Per report from Rui Chen (@chenrui333) ansible homebrew package which
+  depends on libcloud was failing to build with Libcloud 3.8.0.
+
+  Special thanks to Rui Chen for their assistance with troubleshooting the issue
+  and testing v3.9.0 release candidate.
+  (#2047)
+  [Tomaz Muraus - @Kami, Rui Chen - @chenrui333]
+
 Compute
 ~~~~~~~
 

MANIFEST.in

@@ -1,5 +1,10 @@
+# NOTE: We still need to use MANIFEST.in for backward compatibility with past
+# distributions since pyproject.toml + setuptools backend doesn't support
+# as flexible includes and excludes as we need.
 global-exclude *.py[cod]
 global-exclude .pytest_cache
+
+# Include common files at the repository root
 include LICENSE
 include NOTICE
 include example_*.py
@@ -23,3 +28,16 @@ prune docs/
 prune demos/
 prune integration/
 prune pylint_plugins/
+prune __pycache__
+
+# Recursively include all files under the fixture directories
+recursive-include libcloud/test/backup/fixtures *
+recursive-include libcloud/test/common/fixtures *
+recursive-include libcloud/test/compute/fixtures *
+recursive-include libcloud/test/container/fixtures *
+recursive-include libcloud/test/dns/fixtures *
+recursive-include libcloud/test/loadbalancer/fixtures *
+recursive-include libcloud/test/storage/fixtures *
+
+# Exclude __pycache__ directories
+prune **/__pycache__

dist/deploy.sh

@@ -21,7 +21,7 @@ pushd "${SCRIPT_DIR}/../"
 
 # We redirect stderr to /dev/null since sometimes setuptools may print pyproject
 # related warning
-VERSION=$(python setup.py --version 2> /dev/null)
+VERSION=$(grep -Po '(?<=^__version__ = ")[^"]+' libcloud/__init__.py)
 popd
 
 pushd "${SCRIPT_DIR}"

dist/verify_checksums.sh

@@ -30,11 +30,11 @@ TMP_DIR=$(mktemp -d)
 # TODO: Use json endpoint + jq to parse out the url
 # https://pypi.org/pypi/apache-libcloud/3.4.0/json
 EXTENSIONS[0]=".tar.gz"
-EXTENSIONS[1]="-py2.py3-none-any.whl"
+EXTENSIONS[1]="-py3-none-any.whl"
 
 APACHE_MIRROR_URL="http://www.apache.org/dist/libcloud"
 PYPI_MIRROR_URL_SOURCE="https://pypi.python.org/packages/source/a/apache-libcloud"
-PYPI_MIRROR_URL_WHEEL="https://files.pythonhosted.org/packages/py2.py3/a/apache-libcloud"
+PYPI_MIRROR_URL_WHEEL="https://files.pythonhosted.org/packages/py3/a/apache-libcloud"
 
 # From http://tldp.org/LDP/abs/html/debugging.html#ASSERT
 function assert ()                 #  If condition false,
@@ -69,15 +69,15 @@ do
     extension=${EXTENSIONS[$i]}
     file_name="${VERSION}${extension}"
 
-    if [ "${extension}" = "-py2.py3-none-any.whl" ]; then
+    if [ "${extension}" = "-py3-none-any.whl" ]; then
         # shellcheck disable=SC2001
         file_name=$(echo "${file_name}" | sed "s/apache-libcloud/apache_libcloud/g")
     fi
 
     apache_url="${APACHE_MIRROR_URL}/${file_name}"
     pypi_url="${PYPI_MIRROR_URL}/${file_name}"
 
-    if [ "${extension}" = "-py2.py3-none-any.whl" ]; then
+    if [ "${extension}" = "-py3-none-any.whl" ]; then
         pypi_url="${PYPI_MIRROR_URL_WHEEL}/${file_name}"
     else
         pypi_url="${PYPI_MIRROR_URL_SOURCE}/${file_name}"

docs/testing.rst

@@ -77,7 +77,7 @@ To generate the test coverage run the following command:
 
 .. sourcecode:: bash
 
-    PYTHONPATH=. python setup.py coverage
+    tox -e coverage_html_report
 
 When it completes you should see a new ``coverage_html_report`` directory which
 contains the test coverage.

pyproject.toml

@@ -74,7 +74,7 @@ Changelog = "https://github.com/apache/libcloud/blob/trunk/CHANGES.rst"
 
 [project.optional-dependencies]
 build = [
-    "build==1.2.1"
+    "build==1.2.2"
 ]
 publish = [
     "twine==5.1.1"
@@ -89,9 +89,20 @@ where = ["./"]
 include = ["libcloud", "libcloud.test*", "libcloud.*" ]
 
 [tool.setuptools.package-data]
-"*" = ["*.json", "*.xml", "*.pub", "*.key", "*.pem", "*.crt", "*.csv", "*.txt", "*.html"]
-"libcloud.test.compute.fixtures.misc" = ["*"]
-"libcloud.test.dns.fixtures.worldwidedns" = ["*"]
+"libcloud.data" = [
+    "pricing.json"
+]
+"libcloud.test" = [
+    "**/*.json",
+    "**/*.xml",
+    "**/*.pub",
+    "**/*.key",
+    "**/*.pem",
+    "**/*.crt",
+    "**/*.csv",
+    "**/*.txt",
+    "**/*.html",
+]
 
 [tool.setuptools]
 include-package-data = true
@@ -102,7 +113,6 @@ version = {attr = "libcloud.__version__"}
 readme = {file = ["README.rst"], content-type = "text/x-rst"}
 
 [tool.distutils.bdist_wheel]
-universal = true
 
 
 [tool.black]