Merge pull request #71 from octue/python-range

Widen supported django and python ranges, update developer tools

Author: thclark

.devcontainer/devcontainer.json

@@ -12,39 +12,28 @@
     "vscode": {
       "settings": {
         "[python]": {
-          "editor.defaultFormatter": "ms-python.black-formatter",
           "editor.formatOnSave": true,
           "editor.codeActionsOnSave": {
             "source.organizeImports": "always",
             "source.fixAll": "always"
-          }
+          },
+          "editor.defaultFormatter": "charliermarsh.ruff"
         },
-        "black-formatter.args": ["--line-length", "120"],
-        "black-formatter.importStrategy": "fromEnvironment",
-        "isort.args": ["--profile", "black"],
-        "isort.importStrategy": "fromEnvironment",
         "austin.mode": "Wall time",
         "editor.defaultFormatter": "esbenp.prettier-vscode",
         "editor.formatOnSave": true,
-        "jupyter.widgetScriptSources": ["jsdelivr.com", "unpkg.com"],
-        // Line length to match black settings
-        // Disable specific messages:
-        //  - To find the details do: /usr/local/py-utils/bin/pylint --list-msgs
-        //  - Disable missing-module-docstring (C0114) because we don't document modules routinely, just their members
-        //  - Disable invalid-name (C0103) because pylint thinks that eg 'x', 'df', 'np' are invalid due to their lengths
-        "pylint.args": [
-          "--max-line-length=120",
-          "--disable=missing-module-docstring,invalid-name",
-          "--load-plugins=pylint_django"
-        ],
+        "python.locator": "native",
         "python.testing.pytestEnabled": true,
         "python.testing.unittestEnabled": false,
         "terminal.integrated.defaultProfile.linux": "zsh",
-        // Handle this: https://github.com/microsoft/vscode-python/issues/693
-        "python.testing.pytestArgs": ["--no-cov"]
+        "ruff.format.args": ["--line-length", "120"],
+        "ruff.importStrategy": "fromEnvironment"
       },
-      // Add the IDs of extensions you want installed when the container is created.
+
+      // IN ALPHABETIC ORDER: Add the IDs of extensions you want installed when the container is created.
       "extensions": [
+        "4ops.terraform",
+        "charliermarsh.ruff",
         "erikphansen.vscode-toggle-column-selection",
         "esbenp.prettier-vscode",
         "GitHub.copilot",
@@ -53,20 +42,11 @@
         "irongeek.vscode-env",
         "ms-python.python",
         "ms-python.vscode-pylance",
-        "ms-python.pylint",
-        "ms-python.black-formatter",
-        "ms-python.flake8",
-        "ms-python.isort",
-        "ms-toolsai.jupyter",
-        "ms-toolsai.jupyter-renderers",
-        "ms-toolsai.jupyter-keymap",
         "ms-vsliveshare.vsliveshare",
+        "ninoseki.vscode-mogami",
         "p403n1x87.austin-vscode",
         "ritwickdey.liveserver",
-        "shamanu4.django-intellisense",
-        "thebarkman.vscode-djaneiro",
-        "trond-snekvik.simple-rst",
-        "4ops.terraform"
+        "tamasfe.even-better-toml"
       ]
     }
   },

.devcontainer/postattach.sh

@@ -1,6 +1,8 @@
 #!/bin/zsh
 
-# Install dependencies
+
+# Install dependencies to a cache that persists across devcontainer builds
+poetry config cache-dir /workspace/.poetry_cache
 poetry install
 
 # Auto set up remote when pushing new branches
@@ -33,4 +35,5 @@ echo 'alias djm="python manage.py migrate"' >> ~/.zshrc
 echo 'alias djr="python manage.py runserver"' >> ~/.zshrc
 echo 'alias djreset="python manage.py reset_db -c"' >> ~/.zshrc
 echo 'alias djs="python manage.py shell_plus"' >> ~/.zshrc
+echo 'alias djt="pytest backend/test"' >> ~/.zshrc
 echo 'alias dju="python manage.py show_urls"' >> ~/.zshrc

.github/workflows/cd.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   release:
-    if: "${{ github.event.pull_request.merged == true }}"
+    if: '${{ github.event.pull_request.merged == true }}'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository

.github/workflows/ci.yml

@@ -24,20 +24,108 @@ on:
 
 jobs:
   check-semantic-version:
-    if: github.event.pull_request.draft == false
+    if: github.ref != 'refs/heads/main'
     uses: octue/workflows/.github/workflows/check-semantic-version.yml@main
     with:
       path: pyproject.toml
       breaking_change_indicated_by: minor
 
-  run-tests:
+  check-ahead-of-main:
+    if: github.ref != 'refs/heads/main'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check branch is ahead of main
+        run: |
+          if ! git merge-base --is-ancestor origin/main ${{ github.event.pull_request.head.sha }};
+          then echo "::error::This branch is not up-to-date with the latest main branch commit.";
+          exit 1; fi
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install and configure poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Setup virtual environment cache
+        id: cached-poetry-dependencies
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-3.11-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies (if not cached)
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --only dev --no-interaction --no-root
+
+      - name: Install root project
+        run: poetry install --no-interaction
+
+      - name: Run precommit
+        run: SKIP=build-docs,check-branch-name poetry run pre-commit run --all-files
+
+  publish-test:
+    runs-on: ubuntu-latest
+    needs:
+      - lint
+      - check-ahead-of-main
+      - check-semantic-version
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install poetry
+        uses: snok/[email protected]
+
+      - name: Build a binary wheel and a source tarball
+        run: poetry build
+
+      - name: Test package is publishable with PyPI test server
+        uses: pypa/[email protected]
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          skip-existing: true
+          verbose: true
+
+  test:
     if: github.event.pull_request.draft == false
+    needs:
+      - lint
+      - check-ahead-of-main
+      - check-semantic-version
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
-        python: ['3.9', '3.10', '3.11']
-        os: [ubuntu-latest] # [ubuntu-latest, windows-latest, macos-latest] for full coverage but this gets expensive quickly
-    runs-on: ${{ matrix.os }}
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+        django-version: ['>=4.2,<4.3', '>=5.0,<5.1']
+        database-engine: ['sqlite', 'postgres']
+
+        exclude:
+          # Exclude Django 5 with Python 3.9
+          - python-version: '3.9'
+            django-version: '>=5.0,<5.1'
+
+    permissions:
+      id-token: write
+      contents: read
 
     services:
       postgres:
@@ -55,63 +143,85 @@ jobs:
           --health-timeout 5s
           --health-retries 5
 
+    runs-on: 'ubuntu-latest'
+
+    env:
+      DJANGO_SETTINGS_MODULE: tests.server.settings
+      DATABASE_ENGINE: ${{ matrix.database-engine }}
+
     steps:
-      - name: Checkout repository
+      - name: Check out repository
         uses: actions/checkout@v4
 
-      - name: Setup python ${{ matrix.python }}
+      - name: Prepare Integration Test Credentials
+        # Workload Identity Federation works great for using the GCloud API, but the credentials required by
+        # the storages client in integration tests don't like it; they need a private key to sign blobs.
+        # So until the credentials APIs are sensibly applied across the google client libraries, we inject
+        # a private service account key (against recommended practice, but this is the only thing that works).
+        id: application-credentials
+        run: |
+          echo '${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}' > $(pwd)/gha-creds-github-actions.json
+          echo "GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gha-creds-github-actions.json" >> $GITHUB_ENV
+
+      # - name: Authenticate with GCP Workload Identity
+      #   id: auth
+      #   uses: google-github-actions/auth@v2
+      #   with:
+      #     # NOTE: If setting create_credentials_file=true when building docker images,
+      #     # a .dockerignore file must be present and include `gha-creds-*.json` to
+      #     # avoid baking these credentials into the container
+      #     create_credentials_file: true
+      #     workload_identity_provider: projects/134056372703/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider
+      #     service_account: [email protected]
+
+      # - name: Setup gcloud
+      #   uses: 'google-github-actions/setup-gcloud@v2'
+
+      - name: Set up python ${{ matrix.python-version }}
+        id: setup-python
         uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python }}
+          python-version: ${{ matrix.python-version }}
 
-        # See the repo of this action for way more advanced caching strategies than used here
-      - name: Install poetry
-        uses: snok/[email protected]
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
 
-        # For more advanced configuration see https://github.com/ymyzk/tox-gh-actions
-      - name: Install tox and plugins
+      - name: Setup virtual environment cache
+        id: cached-poetry-dependencies
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies (if not cached)
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+
+      - name: Install root project
+        run: poetry install --no-interaction
+
+      - name: Install django ${{ matrix.django-version }}
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install tox==3.24.5 tox-gh-actions==2.9.1 tox-poetry==0.4.1
+          source .venv/bin/activate
+          pip install "django${{ matrix.django-version }}"
 
       - name: Setup tmate session [DEBUG]
         if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true'}}
         uses: mxschmitt/action-tmate@v3
 
-      # For more advanced configuration see https://github.com/ymyzk/tox-gh-actions
-      - name: Run tests using tox
-        run: tox
+      - name: Run tests
+        run: poetry run pytest --cov=django_gcp --cov-report=xml
 
       - name: Upload coverage to Codecov
         # This seems redundant inside the test matrix but actually isn't, since different
         # dependency combinations may cause different lines of code to be hit (e.g. backports)
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           files: coverage.xml
           fail_ci_if_error: false
-          # Token is not required for public repos, but see:
+          # Token is not strictly required for public repos, but see:
           # https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954
           token: ${{ secrets.CODECOV_TOKEN }}
-
-  test-publish:
-    runs-on: ubuntu-latest
-    needs: run-tests
-    permissions:
-      id-token: write
-      contents: read
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Install poetry
-        uses: snok/[email protected]
-
-      - name: Build a binary wheel and a source tarball
-        run: poetry build
-
-      - name: Test package is publishable with PyPI test server
-        uses: pypa/[email protected]
-        with:
-          repository-url: https://test.pypi.org/legacy/
-          skip-existing: true
-          verbose: true