Merge branch 'master' into add_files_endpoint

Author: ml-evs

.docker/docker_config.json

@@ -1,5 +1,6 @@
 {
     "debug": true,
+    "page_limit": 5,
     "default_db": "test_server",
     "base_url": "http://gh_actions_host:3213",
     "implementation": {

.github/dependabot.yml

@@ -8,11 +8,26 @@ updates:
     time: "05:43"
   # Needs to be larger than the number of total requirements (currently 31)
   open-pull-requests-limit: 50
-  target-branch: dependabot_updates
+  target-branch: master
   labels:
   - dependency_updates
   # Turn off automatic rebases so that auto-merge can work without needed N**2 CI runs
   rebase-strategy: "disabled"
+  ignore:
+  - dependency-name: "elasticsearch*"
+    versions: [ ">=8" ]
+  - dependency-name: "jarvis-tools"
+    versions: [ "==2024.4.20", "==2024.4.30" ]
+  groups:
+    python-dependencies:
+      applies-to: version-updates
+      dependency-type: production
+    python-dependencies-dev:
+      applies-to: version-updates
+      dependency-type: development
+    python-dependencies-security:
+      applies-to: security-updates
+      dependency-type: production
 - package-ecosystem: gitsubmodule
   directory: "/"
   schedule:
@@ -26,6 +41,10 @@ updates:
   schedule:
     interval: daily
     time: "05:33"
-  target-branch: dependabot_updates
+  target-branch: master
   labels:
   - CI
+  groups:
+    github-actions:
+      applies-to: version-updates
+      dependency-type: production

.github/fly.toml

@@ -1,11 +1,17 @@
-# fly.toml file generated for broken-field-1976 on 2022-11-21T21:37:55Z
-
 app = "optimade"
 kill_signal = "SIGINT"
 kill_timeout = 5
-processes = []
+swap_size_mb = 1024
+processes = {}
+
+[build]
+  builder = "paketobuildpacks/builder:base"
 
 [env]
+PORT = "5000"
+BPE_OPTIMADE_CONFIG_FILE = "./tests/test_config.json"
+BPL_OPTIMADE_CONFIG_FILE = "./tests/test_config.json"
+OPTIMADE_CONFIG_FILE = "./tests/test_config.json"
 
 [experimental]
   allowed_public_ports = []

.github/utils/single_dependency_pr_body.txt

@@ -20,18 +20,20 @@ jobs:
     if: github.repository_owner == 'Materials-Consortia'
     runs-on: ubuntu-latest
 
-    env:
-      IMAGE_NAME: optimade
+    environment:
+      name: container-image
+      url: https://github.com/${{ github.repository }}/pkgs/container/${{ vars.IMAGE_NAME }}
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
         submodules: true
         ref: ${{ inputs.checkout_ref }}
 
     - name: Retrieve version
+      id: get_version
       run: |
         # Get current OPTIMADE version from optimade/__init__.py
         regex="^__version__ = (\"|')(.*)(\"|')$"
@@ -41,61 +43,63 @@ jobs:
           fi
         done < optimade/__init__.py
 
-        echo "VERSION=${VERSION}" >> $GITHUB_ENV
+        echo "version=${VERSION}" >> $GITHUB_OUTPUT
         echo VERSION=${VERSION}
 
     - name: Set source ref
+      id: set_source_ref
       run: |
         if [[ "${{ inputs.release }}" == "true" ]]; then
           # Use tag for metadata instead of commit SHA
-          SOURCE_REF=v${{ env.VERSION }}
+          SOURCE_REF=v${{ steps.get_version.outputs.version }}
         else
           SOURCE_REF=$(git rev-parse HEAD)
         fi
 
-        echo "SOURCE_REF=${SOURCE_REF}" >> $GITHUB_ENV
+        echo "source_ref=${SOURCE_REF}" >> $GITHUB_OUTPUT
         echo SOURCE_REF=${SOURCE_REF}
 
     - name: Build image
       run: |
         docker build \
           --file Dockerfile \
-          --tag "${IMAGE_NAME}" \
+          --tag "${{ vars.IMAGE_NAME }}" \
           --label "gh_actions_runnumber=${GITHUB_RUN_ID}" \
           --label "org.openctonainers.image.title=OPTIMADE" \
           --label "org.opencontainers.image.description=A server implementation for serving an OPTIMADE API." \
-          --label "org.opencontainers.image.source=https://github.com/${{ github.repository }}/tree/${{ env.SOURCE_REF }}" \
-          --label "org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.SOURCE_REF }}/README.md" \
+          --label "org.opencontainers.image.source=https://github.com/${{ github.repository }}/tree/${{ steps.set_source_ref.outputs.source_ref }}" \
+          --label "org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ steps.set_source_ref.outputs.source_ref }}/README.md" \
           --label "org.opencontainers.image.licenses=MIT" \
-          --label "org.opencontainers.image.url=https://github.com/${{ github.repository }}/pkgs/container/${IMAGE_NAME}" \
+          --label "org.opencontainers.image.url=https://github.com/${{ github.repository }}/pkgs/container/${{ vars.IMAGE_NAME }}" \
           --label "org.opencontainers.image.vendor=${{ github.repository_owner }}" \
-          --label "org.opencontainers.image.version=${{ env.VERSION }}" \
+          --label "org.opencontainers.image.version=${{ steps.get_version.outputs.version }}" \
           --label "org.opencontainers.image.base.name=docker.io/library/python:3.10-slim" \
           .
 
     - name: Create full image ID and tag with 'develop'
+      id: create_tags
       run: |
-        IMAGE_ID=ghcr.io/${{ github.repository_owner }}/${IMAGE_NAME}
+        IMAGE_ID=ghcr.io/${{ github.repository_owner }}/${{ vars.IMAGE_NAME }}
 
         # Change all uppercase to lowercase
         IMAGE_ID=$(echo ${IMAGE_ID} | tr '[A-Z]' '[a-z]')
 
-        echo "IMAGE_ID=${IMAGE_ID}" >> $GITHUB_ENV
+        echo "image_id=${IMAGE_ID}" >> $GITHUB_OUTPUT
         echo IMAGE_ID=${IMAGE_ID}
 
         # Tag with "develop"
-        docker tag ${IMAGE_NAME} ${IMAGE_ID}:develop
+        docker tag ${{ vars.IMAGE_NAME }} ${IMAGE_ID}:develop
 
     - name: Tag with version and `latest` - if it's a release
       if: inputs.release
       run: |
-        docker tag ${IMAGE_NAME} ${{ env.IMAGE_ID }}:${{ env.VERSION }}
-        docker tag ${IMAGE_NAME} ${{ env.IMAGE_ID }}:latest
+        docker tag ${{ vars.IMAGE_NAME }} ${{ steps.create_tags.outputs.image_id }}:${{ steps.get_version.outputs.version }}
+        docker tag ${{ vars.IMAGE_NAME }} ${{ steps.create_tags.outputs.image_id }}:latest
 
     - name: Log in to registry
       run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
 
     - name: Push image
       run: |
-        docker image inspect ${IMAGE_NAME}
-        docker push --all-tags ${{ env.IMAGE_ID }}
+        docker image inspect ${{ vars.IMAGE_NAME }}
+        docker push --all-tags ${{ steps.create_tags.outputs.image_id }}

.github/workflows/cd_container_image.yml

@@ -6,26 +6,46 @@ on:
     - published
 
 env:
-  PUBLISH_UPDATE_BRANCH: master
   GIT_USER_NAME: OPTIMADE Developers
   GIT_USER_EMAIL: "[email protected]"
+  DEFAULT_RELEASE_BRANCH: "master"
 
 jobs:
-
-  publish:
-    name: Publish OPTIMADE Python tools
+  release:
+    name: Release OPTIMADE Python tools
     runs-on: ubuntu-latest
     if: github.repository == 'Materials-Consortia/optimade-python-tools' && startsWith(github.ref, 'refs/tags/v')
+    outputs:
+      publish_branch: ${{ steps.save_branch.outputs.publish_branch }}
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
+    - name: Get triggering branch
+      uses: octokit/[email protected]
+      id: get_release_branch
+      with:
+        route: GET /repos/{owner}/{repo}/releases/tags/${{ github.ref_name }}
+        # Note, the following inputs might give warnings that they are not valid.
+        # This is due to the flexible nature of the `octokit/request-action`.
+        # Fore more information see: https://github.com/octokit/request-action/tree/v2.x?tab=readme-ov-file#warnings
+        owner: Materials-Consortia
+        repo: optimade-python-tools
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Save branch name
+      id: save_branch
+      run: |
+        echo 'publish_branch=${{ fromJson(steps.get_release_branch.outputs.data).target_commitish }}' >> $GITHUB_OUTPUT
+
+    - name: Checkout publish branch
+      uses: actions/checkout@v4
       with:
         submodules: true
         fetch-depth: 0
+        ref: ${{ steps.save_branch.outputs.publish_branch }}
 
     - name: Set up Python 3.10
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.10'
 
@@ -36,70 +56,98 @@ jobs:
         pip install -r requirements.txt -r requirements-dev.txt -r requirements-client.txt -r requirements-http-client.txt -r requirements-docs.txt
         pip install -e .[all]
 
+    - name: Setup changelog configuration
+      id: changelog_config
+      run: |
+        echo "project=$(echo $GITHUB_REPOSITORY | cut -d/ -f2- )" >> $GITHUB_OUTPUT
+        echo "exclude_labels=duplicate,question,invalid,wontfix,dependency_updates,skip_changelog" >> $GITHUB_OUTPUT
+
+        # For the release-specific changelog
+        echo "output_file=release_changelog.md" >> $GITHUB_OUTPUT
+
     - name: Update changelog
-      uses: CharMixer/auto-changelog-action@v1
+      uses: docker://githubchangeloggenerator/github-changelog-generator:1.16.2
       with:
-        token: ${{ secrets.RELEASE_PAT_BOT }}
-        release_branch: ${{ env.PUBLISH_UPDATE_BRANCH }}
-        exclude_labels: "duplicate,question,invalid,wontfix,dependency_updates,skip_changelog"
+        args: --user "${{ github.repository_owner }}" --project "${{ steps.changelog_config.outputs.project }}" --token "${{ secrets.RELEASE_PAT_BOT }}" --release-branch "${{ steps.save_branch.outputs.publish_branch }}" --exclude-labels "${{ steps.changelog_config.outputs.exclude_labels }}"
 
     - name: Update API Reference docs and version - Commit changes and update tag
       run: .github/utils/update_docs.sh
 
-    - name: Update '${{ env.PUBLISH_UPDATE_BRANCH }}'
+    - name: Update triggering branch
       uses: CasperWA/push-protected@v2
       with:
         token: ${{ secrets.RELEASE_PAT_BOT }}
-        branch: ${{ env.PUBLISH_UPDATE_BRANCH }}
+        branch: ${{ steps.save_branch.outputs.publish_branch }}
         unprotect_reviews: true
         sleep: 15
         force: true
         tags: true
 
-    - name: Get tagged versions
-      run: echo "PREVIOUS_VERSION=$(git tag -l --sort -version:refname | sed -n 2p)" >> $GITHUB_ENV
+    - name: Get previous version
+      id: get_previous_version
+      run: echo "previous_version=$(git tag -l --sort -version:refname | sed -n 2p)" >> $GITHUB_OUTPUT
 
     - name: Create release-specific changelog
-      uses: CharMixer/auto-changelog-action@v1
+      uses: docker://githubchangeloggenerator/github-changelog-generator:1.16.2
       with:
-        token: ${{ secrets.RELEASE_PAT_BOT }}
-        release_branch: ${{ env.PUBLISH_UPDATE_BRANCH }}
-        since_tag: "${{ env.PREVIOUS_VERSION }}"
-        output: "release_changelog.md"
-        exclude_labels: "duplicate,question,invalid,wontfix,dependency_updates,skip_changelog"
+        args: --user "${{ github.repository_owner }}" --project "${{ steps.changelog_config.outputs.project }}" --token "${{ secrets.RELEASE_PAT_BOT }}" --release-branch "${{ steps.save_branch.outputs.publish_branch }}" --since-tag "${{ steps.get_previous_version.outputs.previous_version }}" --output "${{ steps.changelog_config.outputs.output_file }}" --usernames-as-github-logins --exclude-labels "${{ steps.changelog_config.outputs.exclude_labels }}"
 
     - name: Append changelog to release body
       run: |
         gh api /repos/${{ github.repository }}/releases/${{ github.event.release.id }} --jq '.body' > release_body.md
-        cat release_changelog.md >> release_body.md
+        cat ${{ steps.changelog_config.outputs.output_file }} >> release_body.md
         gh api /repos/${{ github.repository }}/releases/${{ github.event.release.id }} -X PATCH -F body='@release_body.md'
       env:
         GITHUB_TOKEN: ${{ secrets.RELEASE_PAT_BOT }}
 
+  publish-to-pypi:
+    name: Publish to PyPI
+    needs: release
+    runs-on: ubuntu-latest
+
+    environment:
+      name: pypi
+      url: https://pypi.org/p/optimade
+
+    permissions:
+      id-token: write  # Important for Trusted Publishing on PyPI
+
+    steps:
+    - name: Install prerequisites
+      run: python -m pip install build
+
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        submodules: true
+        fetch-depth: 0
+        ref: ${{ needs.release.outputs.publish_branch }}
+
     - name: Build source distribution
       run: python -m build
 
     - name: Publish package to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
-        user: __token__
-        password: ${{ secrets.PYPI_PASSWORD }}
+        print-hash: true
 
   docs:
     name: Deploy documentation
-    needs: publish
+    needs:
+      - release
+      - publish-to-pypi
     runs-on: ubuntu-latest
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: true
         fetch-depth: 0
-        ref: ${{ env.PUBLISH_UPDATE_BRANCH }}
+        ref: ${{ needs.release.outputs.publish_branch }}
 
     - name: Set up Python 3.10
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.10'
 
@@ -115,18 +163,25 @@ jobs:
         git config --global user.name "${{ env.GIT_USER_NAME }}"
         git config --global user.email "${{ env.GIT_USER_EMAIL }}"
 
-    - name: Deploy documentation
+    - name: Deploy versioned documentation
+      run: |
+        mike deploy --push --remote origin --branch gh-pages --config-file mkdocs.yml ${GITHUB_REF#refs/tags/v}
+
+    - name: Deploy stable/latest documentation
+      if: ${{ needs.release.outputs.publish_branch }} == ${{ env.DEFAULT_RELEASE_BRANCH }}
       run: |
         mike deploy --push --remote origin --branch gh-pages --update-aliases --config-file mkdocs.yml ${GITHUB_REF#refs/tags/v} stable
-        mike deploy --push --remote origin --branch gh-pages --update-aliases --config-file mkdocs.yml latest master
+        mike deploy --push --remote origin --branch gh-pages --update-aliases --config-file mkdocs.yml latest ${{ env.DEFAULT_RELEASE_BRANCH }}
 
   publish_container_image:
     name: Publish container image
-    needs: publish
+    needs:
+      - release
+      - publish-to-pypi
     uses: ./.github/workflows/cd_container_image.yml
     with:
       release: true
-      checkout_ref: master
+      checkout_ref: ${{ needs.release.outputs.publish_branch }}
     secrets: inherit
     permissions:
       packages: write