MRG: Merge pull request #529 from octue/feature/add-topic-and-subscription-creation-command

Resolve and fully support service namespacing and revision tagging

Author: cortadocodes

.github/workflows/python-ci.yml

@@ -18,18 +18,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          # Set fetch-depth to 0 to fetch all tags (necessary for git-mkver to determine the correct semantic version).
           fetch-depth: 0
-      - uses: actions/setup-python@v2
-      - name: Install git-mkver
-        run: |
-          curl -L https://github.com/idc101/git-mkver/releases/download/v1.2.1/git-mkver-linux-amd64-1.2.1.tar.gz \
-          | tar xvz \
-          && sudo mv git-mkver /usr/local/bin
-      - name: Install semantic version checker
-        run: pip install git+https://github.com/octue/conventional-commits
-      - name: Check version
-        run: check-semantic-version pyproject.toml
+      - uses: octue/[email protected]
+        with:
+          version_source_type: pyproject.toml
 
   run-tests:
     if: "!contains(github.event.head_commit.message, 'skipci')"
@@ -38,7 +30,6 @@ jobs:
       USING_COVERAGE: "3.9"
     strategy:
       matrix:
-        python: [3.9]
         os: [ubuntu-latest, windows-latest, macos-latest]
     permissions:
       id-token: write
@@ -49,10 +40,10 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: ${{ matrix.python }}
+          python-version: 3.9
 
       - name: Install Poetry
-        uses: snok/install-poetry@v1
+        uses: snok/install-poetry@v1.3.2
 
       - name: Check pyproject.toml file
         run: poetry check

.github/workflows/release.yml

@@ -18,7 +18,6 @@ jobs:
       USING_COVERAGE: '3.9'
     strategy:
       matrix:
-        python: [ 3.9 ]
         os: [ ubuntu-latest, windows-latest, macos-latest ]
     permissions:
       id-token: write
@@ -29,14 +28,18 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: ${{ matrix.python }}
+          python-version: 3.9
 
       - name: Install Poetry
-        uses: snok/install-poetry@v1
+        uses: snok/install-poetry@v1.3.2
 
       - name: Check pyproject.toml file
         run: poetry check
 
+      - name: Get package version
+        id: get-package-version
+        run: echo "PACKAGE_VERSION=$(poetry version -s)" >> $GITHUB_OUTPUT
+
       - name: Install tox
         run: pip install tox
 
@@ -61,24 +64,21 @@ jobs:
           files: coverage.xml
           fail_ci_if_error: false
 
+    outputs:
+      package_version: ${{ steps.get-package-version.outputs.PACKAGE_VERSION }}
+
   release:
     runs-on: ubuntu-latest
     needs: run-tests
     steps:
     - uses: actions/checkout@v3
 
-    - name: Install Poetry
-      uses: snok/install-poetry@v1
-
-    - name: Get package version
-      run: echo "PACKAGE_VERSION=$(poetry version -s)" >> $GITHUB_ENV
-
     - name: Create Release
       uses: actions/create-release@v1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, no need to create your own.
       with:
-        tag_name: ${{ env.PACKAGE_VERSION }}
+        tag_name: ${{ needs.run-tests.outputs.package_version }}
         release_name: ${{ github.event.pull_request.title }}
         body: ${{ github.event.pull_request.body }}
         draft: false
@@ -100,18 +100,12 @@ jobs:
 
   docker:
     runs-on: ubuntu-latest
-    needs: release
+    needs: [run-tests, release]
     timeout-minutes: 300
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Install Poetry
-        uses: snok/install-poetry@v1
-
-      - name: Get package version
-        run: echo "PACKAGE_VERSION=$(poetry version -s)" >> $GITHUB_ENV
-
       - name: Log in to DockerHub
         uses: docker/login-action@v1
         with:
@@ -123,4 +117,4 @@ jobs:
         with:
           context: .
           push: true
-          tags: octue/octue-sdk-python:${{ env.PACKAGE_VERSION }}-slim,octue/octue-sdk-python:latest
+          tags: octue/octue-sdk-python:${{ needs.run-tests.outputs.package_version }}-slim,octue/octue-sdk-python:latest

.github/workflows/update-pull-request.yml

@@ -18,18 +18,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
+    - uses: octue/[email protected]
       with:
-        # Set fetch-depth to 0 to fetch all commit history (necessary for compiling pull request description).
-        fetch-depth: 0
-    - name: Install release note compiler
-      run: pip install git+https://github.com/octue/conventional-commits
-    - name: Compile new pull request description
-      run: |
-        echo 'PULL_REQUEST_NOTES<<EOF' >> $GITHUB_ENV
-        echo "$(compile-release-notes PULL_REQUEST_START --pull-request-url=${{ github.event.pull_request.url }} --api-token=${{ secrets.GITHUB_TOKEN }})" >> $GITHUB_ENV
-        echo EOF >> $GITHUB_ENV
+        pull_request_url: ${{ github.event.pull_request.url }}
+        api_token: ${{ secrets.GITHUB_TOKEN }}
     - name: Update pull request body
       uses: riskledger/update-pr-description@v2
       with:
-        body: ${{ env.PULL_REQUEST_NOTES }}
+        body: ${{ env.PULL_REQUEST_DESCRIPTION }}
         token: ${{ secrets.GITHUB_TOKEN }}

.pre-commit-config.yaml

@@ -72,7 +72,7 @@ repos:
           - '^dependencies/([a-z][a-z0-9]*)(-[a-z0-9]+)*$'
 
   - repo: https://github.com/octue/conventional-commits
-    rev: 0.6.4
+    rev: 0.8.1
     hooks:
       - id: check-commit-message-is-conventional
         stages: [commit-msg]

Dockerfile

@@ -12,7 +12,7 @@ RUN poetry config virtualenvs.create false
 # install post-create to install the root package, which will change more rapidly than dependencies.
 COPY octue/cloud/deployment/google/dataflow/setup.py setup.py
 COPY pyproject.toml poetry.lock ./
-RUN poetry install --no-ansi --no-interaction --no-root
+RUN poetry install --no-ansi --no-interaction -E dataflow --only=main --no-root -v
 
 COPY . .
-RUN poetry install --no-ansi --no-interaction -E dataflow --no-dev -v
+RUN poetry install --no-ansi --no-interaction -E dataflow --only=main -v

docs/source/creating_services.rst

@@ -114,6 +114,7 @@ octue.yaml
         .. code-block:: yaml
 
             services:
+              - namespace: my-organisation
               - name: my-app
 
         It may also need the following key-value pairs:
@@ -181,6 +182,75 @@ Dockerfile (optional)
 
         As always, if you need help with this, feel free to drop us a message or raise an issue!
 
+Naming services
+===============
+
+.. admonition:: Definitions
+
+    Service revision
+        A specific instance of an Octue service that can be individually addressed. The revision could correspond to a
+        version of the service, a dynamic development branch for it, or a deliberate duplication or variation of it.
+
+    Service revision unique identifier (SRUID)
+        The combination of a service revisions's namespace, name, and revision tag that uniquely identifies it. For
+        example, ``octue/my-service:1.3.0`` where the namespace is ``octue``, the name is ``my-service``, and the
+        revision tag is ``1.3.0``.
+
+    Service namespace
+        The group to which the service belongs e.g. your name or your organisation's name. If in doubt, use the GitHub
+        handle of the user or organisation publishing the services.
+
+        Namespaces must be lower kebab case (i.e. they may contain the letters [a-z], numbers [0-9], and hyphens [-]).
+        They may not begin or end with hyphens.
+
+    Service name
+        A name to uniquely identify the service within its namespace. This usually corresponds to the name of the GitHub
+        repository for the service. Names must be lower kebab case (i.e. they may contain the letters [a-z],
+        numbers [0-9] and hyphens [-]). They may not begin or end with hyphens.
+
+    Service revision tag
+        A tag that uniquely identifies a particular revision of a service. The revision tag could correspond to a commit
+        hash like ``a3eb45``, a release number like ``0.12.4``, a branch name (e.g. ``development``), a particular
+        environment the service is deployed in (e.g. ``production``), or a combination like ``0.12.4-production``. Tags
+        may contain lowercase and uppercase letters, numbers, underscores, periods, and hyphens, but can't start with a
+        period or a dash. They can contain a maximum of 128 characters. These requirements are the same as the `Docker
+        tag format <https://docs.docker.com/engine/reference/commandline/tag/>`_.
+
+    Service ID
+        The SRUID is a special case of the service ID. A service ID can be an SRUID or just the service namespace and
+        name. It can be used to ask a question to a service without specifying a specific revision of it. This enables
+        asking questions to, for example, the service ``octue/my-service`` and automatically having them routed to its
+        latest revision. Note that this will be a future feature; currently, you will still be required to provide a
+        revision tag (i.e. a full SRUID).
+
+Where to specify the namespace, name, and revision tag
+------------------------------------------------------
+
+**Namespace**
+
+- Required: yes
+- Set in:
+
+  - ``octue.yaml``
+  - ``OCTUE_SERVICE_NAMESPACE`` environment variable (takes priority)
+
+**Name**
+
+- Required: yes
+- Set in:
+
+  - ``octue.yaml``
+  - ``OCTUE_SERVICE_NAME`` environment variable (takes priority)
+
+**Revision tag**
+
+- Required: no
+- Default: a random "coolname" (e.g. ``hungry-hippo``)
+- Set in:
+
+  - ``OCTUE_SERVICE_REVISION_TAG`` environment variable
+  - If using ``octue start`` command, the ``--revision-tag`` option (takes priority)
+
 
 Template apps
 =============