Merge pull request #1 from JunAishima/prefect3-pixi-upgrade

Prefect3 pixi upgrade

Author: danielballan

.github/workflows/linting.yml

@@ -9,6 +9,6 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
-    - uses: pre-commit/action@v2.0.3
+      - uses: actions/checkout@v4.3.0
+      - uses: actions/setup-python@v5.6.0
+      - uses: pre-commit/action@v3.0.1

.github/workflows/publish-ghcr.yml

@@ -0,0 +1,59 @@
+#
+name: Create and publish a Docker image
+
+# Configures this workflow to run every time a change is pushed to the branch called `release`.
+on:
+  push:
+    branches: ["main"]
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+      #
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
+      # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see [Usage](https://github.com/docker/build-push-action#usage) in the README of the `docker/build-push-action` repository.
+      # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see [Using artifact attestations to establish provenance for builds](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds).
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v3
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true

.pre-commit-config.yaml

@@ -1,24 +1,92 @@
-# See https://pre-commit.com for more information
-# See https://pre-commit.com/hooks.html for more hooks
+ci:
+  autoupdate_commit_msg: "chore: update pre-commit hooks"
+  autofix_commit_msg: "style: pre-commit fixes"
+
+exclude: ^.cruft.json|.copier-answers.yml$
+
 repos:
+  - repo: https://github.com/adamchainz/blacken-docs
+    rev: "1.19.1"
+    hooks:
+      - id: blacken-docs
+        additional_dependencies: [black==24.*]
+
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: "v5.0.0"
     hooks:
-      - id: trailing-whitespace
-      - id: end-of-file-fixer
-      - id: check-ast
+      - id: check-added-large-files
       - id: check-case-conflict
       - id: check-merge-conflict
       - id: check-symlinks
       - id: check-yaml
       - id: debug-statements
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: name-tests-test
+        args: ["--pytest-test-first"]
+      - id: requirements-txt-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/pre-commit/pygrep-hooks
+    rev: "v1.10.0"
+    hooks:
+      - id: rst-backticks
+      - id: rst-directive-colons
+      - id: rst-inline-touching-normal
+
+  - repo: https://github.com/rbubley/mirrors-prettier
+    rev: "v3.5.3"
+    hooks:
+      - id: prettier
+        types_or: [yaml, markdown, html, css, scss, javascript, json]
+        args: [--prose-wrap=always]
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: "v0.11.11"
+    hooks:
+      - id: ruff
+        args: ["--fix", "--show-fixes"]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: "v1.15.0"
+    hooks:
+      - id: mypy
+        files: src|tests
+        args: []
+        additional_dependencies:
+          - pytest
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: "v2.4.1"
+    hooks:
+      - id: codespell
+        additional_dependencies:
+          - tomli; python_version<'3.11'
+        exclude: "pixi.*"
+
+  - repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: "v0.10.0.1"
+    hooks:
+      - id: shellcheck
+
+  - repo: local
+    hooks:
+      - id: disallow-caps
+        name: Disallow improper capitalization
+        language: pygrep
+        entry: PyBind|Numpy|Cmake|CCache|Github|PyTest
+        exclude: .pre-commit-config.yaml
 
-  - repo: https://github.com/timothycrosley/isort
-    rev: 5.9.3
+  - repo: https://github.com/abravalheri/validate-pyproject
+    rev: "v0.24.1"
     hooks:
-      - id: isort
+      - id: validate-pyproject
+        additional_dependencies: ["validate-pyproject-schema-store[all]"]
 
-  - repo: https://github.com/psf/black
-    rev: 22.3.0
+  - repo: https://github.com/python-jsonschema/check-jsonschema
+    rev: "0.33.0"
     hooks:
-      - id: black
+      - id: check-dependabot
+      - id: check-github-workflows
+      - id: check-readthedocs

Dockerfile

@@ -0,0 +1,27 @@
+FROM ghcr.io/prefix-dev/pixi:latest
+
+ENV TZ="America/New_York"
+
+RUN apt-get -y update && \
+    apt-get -y install git
+
+COPY pixi.toml .
+COPY pixi.lock .
+# use `--locked` to ensure the lockfile is up to date with pixi.toml
+RUN pixi install --locked
+# create the shell-hook bash script to activate the environment
+RUN pixi shell-hook -s bash > /shell-hook
+
+ENV PYTHONUNBUFFERED=1
+
+COPY test.py .
+
+RUN mkdir /etc/tiled
+RUN mkdir /.prefect -m 0777
+RUN mkdir /repo -m 0777
+
+RUN /bin/bash /shell-hook
+
+#now reapply deployment to push the image that is being created
+ENTRYPOINT ["pixi", "run"]
+CMD ["python", "-m", "test", "arg"]

README.md

@@ -1,3 +1,4 @@
 # ucal Workflows
 
-Repository of Prefect workflows for the micro calorimetry endstation at the SST beamline.
+Repository of Prefect workflows for the micro calorimetry endstation at the SST
+beamline.

end_of_run_export.py

@@ -10,7 +10,9 @@
 def get_export_path(run):
     proposal_path = get_proposal_path(run)
 
-    visit_date = datetime.datetime.fromisoformat(run.start.get("start_datetime", datetime.datetime.today().isoformat()))
+    visit_date = datetime.datetime.fromisoformat(
+        run.start.get("start_datetime", datetime.datetime.today().isoformat())
+    )
     visit_dir = visit_date.strftime("%Y%m%d_export")
 
     export_path = join(proposal_path, visit_dir)

export_to_athena.py

@@ -1,5 +1,5 @@
 import numpy as np
-from os.path import exists, join
+from os.path import join
 from export_tools import add_comment_to_lines, get_header_and_data
 from prefect import get_run_logger
 
@@ -12,12 +12,12 @@ def exportToAthena(
     c2="",
     headerUpdates={},
     strict=False,
-    verbose=True
+    verbose=True,
 ):
     """Exports to Graham's ASCII SSRL data format
 
     :param folder: Export folder (filename will be auto-generated)
-    :param data: Numpy array with data of dimensions (npts, ncols)
+    :param data: NumPy array with data of dimensions (npts, ncols)
     :param header: Dictionary with 'scaninfo', 'motors', 'channelinfo' sub-dictionaries
     :param namefmt: Python format string that will be filled with info from 'scaninfo' dictionary
     :param c1: Comment string 1
@@ -59,7 +59,7 @@ def exportToAthena(
     metadata["c1"] = c1
     metadata["c2"] = c2
 
-    headerstring = """NSLS                                  
+    headerstring = """NSLS
 {date}
 PTS:{npts:11d} COLS: {ncols:11d}
 Sample: {sample}   loadid: {loadid}
@@ -71,9 +71,7 @@ def exportToAthena(
 {c1}
 {c2}
 -------------------------------------------------------------------------------
-{cols}""".format(
-        **metadata, **motors
-    )
+{cols}""".format(**metadata, **motors)
     headerstring = add_comment_to_lines(headerstring, "#")
     logger.info(f"Writing Athena to {filename}")
     with open(filename, "w") as f:

export_to_hdf5.py

@@ -14,14 +14,18 @@ def exportToHDF5(folder, run, header_updates={}):
     """
 
     if "primary" not in run:
-        print(f"HDF5 Export does not support streams other than Primary, skipping {run.start['scan_id']}")
+        print(
+            f"HDF5 Export does not support streams other than Primary, skipping {run.start['scan_id']}"
+        )
         return False
     metadata = get_xdi_run_header(run, header_updates)
     print("Got XDI Metadata")
     filename = make_filename(folder, metadata, "hdf5")
     print(f"Exporting HDF5 to {filename}")
 
-    columns, run_data, metadata = get_xdi_normalized_data(run, metadata, omit_array_keys=False)
+    columns, run_data, metadata = get_xdi_normalized_data(
+        run, metadata, omit_array_keys=False
+    )
 
     with h5py.File(filename, "w") as f:
         for name, data in zip(columns, run_data):

export_to_tiled.py

@@ -1,4 +1,4 @@
-from export_to_xdi import get_xdi_normalized_data, get_xdi_run_header, make_filename
+from export_to_xdi import get_xdi_normalized_data, get_xdi_run_header
 import xarray as xr
 
 
@@ -28,20 +28,27 @@ def export_to_tiled(run, header_updates={}):
     """
 
     if "primary" not in run:
-        print(f"Tiled Export does not support streams other than Primary, skipping {run.start['scan_id']}")
+        print(
+            f"Tiled Export does not support streams other than Primary, skipping {run.start['scan_id']}"
+        )
         return False
     metadata = get_xdi_run_header(run, header_updates)
     print("Got XDI Metadata")
 
-    columns, run_data, metadata = get_xdi_normalized_data(run, metadata, omit_array_keys=False)
+    columns, run_data, metadata = get_xdi_normalized_data(
+        run, metadata, omit_array_keys=False
+    )
 
     da_dict = {}
     for name, data in zip(columns, run_data):
         if name == "rixs":
             if len(data) == 3:
                 counts, mono_grid, energy_grid = data
                 rixs = xr.DataArray(
-                    counts.T, coords={"emission": energy_grid[:, 0]}, dims=("time", "emission"), name=name
+                    counts.T,
+                    coords={"emission": energy_grid[:, 0]},
+                    dims=("time", "emission"),
+                    name=name,
                 )
             else:
                 rixs = xr.DataArray(data, dims=("time", "emission"), name=name)