From 42c4fb4e15140c0626d9228bc4bf60a7ecfc4f97 Mon Sep 17 00:00:00 2001 From: Glass Date: Thu, 15 Jan 2026 14:24:06 -0500 Subject: [PATCH 1/2] Add workflow step to trigger gitlab deploy --- .github/workflows/dev_sync.yml | 2 +- .github/workflows/test_and_deploy.yml | 57 +++++++++--- .github/workflows/update-lockfiles.yml | 6 +- pixi.lock | 25 ++--- pyproject.toml | 57 +++++++----- src/usansred/reduce.py | 79 ++++++++-------- src/usansred/reduce_USANS.py | 123 +++++++------------------ src/usansred/summary.py | 33 ++----- tests/conftest.py | 11 +-- tests/usansred/test_reduce.py | 11 ++- 10 files changed, 181 insertions(+), 223 deletions(-) diff --git a/.github/workflows/dev_sync.yml b/.github/workflows/dev_sync.yml index 6ffb232..82ee723 100644 --- a/.github/workflows/dev_sync.yml +++ b/.github/workflows/dev_sync.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 # Needed to switch branches ref: next # Explicitly specify the branch to check out diff --git a/.github/workflows/test_and_deploy.yml b/.github/workflows/test_and_deploy.yml index 2c5acfd..a0f4c2a 100644 --- a/.github/workflows/test_and_deploy.yml +++ b/.github/workflows/test_and_deploy.yml @@ -1,12 +1,16 @@ name: testing - on: workflow_dispatch: pull_request: push: branches: [next, qa, main] - tags: ['v*'] + tags: ["v*"] + +# cancel previous job if new commit is pushed +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true env: PKG_NAME: usansred @@ -16,15 +20,14 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6.0.1 with: fetch-depth: 0 fetch-tags: true - name: Setup pixi - uses: prefix-dev/setup-pixi@v0.8.10 + uses: prefix-dev/setup-pixi@v0.9.3 with: - pixi-version: v0.49.0 manifest-path: pyproject.toml - name: Run unit tests @@ -46,16 +49,14 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6.0.1 with: fetch-depth: 100 fetch-tags: true ref: ${{ github.ref }} - name: Setup Pixi - uses: prefix-dev/setup-pixi@v0.8.10 - with: - pixi-version: v0.49.0 + uses: prefix-dev/setup-pixi@v0.9.3 - name: Build conda package run: | @@ -77,12 +78,11 @@ jobs: pixi run python -c "import finddata" - name: Upload conda package as artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6.0.0 with: name: artifact-conda-package path: ${{ env.PKG_NAME }}-*.conda - # Publish the package as a separate job so that the Github Actions webpage # shows it as a separate step in the CI workflow publish: @@ -92,19 +92,18 @@ jobs: if: startsWith(github.ref, 'refs/tags/v') steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6.0.1 with: fetch-depth: 0 fetch-tags: true - name: Setup Pixi - uses: prefix-dev/setup-pixi@v0.8.10 + uses: prefix-dev/setup-pixi@v0.9.3 with: - pixi-version: v0.49.0 manifest-path: pyproject.toml - name: Download conda package artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v7.0.0 with: name: artifact-conda-package @@ -118,3 +117,31 @@ jobs: if [ "${IS_RC}" = "true" ]; then CONDA_LABEL="rc"; fi echo pushing ${{ github.ref }} with label $CONDA_LABEL pixi run anaconda upload --label $CONDA_LABEL --user neutrons ${{ env.PKG_NAME }}-*.conda + + # Trigger GitLab dev deploy pipeline + deploy-dev: + runs-on: ubuntu-latest + needs: build + # if: github.ref == 'refs/heads/next' + steps: + - name: Get Environment Name + uses: neutrons/branch-mapper@main + id: env_name + with: + prefix: usansred + + - name: Trigger Dev Deploy + # use https://github.com/eic/trigger-gitlab-ci/pull/14 until merged + uses: eic/trigger-gitlab-ci@d984d8d53d871d2fdc1325639d94322da6e8747f + id: trigger + with: + url: https://code.ornl.gov + project_id: 19016 + ref_name: main + token: ${{ secrets.GITLAB_DEPLOY_TOKEN }} + + - name: Annotate commit + uses: peter-evans/commit-comment@v4 + with: + body: | + GitLab pipeline for ${{ steps.env_name.outputs.name }} has been submitted for this commit: ${{ steps.trigger.outputs.web_url }} diff --git a/.github/workflows/update-lockfiles.yml b/.github/workflows/update-lockfiles.yml index faa5671..fd4229e 100644 --- a/.github/workflows/update-lockfiles.yml +++ b/.github/workflows/update-lockfiles.yml @@ -15,10 +15,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup pixi - uses: prefix-dev/setup-pixi@v0.8.10 + uses: prefix-dev/setup-pixi@v0.9.3 with: run-install: false @@ -28,7 +28,7 @@ jobs: pixi update --json | pixi exec pixi-diff-to-markdown >> diff.md - name: Create pull request - uses: peter-evans/create-pull-request@v7 + uses: peter-evans/create-pull-request@v8 with: token: ${{ secrets.GITHUB_TOKEN }} commit-message: Update pixi lockfile diff --git a/pixi.lock b/pixi.lock index 680deab..f6fbaad 100644 --- a/pixi.lock +++ b/pixi.lock @@ -7,6 +7,8 @@ environments: - url: https://conda.anaconda.org/neutrons/ indexes: - https://pypi.org/simple + options: + pypi-prerelease-mode: if-necessary-or-explicit packages: linux-64: - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 @@ -443,8 +445,8 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/zstandard-0.23.0-py311h9ecbd09_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb8e6e7a_2.conda - pypi: https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/42/af/d35718c1ccd68bd2e7b6b26a83c4919516d73610ddf124796797a7858749/regex-2025.7.31-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/7f/fb/44ef7148ed5b55e519c7d205ba4281087fcf947d96a1e6d001ae6c1d4c34/toml_cli-0.8.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a4/3c/87ca0a02736d16b6262921425e84b48984e77d8e4e572c9072ce96e66c30/regex-2026.1.15-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/d2/b8/f0b9b880c03a3db8eaff63d76ca751ac7d8e45483fb7a0bb9f8e5c6ce433/toml_cli-0.8.2-py3-none-any.whl - pypi: ./ docs: channels: @@ -453,6 +455,8 @@ environments: - url: https://conda.anaconda.org/neutrons/ indexes: - https://pypi.org/simple + options: + pypi-prerelease-mode: if-necessary-or-explicit packages: linux-64: - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 @@ -5638,10 +5642,10 @@ packages: - pkg:pypi/referencing?source=hash-mapping size: 51668 timestamp: 1737836872415 -- pypi: https://files.pythonhosted.org/packages/42/af/d35718c1ccd68bd2e7b6b26a83c4919516d73610ddf124796797a7858749/regex-2025.7.31-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl +- pypi: https://files.pythonhosted.org/packages/a4/3c/87ca0a02736d16b6262921425e84b48984e77d8e4e572c9072ce96e66c30/regex-2026.1.15-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl name: regex - version: 2025.7.31 - sha256: 055baef91bb31474bd919fd245cf154db00cbac449596952d3e6bc1e1b226808 + version: 2026.1.15 + sha256: d9ea2604370efc9a174c1b5dcc81784fb040044232150f7f33756049edfc9026 requires_python: '>=3.9' - conda: https://conda.anaconda.org/conda-forge/noarch/requests-2.32.4-pyhd8ed1ab_0.conda sha256: 9866aaf7a13c6cfbe665ec7b330647a0fb10a81e6f9b8fee33642232a1920e18 @@ -6080,10 +6084,10 @@ packages: - pkg:pypi/toml?source=hash-mapping size: 22132 timestamp: 1734091907682 -- pypi: https://files.pythonhosted.org/packages/7f/fb/44ef7148ed5b55e519c7d205ba4281087fcf947d96a1e6d001ae6c1d4c34/toml_cli-0.8.1-py3-none-any.whl +- pypi: https://files.pythonhosted.org/packages/d2/b8/f0b9b880c03a3db8eaff63d76ca751ac7d8e45483fb7a0bb9f8e5c6ce433/toml_cli-0.8.2-py3-none-any.whl name: toml-cli - version: 0.8.1 - sha256: ebd1672e2b512022c4db2ba01658bff17ec9bb868b64ddee29484d1517ce0d98 + version: 0.8.2 + sha256: 7af4679ca04c53ad0f6d300dab26f45a78fedf88e8310305bfe0a8ead37fd000 requires_dist: - jmespath>=1.0.1 - regex>=2020.7.14 @@ -6346,10 +6350,9 @@ packages: timestamp: 1750271478254 - pypi: ./ name: usansred - version: 1.2.0.dev1 - sha256: 7e7fb6a708d10dbd87294f3d1b3896cd49954edb906e974536d882c75d67ebf5 + version: 1.3.0.dev1 + sha256: 4426a6b9eba10d81980de1ddae4b42a7f224c49350c1043772d54c1c6889241d requires_python: '>=3.11' - editable: true - conda: https://conda.anaconda.org/conda-forge/noarch/userpath-1.9.2-pyhd8ed1ab_0.conda sha256: 26e53b42f7fa1127e6115a35b91c20e15f75984648b88f115136f27715d4a440 md5: 946e3571aaa55e0870fec0dea13de3bf diff --git a/pyproject.toml b/pyproject.toml index b50a6cc..48c2aa5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ artifacts = [ "reduction/usansred/_version.py", "reduction/usansred/**/*.yml", "reduction/usansred/**/*.yaml", - "reduction/usansred/**/*.ini" + "reduction/usansred/**/*.ini", ] [tool.hatch.build.targets.wheel] @@ -60,22 +60,30 @@ distance-dirty = "{next_version}.dev{distance}" file = "src/usansred/_version.py" [tool.pytest.ini_options] -pythonpath = [ - ".", "src", "scripts" -] +pythonpath = [".", "src", "scripts"] testpaths = ["tests"] python_files = ["test*.py"] -norecursedirs = [".git", "tmp*", "_tmp*", "__pycache__", "*dataset*", "*data_set*"] +norecursedirs = [ + ".git", + "tmp*", + "_tmp*", + "__pycache__", + "*dataset*", + "*data_set*", +] markers = [ - "datarepo: using repository usansred-data", - "sns_mounted: requires the /SNS filesystem" + "datarepo: using repository usansred-data", + "sns_mounted: requires the /SNS filesystem", ] [tool.ruff] line-length = 120 # https://beta.ruff.rs/docs/rules/ lint.select = ["A", "ARG", "BLE", "E", "F", "I", "PT"] -lint.ignore = ["F403", "F405", "F401", # wild imports and unknown names +lint.ignore = [ + "F403", + "F405", + "F401", # wild imports and unknown names ] ################### @@ -85,16 +93,17 @@ lint.ignore = ["F403", "F405", "F401", # wild imports and unknown names [tool.pixi.workspace] name = "usansred" preview = ["pixi-build"] -channels = [ - "conda-forge", - "mantid/label/main", - "neutrons" -] +channels = ["conda-forge", "mantid/label/main", "neutrons"] platforms = ["linux-64"] [tool.pixi.environments] -default = {features = ["test", "package", "docs", "developer"], solve-group = "default"} -docs = {features = ["docs"], solve-group = "docs"} +default = { features = [ + "test", + "package", + "docs", + "developer", +], solve-group = "default" } +docs = { features = ["docs"], solve-group = "docs" } [tool.pixi.pypi-dependencies] usansred = { path = ".", editable = true } @@ -104,7 +113,7 @@ python = "=3.11" mantidworkbench = ">=6.13.0,<6.14.0" pandas = "*" xlsxwriter = "*" -finddata = {version = "=0.10.1", channel="neutrons"} +finddata = { version = "=0.10.1", channel = "neutrons" } # Conda packages (no PyPi) to be enumerated within the conda package to be built. [tool.pixi.package.run-dependencies] @@ -112,18 +121,18 @@ python = "=3.11" mantidworkbench = ">=6.13.0,<6.14.0" pandas = "*" xlsxwriter = "*" -finddata = {version = "=0.10.1", channel="neutrons"} +finddata = { version = "=0.10.1", channel = "neutrons" } [tool.pixi.package] name = "usansred" -version = "1.0.0" # placeholder, overwritten by sync-version +version = "1.0.0" # placeholder, overwritten by sync-version [tool.pixi.package.build] -backend = { name = "pixi-build-python", version = "0.1.*" } -channels = [ +backend = { name = "pixi-build-python", version = "0.4.*", channels = [ "https://prefix.dev/pixi-build-backends", "https://prefix.dev/conda-forge", -] + +] } [tool.pixi.tasks] # Documentation @@ -131,8 +140,8 @@ build-docs = { cmd = 'sphinx-build -b html docs docs/_build', description = "Bui test-docs = { cmd = "sphinx-build -M doctest docs docs/_build", description = "Test building the documentation" } # Testing test = { description = "Run the test suite", cmd = "pytest" } -unit-test = {description = "Run the unit tests", cmd = "python -m pytest -vv -m \"not datarepo and not sns_mounted\" --cov=src --cov-report=xml --cov-report=term-missing tests/ && mv .coverage .coverage.unit"} -integration-test = {description = "Run the integration tests", cmd = "git submodule update --init && python -m pytest -vv -m \"datarepo\" --cov=src --cov-report=xml --cov-report=term-missing tests/ && mv .coverage .coverage.integration"} +unit-test = { description = "Run the unit tests", cmd = "python -m pytest -vv -m \"not datarepo and not sns_mounted\" --cov=src --cov-report=xml --cov-report=term-missing tests/ && mv .coverage .coverage.unit" } +integration-test = { description = "Run the integration tests", cmd = "git submodule update --init && python -m pytest -vv -m \"datarepo\" --cov=src --cov-report=xml --cov-report=term-missing tests/ && mv .coverage .coverage.integration" } # Packaging pypi-build = { cmd = "hatch build", description = "Build the package for PyPI" } conda-build-command = { cmd = "pixi build", description = "Build the conda package command" } @@ -157,7 +166,7 @@ clean-all = { description = "Clean all artifacts", depends-on = [ "clean-docs", "clean-pypi", ] } -combine-coverage = {description = "Combine coverage reports", cmd = "coverage combine"} +combine-coverage = { description = "Combine coverage reports", cmd = "coverage combine" } sync-version = { cmd = 'version=$(python -m versioningit); toml set tool.pixi.package.version "$version" --toml-path pyproject.toml', description = "Sync pyproject.toml version with Git version" } backup-toml = { cmd = "cp pyproject.toml pyproject.toml.bak", description = "Backup the pyproject.toml file" } reset-toml = { cmd = "cp pyproject.toml.bak pyproject.toml; rm pyproject.toml.bak", description = "Reset the pyproject.toml file to the original state" } diff --git a/src/usansred/reduce.py b/src/usansred/reduce.py index 8d93deb..9dd5189 100755 --- a/src/usansred/reduce.py +++ b/src/usansred/reduce.py @@ -340,11 +340,11 @@ def stitchData(self): for scan in self.scans: it = list( zip( - scan.monitorData["IQData"]["Q"], - scan.monitorData["IQData"]["I"], - scan.monitorData["IQData"]["E"], - scan.detectorData[bank]["IQData"]["I"], - scan.detectorData[bank]["IQData"]["E"], + scan.monitorData["IQData"]["Q"], + scan.monitorData["IQData"]["I"], + scan.monitorData["IQData"]["E"], + scan.detectorData[bank]["IQData"]["I"], + scan.detectorData[bank]["IQData"]["E"], ) ) @@ -445,33 +445,33 @@ def generate_initial_parameters(test_x, test_y): return result.x def clean_iq(qScaled, iScaled, eScaled): - ''' + """ Remove duplicate values in q by: Taking average of all i values with same q Taking standard deviation of error values of e with same q - return - + return - cleaned q, i, and error values - ''' - from collections import defaultdict + """ import math - + from collections import defaultdict + # Dictionary to store sums for averaging I and propagating errors for E - sum_dict = defaultdict(lambda: {'I_sum': 0, 'I_count': 0, 'E_sum_squares': 0}) - + sum_dict = defaultdict(lambda: {"I_sum": 0, "I_count": 0, "E_sum_squares": 0}) + for q, i, e in zip(qScaled, iScaled, eScaled): - sum_dict[q]['I_sum'] += i - sum_dict[q]['I_count'] += 1 - sum_dict[q]['E_sum_squares'] += e ** 2 - + sum_dict[q]["I_sum"] += i + sum_dict[q]["I_count"] += 1 + sum_dict[q]["E_sum_squares"] += e**2 + q_cleaned = [] i_cleaned = [] e_cleaned = [] - + for q, values in sum_dict.items(): q_cleaned.append(q) - i_cleaned.append(values['I_sum'] / values['I_count']) - e_cleaned.append(math.sqrt(values['E_sum_squares'])) - + i_cleaned.append(values["I_sum"] / values["I_count"]) + e_cleaned.append(math.sqrt(values["E_sum_squares"])) + return q_cleaned, i_cleaned, e_cleaned for bb in range(self.numOfBanks): @@ -519,12 +519,7 @@ def clean_iq(qScaled, iScaled, eScaled): qcleaned, icleaned, ecleaned = clean_iq(qScaled, iScaled, eScaled) - dataScaled = { - 'Q': qcleaned.copy(), - 'I': icleaned.copy(), - 'E': ecleaned.copy(), - 'T':[] - } + dataScaled = {"Q": qcleaned.copy(), "I": icleaned.copy(), "E": ecleaned.copy(), "T": []} self.dataScaled.append(dataScaled) q_range = f"{min(self.dataScaled[0]['Q'])} - {max(self.dataScaled[0]['Q'])}" @@ -678,10 +673,10 @@ def logBin(self, momentum_transfer, intensity, energy): def _match_or_interpolate(self, q_data, q_bg, i_bg, e_bg, tolerance=1e-5): """Match q_bg values to q_data directly if close enough, otherwise interpolate""" - + i_bg_matched = numpy.zeros_like(q_data) e_bg_matched = numpy.zeros_like(q_data) - + for i, q in enumerate(q_data): # Find the index in q_bg that is closest to the current q_data value idx = numpy.argmin(numpy.abs(q_bg - q)) @@ -693,9 +688,9 @@ def _match_or_interpolate(self, q_data, q_bg, i_bg, e_bg, tolerance=1e-5): # Otherwise, interpolate i_bg_matched[i] = numpy.interp(q, q_bg, i_bg) e_bg_matched[i] = numpy.interp(q, q_bg, e_bg) - + return i_bg_matched, e_bg_matched - + def subtractBg(self, background, vScale=1.0): """ Subtract the background @@ -703,7 +698,7 @@ def subtractBg(self, background, vScale=1.0): return """ - + if self.experiment.logbin: assert self.isLogBinned msg = ( @@ -739,21 +734,21 @@ def subtractBg(self, background, vScale=1.0): bgScaled = self.experiment.background.dataScaled[0] # Convert things to numpy arrays - q_data = numpy.array(dataScaled['Q']) - i_data = numpy.array(dataScaled['I']) - e_data = numpy.array(dataScaled['E']) - - q_bg = numpy.array(bgScaled['Q']) - i_bg = numpy.array(bgScaled['I']) - e_bg = numpy.array(bgScaled['E']) - + q_data = numpy.array(dataScaled["Q"]) + i_data = numpy.array(dataScaled["I"]) + e_data = numpy.array(dataScaled["E"]) + + q_bg = numpy.array(bgScaled["Q"]) + i_bg = numpy.array(bgScaled["I"]) + e_bg = numpy.array(bgScaled["E"]) + # Interpolate bg I and E values at data Q points i_bg_interp, e_bg_interp = self._match_or_interpolate(q_data, q_bg, i_bg, e_bg) - + # Subtract background i_subtracted = i_data - i_bg_interp e_subtracted = numpy.sqrt(e_data**2 + e_bg_interp**2) - + self.dataBgSubtracted["Q"] = q_data self.dataBgSubtracted["I"] = i_subtracted self.dataBgSubtracted["E"] = e_subtracted @@ -940,7 +935,7 @@ def __init__(self, csvFilePath, logbin=False, outputFolder=None): self.folder = os.path.dirname(csvFilePath) - with open(csvFilePath, newline="", encoding='utf-8-sig') as csvFile: + with open(csvFilePath, newline="", encoding="utf-8-sig") as csvFile: csvReader = csv.reader(csvFile, delimiter=",") for row in csvReader: diff --git a/src/usansred/reduce_USANS.py b/src/usansred/reduce_USANS.py index b6fc9d9..6f0d7a5 100644 --- a/src/usansred/reduce_USANS.py +++ b/src/usansred/reduce_USANS.py @@ -8,20 +8,21 @@ import traceback import warnings +import numpy as np + # third-party imports # from debugpy.common.log import newline from mantid.simpleapi import ( - mtd, ConvertTableToMatrixWorkspace, CropWorkspace, LoadEventNexus, + Rebin, SaveAscii, StepScan, SumSpectra, - Rebin, + mtd, ) from matplotlib import use -import numpy as np # usansred imports from usansred import reduce @@ -93,26 +94,14 @@ def main(): # Get ROI from logs wavelength = [3.6, 1.8, 1.2, 0.9, 0.72, 0.6] - roi_min = ( - mtd["USANS"].getRun().getProperty("BL1A:Det:N1:Det1:TOF:ROI:1:Min").value[-1] - ) + roi_min = mtd["USANS"].getRun().getProperty("BL1A:Det:N1:Det1:TOF:ROI:1:Min").value[-1] # roi_step = mtd["USANS"].getRun().getProperty("BL1A:Det:N1:Det1:TOF:ROI:1:Size").value[-1] # Reference to the item in the wavelength array main_index = 1 for i in range(1, 8): - lower_bound = ( - mtd["USANS"] - .getRun() - .getProperty("BL1A:Det:N1:Det1:TOF:ROI:%s:Min" % i) - .value[-1] - ) - tof_step = ( - mtd["USANS"] - .getRun() - .getProperty("BL1A:Det:N1:Det1:TOF:ROI:%s:Size" % i) - .value[-1] - ) + lower_bound = mtd["USANS"].getRun().getProperty("BL1A:Det:N1:Det1:TOF:ROI:%s:Min" % i).value[-1] + tof_step = mtd["USANS"].getRun().getProperty("BL1A:Det:N1:Det1:TOF:ROI:%s:Size" % i).value[-1] if i > 1 and lower_bound == roi_min: main_index = i - 2 peaks.append([lower_bound * 1000.0, (lower_bound + tof_step) * 1000.0]) @@ -151,9 +140,7 @@ def main(): OutputWorkspace="USANS_monitors", ) file_path = os.path.join(outdir, "%s_monitor.txt" % file_prefix) - SaveAscii( - InputWorkspace="USANS_monitors", Filename=file_path, WriteSpectrumID=False - ) + SaveAscii(InputWorkspace="USANS_monitors", Filename=file_path, WriteSpectrumID=False) # Find whether we have a motor turning short_name = "" @@ -182,9 +169,7 @@ def main(): ColumnE="Error", OutputWorkspace="USANS_scan_monitor", ) - file_path = os.path.join( - outdir, "%s_monitor_scan_%s.txt" % (file_prefix, short_name) - ) + file_path = os.path.join(outdir, "%s_monitor_scan_%s.txt" % (file_prefix, short_name)) SaveAscii( InputWorkspace="USANS_scan_monitor", Filename=file_path, @@ -192,40 +177,19 @@ def main(): ) y_monitor = mtd["USANS_scan_monitor"].readY(0) - iq_file_path_simple = os.path.join( - outdir, "%s_iq_%s_%s.txt" % (file_prefix, short_name, main_wl) - ) + iq_file_path_simple = os.path.join(outdir, "%s_iq_%s_%s.txt" % (file_prefix, short_name, main_wl)) iq_fd_simple = open(iq_file_path_simple, "w") - iq_file_path = os.path.join( - outdir, "%s_iq_%s.txt" % (file_prefix, short_name) - ) + iq_file_path = os.path.join(outdir, "%s_iq_%s.txt" % (file_prefix, short_name)) iq_fd = open(iq_file_path, "w") start_time = mtd["USANS"].getRun().getProperty("start_time").value - experiment = ( - mtd["USANS"].getRun().getProperty("experiment_identifier").value - ) + experiment = mtd["USANS"].getRun().getProperty("experiment_identifier").value run_number = mtd["USANS"].getRun().getProperty("run_number").value run_title = mtd["USANS"].getRun().getProperty("run_title").value - sequence_first_run = ( - mtd["USANS"] - .getRun() - .getProperty("BL1A:CS:Scan:USANS:FirstRun") - .value[-1] - ) - sequence_index = ( - mtd["USANS"] - .getRun() - .getProperty("BL1A:CS:Scan:USANS:Index") - .value[-1] - ) - meta_wavelength = ( - mtd["USANS"] - .getRun() - .getProperty("BL1A:CS:Scan:USANS:Wavelength") - .value[-1] - ) + sequence_first_run = mtd["USANS"].getRun().getProperty("BL1A:CS:Scan:USANS:FirstRun").value[-1] + sequence_index = mtd["USANS"].getRun().getProperty("BL1A:CS:Scan:USANS:Index").value[-1] + meta_wavelength = mtd["USANS"].getRun().getProperty("BL1A:CS:Scan:USANS:Wavelength").value[-1] print("Wavelength: %s [%s]" % (wavelength[main_index], meta_wavelength)) iq_fd.write("# Experiment %s Run %s\n" % (experiment, run_number)) @@ -238,16 +202,12 @@ def main(): "# %-8s %-10s %-10s %-10s %-10s %-10s %-10s %-5s\n" % ("Q", "I(Q)", "dI(Q)", "dQ", "N(Q)", "dN(Q)", "Mon(Q)", "Lambda") ) - iq_fd_simple.write( - "# Experiment %s Run %s\n" % (experiment, run_number) - ) + iq_fd_simple.write("# Experiment %s Run %s\n" % (experiment, run_number)) iq_fd_simple.write("# Run start time: %s\n" % start_time) iq_fd_simple.write("# Title: %s\n" % run_title) iq_fd_simple.write("# Sequence ID: %s\n" % sequence_first_run) iq_fd_simple.write("# Sequence index: %s\n" % sequence_index) - iq_fd_simple.write( - "# Selected wavelength: %s\n" % wavelength[main_index] - ) + iq_fd_simple.write("# Selected wavelength: %s\n" % wavelength[main_index]) iq_fd_simple.write("# %-8s %-10s %-10s\n" % ("Q", "I(Q)", "dI(Q)")) for i in range(len(peaks)): @@ -258,9 +218,7 @@ def main(): XMin=peak[0], XMax=peak[1], ) - StepScan( - InputWorkspace="peak_detector", OutputWorkspace="scan_table" - ) + StepScan(InputWorkspace="peak_detector", OutputWorkspace="scan_table") ConvertTableToMatrixWorkspace( InputWorkspace="scan_table", ColumnX=scan_var, @@ -268,24 +226,22 @@ def main(): ColumnE="Error", OutputWorkspace="USANS_scan_detector", ) - mtd["USANS_scan_detector"].getAxis(1).getUnit().setLabel( - "Counts", "Counts" - ) + mtd["USANS_scan_detector"].getAxis(1).getUnit().setLabel("Counts", "Counts") x_data = mtd["USANS_scan_detector"].readX(0) y_data = mtd["USANS_scan_detector"].readY(0) e_data = mtd["USANS_scan_detector"].readE(0) if i == 0: - file_path = os.path.join( - outdir, "%s_detector_%s.txt" % (file_prefix, main_wl) - ) + file_path = os.path.join(outdir, "%s_detector_%s.txt" % (file_prefix, main_wl)) SaveAscii( InputWorkspace="USANS_scan_detector", Filename=file_path, WriteSpectrumID=False, ) # json_file_path = os.path.join(outdir, "%s_plot_data.json" % file_prefix) - # SavePlot1DAsJson(InputWorkspace="USANS_scan_detector", JsonFilename=json_file_path, PlotName="main_output") + # SavePlot1DAsJson( + # InputWorkspace="USANS_scan_detector", JsonFilename=json_file_path, PlotName="main_output" + # ) try: from postprocessing.publish_plot import plot1d @@ -293,9 +249,7 @@ def main(): try: from finddata.publish_plot import plot1d except: # noqa E722 - logging.error( - "Cannot import postprocessing or finddata." - ) + logging.error("Cannot import postprocessing or finddata.") try: plot1d( @@ -336,15 +290,12 @@ def main(): (e_data[i_theta] / y_monitor[i_theta]) ** 2 + y_data[i_theta] ** 2 / y_monitor[i_theta] ** 3 ) - iq_fd_simple.write( - "%-10.6g %-10.6g %-10.6g\n" % (q, i_q, di_q) - ) + iq_fd_simple.write("%-10.6g %-10.6g %-10.6g\n" % (q, i_q, di_q)) else: file_path = os.path.join( outdir, - "%s_detector_scan_%s_peak_%s.txt" - % (file_prefix, short_name, i), + "%s_detector_scan_%s_peak_%s.txt" % (file_prefix, short_name, i), ) SaveAscii( InputWorkspace="USANS_scan_detector", @@ -352,12 +303,7 @@ def main(): WriteSpectrumID=False, ) for i_theta in range(len(x_data)): - q = ( - 2.0 - * math.pi - * math.sin(x_data[i_theta] * math.pi / 180.0 / 3600.0) - / wavelength[i - 1] - ) + q = 2.0 * math.pi * math.sin(x_data[i_theta] * math.pi / 180.0 / 3600.0) / wavelength[i - 1] # if q<=0: # continue @@ -397,9 +343,7 @@ def main(): ) if i == 0: - file_path = os.path.join( - outdir, "%s_trans_%s.txt" % (file_prefix, main_wl) - ) + file_path = os.path.join(outdir, "%s_trans_%s.txt" % (file_prefix, main_wl)) SaveAscii( InputWorkspace="USANS_scan_trans", Filename=file_path, @@ -408,8 +352,7 @@ def main(): else: file_path = os.path.join( outdir, - "%s_trans_scan_%s_peak_%s.txt" - % (file_prefix, short_name, i), + "%s_trans_scan_%s_peak_%s.txt" % (file_prefix, short_name, i), ) SaveAscii( InputWorkspace="USANS_scan_trans", @@ -422,17 +365,13 @@ def main(): # list all the sequence files json_files = [ - pos_json - for pos_json in os.listdir(outdir) - if pos_json.endswith(".json") and pos_json.startswith("scan") + pos_json for pos_json in os.listdir(outdir) if pos_json.endswith(".json") and pos_json.startswith("scan") ] json_files.sort() json_files = [os.path.join(outdir, ff) for ff in json_files] sample_json_files = [ - pos_json - for pos_json in os.listdir(outdir) - if pos_json.endswith(".json") and pos_json.startswith("sample") + pos_json for pos_json in os.listdir(outdir) if pos_json.endswith(".json") and pos_json.startswith("sample") ] sample_json_files.sort() sample_json_files = [os.path.join(outdir, ff) for ff in json_files] diff --git a/src/usansred/summary.py b/src/usansred/summary.py index 67c31f7..a47ca60 100755 --- a/src/usansred/summary.py +++ b/src/usansred/summary.py @@ -4,9 +4,10 @@ import csv import logging import os -import pandas import sys +import pandas + __author__ = "Yingrui Shang" __copyright__ = "Copyright 2021, NSD, ORNL" @@ -47,36 +48,26 @@ def reportFromCSV(csvFilePath, outputFolder=None): if outputFolder is None: outputFolder = os.path.join(dataFolderName, "reduced") - xlsxWriter = pandas.ExcelWriter( - os.path.join(outputFolder, "summary.xlsx"), engine="xlsxwriter" - ) + xlsxWriter = pandas.ExcelWriter(os.path.join(outputFolder, "summary.xlsx"), engine="xlsxwriter") workbook = xlsxWriter.book # nbFormat = workbook.add_format({"bold": False}) # Create a chart sheet for unscaled data chartsheetUnscaled = workbook.add_chartsheet("Unscaled") - mainChartUnscaled = workbook.add_chart( - {"type": "scatter", "subtype": "smooth_with_markers"} - ) + mainChartUnscaled = workbook.add_chart({"type": "scatter", "subtype": "smooth_with_markers"}) # Create a chart sheet for original data chartsheetOrig = workbook.add_chartsheet("Original") - mainChartOrig = workbook.add_chart( - {"type": "scatter", "subtype": "smooth_with_markers"} - ) + mainChartOrig = workbook.add_chart({"type": "scatter", "subtype": "smooth_with_markers"}) # log binned data chartsheetLogBinned = workbook.add_chartsheet("Log Binned") - mainChartLogBinned = workbook.add_chart( - {"type": "scatter", "subtype": "smooth_with_markers"} - ) + mainChartLogBinned = workbook.add_chart({"type": "scatter", "subtype": "smooth_with_markers"}) # log binned data with background removed chartsheetSubtracted = workbook.add_chartsheet("BG Subtracted") - mainChartSubtracted = workbook.add_chart( - {"type": "scatter", "subtype": "smooth_with_markers"} - ) + mainChartSubtracted = workbook.add_chart({"type": "scatter", "subtype": "smooth_with_markers"}) mainChartUnscaled.set_x_axis({"name": "Q (1/A)", "log_base": 10}) @@ -129,11 +120,7 @@ def reportFromCSV(csvFilePath, outputFolder=None): df = pandas.concat( [ df, - df[ - (df["Q(1/A)"] > 0) - & (df["I(1/cm)"] > 0) - & (df["dI(1/cm)"] > 0) - ], + df[(df["Q(1/A)"] > 0) & (df["I(1/cm)"] > 0) & (df["dI(1/cm)"] > 0)], ], ignore_index=False, axis=1, @@ -156,9 +143,7 @@ def reportFromCSV(csvFilePath, outputFolder=None): worksheet = xlsxWriter.sheets[wn] # worksheet.set_column('A:A', 12, nbFormat) - chart = workbook.add_chart( - {"type": "scatter", "subtype": "smooth_with_markers"} - ) + chart = workbook.add_chart({"type": "scatter", "subtype": "smooth_with_markers"}) chart.add_series( { diff --git a/tests/conftest.py b/tests/conftest.py index db325e3..74f0866 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,9 +2,10 @@ import os import sys +import pytest + # third party imports from mantid.simpleapi import config -import pytest # usansred imports @@ -23,9 +24,7 @@ def data_server(): class _DataServe(object): def __init__(self): - self._directory = os.path.join( - os.path.dirname(this_module_path), "usansred-data" - ) + self._directory = os.path.join(os.path.dirname(this_module_path), "usansred-data") config.appendDataSearchDir(self._directory) config["default.facility"] = "SNS" config["default.instrument"] = "USANS" @@ -51,9 +50,7 @@ def path_to(self, basename: str) -> str: for dirpath, dirnames, filenames in os.walk(self._directory): if basename in filenames: return os.path.join(dirpath, basename) - raise IOError( - f"File {basename} not found in data directory {self._directory}" - ) + raise IOError(f"File {basename} not found in data directory {self._directory}") yield _DataServe() for key, val in _backup.items(): diff --git a/tests/usansred/test_reduce.py b/tests/usansred/test_reduce.py index 1d27c69..16b6fed 100644 --- a/tests/usansred/test_reduce.py +++ b/tests/usansred/test_reduce.py @@ -1,16 +1,17 @@ # import standard import os -import numpy as np import random from unittest.mock import MagicMock from unittest.mock import patch as mock_patch +import numpy as np + # third party packages import pytest +from usansred.reduce import Experiment, Sample # usansred imports from usansred.reduce import main as reduceUSANS -from usansred.reduce import Sample,Experiment def read_numbers_from_file(filename): @@ -74,6 +75,7 @@ def test_main(mock_parse_arguments, data_server, tmp_path): if os.path.exists(expected): # file "UN_EmptyPCell_det_1_lbs.txt" does not exist compare_lines(output, expected) + @pytest.mark.datarepo() def test_sample_match_or_interpolate(data_server, tmp_path): # Get the testing data and temp output directory @@ -86,7 +88,7 @@ def test_sample_match_or_interpolate(data_server, tmp_path): qq = np.array([dd * 1e-5 for dd in range(1, 100)]) ii = -np.log(qq) * 1e3 bb = ii * 0.01 - + # Generate a list of 100 random numbers ee = [random.random() for _ in range(1, 100)] @@ -94,8 +96,9 @@ def test_sample_match_or_interpolate(data_server, tmp_path): iibgmatched, eebgmatched = sample_test._match_or_interpolate(qq, qq, bb, ee) - check = (iibgmatched - bb == 0.) + check = iibgmatched - bb == 0.0 assert np.all(check), "Background interpolation calculation is not right in Sample._match_or_interpolate" + if __name__ == "__main__": pytest.main([__file__]) From 8f7e564ce65b000b868659bc90b4fc7cfc29ca73 Mon Sep 17 00:00:00 2001 From: Glass Date: Thu, 15 Jan 2026 14:29:20 -0500 Subject: [PATCH 2/2] only run on pushes to next again --- .github/workflows/test_and_deploy.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_and_deploy.yml b/.github/workflows/test_and_deploy.yml index a0f4c2a..b817262 100644 --- a/.github/workflows/test_and_deploy.yml +++ b/.github/workflows/test_and_deploy.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6.0.1 + uses: actions/checkout@v6 with: fetch-depth: 0 fetch-tags: true @@ -49,7 +49,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6.0.1 + uses: actions/checkout@v6 with: fetch-depth: 100 fetch-tags: true @@ -92,7 +92,7 @@ jobs: if: startsWith(github.ref, 'refs/tags/v') steps: - name: Checkout - uses: actions/checkout@v6.0.1 + uses: actions/checkout@v6 with: fetch-depth: 0 fetch-tags: true @@ -122,7 +122,7 @@ jobs: deploy-dev: runs-on: ubuntu-latest needs: build - # if: github.ref == 'refs/heads/next' + if: github.ref == 'refs/heads/next' steps: - name: Get Environment Name uses: neutrons/branch-mapper@main