diff --git a/.github/ISSUE_TEMPLATE/audited_events_rule_set_review.md b/.github/ISSUE_TEMPLATE/audited_events_rule_set_review.md index 7f62819970..fff858142e 100644 --- a/.github/ISSUE_TEMPLATE/audited_events_rule_set_review.md +++ b/.github/ISSUE_TEMPLATE/audited_events_rule_set_review.md @@ -1,10 +1,10 @@ --- -name: Annual review of the rule set for audited events +name: Review rule set for audited events about: Issue template for annual review of the rule set for audited events title: Review rule set for audited events labels: -,compliance,doc,no demo,orange assignees: nolunwa-ucsc,bvizzier-ucsc -type: Task +type: Chore _start: 2025-01-10T10:00 _period: 1 year --- diff --git a/.github/ISSUE_TEMPLATE/blank.md b/.github/ISSUE_TEMPLATE/blank.md index 899da05512..39b14d266b 100644 --- a/.github/ISSUE_TEMPLATE/blank.md +++ b/.github/ISSUE_TEMPLATE/blank.md @@ -1,5 +1,5 @@ --- -name: A New Issue +name: A new blank issue about: Template for new issues title: '' labels: orange diff --git a/.github/ISSUE_TEMPLATE/fedramp_inventory_review.md b/.github/ISSUE_TEMPLATE/fedramp_inventory_review.md index 3ce5ef3764..7788f95979 100644 --- a/.github/ISSUE_TEMPLATE/fedramp_inventory_review.md +++ b/.github/ISSUE_TEMPLATE/fedramp_inventory_review.md @@ -1,10 +1,10 @@ --- -name: Review FedRAMP inventory -about: Issue template for the current operator to perform a Port and Protocol review with system administrator -title: Monthly inventory review -labels: +,compliance,infra,no demo,operator,orange +name: Monthly FedRAMP inventory review +about: Issue template for the monthly creation and review of the FedRAMP cloud infrastructure inventory +title: Monthly FedRAMP inventory review +labels: -,compliance,infra,no demo,operator,orange assignees: hannes-ucsc -type: Task +type: Chore _repository: DataBiosphere/azul-private _start: 2024-03-01T09:00 _period: 1 month diff --git a/.github/ISSUE_TEMPLATE/opensearch_updates.md b/.github/ISSUE_TEMPLATE/opensearch_updates.md index 6c41261ee5..1e5cc6302f 100644 --- a/.github/ISSUE_TEMPLATE/opensearch_updates.md +++ b/.github/ISSUE_TEMPLATE/opensearch_updates.md @@ -1,9 +1,9 @@ --- -name: OpenSearch service software update -about: Issue template for operator tasks to update OpenSearch instances software -title: Apply Amazon OpenSearch (ES) Software Update +name: Apply Amazon OpenSearch software updates +about: Issue template for operator to update the service software version on all OpenSearch domains +title: Apply Amazon OpenSearch software updates labels: -,infra,no demo,operator,orange -type: Task +type: Chore _start: 2024-02-26T09:00 _period: 14 days --- diff --git a/.github/ISSUE_TEMPLATE/promotion.md b/.github/ISSUE_TEMPLATE/promotion.md index 7889785177..249bf194f3 100644 --- a/.github/ISSUE_TEMPLATE/promotion.md +++ b/.github/ISSUE_TEMPLATE/promotion.md @@ -1,9 +1,9 @@ --- -name: Promotion pull request -about: Issue template for promoting changes to stable deployments +name: Promotion +about: Issue template for promoting changes to stable deployments on a weekly basis title: Promotion labels: -,infra,no demo,operator,orange -type: Feature +type: Chore _start: 2024-02-27T09:00 _period: 7 days --- diff --git a/.github/ISSUE_TEMPLATE/prune_gitlab_backups.md b/.github/ISSUE_TEMPLATE/prune_gitlab_backups.md index a7676426ca..a0b985eab0 100644 --- a/.github/ISSUE_TEMPLATE/prune_gitlab_backups.md +++ b/.github/ISSUE_TEMPLATE/prune_gitlab_backups.md @@ -3,7 +3,7 @@ name: Prune GitLab data volume backups about: Issue template for the quarterly pruning of GitLab data volume snapshots title: Prune GitLab data volume backups labels: -,infra,operator,orange -type: Task +type: Chore _start: 2025-04-01T09:00 _period: 3 months --- diff --git a/.github/ISSUE_TEMPLATE/public_access_content_review.md b/.github/ISSUE_TEMPLATE/public_access_content_review.md index cefebe29d3..f8b49c4810 100644 --- a/.github/ISSUE_TEMPLATE/public_access_content_review.md +++ b/.github/ISSUE_TEMPLATE/public_access_content_review.md @@ -1,10 +1,10 @@ --- -name: Review publicly accessible content -about: Issue template for the system administrator to perform a review of publicly accessible content. +name: Monthly review of publicly accessible content +about: Issue template for the system administrator to perform a review of publicly accessible content on a monthly basis title: Monthly review of publicly accessible content labels: -,compliance,infra,doc,orange assignees: hannes-ucsc -type: Task +type: Chore _repository: DataBiosphere/azul-private _start: 2024-08-01T09:00 _period: 1 month diff --git a/.github/ISSUE_TEMPLATE/upgrade.md b/.github/ISSUE_TEMPLATE/upgrade.md index d51b6fff0d..c149772aff 100644 --- a/.github/ISSUE_TEMPLATE/upgrade.md +++ b/.github/ISSUE_TEMPLATE/upgrade.md @@ -1,14 +1,14 @@ --- -name: Dependency upgrades -about: Issue template for bi-weekly dependency upgrades -title: Upgrade dependencies -labels: orange,operator,infra,debt -type: Feature +name: Upgrade software dependencies +about: Issue template for the bi-weekly upgrade of Azul's software dependencies +title: Upgrade software dependencies +labels: -,orange,operator,infra,debt +type: Chore _start: 2023-11-27T09:00 _period: 14 days --- - [ ] Update [PyCharm image](https://github.com/DataBiosphere/azul-docker-pycharm) - - [ ] Bump [base image](https://hub.docker.com/_/debian/tags?name=bookworm) tag (only same Debian release), if possible + - [ ] Bump [base image](https://hub.docker.com/_/debian/tags?name=trixie) `-slim` tag (only same Debian release), if possible - [ ] Bump upstream version, if possible - [ ] Bump internal version - [ ] Remove unused dependencies with high or critical CVEs @@ -16,12 +16,12 @@ _period: 14 days - [ ] GH Action workflow succeeded - [ ] Image is available on [DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-pycharm/tags) - [ ] Update [BigQuery Emulator image](https://github.com/DataBiosphere/azul-bigquery-emulator) - - [ ] Bump [base image](https://hub.docker.com/_/debian/tags?name=bookworm) tag, if possible + - [ ] Bump [base image](https://hub.docker.com/_/debian/tags?name=trixie) `-slim` tag, if possible - [ ] Bump internal version - [ ] Push commit to GitHub (directly to `azul` branch, no PR needed) - [ ] GH Action workflow succeeded - [ ] Image is available on [DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-bigquery-emulator/tags) -- [ ] Create Azul PR, connected to this issue, with … +- [ ] Create Azul PR, linked to this issue, with … - [ ] … changes to `requirements*.txt` from open Dependabot PRs, one commit per PR - [ ] … upgrade direct Python dependencies, [reference the operator manual](https://github.com/DataBiosphere/azul/blob/develop/OPERATOR.rst#upgrade-direct-python-dependencies) for instructions or not applicable - [ ] … update to [Python](https://hub.docker.com/_/python/tags) (only patch versions) or no update available @@ -34,8 +34,8 @@ _period: 14 days - [ ] … update to [ClamAV image](https://hub.docker.com/r/clamav/clamav/tags) or no update available - [ ] … update to [GitLab AMI](https://github.com/DataBiosphere/azul/blob/develop/OPERATOR.rst#updating-the-ami-for-gitlab-instances) or no update available - [ ] … update to [Swagger UI](https://github.com/DataBiosphere/azul/blob/develop/OPERATOR.rst#updating-the-swagger-ui) or no update available -- [ ] Created tickets for any deferred updates to … - - [ ] … to next major or minor Python version or such ticket already exists - - [ ] … to next major Docker version or such ticket already exists - - [ ] … to next major or minor Terraform version or such ticket already exists - - [ ] … to next major OpenSearch version or such ticket already exists +- [ ] Created issues for any deferred updates to … + - [ ] … the next major or minor Python version or such an issue already exists + - [ ] … the next major Docker version or such an issue already exists + - [ ] … the next major or minor Terraform version or such an issue already exists + - [ ] … the next major OpenSearch version or such an issue already exists diff --git a/.github/ISSUE_TEMPLATE/vpn_certificate_renewals.md b/.github/ISSUE_TEMPLATE/vpn_certificate_renewals.md index a53b298342..3c9602a266 100644 --- a/.github/ISSUE_TEMPLATE/vpn_certificate_renewals.md +++ b/.github/ISSUE_TEMPLATE/vpn_certificate_renewals.md @@ -1,15 +1,15 @@ --- name: Renew server and client VPN certificates -about: Issue template for the quarterly renewals of VPN certificates +about: Issue template for the quarterly renewal of all VPN certificates title: Renew VPN server and client certificates labels: -,infra,orange -type: Task +type: Chore _start: 2025-08-01T09:00 _period: 3 months --- - [ ] Deploy `tempdev.gitlab` -- [ ] Assign this ticket to the system administrator +- [ ] Assign this issue to the system administrator - [ ] Renew server certificate on `tempdev` - [ ] Renew client certificates on `tempdev` - [ ] Hibernate `tempdev` diff --git a/.github/ISSUE_TEMPLATE/web_app_vulnerability_scan.md b/.github/ISSUE_TEMPLATE/web_app_vulnerability_scan.md index dab5944b3f..88b10604da 100644 --- a/.github/ISSUE_TEMPLATE/web_app_vulnerability_scan.md +++ b/.github/ISSUE_TEMPLATE/web_app_vulnerability_scan.md @@ -1,9 +1,9 @@ --- -name: Run the web app vulnerability scans -about: Issue template for the monthly scanning and triaging of web app vulnerabilities +name: Monthly web app vulnerability scans +about: Issue template for the monthly scanning and triaging of web application vulnerabilities title: Monthly web app vulnerability scans -labels: +,compliance,infra,no demo,orange -type: Task +labels: -,compliance,infra,no demo,orange +type: Chore _repository: DataBiosphere/azul-private _start: 2025-06-01T09:00 _period: 1 month diff --git a/.github/PULL_REQUEST_TEMPLATE/anvilprod-hotfix.md b/.github/PULL_REQUEST_TEMPLATE/anvilprod-hotfix.md index 57a63f4e34..091793ce50 100644 --- a/.github/PULL_REQUEST_TEMPLATE/anvilprod-hotfix.md +++ b/.github/PULL_REQUEST_TEMPLATE/anvilprod-hotfix.md @@ -2,7 +2,7 @@ This is the PR template for hotfix PRs against `anvilprod`. --> -Connected issue: #0000 +Linked issue: #0000 ## Checklist @@ -10,12 +10,13 @@ Connected issue: #0000 ### Author +- [ ] PR is assigned to the author - [ ] Target branch is `anvilprod` - [ ] Name of PR branch matches `hotfixes//--anvilprod` -- [ ] On ZenHub, PR is connected to the issue it hotfixes +- [ ] PR is linked to the issue it hotfixes - [ ] PR description links to connected issue -- [ ] PR title is `Hotfix anvilprod: ` followed by title of connected issue -- [ ] PR title references the connected issue +- [ ] PR title is `Hotfix anvilprod: ` followed by title of linked issue +- [ ] PR title references the linked issue ### Author (hotfixes) @@ -29,9 +30,13 @@ Connected issue: #0000 ### Author (before every review) - [ ] Rebased PR branch on `anvilprod`, squashed fixups from prior reviews -- [ ] Ran `make requirements_update` or this PR does not modify `requirements*.txt`, `common.mk`, `Makefile` and `Dockerfile` +- [ ] Ran `make requirements_update` or this PR does not modify `requirements*.txt`, `common.mk`, `Makefile`, `Dockerfile` or `environment.boot` - [ ] Added `R` tag to commit title or this PR does not modify `requirements*.txt` - [ ] This PR is labeled `reqs` or does not modify `requirements*.txt` +- [ ] PR is not a draft +- [ ] PR is awaiting requested review from system administrator +- [ ] Status of PR is *Review requested* +- [ ] PR is assigned to only the system administrator ### System administrator (after approval) @@ -40,34 +45,48 @@ Connected issue: #0000 - [ ] Decided if PR can be labeled `no sandbox` - [ ] A comment to this PR details the completed security design review - [ ] PR title is appropriate as title of merge commit -- [ ] Moved connected issue to *Approved* column +- [ ] Status of PR is *Approved* - [ ] PR is assigned to only the operator -### Operator (before pushing merge the commit) +### Operator - [ ] Squashed PR branch and rebased onto `anvilprod` - [ ] Sanity-checked history - [ ] Pushed PR branch to GitHub + + +### Operator (deploy runner image) + +- [ ] Ran `_select anvilprod.gitlab && make -C terraform/gitlab/runner` or this PR is not labeled `deploy:runner` + + +### Operator (sandbox build) + - [ ] Added `sandbox` label or PR is labeled `no sandbox` - [ ] Pushed PR branch to GitLab `anvilprod` or PR is labeled `no sandbox` - [ ] Build passes in `hammerbox` deployment or PR is labeled `no sandbox` - [ ] Reviewed build logs for anomalies in `hammerbox` deployment or PR is labeled `no sandbox` + + +### Operator (merge the branch) + - [ ] All status checks passed and the PR is mergeable - [ ] The title of the merge commit starts with the title of this PR - [ ] Added PR # reference to merge commit title - [ ] Collected commit title tags in merge commit title but excluded any `p` tags -- [ ] Moved connected issue to *Merged stable* column in ZenHub - [ ] Pushed merge commit to GitHub +- [ ] Status of PR is *Merged stable* -### Operator (after pushing the merge commit) +### Operator (main build) - [ ] Pushed merge commit to GitLab `anvilprod` - [ ] Build passes on GitLab `anvilprod` - [ ] Reviewed build logs for anomalies on GitLab `anvilprod` - [ ] Deleted PR branch from GitHub - [ ] Deleted PR branch from GitLab `anvilprod` +- [ ] Status of linked issue is *Stable* ### Operator (reindex) diff --git a/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md b/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md index acfeda1390..5aeb91a546 100644 --- a/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md +++ b/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md @@ -2,7 +2,7 @@ This is the PR template for a promotion PR against `anvilprod`. --> -Connected issue: #0000 +Linked issue: #0000 ## Checklist @@ -10,17 +10,17 @@ Connected issue: #0000 ### Author +- [ ] PR is assigned to the author - [ ] Target branch is `anvilprod` - [ ] Name of PR branch matches `promotions/yyyy-mm-dd-anvilprod` -- [ ] On ZenHub, PR is connected to the promotion issue it resolves +- [ ] PR is linked to the promotion issue it resolves - [ ] PR description links to connected issue -- [ ] Title of connected issue matches `Promotion yyyy-mm-dd` -- [ ] PR title starts with title of connected issue followed by ` anvilprod` -- [ ] PR title references the connected issue -- [ ] The promoted issues are part of the same sprint as the connected issue +- [ ] Title of linked issue matches `Promotion yyyy-mm-dd` +- [ ] PR title starts with title of linked issue followed by ` anvilprod` +- [ ] PR title references the linked issue -### Author (reindex, API changes) +### Author (reindex) - [ ] This PR is labeled `reindex:anvilprod` or the changes introduced by it will not require reindexing of `anvilprod` - [ ] This PR is labeled `reindex:partial` and its description documents the specific reindexing procedure for `anvilprod` or requires a full reindex or is not labeled`reindex:anvilprod` @@ -37,19 +37,27 @@ Connected issue: #0000 ### Author (before every review) - [ ] PR branch is up to date (if not, merge `anvilprod` into PR branch to integrate upstream changes) +- [ ] PR is not a draft +- [ ] PR is awaiting requested review from system administrator +- [ ] Status of PR is *Review requested* +- [ ] PR is assigned to only the system administrator ### System administrator (after approval) - [ ] Actually approved the PR - [ ] Decided if PR can be labeled `no sandbox` -- [ ] Moved connected issue to *Approved* column +- [ ] Status of PR is *Approved* - [ ] PR is assigned to only the operator -### Operator (before pushing merge the commit) +### Operator - [ ] Pushed PR branch to GitHub + + +### Operator (deploy `.shared` and `.gitlab` components) + - [ ] Ran `_select anvilprod.shared && CI_COMMIT_REF_NAME=anvilprod make -C terraform/shared apply_keep_unused` or this PR is not labeled `deploy:shared` - [ ] Ran `_select anvilprod.gitlab && python scripts/create_gitlab_snapshot.py --no-restart` (see [operator manual](../blob/develop/OPERATOR.rst#backup-gitlab-volumes) for details) or this PR is not labeled `backup:gitlab` - [ ] Ran `_select anvilprod.gitlab && CI_COMMIT_REF_NAME=anvilprod make -C terraform/gitlab apply` or this PR is not labeled `deploy:gitlab` @@ -57,27 +65,36 @@ Connected issue: #0000 - [ ] PR is assigned to only the system administrator or this PR is not labeled `deploy:gitlab` -### System administrator +### System administrator (post-deploy of `.gitlab` component) - [ ] Background migrations for [`anvilprod.gitlab`](https://gitlab.explore.anvilproject.org/admin/background_migrations) are complete or this PR is not labeled `deploy:gitlab` - [ ] PR is assigned to only the operator -### Operator (before pushing merge the commit) +### Operator (deploy runner image) - [ ] Ran `_select anvilprod.gitlab && make -C terraform/gitlab/runner` or this PR is not labeled `deploy:runner` + + +### Operator (sandbox build) + - [ ] Added `sandbox` label or PR is labeled `no sandbox` - [ ] Pushed PR branch to GitLab `anvilprod` or PR is labeled `no sandbox` - [ ] Build passes in `hammerbox` deployment or PR is labeled `no sandbox` - [ ] Reviewed build logs for anomalies in `hammerbox` deployment or PR is labeled `no sandbox` + + +### Operator (merge the branch) + - [ ] All status checks passed and the PR is mergeable - [ ] The title of the merge commit starts with the title of this PR - [ ] Added PR # reference to merge commit title - [ ] Collected commit title tags in merge commit title but excluded any `p` tags - [ ] Pushed merge commit to GitHub +- [ ] Status of PR is *Merged stable* -### Operator (after pushing the merge commit) +### Operator (main build) - [ ] Pushed merge commit to GitLab `anvilprod` - [ ] Build passes on GitLab `anvilprod` @@ -85,9 +102,14 @@ Connected issue: #0000 - [ ] Ran `_select anvilprod.shared && make -C terraform/shared apply` or this PR is not labeled `deploy:shared` - [ ] Deleted PR branch from GitHub - [ ] Deleted PR branch from GitLab `anvilprod` -- [ ] Moved connected issue to *Merged stable* column on ZenHub -- [ ] Moved promoted issues from *Merged lower* to *Merged stable* column on ZenHub -- [ ] Moved promoted issues from *Lower* to *Stable* column on ZenHub +- [ ] Status of linked issue is *Stable* +- [ ] Status of promoted1 PRs is *Merged stable* +- [ ] Status of promoted1 issues is *Stable* + +1 Promoted issues and PRs are referenced in the titles of the commits +that the promotion branch introduces to the stable branch. Prior to the +promotion, the status of promoted issues (PRs) is *Lower* (*Merged lower*). +Promoted PRs in status *Done* do not need to be moved. ### Operator (reindex) diff --git a/.github/PULL_REQUEST_TEMPLATE/backport.md b/.github/PULL_REQUEST_TEMPLATE/backport.md index 69bdf06619..1d946634b9 100644 --- a/.github/PULL_REQUEST_TEMPLATE/backport.md +++ b/.github/PULL_REQUEST_TEMPLATE/backport.md @@ -8,6 +8,7 @@ This is the PR template for backport PRs against `develop`. ### Author +- [ ] PR is assigned to the author - [ ] Target branch is `develop` - [ ] Name of PR branch matches `backports/<7-digit SHA1 of most recent backported commit>` - [ ] PR title contains the 7-digit SHA1 of the backported commits @@ -18,9 +19,13 @@ This is the PR template for backport PRs against `develop`. ### Author (before every review) - [ ] PR branch is up to date (if not, merge `develop` into PR branch to integrate upstream changes) -- [ ] Ran `make requirements_update` or this PR does not modify `requirements*.txt`, `common.mk`, `Makefile` and `Dockerfile` +- [ ] Ran `make requirements_update` or this PR does not modify `requirements*.txt`, `common.mk`, `Makefile`, `Dockerfile` or `environment.boot` - [ ] Added `R` tag to commit title or this PR does not modify `requirements*.txt` - [ ] This PR is labeled `reqs` or does not modify `requirements*.txt` +- [ ] PR is not a draft +- [ ] PR is awaiting requested review from system administrator +- [ ] Status of PR is *Review requested* +- [ ] PR is assigned to only the system administrator ### System administrator (after approval) @@ -28,14 +33,24 @@ This is the PR template for backport PRs against `develop`. - [ ] Actually approved the PR - [ ] Decided if PR can be labeled `no sandbox` - [ ] PR title is appropriate as title of merge commit -- [ ] Moved connected issue to *Approved* column +- [ ] Status of PR is *Approved* - [ ] PR is assigned to only the operator -### Operator (before pushing merge the commit) +### Operator - [ ] Sanity-checked history - [ ] Pushed PR branch to GitHub + + +### Operator (deploy runner image) + +- [ ] Ran `_select dev.gitlab && make -C terraform/gitlab/runner` or this PR is not labeled `deploy:runner` +- [ ] Ran `_select anvildev.gitlab && make -C terraform/gitlab/runner` or this PR is not labeled `deploy:runner` + + +### Operator (sandbox build) + - [ ] Added `sandbox` label or PR is labeled `no sandbox` - [ ] Pushed PR branch to GitLab `dev` or PR is labeled `no sandbox` - [ ] Pushed PR branch to GitLab `anvildev` or PR is labeled `no sandbox` @@ -43,14 +58,19 @@ This is the PR template for backport PRs against `develop`. - [ ] Build passes in `anvilbox` deployment or PR is labeled `no sandbox` - [ ] Reviewed build logs for anomalies in `sandbox` deployment or PR is labeled `no sandbox` - [ ] Reviewed build logs for anomalies in `anvilbox` deployment or PR is labeled `no sandbox` + + +### Operator (merge the branch) + - [ ] All status checks passed and the PR is mergeable - [ ] The title of the merge commit starts with the title of this PR - [ ] Added PR # reference (to this PR) to merge commit title - [ ] Collected commit title tags in merge commit title but excluded any `p` tags - [ ] Pushed merge commit to GitHub +- [ ] Status of PR is *Merged lower* -### Operator (after pushing the merge commit) +### Operator (main build) - [ ] Pushed merge commit to GitLab `dev` - [ ] Pushed merge commit to GitLab `anvildev` @@ -61,6 +81,7 @@ This is the PR template for backport PRs against `develop`. - [ ] Deleted PR branch from GitHub - [ ] Deleted PR branch from GitLab `dev` - [ ] Deleted PR branch from GitLab `anvildev` +- [ ] Status of linked issue is *Lower* ### Operator diff --git a/.github/PULL_REQUEST_TEMPLATE/prod-hotfix.md b/.github/PULL_REQUEST_TEMPLATE/prod-hotfix.md index c072d04e86..b99e130c9e 100644 --- a/.github/PULL_REQUEST_TEMPLATE/prod-hotfix.md +++ b/.github/PULL_REQUEST_TEMPLATE/prod-hotfix.md @@ -2,7 +2,7 @@ This is the PR template for hotfix PRs against `prod`. --> -Connected issue: #0000 +Linked issue: #0000 ## Checklist @@ -10,12 +10,13 @@ Connected issue: #0000 ### Author +- [ ] PR is assigned to the author - [ ] Target branch is `prod` - [ ] Name of PR branch matches `hotfixes//--prod` -- [ ] On ZenHub, PR is connected to the issue it hotfixes +- [ ] PR is linked to the issue it hotfixes - [ ] PR description links to connected issue -- [ ] PR title is `Hotfix prod: ` followed by title of connected issue -- [ ] PR title references the connected issue +- [ ] PR title is `Hotfix prod: ` followed by title of linked issue +- [ ] PR title references the linked issue ### Author (hotfixes) @@ -29,9 +30,13 @@ Connected issue: #0000 ### Author (before every review) - [ ] Rebased PR branch on `prod`, squashed fixups from prior reviews -- [ ] Ran `make requirements_update` or this PR does not modify `requirements*.txt`, `common.mk`, `Makefile` and `Dockerfile` +- [ ] Ran `make requirements_update` or this PR does not modify `requirements*.txt`, `common.mk`, `Makefile`, `Dockerfile` or `environment.boot` - [ ] Added `R` tag to commit title or this PR does not modify `requirements*.txt` - [ ] This PR is labeled `reqs` or does not modify `requirements*.txt` +- [ ] PR is not a draft +- [ ] PR is awaiting requested review from system administrator +- [ ] Status of PR is *Review requested* +- [ ] PR is assigned to only the system administrator ### System administrator (after approval) @@ -40,29 +45,39 @@ Connected issue: #0000 - [ ] Labeled PR as `no sandbox` - [ ] A comment to this PR details the completed security design review - [ ] PR title is appropriate as title of merge commit -- [ ] Moved connected issue to *Approved* column +- [ ] Status of PR is *Approved* - [ ] PR is assigned to only the operator -### Operator (before pushing merge the commit) +### Operator - [ ] Squashed PR branch and rebased onto `prod` - [ ] Sanity-checked history - [ ] Pushed PR branch to GitHub + + +### Operator (deploy runner image) + +- [ ] Ran `_select prod.gitlab && make -C terraform/gitlab/runner` or this PR is not labeled `deploy:runner` + + +### Operator (merge the branch) + - [ ] All status checks passed and the PR is mergeable - [ ] The title of the merge commit starts with the title of this PR - [ ] Added PR # reference to merge commit title - [ ] Collected commit title tags in merge commit title but excluded any `p` tags -- [ ] Moved connected issue to *Merged stable* column in ZenHub - [ ] Pushed merge commit to GitHub +- [ ] Status of PR is *Merged stable* -### Operator (after pushing the merge commit) +### Operator (main build) - [ ] Pushed merge commit to GitLab `prod` - [ ] Build passes on GitLab `prod` - [ ] Reviewed build logs for anomalies on GitLab `prod` - [ ] Deleted PR branch from GitHub +- [ ] Status of linked issue is *Stable* ### Operator (reindex) diff --git a/.github/PULL_REQUEST_TEMPLATE/prod-promotion.md b/.github/PULL_REQUEST_TEMPLATE/prod-promotion.md index f2ef35324e..a3ef7be894 100644 --- a/.github/PULL_REQUEST_TEMPLATE/prod-promotion.md +++ b/.github/PULL_REQUEST_TEMPLATE/prod-promotion.md @@ -2,7 +2,7 @@ This is the PR template for a promotion PR against `prod`. --> -Connected issue: #0000 +Linked issue: #0000 ## Checklist @@ -10,17 +10,17 @@ Connected issue: #0000 ### Author +- [ ] PR is assigned to the author - [ ] Target branch is `prod` - [ ] Name of PR branch matches `promotions/yyyy-mm-dd-prod` -- [ ] On ZenHub, PR is connected to the promotion issue it resolves +- [ ] PR is linked to the promotion issue it resolves - [ ] PR description links to connected issue -- [ ] Title of connected issue matches `Promotion yyyy-mm-dd` -- [ ] PR title starts with title of connected issue followed by ` prod` -- [ ] PR title references the connected issue -- [ ] The promoted issues are part of the same sprint as the connected issue +- [ ] Title of linked issue matches `Promotion yyyy-mm-dd` +- [ ] PR title starts with title of linked issue followed by ` prod` +- [ ] PR title references the linked issue -### Author (reindex, API changes) +### Author (reindex) - [ ] This PR is labeled `reindex:prod` or the changes introduced by it will not require reindexing of `prod` - [ ] This PR is labeled `reindex:partial` and its description documents the specific reindexing procedure for `prod` or requires a full reindex or is not labeled`reindex:prod` @@ -37,19 +37,27 @@ Connected issue: #0000 ### Author (before every review) - [ ] PR branch is up to date (if not, merge `prod` into PR branch to integrate upstream changes) +- [ ] PR is not a draft +- [ ] PR is awaiting requested review from system administrator +- [ ] Status of PR is *Review requested* +- [ ] PR is assigned to only the system administrator ### System administrator (after approval) - [ ] Actually approved the PR - [ ] Labeled PR as `no sandbox` -- [ ] Moved connected issue to *Approved* column +- [ ] Status of PR is *Approved* - [ ] PR is assigned to only the operator -### Operator (before pushing merge the commit) +### Operator - [ ] Pushed PR branch to GitHub + + +### Operator (deploy `.shared` and `.gitlab` components) + - [ ] Ran `_select prod.shared && CI_COMMIT_REF_NAME=prod make -C terraform/shared apply_keep_unused` or this PR is not labeled `deploy:shared` - [ ] Ran `_select prod.gitlab && python scripts/create_gitlab_snapshot.py --no-restart` (see [operator manual](../blob/develop/OPERATOR.rst#backup-gitlab-volumes) for details) or this PR is not labeled `backup:gitlab` - [ ] Ran `_select prod.gitlab && CI_COMMIT_REF_NAME=prod make -C terraform/gitlab apply` or this PR is not labeled `deploy:gitlab` @@ -57,32 +65,42 @@ Connected issue: #0000 - [ ] PR is assigned to only the system administrator or this PR is not labeled `deploy:gitlab` -### System administrator +### System administrator (post-deploy of `.gitlab` component) - [ ] Background migrations for [`prod.gitlab`](https://gitlab.azul.data.humancellatlas.org/admin/background_migrations) are complete or this PR is not labeled `deploy:gitlab` - [ ] PR is assigned to only the operator -### Operator (before pushing merge the commit) +### Operator (deploy runner image) - [ ] Ran `_select prod.gitlab && make -C terraform/gitlab/runner` or this PR is not labeled `deploy:runner` + + +### Operator (merge the branch) + - [ ] All status checks passed and the PR is mergeable - [ ] The title of the merge commit starts with the title of this PR - [ ] Added PR # reference to merge commit title - [ ] Collected commit title tags in merge commit title but excluded any `p` tags - [ ] Pushed merge commit to GitHub +- [ ] Status of PR is *Merged stable* -### Operator (after pushing the merge commit) +### Operator (main build) - [ ] Pushed merge commit to GitLab `prod` - [ ] Build passes on GitLab `prod` - [ ] Reviewed build logs for anomalies on GitLab `prod` - [ ] Ran `_select prod.shared && make -C terraform/shared apply` or this PR is not labeled `deploy:shared` - [ ] Deleted PR branch from GitHub -- [ ] Moved connected issue to *Merged stable* column on ZenHub -- [ ] Moved promoted issues from *Merged lower* to *Merged stable* column on ZenHub -- [ ] Moved promoted issues from *Lower* to *Stable* column on ZenHub +- [ ] Status of linked issue is *Stable* +- [ ] Status of promoted1 PRs is *Merged stable* +- [ ] Status of promoted1 issues is *Stable* + +1 Promoted issues and PRs are referenced in the titles of the commits +that the promotion branch introduces to the stable branch. Prior to the +promotion, the status of promoted issues (PRs) is *Lower* (*Merged lower*). +Promoted PRs in status *Done* do not need to be moved. ### Operator (reindex) diff --git a/.github/PULL_REQUEST_TEMPLATE/upgrade.md b/.github/PULL_REQUEST_TEMPLATE/upgrade.md index ebd885fab9..535d72631c 100644 --- a/.github/PULL_REQUEST_TEMPLATE/upgrade.md +++ b/.github/PULL_REQUEST_TEMPLATE/upgrade.md @@ -2,7 +2,7 @@ This is the PR template for upgrading Azul dependencies. --> -Connected issue: #0000 +Linked issue: #0000 ## Checklist @@ -10,11 +10,12 @@ Connected issue: #0000 ### Author +- [ ] PR is assigned to the author - [ ] Target branch is `develop` - [ ] Name of PR branch matches `upgrades/yyyy-mm-dd` -- [ ] On ZenHub, PR is connected to the upgrade issue it resolves +- [ ] PR is linked to the upgrade issue it resolves - [ ] PR title matches `Upgrade dependencies yyyy-mm-dd` -- [ ] PR title references the connected issue +- [ ] PR title references the linked issue ### Author (upgrading deployments) @@ -32,27 +33,35 @@ Connected issue: #0000 ### Author (before every review) - [ ] Rebased PR branch on `develop`, squashed fixups from prior reviews -- [ ] Ran `make requirements_update` or this PR does not modify `requirements*.txt`, `common.mk`, `Makefile` and `Dockerfile` +- [ ] Ran `make requirements_update` or this PR does not modify `requirements*.txt`, `common.mk`, `Makefile`, `Dockerfile` or `environment.boot` - [ ] Added `R` tag to commit title or this PR does not modify `requirements*.txt` - [ ] This PR is labeled `reqs` or does not modify `requirements*.txt` - [ ] `make integration_test` passes in personal deployment or this PR does not modify functionality that could affect the IT outcome +- [ ] PR is not a draft +- [ ] PR is awaiting requested review from system administrator +- [ ] Status of PR is *Review requested* +- [ ] PR is assigned to only the system administrator ### System administrator (after approval) - [ ] Actually approved the PR -- [ ] Labeled connected issue as `no demo` +- [ ] Labeled linked issue as `no demo` - [ ] A comment to this PR details the completed security design review - [ ] PR title is appropriate as title of merge commit -- [ ] Moved connected issue to *Approved* column +- [ ] Status of PR is *Approved* - [ ] PR is assigned to only the operator -### Operator (before pushing merge the commit) +### Operator - [ ] Squashed PR branch and rebased onto `develop` - [ ] Sanity-checked history - [ ] Pushed PR branch to GitHub + + +### Operator (deploy `.shared` and `.gitlab` components) + - [ ] Ran `_select dev.shared && CI_COMMIT_REF_NAME=develop make -C terraform/shared apply_keep_unused` or this PR is not labeled `deploy:shared` - [ ] Ran `_select dev.gitlab && python scripts/create_gitlab_snapshot.py --no-restart` (see [operator manual](../blob/develop/OPERATOR.rst#backup-gitlab-volumes) for details) or this PR is not labeled `backup:gitlab` - [ ] Ran `_select dev.gitlab && CI_COMMIT_REF_NAME=develop make -C terraform/gitlab apply` or this PR is not labeled `deploy:gitlab` @@ -63,17 +72,21 @@ Connected issue: #0000 - [ ] PR is assigned to only the system administrator or this PR is not labeled `deploy:gitlab` -### System administrator +### System administrator (post-deploy of `.gitlab` component) - [ ] Background migrations for [`dev.gitlab`](https://gitlab.dev.singlecell.gi.ucsc.edu/admin/background_migrations) are complete or this PR is not labeled `deploy:gitlab` - [ ] Background migrations for [`anvildev.gitlab`](https://gitlab.anvil.gi.ucsc.edu/admin/background_migrations) are complete or this PR is not labeled `deploy:gitlab` - [ ] PR is assigned to only the operator -### Operator (before pushing merge the commit) +### Operator (deploy runner image) - [ ] Ran `_select dev.gitlab && make -C terraform/gitlab/runner` or this PR is not labeled `deploy:runner` - [ ] Ran `_select anvildev.gitlab && make -C terraform/gitlab/runner` or this PR is not labeled `deploy:runner` + + +### Operator (sandbox build) + - [ ] Added `sandbox` label - [ ] Pushed PR branch to GitLab `dev` - [ ] Pushed PR branch to GitLab `anvildev` @@ -81,17 +94,21 @@ Connected issue: #0000 - [ ] Build passes in `anvilbox` deployment - [ ] Reviewed build logs for anomalies in `sandbox` deployment - [ ] Reviewed build logs for anomalies in `anvilbox` deployment + + +### Operator (merge the branch) + - [ ] All status checks passed and the PR is mergeable - [ ] The title of the merge commit starts with the title of this PR - [ ] Added PR # reference to merge commit title - [ ] Collected commit title tags in merge commit title but excluded any `p` tags -- [ ] Moved connected issue to *Merged lower* column in ZenHub -- [ ] Moved blocked issues to *Triage* or no issues are blocked on the connected issue - [ ] Closed related Dependabot PRs with a comment referencing the corresponding commit in this PR or this PR does not include any such commits - [ ] Pushed merge commit to GitHub +- [ ] Status of PR is *Merged lower* +- [ ] Status of blocked issues is *Triage* or no issues are blocked on the linked issue -### Operator (after pushing the merge commit) +### Operator (main build) - [ ] Pushed merge commit to GitLab `dev` - [ ] Pushed merge commit to GitLab `anvildev` @@ -104,12 +121,13 @@ Connected issue: #0000 - [ ] Deleted PR branch from GitHub - [ ] Deleted PR branch from GitLab `dev` - [ ] Deleted PR branch from GitLab `anvildev` +- [ ] Status of linked issue is *Lower* ### Operator - [ ] At least 24 hours have passed since `anvildev.shared` was last deployed -- [ ] Ran `scripts/export_inspector_findings.py` against `anvildev`, imported results to [Google Sheet](https://docs.google.com/spreadsheets/d/1RWF7g5wRKWPGovLw4jpJGX_XMi8aWLXLOvvE5rxqgH8) and posted screenshot of relevant1 findings as a comment on the connected issue. +- [ ] Ran `scripts/export_inspector_findings.py` against `anvildev`, imported results to [Google Sheet](https://docs.google.com/spreadsheets/d/1RWF7g5wRKWPGovLw4jpJGX_XMi8aWLXLOvvE5rxqgH8) and posted screenshot of relevant1 findings as a comment on the linked issue. - [ ] Propagated the `deploy:shared`, `deploy:gitlab`, `deploy:runner` and `backup:gitlab` labels to the next promotion PRs or this PR carries none of these labels - [ ] Propagated any specific instructions related to the `deploy:shared`, `deploy:gitlab`, `deploy:runner` and `backup:gitlab` labels, from the description of this PR to that of the next promotion PRs or this PR carries none of these labels - [ ] PR is assigned to only the system administrator diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 053e2176d9..5ee77f6570 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -6,7 +6,7 @@ hotfix.md`, `&template=backport.md` or `&template=upgrade.md` to switch the template. --> -Connected issues: #0000 +Linked issues: #0000 ## Checklist @@ -14,14 +14,15 @@ Connected issues: #0000 ### Author +- [ ] PR is assigned to the author - [ ] PR is a draft - [ ] Target branch is `develop` - [ ] Name of PR branch matches `issues//-` -- [ ] On ZenHub, PR is connected to all issues it (partially) resolves +- [ ] PR is linked to all issues it (partially) resolves - [ ] PR description links to connected issues -- [ ] PR title matches1 that of a connected issue or comment in PR explains why they're different -- [ ] PR title references all connected issues -- [ ] For each connected issue, there is at least one commit whose title references that issue +- [ ] PR title matches1 that of a linked issue or comment in PR explains why they're different +- [ ] PR title references all linked issues +- [ ] For each linked issue, there is at least one commit whose title references that issue 1 when the issue title describes a problem, the corresponding PR title is `Fix: ` followed by the issue title @@ -30,18 +31,11 @@ title is `Fix: ` followed by the issue title ### Author (partiality) - [ ] Added `p` tag to titles of partial commits -- [ ] This PR is labeled `partial` or completely resolves all connected issues -- [ ] This PR partially resolves each of the connected issues or does not have the `partial` label +- [ ] This PR is labeled `partial` or completely resolves all linked issues +- [ ] This PR partially resolves each of the linked issues or does not have the `partial` label -### Author (chains) - -- [ ] This PR is blocked by previous PR in the chain or is not chained to another PR -- [ ] The blocking PR is labeled `base` or this PR is not chained to another PR -- [ ] This PR is labeled `chained` or is not chained to another PR - - -### Author (reindex, API changes) +### Author (reindex) - [ ] Added `r` tag to commit title or the changes introduced by this PR will not require reindexing of any deployment - [ ] This PR is labeled `reindex:dev` or the changes introduced by it will not require reindexing of `dev` @@ -49,7 +43,11 @@ title is `Fix: ` followed by the issue title - [ ] This PR is labeled `reindex:anvilprod` or the changes introduced by it will not require reindexing of `anvilprod` - [ ] This PR is labeled `reindex:prod` or the changes introduced by it will not require reindexing of `prod` - [ ] This PR is labeled `reindex:partial` and its description documents the specific reindexing procedure for `dev`, `anvildev`, `anvilprod` and `prod` or requires a full reindex or carries none of the labels `reindex:dev`, `reindex:anvildev`, `reindex:anvilprod` and `reindex:prod` -- [ ] This PR and its connected issues are labeled `API` or this PR does not modify a REST API + + +### Author (API changes) + +- [ ] This PR and its linked issues are labeled `API` or this PR does not modify a REST API - [ ] Added `a` (`A`) tag to commit title for backwards (in)compatible changes or this PR does not modify a REST API - [ ] Updated REST API version number in `app.py` or this PR does not modify a REST API @@ -68,47 +66,56 @@ title is `Fix: ` followed by the issue title ### Author (hotfixes) - [ ] Added `F` tag to main commit title or this PR does not include permanent fix for a temporary hotfix -- [ ] Reverted the temporary hotfixes for any connected issues or the none of the stable branches (`anvilprod` and `prod`) have temporary hotfixes for any of the issues connected to this PR +- [ ] Reverted the temporary hotfixes for any linked issues or the none of the stable branches (`anvilprod` and `prod`) have temporary hotfixes for any of the issues linked to this PR ### Author (before every review) - [ ] Rebased PR branch on `develop`, squashed fixups from prior reviews -- [ ] Ran `make requirements_update` or this PR does not modify `requirements*.txt`, `common.mk`, `Makefile` and `Dockerfile` +- [ ] Ran `make requirements_update` or this PR does not modify `requirements*.txt`, `common.mk`, `Makefile`, `Dockerfile` or `environment.boot` - [ ] Added `R` tag to commit title or this PR does not modify `requirements*.txt` - [ ] This PR is labeled `reqs` or does not modify `requirements*.txt` - [ ] `make integration_test` passes in personal deployment or this PR does not modify functionality that could affect the IT outcome +- [ ] PR is awaiting requested review from a peer +- [ ] Status of PR is *Review requested* +- [ ] PR is assigned to only the peer ### Peer reviewer (after approval) +Note that when requesting changes, the PR must be assigned back to the author. + - [ ] Actually approved the PR - [ ] PR is not a draft -- [ ] Ticket is in *Review requested* column - [ ] PR is awaiting requested review from system administrator +- [ ] Status of PR is *Review requested* - [ ] PR is assigned to only the system administrator ### System administrator (after approval) - [ ] Actually approved the PR -- [ ] Labeled connected issues as `demo` or `no demo` -- [ ] Commented on connected issues about demo expectations or all connected issues are labeled `no demo` +- [ ] Labeled linked issues as `demo` or `no demo` +- [ ] Commented on linked issues about demo expectations or all linked issues are labeled `no demo` - [ ] Decided if PR can be labeled `no sandbox` - [ ] A comment to this PR details the completed security design review - [ ] PR title is appropriate as title of merge commit - [ ] `N reviews` label is accurate -- [ ] Moved connected issues to *Approved* column +- [ ] Status of PR is *Approved* - [ ] PR is assigned to only the operator -### Operator (before pushing merge the commit) +### Operator - [ ] Checked `reindex:…` labels and `r` commit title tag -- [ ] Checked that demo expectations are clear or all connected issues are labeled `no demo` +- [ ] Checked that demo expectations are clear or all linked issues are labeled `no demo` - [ ] Squashed PR branch and rebased onto `develop` - [ ] Sanity-checked history - [ ] Pushed PR branch to GitHub + + +### Operator (deploy `.shared` and `.gitlab` components) + - [ ] Ran `_select dev.shared && CI_COMMIT_REF_NAME=develop make -C terraform/shared apply_keep_unused` or this PR is not labeled `deploy:shared` - [ ] Ran `_select dev.gitlab && CI_COMMIT_REF_NAME=develop make -C terraform/gitlab apply` or this PR is not labeled `deploy:gitlab` - [ ] Ran `_select anvildev.shared && CI_COMMIT_REF_NAME=develop make -C terraform/shared apply_keep_unused` or this PR is not labeled `deploy:shared` @@ -117,17 +124,21 @@ title is `Fix: ` followed by the issue title - [ ] PR is assigned to only the system administrator or this PR is not labeled `deploy:gitlab` -### System administrator +### System administrator (post-deploy of `.gitlab` component) - [ ] Background migrations for [`dev.gitlab`](https://gitlab.dev.singlecell.gi.ucsc.edu/admin/background_migrations) are complete or this PR is not labeled `deploy:gitlab` - [ ] Background migrations for [`anvildev.gitlab`](https://gitlab.anvil.gi.ucsc.edu/admin/background_migrations) are complete or this PR is not labeled `deploy:gitlab` - [ ] PR is assigned to only the operator -### Operator (before pushing merge the commit) +### Operator (deploy runner image) - [ ] Ran `_select dev.gitlab && make -C terraform/gitlab/runner` or this PR is not labeled `deploy:runner` - [ ] Ran `_select anvildev.gitlab && make -C terraform/gitlab/runner` or this PR is not labeled `deploy:runner` + + +### Operator (sandbox build) + - [ ] Added `sandbox` label or PR is labeled `no sandbox` - [ ] Pushed PR branch to GitLab `dev` or PR is labeled `no sandbox` - [ ] Pushed PR branch to GitLab `anvildev` or PR is labeled `no sandbox` @@ -141,24 +152,20 @@ title is `Fix: ` followed by the issue title - [ ] Started reindex in `anvilbox` or this PR is not labeled `reindex:anvildev` - [ ] Checked for failures in `sandbox` or this PR is not labeled `reindex:dev` - [ ] Checked for failures in `anvilbox` or this PR is not labeled `reindex:anvildev` + + +### Operator (merge the branch) + - [ ] All status checks passed and the PR is mergeable - [ ] The title of the merge commit starts with the title of this PR - [ ] Added PR # reference to merge commit title - [ ] Collected commit title tags in merge commit title but only included `p` if the PR is also labeled `partial` -- [ ] Moved connected issues to *Merged lower* column in ZenHub -- [ ] Moved blocked issues to *Triage* or no issues are blocked on the connected issues - [ ] Pushed merge commit to GitHub +- [ ] Status of PR is *Merged lower* +- [ ] Status of blocked issues is *Triage* or no issues are blocked on the linked issues -### Operator (chain shortening) - -- [ ] Changed the target branch of the blocked PR to `develop` or this PR is not labeled `base` -- [ ] Removed the `chained` label from the blocked PR or this PR is not labeled `base` -- [ ] Removed the blocking relationship from the blocked PR or this PR is not labeled `base` -- [ ] Removed the `base` label from this PR or this PR is not labeled `base` - - -### Operator (after pushing the merge commit) +### Operator (main build) - [ ] Pushed merge commit to GitLab `dev` - [ ] Pushed merge commit to GitLab `anvildev` @@ -171,6 +178,7 @@ title is `Fix: ` followed by the issue title - [ ] Deleted PR branch from GitHub - [ ] Deleted PR branch from GitLab `dev` - [ ] Deleted PR branch from GitLab `anvildev` +- [ ] Status of linked issues is *Lower* ### Operator (reindex) diff --git a/.github/pull_request_template.md.template.py b/.github/pull_request_template.md.template.py index ba2a7271e4..276417bed9 100644 --- a/.github/pull_request_template.md.template.py +++ b/.github/pull_request_template.md.template.py @@ -292,8 +292,7 @@ def emit(t: T, target_branch: str): }, iif(t is not T.backport, { 'type': 'p', - 'content': f'Connected {t.issues}: #0000' - + 'content': f'Linked {t.issues}: #0000' }), { 'type': 'h1', @@ -303,6 +302,10 @@ def emit(t: T, target_branch: str): 'type': 'h2', 'content': 'Author' }, + { + 'type': 'cli', + 'content': 'PR is assigned to the author' + }, iif(t is T.default, { 'type': 'cli', 'content': 'PR is a draft' @@ -324,10 +327,10 @@ def emit(t: T, target_branch: str): iif(t is not t.backport, { 'type': 'cli', 'content': { - T.default: 'On ZenHub, PR is connected to all issues it (partially) resolves', - T.upgrade: 'On ZenHub, PR is connected to the upgrade issue it resolves', - T.hotfix: 'On ZenHub, PR is connected to the issue it hotfixes', - T.promotion: 'On ZenHub, PR is connected to the promotion issue it resolves', + T.default: 'PR is linked to all issues it (partially) resolves', + T.upgrade: 'PR is linked to the upgrade issue it resolves', + T.hotfix: 'PR is linked to the issue it hotfixes', + T.promotion: 'PR is linked to the promotion issue it resolves', T.backport: None }[t] }), @@ -337,16 +340,16 @@ def emit(t: T, target_branch: str): }), iif(t is T.promotion, { 'type': 'cli', - 'content': 'Title of connected issue matches `Promotion yyyy-mm-dd`' + 'content': 'Title of linked issue matches `Promotion yyyy-mm-dd`' }), { 'type': 'cli', 'content': { - t.default: 'PR title matches that of a connected issue', - t.promotion: f'PR title starts with title of connected issue ' + t.default: 'PR title matches that of a linked issue', + t.promotion: f'PR title starts with title of linked issue ' f'followed by ` {target_branch}`', t.hotfix: f'PR title is `Hotfix {target_branch}: ` ' - f'followed by title of connected issue', + f'followed by title of linked issue', t.upgrade: 'PR title matches `Upgrade dependencies yyyy-mm-dd`', t.backport: 'PR title contains the 7-digit SHA1 of the backported commits' }[t], @@ -354,12 +357,7 @@ def emit(t: T, target_branch: str): }, iif(t is not T.backport, { 'type': 'cli', - 'content': f'PR title references {t.issues('all', 'the')} connected {t.issues}' - }), - iif(t is T.promotion, { - 'type': 'cli', - 'content': 'The promoted issues are part of the same sprint as the connected ' - 'issue' + 'content': f'PR title references {t.issues('all', 'the')} linked {t.issues}' }), *( [ @@ -378,7 +376,7 @@ def emit(t: T, target_branch: str): *iif(t is T.default, [ { 'type': 'cli', - 'content': 'For each connected issue, there is at least one commit whose title ' + 'content': 'For each linked issue, there is at least one commit whose title ' 'references that issue' }, { @@ -397,39 +395,18 @@ def emit(t: T, target_branch: str): { 'type': 'cli', 'content': 'This PR is labeled `partial`', - 'alt': 'or completely resolves all connected issues' + 'alt': 'or completely resolves all linked issues' }, { 'type': 'cli', - 'content': 'This PR partially resolves each of the connected issues', + 'content': 'This PR partially resolves each of the linked issues', 'alt': 'or does not have the `partial` label' } ]), - *iif(t is T.default, [ - { - 'type': 'h2', - 'content': 'Author (chains)' - }, - { - 'type': 'cli', - 'content': 'This PR is blocked by previous PR in the chain', - 'alt': 'or is not chained to another PR' - }, - { - 'type': 'cli', - 'content': 'The blocking PR is labeled `base`', - 'alt': 'or this PR is not chained to another PR' - }, - { - 'type': 'cli', - 'content': 'This PR is labeled `chained`', - 'alt': 'or is not chained to another PR' - } - ]), *iif(t in (T.default, T.promotion), [ { 'type': 'h2', - 'content': 'Author (reindex, API changes)' + 'content': 'Author (reindex)' }, iif(t is T.default, { 'type': 'cli', @@ -465,9 +442,13 @@ def emit(t: T, target_branch: str): ) }, *iif(t is T.default, [ + { + 'type': 'h2', + 'content': 'Author (API changes)' + }, { 'type': 'cli', - 'content': 'This PR and its connected issues are labeled `API`', + 'content': 'This PR and its linked issues are labeled `API`', 'alt': 'or this PR does not modify a REST API' }, { @@ -545,12 +526,14 @@ def emit(t: T, target_branch: str): }, { 'type': 'cli', - 'content': 'Reverted the temporary hotfixes for any connected issues', + 'content': 'Reverted the temporary hotfixes for any linked issues', 'alt': 'or the none of the stable branches (' + join_grammatically(list(map(bq, T.promotion.target_branches))) + - ') have temporary hotfixes for any of the issues connected to this PR' + ') have temporary hotfixes for any of the issues linked to this PR' } - ] if t is T.default else [ + ] + if t is T.default else + [ { 'type': 'cli', 'content': 'Added `h` tag to commit title', @@ -570,8 +553,10 @@ def emit(t: T, target_branch: str): 'content': 'This PR is labeled `partial`', 'alt': 'or represents a permanent hotfix' }, - ] if t is T.hotfix else [ - ]), + ] + if t is T.hotfix else + [] + ), ]), { 'type': 'h2', @@ -586,11 +571,12 @@ def emit(t: T, target_branch: str): ), f'Rebased PR branch on `{target_branch}`, squashed fixups from prior reviews') }, - *iif(t is not T.promotion, [ + *iif(target_branch == 'develop' or t is T.hotfix, [ { 'type': 'cli', 'content': 'Ran `make requirements_update`', - 'alt': 'or this PR does not modify `requirements*.txt`, `common.mk`, `Makefile` and `Dockerfile`' + 'alt': 'or this PR does not modify `requirements*.txt`, ' + '`common.mk`, `Makefile`, `Dockerfile` or `environment.boot`' }, { 'type': 'cli', @@ -602,7 +588,7 @@ def emit(t: T, target_branch: str): 'content': 'This PR is labeled `reqs`', 'alt': 'or does not modify `requirements*.txt`' }, - iif(t in (T.default, T.upgrade), { + iif(t not in (T.backport, T.hotfix), { 'type': 'cli', 'content': '`make integration_test` passes in personal deployment', 'alt': 'or this PR does not modify functionality that could affect the IT outcome' @@ -610,30 +596,46 @@ def emit(t: T, target_branch: str): ]), *iif(t is T.default, [ { - 'type': 'h2', - 'content': 'Peer reviewer (after approval)' + 'type': 'cli', + 'content': 'PR is awaiting requested review from a peer' }, { 'type': 'cli', - 'content': 'Actually approved the PR' + 'content': 'Status of PR is *Review requested*' }, { 'type': 'cli', - 'content': 'PR is not a draft' + 'content': 'PR is assigned to only the peer' }, { - 'type': 'cli', - 'content': 'Ticket is in *Review requested* column' + 'type': 'h2', + 'content': 'Peer reviewer (after approval)' }, { - 'type': 'cli', - 'content': 'PR is awaiting requested review from system administrator' + 'type': 'p', + 'content': 'Note that when requesting changes, the PR must be assigned back to the author.' }, { 'type': 'cli', - 'content': 'PR is assigned to only the system administrator' - } + 'content': 'Actually approved the PR' + }, ]), + { + 'type': 'cli', + 'content': 'PR is not a draft' + }, + { + 'type': 'cli', + 'content': 'PR is awaiting requested review from system administrator' + }, + { + 'type': 'cli', + 'content': 'Status of PR is *Review requested*' + }, + { + 'type': 'cli', + 'content': 'PR is assigned to only the system administrator' + }, { 'type': 'h2', 'content': 'System administrator (after approval)' @@ -644,16 +646,16 @@ def emit(t: T, target_branch: str): }, iif(t is T.default, { 'type': 'cli', - 'content': 'Labeled connected issues as `demo` or `no demo`' + 'content': 'Labeled linked issues as `demo` or `no demo`' }), iif(t is T.upgrade, { 'type': 'cli', - 'content': 'Labeled connected issue as `no demo`' + 'content': 'Labeled linked issue as `no demo`' }), iif(t is T.default, { 'type': 'cli', - 'content': 'Commented on connected issues about demo expectations', - 'alt': 'or all connected issues are labeled `no demo`' + 'content': 'Commented on linked issues about demo expectations', + 'alt': 'or all linked issues are labeled `no demo`' }), iif(t is not T.upgrade, { 'type': 'cli', @@ -677,7 +679,7 @@ def emit(t: T, target_branch: str): }), { 'type': 'cli', - 'content': f'Moved connected {t.issues} to *Approved* column' + 'content': 'Status of PR is *Approved*' }, { 'type': 'cli', @@ -685,7 +687,7 @@ def emit(t: T, target_branch: str): }, { 'type': 'h2', - 'content': 'Operator (before pushing merge the commit)' + 'content': 'Operator' }, *iif(t is T.default, [ { @@ -695,7 +697,7 @@ def emit(t: T, target_branch: str): { 'type': 'cli', 'content': 'Checked that demo expectations are clear', - 'alt': 'or all connected issues are labeled `no demo`' + 'alt': 'or all linked issues are labeled `no demo`' } ]), iif(t not in (T.promotion, T.backport), { @@ -711,6 +713,10 @@ def emit(t: T, target_branch: str): 'content': 'Pushed PR branch to GitHub' }, *iif(t.needs_shared_deploy, [ + { + 'type': 'h2', + 'content': 'Operator (deploy `.shared` and `.gitlab` components)' + }, *flatten([ [ { @@ -755,7 +761,7 @@ def emit(t: T, target_branch: str): }, { 'type': 'h2', - 'content': 'System administrator' + 'content': 'System administrator (post-deploy of `.gitlab` component)' }, *[ { @@ -771,27 +777,33 @@ def emit(t: T, target_branch: str): 'type': 'cli', 'content': 'PR is assigned to only the operator', }, + ]), + { + 'type': 'h2', + 'content': 'Operator (deploy runner image)' + }, + *[ + { + 'type': 'cli', + 'content': 'Ran ' + bq( + f'_select {d}.gitlab && ' + f'make -C terraform/gitlab/runner' + ), + 'alt': 'or this PR is not labeled `deploy:runner`' + } + for d in t.target_deployments(target_branch) + ], + *iif(t.has_sandbox_for(target_branch), [ { 'type': 'h2', - 'content': 'Operator (before pushing merge the commit)' + 'content': 'Operator (sandbox build)' }, - *[ - { - 'type': 'cli', - 'content': 'Ran ' + bq( - f'_select {d}.gitlab && ' - f'make -C terraform/gitlab/runner' - ), - 'alt': 'or this PR is not labeled `deploy:runner`' - } - for d in t.target_deployments(target_branch) - ], + { + 'type': 'cli', + 'content': 'Added `sandbox` label', + 'alt': iif(t is T.upgrade, None, 'or PR is labeled `no sandbox`') + } ]), - iif(t.has_sandbox_for(target_branch), { - 'type': 'cli', - 'content': 'Added `sandbox` label', - 'alt': iif(t is T.upgrade, None, 'or PR is labeled `no sandbox`') - }), # zip() is used to interleave the steps for each deployment so # that first, step 1 is done for all deployments, then step 2 # for all of them, and so on. @@ -834,6 +846,10 @@ def emit(t: T, target_branch: str): for i, (d, s) in enumerate(t.target_deployments(target_branch).items()) if s is not None ))), + { + 'type': 'h2', + 'content': 'Operator (merge the branch)' + }, { 'type': 'cli', 'content': 'All status checks passed and the PR is mergeable' @@ -855,17 +871,6 @@ def emit(t: T, target_branch: str): 'but only included `p` if the PR is also labeled `partial`', 'but excluded any `p` tags') }, - iif(t in (T.default, T.upgrade, T.hotfix), { - 'type': 'cli', - 'content': iif(t is t.hotfix, - 'Moved connected issue to *Merged stable* column in ZenHub', - f'Moved connected {t.issues} to *Merged lower* column in ZenHub') - }), - iif(target_branch == 'develop' and t is not T.backport, { - 'type': 'cli', - 'content': 'Moved blocked issues to *Triage*', - 'alt': f'or no issues are blocked on the connected {t.issues}' - }), iif(t is T.upgrade, { 'type': 'cli', @@ -878,28 +883,19 @@ def emit(t: T, target_branch: str): 'type': 'cli', 'content': 'Pushed merge commit to GitHub' }, - *iif(t is T.default, [ - { - 'type': 'h2', - 'content': 'Operator (chain shortening)' - }, - *[ - { - 'type': 'cli', - 'content': content, - 'alt': 'or this PR is not labeled `base`' - } - for content in [ - f'Changed the target branch of the blocked PR to {bq(target_branch)}', - 'Removed the `chained` label from the blocked PR', - 'Removed the blocking relationship from the blocked PR', - 'Removed the `base` label from this PR' - ] - ] - ]), + { + 'type': 'cli', + 'content': f'Status of PR is ' + f'*Merged {'lower' if target_branch == 'develop' else 'stable'}*' + }, + iif(target_branch == 'develop' and t is not T.backport, { + 'type': 'cli', + 'content': 'Status of blocked issues is *Triage*', + 'alt': f'or no issues are blocked on the linked {t.issues}' + }), { 'type': 'h2', - 'content': 'Operator (after pushing the merge commit)' + 'content': 'Operator (main build)' }, *[ { @@ -944,20 +940,38 @@ def emit(t: T, target_branch: str): for d, s in t.target_deployments(target_branch).items() if s is not None ), - *iif(t is T.promotion, [ - { - 'type': 'cli', - 'content': 'Moved connected issue to *Merged stable* column on ZenHub' - }, - { - 'type': 'cli', - 'content': 'Moved promoted issues from *Merged lower* to *Merged stable* column on ZenHub' - }, - { - 'type': 'cli', - 'content': 'Moved promoted issues from *Lower* to *Stable* column on ZenHub' - } - ]), + *( + [ + { + 'type': 'cli', + 'content': f'Status of linked {t.issues} is ' + f'*{'Lower' if target_branch == 'develop' else 'Stable'}*' + } + ] + if t is not T.promotion else + [ + { + 'type': 'cli', + 'content': 'Status of linked issue is *Stable*' + }, + { + 'type': 'cli', + 'content': 'Status of promoted PRs is *Merged stable*' + }, + { + 'type': 'cli', + 'content': 'Status of promoted issues is *Stable*' + }, + { + 'type': 'p', + 'content': ' Promoted issues and PRs are referenced in ' + 'the titles of the commits that the promotion branch introduces to ' + 'the stable branch. Prior to the promotion, the status of promoted ' + 'issues (PRs) is *Lower* (*Merged lower*). Promoted PRs in status ' + '*Done* do not need to be moved.' + } + ] + ), *iif(t in (T.default, T.hotfix, T.promotion), [ { 'type': 'h2', @@ -1072,7 +1086,7 @@ def emit(t: T, target_branch: str): 'content': 'Ran `scripts/export_inspector_findings.py` against `anvildev`, imported results ' 'to [Google Sheet](https://docs.google.com/spreadsheets/d/' '1RWF7g5wRKWPGovLw4jpJGX_XMi8aWLXLOvvE5rxqgH8) and posted screenshot of ' - 'relevant findings as a comment on the connected issue.' + 'relevant findings as a comment on the linked issue.' } ]), *iif(target_branch == 'develop' and t is not T.backport, [ diff --git a/docker_images.json b/docker_images.json index 0db70600df..a52978c03a 100644 --- a/docker_images.json +++ b/docker_images.json @@ -15,66 +15,66 @@ } } }, - "docker.io/library/python:3.12.11-slim-bookworm": { - "digest": "sha256:3ad2a947749a3eb74acd9e00636ffa0def5aae0bbbd9fa4fff6253e404e2fe15", - "mirror_digest": "sha256:a9773611e52fb17803b9c3f50b2ff2fb027fe46e35d3fed4dfeadc8e06c5c847", + "docker.io/library/python:3.12.11-slim-trixie": { + "digest": "sha256:abc799c7ee22b0d66f46c367643088a35e048bbabd81212d73c2323aed38c64f", + "mirror_digest": "sha256:25cb89337ae6946bb067f236617afd4dea60339b2349cd88ae78070d9f30dd10", "parts": { "linux/amd64": { - "digest": "sha256:f8fcabde4d56a318e6c117d02868546f7b033baae99488721782d4cae727edc3", - "id": "sha256:c762e87affa389d08d073446be444f06c9bef07d687687f6bd79a86a7c223525", + "digest": "sha256:a2dd6679f8a13a533f44782cd31584d4ca4ec7170c84e7e726dc9cc3f862ed42", + "id": "sha256:f35c889e4f8eec7a5ccec31926201e4a63df72a87cbd59e7af1bfe2689397354", "platform": "linux/amd64" }, "linux/arm64": { - "digest": "sha256:41816cb704043c3aab4d2a88c92fe1955ef99f05080451feb57159eb9e5055ef", - "id": "sha256:16004f71935df31ed395523696c406cda39f930985d98354b92ce4c479abfd9b", + "digest": "sha256:0467c51f4df7a7f4437d09713b2a6e17bdef2c2f97fb76a2292759ab8e4a33ed", + "id": "sha256:12201b64d1f192c962c7b74947706000df3ef3f994cf6f4e3206060938a0faa0", "platform": "linux/arm64" } } }, - "docker.io/ucscgi/azul-pycharm:2024.3.6-61": { - "digest": "sha256:f91b917a9c3241d252d85ef57d104c6a288427b67ea5e6a66d0f01845a713481", - "mirror_digest": "sha256:4308c62acc87cf0d9a10d73c7afcee2b443dc85f9e95c0c7f74b030715a30293", + "docker.io/ucscgi/azul-pycharm:2024.3.6-62": { + "digest": "sha256:d6565a6e2c95754ae6774c55d39ab0045f6d745a22dd5087cf97eb3d520359e7", + "mirror_digest": "sha256:33eb850387b77426464716fa4e1b7d645fc76e426b4e877c555a7241a50f5dc9", "parts": { "linux/amd64": { - "digest": "sha256:7475eaf0ce25afeb6b93c2101e7681d4de3648e52dac6b37fb66ea03e1f2eafe", - "id": "sha256:860ca80be478e21a5b5654ca70dad6d358ed9957d8eedd409cdad64ad495dd9c", + "digest": "sha256:492330e0cc96d887aad121894f951c565f9a79595e1866dc986f2566a2e3dfb3", + "id": "sha256:1e74bfc887188e1b1d63c6535ba0f767b479a3ff9e403362a5fb52d17d9723cf", "platform": "linux/amd64" }, "linux/arm64": { - "digest": "sha256:44fdbd8cd78a563f4f0f3bcf3feee8ca19c4db030156bf0efdb586d2de463072", - "id": "sha256:b6a03926deadfc5973bd799c8cd73dbd0c5fe2c797f33ac0092202712d13b1c6", + "digest": "sha256:e6849540507beec1e58ed69e77c26f1978630c299d8004f2fe259bd43caaf936", + "id": "sha256:dd219cae1939d82a047273c4a5252117c1bd1a2e3bb3458e8c6b980eaa5dd5bd", "platform": "linux/arm64" } } }, "docker.io/opensearchproject/opensearch:2.19.3": { - "digest": "sha256:674ffaf327f9be0088104132da24b627ff8d922f1a5de21f3e85d8ca8adf729b", - "mirror_digest": "sha256:0849d075ad68d441b3c2b953d995c05857dc227a49f6f6cf86e01588ab5b6c46", + "digest": "sha256:7350d82989e478700cb83107db909024f2c07dfd72ee5aa7c41804f2fd27bafd", + "mirror_digest": "sha256:c872c196d8772a9fb904821a873719c8ab99d7ac62e4fbe6e22b031c4a1bdf5a", "parts": { "linux/amd64": { - "digest": "sha256:d061ea683129a91adfe42abc6b5aff6d945855657f7409a89cbbd1bb010a2632", - "id": "sha256:b0317fd5558dcef16fae1cd022a79917b061d331be069b11f927595931d6ffb5", + "digest": "sha256:96a23ccb75d82edadf2aaeb1ba65aa9914459af35fe7045f3ba6acc566c25ed6", + "id": "sha256:f39b21c64beed147e385d4487d10a8ab1ee10329b9f9db7816ecbaa5a2827c01", "platform": "linux/amd64" }, "linux/arm64": { - "digest": "sha256:e45bb17e95f37b85b74e8ca48b28cd763a14e4224cc7358bddda7c95cb016384", - "id": "sha256:30d9305227ddc47261cf7806d017bed626cea001079bac41862344d8d51c03a1", + "digest": "sha256:639690ed70040ef6760e24e3008f9b8216b92ed31bcba992853d17f3785dc2eb", + "id": "sha256:bcbbe8ee2b9efa77558626a6992e578f56a106baa1591a0e36d210d1c32b60b4", "platform": "linux/arm64" } } }, - "docker.io/ucscgi/azul-bigquery-emulator:0.4.4-43": { - "digest": "sha256:fed995388fb7b69e709d9ec0de8d394160242f635ca3f63ce5ec4266b02da646", - "mirror_digest": "sha256:e2d0b0afd2c6897d43324b08c04b45cec6368aa3bab7c32c716be0251140f1ee", + "docker.io/ucscgi/azul-bigquery-emulator:0.4.4-44": { + "digest": "sha256:81f8be0f5eef8fd54a0a30f031430dbf9bdd7805c331c42140208b89d81165ea", + "mirror_digest": "sha256:2406953397c8f81075d967adc1fb20855a2eb86d333306ae411b7f779fda3e1e", "parts": { "linux/amd64": { - "digest": "sha256:06391bdacd0826f747dbbbea01b416f9224164b29656ec2c4708ff259c7c3c37", - "id": "sha256:44d6187086588536fd1f635f6bf64762bc6bcbc781ae99d2f11d65aa53f01234", + "digest": "sha256:377605705eeff97d685be89bfbfc79f8204d5fa213ac49d7fcc62186d1e971fe", + "id": "sha256:898250140d1015d765fb2cd69c19c104ee738588ca0f04d43b25023e0c0ffed3", "platform": "linux/amd64" }, "linux/arm64": { - "digest": "sha256:c7cea89fcfa60e7e93f8b35f213f2294b1c25d8c7caa8b41bb17009d4a8a4b8c", - "id": "sha256:0141eaf38c437af3db6007f4b3b9a6d4febd872326abaa50fb6cac5f9327445e", + "digest": "sha256:af32d345d49926733bebc43b895484fd851ab4514192fc476a41350fe5ca0d75", + "id": "sha256:6cef2a4ce964eb482f3262184f80a617eff870a5d3a11cdc48ee8e5cf9ffa7f0", "platform": "linux/arm64" } } @@ -149,17 +149,17 @@ "platform": "linux/amd64" }, "docker.io/opensearchproject/opensearch-dashboards:2.19.3": { - "digest": "sha256:a6b92ab4ccdb71a056347c1ca1022c4266e2cfd20fc0a74a6687433cb258ae71", - "mirror_digest": "sha256:0654300473c9f3b3e6dccbbf906eafb996293bcf9b42208fcf353cfb19a357fb", + "digest": "sha256:e7ec398ed591f1973d950ba96427f9dc9b7451bc30a3e50b92b8cdbf8901bce4", + "mirror_digest": "sha256:8377e58e76a3f453b607dc657a5dc3b0b7c41938a7374acc820dd791ff14bee5", "parts": { "linux/amd64": { - "digest": "sha256:b51521e7ff99d6dbdc160835a2fbeb90465bfc58f92f3aa4387ecd56ae051d2f", - "id": "sha256:1534f70025563f31d9dea585dae5aeb508e7fbd24b4d40672bdab61f1351b094", + "digest": "sha256:f2ecd5812e96d67d1e6a1260d79683abbda969efd3dd352c5961712ab8e76ead", + "id": "sha256:6fbefed06bf7a7937f7ad32809ef5a6804b565104a73f9e890ce53b4670f240b", "platform": "linux/amd64" }, "linux/arm64": { - "digest": "sha256:4de79bb89763da9dd068335c8f57d6190dcc4e97b43d6440bab21a495a630e55", - "id": "sha256:cb69221a8c12e29464f69660ec60e4776d4b03fb0a8a9acc3b80f8fd4d312086", + "digest": "sha256:f2d47a0578549e647f560a9f80728f57c2f438e805f271a29e77df1f1d92653c", + "id": "sha256:8ba630384c16a3058f3933b1fc09a6d8528e9816fd15e3809d48679d06d5f931", "platform": "linux/arm64" } } diff --git a/environment.boot b/environment.boot index e6a594392c..5396098bad 100644 --- a/environment.boot +++ b/environment.boot @@ -1,4 +1,4 @@ azul_python_version=3.12.11 -azul_python_image=docker.io/library/python@sha256:a9773611e52fb17803b9c3f50b2ff2fb027fe46e35d3fed4dfeadc8e06c5c847 +azul_python_image=docker.io/library/python@sha256:25cb89337ae6946bb067f236617afd4dea60339b2349cd88ae78070d9f30dd10 azul_docker_version=28.4.0 azul_terraform_version=1.12.2 diff --git a/environment.py b/environment.py index 9edd8326e1..069120388b 100644 --- a/environment.py +++ b/environment.py @@ -296,11 +296,11 @@ def env() -> Mapping[str, Optional[str]]: # See `azul_python_version` above about what actions are required # after modifying this entry. 'python': { - 'ref': 'docker.io/library/python:{azul_python_version}-slim-bookworm', + 'ref': 'docker.io/library/python:{azul_python_version}-slim-trixie', 'url': 'https://hub.docker.com/_/python', }, 'pycharm': { - 'ref': 'docker.io/ucscgi/azul-pycharm:2024.3.6-61', + 'ref': 'docker.io/ucscgi/azul-pycharm:2024.3.6-62', 'url': 'https://hub.docker.com/repository/docker/ucscgi/azul-pycharm', 'is_custom': True }, @@ -310,7 +310,7 @@ def env() -> Mapping[str, Optional[str]]: 'is_custom': False }, 'bigquery_emulator': { - 'ref': 'docker.io/ucscgi/azul-bigquery-emulator:0.4.4-43', + 'ref': 'docker.io/ucscgi/azul-bigquery-emulator:0.4.4-44', 'url': 'https://hub.docker.com/repository/docker/ucscgi/azul-bigquery-emulator', 'is_custom': True }, diff --git a/lambdas/service/.chalice/config.json.template.py b/lambdas/service/.chalice/config.json.template.py index c40dff9442..93b409e7be 100644 --- a/lambdas/service/.chalice/config.json.template.py +++ b/lambdas/service/.chalice/config.json.template.py @@ -34,7 +34,8 @@ "lambda_functions": { "api_handler": chalice.vpc_lambda_config(app_name), service.generate_manifest.name: { - "lambda_timeout": config.service_lambda_timeout + "lambda_timeout": config.service_lambda_timeout, + "lambda_memory_size": 3072 if config.deployment.is_stable else 2048, }, service.update_health_cache.name: { "lambda_memory_size": 128, diff --git a/lambdas/service/app.py b/lambdas/service/app.py index 3be85bd6c3..5fd5f747bf 100644 --- a/lambdas/service/app.py +++ b/lambdas/service/app.py @@ -57,7 +57,6 @@ ) from azul.health import ( HealthApp, - HealthController, ) from azul.indexer.document import ( EntityType, @@ -126,7 +125,7 @@ # changes and reset the minor version to zero. Otherwise, increment only # the minor version for backwards compatible changes. A backwards # compatible change is one that does not require updates to clients. - 'version': '13.0', + 'version': '14.2', 'description': fd(f''' # Overview @@ -303,10 +302,6 @@ def _oauth2_spec(self) -> JSON: def drs_controller(self) -> DRSController: return DRSController(app=self, file_url_func=self.file_url) - @cached_property - def health_controller(self) -> HealthController: - return HealthController(app=self, lambda_name=self.unqualified_app_name) - @cached_property def catalog_controller(self) -> CatalogController: return CatalogController(app=self, file_url_func=self.file_url) diff --git a/lambdas/service/openapi.json b/lambdas/service/openapi.json index cd07900746..d8347cc261 100644 --- a/lambdas/service/openapi.json +++ b/lambdas/service/openapi.json @@ -2,7 +2,7 @@ "openapi": "3.0.1", "info": { "title": "azul-service-dev", - "version": "13.0", + "version": "14.2", "description": "\n# Overview\n\nAzul is a REST web service for querying metadata associated with\nboth experimental and analysis data from a data repository. In order\nto deliver response times that make it suitable for interactive use\ncases, the set of metadata properties that it exposes for sorting,\nfiltering, and aggregation is limited. Azul provides a uniform view\nof the metadata over a range of diverse schemas, effectively\nshielding clients from changes in the schemas as they occur over\ntime. It does so, however, at the expense of detail in the set of\nmetadata properties it exposes and in the accuracy with which it\naggregates them.\n\nAzul denormalizes and aggregates metadata into several different\nindices for selected entity types. Metadata entities can be queried\nusing the [Index](#operations-tag-Index) endpoints.\n\nA set of indices forms a catalog. There is a default catalog called\n`dcp2` which will be used unless a\ndifferent catalog name is specified using the `catalog` query\nparameter. Metadata from different catalogs is completely\nindependent: a response obtained by querying one catalog does not\nnecessarily correlate to a response obtained by querying another\none. Two catalogs can contain metadata from the same sources or\ndifferent sources. It is only guaranteed that the body of a\nresponse by any given endpoint adheres to one schema,\nindependently of which catalog was specified in the request.\n\nAzul provides the ability to download data and metadata via the\n[Manifests](#operations-tag-Manifests) endpoints. The\n`curl` format manifests can be used to\ndownload data files. Other formats provide various views of the\nmetadata. Manifests can be generated for a selection of files using\nfilters. These filters are interchangeable with the filters used by\nthe [Index](#operations-tag-Index) endpoints.\n\nAzul also provides a [summary](#operations-Index-get_index_summary)\nview of indexed data.\n\n## Data model\n\nAny index, when queried, returns a JSON array of hits. Each hit\nrepresents a metadata entity. Nested in each hit is a summary of the\nproperties of entities associated with the hit. An entity is\nassociated either by a direct edge in the original metadata graph,\nor indirectly as a series of edges. The nested properties are\ngrouped by the type of the associated entity. The properties of all\ndata files associated with a particular sample, for example, are\nlisted under `hits[*].files` in a `/index/samples` response. It is\nimportant to note that while each _hit_ represents a discrete\nentity, the properties nested within that hit are the result of an\naggregation over potentially many associated entities.\n\nTo illustrate this, consider a data file that is part of two\nprojects (a project is a group of related experiments, typically by\none laboratory, institution or consortium). Querying the `files`\nindex for this file yields a hit looking something like:\n\n```\n{\n \"projects\": [\n {\n \"projectTitle\": \"Project One\"\n \"laboratory\": ...,\n ...\n },\n {\n \"projectTitle\": \"Project Two\"\n \"laboratory\": ...,\n ...\n }\n ],\n \"files\": [\n {\n \"format\": \"pdf\",\n \"name\": \"Team description.pdf\",\n ...\n }\n ]\n}\n```\n\nThis example hit contains two kinds of nested entities (a hit in an\nactual Azul response will contain more): There are the two projects\nentities, and the file itself. These nested entities contain\nselected metadata properties extracted in a consistent way. This\nmakes filtering and sorting simple.\n\nAlso notice that there is only one file. When querying a particular\nindex, the corresponding entity will always be a singleton like\nthis.\n\n\n## Contact us\n\nFor technical support please file an issue at\n[GitHub](https://github.com/DataBiosphere/azul/issues) or email\n`azul-group@ucsc.edu`. To report a security concern or misconduct please email\n`azul-group@ucsc.edu`.\n" }, "tags": [ diff --git a/requirements.all.txt b/requirements.all.txt index 843bdf6f12..1042b34d83 100644 --- a/requirements.all.txt +++ b/requirements.all.txt @@ -64,7 +64,7 @@ mypy==1.17.1 mypy-boto3-apigateway==1.40.0 mypy-boto3-cloudwatch==1.40.27 mypy-boto3-dynamodb==1.40.20 -mypy-boto3-ec2==1.40.34 +mypy-boto3-ec2==1.40.37 mypy-boto3-ecr==1.40.0 mypy-boto3-es==1.40.15 mypy-boto3-iam==1.40.0 @@ -74,8 +74,8 @@ mypy-boto3-s3==1.40.26 mypy-boto3-secretsmanager==1.40.0 mypy-boto3-securityhub==1.40.26 mypy-boto3-sns==1.40.1 -mypy-boto3-sqs==1.40.17 -mypy-boto3-ssm==1.40.0 +mypy-boto3-sqs==1.40.35 +mypy-boto3-ssm==1.40.37 mypy-boto3-stepfunctions==1.40.0 mypy-boto3-sts==1.40.0 mypy_extensions==1.1.0 @@ -101,7 +101,7 @@ pygithub==2.8.1 pyjwt==2.10.1 pynacl==1.6.0 pyopenssl==25.3.0 -pyparsing==3.2.4 +pyparsing==3.2.5 python-dateutil==2.9.0.post0 python-dxf==12.1.1 python-gitlab==6.3.0 @@ -131,7 +131,7 @@ typing_extensions==4.15.0 uritemplate==4.2.0 urllib3==1.26.20 watchdog==6.0.0 -wcwidth==0.2.13 +wcwidth==0.2.14 werkzeug==3.1.3 wheel==0.45.1 wrapt==1.17.3 diff --git a/requirements.dev.trans.txt b/requirements.dev.trans.txt index b820ffc58c..f89f033345 100644 --- a/requirements.dev.trans.txt +++ b/requirements.dev.trans.txt @@ -18,7 +18,7 @@ mccabe==0.7.0 mypy-boto3-apigateway==1.40.0 mypy-boto3-cloudwatch==1.40.27 mypy-boto3-dynamodb==1.40.20 -mypy-boto3-ec2==1.40.34 +mypy-boto3-ec2==1.40.37 mypy-boto3-ecr==1.40.0 mypy-boto3-es==1.40.15 mypy-boto3-iam==1.40.0 @@ -28,8 +28,8 @@ mypy-boto3-s3==1.40.26 mypy-boto3-secretsmanager==1.40.0 mypy-boto3-securityhub==1.40.26 mypy-boto3-sns==1.40.1 -mypy-boto3-sqs==1.40.17 -mypy-boto3-ssm==1.40.0 +mypy-boto3-sqs==1.40.35 +mypy-boto3-ssm==1.40.37 mypy-boto3-stepfunctions==1.40.0 mypy-boto3-sts==1.40.0 mypy_extensions==1.1.0 @@ -41,7 +41,7 @@ pycodestyle==2.14.0 pyflakes==3.4.0 pyjwt==2.10.1 pynacl==1.6.0 -pyparsing==3.2.4 +pyparsing==3.2.5 readchar==4.2.1 referencing==0.36.2 requests-toolbelt==1.0.0 @@ -53,7 +53,7 @@ tqdm==4.67.1 types-awscrt==0.27.6 types-s3transfer==0.13.1 uritemplate==4.2.0 -wcwidth==0.2.13 +wcwidth==0.2.14 www-authenticate==0.9.2 xmltodict==1.0.2 xmod==1.8.1 diff --git a/scripts/zenhub_to_github.py b/scripts/zenhub_to_github.py index fa2d99959e..7797344a5e 100644 --- a/scripts/zenhub_to_github.py +++ b/scripts/zenhub_to_github.py @@ -66,6 +66,9 @@ class Main: owner = 'DataBiosphere' + #: Only issues with this label will be migrated + label = 'orange' + project_title = 'Azul' status_field_name = 'Status' @@ -80,7 +83,7 @@ class Main: 'Epics': 'Backlog', 'Parked': 'Parked', 'Debt': 'Backlog', - 'Compliance controls': 'Backlog', # add label + 'Compliance controls': 'Backlog', 'Compliance': 'Backlog', 'Backlog': 'Backlog', 'Up next': 'Up next', @@ -248,7 +251,8 @@ def main(self): zh_blockees = {} for blockee in nodes(zh_issue, 'blockedIssues'): if json_str(blockee['ghNodeId']).startswith('PR_'): - log.warning('GitHub cannot block PRs on issues, ignoring blockee %s') + log.warning('GitHub cannot block PRs on issues. ' + 'Ignoring blockee %s', blockee['htmlUrl']) elif any(json_str(blockee['htmlUrl']).startswith(repo) for repo in self.archived_repos): log.warning('GitHub cannot block issues in archived repositories. ' 'Ignoring blockee %s', blockee['htmlUrl']) @@ -404,7 +408,7 @@ def _zenhub_issues(self, workspace_id: str, pipeline: JSON) -> dict[str, JSON]: while True: response = self._zenhub(self._body( container_id=container_id, - label='orange', + label=self.label, cursor=cursor, query=query )) diff --git a/src/azul/__init__.py b/src/azul/__init__.py index b780c53eab..515cf8dc80 100644 --- a/src/azul/__init__.py +++ b/src/azul/__init__.py @@ -1963,6 +1963,10 @@ def __repr__(self): case args: return class_name + repr(args) + @final + def __eq__(self, other: object): + return isinstance(other, R) and self.args == other.args + @deprecated("Use 'assert False, R(…)' instead", category=None) class RequirementError(AssertionError): diff --git a/src/azul/collections.py b/src/azul/collections.py index 58421258da..3c6190865d 100644 --- a/src/azul/collections.py +++ b/src/azul/collections.py @@ -7,6 +7,9 @@ Mapping, MutableSet, ) +from enum import ( + Enum, +) from functools import ( partial, ) @@ -551,3 +554,77 @@ def update(self, members: Iterable[K] = (), /) -> None: def singleton[T](x: T) -> frozenset[T]: return frozenset((x,)) + + +class LookupDefault(Enum): + RAISE = 0 + + +def lookup[K, V](d: Mapping[K, V], + k: K, + *ks: K, + default: V | LookupDefault | None = LookupDefault.RAISE + ) -> V | None: + """ + Look up a value in the specified dictionary given one or more candidate + keys. + + This function raises a key error for the first (!) key if none of the keys + are present and the `default` keyword argument absent. If the `default` + keyword argument is present (None is a valid default), this function returns + that argument instead of raising an KeyError in that case. This is notably + different to dict.get() whose default default is `None`. This function does + not have a default default. + + If the first key is present, return its value ... + + >>> lookup({1:2}, 1) + 2 + + ... and ignore the other keys. + + >>> lookup({1:2}, 1, 3) + 2 + + If the first key is absent, try the fallbacks. + + >>> lookup({1:2}, 3, 1) + 2 + + If the key isn't present, raise a KeyError referring to that key. + + >>> lookup({1:2}, 3) + Traceback (most recent call last): + ... + KeyError: 3 + + If neither the first key nor the fallbacks are present, raise a KeyError + referring to the first key. + + >>> lookup({1:2}, 3, 4) + Traceback (most recent call last): + ... + KeyError: 3 + + If the key isn't present but a default was passed, return the default. + + >>> lookup({1:2}, 3, default=4) + 4 + + None is a valid default. + + >>> lookup({1:2}, 3, 4, default=None) is None + True + """ + try: + return d[k] + except KeyError: + for k in ks: + try: + return d[k] + except KeyError: + pass + if default is LookupDefault.RAISE: + raise + else: + return default diff --git a/src/azul/indexer/document_service.py b/src/azul/indexer/document_service.py index 27756e4421..86b7ff222f 100644 --- a/src/azul/indexer/document_service.py +++ b/src/azul/indexer/document_service.py @@ -34,6 +34,7 @@ from azul.plugins import ( FieldPath, MetadataPlugin, + RepositoryPlugin, ) from azul.types import ( AnyJSON, @@ -47,6 +48,10 @@ class DocumentService: def metadata_plugin(self, catalog: CatalogName) -> MetadataPlugin: return MetadataPlugin.load(catalog).create() + @cache + def repository_plugin(self, catalog: CatalogName) -> RepositoryPlugin: + return RepositoryPlugin.load(catalog).create(catalog) + @cache def aggregate_class(self, catalog: CatalogName) -> Type[Aggregate]: return self.metadata_plugin(catalog).aggregate_class() diff --git a/src/azul/indexer/index_service.py b/src/azul/indexer/index_service.py index 30cbf29a41..d512e3b524 100644 --- a/src/azul/indexer/index_service.py +++ b/src/azul/indexer/index_service.py @@ -39,6 +39,7 @@ from azul import ( CatalogName, + R, config, ) from azul.deployment import ( @@ -484,6 +485,13 @@ def aggregate(self, tallies: CataloguedTallies): old_aggregate.contents = {} new_aggregates.append(old_aggregate) + for aggregate in new_aggregates: + assert len(aggregate.sources) == 1, R( + 'Entity has an invalid number of sources', + aggregate.entity, + aggregate.sources + ) + # Write new aggregates writer.write(new_aggregates) diff --git a/src/azul/plugins/__init__.py b/src/azul/plugins/__init__.py index 3b8fde66a7..7b9d510682 100644 --- a/src/azul/plugins/__init__.py +++ b/src/azul/plugins/__init__.py @@ -837,7 +837,7 @@ class File(SerializableAttrs, metaclass=ABCMeta): @classmethod @abstractmethod - def from_hit(cls, hit: JSON) -> Self: + def from_index(cls, hit: JSON) -> Self: """ Instantiate this class from an entity aggregate document retrieved from Elasticsearch. diff --git a/src/azul/plugins/metadata/anvil/__init__.py b/src/azul/plugins/metadata/anvil/__init__.py index 487582e4b8..4a00c19e3a 100644 --- a/src/azul/plugins/metadata/anvil/__init__.py +++ b/src/azul/plugins/metadata/anvil/__init__.py @@ -346,7 +346,7 @@ def recurse(mapping: MetadataPlugin._FieldMapping, path: FieldPath): # The file URL is synthesized from the `uuid` and `version` fields. # Above, we already configured these two fields to be omitted from the # manifest since they are not informative to the user. - result[('contents', 'files')]['file_url'] = 'files.azul_file_url' + result[('contents', 'files')]['file_url'] = 'files.azul_url' return result primary_keys_by_table = { @@ -524,7 +524,7 @@ class AnvilFile(File): md5: str @classmethod - def from_hit(cls, hit: JSON) -> Self: + def from_index(cls, hit: JSON) -> Self: return cls(uuid=hit['document_id'], version=hit['version'], name=hit['file_name'], diff --git a/src/azul/plugins/metadata/anvil/indexer/transform.py b/src/azul/plugins/metadata/anvil/indexer/transform.py index 8bd12cb724..5a9fd73a29 100644 --- a/src/azul/plugins/metadata/anvil/indexer/transform.py +++ b/src/azul/plugins/metadata/anvil/indexer/transform.py @@ -316,8 +316,6 @@ def _file_types(cls) -> FieldTypes: # FIXME: Redundant file fields for AnVIL are no longer needed # https://github.com/DataBiosphere/azul/issues/7005 'uuid': null_str, - 'size': null_int, - 'name': null_str, 'crc32': null_str, 'sha256': null_str, 'drs_uri': null_str @@ -413,11 +411,8 @@ def _donor(self, donor: EntityReference) -> MutableJSON: return self._entity(donor, self._donor_types()) def _file(self, file: EntityReference) -> MutableJSON: - metadata = self.bundle.entities[file] return self._entity(file, self._file_types(), - size=metadata['file_size'], - name=metadata['file_name'], uuid=file.entity_id) def _only_dataset(self) -> EntityReference: diff --git a/src/azul/plugins/metadata/anvil/service/aggregation.py b/src/azul/plugins/metadata/anvil/service/aggregation.py index 02452feb51..f3321f0078 100644 --- a/src/azul/plugins/metadata/anvil/service/aggregation.py +++ b/src/azul/plugins/metadata/anvil/service/aggregation.py @@ -31,7 +31,7 @@ def prepare_request(self, request: Search) -> Search: if self.entity_type == 'files': request.aggs.metric('totalFileSize', 'sum', - field='contents.files.size_') + field='contents.files.file_size_') return request def process_response(self, response: MutableJSON) -> MutableJSON: diff --git a/src/azul/plugins/metadata/anvil/service/response.py b/src/azul/plugins/metadata/anvil/service/response.py index 23ac4deb43..825b8a9eb2 100644 --- a/src/azul/plugins/metadata/anvil/service/response.py +++ b/src/azul/plugins/metadata/anvil/service/response.py @@ -181,7 +181,14 @@ def _pivotal_entity(self, ) -> MutableJSON: inner_entity = copy_json(inner_entity) if inner_entity_type == 'files': - inner_entity['uuid'] = inner_entity['document_id'] + inner_entity['azul_url'] = self._file_url(uuid=inner_entity['uuid'], + version=inner_entity['version'], + drs_uri=inner_entity['drs_uri']) + # FIXME: https://github.com/DataBiosphere/azul/issues/6549 + # Remove files.url + inner_entity['url'] = inner_entity['azul_url'] + inner_entity.pop('uuid', None) + inner_entity.pop('version', None) return inner_entity def _non_pivotal_entity(self, diff --git a/src/azul/plugins/metadata/hca/__init__.py b/src/azul/plugins/metadata/hca/__init__.py index fb665d6f4d..324357f9f5 100644 --- a/src/azul/plugins/metadata/hca/__init__.py +++ b/src/azul/plugins/metadata/hca/__init__.py @@ -369,7 +369,7 @@ def manifest_config(self) -> ManifestConfig: 'sha256': 'file_sha256', 'content-type': 'file_content_type', 'drs_uri': 'file_drs_uri', - 'file_url': 'file_url' + 'file_url': 'file_azul_url' }, ('contents', 'cell_suspensions'): { 'document_id': 'cell_suspension.provenance.document_id', @@ -475,7 +475,7 @@ class HCAFile(File): s3_etag: str | None = None @classmethod - def from_hit(cls, hit: JSON) -> Self: + def from_index(cls, hit: JSON) -> Self: return cls(uuid=hit['uuid'], version=hit['version'], name=hit['name'], @@ -488,12 +488,12 @@ def from_hit(cls, hit: JSON) -> Self: s3_etag=hit.get('s3_etag')) @classmethod - def from_descriptor(cls, - descriptor: JSON, - *, - uuid: str, - name: str, - drs_uri: str | None) -> Self: + def from_metadata(cls, + descriptor: JSON, + *, + uuid: str, + name: str, + drs_uri: str | None) -> Self: content_type = descriptor['content_type'] # FIXME: Obsolete MIME parameter in file content types # https://github.com/HumanCellAtlas/dcp2/issues/73 diff --git a/src/azul/plugins/metadata/hca/service/response.py b/src/azul/plugins/metadata/hca/service/response.py index 0ab6f72243..bf8b8cf223 100644 --- a/src/azul/plugins/metadata/hca/service/response.py +++ b/src/azul/plugins/metadata/hca/service/response.py @@ -392,8 +392,14 @@ def make_translated_file(self, file: JSON) -> JSON: 'uuid': file.get('uuid'), 'version': file.get('version'), 'matrixCellCount': file.get('matrix_cell_count'), - 'drs_uri': file.get('drs_uri') + 'drs_uri': file.get('drs_uri'), + 'azul_url': self._file_url(uuid=file['uuid'], + version=file['version'], + drs_uri=file['drs_uri']) } + # FIXME: https://github.com/DataBiosphere/azul/issues/6549 + # Remove files.url + translated_file['url'] = translated_file['azul_url'] return translated_file def make_specimen(self, specimen) -> MutableJSON: diff --git a/src/azul/plugins/repository/canned/__init__.py b/src/azul/plugins/repository/canned/__init__.py index 093cc5a9c4..cb0869b540 100644 --- a/src/azul/plugins/repository/canned/__init__.py +++ b/src/azul/plugins/repository/canned/__init__.py @@ -209,10 +209,10 @@ def list_files(self, source: CannedSourceRef, prefix: str) -> list[HCAFile]: assert prefix == prefix.lower(), prefix staging_area = self.staging_area(source.spec.name) return [ - HCAFile.from_descriptor(descriptor.content, - uuid=file_uuid, - name=descriptor.content['file_name'], - drs_uri=None) + HCAFile.from_metadata(descriptor.content, + uuid=file_uuid, + name=descriptor.content['file_name'], + drs_uri=None) for file_uuid, descriptor in staging_area.descriptors.items() if descriptor.content['sha256'].lower().startswith(prefix) ] diff --git a/src/azul/plugins/repository/tdr_hca/__init__.py b/src/azul/plugins/repository/tdr_hca/__init__.py index c1541a75bd..11a04b0034 100644 --- a/src/azul/plugins/repository/tdr_hca/__init__.py +++ b/src/azul/plugins/repository/tdr_hca/__init__.py @@ -188,10 +188,10 @@ def file_from_row(cls, row: BigQueryRow) -> HCAFile: # FIXME: Move validation of descriptor to the metadata API # https://github.com/DataBiosphere/azul/issues/6299 api.Entity.validate_described_by(descriptor) - return HCAFile.from_descriptor(descriptor, - uuid=descriptor['file_id'], - name=row['file_name'], - drs_uri=cls._parse_drs_uri(row['file_id'], descriptor)) + return HCAFile.from_metadata(descriptor, + uuid=descriptor['file_id'], + name=row['file_name'], + drs_uri=cls._parse_drs_uri(row['file_id'], descriptor)) def _add_manifest_entry(self, entity: EntityReference, diff --git a/src/azul/service/avro_pfb.py b/src/azul/service/avro_pfb.py index 604fc0f8da..91782e50b5 100644 --- a/src/azul/service/avro_pfb.py +++ b/src/azul/service/avro_pfb.py @@ -485,7 +485,7 @@ def avro_pfb_schema(azul_avro_schema: Iterable[JSON]) -> JSON: def _inject_reference_handover_columns(field_types: FieldTypes) -> FieldTypes: return { entity_type: ( - dict(fields, datarepo_row_id=null_str, source_datarepo_snapshot_id=null_str) + dict(fields, datarepo_row_id=null_str, datarepo_snapshot_id=null_str) if isinstance(fields, dict) and 'source_datarepo_row_ids' in fields else fields ) @@ -496,7 +496,7 @@ def _inject_reference_handover_columns(field_types: FieldTypes) -> FieldTypes: def _inject_reference_handover_values(entity: MutableJSON, doc: JSON): if 'source_datarepo_row_ids' in entity: entity['datarepo_row_id'] = entity['document_id'] - entity['source_datarepo_snapshot_id'] = one(doc['sources'])['id'] + entity['datarepo_snapshot_id'] = one(doc['sources'])['id'] # FIXME: It's not obvious as to why these are union types. Explain or change. @@ -570,7 +570,7 @@ def _entity_schema_recursive(field_types: FieldTypes, 'estimated_cell_count', 'total_estimated_cells', 'total_estimated_cells_redundant', - 'source_datarepo_snapshot_id', + 'datarepo_snapshot_id', ) path_exceptions = ( ('projects', 'accessions'), diff --git a/src/azul/service/manifest_service.py b/src/azul/service/manifest_service.py index b7eb693d9d..729a4873c0 100644 --- a/src/azul/service/manifest_service.py +++ b/src/azul/service/manifest_service.py @@ -3,6 +3,9 @@ abstractmethod, ) import base64 +from bisect import ( + insort, +) from collections.abc import ( Iterable, Mapping, @@ -2059,6 +2062,20 @@ def create_file(self) -> tuple[str, str | None]: replicas = list(self._list_replicas()) plugin = self.metadata_plugin replica_schemas = plugin.verbatim_pfb_schema(replicas) + # FIXME: Move injection of snapshot ID field to metadata plugin + # https://github.com/DataBiosphere/azul/issues/7411 + if config.is_anvil_enabled(self.catalog): + for replica in replicas: + if replica['replica_type'] == 'anvil_dataset': + source_id = replica['source']['id'] + replica['contents']['datarepo_snapshot_id'] = source_id + for schema in replica_schemas: + if schema['name'] == 'anvil_dataset': + field_schema = plugin._pfb_schema_from_anvil_column(table_name='anvil_dataset', + column_name='datarepo_snapshot_id', + anvil_datatype='string', + is_optional=False) + insort(schema['fields'], field_schema, key=itemgetter('name')) # Ensure field order is consistent for unit tests replica_schemas.sort(key=itemgetter('name')) links = { diff --git a/src/azul/service/repository_service.py b/src/azul/service/repository_service.py index eb20e0a2da..a5e748dc3b 100644 --- a/src/azul/service/repository_service.py +++ b/src/azul/service/repository_service.py @@ -15,6 +15,7 @@ TYPE_CHECKING, ) +import attrs from more_itertools import ( first, one, @@ -29,7 +30,6 @@ from azul import ( CatalogName, - cache, config, ) from azul.plugins import ( @@ -53,7 +53,6 @@ _ElasticsearchStage, ) from azul.types import ( - AnyMutableJSON, JSON, MutableJSON, ) @@ -70,12 +69,30 @@ def __init__(self, entity_type: str, entity_id: str): super().__init__(f"Can't find an entity in {entity_type} with an uuid, {entity_id}.") +@attrs.frozen(auto_attribs=True, kw_only=True) class SearchResponseStage(_ElasticsearchStage[ResponseTriple, MutableJSON], metaclass=ABCMeta): + file_url_func: FileUrlFunc def prepare_request(self, request: Search) -> Search: return request + @property + def repository_plugin(self) -> RepositoryPlugin: + return self.service.repository_plugin(self.catalog) + + def _file_url(self, *, uuid: str, version: str, drs_uri: str | None) -> str | None: + plugin = self.repository_plugin + if drs_uri is None and plugin.file_download_class().needs_drs_uri: + # Don't emit a download URL if a DRS URI is needed for downloading a + # file, but none is available + return None + else: + return str(self.file_url_func(catalog=self.catalog, + fetch=False, + file_uuid=uuid, + version=version)) + class SummaryResponseStage(ElasticsearchStage[JSON, MutableJSON], metaclass=ABCMeta): @@ -91,10 +108,6 @@ def prepare_request(self, request: Search) -> Search: class RepositoryService(ElasticsearchService): - @cache - def repository_plugin(self, catalog: CatalogName) -> RepositoryPlugin: - return RepositoryPlugin.load(catalog).create(catalog) - def search(self, *, catalog: CatalogName, @@ -124,7 +137,8 @@ def search(self, filters=filters, pagination=pagination, aggregate=item_id is None, - entity_type=entity_type) + entity_type=entity_type, + file_url_func=file_url_func) special_fields = self.metadata_plugin(catalog).special_fields for hit in response['hits']: @@ -132,49 +146,6 @@ def search(self, source_id = one(hit['sources'])[special_fields.source_id] entity[special_fields.accessible] = source_id in filters.source_ids - def inject_file_urls(node: AnyMutableJSON, *path: str) -> None: - if node is None: - pass - elif isinstance(node, (str, int, float, bool)): - pass - elif isinstance(node, list): - for child in node: - inject_file_urls(child, *path) - elif isinstance(node, dict): - if path: - try: - next_node = node[path[0]] - except KeyError: - # Not all node trees will match the given path. (e.g. a - # response from the 'files' index won't have a - # 'matrices' in its 'hits[].projects' inner entities. - pass - else: - inject_file_urls(next_node, *path[1:]) - else: - try: - version = node['version'] - uuid = node['uuid'] - drs_uri = node['drs_uri'] - except KeyError: - for child in node.values(): - inject_file_urls(child, *path) - else: - plugin = self.repository_plugin(catalog) - if drs_uri is None and plugin.file_download_class().needs_drs_uri: - node['url'] = None - else: - node['url'] = str(file_url_func(catalog=catalog, - fetch=False, - file_uuid=uuid, - version=version)) - else: - assert False - - inject_file_urls(response['hits'], 'projects', 'contributedAnalyses') - inject_file_urls(response['hits'], 'projects', 'matrices') - inject_file_urls(response['hits'], 'files') - if item_id is not None: response = one(response['hits'], too_short=EntityNotFoundError(entity_type, item_id)) return response @@ -185,7 +156,8 @@ def _search(self, entity_type: str, aggregate: bool, filters: Filters, - pagination: Pagination + pagination: Pagination, + file_url_func: FileUrlFunc ) -> MutableJSON: """ This function does the whole transformation process. It takes the path @@ -242,7 +214,8 @@ def _search(self, response_stage_cls = SearchResponseStage chain = response_stage_cls(service=self, catalog=catalog, - entity_type=entity_type).wrap(chain) + entity_type=entity_type, + file_url_func=file_url_func).wrap(chain) request = self.create_request(catalog, entity_type) request = chain.prepare_request(request) @@ -371,7 +344,7 @@ def _hit_to_doc(hit: Hit) -> JSON: assert file_version is None, len(hits) file = one(first(hits)['contents']['files']) - file = plugin.file_class.from_hit(file) + file = plugin.file_class.from_index(file) if file_version is not None: assert file_version == file.version return file diff --git a/src/humancellatlas/data/metadata/api.py b/src/humancellatlas/data/metadata/api.py index 757bc8a70e..6051d88e9c 100644 --- a/src/humancellatlas/data/metadata/api.py +++ b/src/humancellatlas/data/metadata/api.py @@ -39,9 +39,11 @@ cached_property, ) from azul.collections import ( + LookupDefault, OrderedSet, adict, dict_merge, + lookup, ) from azul.indexer.document import ( @@ -58,10 +60,6 @@ from humancellatlas.data.metadata.datetime import ( parse_jsonschema_date_time, ) -from humancellatlas.data.metadata.lookup import ( - LookupDefault, - lookup, -) # A few helpful type aliases # diff --git a/src/humancellatlas/data/metadata/lookup.py b/src/humancellatlas/data/metadata/lookup.py deleted file mode 100644 index 41fd21f679..0000000000 --- a/src/humancellatlas/data/metadata/lookup.py +++ /dev/null @@ -1,73 +0,0 @@ -from enum import ( - Enum, -) -from typing import ( - Mapping, - TypeVar, -) - -K = TypeVar('K') -V = TypeVar('V') - - -class LookupDefault(Enum): - RAISE = 0 - - -def lookup(d: Mapping[K, V], - k: K, - *ks: K, - default: V | LookupDefault | None = LookupDefault.RAISE - ) -> V: - """ - Look up a value in the specified dictionary given one or more candidate keys. - - This function raises a key error for the first (!) key if none of the keys are present and the `default` keyword - argument absent. If the `default` keyword argument is present (None is a valid default), this function returns - that argument instead of raising an KeyError in that case. This is notably different to dict.get() whose default - default is `None`. This function does not have a default default. - - If the first key is present, return its value ... - >>> lookup({1:2}, 1) - 2 - - ... and ignore the other keys. - >>> lookup({1:2}, 1, 3) - 2 - - If the first key is absent, try the fallbacks. - >>> lookup({1:2}, 3, 1) - 2 - - If the key isn't present, raise a KeyError referring to that key. - >>> lookup({1:2}, 3) - Traceback (most recent call last): - ... - KeyError: 3 - - If neither the first key nor the fallbacks are present, raise a KeyError referring to the first key. - >>> lookup({1:2}, 3, 4) - Traceback (most recent call last): - ... - KeyError: 3 - - If the key isn't present but a default was passed, return the default. - >>> lookup({1:2}, 3, default=4) - 4 - - None is a valid default. - >>> lookup({1:2}, 3, 4, default=None) is None - True - """ - try: - return d[k] - except KeyError: - for k in ks: - try: - return d[k] - except KeyError: - pass - if default is LookupDefault.RAISE: - raise - else: - return default diff --git a/test/hca_metadata_api/test.py b/test/hca_metadata_api/test.py index f7b83b3973..ff1cbe7c4f 100644 --- a/test/hca_metadata_api/test.py +++ b/test/hca_metadata_api/test.py @@ -818,7 +818,6 @@ def load_tests(_loader, tests, _ignore): 'humancellatlas.data.metadata.age_range', 'humancellatlas.data.metadata.api', 'humancellatlas.data.metadata.datetime', - 'humancellatlas.data.metadata.lookup', ) for module in modules: tests.addTests(doctest.DocTestSuite(module)) diff --git a/test/indexer/data/826dea02-e274-affe-aabc-eb3db63ad068.results.json b/test/indexer/data/826dea02-e274-affe-aabc-eb3db63ad068.results.json index 1c0a8283d4..ebc36bc399 100644 --- a/test/indexer/data/826dea02-e274-affe-aabc-eb3db63ad068.results.json +++ b/test/indexer/data/826dea02-e274-affe-aabc-eb3db63ad068.results.json @@ -200,15 +200,6 @@ "uuid": [ "15b76f9c-6b46-433f-851d-34e89f1b9ba6" ], - "size": [ - 213021639 - ], - "size_": [ - 213021639 - ], - "name": [ - "307500.merged.matefixed.sorted.markeddups.recal.g.vcf.gz" - ], "crc32": [ "" ], @@ -405,9 +396,6 @@ "is_supplementary": 0, "version": "2022-06-01T00:00:00.000000Z", "uuid": "15b76f9c-6b46-433f-851d-34e89f1b9ba6", - "size": 213021639, - "size_": 213021639, - "name": "307500.merged.matefixed.sorted.markeddups.recal.g.vcf.gz", "crc32": "", "sha256": "", "drs_uri": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_1e269f04-4347-4188-b060-1dcc69e71d67" @@ -651,9 +639,6 @@ "is_supplementary": 0, "version": "2022-06-01T00:00:00.000000Z", "uuid": "15b76f9c-6b46-433f-851d-34e89f1b9ba6", - "size": 213021639, - "size_": 213021639, - "name": "307500.merged.matefixed.sorted.markeddups.recal.g.vcf.gz", "crc32": "", "sha256": "", "drs_uri": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_1e269f04-4347-4188-b060-1dcc69e71d67" @@ -843,9 +828,6 @@ "is_supplementary": 0, "version": "2022-06-01T00:00:00.000000Z", "uuid": "15b76f9c-6b46-433f-851d-34e89f1b9ba6", - "size": 213021639, - "size_": 213021639, - "name": "307500.merged.matefixed.sorted.markeddups.recal.g.vcf.gz", "crc32": "", "sha256": "", "drs_uri": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_1e269f04-4347-4188-b060-1dcc69e71d67" @@ -1144,15 +1126,6 @@ "uuid": [ "15b76f9c-6b46-433f-851d-34e89f1b9ba6" ], - "size": [ - 213021639 - ], - "size_": [ - 213021639 - ], - "name": [ - "307500.merged.matefixed.sorted.markeddups.recal.g.vcf.gz" - ], "crc32": [ "" ], @@ -1200,15 +1173,6 @@ "uuid": [ "3b17377b-16b1-431c-9967-e5d01fc5923f" ], - "size": [ - 3306845592 - ], - "size_": [ - 3306845592 - ], - "name": [ - "307500.merged.matefixed.sorted.markeddups.recal.bam" - ], "crc32": [ "" ], @@ -1423,9 +1387,6 @@ "is_supplementary": 0, "version": "2022-06-01T00:00:00.000000Z", "uuid": "15b76f9c-6b46-433f-851d-34e89f1b9ba6", - "size": 213021639, - "size_": 213021639, - "name": "307500.merged.matefixed.sorted.markeddups.recal.g.vcf.gz", "crc32": "", "sha256": "", "drs_uri": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_1e269f04-4347-4188-b060-1dcc69e71d67" @@ -1450,9 +1411,6 @@ "is_supplementary": 0, "version": "2022-06-01T00:00:00.000000Z", "uuid": "3b17377b-16b1-431c-9967-e5d01fc5923f", - "size": 3306845592, - "size_": 3306845592, - "name": "307500.merged.matefixed.sorted.markeddups.recal.bam", "crc32": "", "sha256": "", "drs_uri": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_8b722e88-8103-49c1-b351-e64fa7c6ab37" @@ -1701,9 +1659,6 @@ "is_supplementary": 0, "version": "2022-06-01T00:00:00.000000Z", "uuid": "3b17377b-16b1-431c-9967-e5d01fc5923f", - "size": 3306845592, - "size_": 3306845592, - "name": "307500.merged.matefixed.sorted.markeddups.recal.bam", "crc32": "", "sha256": "", "drs_uri": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_8b722e88-8103-49c1-b351-e64fa7c6ab37" @@ -1929,9 +1884,6 @@ "is_supplementary": 0, "version": "2022-06-01T00:00:00.000000Z", "uuid": "3b17377b-16b1-431c-9967-e5d01fc5923f", - "size": 3306845592, - "size_": 3306845592, - "name": "307500.merged.matefixed.sorted.markeddups.recal.bam", "crc32": "", "sha256": "", "drs_uri": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_8b722e88-8103-49c1-b351-e64fa7c6ab37" @@ -2149,15 +2101,6 @@ "uuid": [ "3b17377b-16b1-431c-9967-e5d01fc5923f" ], - "size": [ - 3306845592 - ], - "size_": [ - 3306845592 - ], - "name": [ - "307500.merged.matefixed.sorted.markeddups.recal.bam" - ], "crc32": [ "" ], @@ -2388,9 +2331,6 @@ "is_supplementary": 0, "version": "2022-06-01T00:00:00.000000Z", "uuid": "3b17377b-16b1-431c-9967-e5d01fc5923f", - "size": 3306845592, - "size_": 3306845592, - "name": "307500.merged.matefixed.sorted.markeddups.recal.bam", "crc32": "", "sha256": "", "drs_uri": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_8b722e88-8103-49c1-b351-e64fa7c6ab37" @@ -2605,15 +2545,6 @@ "uuid": [ "15b76f9c-6b46-433f-851d-34e89f1b9ba6" ], - "size": [ - 213021639 - ], - "size_": [ - 213021639 - ], - "name": [ - "307500.merged.matefixed.sorted.markeddups.recal.g.vcf.gz" - ], "crc32": [ "" ], @@ -2661,15 +2592,6 @@ "uuid": [ "3b17377b-16b1-431c-9967-e5d01fc5923f" ], - "size": [ - 3306845592 - ], - "size_": [ - 3306845592 - ], - "name": [ - "307500.merged.matefixed.sorted.markeddups.recal.bam" - ], "crc32": [ "" ], @@ -2884,9 +2806,6 @@ "is_supplementary": 0, "version": "2022-06-01T00:00:00.000000Z", "uuid": "15b76f9c-6b46-433f-851d-34e89f1b9ba6", - "size": 213021639, - "size_": 213021639, - "name": "307500.merged.matefixed.sorted.markeddups.recal.g.vcf.gz", "crc32": "", "sha256": "", "drs_uri": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_1e269f04-4347-4188-b060-1dcc69e71d67" @@ -2911,9 +2830,6 @@ "is_supplementary": 0, "version": "2022-06-01T00:00:00.000000Z", "uuid": "3b17377b-16b1-431c-9967-e5d01fc5923f", - "size": 3306845592, - "size_": 3306845592, - "name": "307500.merged.matefixed.sorted.markeddups.recal.bam", "crc32": "", "sha256": "", "drs_uri": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_8b722e88-8103-49c1-b351-e64fa7c6ab37" @@ -3184,15 +3100,6 @@ "uuid": [ "15b76f9c-6b46-433f-851d-34e89f1b9ba6" ], - "size": [ - 213021639 - ], - "size_": [ - 213021639 - ], - "name": [ - "307500.merged.matefixed.sorted.markeddups.recal.g.vcf.gz" - ], "crc32": [ "" ], @@ -3240,15 +3147,6 @@ "uuid": [ "3b17377b-16b1-431c-9967-e5d01fc5923f" ], - "size": [ - 3306845592 - ], - "size_": [ - 3306845592 - ], - "name": [ - "307500.merged.matefixed.sorted.markeddups.recal.bam" - ], "crc32": [ "" ], @@ -3463,9 +3361,6 @@ "is_supplementary": 0, "version": "2022-06-01T00:00:00.000000Z", "uuid": "15b76f9c-6b46-433f-851d-34e89f1b9ba6", - "size": 213021639, - "size_": 213021639, - "name": "307500.merged.matefixed.sorted.markeddups.recal.g.vcf.gz", "crc32": "", "sha256": "", "drs_uri": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_1e269f04-4347-4188-b060-1dcc69e71d67" @@ -3490,9 +3385,6 @@ "is_supplementary": 0, "version": "2022-06-01T00:00:00.000000Z", "uuid": "3b17377b-16b1-431c-9967-e5d01fc5923f", - "size": 3306845592, - "size_": 3306845592, - "name": "307500.merged.matefixed.sorted.markeddups.recal.bam", "crc32": "", "sha256": "", "drs_uri": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_8b722e88-8103-49c1-b351-e64fa7c6ab37" @@ -3753,15 +3645,6 @@ "uuid": [ "15b76f9c-6b46-433f-851d-34e89f1b9ba6" ], - "size": [ - 213021639 - ], - "size_": [ - 213021639 - ], - "name": [ - "307500.merged.matefixed.sorted.markeddups.recal.g.vcf.gz" - ], "crc32": [ "" ], @@ -3809,15 +3692,6 @@ "uuid": [ "3b17377b-16b1-431c-9967-e5d01fc5923f" ], - "size": [ - 3306845592 - ], - "size_": [ - 3306845592 - ], - "name": [ - "307500.merged.matefixed.sorted.markeddups.recal.bam" - ], "crc32": [ "" ], @@ -4032,9 +3906,6 @@ "is_supplementary": 0, "version": "2022-06-01T00:00:00.000000Z", "uuid": "15b76f9c-6b46-433f-851d-34e89f1b9ba6", - "size": 213021639, - "size_": 213021639, - "name": "307500.merged.matefixed.sorted.markeddups.recal.g.vcf.gz", "crc32": "", "sha256": "", "drs_uri": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_1e269f04-4347-4188-b060-1dcc69e71d67" @@ -4059,9 +3930,6 @@ "is_supplementary": 0, "version": "2022-06-01T00:00:00.000000Z", "uuid": "3b17377b-16b1-431c-9967-e5d01fc5923f", - "size": 3306845592, - "size_": 3306845592, - "name": "307500.merged.matefixed.sorted.markeddups.recal.bam", "crc32": "", "sha256": "", "drs_uri": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_8b722e88-8103-49c1-b351-e64fa7c6ab37" diff --git a/test/indexer/test_indexer.py b/test/indexer/test_indexer.py index 2b858da701..a06d579d76 100644 --- a/test/indexer/test_indexer.py +++ b/test/indexer/test_indexer.py @@ -33,6 +33,7 @@ ) import attr +import attrs from more_itertools import ( bucket, ilen, @@ -60,6 +61,7 @@ CataloguedEntityReference, Contribution, ContributionCoordinates, + DocumentSource, DocumentType, EntityReference, EntityType, @@ -95,6 +97,7 @@ ) from azul.plugins.repository.dss import ( DSSBundle, + DSSBundleFQID, ) from azul.threads import ( Latch, @@ -467,6 +470,32 @@ def test_zero_tallies(self): fr'.*_aggregate/_create/{doc_id}.*') self.assertTrue(any(message_re.fullmatch(message) for message in logs.output)) + def test_entity_disjunctivity(self): + bundle1 = self._load_canned_bundle(self.old_bundle) + self._index_bundle(bundle1) + # For bundle2 we create a new bundle using bundle1's contents, however + # with a different source and bundle UUID, which, when indexed, results + # in aggregates with two sources. + source1 = bundle1.fqid.source + source2 = attrs.evolve(source1, id='6b26d8d7-6e5a-4bcb-910d-1804cc59e32e') + fqid2 = DSSBundleFQID(source=source2, + uuid='3ccbc8a7-009e-4bc6-8854-da47bffae862', + version=bundle1.version) + bundle2 = attrs.evolve(bundle1, fqid=fqid2) + with self.assertRaises(AssertionError) as cm: + self._index_bundle(bundle2) + expected = R( + 'Entity has an invalid number of sources', + CataloguedEntityReference(entity_type='cell_suspensions', + entity_id='412898c5-5b9b-4907-b07c-e9b89666e204', + catalog='test'), + { + DocumentSource(id=source1.id, spec=source1.spec), + DocumentSource(id=source2.id, spec=source2.spec) + } + ) + self.assertEqual(expected, one(cm.exception.args)) + def test_deletion_before_addition(self): self._index_canned_bundle(self.new_bundle, delete=True) self._assert_index_counts(just_deletion=True) diff --git a/test/integration_test.py b/test/integration_test.py index 88dae78f18..17b835ef47 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -38,7 +38,6 @@ ContextManager, IO, Protocol, - TypedDict, cast, ) from unittest import ( @@ -100,6 +99,7 @@ ) from azul.collections import ( alist, + lookup, ) from azul.csp import ( CSP, @@ -211,13 +211,6 @@ def read(self, amount: int) -> bytes: ... def seek(self, amount: int) -> Any: ... -class FileInnerEntity(TypedDict): - uuid: str - version: str - name: str - size: int - - GET = 'GET' HEAD = 'HEAD' PUT = 'PUT' @@ -736,19 +729,11 @@ def _manifest_urls(self, urls.append(furl(response.headers['Location'])) return urls - def _get_one_inner_file(self, catalog: CatalogName) -> tuple[JSON, FileInnerEntity]: + def _get_one_inner_file(self, catalog: CatalogName) -> tuple[JSON, JSON]: outer_file = self._get_one_outer_file(catalog) inner_files: JSONs = outer_file['files'] inner_file = one(inner_files) - # FIXME: Two AnVIL snapshots with null in anvil_file.file_size column - # https://github.com/DataBiosphere/azul/issues/7243 - if inner_file['size'] is None: - inner_file = dict(inner_file, size=1) - assert isinstance(inner_file['uuid'], str), inner_file - assert isinstance(inner_file['version'], str), inner_file - assert isinstance(inner_file['name'], str), inner_file - assert isinstance(inner_file['size'], int), inner_file - return outer_file, cast(FileInnerEntity, inner_file) + return outer_file, inner_file @cache def _get_one_outer_file(self, catalog: CatalogName) -> JSON: @@ -1119,20 +1104,17 @@ def _test_repository_files(self, catalog: CatalogName): with self.subTest('repository_files', catalog=catalog): outer_file, inner_file = self._get_one_inner_file(catalog) source = self._source_spec(catalog, outer_file) - file_uuid, file_version = inner_file['uuid'], inner_file['version'] - endpoint_url = config.service_endpoint - file_url = endpoint_url.set(path=f'/fetch/repository/files/{file_uuid}', - args=dict(catalog=catalog, - version=file_version)) + file_url = furl(inner_file['url']) + self.assertEqual(file_url.path.segments[0], 'repository') + # FIXME: Use _check_endpoint() instead + # https://github.com/DataBiosphere/azul/issues/7373 + file_url.path.segments.insert(0, 'fetch') response = self._get_url_unchecked(GET, file_url) if response.status == 404: response = json.loads(response.data) # Phantom files lack DRS URIs and cannot be downloaded self.assertEqual('NotFoundError', response['Code']) - self.assertEqual(response['Message'], - f'File {file_uuid!r} with version {file_version!r} ' - f'was found in catalog {catalog!r}, ' - f'however no download is currently available') + self.assertIn('no download is currently available', response['Message']) else: self.assertEqual(200, response.status) response = json.loads(response.data) @@ -1144,15 +1126,16 @@ def _test_repository_files(self, catalog: CatalogName): response = self._get_url(GET, furl(response['Location']), stream=True) self._validate_file_response(response, source, inner_file) - def _file_ext(self, file: FileInnerEntity) -> str: + def _file_ext(self, file: JSON) -> str: # We believe that the file extension is a more reliable indicator than # the `format` metadata field. Note that this method preserves multipart # extensions and includes the leading '.', so the extension of # "foo.fastq.gz" is ".fastq.gz" instead of "gz" - suffixes = PurePath(file['name']).suffixes + file_name = lookup(file, 'file_name', 'name') + suffixes = PurePath(file_name).suffixes return ''.join(suffixes).lower() - def _validate_file_content(self, content: ReadableFileObject, file: FileInnerEntity): + def _validate_file_content(self, content: ReadableFileObject, file: JSON): file_ext = self._file_ext(file) if file_ext == '.fastq': self._validate_fastq_content(content) @@ -1160,12 +1143,17 @@ def _validate_file_content(self, content: ReadableFileObject, file: FileInnerEnt with gzip.open(content) as buf: self._validate_fastq_content(buf) else: - self.assertEqual(1 if file['size'] > 0 else 0, len(content.read(1))) + file_size = lookup(file, 'file_size', 'size') + # FIXME: Two AnVIL snapshots with null in anvil_file.file_size column + # https://github.com/DataBiosphere/azul/issues/7243 + if file_size is None: + file_size = 1 + self.assertEqual(1 if file_size > 0 else 0, len(content.read(1))) def _validate_file_response(self, response: urllib3.HTTPResponse, source: SourceSpec, - file: FileInnerEntity): + file: JSON): """ Note: The response object must have been obtained with stream=True """ @@ -1184,14 +1172,15 @@ def _validate_file_response(self, def _test_drs(self, catalog: CatalogName, source: SourceSpec, - file: FileInnerEntity + file: JSON ) -> None: repository_plugin = self.azul_client.repository_plugin(catalog) drs = repository_plugin.drs_client() + file_uuid = lookup(file, 'document_id', 'uuid') for access_method in AccessMethod: with self.subTest('drs', catalog=catalog, access_method=AccessMethod.https): - log.info('Resolving file %r with DRS using %r', file['uuid'], access_method) - drs_uri = f'drs://{config.api_lambda_domain("service")}/{file["uuid"]}' + log.info('Resolving file %r with DRS using %r', file_uuid, access_method) + drs_uri = f'drs://{config.api_lambda_domain("service")}/{file_uuid}' access = drs.get_object(drs_uri, access_method=access_method) self.assertIsNone(access.headers) if access.method is AccessMethod.https: @@ -1203,11 +1192,12 @@ def _test_drs(self, else: self.fail(access_method) - def _test_dos(self, catalog: CatalogName, file: FileInnerEntity): + def _test_dos(self, catalog: CatalogName, file: JSON): with self.subTest('dos', catalog=catalog): - log.info('Resolving file %s with DOS', file['uuid']) + file_uuid = lookup(file, 'document_id', 'uuid') + log.info('Resolving file %s with DOS', file_uuid) response = self._check_endpoint(method=GET, - path=drs.dos_object_url_path(file['uuid']), + path=drs.dos_object_url_path(file_uuid), args=dict(catalog=catalog)) json_data = json.loads(response)['data_object'] file_url = first(json_data['urls'])['url'] diff --git a/test/service/data/manifest/verbatim/pfb/anvil/pfb_entities.json b/test/service/data/manifest/verbatim/pfb/anvil/pfb_entities.json index c621824d6d..c4c5d418da 100644 --- a/test/service/data/manifest/verbatim/pfb/anvil/pfb_entities.json +++ b/test/service/data/manifest/verbatim/pfb/anvil/pfb_entities.json @@ -425,6 +425,7 @@ "DS-BDIS" ], "datarepo_row_id": "2370f948-2783-4eb6-afea-e022897f4dcf", + "datarepo_snapshot_id": "6c87f0e1-509d-46a4-b845-7584df39263b", "dataset_id": "52ee7665-7033-63f2-a8d9-ce8e32666739", "owner": [ "Debbie Nickerson" diff --git a/test/service/data/manifest/verbatim/pfb/anvil/pfb_schema.json b/test/service/data/manifest/verbatim/pfb/anvil/pfb_schema.json index 145d3d9f2b..cf6cfa43ca 100644 --- a/test/service/data/manifest/verbatim/pfb/anvil/pfb_schema.json +++ b/test/service/data/manifest/verbatim/pfb/anvil/pfb_schema.json @@ -532,6 +532,11 @@ "namespace": "anvil_dataset", "type": "string" }, + { + "name": "datarepo_snapshot_id", + "namespace": "anvil_dataset", + "type": "string" + }, { "name": "dataset_id", "namespace": "anvil_dataset", diff --git a/test/service/test_manifest.py b/test/service/test_manifest.py index 4371a0901e..4641bf1cec 100644 --- a/test/service/test_manifest.py +++ b/test/service/test_manifest.py @@ -504,7 +504,7 @@ def test_compact_manifest(self): self._drs_uri('f2b6c6f0-8d25-4aae-b255-1974cc110cfe', '2018-09-14T12:33:43.720332Z')), - ('file_url', + ('file_azul_url', self._file_url('5f9b45af-9a26-4b16-a785-7f2d1053dd7c', '2018-09-14T12:33:47.012715Z'), self._file_url('f2b6c6f0-8d25-4aae-b255-1974cc110cfe', @@ -653,8 +653,8 @@ def test_manifest_zarr(self): 'file_uuid': 'c1c4a2bc-b5fb-4083-af64-f5dec70d7f9d', 'file_drs_uri': self._drs_uri('c1c4a2bc-b5fb-4083-af64-f5dec70d7f9d', '2018-10-10T03:10:37.983672Z'), - 'file_url': self._file_url('c1c4a2bc-b5fb-4083-af64-f5dec70d7f9d', - '2018-10-10T03:10:37.983672Z'), + 'file_azul_url': self._file_url('c1c4a2bc-b5fb-4083-af64-f5dec70d7f9d', + '2018-10-10T03:10:37.983672Z'), 'specimen_from_organism.organ': 'brain' }, # Related files from zarray store @@ -664,8 +664,8 @@ def test_manifest_zarr(self): 'file_uuid': '54541cc5-9010-425b-9037-22e43948c97c', 'file_drs_uri': self._drs_uri('54541cc5-9010-425b-9037-22e43948c97c', '2018-10-10T03:10:38.239541Z'), - 'file_url': self._file_url('54541cc5-9010-425b-9037-22e43948c97c', - '2018-10-10T03:10:38.239541Z'), + 'file_azul_url': self._file_url('54541cc5-9010-425b-9037-22e43948c97c', + '2018-10-10T03:10:38.239541Z'), 'specimen_from_organism.organ': 'brain' }, { @@ -674,8 +674,8 @@ def test_manifest_zarr(self): 'file_uuid': '66b8f976-6f1e-45b3-bd97-069658c3c847', 'file_drs_uri': self._drs_uri('66b8f976-6f1e-45b3-bd97-069658c3c847', '2018-10-10T03:10:38.474167Z'), - 'file_url': self._file_url('66b8f976-6f1e-45b3-bd97-069658c3c847', - '2018-10-10T03:10:38.474167Z'), + 'file_azul_url': self._file_url('66b8f976-6f1e-45b3-bd97-069658c3c847', + '2018-10-10T03:10:38.474167Z'), 'specimen_from_organism.organ': 'brain' }, { @@ -684,8 +684,8 @@ def test_manifest_zarr(self): 'file_uuid': 'ac05d7fb-d6b9-4ab1-8c04-6211450dbb62', 'file_drs_uri': self._drs_uri('ac05d7fb-d6b9-4ab1-8c04-6211450dbb62', '2018-10-10T03:10:38.714461Z'), - 'file_url': self._file_url('ac05d7fb-d6b9-4ab1-8c04-6211450dbb62', - '2018-10-10T03:10:38.714461Z'), + 'file_azul_url': self._file_url('ac05d7fb-d6b9-4ab1-8c04-6211450dbb62', + '2018-10-10T03:10:38.714461Z'), 'specimen_from_organism.organ': 'brain' }, { @@ -694,8 +694,8 @@ def test_manifest_zarr(self): 'file_uuid': '0c518a52-f315-4ea2-beed-1c9d8f2d802b', 'file_drs_uri': self._drs_uri('0c518a52-f315-4ea2-beed-1c9d8f2d802b', '2018-10-10T03:10:39.039270Z'), - 'file_url': self._file_url('0c518a52-f315-4ea2-beed-1c9d8f2d802b', - '2018-10-10T03:10:39.039270Z'), + 'file_azul_url': self._file_url('0c518a52-f315-4ea2-beed-1c9d8f2d802b', + '2018-10-10T03:10:39.039270Z'), 'specimen_from_organism.organ': 'brain' }, { @@ -704,8 +704,8 @@ def test_manifest_zarr(self): 'file_uuid': '136108ab-277e-47a4-acc3-1feed8fb2f25', 'file_drs_uri': self._drs_uri('136108ab-277e-47a4-acc3-1feed8fb2f25', '2018-10-10T03:10:39.426609Z'), - 'file_url': self._file_url('136108ab-277e-47a4-acc3-1feed8fb2f25', - '2018-10-10T03:10:39.426609Z'), + 'file_azul_url': self._file_url('136108ab-277e-47a4-acc3-1feed8fb2f25', + '2018-10-10T03:10:39.426609Z'), 'specimen_from_organism.organ': 'brain' }, { @@ -714,8 +714,8 @@ def test_manifest_zarr(self): 'file_uuid': '0bef5419-739c-4a2c-aedb-43754d55d51c', 'file_drs_uri': self._drs_uri('0bef5419-739c-4a2c-aedb-43754d55d51c', '2018-10-10T03:10:39.642846Z'), - 'file_url': self._file_url('0bef5419-739c-4a2c-aedb-43754d55d51c', - '2018-10-10T03:10:39.642846Z'), + 'file_azul_url': self._file_url('0bef5419-739c-4a2c-aedb-43754d55d51c', + '2018-10-10T03:10:39.642846Z'), 'specimen_from_organism.organ': 'brain' }, { @@ -724,8 +724,8 @@ def test_manifest_zarr(self): 'file_uuid': '3a5f7299-1aa1-4060-9631-212c29b4d807', 'file_drs_uri': self._drs_uri('3a5f7299-1aa1-4060-9631-212c29b4d807', '2018-10-10T03:10:39.899615Z'), - 'file_url': self._file_url('3a5f7299-1aa1-4060-9631-212c29b4d807', - '2018-10-10T03:10:39.899615Z'), + 'file_azul_url': self._file_url('3a5f7299-1aa1-4060-9631-212c29b4d807', + '2018-10-10T03:10:39.899615Z'), 'specimen_from_organism.organ': 'brain' }, { @@ -734,8 +734,8 @@ def test_manifest_zarr(self): 'file_uuid': 'a8f0dc39-6019-4fc7-899d-4e34a48d03e5', 'file_drs_uri': self._drs_uri('a8f0dc39-6019-4fc7-899d-4e34a48d03e5', '2018-10-10T03:10:40.113268Z'), - 'file_url': self._file_url('a8f0dc39-6019-4fc7-899d-4e34a48d03e5', - '2018-10-10T03:10:40.113268Z'), + 'file_azul_url': self._file_url('a8f0dc39-6019-4fc7-899d-4e34a48d03e5', + '2018-10-10T03:10:40.113268Z'), 'specimen_from_organism.organ': 'brain' }, { @@ -744,8 +744,8 @@ def test_manifest_zarr(self): 'file_uuid': '68ba4711-1447-42ac-aa40-9c0e4cda1666', 'file_drs_uri': self._drs_uri('68ba4711-1447-42ac-aa40-9c0e4cda1666', '2018-10-10T03:10:40.583439Z'), - 'file_url': self._file_url('68ba4711-1447-42ac-aa40-9c0e4cda1666', - '2018-10-10T03:10:40.583439Z'), + 'file_azul_url': self._file_url('68ba4711-1447-42ac-aa40-9c0e4cda1666', + '2018-10-10T03:10:40.583439Z'), 'specimen_from_organism.organ': 'brain' }, { @@ -754,8 +754,8 @@ def test_manifest_zarr(self): 'file_uuid': '27e66328-e337-4bcd-ba15-7893ecaf841f', 'file_drs_uri': self._drs_uri('27e66328-e337-4bcd-ba15-7893ecaf841f', '2018-10-10T03:10:40.801631Z'), - 'file_url': self._file_url('27e66328-e337-4bcd-ba15-7893ecaf841f', - '2018-10-10T03:10:40.801631Z'), + 'file_azul_url': self._file_url('27e66328-e337-4bcd-ba15-7893ecaf841f', + '2018-10-10T03:10:40.801631Z'), 'specimen_from_organism.organ': 'brain' }, { @@ -764,8 +764,8 @@ def test_manifest_zarr(self): 'file_uuid': '2ab1a516-ef36-41b6-a78f-513361658feb', 'file_drs_uri': self._drs_uri('2ab1a516-ef36-41b6-a78f-513361658feb', '2018-10-10T03:10:40.958708Z'), - 'file_url': self._file_url('2ab1a516-ef36-41b6-a78f-513361658feb', - '2018-10-10T03:10:40.958708Z'), + 'file_azul_url': self._file_url('2ab1a516-ef36-41b6-a78f-513361658feb', + '2018-10-10T03:10:40.958708Z'), 'specimen_from_organism.organ': 'brain' }, { @@ -774,8 +774,8 @@ def test_manifest_zarr(self): 'file_uuid': '351970aa-bc4c-405e-a274-be9e08e42e98', 'file_drs_uri': self._drs_uri('351970aa-bc4c-405e-a274-be9e08e42e98', '2018-10-10T03:10:41.135992Z'), - 'file_url': self._file_url('351970aa-bc4c-405e-a274-be9e08e42e98', - '2018-10-10T03:10:41.135992Z'), + 'file_azul_url': self._file_url('351970aa-bc4c-405e-a274-be9e08e42e98', + '2018-10-10T03:10:41.135992Z'), 'specimen_from_organism.organ': 'brain' } ] @@ -1740,7 +1740,7 @@ def test_compact_manifest(self): self._drs_uri('v1_6c87f0e1-509d-46a4-b845-7584df39263b_8b722e88-8103-49c1-b351-e64fa7c6ab37') ), ( - 'files.azul_file_url', + 'files.azul_url', self._file_url('6b0f6c0f-5d80-4242-accb-840921351cd5', self.version), self._file_url('15b76f9c-6b46-433f-851d-34e89f1b9ba6', self.version), self._file_url('3b17377b-16b1-431c-9967-e5d01fc5923f', self.version) diff --git a/test/service/test_response.py b/test/service/test_response.py index 1d857c00f4..02e2adf8a2 100644 --- a/test/service/test_response.py +++ b/test/service/test_response.py @@ -137,6 +137,10 @@ def tearDownClass(cls): cls._teardown_indices() super().tearDownClass() + @property + def file_url_func(self): + return self.app_module.app.file_url + def _get_hits(self, entity_type: str, entity_id: str): """ Fetches hits from ES instance searching for a particular entity ID @@ -236,6 +240,12 @@ def test_response_stage_files(self): "sha256": "77337cb51b2e584b5ae1b99db6c163b988cbc5b894dda2f5d22424978c3bfc7a", "size": 195142097, "fileSource": None, + **{ + field: f'{self.base_url}/repository/files/' + f'7b07f99e-4a8a-4ad0-bd4f-db0d7a00c7bb' + f'?catalog=test&version=2018-11-02T11%3A33%3A44.698028Z' + for field in ['azul_url', 'url'] + }, "drs_uri": f"drs://{self._drs_domain_name}/" f"7b07f99e-4a8a-4ad0-bd4f-db0d7a00c7bb?version=2018-11-02T11%3A33%3A44.698028Z", "uuid": "7b07f99e-4a8a-4ad0-bd4f-db0d7a00c7bb", @@ -351,7 +361,8 @@ def test_response_stage_files(self): hits = self._get_hits('files', '0c5ac7c0-817e-40d4-b1b1-34c3d5cfecdb') stage = HCASearchResponseStage(service=self.index_service, entity_type='files', - catalog=self.catalog) + catalog=self.catalog, + file_url_func=self.file_url_func) response = stage.process_response((hits, self.paginations[n], {})) self.assertElasticEqual(responses[n], response) @@ -364,7 +375,8 @@ def test_response_stage_files_summaries(self): hits = self._get_hits('samples', 'a21dc760-a500-4236-bcff-da34a0e873d2') stage = HCASearchResponseStage(service=self.index_service, entity_type='samples', - catalog=self.catalog) + catalog=self.catalog, + file_url_func=self.file_url_func) response = stage.process_response((hits, self.paginations[0], {})) for hit in response['hits']: @@ -421,7 +433,8 @@ def test_response_stage_files_facets(self): # https://github.com/DataBiosphere/azul/issues/2970 stage = HCASearchResponseStage(service=self.index_service, entity_type='files', - catalog=self.catalog) + catalog=self.catalog, + file_url_func=self.file_url_func) facets = stage.make_facets(self.canned_aggs) expected_output = { "organ": { @@ -465,7 +478,8 @@ def test_response_stage_projects(self): hits = self._get_hits('projects', 'e8642221-4c2c-4fd7-b926-a68bce363c88') stage = HCASearchResponseStage(service=self.index_service, entity_type='projects', - catalog=self.catalog) + catalog=self.catalog, + file_url_func=self.file_url_func) response = stage.process_response((hits, self.paginations[0], self.canned_aggs)) expected_response = { @@ -699,7 +713,8 @@ def test_response_stage_projects_accessions(self): hits = self._get_hits('projects', '627cb0ba-b8a1-405a-b58f-0add82c3d635') stage = HCASearchResponseStage(service=self.index_service, entity_type='projects', - catalog=self.catalog) + catalog=self.catalog, + file_url_func=self.file_url_func) response = stage.process_response((hits, self.paginations[0], {})) expected_hits = [ { @@ -921,7 +936,8 @@ def test_response_stage_projects_celltype(self): hits = self._get_hits('projects', '250aef61-a15b-4d97-b8b4-54bb997c1d7d') stage = HCASearchResponseStage(service=self.index_service, entity_type='projects', - catalog=self.catalog) + catalog=self.catalog, + file_url_func=self.file_url_func) response = stage.process_response((hits, self.paginations[0], {})) cell_suspension = one(response['hits'][0]['cellSuspensions']) self.assertEqual(["Plasma cells"], cell_suspension['selectedCellType']) @@ -935,7 +951,8 @@ def test_response_stage_projects_cell_line(self): hits = self._get_hits('projects', 'c765e3f9-7cfc-4501-8832-79e5f7abd321') stage = HCASearchResponseStage(service=self.index_service, entity_type='projects', - catalog=self.catalog) + catalog=self.catalog, + file_url_func=self.file_url_func) response = stage.process_response((hits, self.paginations[0], {})) expected_cell_lines = { 'id': ['cell_line_Day7_hiPSC-CM_BioRep2', 'cell_line_GM18517'], @@ -964,7 +981,8 @@ def test_response_stage_files_file(self): hits = self._get_hits('files', '4015da8b-18d8-4f3c-b2b0-54f0b77ae80a') stage = HCASearchResponseStage(service=self.index_service, entity_type='files', - catalog=self.catalog) + catalog=self.catalog, + file_url_func=self.file_url_func) response = stage.process_response((hits, self.paginations[0], {})) expected_file = { 'contentDescription': ['RNA sequence'], @@ -975,6 +993,12 @@ def test_response_stage_files_file(self): 'sha256': '709fede4736213f0f71ae4d76719fd51fa402a9112582a4c52983973cb7d7e47', 'size': 22819025, 'fileSource': None, + **{ + field: f'{self.base_url}/repository/files/' + f'a8b8479d-cfa9-4f74-909f-49552439e698' + f'?catalog=test&version=2019-10-09T17%3A22%3A51.560099Z' + for field in ['azul_url', 'url'] + }, 'drs_uri': f'drs://{self._drs_domain_name}/' f'a8b8479d-cfa9-4f74-909f-49552439e698?version=2019-10-09T17%3A22%3A51.560099Z', 'uuid': 'a8b8479d-cfa9-4f74-909f-49552439e698', @@ -2947,10 +2971,13 @@ def test_matrices_tree(self): path='bd98f428-881e-501a-ac16-24f27a68ce2f', args=dict(version='2021-02-11T23:11:45.000000Z') )), - 'url': str(self.base_url.set( - path='/repository/files/bd98f428-881e-501a-ac16-24f27a68ce2f', - args=dict(catalog='test', version='2021-02-11T23:11:45.000000Z') - )) + **{ + field: str(self.base_url.set( + path='/repository/files/bd98f428-881e-501a-ac16-24f27a68ce2f', + args=dict(catalog='test', version='2021-02-11T23:11:45.000000Z') + )) + for field in ['azul_url', 'url'] + } } ] } @@ -2979,10 +3006,13 @@ def test_matrices_tree(self): path='538faa28-3235-5e4b-a998-5672e2d964e8', args=dict(version='2020-12-03T10:39:17.144517Z') )), - 'url': str(self.base_url.set( - path='/repository/files/538faa28-3235-5e4b-a998-5672e2d964e8', - args=dict(catalog='test', version='2020-12-03T10:39:17.144517Z') - )) + **{ + field: str(self.base_url.set( + path='/repository/files/538faa28-3235-5e4b-a998-5672e2d964e8', + args=dict(catalog='test', version='2020-12-03T10:39:17.144517Z') + )) + for field in ['azul_url', 'url'] + } }, { # Supplementary file, source from submitter_id @@ -3001,10 +3031,13 @@ def test_matrices_tree(self): path='6c142250-567c-5b63-bd4f-0d78499863f8', args=dict(version='2020-12-03T10:39:17.144517Z') )), - 'url': str(self.base_url.set( - path='/repository/files/6c142250-567c-5b63-bd4f-0d78499863f8', - args=dict(catalog='test', version='2020-12-03T10:39:17.144517Z') - )) + **{ + field: str(self.base_url.set( + path='/repository/files/6c142250-567c-5b63-bd4f-0d78499863f8', + args=dict(catalog='test', version='2020-12-03T10:39:17.144517Z') + )) + for field in ['azul_url', 'url'] + } }, { # Supplementary file, source from submitter_id @@ -3023,10 +3056,13 @@ def test_matrices_tree(self): path='8d2ba1c1-bc9f-5c2a-a74d-fe5e09bdfb18', args=dict(version='2020-12-03T10:39:17.144517Z') )), - 'url': str(self.base_url.set( - path='/repository/files/8d2ba1c1-bc9f-5c2a-a74d-fe5e09bdfb18', - args=dict(catalog='test', version='2020-12-03T10:39:17.144517Z') - )) + **{ + field: str(self.base_url.set( + path='/repository/files/8d2ba1c1-bc9f-5c2a-a74d-fe5e09bdfb18', + args=dict(catalog='test', version='2020-12-03T10:39:17.144517Z') + )) + for field in ['azul_url', 'url'] + } } ] } @@ -3064,10 +3100,13 @@ def test_matrices_tree(self): path='87f31102-ebbc-5875-abdf-4fa5cea48e8d', args=dict(version='2021-02-10T16:56:40.419579Z') )), - 'url': str(self.base_url.set( - path='/repository/files/87f31102-ebbc-5875-abdf-4fa5cea48e8d', - args=dict(catalog='test', version='2021-02-10T16:56:40.419579Z') - )) + **{ + field: str(self.base_url.set( + path='/repository/files/87f31102-ebbc-5875-abdf-4fa5cea48e8d', + args=dict(catalog='test', version='2021-02-10T16:56:40.419579Z') + )) + for field in ['azul_url', 'url'] + } }, { # Supplementary file, source from submitter_id @@ -3086,10 +3125,13 @@ def test_matrices_tree(self): path='733318e0-19c2-51e8-9ad6-d94ad562dd46', args=dict(version='2021-02-10T16:56:40.419579Z') )), - 'url': str(self.base_url.set( - path='/repository/files/733318e0-19c2-51e8-9ad6-d94ad562dd46', - args=dict(catalog='test', version='2021-02-10T16:56:40.419579Z') - )) + **{ + field: str(self.base_url.set( + path='/repository/files/733318e0-19c2-51e8-9ad6-d94ad562dd46', + args=dict(catalog='test', version='2021-02-10T16:56:40.419579Z') + )) + for field in ['azul_url', 'url'] + } }, { # Supplementary file, source from submitter_id @@ -3108,10 +3150,13 @@ def test_matrices_tree(self): path='c59e2de5-01fe-56eb-be56-679ed14161bf', args=dict(version='2021-02-10T16:56:40.419579Z') )), - 'url': str(self.base_url.set( - path='/repository/files/c59e2de5-01fe-56eb-be56-679ed14161bf', - args=dict(catalog='test', version='2021-02-10T16:56:40.419579Z') - )) + **{ + field: str(self.base_url.set( + path='/repository/files/c59e2de5-01fe-56eb-be56-679ed14161bf', + args=dict(catalog='test', version='2021-02-10T16:56:40.419579Z') + )) + for field in ['azul_url', 'url'] + } }, { # Supplementary file, source from submitter_id @@ -3130,10 +3175,13 @@ def test_matrices_tree(self): path='68bda896-3b3e-5f2a-9212-f4030a0f37e2', args=dict(version='2021-02-10T16:56:40.419579Z') )), - 'url': str(self.base_url.set( - path='/repository/files/68bda896-3b3e-5f2a-9212-f4030a0f37e2', - args=dict(catalog='test', version='2021-02-10T16:56:40.419579Z') - )) + **{ + field: str(self.base_url.set( + path='/repository/files/68bda896-3b3e-5f2a-9212-f4030a0f37e2', + args=dict(catalog='test', version='2021-02-10T16:56:40.419579Z') + )) + for field in ['azul_url', 'url'] + } }, { # Supplementary file, source from submitter_id @@ -3152,10 +3200,13 @@ def test_matrices_tree(self): path='0c5ab869-da2d-5c11-b4ae-f978a052899f', args=dict(version='2021-02-10T16:56:40.419579Z') )), - 'url': str(self.base_url.set( - path='/repository/files/0c5ab869-da2d-5c11-b4ae-f978a052899f', - args=dict(catalog='test', version='2021-02-10T16:56:40.419579Z') - )) + **{ + field: str(self.base_url.set( + path='/repository/files/0c5ab869-da2d-5c11-b4ae-f978a052899f', + args=dict(catalog='test', version='2021-02-10T16:56:40.419579Z') + )) + for field in ['azul_url', 'url'] + } }, { # Supplementary file, source from submitter_id @@ -3174,10 +3225,13 @@ def test_matrices_tree(self): path='cade4593-bfba-56ed-80ab-080d0de7d5a4', args=dict(version='2021-02-10T16:56:40.419579Z') )), - 'url': str(self.base_url.set( - path='/repository/files/cade4593-bfba-56ed-80ab-080d0de7d5a4', - args=dict(catalog='test', version='2021-02-10T16:56:40.419579Z') - )) + **{ + field: str(self.base_url.set( + path='/repository/files/cade4593-bfba-56ed-80ab-080d0de7d5a4', + args=dict(catalog='test', version='2021-02-10T16:56:40.419579Z') + )) + for field in ['azul_url', 'url'] + } }, { # Supplementary file, source from submitter_id @@ -3196,10 +3250,13 @@ def test_matrices_tree(self): path='5b465aad-0981-5152-b468-e615e20f5884', args=dict(version='2021-02-10T16:56:40.419579Z') )), - 'url': str(self.base_url.set( - path='/repository/files/5b465aad-0981-5152-b468-e615e20f5884', - args=dict(catalog='test', version='2021-02-10T16:56:40.419579Z') - )) + **{ + field: str(self.base_url.set( + path='/repository/files/5b465aad-0981-5152-b468-e615e20f5884', + args=dict(catalog='test', version='2021-02-10T16:56:40.419579Z') + )) + for field in ['azul_url', 'url'] + } }, { # Supplementary file, source from submitter_id @@ -3218,10 +3275,13 @@ def test_matrices_tree(self): path='b905c8be-2e2d-592c-8481-3eb7a87c6484', args=dict(version='2021-02-10T16:56:40.419579Z') )), - 'url': str(self.base_url.set( - path='/repository/files/b905c8be-2e2d-592c-8481-3eb7a87c6484', - args=dict(catalog='test', version='2021-02-10T16:56:40.419579Z') - )) + **{ + field: str(self.base_url.set( + path='/repository/files/b905c8be-2e2d-592c-8481-3eb7a87c6484', + args=dict(catalog='test', version='2021-02-10T16:56:40.419579Z') + )) + for field in ['azul_url', 'url'] + } } ] } diff --git a/test/service/test_response_anvil.py b/test/service/test_response_anvil.py index 7b34f71b26..6531f90453 100644 --- a/test/service/test_response_anvil.py +++ b/test/service/test_response_anvil.py @@ -1257,19 +1257,18 @@ def test_entity_indices(self): 'reference_assembly': [None], 'file_name': '307500.merged.matefixed.sorted.markeddups.recal.g.vcf.gz', 'is_supplementary': False, - 'version': self.version, - 'uuid': '15b76f9c-6b46-433f-851d-34e89f1b9ba6', - 'size': 213021639, - 'name': '307500.merged.matefixed.sorted.markeddups.recal.g.vcf.gz', 'crc32': '', 'sha256': '', 'accessible': True, 'drs_uri': f'drs://{self._drs_domain_name}/v1_6c87f0e1-509d-46a4-b845-7584df39263b_' f'1e269f04-4347-4188-b060-1dcc69e71d67', - 'url': str(self.base_url.set( - path='/repository/files/15b76f9c-6b46-433f-851d-34e89f1b9ba6', - args=dict(catalog='test', version=self.version) - )) + **{ + field: str(self.base_url.set( + path='/repository/files/15b76f9c-6b46-433f-851d-34e89f1b9ba6', + args=dict(catalog='test', version=self.version) + )) + for field in ['azul_url', 'url'] + } } ] }, @@ -1340,19 +1339,18 @@ def test_entity_indices(self): 'reference_assembly': [None], 'file_name': '307500.merged.matefixed.sorted.markeddups.recal.bam', 'is_supplementary': False, - 'version': self.version, - 'uuid': '3b17377b-16b1-431c-9967-e5d01fc5923f', - 'size': 3306845592, - 'name': '307500.merged.matefixed.sorted.markeddups.recal.bam', 'crc32': '', 'sha256': '', 'accessible': True, 'drs_uri': f'drs://{self._drs_domain_name}/v1_6c87f0e1-509d-46a4-b845-7584df39263b_' f'8b722e88-8103-49c1-b351-e64fa7c6ab37', - 'url': str(self.base_url.set( - path='/repository/files/3b17377b-16b1-431c-9967-e5d01fc5923f', - args=dict(catalog='test', version=self.version) - )) + **{ + field: str(self.base_url.set( + path='/repository/files/3b17377b-16b1-431c-9967-e5d01fc5923f', + args=dict(catalog='test', version=self.version) + )) + for field in ['azul_url', 'url'] + } } ] }