Merger pipeline (ACCC register update email) #1992
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Merger Pipeline | |
| run-name: >- | |
| ${{ | |
| github.event_name == 'repository_dispatch' && github.event.client_payload.custom_title || 'Merger pipeline' | |
| }} | |
| on: | |
| repository_dispatch: | |
| types: [new_merger_detected] | |
| workflow_dispatch: | |
| inputs: | |
| all_mergers: | |
| description: 'Process all mergers (not just new/changed ones)' | |
| required: false | |
| default: false | |
| type: boolean | |
| conflict_retry_count: | |
| description: 'Internal: chained-retry counter after a rebase conflict against main. Leave as 0.' | |
| required: false | |
| default: '0' | |
| type: string | |
| push: | |
| branches: | |
| - main | |
| schedule: | |
| - cron: '8 7,11,15,19 * * 1-5' | |
| timezone: 'Australia/Sydney' | |
| - cron: '8 7 * * 0' | |
| timezone: 'Australia/Sydney' | |
| permissions: | |
| contents: write | |
| issues: write | |
| pull-requests: write # needed by the folded-in detect-* PR steps | |
| actions: write # needed to trigger a fresh run after a rebase conflict | |
| # Serialise pipeline runs on main. Two concurrent runs can otherwise race | |
| # on the same files and rely on the post-rebase HTML re-clean / commit-drop | |
| # logic below to recover. Forcing them to queue keeps that recovery code as | |
| # a belt-and-braces guard rather than a hot path. | |
| concurrency: | |
| group: pipeline-main | |
| cancel-in-progress: false | |
| jobs: | |
| pipeline: | |
| runs-on: ubuntu-latest | |
| # Without this, a hung step (e.g. an apt-get fetch that stalls) runs to | |
| # GitHub's default 6-hour job timeout before failing. Because runs are | |
| # serialised via the concurrency group above, that also blocks every | |
| # queued run behind it for the same 6 hours. Normal runs take ~1-3 | |
| # minutes; with the apt cache above, even a LibreOffice install run | |
| # should stay well under this, with headroom for a cold-cache run. | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v7 | |
| # Go is only ever used to build pup, so the cache is checked first and | |
| # the toolchain is installed only when the binary has to be rebuilt. | |
| - name: Cache pup binary | |
| id: cache-pup | |
| uses: actions/cache@v6 | |
| with: | |
| path: ~/go/bin/pup | |
| key: ${{ runner.os }}-pup-v0.4.0 | |
| # The floor is set by pup's dependencies, not by pup: it ships no go.mod, | |
| # so `go install` resolves golang.org/x/net, fatih/color et al at their | |
| # latest versions, and those now require Go 1.25. setup-go pins | |
| # GOTOOLCHAIN=local, so too low a floor fails a cold-cache install. | |
| - name: Set up Go | |
| if: steps.cache-pup.outputs.cache-hit != 'true' | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: '>=1.25' | |
| cache: false | |
| # Pinned rather than @latest, which resolves to this same version today: | |
| # the cache key names v0.4.0, so a future pup release must not be able to | |
| # install itself under a key that claims otherwise. | |
| - name: Install pup HTML parser | |
| if: steps.cache-pup.outputs.cache-hit != 'true' | |
| run: go install github.com/ericchiang/pup@v0.4.0 | |
| - name: Add Go bin to PATH | |
| run: echo "${HOME}/go/bin" >> $GITHUB_PATH | |
| - name: Set up Python | |
| uses: actions/setup-python@v7 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r scripts/requirements.txt | |
| # --- Scrape --- | |
| - name: Run scraper | |
| env: | |
| SCRAPE_REPORT_DIR: ${{ runner.temp }}/scrape-report | |
| run: | | |
| chmod +x ./scripts/scrape/scrape.sh | |
| if [ "${{ inputs.all_mergers }}" = "true" ]; then | |
| ./scripts/scrape/scrape.sh --all | |
| else | |
| ./scripts/scrape/scrape.sh | |
| fi | |
| # Record which merger IDs this run actually fetched, which of their | |
| # pages changed, and which matters were skipped past cutoff, so a run | |
| # can be spot checked against an ACCC register notification email. | |
| # Runs even when the scraper failed, to summarise what it got through. | |
| - name: Summarise scraped mergers | |
| if: ${{ !cancelled() }} | |
| env: | |
| SCRAPE_REPORT_DIR: ${{ runner.temp }}/scrape-report | |
| # Passed through env, never interpolated into the script: the | |
| # subject/matter IDs are attacker-influencable content from an | |
| # inbound email (matter_ids is regex-extracted from the body by | |
| # the accc-register-watcher Worker, but treat it the same way). | |
| TRIGGER_EVENT: ${{ github.event_name }} | |
| EMAIL_SUBJECT: ${{ github.event.client_payload.email_subject }} | |
| EMAIL_MATTER_IDS: ${{ github.event.client_payload.matter_ids }} | |
| run: | | |
| { | |
| echo "### Scrape trigger" | |
| echo "" | |
| printf -- '- Event: `%s`\n' "$TRIGGER_EVENT" | |
| if [ -n "$EMAIL_SUBJECT" ]; then | |
| printf -- '- ACCC register email: %s\n' "$EMAIL_SUBJECT" | |
| fi | |
| if [ -n "$EMAIL_MATTER_IDS" ]; then | |
| printf -- '- Matter IDs mentioned in email: `%s`\n' "$EMAIL_MATTER_IDS" | |
| fi | |
| echo "" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| git status --porcelain -- data/raw/matters/ \ | |
| | awk '{print $NF}' > "${RUNNER_TEMP}/scrape-changed.txt" | |
| python -m scripts.scrape.scrape_summary \ | |
| --report-dir "$SCRAPE_REPORT_DIR" \ | |
| --changed-paths "${RUNNER_TEMP}/scrape-changed.txt" \ | |
| | tee -a "$GITHUB_STEP_SUMMARY" | |
| - name: Verify HTML cleaning | |
| run: | | |
| # Report any matter HTML files modified by this scrape run that still | |
| # contain known uncleaned dynamic tokens. This catches cases where | |
| # clean_file did not run, or where a git rebase will later undo its | |
| # substitutions when two concurrent runs touched the same file. | |
| modified_html=() | |
| while IFS= read -r f; do | |
| modified_html+=("$f") | |
| done < <(git status --porcelain data/raw/matters/ 2>/dev/null \ | |
| | awk '{print $NF}' | grep '\.html$' || true) | |
| if [ ${#modified_html[@]} -eq 0 ]; then | |
| echo "No matter HTML files were modified by this scrape run." | |
| else | |
| echo "Checking ${#modified_html[@]} modified matter page(s) for uncleaned tokens..." | |
| bad_files=() | |
| for f in "${modified_html[@]}"; do | |
| if grep -qE \ | |
| 'js-view-dom-id-[a-f0-9]{64}|/(css|js)/[a-zA-Z]+_[A-Za-z0-9_-]{30,}\.(css|js)(\?|")' \ | |
| "$f" 2>/dev/null; then | |
| bad_files+=("$f") | |
| fi | |
| done | |
| if [ ${#bad_files[@]} -gt 0 ]; then | |
| # ::warning:: so this surfaces as an annotation on the run rather | |
| # than as a line of plain text nobody scrolls to. | |
| for f in "${bad_files[@]}"; do | |
| echo "::warning file=${f}::Uncleaned dynamic content in ${f} — clean_file may not have run, or a concurrent pipeline run touching the same file will reintroduce raw tokens on rebase." | |
| done | |
| echo "WARNING: The following matter pages appear to have uncleaned dynamic content:" | |
| printf ' %s\n' "${bad_files[@]}" | |
| else | |
| echo "HTML cleaning check passed (${#modified_html[@]} file(s) verified)." | |
| fi | |
| fi | |
| - name: Check for scrape changes | |
| id: scrape-changes | |
| run: | | |
| git add -A | |
| if git diff --staged --quiet; then | |
| echo "changed=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "changed=true" >> $GITHUB_OUTPUT | |
| fi | |
| # --- Extract phase 1: HTML parse + attachment download --- | |
| # extract_mergers.py downloads attachment PDFs / DOCX files as a side | |
| # effect of parsing each matter page. We skip questionnaire/NOCC PDF | |
| # parsing here so we can convert any newly downloaded DOCX files first, | |
| # then do the PDF-parse pass in phase 2 below. | |
| - name: Run extraction (download phase) | |
| if: steps.scrape-changes.outputs.changed == 'true' || github.event_name != 'schedule' | |
| run: | | |
| if [ "${{ inputs.all_mergers }}" = "true" ]; then | |
| python -m scripts.extract_mergers --all --skip-pdf-enrich | |
| else | |
| python -m scripts.extract_mergers --skip-pdf-enrich | |
| fi | |
| # --- Convert DOCX to PDF --- | |
| # Runs whenever there are unconverted files, catching both new downloads and stragglers. | |
| - name: Find unconverted DOCX files | |
| id: find-docx | |
| # Always run so we also catch stragglers from a previous conversion | |
| # failure, even on scheduled runs that found no scrape changes. | |
| run: | | |
| > "${RUNNER_TEMP}/unconverted.txt" | |
| while IFS= read -r -d '' docx; do | |
| pdf="${docx%.docx}.pdf" | |
| if [ ! -f "$pdf" ]; then | |
| echo "$docx" >> "${RUNNER_TEMP}/unconverted.txt" | |
| fi | |
| done < <(find data/raw/matters -name "*.docx" -type f -print0 2>/dev/null) | |
| if [ -s "${RUNNER_TEMP}/unconverted.txt" ]; then | |
| echo "Found unconverted DOCX files:" | |
| cat "${RUNNER_TEMP}/unconverted.txt" | |
| echo "has_unconverted=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "No unconverted DOCX files found" | |
| echo "has_unconverted=false" >> $GITHUB_OUTPUT | |
| fi | |
| # Everything from the enrich phase to the commit shares one condition: | |
| # phase 1 ran (scrape changes, or any trigger other than the schedule), | |
| # or we are about to convert DOCX stragglers whose PDFs the enrich pass | |
| # needs to see. Evaluate it once here rather than repeating the three | |
| # clauses on each of those steps, where they could drift apart. | |
| - name: Decide whether to run the enrich phase | |
| id: enrich-gate | |
| env: | |
| SCRAPE_CHANGED: ${{ steps.scrape-changes.outputs.changed }} | |
| HAS_UNCONVERTED: ${{ steps.find-docx.outputs.has_unconverted }} | |
| IS_SCHEDULE: ${{ github.event_name == 'schedule' }} | |
| run: | | |
| if [ "$SCRAPE_CHANGED" = "true" ] \ | |
| || [ "$HAS_UNCONVERTED" = "true" ] \ | |
| || [ "$IS_SCHEDULE" != "true" ]; then | |
| echo "run=true" >> "$GITHUB_OUTPUT" | |
| echo "Enrich phase will run (scrape_changed=${SCRAPE_CHANGED}, unconverted_docx=${HAS_UNCONVERTED}, scheduled=${IS_SCHEDULE})" | |
| else | |
| echo "run=false" >> "$GITHUB_OUTPUT" | |
| echo "Scheduled run with no scrape changes and no unconverted DOCX — skipping enrich phase" | |
| fi | |
| # The runner's apt mirror has been slow enough on some days to make this | |
| # install take 10+ minutes on its own (run #1690: 96.7 MB at 160 kB/s; | |
| # run #1693 stalled on it entirely and hit the 6-hour job timeout). | |
| # libreoffice-writer's package set rarely changes, so caching the | |
| # downloaded .debs sidesteps the mirror on a cache hit. | |
| # | |
| # The cache is keyed on the candidate version apt would install, so it | |
| # is an exact hit on an unchanged package set (actions/cache skips the | |
| # save on an exact hit) and misses cleanly on an upstream version bump. | |
| # Resolving that needs the package lists, so the `apt-get update` the | |
| # install used to do lives here instead of being run twice. | |
| - name: Resolve LibreOffice package version | |
| id: libreoffice-version | |
| if: steps.find-docx.outputs.has_unconverted == 'true' | |
| run: | | |
| sudo apt-get update | |
| version=$(apt-cache policy libreoffice-writer | awk '/Candidate:/ {print $2}') | |
| # A missing candidate would silently key every run the same; fail | |
| # loudly instead so a broken mirror doesn't poison the cache. | |
| if [ -z "$version" ] || [ "$version" = "(none)" ]; then | |
| echo "Could not resolve a libreoffice-writer candidate version" >&2 | |
| exit 1 | |
| fi | |
| echo "Candidate libreoffice-writer version: ${version}" | |
| echo "version=${version}" >> "$GITHUB_OUTPUT" | |
| # Cached into a runner-owned directory, NOT /var/cache/apt/archives: | |
| # actions/cache runs unprivileged, so untarring straight into that | |
| # root-owned directory failed with "Cannot open: Permission denied" on | |
| # every .deb and the restore was abandoned ("Failed to restore ... exit | |
| # code 2"). The install then re-downloaded the whole ~96 MB set on every | |
| # run while the post-job step happily re-saved it, so the cache had been | |
| # costing a 96 MB upload per run and never once serving one. | |
| - name: Cache LibreOffice apt packages | |
| if: steps.find-docx.outputs.has_unconverted == 'true' | |
| uses: actions/cache@v6 | |
| with: | |
| path: ${{ runner.temp }}/apt-libreoffice | |
| key: ${{ runner.os }}-apt-libreoffice-writer-${{ steps.libreoffice-version.outputs.version }} | |
| restore-keys: | | |
| ${{ runner.os }}-apt-libreoffice-writer- | |
| - name: Install LibreOffice | |
| if: steps.find-docx.outputs.has_unconverted == 'true' | |
| env: | |
| APT_CACHE_DIR: ${{ runner.temp }}/apt-libreoffice | |
| run: | | |
| set -euo pipefail | |
| mkdir -p "$APT_CACHE_DIR" | |
| # Seed apt's own archive directory from the cache. apt serves any | |
| # .deb already sitting there and downloads only what is missing or | |
| # superseded, so a partial (restore-keys) hit still helps. | |
| shopt -s nullglob | |
| cached=("$APT_CACHE_DIR"/*.deb) | |
| shopt -u nullglob | |
| if [ ${#cached[@]} -gt 0 ]; then | |
| echo "Seeding apt archives with ${#cached[@]} cached .deb(s)" | |
| sudo cp -f "${cached[@]}" /var/cache/apt/archives/ | |
| else | |
| echo "No cached .deb files — installing from the mirror" | |
| fi | |
| sudo apt-get install -y libreoffice-writer --no-install-recommends | |
| # Hand the .debs back to the runner-owned directory so the post-job | |
| # save can read them without sudo. | |
| shopt -s nullglob | |
| fetched=(/var/cache/apt/archives/*.deb) | |
| shopt -u nullglob | |
| if [ ${#fetched[@]} -gt 0 ]; then | |
| sudo cp -f "${fetched[@]}" "$APT_CACHE_DIR"/ | |
| fi | |
| sudo chown -R "$(id -u):$(id -g)" "$APT_CACHE_DIR" | |
| echo "Cache directory now holds $(ls -1 "$APT_CACHE_DIR"/*.deb 2>/dev/null | wc -l) .deb file(s)" | |
| - name: Convert DOCX to PDF | |
| if: steps.find-docx.outputs.has_unconverted == 'true' | |
| run: | | |
| while IFS= read -r docx; do | |
| if [ -n "$docx" ]; then | |
| echo "Converting: $docx" | |
| dir=$(dirname "$docx") | |
| libreoffice --headless --convert-to pdf --outdir "$dir" "$docx" | |
| fi | |
| done < "${RUNNER_TEMP}/unconverted.txt" | |
| # --- Install Tesseract OCR, only if a pending Phase 2 Notice needs it --- | |
| # Most notices have a full text layer and never touch the OCR fallback | |
| # in parse_phase2_notice.py; a matter is only pending here the first | |
| # time its notice is seen, so this (and the OCR itself) never re-runs | |
| # for a matter that's already been parsed, e.g. Ampol-EG Australia. | |
| - name: Check if Phase 2 Notice OCR is needed | |
| id: check-ocr | |
| run: | | |
| needs_ocr=$(python -m scripts.check_phase2_notice_ocr_needed) | |
| echo "needs_ocr=$needs_ocr" >> "$GITHUB_OUTPUT" | |
| - name: Install Tesseract OCR | |
| if: steps.check-ocr.outputs.needs_ocr == 'true' | |
| run: sudo apt-get update && sudo apt-get install -y tesseract-ocr --no-install-recommends | |
| # --- Extract phase 2: PDF parsing + downstream generation --- | |
| # Loads the mergers.json written by phase 1, runs questionnaire/NOCC | |
| # parsing (now seeing any newly-converted PDFs), auto-fixes missing | |
| # questionnaire dates, and rewrites mergers.json. Downstream generators | |
| # run exactly once here. | |
| - name: Run extraction (enrich phase) and downstream generation | |
| # Run if phase 1 ran (scrape changes or non-schedule trigger) OR if | |
| # we just converted DOCX stragglers (so questionnaire/NOCC parsing | |
| # picks them up). | |
| if: steps.enrich-gate.outputs.run == 'true' | |
| run: | | |
| python -m scripts.enrich_pdfs | |
| python -m scripts.generate.generate_similar_mergers | |
| python -m scripts.generate.generate_static_data | |
| ./scripts/generate/generate-cli-data.sh | |
| python -m scripts.generate.generate_rss_feed | |
| # Cloudflare Pages rejects an entire deployment if any single asset is | |
| # over 25 MiB, so a scanned exhibit that lands over it takes the site | |
| # build down until it's shrunk. Do that before the files are committed. | |
| # Ghostscript is only installed when there's something to compress, which | |
| # is rare — the same pattern as the Tesseract step above. | |
| - name: Compress oversized PDFs | |
| if: steps.enrich-gate.outputs.run == 'true' | |
| run: | | |
| if find data/raw/matters -type f -name '*.pdf' -size +25M | grep -q .; then | |
| sudo apt-get update | |
| sudo apt-get install -y ghostscript --no-install-recommends | |
| python -m scripts.compress_pdfs | |
| else | |
| echo "No PDFs over the 25 MiB Pages asset limit" | |
| fi | |
| - name: Stage extracted data | |
| if: steps.enrich-gate.outputs.run == 'true' | |
| run: | | |
| git add data/processed/mergers.json data/processed/questionnaire_data.json \ | |
| data/processed/nocc_data.json data/processed/similar_mergers.json \ | |
| data/processed/phase1_estimates.json \ | |
| data/raw/matters/ frontend/public/data/ \ | |
| frontend/public/feed.xml data/output/ \ | |
| data/frozen_events_mergers.json | |
| # --- Single commit --- | |
| - name: Commit all changes | |
| id: commit | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| RETRY_COUNT: ${{ inputs.conflict_retry_count || '0' }} | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| timestamp=$(TZ='Australia/Sydney' date) | |
| if git diff --staged --quiet; then | |
| echo "No changes to commit" | |
| else | |
| git commit -m "Update merger data: ${timestamp}" | |
| # Scoped to main on purpose. A bare `git pull` uses the default | |
| # refspec and fetches *every* branch into this depth-1 clone — | |
| # including cli-dist, whose multi-MB cli.sqlite this same job | |
| # force-pushes fresh each run, so it is never already present. | |
| # Measured at 42s of a 105s run (run #1847) to fetch two branches | |
| # this step does not read, only to report main hadn't moved. | |
| if ! git pull --rebase --autostash origin main; then | |
| git rebase --abort | |
| # A real conflict here means something else pushed to main while | |
| # this run was scraping/extracting - hand-merging the JSON isn't | |
| # safe, so give up on this run's commit and kick off a fresh run | |
| # instead. It'll check out the now-current main and re-scrape/ | |
| # re-extract on top of it, so it won't hit the same conflict. | |
| # Capped so a misbehaving concurrent writer can't cause an | |
| # infinite chain of retriggers. | |
| next_retry=$((RETRY_COUNT + 1)) | |
| if [ "$next_retry" -le 3 ]; then | |
| echo "Rebase conflict against a concurrent push to main (retry ${next_retry}/3) - triggering a fresh pipeline run" | |
| gh workflow run pipeline.yml --ref main -f "conflict_retry_count=${next_retry}" | |
| else | |
| echo "Rebase conflict against a concurrent push to main - already retried ${RETRY_COUNT} times, giving up and leaving this to the next scheduled run" | |
| fi | |
| exit 1 | |
| fi | |
| # After rebase, re-clean any HTML files that changed. A 3-way merge | |
| # during rebase can reintroduce raw dynamic tokens if both the local | |
| # commit and the pulled commits touched the same file. | |
| rebase_html=$(git diff HEAD^..HEAD --name-only 2>/dev/null | grep '\.html$' || true) | |
| if [ -n "$rebase_html" ]; then | |
| echo "HTML files in rebased commit:" | |
| echo "$rebase_html" | sed 's/^/ /' | |
| while IFS= read -r f; do | |
| if [ -f "$f" ]; then | |
| ./scripts/scrape/scrape.sh --clean-file "$f" | |
| git add "$f" | |
| echo "Re-cleaned: $f" | |
| fi | |
| done <<< "$rebase_html" | |
| if ! git diff --staged --quiet; then | |
| echo "WARNING: Rebase introduced uncleaned HTML — amending commit to fix" | |
| git commit --amend --no-edit | |
| fi | |
| fi | |
| git push | |
| # Flag downstream cli-sqlite publish if cli-manifest.json is in | |
| # the pushed commit. The manifest is the tracked stand-in for the | |
| # bundle, which is gitignored — generate-cli-data.sh only rewrites | |
| # the manifest when the bundle content actually changed, so it is | |
| # an exact signal. Pushes from GITHUB_TOKEN don't fire other | |
| # workflows, so we publish inline rather than relying on a | |
| # paths-filtered push trigger on publish-cli-sqlite.yml. | |
| if git diff HEAD^..HEAD --name-only \ | |
| | grep -qx "data/output/cli/cli-manifest.json"; then | |
| echo "publish_cli=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| fi | |
| # --- Publish cli.sqlite to the cli-dist orphan branch --- | |
| # Inlined here (rather than a separate workflow triggered by push to | |
| # cli-bundle.json) because pushes from the default GITHUB_TOKEN do not | |
| # fire downstream workflow runs. | |
| - name: Build CLI SQLite | |
| if: steps.commit.outputs.publish_cli == 'true' && github.ref == 'refs/heads/main' | |
| run: | | |
| mkdir -p "${RUNNER_TEMP}/cli-dist" | |
| python -m scripts.generate.build_cli_sqlite \ | |
| --bundle data/output/cli/cli-bundle.json \ | |
| --output-dir "${RUNNER_TEMP}/cli-dist" | |
| echo "--- Manifest ---" | |
| cat "${RUNNER_TEMP}/cli-dist/cli-manifest.json" | |
| - name: Force-push cli.sqlite to cli-dist branch | |
| if: steps.commit.outputs.publish_cli == 'true' && github.ref == 'refs/heads/main' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| work="${RUNNER_TEMP}/cli-dist-publish" | |
| mkdir -p "$work" | |
| cp "${RUNNER_TEMP}/cli-dist/cli.sqlite" "$work/cli.sqlite" | |
| cp "${RUNNER_TEMP}/cli-dist/cli-manifest.json" "$work/cli-manifest.json" | |
| cd "$work" | |
| git init -q -b cli-dist | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| version=$(python -c 'import json; print(json.load(open("cli-manifest.json"))["version"])') | |
| git add cli.sqlite cli-manifest.json | |
| git commit -q -m "Publish cli.sqlite (data version ${version})" | |
| remote="https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" | |
| for attempt in 1 2 3 4; do | |
| if git push --force "$remote" HEAD:cli-dist; then | |
| echo "Push succeeded on attempt ${attempt}" | |
| break | |
| fi | |
| if [ "$attempt" -eq 4 ]; then | |
| echo "Push failed after 4 attempts" >&2 | |
| exit 1 | |
| fi | |
| sleep $((2 ** attempt)) | |
| done | |
| # --- Open GitHub issues for auto-fixed event dates --- | |
| # The extraction script writes missing_event_dates.json when it auto-sets | |
| # a date on a catchable event (questionnaire, remedy offer, etc.). | |
| # We create one issue per affected merger so the owner can confirm the date. | |
| - name: Create issues for auto-fixed event dates | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| if [ ! -f data/processed/missing_event_dates.json ]; then | |
| echo "No missing event dates to report" | |
| exit 0 | |
| fi | |
| COUNT=$(jq '.issues | length' data/processed/missing_event_dates.json) | |
| if [ "$COUNT" -eq 0 ]; then | |
| echo "No issues to create" | |
| exit 0 | |
| fi | |
| echo "Creating $COUNT issue(s) for missing event dates..." | |
| gh label create "missing-event-date" \ | |
| --description "Event date was missing; auto-set by pipeline" \ | |
| --color "e4e669" \ | |
| --force | |
| # List the open issues once and match locally. Asking per merger cost | |
| # an API call each to almost always learn nothing, and gh's default | |
| # page size of 30 meant the guard silently stopped seeing older | |
| # issues once more than 30 were open, re-creating them. | |
| OPEN_TITLES="${RUNNER_TEMP}/missing-event-date-titles.txt" | |
| gh issue list \ | |
| --label "missing-event-date" \ | |
| --state open \ | |
| --limit 500 \ | |
| --json title \ | |
| --jq '.[].title' > "$OPEN_TITLES" | |
| for i in $(seq 0 $((COUNT - 1))); do | |
| MERGER_ID=$(jq -r ".issues[$i].merger_id" data/processed/missing_event_dates.json) | |
| # Skip if an open issue already exists for this merger (idempotent guard) | |
| if grep -qF -- "$MERGER_ID" "$OPEN_TITLES"; then | |
| echo "Issue already open for $MERGER_ID — skipping" | |
| continue | |
| fi | |
| TITLE=$(jq -r ".issues[$i].title" data/processed/missing_event_dates.json) | |
| jq -n \ | |
| --argjson idx "$i" \ | |
| --slurpfile data data/processed/missing_event_dates.json \ | |
| '{"title": $data[0].issues[$idx].title, | |
| "body": $data[0].issues[$idx].body, | |
| "labels": ["missing-event-date"]}' | \ | |
| gh api "repos/$GITHUB_REPOSITORY/issues" \ | |
| --method POST --input - | |
| # Keep the local guard true for the rest of this run. | |
| echo "$TITLE" >> "$OPEN_TITLES" | |
| echo "Created issue: $TITLE" | |
| done | |
| # --- Manage inferred-Phase-2 tracking issues --- | |
| # detect_inferred_phase_2 writes inferred_phase_2.json when a merger has a | |
| # Phase 2 notice but the ACCC register still shows Phase 1 (the site treats | |
| # it as Phase 2). We open a tracking issue so the owner can confirm, and | |
| # auto-close it once the register's stage catches up to Phase 2. | |
| - name: Manage inferred Phase 2 issues | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| FILE=data/processed/inferred_phase_2.json | |
| if [ ! -f "$FILE" ]; then | |
| echo "No inferred Phase 2 data to process" | |
| exit 0 | |
| fi | |
| # List this label's issues once, in both states, and match locally. | |
| # Asking per merger cost an API call each, and gh's default page size | |
| # of 30 meant both guards silently stopped seeing older issues once | |
| # the label had more than 30. One listing covers both guards: the | |
| # open-issue view is filtered out of it locally rather than costing a | |
| # second identical query. | |
| ALL_ISSUES="${RUNNER_TEMP}/inferred-phase-2-all.json" | |
| gh issue list \ | |
| --label "inferred-phase-2" \ | |
| --state all \ | |
| --limit 500 \ | |
| --json number,title,state > "$ALL_ISSUES" | |
| ALL_TITLES="${RUNNER_TEMP}/inferred-phase-2-titles.txt" | |
| jq -r '.[].title' "$ALL_ISSUES" > "$ALL_TITLES" | |
| OPEN_ISSUES="${RUNNER_TEMP}/inferred-phase-2-open.json" | |
| jq '[.[] | select(.state | ascii_upcase == "OPEN")]' "$ALL_ISSUES" > "$OPEN_ISSUES" | |
| # Open a tracking issue for each newly inferred Phase 2 merger. | |
| OPEN_COUNT=$(jq '.open | length' "$FILE") | |
| echo "Inferred Phase 2 mergers to open: $OPEN_COUNT" | |
| # Only needed to label an issue we are about to create. `confirmed` | |
| # only ever grows, so this step runs on every pipeline run with | |
| # nothing to open — no reason to spend the call then. | |
| if [ "$OPEN_COUNT" -gt 0 ]; then | |
| gh label create "inferred-phase-2" \ | |
| --description "Merger inferred as Phase 2 from a notice; awaiting ACCC stage update" \ | |
| --color "d4c5f9" \ | |
| --force | |
| fi | |
| for i in $(seq 0 $((OPEN_COUNT - 1))); do | |
| MERGER_ID=$(jq -r ".open[$i].merger_id" "$FILE") | |
| # Skip if an issue already exists in ANY state, so we never reopen one | |
| # the owner deliberately closed (e.g. after parties dropped out). | |
| if grep -qF -- "$MERGER_ID" "$ALL_TITLES"; then | |
| echo "Issue already exists for $MERGER_ID — skipping" | |
| continue | |
| fi | |
| TITLE=$(jq -r ".open[$i].title" "$FILE") | |
| jq -n \ | |
| --argjson idx "$i" \ | |
| --slurpfile data "$FILE" \ | |
| '{"title": $data[0].open[$idx].title, | |
| "body": $data[0].open[$idx].body, | |
| "labels": ["inferred-phase-2"]}' | \ | |
| gh api "repos/$GITHUB_REPOSITORY/issues" --method POST --input - | |
| # Keep the local guard true for the rest of this run. | |
| echo "$TITLE" >> "$ALL_TITLES" | |
| echo "Created inferred Phase 2 issue: $TITLE" | |
| done | |
| # Auto-close tracking issues where the ACCC register now shows Phase 2. | |
| # `confirmed` names every merger whose stage has caught up, so it only | |
| # grows: nearly all of them were closed on an earlier run. Match it | |
| # against the open issues already listed above rather than asking | |
| # GitHub about each one, and report what was actually closed. | |
| CLOSE_COUNT=$(jq '.confirmed | length' "$FILE") | |
| CLOSED=0 | |
| for i in $(seq 0 $((CLOSE_COUNT - 1))); do | |
| MERGER_ID=$(jq -r ".confirmed[$i]" "$FILE") | |
| NUMBER=$(jq -r --arg id "$MERGER_ID" \ | |
| 'map(select(.title | contains($id))) | .[0].number // empty' \ | |
| "$OPEN_ISSUES") | |
| if [ -n "$NUMBER" ]; then | |
| gh issue comment "$NUMBER" \ | |
| --body "The ACCC register now shows this merger at Phase 2 — closing automatically." | |
| gh issue close "$NUMBER" | |
| echo "Closed inferred Phase 2 issue #$NUMBER for $MERGER_ID" | |
| CLOSED=$((CLOSED + 1)) | |
| fi | |
| done | |
| echo "Confirmed Phase 2 mergers: ${CLOSE_COUNT}; tracking issues closed: ${CLOSED}" | |
| # --- Duplicate & related detection (create/refresh review PRs) --- | |
| # These blocks mirror the former standalone detect-duplicates.yml, | |
| # detect-related-mergers.yml and detect-related-parties.yml workflows, | |
| # which now only run on manual dispatch. Each one branches off the | |
| # latest main (origin/main, i.e. this run's push), applies its own | |
| # suggestions to its own processed-data file, force-pushes a well-known | |
| # fix branch and opens/updates — or auto-closes — a review PR. They edit | |
| # different files (mergers.json / related_mergers.json / | |
| # related_parties.json) and each commits only its own file, so they never | |
| # collide with each other or with the pipeline's own commit to main above. | |
| # Running last keeps main's working tree intact for every step before this. | |
| - name: Record base for detection | |
| id: detect_base | |
| run: | | |
| # Base the fix branches off the true latest main so the PR diffs are | |
| # clean whether or not this run committed. With the pipeline | |
| # serialised on main, origin/main is stable for the rest of the job. | |
| git fetch origin main | |
| echo "sha=$(git rev-parse origin/main)" >> "$GITHUB_OUTPUT" | |
| # -- Duplicate events → fix/duplicate-events -- | |
| - name: Duplicate events review PR | |
| uses: ./.github/actions/detection-pr | |
| with: | |
| base-sha: ${{ steps.detect_base.outputs.sha }} | |
| branch: fix/duplicate-events | |
| heading: Duplicate events detection | |
| command: >- | |
| python -m scripts.detect.detect_duplicates | |
| --apply-fixes | |
| --pr-markdown pr_body.md | |
| data-file: data/processed/mergers.json | |
| commit-message: "fix: remove duplicate events detected on {date}" | |
| pr-title: "fix: remove duplicate events ({date})" | |
| close-comment: >- | |
| All duplicate events resolved as of {date}. Closing automatically. | |
| github-token: ${{ github.token }} | |
| ntfy-topic: ${{ secrets.NTFY_TOPIC }} | |
| ntfy-token: ${{ secrets.NTFY_TOKEN }} | |
| ntfy-server: ${{ vars.NTFY_SERVER || 'https://ntfy.sh' }} | |
| # -- Related mergers → fix/related-mergers -- | |
| - name: Related mergers review PR | |
| id: relm | |
| uses: ./.github/actions/detection-pr | |
| with: | |
| base-sha: ${{ steps.detect_base.outputs.sha }} | |
| branch: fix/related-mergers | |
| heading: Related mergers detection | |
| command: >- | |
| python -m scripts.detect.detect_related_mergers | |
| --summary | |
| --apply-suggestions | |
| --pr-markdown pr_body.md | |
| --issue-markdown pr_issue_body.md | |
| --meta-json pr_meta.json | |
| data-file: data/processed/related_mergers.json | |
| commit-message: "suggest: candidate related mergers detected on {date}" | |
| pr-title: "suggest: link related mergers ({date})" | |
| close-comment: >- | |
| All detected related-merger pairs have been recorded as of {date}. | |
| Closing automatically. | |
| github-token: ${{ github.token }} | |
| ntfy-topic: ${{ secrets.NTFY_TOPIC }} | |
| ntfy-token: ${{ secrets.NTFY_TOKEN }} | |
| ntfy-server: ${{ vars.NTFY_SERVER || 'https://ntfy.sh' }} | |
| # An exact (100%) name match on acquirer, target, and merger name for a | |
| # refiled waiver leaves essentially no room for a false positive (see | |
| # detect_related_mergers.is_certain_match), so that case is merged | |
| # immediately instead of waiting on manual review. A follow-up issue | |
| # still asks for a sanity check. Mixed batches (some exact, some not) | |
| # are conservatively left for manual review in their entirety, since a | |
| # PR merge can't be split pair-by-pair. | |
| - name: Auto-merge exact-match waiver-refile PR | |
| id: automerge | |
| if: steps.relm.outputs.found == 'true' && steps.relm.outputs.changed == 'true' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ steps.relm.outputs.pr-number }} | |
| run: | | |
| if [ -z "$PR_NUMBER" ] || [ "$(jq -r '.all_certain' pr_meta.json)" != "true" ]; then | |
| echo "Not an all-exact-match batch (or PR number missing) — leaving PR #$PR_NUMBER for manual review." | |
| exit 0 | |
| fi | |
| gh pr merge "$PR_NUMBER" --squash | |
| gh label create "needs-verification" \ | |
| --description "Auto-merged by automation; needs a human to double-check" \ | |
| --color "fbca04" \ | |
| --force | |
| TITLE="Verify auto-merged waiver refile(s) ($(date -u +%Y-%m-%d))" | |
| jq -n \ | |
| --arg title "$TITLE" \ | |
| --rawfile body pr_issue_body.md \ | |
| '{"title": $title, "body": $body, "labels": ["needs-verification"]}' | \ | |
| gh api "repos/$GITHUB_REPOSITORY/issues" --method POST --input - | |
| echo "Merged PR #$PR_NUMBER automatically and opened a verification issue" | |
| echo "merged=true" >> "$GITHUB_OUTPUT" | |
| # This one merged itself into main without review, so it is the detection | |
| # outcome most worth interrupting for — hence priority 4 rather than the | |
| # default the review PRs get. | |
| - name: Notify about the auto-merged waiver refile | |
| if: steps.automerge.outputs.merged == 'true' | |
| uses: ./.github/actions/ntfy | |
| with: | |
| topic: ${{ secrets.NTFY_TOPIC }} | |
| token: ${{ secrets.NTFY_TOKEN }} | |
| server: ${{ vars.NTFY_SERVER || 'https://ntfy.sh' }} | |
| title: Waiver refile auto-merged | |
| message: >- | |
| PR #${{ steps.relm.outputs.pr-number }} was an exact name match and | |
| has been merged into main. A verification issue is open for a | |
| sanity check. | |
| tags: robot,white_check_mark | |
| priority: '4' | |
| click: ${{ github.server_url }}/${{ github.repository }}/issues?q=is%3Aopen+label%3Aneeds-verification | |
| # -- Related parties → fix/related-parties -- | |
| - name: Related parties review PR | |
| uses: ./.github/actions/detection-pr | |
| with: | |
| base-sha: ${{ steps.detect_base.outputs.sha }} | |
| branch: fix/related-parties | |
| heading: Related parties detection | |
| command: >- | |
| python -m scripts.detect.detect_related_parties | |
| --summary | |
| --apply-suggestions | |
| --pr-markdown pr_body.md | |
| data-file: data/processed/related_parties.json | |
| commit-message: "suggest: candidate related parties detected on {date}" | |
| pr-title: "suggest: link related parties ({date})" | |
| close-comment: >- | |
| All detected related-party groups have been recorded as of {date}. | |
| Closing automatically. | |
| github-token: ${{ github.token }} | |
| ntfy-topic: ${{ secrets.NTFY_TOPIC }} | |
| ntfy-token: ${{ secrets.NTFY_TOKEN }} | |
| ntfy-server: ${{ vars.NTFY_SERVER || 'https://ntfy.sh' }} |