From cec868a833ae0f2f9a881ac6e0dc22dec093fe59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 6 Jun 2025 11:51:21 +0200 Subject: [PATCH] no longer proxy to hydra from netlify The infra team would like to have the freedom to have longer maintainace periods of hydra if necessary. Also we want to reduce the load on the service. This commit makes nix.dev mirror the manual as snapshots. During the build process we download the tarballs from github releases --- .../workflows/update-development-manual.yml | 25 ++++ default.nix | 14 +++ netlify.toml | 15 --- nix/development-manual.json | 6 + nix/update-development-manual.nix | 115 ++++++++++++++++++ 5 files changed, 160 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/update-development-manual.yml create mode 100644 nix/development-manual.json create mode 100644 nix/update-development-manual.nix diff --git a/.github/workflows/update-development-manual.yml b/.github/workflows/update-development-manual.yml new file mode 100644 index 000000000..d40c722a9 --- /dev/null +++ b/.github/workflows/update-development-manual.yml @@ -0,0 +1,25 @@ +name: "Update development manual" +on: + repository_dispatch: + workflow_dispatch: + schedule: + # Run weekly on Mondays at 3:31 AM UTC + - cron: "31 3 * * 1" + +jobs: + update-development-manual: + runs-on: ubuntu-latest + permissions: + contents: write # Required to create releases and push changes + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v31 + - name: Update development manual + run: nix-shell --run update-development-manual + env: + GH_TOKEN: ${{ github.token }} + - name: Create Pull Request + uses: peter-evans/create-pull-request@v7 + with: + title: Update development manual + labels: dependencies \ No newline at end of file diff --git a/default.nix b/default.nix index e6b556457..326e3dd59 100644 --- a/default.nix +++ b/default.nix @@ -11,6 +11,13 @@ let lib = pkgs.lib; releases = import ./nix/releases.nix { inherit lib inputs system; }; + + # Fetch the development manual from GitHub releases + developmentManualInfo = lib.importJSON ./nix/development-manual.json; + developmentManual = pkgs.fetchurl { + url = developmentManualInfo.url; + sha256 = developmentManualInfo.sha256; + }; pkgs-unstable = import inputs.main.nixpkgs-rolling { config = { }; overlays = [ ]; @@ -83,6 +90,11 @@ let '' + lib.optionalString withManuals '' ${lib.concatStringsSep "\n" (lib.mapAttrsToList release releases.nixReleases)} ${lib.concatStringsSep "\n" (lib.mapAttrsToList mutableRedirect releases.mutableNixManualRedirects)} + + # Extract and install development manual + echo "Installing development manual..." + mkdir -p $out/manual/nix/development + tar -xf ${developmentManual} -C $out/manual/nix/development --strip-components=1 ''; }; @@ -92,6 +104,7 @@ let }; update-nix-releases = pkgs-unstable.callPackage ./nix/update-nix-releases.nix { }; update-nixpkgs-releases = pkgs-unstable.callPackage ./nix/update-nixpkgs-releases.nix { }; + update-development-manual = pkgs-unstable.callPackage ./nix/update-development-manual.nix { }; in { # build with `nix-build -A build` @@ -103,6 +116,7 @@ in devmode update-nix-releases update-nixpkgs-releases + update-development-manual pkgs.npins pkgs.python3.pkgs.black pkgs.vale diff --git a/netlify.toml b/netlify.toml index d98a98198..e69de29bb 100644 --- a/netlify.toml +++ b/netlify.toml @@ -1,15 +0,0 @@ -[[redirects]] -from = "/manual/nix/unstable/*" -to = "https://hydra.nixos.org/job/nix/master/manual/latest/download/1/manual/:splat" -status = 200 -force = true -[redirects.headers] -X-From = "nix.dev-Uogho3gi" - -[[redirects]] -from = "/manual/nix/development/*" -to = "https://hydra.nixos.org/job/nix/master/manual/latest/download/1/manual/:splat" -status = 200 -force = true -[redirects.headers] -X-From = "nix.dev-Uogho3gi" diff --git a/nix/development-manual.json b/nix/development-manual.json new file mode 100644 index 000000000..fd7f51757 --- /dev/null +++ b/nix/development-manual.json @@ -0,0 +1,6 @@ +{ + "version": "2025.01.06", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "url": "https://github.com/NixOS/nix.dev/releases/download/development-manual-2025.01.06/manual.tar.xz", + "updated": "2025-01-06 00:00:00 UTC" +} \ No newline at end of file diff --git a/nix/update-development-manual.nix b/nix/update-development-manual.nix new file mode 100644 index 000000000..8bc6602f9 --- /dev/null +++ b/nix/update-development-manual.nix @@ -0,0 +1,115 @@ +{ + writeShellApplication, + curl, + jq, + gh, + coreutils, + gnused, + gnutar, + gzip, + git, +}: +# Update mechanism for development manual from Hydra, mirrored to GitHub releases +writeShellApplication { + name = "update-development-manual"; + runtimeInputs = [ + curl + jq + gh + coreutils + gnused + gnutar + gzip + git + ]; + text = '' + set -euo pipefail + + echo >&2 "Downloading development manual from Hydra..." + + # Try multiple fallback URLs in case Hydra structure changes + urls=( + "https://hydra.nixos.org/job/nix/master/manual/latest/download/1/manual.tar.xz" + "https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download/2/nix-*-x86_64-linux/share/doc/nix/manual" + ) + + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + + downloaded=false + for url in "''${urls[@]}"; do + echo >&2 "Trying to download from: $url" + if curl -f -L -s "$url" -o "$tmp/manual.tar.xz"; then + echo >&2 "Successfully downloaded from: $url" + downloaded=true + break + fi + done + + if [ "$downloaded" = false ]; then + echo >&2 "Failed to download manual from any URL" + exit 1 + fi + + # Verify we got a valid archive + if ! tar -tf "$tmp/manual.tar.xz" > /dev/null 2>&1; then + echo >&2 "Downloaded file is not a valid tar archive" + exit 1 + fi + + # Calculate hash for comparison + current_hash=$(sha256sum "$tmp/manual.tar.xz" | cut -d' ' -f1) + + # Check existing version file + if [ -f "nix/development-manual.json" ]; then + existing_hash=$(jq -r '.sha256' nix/development-manual.json) + if [ "$existing_hash" = "$current_hash" ]; then + echo >&2 "Manual with same content already exists" + exit 0 + fi + fi + + # Use date-based versioning for stability + # Format: YYYY.MM.DD (with optional .N for multiple updates same day) + base_version=$(date +%Y.%m.%d) + + # Check if this version already exists and increment if needed + version="$base_version" + counter=1 + while gh release view "development-manual-$version" >/dev/null 2>&1; do + version="$base_version.$counter" + counter=$((counter + 1)) + done + + echo >&2 "Creating new version: $version" + + current_time=$(date -u +"%Y-%m-%d %H:%M:%S UTC") + + # Create new versioned release + release_tag="development-manual-$version" + gh release create "$release_tag" "$tmp/manual.tar.xz" \ + --title "Development Manual $version" \ + --notes "Nix development manual from master branch ($current_time) + + Hash: $current_hash" \ + --prerelease + + # Update the version file in the repository + jq -n \ + --arg version "$version" \ + --arg sha256 "$current_hash" \ + --arg url "https://github.com/NixOS/nix.dev/releases/download/$release_tag/manual.tar.xz" \ + --arg updated "$current_time" \ + '{ + version: $version, + sha256: $sha256, + url: $url, + updated: $updated + }' > nix/development-manual.json + + echo >&2 "Development manual updated successfully!" + echo >&2 "Version: $version" + echo >&2 "Hash: $current_hash" + echo >&2 "Updated: nix/development-manual.json" + ''; +}