From 2881477e1b06ce2493c254b865f6675c5abec4f9 Mon Sep 17 00:00:00 2001
From: David <dgx.arnold@gmail.com>
Date: Fri, 10 May 2024 02:52:31 +0200
Subject: [PATCH 01/10] refactor: bash+jq instead of python closure

---
 nix/kexec-installer/module.nix        |   5 +-
 nix/kexec-installer/restore_routes.py | 118 -------------------------
 nix/kexec-installer/restore_routes.sh | 121 ++++++++++++++++++++++++++
 3 files changed, 124 insertions(+), 120 deletions(-)
 delete mode 100644 nix/kexec-installer/restore_routes.py
 create mode 100755 nix/kexec-installer/restore_routes.sh

diff --git a/nix/kexec-installer/module.nix b/nix/kexec-installer/module.nix
index a10e6a5..5ae42c2 100644
--- a/nix/kexec-installer/module.nix
+++ b/nix/kexec-installer/module.nix
@@ -1,7 +1,7 @@
 { config, lib, modulesPath, pkgs, ... }:
 let
-  restore-network = pkgs.writers.writePython3 "restore-network" { flakeIgnore = [ "E501" ]; }
-    ./restore_routes.py;
+
+  restore-network = pkgs.writers.writeBash "restore-network" ./restore_routes.sh;
 
   # does not link with iptables enabled
   iprouteStatic = pkgs.pkgsStatic.iproute2.override { iptables = null; };
@@ -56,6 +56,7 @@ in
     environment.etc.is_kexec.text = "true";
 
     systemd.services.restore-network = {
+      path = [pkgs.jq];
       before = [ "network-pre.target" ];
       wants = [ "network-pre.target" ];
       wantedBy = [ "multi-user.target" ];
diff --git a/nix/kexec-installer/restore_routes.py b/nix/kexec-installer/restore_routes.py
deleted file mode 100644
index 1635376..0000000
--- a/nix/kexec-installer/restore_routes.py
+++ /dev/null
@@ -1,118 +0,0 @@
-import json
-import sys
-from pathlib import Path
-from typing import Any
-
-
-def filter_interfaces(network: list[dict[str, Any]]) -> list[dict[str, Any]]:
-    output = []
-    for net in network:
-        if net.get("link_type") == "loopback":
-            continue
-        if not net.get("address"):
-            # We need a mac address to match devices reliable
-            continue
-        addr_info = []
-        has_dynamic_address = False
-        for addr in net.get("addr_info", []):
-            # no link-local ipv4/ipv6
-            if addr.get("scope") == "link":
-                continue
-            # do not explicitly configure addresses from dhcp or router advertisement
-            if addr.get("dynamic", False):
-                has_dynamic_address = True
-                continue
-            else:
-                addr_info.append(addr)
-        if addr_info != [] or has_dynamic_address:
-            net["addr_info"] = addr_info
-            output.append(net)
-
-    return output
-
-
-def filter_routes(routes: list[dict[str, Any]]) -> list[dict[str, Any]]:
-    filtered = []
-    for route in routes:
-        # Filter out routes set by addresses with subnets, dhcp and router advertisement
-        if route.get("protocol") in ["dhcp", "kernel", "ra"]:
-            continue
-        filtered.append(route)
-
-    return filtered
-
-
-def generate_networkd_units(
-    interfaces: list[dict[str, Any]], routes: list[dict[str, Any]], directory: Path
-) -> None:
-    directory.mkdir(exist_ok=True)
-    for interface in interfaces:
-        name = f"00-{interface['ifname']}.network"
-        addresses = [
-            f"Address = {addr['local']}/{addr['prefixlen']}"
-            for addr in interface.get("addr_info", [])
-        ]
-
-        route_sections = []
-        for route in routes:
-            if route.get("dev", "nodev") != interface.get("ifname", "noif"):
-                continue
-
-            route_section = "[Route]\n"
-            if route.get("dst") != "default":
-                # can be skipped for default routes
-                route_section += f"Destination = {route['dst']}\n"
-            gateway = route.get("gateway")
-            if gateway:
-                route_section += f"Gateway = {gateway}\n"
-
-            # we may ignore on-link default routes here, but I don't see how
-            # they would be useful for internet connectivity anyway
-            route_sections.append(route_section)
-
-        # FIXME in some networks we might not want to trust dhcp or router advertisements
-        unit = f"""
-[Match]
-MACAddress = {interface["address"]}
-
-[Network]
-# both ipv4 and ipv6
-DHCP = yes
-# lets us discover the switch port we're connected to
-LLDP = yes
-# ipv6 router advertisements
-IPv6AcceptRA = yes
-# allows us to ping "nixos.local"
-MulticastDNS = yes
-
-"""
-        unit += "\n".join(addresses)
-        unit += "\n" + "\n".join(route_sections)
-        (directory / name).write_text(unit)
-
-
-def main() -> None:
-    if len(sys.argv) < 5:
-        print(
-            f"USAGE: {sys.argv[0]} addresses routes-v4 routes-v6 networkd-directory",
-            file=sys.stderr,
-        )
-        sys.exit(1)
-
-    with open(sys.argv[1]) as f:
-        addresses = json.load(f)
-    with open(sys.argv[2]) as f:
-        v4_routes = json.load(f)
-    with open(sys.argv[3]) as f:
-        v6_routes = json.load(f)
-
-    networkd_directory = Path(sys.argv[4])
-
-    relevant_interfaces = filter_interfaces(addresses)
-    relevant_routes = filter_routes(v4_routes) + filter_routes(v6_routes)
-
-    generate_networkd_units(relevant_interfaces, relevant_routes, networkd_directory)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/nix/kexec-installer/restore_routes.sh b/nix/kexec-installer/restore_routes.sh
new file mode 100755
index 0000000..f9ce3f8
--- /dev/null
+++ b/nix/kexec-installer/restore_routes.sh
@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+
+# filter_interfaces function
+filter_interfaces() {
+    # This function takes a list of network interfaces as input and filters
+    # out loopback interfaces, interfaces without a MAC address, and addresses
+    # with a "link" scope or marked as dynamic (from DHCP or router
+    # advertisements). The filtered interfaces are returned as an array.
+    local network=("$@")
+
+    for net in "${network[@]}"; do
+        local link_type="$(jq -r '.link_type' <<< "$net")"
+        local address="$(jq -r '.address // ""' <<< "$net")"
+        local addr_info="$(jq -r '.addr_info | map(select(.scope != "link" and (.dynamic | not)))' <<< "$net")"
+        local has_dynamic_address=$(jq -r '.addr_info | any(.dynamic)' <<< "$net")
+
+        # echo "Link Type: $link_type -- Address: $address -- Has Dynamic Address: $has_dynamic_address  -- Addr Info: $addr_info"
+
+        if [[ "$link_type" != "loopback" && -n "$address" && ("$addr_info" != "[]" || "$has_dynamic_address" == "true") ]]; then
+            net=$(jq -c --argjson addr_info "$addr_info" '.addr_info = $addr_info' <<< "$net")
+            echo "$net" # "return"
+        fi
+    done
+}
+
+# filter_routes function
+filter_routes() {
+    # This function takes a list of routes as input and filters out routes
+    # with protocols "dhcp", "kernel", or "ra". The filtered routes are
+    # returned as an array.
+    local routes=("$@")
+
+    for route in "${routes[@]}"; do
+        local protocol=$(jq -r '.protocol' <<< "$route")
+        if [[ $protocol != "dhcp" && $protocol != "kernel" && $protocol != "ra" ]]; then
+            echo "$route" # "return"
+        fi
+    done
+}
+
+# generate_networkd_units function
+generate_networkd_units() {
+    # This function takes the filtered interfaces and routes, along with a
+    # directory path. It generates systemd-networkd unit files for each interface,
+    # including the configured addresses and routes. The unit files are written
+    # to the specified directory with the naming convention 00-<ifname>.network.
+    local -n interfaces=$1
+    local -n routes=$2
+    local directory="$3"
+
+    mkdir -p "$directory"
+
+    for interface in "${interfaces[@]}"; do
+        local ifname=$(jq -r '.ifname' <<< "$interface")
+        local address=$(jq -r '.address' <<< "$interface")
+        local addresses=$(jq -r '.addr_info | map("Address = \(.local)/\(.prefixlen)") | join("\n")' <<< "$interface")
+        local route_sections=()
+
+        for route in "${routes[@]}"; do
+            local dev=$(jq -r '.dev' <<< "$route")
+            if [[ $dev == $ifname ]]; then
+                local route_section="[Route]"
+                local dst=$(jq -r '.dst' <<< "$route")
+                if [[ $dst != "default" ]]; then
+                    route_section+="\nDestination = $dst"
+                fi
+                local gateway=$(jq -r '.gateway // ""' <<< "$route")
+                if [[ -n $gateway ]]; then
+                    route_section+="\nGateway = $gateway"
+                fi
+                route_sections+=("$route_section")
+            fi
+        done
+
+        local unit=$(cat <<-EOF
+[Match]
+MACAddress = $address
+
+[Network]
+DHCP = yes
+LLDP = yes
+IPv6AcceptRA = yes
+MulticastDNS = yes
+
+$addresses
+$(printf '%s\n' "${route_sections[@]}")
+EOF
+)
+        echo -e "$unit" > "$directory/00-$ifname.network"
+    done
+}
+
+# main function
+main() {
+    if [[ $# -lt 4 ]]; then
+        echo "USAGE: $0 addresses routes-v4 routes-v6 networkd-directory" >&2
+        # exit 1
+        return 1
+    fi
+
+    local addresses
+    readarray -t addresses < <(jq -c '.[]' "$1") # Read JSON data into array
+
+    local v4_routes
+    readarray -t v4_routes < <(jq -c '.[]' "$2")
+
+    local v6_routes
+    readarray -t v6_routes < <(jq -c '.[]' "$3")
+
+    local networkd_directory="$4"
+
+    local relevant_interfaces
+    readarray -t relevant_interfaces < <(filter_interfaces "${addresses[@]}")
+
+    local relevant_routes
+    readarray -t relevant_routes < <(filter_routes "${v4_routes[@]}" "${v6_routes[@]}")
+
+    generate_networkd_units relevant_interfaces relevant_routes "$networkd_directory"
+}
+
+main "$@"

From deff4302a7434665ed7e402c49f7d9e4c41d5be2 Mon Sep 17 00:00:00 2001
From: David <dgx.arnold@gmail.com>
Date: Fri, 10 May 2024 12:39:00 +0200
Subject: [PATCH 02/10] chore: anticipate expected delay in tests

---
 nix/kexec-installer/test.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/nix/kexec-installer/test.nix b/nix/kexec-installer/test.nix
index 5aa302a..3ff9b11 100644
--- a/nix/kexec-installer/test.nix
+++ b/nix/kexec-installer/test.nix
@@ -137,6 +137,8 @@ makeTest' {
     node1.succeed('/root/kexec/kexec --version >&2')
     node1.succeed('/root/kexec/run >&2')
 
+    time.sleep(6)
+
     # wait for kexec to finish
     while ssh(["true"], check=False).returncode == 0:
         print("Waiting for kexec to finish...")

From f274f19bbc2ddaac77e62b2692d5932c0811e2bc Mon Sep 17 00:00:00 2001
From: David <dgx.arnold@gmail.com>
Date: Fri, 10 May 2024 12:39:48 +0200
Subject: [PATCH 03/10] feat: free caches and log free mem before kexec

---
 nix/kexec-installer/kexec-run.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/nix/kexec-installer/kexec-run.sh b/nix/kexec-installer/kexec-run.sh
index 5d4a1ad..b099713 100644
--- a/nix/kexec-installer/kexec-run.sh
+++ b/nix/kexec-installer/kexec-run.sh
@@ -74,6 +74,9 @@ if ! "$SCRIPT_DIR/kexec" --load "$SCRIPT_DIR/bzImage" \
   exit 1
 fi
 
+sync; echo 3 > /proc/sys/vm/drop_caches
+echo "current available memory: $(free -h | awk '/^Mem/ {print $7}')"
+
 # Disconnect our background kexec from the terminal
 echo "machine will boot into nixos in 6s..."
 if test -e /dev/kmsg; then

From a087c0d13569fcaa634334b3e748927a105f5538 Mon Sep 17 00:00:00 2001
From: David <dgx.arnold@gmail.com>
Date: Fri, 10 May 2024 00:03:43 +0200
Subject: [PATCH 04/10] remove some extra weight

---
 nix/installer.nix                | 41 ++++++++++++++++++++++++++++++++
 nix/kexec-installer/module.nix   |  1 +
 nix/netboot-installer/module.nix |  1 +
 nix/no-grub.nix                  | 10 ++++++++
 nix/noninteractive.nix           |  3 +++
 5 files changed, 56 insertions(+)
 create mode 100644 nix/no-grub.nix

diff --git a/nix/installer.nix b/nix/installer.nix
index 8ed0157..b7cf866 100644
--- a/nix/installer.nix
+++ b/nix/installer.nix
@@ -25,11 +25,52 @@ in
   documentation.enable = false;
   documentation.man.man-db.enable = false;
 
+  # reduce closure size through package set crafting
+  # where there's no otherwise globally effective
+  # config setting available
+  # TODO: some are candidates for a long-term upstream solution
+  nixpkgs.overlays = [
+    (final: prev: {
+      # save ~12MB by not bundling manpages
+      coreutils-full = prev.coreutils;
+      # save ~20MB by making them minimal
+      util-linux = prev.util-linux.override {
+        nlsSupport = false;
+        ncursesSupport = false;
+        systemdSupport = false;
+        translateManpages = false;
+      };
+      # save ~6MB by removing one bash 
+      bashInteractive = prev.bash;
+      # saves ~25MB
+      systemd = prev.systemd.override {
+        pname = "systemd-slim";
+        withDocumentation = false;
+        withCoredump = false;
+        withFido2 = false;
+        withRepart = false;
+        withMachined = false;
+        withRemote = false;
+        withTpm2Tss = false;
+        withLibBPF = false;
+        withAudit = false;
+        withCompression = false;
+        withImportd = false;
+        withPortabled = false;
+      };
+    })
+  ];
+  systemd.coredump.enable = false;
+
+
   environment.systemPackages = [
     # for zapping of disko
     pkgs.jq
     # for copying extra files of nixos-anywhere
     pkgs.rsync
+    # for installing nixos via nixos-anywhere
+    config.system.build.nixos-enter
+    config.system.build.nixos-install
   ];
 
   imports = [
diff --git a/nix/kexec-installer/module.nix b/nix/kexec-installer/module.nix
index 5ae42c2..9b9fcb5 100644
--- a/nix/kexec-installer/module.nix
+++ b/nix/kexec-installer/module.nix
@@ -13,6 +13,7 @@ in
     ../networkd.nix
     ../serial.nix
     ../restore-remote-access.nix
+    ../no-grub.nix
   ];
   options = {
     system.kexec-installer.name = lib.mkOption {
diff --git a/nix/netboot-installer/module.nix b/nix/netboot-installer/module.nix
index 590bf56..3d1038e 100644
--- a/nix/netboot-installer/module.nix
+++ b/nix/netboot-installer/module.nix
@@ -6,6 +6,7 @@
     ../networkd.nix
     ../serial.nix
     ../restore-remote-access.nix
+    ../no-grub.nix
   ];
 
   # We are stateless, so just default to latest.
diff --git a/nix/no-grub.nix b/nix/no-grub.nix
new file mode 100644
index 0000000..3a5dad2
--- /dev/null
+++ b/nix/no-grub.nix
@@ -0,0 +1,10 @@
+{lib, ...}:{
+  # when grub ends up being bloat: kexec & netboot
+  nixpkgs.overlays = [
+    (final: prev: {
+      # we don't need grub: save ~ 60MB
+      grub2 = prev.coreutils;
+      grub2_efi = prev.coreutils;
+    })
+  ];
+}
diff --git a/nix/noninteractive.nix b/nix/noninteractive.nix
index 0f6e954..1e8d588 100644
--- a/nix/noninteractive.nix
+++ b/nix/noninteractive.nix
@@ -23,6 +23,9 @@
   # would pull in nano
   programs.nano.enable = false;
 
+  # also avoids placing man pages of systemPackages
+  documentation.man.man-db.enable = false;
+
   # prevents strace
   environment.defaultPackages = lib.mkForce [ pkgs.rsync pkgs.parted pkgs.gptfdisk ];
 

From 51f092ea6e8dfcf940cffe9e8ab87cbaa7ecd2aa Mon Sep 17 00:00:00 2001
From: David <dgx.arnold@gmail.com>
Date: Fri, 10 May 2024 16:02:29 +0200
Subject: [PATCH 05/10] chore: iterate systemd slim

---
 nix/installer.nix | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/nix/installer.nix b/nix/installer.nix
index b7cf866..0b7a60f 100644
--- a/nix/installer.nix
+++ b/nix/installer.nix
@@ -57,6 +57,13 @@ in
         withCompression = false;
         withImportd = false;
         withPortabled = false;
+        withSysupdate = false;
+        withHomed = false;
+        withLocaled = false;
+        withPolkit = false;
+        # withQrencode = false;
+        # withVmspawn = false;
+        withPasswordQuality = false;
       };
     })
   ];

From 6275988baba9f1b1ba74fd9c0ebcc7cfa8e9ce31 Mon Sep 17 00:00:00 2001
From: David <dgx.arnold@gmail.com>
Date: Sun, 19 May 2024 14:57:03 +0200
Subject: [PATCH 06/10] chore: make util-linux modifications more concise

---
 nix/installer.nix | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/nix/installer.nix b/nix/installer.nix
index 0b7a60f..ef97b59 100644
--- a/nix/installer.nix
+++ b/nix/installer.nix
@@ -33,12 +33,11 @@ in
     (final: prev: {
       # save ~12MB by not bundling manpages
       coreutils-full = prev.coreutils;
-      # save ~20MB by making them minimal
+      # save ~16MB by making them minimal
       util-linux = prev.util-linux.override {
         nlsSupport = false;
         ncursesSupport = false;
         systemdSupport = false;
-        translateManpages = false;
       };
       # save ~6MB by removing one bash 
       bashInteractive = prev.bash;

From eeb2377839ec73e3ca427882ebd4a2f08a276a85 Mon Sep 17 00:00:00 2001
From: David <dgx.arnold@gmail.com>
Date: Sun, 19 May 2024 16:39:49 +0200
Subject: [PATCH 07/10] refactor: adapt to cherry pick

---
 nix/kexec-installer/module.nix   |  1 -
 nix/netboot-installer/module.nix |  1 -
 nix/no-grub.nix                  | 10 ----------
 3 files changed, 12 deletions(-)
 delete mode 100644 nix/no-grub.nix

diff --git a/nix/kexec-installer/module.nix b/nix/kexec-installer/module.nix
index 9b9fcb5..5ae42c2 100644
--- a/nix/kexec-installer/module.nix
+++ b/nix/kexec-installer/module.nix
@@ -13,7 +13,6 @@ in
     ../networkd.nix
     ../serial.nix
     ../restore-remote-access.nix
-    ../no-grub.nix
   ];
   options = {
     system.kexec-installer.name = lib.mkOption {
diff --git a/nix/netboot-installer/module.nix b/nix/netboot-installer/module.nix
index 3d1038e..590bf56 100644
--- a/nix/netboot-installer/module.nix
+++ b/nix/netboot-installer/module.nix
@@ -6,7 +6,6 @@
     ../networkd.nix
     ../serial.nix
     ../restore-remote-access.nix
-    ../no-grub.nix
   ];
 
   # We are stateless, so just default to latest.
diff --git a/nix/no-grub.nix b/nix/no-grub.nix
deleted file mode 100644
index 3a5dad2..0000000
--- a/nix/no-grub.nix
+++ /dev/null
@@ -1,10 +0,0 @@
-{lib, ...}:{
-  # when grub ends up being bloat: kexec & netboot
-  nixpkgs.overlays = [
-    (final: prev: {
-      # we don't need grub: save ~ 60MB
-      grub2 = prev.coreutils;
-      grub2_efi = prev.coreutils;
-    })
-  ];
-}

From a8032e04280e2cf80c06d11b948c0ea9f260a43a Mon Sep 17 00:00:00 2001
From: David <dgx.arnold@gmail.com>
Date: Sun, 19 May 2024 19:22:48 +0200
Subject: [PATCH 08/10] fix: docstrings

---
 nix/kexec-installer/restore_routes.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nix/kexec-installer/restore_routes.sh b/nix/kexec-installer/restore_routes.sh
index f9ce3f8..7ab5940 100755
--- a/nix/kexec-installer/restore_routes.sh
+++ b/nix/kexec-installer/restore_routes.sh
@@ -5,7 +5,7 @@ filter_interfaces() {
     # This function takes a list of network interfaces as input and filters
     # out loopback interfaces, interfaces without a MAC address, and addresses
     # with a "link" scope or marked as dynamic (from DHCP or router
-    # advertisements). The filtered interfaces are returned as an array.
+    # advertisements). The filtered interfaces are returned one by one on stdout.
     local network=("$@")
 
     for net in "${network[@]}"; do
@@ -27,7 +27,7 @@ filter_interfaces() {
 filter_routes() {
     # This function takes a list of routes as input and filters out routes
     # with protocols "dhcp", "kernel", or "ra". The filtered routes are
-    # returned as an array.
+    # returned one by one on stdout.
     local routes=("$@")
 
     for route in "${routes[@]}"; do

From b9eec5585ae627706f6c431c6ed87444c96419cf Mon Sep 17 00:00:00 2001
From: David <dgx.arnold@gmail.com>
Date: Sun, 19 May 2024 19:25:23 +0200
Subject: [PATCH 09/10] fix: cleanup after rebase

---
 nix/noninteractive.nix | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/nix/noninteractive.nix b/nix/noninteractive.nix
index 1e8d588..0f6e954 100644
--- a/nix/noninteractive.nix
+++ b/nix/noninteractive.nix
@@ -23,9 +23,6 @@
   # would pull in nano
   programs.nano.enable = false;
 
-  # also avoids placing man pages of systemPackages
-  documentation.man.man-db.enable = false;
-
   # prevents strace
   environment.defaultPackages = lib.mkForce [ pkgs.rsync pkgs.parted pkgs.gptfdisk ];
 

From 2251e8485cc3b95716ec57c05000cfbfe4f799ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <Mic92@users.noreply.github.com>
Date: Sun, 19 May 2024 19:28:59 +0200
Subject: [PATCH 10/10] nix/kexec-installer/test.nix: comment why we wait 6s

---
 nix/kexec-installer/test.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nix/kexec-installer/test.nix b/nix/kexec-installer/test.nix
index 3ff9b11..0c127eb 100644
--- a/nix/kexec-installer/test.nix
+++ b/nix/kexec-installer/test.nix
@@ -137,6 +137,7 @@ makeTest' {
     node1.succeed('/root/kexec/kexec --version >&2')
     node1.succeed('/root/kexec/run >&2')
 
+    # the kexec script will sleep 6s before doing anything, so do we here.
     time.sleep(6)
 
     # wait for kexec to finish