feat(jeeves): share one system closure across builders

Reuse a single NixOS container build for all runners while preserving
unique runtime hostnames and existing Gitea registrations.

systems/jeeves/runners/nix_builder.nix

@@ -1,8 +1,7 @@
 {
   config,
   lib,
-  outputs,
+  pkgs,
-  utils,
   ...
 }:
 
@@ -12,6 +11,121 @@ let
   cfg = config.services.nix_builder;
   runnerUsername = "gitea-runner";
   runnerUserid = 601;
+  runnerLabels = [
+    "self-hosted:host"
+    "nixos:host"
+  ];
+  containerConfig =
+    {
+      config,
+      pkgs,
+      lib,
+      ...
+    }:
+    let
+      runnerConfigFile = (pkgs.formats.yaml { }).generate "gitea-runner.yaml" { };
+      registerRunner = pkgs.writeShellApplication {
+        name = "register-gitea-runner";
+        runtimeInputs = [
+          pkgs.coreutils
+          pkgs.hostname
+          config.services.gitea-actions-runner.package
+        ];
+        text = builtins.readFile ./register_gitea_runner.sh;
+      };
+    in
+    {
+      networking = {
+        hostName = "nix-builder";
+        useDHCP = lib.mkDefault true;
+        interfaces.eth0.useDHCP = true;
+        # Ensure containers don't inherit the host's stub resolver (127.0.0.53) which was causing issues
+        useHostResolvConf = false;
+      };
+      nix.settings = {
+        system-features = lib.mkAfter [
+          "gccarch-x86-64-v2"
+          "gccarch-x86-64-v3"
+        ];
+        trusted-substituters = [
+          "https://cache.nixos.org"
+          "https://cache.tmmworkshop.com"
+          "https://nix-community.cachix.org"
+        ];
+        substituters = [
+          "https://cache.nixos.org/?priority=2&want-mass-query=true"
+          "https://cache.tmmworkshop.com/?priority=2&want-mass-query=true"
+          "https://nix-community.cachix.org/?priority=10&want-mass-query=true"
+        ];
+        trusted-public-keys = [
+          "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
+          "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA="
+          "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
+        ];
+        experimental-features = [
+          "flakes"
+          "nix-command"
+        ];
+        sandbox = true;
+        allowed-users = [ runnerUsername ];
+        trusted-users = [
+          "root"
+          runnerUsername
+        ];
+      };
+      users = {
+        users.${runnerUsername} = {
+          isSystemUser = true;
+          group = runnerUsername;
+          uid = runnerUserid;
+        };
+        groups.${runnerUsername}.gid = runnerUserid;
+      };
+      services.gitea-actions-runner.instances.builder = {
+        enable = true;
+        name = "jeeves-nix-builder";
+        url = "http://192.168.99.14:6443/";
+        labels = runnerLabels;
+        tokenFile = "/run/secrets/gitea-runners/registration-token";
+        hostPackages = with pkgs; [
+          bash
+          coreutils
+          curl
+          gawk
+          gitMinimal
+          gnused
+          my_python
+          nix
+          nixfmt
+          nixos-rebuild
+          nodejs
+          treefmt
+          wget
+        ];
+      };
+      systemd.services."gitea-runner-builder" = {
+        serviceConfig = {
+          DynamicUser = mkForce false;
+          User = mkForce runnerUsername;
+          Group = mkForce runnerUsername;
+          ExecStartPre = mkForce [
+            "${getExe registerRunner} builder http://192.168.99.14:6443/ ${runnerConfigFile} ${escapeShellArgs runnerLabels}"
+          ];
+        };
+      };
+      system.stateVersion = "24.05";
+    };
+  sharedContainerPath =
+    (import "${pkgs.path}/nixos/lib/eval-config.nix" {
+      modules = [
+        {
+          boot.isNspawnContainer = true;
+          nixpkgs.pkgs = pkgs;
+        }
+        containerConfig
+      ];
+      system = null;
+    }).config.system.build.toplevel;
 in
 {
   options.services.nix_builder = {
@@ -48,9 +162,11 @@ in
     containers = mapAttrs (
       name: containerCfg:
       mkIf containerCfg.enable {
+        path = sharedContainerPath;
         autoStart = true;
         privateNetwork = true;
         hostBridge = cfg.bridgeName;
+        extraFlags = [ "--hostname=${name}" ];
         bindMounts = {
           host-nix = {
             mountPoint = "/host-nix/var/nix/daemon-socket";
@@ -68,93 +184,6 @@ in
             isReadOnly = false;
           };
         };
-        config =
-          {
-            config,
-            pkgs,
-            lib,
-            ...
-          }:
-          {
-            networking = {
-              useDHCP = lib.mkDefault true;
-              interfaces.eth0.useDHCP = true;
-              # Ensure containers don't inherit the host's stub resolver (127.0.0.53) which was causing issues
-              useHostResolvConf = false;
-            };
-            nix.settings = {
-              trusted-substituters = [
-                "https://cache.nixos.org"
-                "https://cache.tmmworkshop.com"
-                "https://nix-community.cachix.org"
-              ];
-              substituters = [
-                "https://cache.nixos.org/?priority=2&want-mass-query=true"
-                "https://cache.tmmworkshop.com/?priority=2&want-mass-query=true"
-                "https://nix-community.cachix.org/?priority=10&want-mass-query=true"
-              ];
-              trusted-public-keys = [
-                "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
-                "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA="
-                "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
-              ];
-              experimental-features = [
-                "flakes"
-                "nix-command"
-              ];
-              sandbox = true;
-              allowed-users = [ "gitea-runner" ];
-              trusted-users = [
-                "root"
-                "gitea-runner"
-              ];
-            };
-            nixpkgs = {
-              overlays = builtins.attrValues outputs.overlays;
-              config.allowUnfree = true;
-            };
-            users = {
-              users.${runnerUsername} = {
-                isSystemUser = true;
-                group = runnerUsername;
-                uid = runnerUserid;
-              };
-              groups.${runnerUsername}.gid = runnerUserid;
-            };
-            services.gitea-actions-runner.instances.${name} = {
-              enable = true;
-              name = "jeeves-${name}";
-              url = "http://192.168.99.14:6443/";
-              labels = [
-                "self-hosted:host"
-                "nixos:host"
-              ];
-              tokenFile = "/run/secrets/gitea-runners/registration-token";
-              hostPackages = with pkgs; [
-                bash
-                coreutils
-                curl
-                gawk
-                gitMinimal
-                gnused
-                my_python
-                nix
-                nixfmt
-                nixos-rebuild
-                nodejs
-                treefmt
-                wget
-              ];
-            };
-            systemd.services."gitea-runner-${utils.escapeSystemdPath name}" = {
-              serviceConfig = {
-                DynamicUser = mkForce false;
-                User = mkForce runnerUsername;
-                Group = mkForce runnerUsername;
-              };
-            };
-            system.stateVersion = "24.05";
-          };
       }
     ) cfg.containers;
 

systems/jeeves/runners/register_gitea_runner.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+legacy_instance_dir="$STATE_DIRECTORY/$(hostname)"
+instance_dir="$STATE_DIRECTORY/$1"
+runner_url="$2"
+runner_config_file="$3"
+shift 3
+runner_labels="$(IFS=,; printf '%s' "$*")"
+
+if [ ! -e "$instance_dir" ] && [ -d "$legacy_instance_dir" ]; then
+  mv "$legacy_instance_dir" "$instance_dir"
+fi
+
+mkdir -vp "$instance_dir"
+cd "$instance_dir" || exit 1
+
+token_hash_file="$instance_dir/.token-hash"
+token_hash_current="$(printf '%s' "$TOKEN" | sha256sum | cut -d' ' -f1)"
+token_hash_stored="$(cat "$token_hash_file" 2>/dev/null || echo "")"
+labels_file="$instance_dir/.labels"
+labels_wanted="$(printf '%s\n' "$@" | sort)"
+labels_current="$(cat "$labels_file" 2>/dev/null || echo 0)"
+
+if [ ! -e "$instance_dir/.runner" ] || [ "$labels_wanted" != "$labels_current" ] || [ "$token_hash_current" != "$token_hash_stored" ]; then
+  rm -v "$instance_dir/.runner" || true
+
+  gitea-runner register --no-interactive \
+    --instance "$runner_url" \
+    --token "$TOKEN" \
+    --name "jeeves-$(hostname)" \
+    --labels "$runner_labels" \
+    --config "$runner_config_file"
+
+  printf '%s' "$token_hash_current" > "$token_hash_file"
+  printf '%s\n' "$labels_wanted" > "$labels_file"
+fi