From 5514047eda84d86b774b4263220887aeea07eda7 Mon Sep 17 00:00:00 2001 From: Richie Cahill Date: Tue, 8 Sep 2026 19:32:39 -0400 Subject: [PATCH] feat(jeeves): share one system closure across builders Reuse a single NixOS container build for all runners while preserving unique runtime hostnames and existing Gitea registrations. --- systems/jeeves/runners/nix_builder.nix | 207 ++++++++++-------- .../jeeves/runners/register_gitea_runner.sh | 36 +++ 2 files changed, 154 insertions(+), 89 deletions(-) create mode 100644 systems/jeeves/runners/register_gitea_runner.sh diff --git a/systems/jeeves/runners/nix_builder.nix b/systems/jeeves/runners/nix_builder.nix index 239562d..9d89083 100644 --- a/systems/jeeves/runners/nix_builder.nix +++ b/systems/jeeves/runners/nix_builder.nix @@ -1,8 +1,7 @@ { config, lib, - outputs, - utils, + pkgs, ... }: @@ -12,6 +11,121 @@ let cfg = config.services.nix_builder; runnerUsername = "gitea-runner"; runnerUserid = 601; + runnerLabels = [ + "self-hosted:host" + "nixos:host" + ]; + containerConfig = + { + config, + pkgs, + lib, + ... + }: + let + runnerConfigFile = (pkgs.formats.yaml { }).generate "gitea-runner.yaml" { }; + registerRunner = pkgs.writeShellApplication { + name = "register-gitea-runner"; + runtimeInputs = [ + pkgs.coreutils + pkgs.hostname + config.services.gitea-actions-runner.package + ]; + text = builtins.readFile ./register_gitea_runner.sh; + }; + in + { + networking = { + hostName = "nix-builder"; + useDHCP = lib.mkDefault true; + interfaces.eth0.useDHCP = true; + # Ensure containers don't inherit the host's stub resolver (127.0.0.53) which was causing issues + useHostResolvConf = false; + }; + nix.settings = { + system-features = lib.mkAfter [ + "gccarch-x86-64-v2" + "gccarch-x86-64-v3" + ]; + trusted-substituters = [ + "https://cache.nixos.org" + "https://cache.tmmworkshop.com" + "https://nix-community.cachix.org" + ]; + substituters = [ + "https://cache.nixos.org/?priority=2&want-mass-query=true" + "https://cache.tmmworkshop.com/?priority=2&want-mass-query=true" + "https://nix-community.cachix.org/?priority=10&want-mass-query=true" + ]; + trusted-public-keys = [ + "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" + "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" + "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" + ]; + experimental-features = [ + "flakes" + "nix-command" + ]; + sandbox = true; + allowed-users = [ runnerUsername ]; + trusted-users = [ + "root" + runnerUsername + ]; + }; + users = { + users.${runnerUsername} = { + isSystemUser = true; + group = runnerUsername; + uid = runnerUserid; + }; + groups.${runnerUsername}.gid = runnerUserid; + }; + services.gitea-actions-runner.instances.builder = { + enable = true; + name = "jeeves-nix-builder"; + url = "http://192.168.99.14:6443/"; + labels = runnerLabels; + tokenFile = "/run/secrets/gitea-runners/registration-token"; + hostPackages = with pkgs; [ + bash + coreutils + curl + gawk + gitMinimal + gnused + my_python + nix + nixfmt + nixos-rebuild + nodejs + treefmt + wget + ]; + }; + systemd.services."gitea-runner-builder" = { + serviceConfig = { + DynamicUser = mkForce false; + User = mkForce runnerUsername; + Group = mkForce runnerUsername; + ExecStartPre = mkForce [ + "${getExe registerRunner} builder http://192.168.99.14:6443/ ${runnerConfigFile} ${escapeShellArgs runnerLabels}" + ]; + }; + }; + system.stateVersion = "24.05"; + }; + sharedContainerPath = + (import "${pkgs.path}/nixos/lib/eval-config.nix" { + modules = [ + { + boot.isNspawnContainer = true; + nixpkgs.pkgs = pkgs; + } + containerConfig + ]; + system = null; + }).config.system.build.toplevel; in { options.services.nix_builder = { @@ -48,9 +162,11 @@ in containers = mapAttrs ( name: containerCfg: mkIf containerCfg.enable { + path = sharedContainerPath; autoStart = true; privateNetwork = true; hostBridge = cfg.bridgeName; + extraFlags = [ "--hostname=${name}" ]; bindMounts = { host-nix = { mountPoint = "/host-nix/var/nix/daemon-socket"; @@ -68,93 +184,6 @@ in isReadOnly = false; }; }; - config = - { - config, - pkgs, - lib, - ... - }: - { - networking = { - useDHCP = lib.mkDefault true; - interfaces.eth0.useDHCP = true; - # Ensure containers don't inherit the host's stub resolver (127.0.0.53) which was causing issues - useHostResolvConf = false; - }; - nix.settings = { - trusted-substituters = [ - "https://cache.nixos.org" - "https://cache.tmmworkshop.com" - "https://nix-community.cachix.org" - ]; - substituters = [ - "https://cache.nixos.org/?priority=2&want-mass-query=true" - "https://cache.tmmworkshop.com/?priority=2&want-mass-query=true" - "https://nix-community.cachix.org/?priority=10&want-mass-query=true" - ]; - trusted-public-keys = [ - "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" - "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA=" - "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" - ]; - experimental-features = [ - "flakes" - "nix-command" - ]; - sandbox = true; - allowed-users = [ "gitea-runner" ]; - trusted-users = [ - "root" - "gitea-runner" - ]; - }; - nixpkgs = { - overlays = builtins.attrValues outputs.overlays; - config.allowUnfree = true; - }; - users = { - users.${runnerUsername} = { - isSystemUser = true; - group = runnerUsername; - uid = runnerUserid; - }; - groups.${runnerUsername}.gid = runnerUserid; - }; - services.gitea-actions-runner.instances.${name} = { - enable = true; - name = "jeeves-${name}"; - url = "http://192.168.99.14:6443/"; - labels = [ - "self-hosted:host" - "nixos:host" - ]; - tokenFile = "/run/secrets/gitea-runners/registration-token"; - hostPackages = with pkgs; [ - bash - coreutils - curl - gawk - gitMinimal - gnused - my_python - nix - nixfmt - nixos-rebuild - nodejs - treefmt - wget - ]; - }; - systemd.services."gitea-runner-${utils.escapeSystemdPath name}" = { - serviceConfig = { - DynamicUser = mkForce false; - User = mkForce runnerUsername; - Group = mkForce runnerUsername; - }; - }; - system.stateVersion = "24.05"; - }; } ) cfg.containers; diff --git a/systems/jeeves/runners/register_gitea_runner.sh b/systems/jeeves/runners/register_gitea_runner.sh new file mode 100644 index 0000000..205bd93 --- /dev/null +++ b/systems/jeeves/runners/register_gitea_runner.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +legacy_instance_dir="$STATE_DIRECTORY/$(hostname)" +instance_dir="$STATE_DIRECTORY/$1" +runner_url="$2" +runner_config_file="$3" +shift 3 +runner_labels="$(IFS=,; printf '%s' "$*")" + +if [ ! -e "$instance_dir" ] && [ -d "$legacy_instance_dir" ]; then + mv "$legacy_instance_dir" "$instance_dir" +fi + +mkdir -vp "$instance_dir" +cd "$instance_dir" || exit 1 + +token_hash_file="$instance_dir/.token-hash" +token_hash_current="$(printf '%s' "$TOKEN" | sha256sum | cut -d' ' -f1)" +token_hash_stored="$(cat "$token_hash_file" 2>/dev/null || echo "")" +labels_file="$instance_dir/.labels" +labels_wanted="$(printf '%s\n' "$@" | sort)" +labels_current="$(cat "$labels_file" 2>/dev/null || echo 0)" + +if [ ! -e "$instance_dir/.runner" ] || [ "$labels_wanted" != "$labels_current" ] || [ "$token_hash_current" != "$token_hash_stored" ]; then + rm -v "$instance_dir/.runner" || true + + gitea-runner register --no-interactive \ + --instance "$runner_url" \ + --token "$TOKEN" \ + --name "jeeves-$(hostname)" \ + --labels "$runner_labels" \ + --config "$runner_config_file" + + printf '%s' "$token_hash_current" > "$token_hash_file" + printf '%s\n' "$labels_wanted" > "$labels_file" +fi -- 2.55.0