diff --git a/.github/workflows/build_systems.yml b/.github/workflows/build_systems.yml index 8feff62..cc2609b 100644 --- a/.github/workflows/build_systems.yml +++ b/.github/workflows/build_systems.yml @@ -8,8 +8,23 @@ on: - cron: "0 22 * * *" jobs: + prebuild-common: + name: prebuild-common-x86-64-v3 + runs-on: nix-cache-builder + steps: + - uses: actions/checkout@v4 + # portal-1 is the smallest system closure: 95% of its derivations are + # shared by all five systems, so it is a maintainable common cache seed. + # Keep going so one failing package does not stop unrelated cache entries + # from being built. + - name: Build common packages + run: nixos-rebuild build --keep-going --accept-flake-config --flake ./#portal-1 + - name: Copy common packages to nix-cache + run: nix copy --accept-flake-config --to unix:///host-nix/var/nix/daemon-socket/socket .#nixosConfigurations.portal-1.config.system.build.toplevel + build: name: build-${{ matrix.system }} + needs: prebuild-common runs-on: self-hosted strategy: matrix: diff --git a/systems/jeeves/runners/default.nix b/systems/jeeves/runners/default.nix index 60911a6..fffc37d 100644 --- a/systems/jeeves/runners/default.nix +++ b/systems/jeeves/runners/default.nix @@ -21,5 +21,16 @@ nix-builder-12.enable = true; nix-builder-13.enable = true; nix-builder-14.enable = true; + + # Warm the shared x86-64-v3 cache before the smaller per-system runners + # start. Eight jobs with eight cores each can use Jeeves' 64 logical CPUs, + # while the 6000% quota leaves some capacity for its normal services. + nix-cache-builder = { + enable = true; + labels = [ "nix-cache-builder:host" ]; + cores = 8; + maxJobs = 8; + cpuQuota = "6000%"; + }; }; } diff --git a/systems/jeeves/runners/nix_builder.nix b/systems/jeeves/runners/nix_builder.nix index 8667f8f..03ea1e1 100644 --- a/systems/jeeves/runners/nix_builder.nix +++ b/systems/jeeves/runners/nix_builder.nix @@ -11,11 +11,8 @@ let cfg = config.services.nix_builder; runnerUsername = "gitea-runner"; runnerUserid = 601; - runnerLabels = [ - "self-hosted:host" - "nixos:host" - ]; containerConfig = + containerCfg: { config, pkgs, @@ -50,8 +47,8 @@ let useHostResolvConf = false; }; nix.settings = { - cores = 8; - max-jobs = 2; + inherit (containerCfg) cores; + max-jobs = containerCfg.maxJobs; system-features = lib.mkAfter [ "gccarch-x86-64-v2" "gccarch-x86-64-v3" @@ -94,7 +91,7 @@ let enable = true; name = "jeeves-nix-builder"; url = "http://192.168.99.14:6443/"; - labels = runnerLabels; + labels = containerCfg.labels; tokenFile = "/run/secrets/gitea-runners/registration-token"; settings.runner.timeout = "12h"; hostPackages = with pkgs; [ @@ -120,20 +117,21 @@ let User = mkForce runnerUsername; Group = mkForce runnerUsername; ExecStartPre = mkForce [ - "${getExe registerRunner} builder http://192.168.99.14:6443/ ${runnerConfigFile} ${escapeShellArgs runnerLabels}" + "${getExe registerRunner} builder http://192.168.99.14:6443/ ${runnerConfigFile} ${escapeShellArgs containerCfg.labels}" ]; }; }; system.stateVersion = "24.05"; }; - sharedContainerPath = + mkContainerPath = + containerCfg: (import "${pkgs.path}/nixos/lib/eval-config.nix" { modules = [ { boot.isNspawnContainer = true; nixpkgs.pkgs = pkgs; } - containerConfig + (containerConfig containerCfg) ]; system = null; }).config.system.build.toplevel; @@ -151,7 +149,36 @@ in types.submodule ( { name, ... }: { - options.enable = mkEnableOption "Gitea runner container"; + options = { + enable = mkEnableOption "Gitea runner container"; + + labels = mkOption { + type = types.listOf types.str; + default = [ + "self-hosted:host" + "nixos:host" + ]; + description = "Gitea Actions labels advertised by this runner."; + }; + + cores = mkOption { + type = types.ints.positive; + default = 8; + description = "Number of cores made available to each Nix build job."; + }; + + maxJobs = mkOption { + type = types.ints.positive; + default = 2; + description = "Maximum number of Nix build jobs run in parallel."; + }; + + cpuQuota = mkOption { + type = types.str; + default = "800%"; + description = "systemd CPU quota for the runner container."; + }; + }; } ) ); @@ -173,7 +200,7 @@ in containers = mapAttrs ( name: containerCfg: mkIf containerCfg.enable { - path = sharedContainerPath; + path = mkContainerPath containerCfg; autoStart = true; privateNetwork = true; hostBridge = cfg.bridgeName; @@ -199,16 +226,14 @@ in ) cfg.containers; systemd = { - services = builtins.listToAttrs ( - map (name: { - name = "container@${name}"; - value = { - requires = [ "gitea.service" ]; - after = [ "gitea.service" ]; - serviceConfig.CPUQuota = "800%"; - }; - }) (builtins.attrNames (filterAttrs (_: c: c.enable) cfg.containers)) - ); + services = mapAttrs' ( + name: containerCfg: + nameValuePair "container@${name}" { + requires = [ "gitea.service" ]; + after = [ "gitea.service" ]; + serviceConfig.CPUQuota = containerCfg.cpuQuota; + } + ) (filterAttrs (_: c: c.enable) cfg.containers); tmpfiles.rules = [ "d ${vars.uv_cache} 0755 ${runnerUsername} ${runnerUsername} - -"