setup a isolated vlan for the runners

systems/jeeves/networking.nix

@@ -2,31 +2,71 @@
   networking = {
     hostName = "jeeves";
     hostId = "0e15ce35";
-    firewall.enable = true;
+    firewall = {
+      enable = true;
+      interfaces.br-nix-builder = {
+        allowedTCPPorts = [ ];
+        allowedUDPPorts = [ ];
+      };
+    };
     useNetworkd = true;
   };
 
   systemd.network = {
     enable = true;
+    wait-online = {
+      enable = false;
+      anyInterface = true;
+    };
+    netdevs = {
+      "20-br-nix-builder" = {
+        netdevConfig = {
+          Kind = "bridge";
+          Name = "br-nix-builder";
+        };
+      };
+      "30-internet-vlan" = {
+        netdevConfig = {
+          Kind = "vlan";
+          Name = "internet-vlan";
+        };
+        vlanConfig.Id = 100;
+      };
+    };
     networks = {
       "10-1GB_Primary" = {
-        matchConfig.Name = "enp98s0f0";
+        matchConfig.Name = "enp97s0f1";
         address = [ "192.168.99.14/24" ];
         routes = [ { Gateway = "192.168.99.1"; } ];
+        vlan = [ "internet-vlan" ];
         linkConfig.RequiredForOnline = "routable";
       };
-      "10-1GB_Secondary" = {
+      "50-internet-vlan" = {
-        matchConfig.Name = "enp98s0f1";
+        matchConfig.Name = "internet-vlan";
-        DHCP = "yes";
+        bridge = [ "br-nix-builder" ];
+        linkConfig.RequiredForOnline = "no";
       };
-      "10-10GB_Primary" = {
+      "60-br-nix-builder" = {
-        matchConfig.Name = "enp97s0f0np0";
+        matchConfig.Name = "br-nix-builder";
-        DHCP = "yes";
+        bridgeConfig = { };
-        linkConfig.RequiredForOnline = "routable";
+        address = [ "192.168.3.10/24" ];
-      };
+        routingPolicyRules = [
-      "10-10GB_Secondary" = {
+          {
-        matchConfig.Name = "enp97s0f1np1";
+            From = "192.168.3.0/24";
-        DHCP = "yes";
+            Table = 100;
+            Priority = 100;
+          }
+        ];
+        routes = [
+          {
+            Gateway = "192.168.3.1";
+            Table = 100;
+            GatewayOnLink = false;
+            Metric = 2048;
+            PreferredSource = "192.168.3.10";
+          }
+        ];
+        linkConfig.RequiredForOnline = "no";
       };
     };
   };

systems/jeeves/runners/nix_builder.nix

@@ -9,108 +9,126 @@ with lib;
 
 let
   vars = import ../vars.nix;
+  cfg = config.services.nix_builder;
 in
 {
-  options.services.nix_builder.containers = mkOption {
+  options.services.nix_builder = {
-    type = types.attrsOf (
+    bridgeName = mkOption {
-      types.submodule (
+      type = types.str;
-        { name, ... }:
+      default = "br-nix-builder";
-        {
+      description = "Bridge name for the builder containers.";
-          options.enable = mkEnableOption "GitHub runner container";
+    };
-        }
+
-      )
+    containers = mkOption {
-    );
+      type = types.attrsOf (
-    default = { };
+        types.submodule (
-    description = "GitHub runner container configurations";
+          { name, ... }:
+          {
+            options.enable = mkEnableOption "GitHub runner container";
+          }
+        )
+      );
+      default = { };
+      description = "GitHub runner container configurations";
+    };
   };
 
-  config.containers = mapAttrs (
+  config = {
-    name: cfg:
+    containers = mapAttrs (
-    mkIf cfg.enable {
+      name: containerCfg:
-      autoStart = true;
+      mkIf containerCfg.enable {
-      privateNetwork = true;
+        autoStart = true;
-      ephemeral = true;
+        privateNetwork = true;
-      bindMounts = {
+        hostBridge = cfg.bridgeName;
-        storage = {
+        ephemeral = true;
-          mountPoint = "/zfs/media/github-runners/${name}";
+        bindMounts = {
-          isReadOnly = false;
+          storage = {
-        };
+            hostPath = "/zfs/media/github-runners/${name}";
-        host-nix = {
+            mountPoint = "/zfs/media/github-runners/${name}";
-          mountPoint = "/host-nix/var/nix/daemon-socket";
+            isReadOnly = false;
-          hostPath = "/nix/var/nix/daemon-socket";
-          isReadOnly = false;
-        };
-        secrets = {
-          mountPoint = "${vars.secrets}/services/github-runners/${name}";
-          isReadOnly = true;
-        };
-      };
-      config =
-        {
-          config,
-          pkgs,
-          lib,
-          ...
-        }:
-        {
-          nix.settings = {
-            trusted-substituters = [
-              "https://cache.nixos.org"
-              "https://cache.tmmworkshop.com"
-              "https://nix-community.cachix.org"
-            ];
-            substituters = [
-              "https://cache.nixos.org/?priority=2&want-mass-query=true"
-              "https://cache.tmmworkshop.com/?priority=2&want-mass-query=true"
-              "https://nix-community.cachix.org/?priority=10&want-mass-query=true"
-            ];
-            trusted-public-keys = [
-              "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
-              "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA="
-              "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
-            ];
-            experimental-features = [
-              "flakes"
-              "nix-command"
-            ];
-            sandbox = true;
-            allowed-users = [ "github-runners" ];
-            trusted-users = [
-              "root"
-              "github-runners"
-            ];
           };
-          nixpkgs = {
+          host-nix = {
-            overlays = builtins.attrValues outputs.overlays;
+            mountPoint = "/host-nix/var/nix/daemon-socket";
-            config.allowUnfree = true;
+            hostPath = "/nix/var/nix/daemon-socket";
+            isReadOnly = false;
           };
-          services.github-runners.${name} = {
+          secrets = {
-            enable = true;
+            hostPath = "${vars.secrets}/services/github-runners/${name}";
-            replace = true;
+            mountPoint = "${vars.secrets}/services/github-runners/${name}";
-            workDir = "/zfs/media/github-runners/${name}";
+            isReadOnly = true;
-            url = "https://github.com/RichieCahill/dotfiles";
-            extraLabels = [ "nixos" ];
-            tokenFile = "${vars.secrets}/services/github-runners/${name}";
-            user = "github-runners";
-            group = "github-runners";
-            extraPackages = with pkgs; [
-              nixfmt-rfc-style
-              nixos-rebuild
-              treefmt
-              my_python
-            ];
           };
-          users = {
+        };
-            users.github-runners = {
+        config =
-              shell = pkgs.bash;
+          {
-              isSystemUser = true;
+            config,
-              group = "github-runners";
+            pkgs,
-              uid = 601;
+            lib,
+            ...
+          }:
+          {
+            networking = {
+              useDHCP = lib.mkDefault true;
+              interfaces.eth0.useDHCP = true;
             };
-            groups.github-runners.gid = 601;
+            nix.settings = {
+              trusted-substituters = [
+                "https://cache.nixos.org"
+                "https://cache.tmmworkshop.com"
+                "https://nix-community.cachix.org"
+              ];
+              substituters = [
+                "https://cache.nixos.org/?priority=2&want-mass-query=true"
+                "https://cache.tmmworkshop.com/?priority=2&want-mass-query=true"
+                "https://nix-community.cachix.org/?priority=10&want-mass-query=true"
+              ];
+              trusted-public-keys = [
+                "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
+                "cache.tmmworkshop.com:jHffkpgbmEdstQPoihJPYW9TQe6jnQbWR2LqkNGV3iA="
+                "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
+              ];
+              experimental-features = [
+                "flakes"
+                "nix-command"
+              ];
+              sandbox = true;
+              allowed-users = [ "github-runners" ];
+              trusted-users = [
+                "root"
+                "github-runners"
+              ];
+            };
+            nixpkgs = {
+              overlays = builtins.attrValues outputs.overlays;
+              config.allowUnfree = true;
+            };
+            services.github-runners.${name} = {
+              enable = true;
+              replace = true;
+              workDir = "/zfs/media/github-runners/${name}";
+              url = "https://github.com/RichieCahill/dotfiles";
+              extraLabels = [ "nixos" ];
+              tokenFile = "${vars.secrets}/services/github-runners/${name}";
+              user = "github-runners";
+              group = "github-runners";
+              extraPackages = with pkgs; [
+                nixfmt-rfc-style
+                nixos-rebuild
+                treefmt
+                my_python
+              ];
+            };
+            users = {
+              users.github-runners = {
+                shell = pkgs.bash;
+                isSystemUser = true;
+                group = "github-runners";
+                uid = 601;
+              };
+              groups.github-runners.gid = 601;
+            };
+            system.stateVersion = "24.11";
           };
-          system.stateVersion = "24.11";
+      }
-        };
+    ) cfg.containers;
-    }
+  };
-  ) config.services.nix_builder.containers;
 }