setup a isolated vlan for the runners

systems/jeeves/networking.nix

@@ -2,31 +2,71 @@
   networking = {
     hostName = "jeeves";
     hostId = "0e15ce35";
-    firewall.enable = true;
+    firewall = {
+      enable = true;
+      interfaces.br-nix-builder = {
+        allowedTCPPorts = [ ];
+        allowedUDPPorts = [ ];
+      };
+    };
     useNetworkd = true;
   };
 
   systemd.network = {
     enable = true;
+    wait-online = {
+      enable = false;
+      anyInterface = true;
+    };
+    netdevs = {
+      "20-br-nix-builder" = {
+        netdevConfig = {
+          Kind = "bridge";
+          Name = "br-nix-builder";
+        };
+      };
+      "30-internet-vlan" = {
+        netdevConfig = {
+          Kind = "vlan";
+          Name = "internet-vlan";
+        };
+        vlanConfig.Id = 100;
+      };
+    };
     networks = {
       "10-1GB_Primary" = {
-        matchConfig.Name = "enp98s0f0";
+        matchConfig.Name = "enp97s0f1";
         address = [ "192.168.99.14/24" ];
         routes = [ { Gateway = "192.168.99.1"; } ];
+        vlan = [ "internet-vlan" ];
         linkConfig.RequiredForOnline = "routable";
       };
-      "10-1GB_Secondary" = {
-        matchConfig.Name = "enp98s0f1";
-        DHCP = "yes";
+      "50-internet-vlan" = {
+        matchConfig.Name = "internet-vlan";
+        bridge = [ "br-nix-builder" ];
+        linkConfig.RequiredForOnline = "no";
       };
-      "10-10GB_Primary" = {
-        matchConfig.Name = "enp97s0f0np0";
-        DHCP = "yes";
-        linkConfig.RequiredForOnline = "routable";
-      };
-      "10-10GB_Secondary" = {
-        matchConfig.Name = "enp97s0f1np1";
-        DHCP = "yes";
+      "60-br-nix-builder" = {
+        matchConfig.Name = "br-nix-builder";
+        bridgeConfig = { };
+        address = [ "192.168.3.10/24" ];
+        routingPolicyRules = [
+          {
+            From = "192.168.3.0/24";
+            Table = 100;
+            Priority = 100;
+          }
+        ];
+        routes = [
+          {
+            Gateway = "192.168.3.1";
+            Table = 100;
+            GatewayOnLink = false;
+            Metric = 2048;
+            PreferredSource = "192.168.3.10";
+          }
+        ];
+        linkConfig.RequiredForOnline = "no";
       };
     };
   };

systems/jeeves/runners/nix_builder.nix

@@ -9,9 +9,17 @@ with lib;
 
 let
   vars = import ../vars.nix;
+  cfg = config.services.nix_builder;
 in
 {
-  options.services.nix_builder.containers = mkOption {
+  options.services.nix_builder = {
+    bridgeName = mkOption {
+      type = types.str;
+      default = "br-nix-builder";
+      description = "Bridge name for the builder containers.";
+    };
+
+    containers = mkOption {
       type = types.attrsOf (
         types.submodule (
           { name, ... }:
@@ -23,15 +31,19 @@ in
       default = { };
       description = "GitHub runner container configurations";
     };
+  };
 
-  config.containers = mapAttrs (
-    name: cfg:
-    mkIf cfg.enable {
+  config = {
+    containers = mapAttrs (
+      name: containerCfg:
+      mkIf containerCfg.enable {
         autoStart = true;
         privateNetwork = true;
+        hostBridge = cfg.bridgeName;
         ephemeral = true;
         bindMounts = {
           storage = {
+            hostPath = "/zfs/media/github-runners/${name}";
             mountPoint = "/zfs/media/github-runners/${name}";
             isReadOnly = false;
           };
@@ -41,6 +53,7 @@ in
             isReadOnly = false;
           };
           secrets = {
+            hostPath = "${vars.secrets}/services/github-runners/${name}";
             mountPoint = "${vars.secrets}/services/github-runners/${name}";
             isReadOnly = true;
           };
@@ -53,6 +66,10 @@ in
             ...
           }:
           {
+            networking = {
+              useDHCP = lib.mkDefault true;
+              interfaces.eth0.useDHCP = true;
+            };
             nix.settings = {
               trusted-substituters = [
                 "https://cache.nixos.org"
@@ -112,5 +129,6 @@ in
             system.stateVersion = "24.11";
           };
       }
-  ) config.services.nix_builder.containers;
+    ) cfg.containers;
+  };
 }