adding robots.txt

systems/jeeves/web_services/haproxy.cfg

@@ -29,6 +29,13 @@ frontend ContentSwitching
   # ACME challenge routing (must be first)
   acl is_acme path_beg /.well-known/acme-challenge/
 
+  # --- robots.txt ---
+  # Serve a single global robots.txt for every vhost (asks crawlers to wait
+  # 10s between requests via Crawl-delay). Returned for both HTTP and HTTPS.
+  # File is deployed to /etc/haproxy/robots.txt by haproxy.nix.
+  acl is_robots path /robots.txt
+  http-request return status 200 content-type "text/plain" file /etc/haproxy/robots.txt if is_robots
+
   # tmmworkshop.com
   acl host_audiobookshelf  hdr(host) -i audiobookshelf.tmmworkshop.com
   acl host_cache  hdr(host) -i cache.tmmworkshop.com

systems/jeeves/web_services/haproxy.nix

@@ -4,6 +4,9 @@
     443
   ];
 
+  # Global robots.txt served by HAProxy for every vhost (see haproxy.cfg).
+  environment.etc."haproxy/robots.txt".source = ./robots.txt;
+
   services.haproxy = {
     enable = true;
     config = builtins.readFile ./haproxy.cfg;

systems/jeeves/web_services/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Crawl-delay: 10