adding robots.txt

systems/jeeves/web_services/haproxy.cfg

@@ -29,6 +29,13 @@ frontend ContentSwitching
   # ACME challenge routing (must be first)
   acl is_acme path_beg /.well-known/acme-challenge/
 
+  # --- robots.txt ---
+  # Serve a single global robots.txt for every vhost (asks crawlers to wait
+  # 10s between requests via Crawl-delay). Returned for both HTTP and HTTPS.
+  # File is deployed to /etc/haproxy/robots.txt by haproxy.nix.
+  acl is_robots path /robots.txt
+  http-request return status 200 content-type "text/plain" file /etc/haproxy/robots.txt if is_robots
+
   # tmmworkshop.com
   acl host_audiobookshelf  hdr(host) -i audiobookshelf.tmmworkshop.com
   acl host_cache  hdr(host) -i cache.tmmworkshop.com