From e1c4ae0d6e79ed6b4b1d78f8cfaa617662fa5b47 Mon Sep 17 00:00:00 2001 From: Richie Cahill Date: Mon, 22 Jun 2026 19:48:53 -0400 Subject: [PATCH] adding robots.txt --- systems/jeeves/web_services/haproxy.cfg | 7 +++++++ systems/jeeves/web_services/haproxy.nix | 3 +++ systems/jeeves/web_services/robots.txt | 2 ++ 3 files changed, 12 insertions(+) create mode 100644 systems/jeeves/web_services/robots.txt diff --git a/systems/jeeves/web_services/haproxy.cfg b/systems/jeeves/web_services/haproxy.cfg index 55eba7c..0ae1794 100644 --- a/systems/jeeves/web_services/haproxy.cfg +++ b/systems/jeeves/web_services/haproxy.cfg @@ -29,6 +29,13 @@ frontend ContentSwitching # ACME challenge routing (must be first) acl is_acme path_beg /.well-known/acme-challenge/ + # --- robots.txt --- + # Serve a single global robots.txt for every vhost (asks crawlers to wait + # 10s between requests via Crawl-delay). Returned for both HTTP and HTTPS. + # File is deployed to /etc/haproxy/robots.txt by haproxy.nix. + acl is_robots path /robots.txt + http-request return status 200 content-type "text/plain" file /etc/haproxy/robots.txt if is_robots + # tmmworkshop.com acl host_audiobookshelf hdr(host) -i audiobookshelf.tmmworkshop.com acl host_cache hdr(host) -i cache.tmmworkshop.com diff --git a/systems/jeeves/web_services/haproxy.nix b/systems/jeeves/web_services/haproxy.nix index cf87e3f..47c7c1f 100644 --- a/systems/jeeves/web_services/haproxy.nix +++ b/systems/jeeves/web_services/haproxy.nix @@ -4,6 +4,9 @@ 443 ]; + # Global robots.txt served by HAProxy for every vhost (see haproxy.cfg). + environment.etc."haproxy/robots.txt".source = ./robots.txt; + services.haproxy = { enable = true; config = builtins.readFile ./haproxy.cfg; diff --git a/systems/jeeves/web_services/robots.txt b/systems/jeeves/web_services/robots.txt new file mode 100644 index 0000000..93d096d --- /dev/null +++ b/systems/jeeves/web_services/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Crawl-delay: 10