From 2cc6baa43f941782c21ef6ceeedc8e07c123af2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Asger=20Geel=20Weirs=C3=B8e?= <cla@weircon.dk>
Date: Mon, 5 Oct 2026 21:30:00 +0200
Subject: [PATCH] Keep crawlers off the tarball archives, not the tarball index

SEO crawlers were downloading the release archives (up to about 2 GB each)
even though they are served noindex. Disallow only *.tar.zst and *.zip under
/tarballz/; the index page and the PKGBUILDs stay crawlable. Repeated in every
group, because a crawler with its own group ignores User-agent: *.

Claude-Session: https://claude.ai/code/session_01GGe7kUhooMFuzY6GaDb2w9
---
 public/robots.txt | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/public/robots.txt b/public/robots.txt
index 48f4f64..46b143c 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -1,43 +1,72 @@
 # Asger Weirsøe — asger.weirsoe.dk
 # Alle crawlere er velkomne, inkl. AI/LLM-crawlere (vi vil gerne findes og citeres).
+# Undtagelse: selve arkivfilerne under /tarballz/ (op til ~2 GB stykket). Indeks og
+# PKGBUILD'er må gerne crawles. Står i hver gruppe, fordi en crawler med sin egen
+# gruppe ignorerer User-agent: *.
 User-agent: *
 Allow: /
+Disallow: /tarballz/*.tar.zst
+Disallow: /tarballz/*.zip
 
 # Eksplicit velkomst til AI/LLM-crawlere
 User-agent: GPTBot
 Allow: /
+Disallow: /tarballz/*.tar.zst
+Disallow: /tarballz/*.zip
 
 User-agent: OAI-SearchBot
 Allow: /
+Disallow: /tarballz/*.tar.zst
+Disallow: /tarballz/*.zip
 
 User-agent: ChatGPT-User
 Allow: /
+Disallow: /tarballz/*.tar.zst
+Disallow: /tarballz/*.zip
 
 User-agent: ClaudeBot
 Allow: /
+Disallow: /tarballz/*.tar.zst
+Disallow: /tarballz/*.zip
 
 User-agent: Claude-Web
 Allow: /
+Disallow: /tarballz/*.tar.zst
+Disallow: /tarballz/*.zip
 
 User-agent: anthropic-ai
 Allow: /
+Disallow: /tarballz/*.tar.zst
+Disallow: /tarballz/*.zip
 
 User-agent: PerplexityBot
 Allow: /
+Disallow: /tarballz/*.tar.zst
+Disallow: /tarballz/*.zip
 
 User-agent: Perplexity-User
 Allow: /
+Disallow: /tarballz/*.tar.zst
+Disallow: /tarballz/*.zip
 
 User-agent: Google-Extended
 Allow: /
+Disallow: /tarballz/*.tar.zst
+Disallow: /tarballz/*.zip
 
 User-agent: Applebot-Extended
 Allow: /
+Disallow: /tarballz/*.tar.zst
+Disallow: /tarballz/*.zip
 
 User-agent: CCBot
 Allow: /
+Disallow: /tarballz/*.tar.zst
+Disallow: /tarballz/*.zip
 
 User-agent: cohere-ai
 Allow: /
+Disallow: /tarballz/*.tar.zst
+Disallow: /tarballz/*.zip
 
 Sitemap: https://asger.xn--weirse-fya.dk/sitemap-index.xml