From 40851172621db1940ba5e7b63ae001910eb34bd0 Mon Sep 17 00:00:00 2001 From: nthmost Date: Wed, 3 Jun 2026 18:12:56 -0700 Subject: [PATCH 1/2] Redirect named AI crawlers to wiki dumps - Add @dump_crawlers matcher in Caddy (before Anubis) to 302-redirect known bulk AI scrapers to dumps.noisebridge.net - Add Anubis impressum footer linking to dumps on the challenge/deny page Closes noisebridge/noisebridge-wiki#10 --- roles/anubis/templates/policy.yaml.j2 | 19 ++++++++++++++ templates/caddy/m3/Caddyfile.j2 | 37 +++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/roles/anubis/templates/policy.yaml.j2 b/roles/anubis/templates/policy.yaml.j2 index bb9ee893..fcf0dd36 100644 --- a/roles/anubis/templates/policy.yaml.j2 +++ b/roles/anubis/templates/policy.yaml.j2 @@ -12,6 +12,25 @@ # username: "{{ anubis_metrics_username }}" # password: "{{ anubis_metrics_password }}" +impressum: + # Footer shown on every Anubis challenge/deny page — points bots to the dump. + footer: >- + Looking for bulk wiki content? + Download the full wiki dump instead of scraping. + page: + title: "Noisebridge Wiki — Bot & Scraper Policy" + body: >- +

The Noisebridge wiki is protected + by Anubis to prevent scraper bots + from overloading the server.

+ +

If you are a bulk data consumer or AI training pipeline, please use the + public wiki dump instead of + scraping live pages. The dump is updated daily and is free to use.

+ +

If you believe you have been incorrectly challenged, contact + {{ anubis_webmaster_email }}.

+ store: backend: bbolt parameters: diff --git a/templates/caddy/m3/Caddyfile.j2 b/templates/caddy/m3/Caddyfile.j2 index a57b122a..cdf65ef0 100644 --- a/templates/caddy/m3/Caddyfile.j2 +++ b/templates/caddy/m3/Caddyfile.j2 @@ -119,6 +119,43 @@ www.noisebridge.net { } {{ noisebridge_caddy_secure_header | indent(width=2) }} + # Named AI/bulk crawlers: redirect to wiki dumps before they hit Anubis. + # These UAs are identifiable bulk scrapers — better to route them to the + # structured dump than wall them off with a challenge they can't solve. + # Source: https://github.com/noisebridge/noisebridge-wiki/issues/10 + @dump_crawlers { + header User-Agent *GPTBot* + header User-Agent *OAI-SearchBot* + header User-Agent *ChatGPT-User* + header User-Agent *ClaudeBot* + header User-Agent *Claude-SearchBot* + header User-Agent *Claude-User* + header User-Agent *Claude-Web* + header User-Agent *anthropic-ai* + header User-Agent *CCBot* + header User-Agent *Bytespider* + header User-Agent *Amazonbot* + header User-Agent *cohere-ai* + header User-Agent *cohere-training-data-crawler* + header User-Agent *PerplexityBot* + header User-Agent *Perplexity-User* + header User-Agent *FacebookBot* + header User-Agent *meta-externalagent* + header User-Agent *Meta-ExternalAgent* + header User-Agent *meta-externalfetcher* + header User-Agent *Meta-ExternalFetcher* + header User-Agent *Diffbot* + header User-Agent *AI2Bot* + header User-Agent *Ai2Bot-Dolma* + header User-Agent *ImagesiftBot* + header User-Agent *DuckAssistBot* + header User-Agent *FriendlyCrawler* + header User-Agent *YouBot* + header User-Agent *TikTokSpider* + header User-Agent *PetalBot* + } + redir @dump_crawlers https://dumps.noisebridge.net 302 + # Anubis challenge assets (served directly from Anubis) handle /.within.website/* { reverse_proxy unix//run/anubis/anubis.sock { From 7039c6f464bf7d5e8971edc315ad470bd579732e Mon Sep 17 00:00:00 2001 From: nthmost Date: Fri, 5 Jun 2026 12:21:43 -0700 Subject: [PATCH 2/2] =?UTF-8?q?Drop=20@dump=5Fcrawlers=20redirect=20?= =?UTF-8?q?=E2=80=94=20wiki=20content=20should=20be=20indexed=20by=20LLMs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- templates/caddy/m3/Caddyfile.j2 | 37 --------------------------------- 1 file changed, 37 deletions(-) diff --git a/templates/caddy/m3/Caddyfile.j2 b/templates/caddy/m3/Caddyfile.j2 index cdf65ef0..a57b122a 100644 --- a/templates/caddy/m3/Caddyfile.j2 +++ b/templates/caddy/m3/Caddyfile.j2 @@ -119,43 +119,6 @@ www.noisebridge.net { } {{ noisebridge_caddy_secure_header | indent(width=2) }} - # Named AI/bulk crawlers: redirect to wiki dumps before they hit Anubis. - # These UAs are identifiable bulk scrapers — better to route them to the - # structured dump than wall them off with a challenge they can't solve. - # Source: https://github.com/noisebridge/noisebridge-wiki/issues/10 - @dump_crawlers { - header User-Agent *GPTBot* - header User-Agent *OAI-SearchBot* - header User-Agent *ChatGPT-User* - header User-Agent *ClaudeBot* - header User-Agent *Claude-SearchBot* - header User-Agent *Claude-User* - header User-Agent *Claude-Web* - header User-Agent *anthropic-ai* - header User-Agent *CCBot* - header User-Agent *Bytespider* - header User-Agent *Amazonbot* - header User-Agent *cohere-ai* - header User-Agent *cohere-training-data-crawler* - header User-Agent *PerplexityBot* - header User-Agent *Perplexity-User* - header User-Agent *FacebookBot* - header User-Agent *meta-externalagent* - header User-Agent *Meta-ExternalAgent* - header User-Agent *meta-externalfetcher* - header User-Agent *Meta-ExternalFetcher* - header User-Agent *Diffbot* - header User-Agent *AI2Bot* - header User-Agent *Ai2Bot-Dolma* - header User-Agent *ImagesiftBot* - header User-Agent *DuckAssistBot* - header User-Agent *FriendlyCrawler* - header User-Agent *YouBot* - header User-Agent *TikTokSpider* - header User-Agent *PetalBot* - } - redir @dump_crawlers https://dumps.noisebridge.net 302 - # Anubis challenge assets (served directly from Anubis) handle /.within.website/* { reverse_proxy unix//run/anubis/anubis.sock {