Merge pull request #10397 from internetarchive/robots-txt-crawl-delays

Add crawl delays to robots.txt
internetarchive · Jan 28, 2025 · 67903f4 · 67903f4
2 parents c8e2621 + e6436fa
commit 67903f4
Showing 1 changed file with 32 additions and 1 deletion.
diff --git a/static/robots.txt b/static/robots.txt
@@ -2,13 +2,44 @@
 User-agent: *
 Disallow: /api
 Disallow: /edit
+Disallow: /account
 Disallow: /search
 Disallow: /search/inside
+Disallow: */borrow*
+Disallow: /*.rdf$
 
 Sitemap: https://openlibrary.org/static/sitemaps/siteindex.xml.gz
 
 User-agent: Baiduspider
-Crawl-delay: 0.5
+Crawl-delay: 10
 
 User-agent: Googlebot
 Disallow: /*.rdf$
+Crawl-delay: 10
+
+User-agent: AhrefsBot
+Crawl-delay: 10
+
+User-agent: meta-externalagent
+Crawl-delay: 10
+
+User-agent: anthropic-ai
+Crawl-delay: 10
+
+User-agent: ClaudeBot
+Crawl-delay: 10
+
+User-agent: openai
+Crawl-delay: 10
+
+User-agent: ChatGPT
+Crawl-delay: 10
+
+User-agent: bingbot
+Crawl-delay: 10
+
+User-agent: Yandex
+Crawl-delay: 10
+
+User-agent: *bot
+Crawl-delay: 10