Robots: block more AI scrapers

Savjee · Jul 19, 2024 · dbce16f · dbce16f
1 parent f40326c
commit dbce16f
Showing 1 changed file with 13 additions and 3 deletions.
diff --git a/src/site/robots.txt b/src/site/robots.txt
@@ -1,9 +1,19 @@
-User-agent: *
-Disallow: /*?*utm_source=
-
 User-agent: GPTBot
 Disallow: /
 
+User-agent: CCBot
+Disallow: /
+
+User-agent: PerplexityBot
+Disallow: /
+
+User-agent: anthropic-ai
+Disallow: /
+‍User-agent: Claude-Web
+Disallow: /
+User-agent: ClaudeBot
+Disallow: /
+
 sitemap: https://simplyexplained.com/sitemap-blogposts.txt
 sitemap: https://simplyexplained.com/sitemap-videos.txt
 sitemap: https://simplyexplained.com/sitemap.txt