Merge pull request #427 from datacite/robots-tweak

Allows crawling of search tab landing pages
datacite · Dec 11, 2024 · f9962b3 · f9962b3
2 parents 021c17e + f7a2d35
commit f9962b3
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/public/robots.txt b/public/robots.txt
@@ -4,7 +4,11 @@
 
 # See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
 
-# Disallow every robot except for the landing page
+# Disallow every robot except for the landing pages
 User-agent: *
 Allow: /$
+Allow: /doi.org$
+Allow: /orcid.org$
+Allow: /ror.org$
+Allow: /repositories$
 Disallow: /