diff --git a/public/robots.txt b/public/robots.txt index d731573e..220f42a2 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -4,14 +4,7 @@ # See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file -# Disallow every other robot +# Disallow every robot except for the landing page User-agent: * +Allow: /$ Disallow: / - -# Google can only index individual works pages -# Allow syntax is only really listened to by google and a few others -User-agent: Googlebot -Allow: /doi.org/ - -# Sitemap specific location -Sitemap: https://commons.datacite.org/sitemaps/sitemap.xml.gz