From ec94fa34a3a0e6dbd782dbc9e14988045aaf4c4b Mon Sep 17 00:00:00 2001
From: Bryceson Laing <bryceson.laing@datacite.org>
Date: Wed, 16 Oct 2024 08:27:28 -0500
Subject: [PATCH] Disable all bots and remove sitemap (#402)

* Disable all bots and remove sitemap

* allow bots to access only the landing page
---
 public/robots.txt | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/public/robots.txt b/public/robots.txt
index d731573e..220f42a2 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -4,14 +4,7 @@
 
 # See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
 
-# Disallow every other robot
+# Disallow every robot except for the landing page
 User-agent: *
+Allow: /$
 Disallow: /
-
-# Google can only index individual works pages
-# Allow syntax is only really listened to by google and a few others
-User-agent: Googlebot
-Allow: /doi.org/
-
-# Sitemap specific location
-Sitemap: https://commons.datacite.org/sitemaps/sitemap.xml.gz