Adds auto generated robots.txt

Related to Issues #25

nuxt.config.js

@@ -1,3 +1,4 @@
+
 const locales = [
   { code: 'ca', iso: 'ca', file: 'ca.json' },
   { code: 'cy', iso: 'cy', file: 'cy.json' },
@@ -62,6 +63,7 @@ export default {
   'modules': [
     '@nuxt/http',
     '@nuxtjs/google-gtag',
+    '@nuxtjs/robots',
     'nuxt-i18n',
     'nuxt-ssr-cache',
     // sitemap should always come last
@@ -101,6 +103,20 @@ export default {
     ],
   },
 
+  // robots configuration: https://github.com/nuxt-community/robots-module
+  // Provides an automated /robots.txt
+  'robots': {
+    UserAgent: '*',
+    Disallow: [
+      'login/',
+      // Disallow both the page and directory
+      'user/settings',
+      'user/settings/',
+    ].reduce((rules, page) => rules.concat(locales.map(locale => `/${locale.code}/${page}`)), []),
+    // TODO Should be a dynamically bound URL to be fed through ENV
+    Sitemap: 'https://v2.oengus.dev/sitemap.xml',
+  },
+
   // i18n configuration: https://i18n.nuxtjs.org/
   'i18n': {
     locales,
@@ -147,6 +163,7 @@ export default {
   },
 
   // Sitemap configuration: https://sitemap.nuxtjs.org/
+  // Provides an automated /sitemap.xml
   'sitemap': {
     hostname: 'https://v2.oengus.dev',
     i18n: {

package.json

@@ -14,6 +14,7 @@
   "dependencies": {
     "@nuxt/http": "^0.6.4",
     "@nuxtjs/google-gtag": "^1.0.4",
+    "@nuxtjs/robots": "^2.5.0",
     "@nuxtjs/sitemap": "^2.4.0",
     "core-js": "^3.15.2",
     "nuxt": "^2.15.7",