feat: config robots-txt, sitemap

astro.config.mjs

@@ -1,9 +1,11 @@
 import { defineConfig } from 'astro/config';
 import tailwind from '@astrojs/tailwind';
-
-import sitemap from "@astrojs/sitemap";
+import robotsTxt from 'astro-robots-txt';
+import robotsConfig from './robots-txt.config';
+import sitemap from '@astrojs/sitemap';
 
 // https://astro.build/config
 export default defineConfig({
-  integrations: [tailwind(), sitemap()]
-});
+  site: 'https://iztechai.com',
+  integrations: [tailwind(), robotsTxt(robotsConfig), sitemap()],
+});

robots-txt.config.ts

@@ -0,0 +1,20 @@
+import type { RobotsTxtOptions } from 'astro-robots-txt';
+import siteConfig from './site.config';
+
+const robotsConfig: RobotsTxtOptions = {
+  policy: [
+    {
+      userAgent: '*',
+      // The next line enables or disables the crawling on the `robots.txt` level
+      disallow: siteConfig.disableIndexing ? '/' : '',
+    },
+  ],
+  sitemapBaseFileName: 'sitemap-index', // default 'sitemap-index'
+  // transform(content) {
+  //   const commentsList = ['Some text before main content', 'Second line', 'Third line'];
+  //   const comments = commentsList.map((item) => `# ${item}.`).join('\n');
+  //   return `${comments}\n\n${content}`;
+  // },
+};
+
+export default robotsConfig;

site.config.ts

@@ -0,0 +1,3 @@
+export default {
+  disableIndexing: false,
+};