From d048494b1b1d87a703aba61e12cb201e5e5abed2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arda=20Sevin=C3=A7?= <ardasevinc.technical@gmail.com>
Date: Tue, 10 Oct 2023 16:35:20 +0300
Subject: [PATCH] feat: config robots-txt, sitemap

---
 astro.config.mjs     | 10 ++++++----
 robots-txt.config.ts | 20 ++++++++++++++++++++
 site.config.ts       |  3 +++
 3 files changed, 29 insertions(+), 4 deletions(-)
 create mode 100644 robots-txt.config.ts
 create mode 100644 site.config.ts

diff --git a/astro.config.mjs b/astro.config.mjs
index 63281b7..b900ca8 100644
--- a/astro.config.mjs
+++ b/astro.config.mjs
@@ -1,9 +1,11 @@
 import { defineConfig } from 'astro/config';
 import tailwind from '@astrojs/tailwind';
-
-import sitemap from "@astrojs/sitemap";
+import robotsTxt from 'astro-robots-txt';
+import robotsConfig from './robots-txt.config';
+import sitemap from '@astrojs/sitemap';
 
 // https://astro.build/config
 export default defineConfig({
-  integrations: [tailwind(), sitemap()]
-});
\ No newline at end of file
+  site: 'https://iztechai.com',
+  integrations: [tailwind(), robotsTxt(robotsConfig), sitemap()],
+});
diff --git a/robots-txt.config.ts b/robots-txt.config.ts
new file mode 100644
index 0000000..96f241b
--- /dev/null
+++ b/robots-txt.config.ts
@@ -0,0 +1,20 @@
+import type { RobotsTxtOptions } from 'astro-robots-txt';
+import siteConfig from './site.config';
+
+const robotsConfig: RobotsTxtOptions = {
+  policy: [
+    {
+      userAgent: '*',
+      // The next line enables or disables the crawling on the `robots.txt` level
+      disallow: siteConfig.disableIndexing ? '/' : '',
+    },
+  ],
+  sitemapBaseFileName: 'sitemap-index', // default 'sitemap-index'
+  // transform(content) {
+  //   const commentsList = ['Some text before main content', 'Second line', 'Third line'];
+  //   const comments = commentsList.map((item) => `# ${item}.`).join('\n');
+  //   return `${comments}\n\n${content}`;
+  // },
+};
+
+export default robotsConfig;
diff --git a/site.config.ts b/site.config.ts
new file mode 100644
index 0000000..861b5f0
--- /dev/null
+++ b/site.config.ts
@@ -0,0 +1,3 @@
+export default {
+  disableIndexing: false,
+};