Add sitemap generator

.gitignore

@@ -64,5 +64,6 @@ _crowdin
 *.__*
 dist
 public/data
+public/sitemap.xml
 !_crowdin/translation
 .wrangler/

workspaces/cms-scripts/package.json

@@ -4,9 +4,10 @@
   "private": true,
   "scripts": {
     "update-algolia-index": "vite-node src/algolia.ts",
-    "update-dynamic-data": "vite-node src/index.ts",
+    "update-dynamic-data": "vite-node src/index.ts && yarn sitemap",
     "update-jobs": "vite-node src/jobs-update.ts",
     "pre-crowdin": "vite-node src/pre-crowdin.ts",
+    "sitemap": "vite-node src/sitemap.ts",
     "build": "tsc"
   },
   "dependencies": {

workspaces/cms-scripts/src/sitemap.ts

@@ -0,0 +1,149 @@
+import fs from 'fs/promises';
+import path from 'path';
+
+process.chdir(path.resolve(__dirname, '../../..'));
+
+import { locales } from '@starknet-io/cms-data/src/i18n/config';
+import { getPosts } from './data';
+
+const domain = 'https://www.starknet.io';
+const changefreqByDepth = ['daily', 'weekly', 'monthly', 'yearly'];
+const priorityByDepth = [1, 0.8, 0.6, 0.4];
+const customPages = ['announcements', 'events', 'jobs', 'posts', 'roadmap', 'tutorials'];
+
+type SitemapUrl = {
+  changefreq: string;
+  lastmod?: string;
+  priority: number;
+  url: string;
+};
+
+const sitemapUrls: SitemapUrl[] = locales.flatMap(locale => [
+  {
+    changefreq: 'weekly',
+    priority: 1,
+    url: `/${locale}`
+  },
+  ...customPages.map(url => ({
+    changefreq: 'weekly',
+    priority: 0.8,
+    url: `/${locale}/${url}`
+  }))
+]);
+
+const parsePageDirectory = async (dir: string, depth = 0) => {
+  for (const filepath of await fs.readdir(dir)) {
+    const file = path.join(dir, filepath);
+
+    if ((await fs.stat(file))?.isDirectory()) {
+      await parsePageDirectory(file, depth + 1);
+
+      continue;
+    }
+
+    const page = await fs.readFile(file, 'utf8');
+
+    try {
+      const { hidden_page, link } = JSON.parse(page);
+
+      if (!hidden_page && link) {
+        sitemapUrls.push({
+          url: link,
+          changefreq: changefreqByDepth[depth],
+          priority: priorityByDepth[depth]
+        });
+      }
+    } catch {
+      console.error('Error parsing page', file);
+    }
+  }
+};
+
+const parsePosts = async () => {
+  const { filenameMap } = await getPosts();
+  const categories: string[] = [];
+
+  filenameMap.forEach(({ locale, category, slug, published_date }) => {
+    if (!categories.includes(category)) {
+      categories.push(category);
+
+      sitemapUrls.push({
+        url: `/${locale}/posts/${category}`,
+        changefreq: 'weekly',
+        priority: 0.8
+      });
+    }
+
+    sitemapUrls.push({
+      url: `/${locale}/posts/${category}/${slug}`,
+      changefreq: 'monthly',
+      priority: 0.6,
+      lastmod: published_date?.split('T')?.[0]
+    });
+  });
+};
+
+const parseTutorials = async () => {
+  for (const locale of locales) {
+    for (const filename of await fs.readdir(`./public/data/tutorials/${locale}`)) {
+      const file = await fs.readFile(`./public/data/tutorials/${locale}/${filename}`, 'utf8');
+
+      try {
+        const tutorial = JSON.parse(file);
+
+        if (tutorial.type === 'youtube') {
+          sitemapUrls.push({
+            url: `/${locale}/tutorials/video/${tutorial.id}`,
+            changefreq: 'monthly',
+            priority: 0.6
+          });
+        }
+      } catch {
+        console.error(`Error parsing tutorial`, filename);
+      }
+    }
+  }
+};
+
+const parseDetails = async (name: string) => {
+  for (const locale of locales) {
+    const file = await fs.readFile(`./public/data/${name}-details/${locale}.json`, 'utf8');
+
+    try {
+      const roadmaps = JSON.parse(file);
+      sitemapUrls.push(
+        ...roadmaps.map((roadmap: { slug: string }) => ({
+          url: `/${locale}/${name}/${roadmap.slug}`,
+          changefreq: 'monthly',
+          priority: 0.6
+        }))
+      );
+    } catch {
+      console.error(`Error parsing ${name} for locale`, locale);
+    }
+  }
+};
+
+await parsePageDirectory('./public/data/pages');
+await parsePosts();
+await parseDetails('announcements');
+await parseDetails('roadmap');
+await parseTutorials();
+
+let sitemap = `<?xml version="1.0" encoding="utf-8" standalone="yes" ?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`;
+
+sitemapUrls.forEach(({ url, changefreq, priority, lastmod }) => {
+  sitemap += `
+  <url>
+    <loc>${domain}${url}</loc>
+    <changefreq>${changefreq}</changefreq>
+    <priority>${priority}</priority>${lastmod ? `
+    <lastmod>${lastmod}</lastmod>` : ''}
+  </url>`;
+});
+
+sitemap += `
+</urlset>`;
+
+await fs.writeFile('./public/sitemap.xml', sitemap);

workspaces/website/functions/[[route]].ts

@@ -34,6 +34,7 @@ router.get("/*.svg", ittyAssetshandler);
 router.get("/*.ico", ittyAssetshandler);
 router.get("/*.txt", ittyAssetshandler);
 router.get("/assets/*", ittyAssetshandler);
+router.get("/sitemap.xml", ittyAssetshandler);
 
 router.all("/data/*", preflight);
 router.get("/data/*", async (req, context: EventContext<{}, any, Record<string, unknown>>) => {