[DOP-5031]: Download and install the persistence module integration (#12

)

* Add persistence module code

* Add persistence module download code

persistence-module/src/index.ts

@@ -1,49 +1,18 @@
 // Documentation: https://sdk.netlify.com
 import { NetlifyIntegration } from '@netlify/sdk';
-import { deserialize } from 'bson';
-import { readdir, readFile, existsSync } from 'fs';
-import { promisify } from 'util';
-import { type Page, updatePages } from './update-pages';
-
-const readdirAsync = promisify(readdir);
-const readFileAsync = promisify(readFile);
+import { downloadPersistenceModule } from './persistence';
 
 const integration = new NetlifyIntegration();
 const ZIP_PATH = `${process.cwd()}/bundle/documents`;
 
 integration.addBuildEventHandler(
-	'onSuccess',
-	async ({ utils: { run, git } }) => {
-		/**
-		 * Minor note that persistence module also handles merging of ToCs for embedded products
-		 */
-		console.log('=========== Chatbot Data Upload Integration ================');
-
-		const bundleDirExists = existsSync(`${process.cwd()}/bundle`);
-
-		if (!bundleDirExists) await run.command('unzip -o bundle.zip -d bundle');
-
-		const zipContents = await readdirAsync(ZIP_PATH, {
-			recursive: true,
-		});
-
-		const bsonPages = zipContents.filter((fileName) => {
-			const splitFile = fileName.toString().split('.');
-
-			return splitFile[splitFile.length - 1] === 'bson';
-		});
-
-		const pageAstObjects = await Promise.all(
-			bsonPages.map(async (bsonFileName) => {
-				const rawData = await readFileAsync(`${ZIP_PATH}/${bsonFileName}`);
-
-				return deserialize(rawData) as Page;
-			}),
-		);
-
-		await updatePages(pageAstObjects, 'updated_documents');
-		console.log('=========== Chatbot Data Upload Integration ================');
-	},
+  'onPreBuild',
+  async ({ utils: { cache, run } }) => {
+    try {
+      await downloadPersistenceModule(run);
+    } catch (e) {
+      console.error('Unable to run the persistence module', e);
+    }
+  },
 );
-
 export { integration };

persistence-module/src/persistence.ts

@@ -0,0 +1,29 @@
+import type { NetlifyPluginUtils } from '@netlify/build';
+import { existsSync } from 'node:fs';
+
+const WORKER_POOL_PATH = `${process.cwd()}/docs-worker-pool`;
+const PERSISTENCE_PATH = `${WORKER_POOL_PATH}/modules/persistence`;
+
+export async function downloadPersistenceModule(
+  run: NetlifyPluginUtils['run'],
+): Promise<void> {
+  const isModuleDownloaded = existsSync(WORKER_POOL_PATH);
+
+  if (isModuleDownloaded) return;
+
+  await run.command(
+    'git clone --depth 1 --filter=tree:0 https://github.com/mongodb/docs-worker-pool.git --sparse',
+  );
+
+  await run.command('git sparse-checkout set --no-cone modules/persistence', {
+    cwd: WORKER_POOL_PATH,
+  });
+
+  await run.command('npm ci', {
+    cwd: PERSISTENCE_PATH,
+  });
+
+  await run.command('npm run build', {
+    cwd: PERSISTENCE_PATH,
+  });
+}