From 1f7f36abcc2af43f1b10c4bc299a7c91880b0461 Mon Sep 17 00:00:00 2001
From: dhrumit parmar <dgparmar1406@gmail.com>
Date: Mon, 22 Jul 2024 18:28:36 +0530
Subject: [PATCH] Modify testing if the discussion.json is empty or notpresent

---
 schemas/discussion-data.yaml             |  1 +
 scraper/src/github-scraper/discussion.ts | 12 ++++--------
 scraper/src/github-scraper/utils.ts      | 10 ++++++----
 tests/github-discussion-schema.test.mjs  |  3 +++
 4 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/schemas/discussion-data.yaml b/schemas/discussion-data.yaml
index ca89a6e3..f86bd399 100644
--- a/schemas/discussion-data.yaml
+++ b/schemas/discussion-data.yaml
@@ -11,6 +11,7 @@ required:
   - category
   - time
   - repoName
+  - participants
 
 properties:
     source:
diff --git a/scraper/src/github-scraper/discussion.ts b/scraper/src/github-scraper/discussion.ts
index f415fb85..445ecdf7 100644
--- a/scraper/src/github-scraper/discussion.ts
+++ b/scraper/src/github-scraper/discussion.ts
@@ -138,14 +138,10 @@ export async function scrapeDiscussions(
       endDate,
       startDate,
     );
-    if (allDiscussions) {
-      const parsedDiscussions = await parseDiscussionData(
-        allDiscussions,
-        endDate,
-        startDate,
-      );
-      await saveDiscussionData(parsedDiscussions, dataDir);
-    }
+    const parsedDiscussions =
+      allDiscussions &&
+      (await parseDiscussionData(allDiscussions, endDate, startDate));
+    await saveDiscussionData(parsedDiscussions, dataDir);
   } catch (error: any) {
     throw new Error(`Error fetching discussions: ${error.message}`);
   }
diff --git a/scraper/src/github-scraper/utils.ts b/scraper/src/github-scraper/utils.ts
index 17d9cf71..a1fd6eb1 100644
--- a/scraper/src/github-scraper/utils.ts
+++ b/scraper/src/github-scraper/utils.ts
@@ -177,7 +177,9 @@ export async function mergeDiscussions(
   newDiscussions: ParsedDiscussion[],
 ) {
   const mergedDiscussions = [...oldData];
-
+  if (!newDiscussions) {
+    return mergedDiscussions;
+  }
   newDiscussions.forEach((newDiscussion) => {
     const oldIndex = oldData.findIndex(
       (oldDiscussion) => oldDiscussion.link === newDiscussion.link,
@@ -203,14 +205,14 @@ export async function saveDiscussionData(
   dataDir: string,
 ) {
   // check data dir present or not and file is present or not if not then create it
-  await mkdir(dataDir + "/discussions", { recursive: true });
+  const discussionsDir = path.join(dataDir, "discussions");
+  await mkdir(discussionsDir, { recursive: true });
 
-  const file = path.join(dataDir + "/discussions", "discussions.json");
+  const file = path.join(discussionsDir, "discussions.json");
   try {
     // Try reading the file
     const response = await readFile(file);
     const oldData = JSON.parse(response.toString());
-
     const newData = await mergeDiscussions(oldData, discussions);
     const jsonData = JSON.stringify(newData, null, 2);
     await writeFile(file, jsonData);
diff --git a/tests/github-discussion-schema.test.mjs b/tests/github-discussion-schema.test.mjs
index 8b58a93a..8acec43b 100644
--- a/tests/github-discussion-schema.test.mjs
+++ b/tests/github-discussion-schema.test.mjs
@@ -26,6 +26,9 @@ const filesInDir = fs
 
 filesInDir.forEach((file) => {
   const content = fs.readFileSync(join(GH_DATA, file)).toString();
+  if (content === "null") {
+    process.exit(1);
+  }
   const data = JSON.parse(stripJsonComments(content));
 
   describe(`Validate '${file}'`, function () {