Modify testing if the discussion.json is empty or notpresent

ohcnetwork · Jul 22, 2024 · 1f7f36a · 1f7f36a
1 parent 1a16748
commit 1f7f36a
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 12 deletions.
diff --git a/schemas/discussion-data.yaml b/schemas/discussion-data.yaml
@@ -11,6 +11,7 @@ required:
   - category
   - time
   - repoName
+  - participants
 
 properties:
     source:

diff --git a/scraper/src/github-scraper/discussion.ts b/scraper/src/github-scraper/discussion.ts
@@ -138,14 +138,10 @@ export async function scrapeDiscussions(
       endDate,
       startDate,
     );
-    if (allDiscussions) {
-      const parsedDiscussions = await parseDiscussionData(
-        allDiscussions,
-        endDate,
-        startDate,
-      );
-      await saveDiscussionData(parsedDiscussions, dataDir);
-    }
+    const parsedDiscussions =
+      allDiscussions &&
+      (await parseDiscussionData(allDiscussions, endDate, startDate));
+    await saveDiscussionData(parsedDiscussions, dataDir);
   } catch (error: any) {
     throw new Error(`Error fetching discussions: ${error.message}`);
   }

diff --git a/scraper/src/github-scraper/utils.ts b/scraper/src/github-scraper/utils.ts
@@ -177,7 +177,9 @@ export async function mergeDiscussions(
   newDiscussions: ParsedDiscussion[],
 ) {
   const mergedDiscussions = [...oldData];
-
+  if (!newDiscussions) {
+    return mergedDiscussions;
+  }
   newDiscussions.forEach((newDiscussion) => {
     const oldIndex = oldData.findIndex(
       (oldDiscussion) => oldDiscussion.link === newDiscussion.link,
@@ -203,14 +205,14 @@ export async function saveDiscussionData(
   dataDir: string,
 ) {
   // check data dir present or not and file is present or not if not then create it
-  await mkdir(dataDir + "/discussions", { recursive: true });
+  const discussionsDir = path.join(dataDir, "discussions");
+  await mkdir(discussionsDir, { recursive: true });
 
-  const file = path.join(dataDir + "/discussions", "discussions.json");
+  const file = path.join(discussionsDir, "discussions.json");
   try {
     // Try reading the file
     const response = await readFile(file);
     const oldData = JSON.parse(response.toString());
-
     const newData = await mergeDiscussions(oldData, discussions);
     const jsonData = JSON.stringify(newData, null, 2);
     await writeFile(file, jsonData);

diff --git a/tests/github-discussion-schema.test.mjs b/tests/github-discussion-schema.test.mjs
@@ -26,6 +26,9 @@ const filesInDir = fs
 
 filesInDir.forEach((file) => {
   const content = fs.readFileSync(join(GH_DATA, file)).toString();
+  if (content === "null") {
+    process.exit(1);
+  }
   const data = JSON.parse(stripJsonComments(content));
 
   describe(`Validate '${file}'`, function () {