diff --git a/schemas/discussion-data.yaml b/schemas/discussion-data.yaml index ca89a6e3..f86bd399 100644 --- a/schemas/discussion-data.yaml +++ b/schemas/discussion-data.yaml @@ -11,6 +11,7 @@ required: - category - time - repoName + - participants properties: source: diff --git a/scraper/src/github-scraper/discussion.ts b/scraper/src/github-scraper/discussion.ts index f415fb85..445ecdf7 100644 --- a/scraper/src/github-scraper/discussion.ts +++ b/scraper/src/github-scraper/discussion.ts @@ -138,14 +138,10 @@ export async function scrapeDiscussions( endDate, startDate, ); - if (allDiscussions) { - const parsedDiscussions = await parseDiscussionData( - allDiscussions, - endDate, - startDate, - ); - await saveDiscussionData(parsedDiscussions, dataDir); - } + const parsedDiscussions = + allDiscussions && + (await parseDiscussionData(allDiscussions, endDate, startDate)); + await saveDiscussionData(parsedDiscussions, dataDir); } catch (error: any) { throw new Error(`Error fetching discussions: ${error.message}`); } diff --git a/scraper/src/github-scraper/utils.ts b/scraper/src/github-scraper/utils.ts index 17d9cf71..a1fd6eb1 100644 --- a/scraper/src/github-scraper/utils.ts +++ b/scraper/src/github-scraper/utils.ts @@ -177,7 +177,9 @@ export async function mergeDiscussions( newDiscussions: ParsedDiscussion[], ) { const mergedDiscussions = [...oldData]; - + if (!newDiscussions) { + return mergedDiscussions; + } newDiscussions.forEach((newDiscussion) => { const oldIndex = oldData.findIndex( (oldDiscussion) => oldDiscussion.link === newDiscussion.link, @@ -203,14 +205,14 @@ export async function saveDiscussionData( dataDir: string, ) { // check data dir present or not and file is present or not if not then create it - await mkdir(dataDir + "/discussions", { recursive: true }); + const discussionsDir = path.join(dataDir, "discussions"); + await mkdir(discussionsDir, { recursive: true }); - const file = path.join(dataDir + "/discussions", "discussions.json"); + const file = path.join(discussionsDir, "discussions.json"); try { // Try reading the file const response = await readFile(file); const oldData = JSON.parse(response.toString()); - const newData = await mergeDiscussions(oldData, discussions); const jsonData = JSON.stringify(newData, null, 2); await writeFile(file, jsonData); diff --git a/tests/github-discussion-schema.test.mjs b/tests/github-discussion-schema.test.mjs index 8b58a93a..8acec43b 100644 --- a/tests/github-discussion-schema.test.mjs +++ b/tests/github-discussion-schema.test.mjs @@ -26,6 +26,9 @@ const filesInDir = fs filesInDir.forEach((file) => { const content = fs.readFileSync(join(GH_DATA, file)).toString(); + if (content === "null") { + process.exit(1); + } const data = JSON.parse(stripJsonComments(content)); describe(`Validate '${file}'`, function () {