feat: ✨ force GC before enrollment history, add logging

icssc · Nov 20, 2023 · b8f58df · b8f58df
1 parent 09eb727
commit b8f58df
Showing 1 changed file with 20 additions and 13 deletions.
diff --git a/services/websoc-scraper-v2/index.ts b/services/websoc-scraper-v2/index.ts
@@ -473,13 +473,30 @@ async function scrape(name: string, term: Term) {
     },
   };
 
+  const [instructorsDeleted, buildingsDeleted, meetingsDeleted, sectionsDeleted] =
+    await prisma.$transaction([
+      prisma.websocSectionInstructor.deleteMany(params),
+      prisma.websocSectionMeetingBuilding.deleteMany(params),
+      prisma.websocSectionMeeting.deleteMany(params),
+      prisma.websocSection.deleteMany(params),
+    ]);
+
+  logger.info(`Removed ${instructorsDeleted.count} instructors`);
+  logger.info(`Removed ${buildingsDeleted.count} buildings`);
+  logger.info(`Removed ${meetingsDeleted.count} meetings`);
+  logger.info(`Removed ${sectionsDeleted.count} sections`);
+
+  forceGC();
+
   const enrollmentHistory = Object.fromEntries(
     (await prisma.websocEnrollmentHistory.findMany(params)).map((x) => [
       `${x.year}-${x.quarter}-${x.sectionCode}`,
       x,
     ]),
   );
 
+  logger.info(`Retrieved ${enrollmentHistory.length} enrollment history entries`);
+
   for (const { data } of Object.values(res)) {
     const key = `${data.year}-${data.quarter}-${data.sectionCode}`;
     if (key in enrollmentHistory) {
@@ -550,26 +567,16 @@ async function scrape(name: string, term: Term) {
     }
   }
 
-  await prisma.$transaction([
+  const [enrollmentInserted, enrollmentRemoved] = await prisma.$transaction([
     prisma.websocEnrollmentHistory.createMany({
       data: Object.values(enrollmentHistory) as Prisma.WebsocEnrollmentHistoryCreateManyInput[],
       skipDuplicates: true,
     }),
     prisma.websocEnrollmentHistory.deleteMany(params),
   ]);
 
-  const [instructorsDeleted, buildingsDeleted, meetingsDeleted, sectionsDeleted] =
-    await prisma.$transaction([
-      prisma.websocSectionInstructor.deleteMany(params),
-      prisma.websocSectionMeetingBuilding.deleteMany(params),
-      prisma.websocSectionMeeting.deleteMany(params),
-      prisma.websocSection.deleteMany(params),
-    ]);
-
-  logger.info(`Removed ${instructorsDeleted.count} instructors`);
-  logger.info(`Removed ${buildingsDeleted.count} buildings`);
-  logger.info(`Removed ${meetingsDeleted.count} meetings`);
-  logger.info(`Removed ${sectionsDeleted.count} sections`);
+  logger.info(`Inserted ${enrollmentInserted.count} enrollment history entries`);
+  logger.info(`Removed ${enrollmentRemoved.count} enrollment history entries`);
   logger.info("Sleeping for 3 minutes");
 
   await sleep(SLEEP_DURATION);