From 1a9195cc59546f307a6a1f72af7b64321a25dc48 Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Thu, 28 Dec 2023 00:52:51 -0800 Subject: [PATCH 1/8] =?UTF-8?q?feat:=20=E2=9C=A8=20add=20course/instructor?= =?UTF-8?q?=20previews=20to=20scraper/db?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- libs/db/prisma/schema.prisma | 20 +++++---- tools/registrar-scraper/src/index.ts | 61 ++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/libs/db/prisma/schema.prisma b/libs/db/prisma/schema.prisma index 7e357fd4..1ab67d64 100644 --- a/libs/db/prisma/schema.prisma +++ b/libs/db/prisma/schema.prisma @@ -70,11 +70,14 @@ model Course { maxUnits Float description String @db.Text departmentName String - instructorHistory Json + instructorHistory String[] + instructors Json @default("[]") prerequisiteTree Json - prerequisiteList Json + prerequisiteList String[] prerequisiteText String @db.Text - prerequisiteFor Json + prerequisiteFor String[] + prerequisites Json @default("[]") + dependencies Json @default("[]") repeatability String gradingOption String concurrent String @@ -82,21 +85,22 @@ model Course { restriction String @db.Text overlap String corequisites String - geList Json + geList String[] geText String - terms Json + terms String[] } model Instructor { - ucinetid String @id + ucinetid String @id name String shortenedName String title String email String department String - schools Json - relatedDepartments Json + schools String[] + relatedDepartments String[] courseHistory Json + courses Json @default("[]") } model GradesInstructor { diff --git a/tools/registrar-scraper/src/index.ts b/tools/registrar-scraper/src/index.ts index a0755241..aaaf255f 100644 --- a/tools/registrar-scraper/src/index.ts +++ b/tools/registrar-scraper/src/index.ts @@ -52,6 +52,67 @@ async function main() { data: Object.values(instructorInfo), }), ]); + const courses = Object.fromEntries((await prisma.course.findMany()).map((x) => [x.id, x])); + const instructors = Object.fromEntries( + (await prisma.instructor.findMany()).map((x) => [x.ucinetid, x]), + ); + const newCourses = []; + for (const course of Object.values(courses)) { + newCourses.push({ + ...course, + prerequisiteTree: course.prerequisiteTree as object, + instructors: course.instructorHistory + .map((x) => instructors[x]) + .filter((x) => x) + .map(({ ucinetid, name, shortenedName }) => ({ ucinetid, name, shortenedName })), + prerequisites: course.prerequisiteList + .map((x) => courses[x.replace(/ /g, "")]) + .filter((x) => x) + .map(({ id, title, department, courseNumber }) => ({ + id, + title, + department, + courseNumber, + })), + dependencies: course.prerequisiteFor + .map((x) => courses[x.replace(/ /g, "")]) + .filter((x) => x) + .map(({ id, title, department, courseNumber }) => ({ + id, + title, + department, + courseNumber, + })), + }); + } + const newInstructors = []; + for (const instructor of Object.values(instructors)) { + newInstructors.push({ + ...instructor, + courseHistory: instructor.courseHistory as object, + courses: Object.keys(instructor.courseHistory!) + .map((x) => courses[x.replace(/ /g, "")]) + .filter((x) => x) + .map(({ id, title, department, courseNumber }) => ({ + id, + title, + department, + courseNumber, + })), + }); + } + await prisma.$transaction([ + prisma.course.deleteMany({ where: { id: { in: Object.keys(courses) } } }), + prisma.instructor.deleteMany({ where: { ucinetid: { in: Object.keys(instructors) } } }), + prisma.course.createMany({ + data: newCourses, + skipDuplicates: true, + }), + prisma.instructor.createMany({ + data: newInstructors, + skipDuplicates: true, + }), + ]); } main().then(); From fa9910e062055c8e910d9c58eb2dbaecf4161b34 Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Thu, 28 Dec 2023 00:58:04 -0800 Subject: [PATCH 2/8] =?UTF-8?q?feat(types):=20=E2=9C=A8=20add=20previews?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/types/types/courses.ts | 22 ++++++++++++++++++++-- packages/types/types/instructor.ts | 8 ++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/packages/types/types/courses.ts b/packages/types/types/courses.ts index 5a8674c6..540c0dbe 100644 --- a/packages/types/types/courses.ts +++ b/packages/types/types/courses.ts @@ -1,4 +1,5 @@ import { CourseLevel, GECategory } from "./constants"; +import { InstructorPreview } from "./instructor"; /** * An object representing a prerequisite. @@ -44,6 +45,11 @@ export type PrerequisiteTree = { NOT?: Array; }; +/** + * An object that contains a subset of a course's metadata, for preview purposes. + */ +export type CoursePreview = Pick; + /** * An object that represents a course. * The type of the payload returned on a successful response from querying @@ -103,7 +109,7 @@ export type Course = { */ prerequisiteTree: PrerequisiteTree; /** - * The list of prerequisites for the course. + * The list of prerequisites' IDs for the course. */ prerequisiteList: string[]; /** @@ -111,7 +117,7 @@ export type Course = { */ prerequisiteText: string; /** - * The courses for which this course is a prerequisite. + * The IDs of the courses for which this course is a prerequisite. */ prerequisiteFor: string[]; /** @@ -154,6 +160,18 @@ export type Course = { * The list of terms in which this course was offered. */ terms: string[]; + /** + * + */ + instructors: InstructorPreview[]; + /** + * + */ + prerequisites: CoursePreview[]; + /** + * + */ + dependencies: CoursePreview[]; }; /** diff --git a/packages/types/types/instructor.ts b/packages/types/types/instructor.ts index e58a6f5d..9d90c99c 100644 --- a/packages/types/types/instructor.ts +++ b/packages/types/types/instructor.ts @@ -1,3 +1,5 @@ +import { CoursePreview } from "./courses"; + /** * An object representing an instructor. * The type of the payload returned on a successful response from querying @@ -42,6 +44,10 @@ export type Instructor = { * the instructor taught the corresponding course. */ courseHistory: Record; + /** + * + */ + courses: CoursePreview[]; }; /** @@ -49,3 +55,5 @@ export type Instructor = { * ``/v1/rest/instructors/all``. */ export type Instructors = Instructor[]; + +export type InstructorPreview = Pick; From d3c26f8a81e2e02fa0d73de3670d41da8f3b8424 Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Thu, 28 Dec 2023 01:04:45 -0800 Subject: [PATCH 3/8] =?UTF-8?q?docs:=20=F0=9F=93=9A=EF=B8=8F=20add=20comme?= =?UTF-8?q?nts=20to=20types?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/types/types/courses.ts | 6 +++--- packages/types/types/instructor.ts | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/types/types/courses.ts b/packages/types/types/courses.ts index 540c0dbe..6e08779e 100644 --- a/packages/types/types/courses.ts +++ b/packages/types/types/courses.ts @@ -161,15 +161,15 @@ export type Course = { */ terms: string[]; /** - * + * The previews for the instructors that have taught this course in the past. */ instructors: InstructorPreview[]; /** - * + * The previews for the courses that are required to take this course. */ prerequisites: CoursePreview[]; /** - * + * The previews for the courses that require this course. */ dependencies: CoursePreview[]; }; diff --git a/packages/types/types/instructor.ts b/packages/types/types/instructor.ts index 9d90c99c..f78822a7 100644 --- a/packages/types/types/instructor.ts +++ b/packages/types/types/instructor.ts @@ -45,7 +45,7 @@ export type Instructor = { */ courseHistory: Record; /** - * + * The previews for the course(s) this instructor has taught in the past. */ courses: CoursePreview[]; }; @@ -56,4 +56,7 @@ export type Instructor = { */ export type Instructors = Instructor[]; +/** + * An object that contains a subset of an instructor's metadata, for preview purposes. + */ export type InstructorPreview = Pick; From a97f16eb65a6bee1718adc7a9e922322ba5a1eee Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Thu, 28 Dec 2023 01:07:56 -0800 Subject: [PATCH 4/8] =?UTF-8?q?feat(graphql):=20=E2=9C=A8=20add=20graphql?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../routes/v1/graphql/schema/courses.graphql | 18 ++++++++++++++++++ .../v1/graphql/schema/instructors.graphql | 10 ++++++++++ 2 files changed, 28 insertions(+) diff --git a/apps/api/src/routes/v1/graphql/schema/courses.graphql b/apps/api/src/routes/v1/graphql/schema/courses.graphql index ca496793..cc1696f8 100644 --- a/apps/api/src/routes/v1/graphql/schema/courses.graphql +++ b/apps/api/src/routes/v1/graphql/schema/courses.graphql @@ -1,3 +1,15 @@ +"An object that contains a subset of a course's metadata, for preview purposes." +type CoursePreview { + "The course ID." + id: String! + "The department code that the course belongs to." + department: String! + "The course number of the course." + courseNumber: String! + "The title of the course." + title: String! +} + "An object that represents a course." type Course { "The course ID." @@ -52,6 +64,12 @@ type Course { geText: String! "The list of terms in which this course was offered." terms: [String!]! + "The previews for the instructors that have taught this course in the past." + instructors: [InstructorPreview!]! + "The previews for the courses that are required to take this course." + prerequisites: [CoursePreview!]! + "The previews for the courses that require this course." + dependencies: [CoursePreview!]! } extend type Query { diff --git a/apps/api/src/routes/v1/graphql/schema/instructors.graphql b/apps/api/src/routes/v1/graphql/schema/instructors.graphql index d18005fd..843ad689 100644 --- a/apps/api/src/routes/v1/graphql/schema/instructors.graphql +++ b/apps/api/src/routes/v1/graphql/schema/instructors.graphql @@ -1,3 +1,13 @@ +"An object that contains a subset of an instructor's metadata, for preview purposes." +type InstructorPreview { + "The instructor's UCINetID." + ucinetid: String! + "The full name of the instructor." + name: String! + "The shortened name (or WebSoc name; e.g. ``SHINDLER, M.``) of the instructor." + shortenedName: String! +} + "An object representing an instructor." type Instructor { "The instructor's UCINetID." From 04aa52effe5942d1fc245961977dade1a482087b Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Thu, 28 Dec 2023 01:33:51 -0800 Subject: [PATCH 5/8] =?UTF-8?q?fix:=20=F0=9F=90=9B=20type=20errors=20with?= =?UTF-8?q?=20new=20schema?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/src/lib/utils.ts | 20 ++++++++++++++----- apps/api/src/routes/v1/rest/courses/lib.ts | 6 +++--- .../api/src/routes/v1/rest/instructors/lib.ts | 4 ++-- .../src/instructor-scraper/index.ts | 8 ++++---- 4 files changed, 24 insertions(+), 14 deletions(-) diff --git a/apps/api/src/lib/utils.ts b/apps/api/src/lib/utils.ts index 1ed1a295..4d28e479 100644 --- a/apps/api/src/lib/utils.ts +++ b/apps/api/src/lib/utils.ts @@ -1,5 +1,12 @@ import { Course as PrismaCourse } from "@libs/db"; -import { Course, CourseLevel, GECategory, PrerequisiteTree } from "@peterportal-api/types"; +import { + Course, + CourseLevel, + CoursePreview, + GECategory, + InstructorPreview, + PrerequisiteTree, +} from "@peterportal-api/types"; const days = ["Su", "M", "Tu", "W", "Th", "F", "Sa"]; @@ -64,11 +71,14 @@ export function normalizeCourse(course: PrismaCourse): Course { return { ...course, courseLevel, - instructorHistory: course.instructorHistory as unknown as string[], + instructorHistory: course.instructorHistory, + instructors: course.instructors as unknown as InstructorPreview[], prerequisiteTree: course.prerequisiteTree as unknown as PrerequisiteTree, - prerequisiteList: course.prerequisiteList as unknown as string[], - prerequisiteFor: course.prerequisiteFor as unknown as string[], + prerequisiteList: course.prerequisiteList, + prerequisiteFor: course.prerequisiteFor, + prerequisites: course.prerequisites as unknown as CoursePreview[], + dependencies: course.prerequisites as unknown as CoursePreview[], geList, - terms: course.terms as unknown as string[], + terms: course.terms, }; } diff --git a/apps/api/src/routes/v1/rest/courses/lib.ts b/apps/api/src/routes/v1/rest/courses/lib.ts index aa13a485..1292fc78 100644 --- a/apps/api/src/routes/v1/rest/courses/lib.ts +++ b/apps/api/src/routes/v1/rest/courses/lib.ts @@ -37,16 +37,16 @@ export function constructPrismaQuery(parsedQuery: Query): Prisma.CourseWhereInpu if (parsedQuery.taughtByInstructors) AND.push({ OR: parsedQuery.taughtByInstructors.map((instructor) => ({ - instructorHistory: { array_contains: [instructor.toLowerCase()] }, + instructorHistory: { has: instructor.toLowerCase() }, })), }); if (parsedQuery.geCategory && parsedQuery.geCategory !== "ANY") - AND.push({ geList: { array_contains: [parsedQuery.geCategory] } }); + AND.push({ geList: { has: parsedQuery.geCategory } }); if (parsedQuery.taughtInTerms) AND.push({ - OR: parsedQuery.taughtInTerms.map((term) => ({ terms: { array_contains: [term] } })), + OR: parsedQuery.taughtInTerms.map((term) => ({ terms: { has: term } })), }); return { AND: AND.length ? AND : undefined }; diff --git a/apps/api/src/routes/v1/rest/instructors/lib.ts b/apps/api/src/routes/v1/rest/instructors/lib.ts index e76d553f..88070d4a 100644 --- a/apps/api/src/routes/v1/rest/instructors/lib.ts +++ b/apps/api/src/routes/v1/rest/instructors/lib.ts @@ -23,13 +23,13 @@ export function constructPrismaQuery(parsedQuery: Query): Prisma.InstructorWhere if (parsedQuery.schoolsContains) AND.push({ - OR: parsedQuery.schoolsContains.map((school) => ({ schools: { array_contains: [school] } })), + OR: parsedQuery.schoolsContains.map((school) => ({ schools: { has: school } })), }); if (parsedQuery.relatedDepartmentsContains) AND.push({ OR: parsedQuery.relatedDepartmentsContains.map((dept) => ({ - relatedDepartments: { array_contains: [dept.toUpperCase()] }, + relatedDepartments: { has: dept.toUpperCase() }, })), }); diff --git a/tools/registrar-scraper/src/instructor-scraper/index.ts b/tools/registrar-scraper/src/instructor-scraper/index.ts index b1dda259..42c407a6 100644 --- a/tools/registrar-scraper/src/instructor-scraper/index.ts +++ b/tools/registrar-scraper/src/instructor-scraper/index.ts @@ -26,7 +26,7 @@ type InstructorsData = { }; type InstructorsInfo = { - [ucinetid: string]: Instructor; + [ucinetid: string]: Omit; }; type InstructorsLog = { @@ -139,7 +139,7 @@ export async function getInstructors( } }); }); - const instructorPromises: Promise<[string, Instructor]>[] = []; + const instructorPromises: Promise<[string, Omit]>[] = []; Object.keys(instructorsDict).forEach((name) => { const schools = instructorsDict[name].schools; const related_departments = Array.from(instructorsDict[name].courses); @@ -214,9 +214,9 @@ async function getInstructor( relatedDepartments: string[], attempts: number, year_threshold: number, -): Promise<[string, Instructor]> { +): Promise<[string, Omit]> { logger.info(`Scraping data for ${instructorName}`); - const instructorObject: Instructor = { + const instructorObject: Omit = { name: instructorName, ucinetid: "", title: "", From 7eccbc24543b1045e05af8af6b51eaf393aa7eda Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Thu, 28 Dec 2023 05:11:13 -0800 Subject: [PATCH 6/8] =?UTF-8?q?fix:=20=F0=9F=90=9B=20im=20dumb?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/src/lib/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/lib/utils.ts b/apps/api/src/lib/utils.ts index 4d28e479..03415960 100644 --- a/apps/api/src/lib/utils.ts +++ b/apps/api/src/lib/utils.ts @@ -77,7 +77,7 @@ export function normalizeCourse(course: PrismaCourse): Course { prerequisiteList: course.prerequisiteList, prerequisiteFor: course.prerequisiteFor, prerequisites: course.prerequisites as unknown as CoursePreview[], - dependencies: course.prerequisites as unknown as CoursePreview[], + dependencies: course.dependencies as unknown as CoursePreview[], geList, terms: course.terms, }; From 7c510ee2a06598686a4dcfa2dc835af830a18e8b Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Wed, 10 Jan 2024 10:10:45 -0800 Subject: [PATCH 7/8] =?UTF-8?q?fix:=20=F0=9F=90=9B=20misc=20improvements?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Aponia --- apps/api/src/lib/utils.ts | 8 +- tools/registrar-scraper/src/index.ts | 84 +++++++++---------- .../src/instructor-scraper/index.ts | 12 +-- 3 files changed, 49 insertions(+), 55 deletions(-) diff --git a/apps/api/src/lib/utils.ts b/apps/api/src/lib/utils.ts index 03415960..af00584c 100644 --- a/apps/api/src/lib/utils.ts +++ b/apps/api/src/lib/utils.ts @@ -72,12 +72,12 @@ export function normalizeCourse(course: PrismaCourse): Course { ...course, courseLevel, instructorHistory: course.instructorHistory, - instructors: course.instructors as unknown as InstructorPreview[], - prerequisiteTree: course.prerequisiteTree as unknown as PrerequisiteTree, + instructors: course.instructors as InstructorPreview[], + prerequisiteTree: course.prerequisiteTree as PrerequisiteTree, prerequisiteList: course.prerequisiteList, prerequisiteFor: course.prerequisiteFor, - prerequisites: course.prerequisites as unknown as CoursePreview[], - dependencies: course.dependencies as unknown as CoursePreview[], + prerequisites: course.prerequisites as CoursePreview[], + dependencies: course.dependencies as CoursePreview[], geList, terms: course.terms, }; diff --git a/tools/registrar-scraper/src/index.ts b/tools/registrar-scraper/src/index.ts index aaaf255f..c13be550 100644 --- a/tools/registrar-scraper/src/index.ts +++ b/tools/registrar-scraper/src/index.ts @@ -56,51 +56,45 @@ async function main() { const instructors = Object.fromEntries( (await prisma.instructor.findMany()).map((x) => [x.ucinetid, x]), ); - const newCourses = []; - for (const course of Object.values(courses)) { - newCourses.push({ - ...course, - prerequisiteTree: course.prerequisiteTree as object, - instructors: course.instructorHistory - .map((x) => instructors[x]) - .filter((x) => x) - .map(({ ucinetid, name, shortenedName }) => ({ ucinetid, name, shortenedName })), - prerequisites: course.prerequisiteList - .map((x) => courses[x.replace(/ /g, "")]) - .filter((x) => x) - .map(({ id, title, department, courseNumber }) => ({ - id, - title, - department, - courseNumber, - })), - dependencies: course.prerequisiteFor - .map((x) => courses[x.replace(/ /g, "")]) - .filter((x) => x) - .map(({ id, title, department, courseNumber }) => ({ - id, - title, - department, - courseNumber, - })), - }); - } - const newInstructors = []; - for (const instructor of Object.values(instructors)) { - newInstructors.push({ - ...instructor, - courseHistory: instructor.courseHistory as object, - courses: Object.keys(instructor.courseHistory!) - .map((x) => courses[x.replace(/ /g, "")]) - .filter((x) => x) - .map(({ id, title, department, courseNumber }) => ({ - id, - title, - department, - courseNumber, - })), - }); - } + const newCourses = Object.values(courses).map((course) => ({ + ...course, + prerequisiteTree: course.prerequisiteTree as object, + instructors: course.instructorHistory + .map((x) => instructors[x]) + .filter((x) => x) + .map(({ ucinetid, name, shortenedName }) => ({ ucinetid, name, shortenedName })), + prerequisites: course.prerequisiteList + .map((x) => courses[x.replace(/ /g, "")]) + .filter((x) => x) + .map(({ id, title, department, courseNumber }) => ({ + id, + title, + department, + courseNumber, + })), + dependencies: course.prerequisiteFor + .map((x) => courses[x.replace(/ /g, "")]) + .filter((x) => x) + .map(({ id, title, department, courseNumber }) => ({ + id, + title, + department, + courseNumber, + })), + })); + const newInstructors = Object.values(instructors).map((instructor) => ({ + ...instructor, + courseHistory: instructor.courseHistory as object, + courses: Object.keys(instructor.courseHistory!) + .map((x) => courses[x.replace(/ /g, "")]) + .filter((x) => x) + .map(({ id, title, department, courseNumber }) => ({ + id, + title, + department, + courseNumber, + })), + })); await prisma.$transaction([ prisma.course.deleteMany({ where: { id: { in: Object.keys(courses) } } }), prisma.instructor.deleteMany({ where: { ucinetid: { in: Object.keys(instructors) } } }), diff --git a/tools/registrar-scraper/src/instructor-scraper/index.ts b/tools/registrar-scraper/src/instructor-scraper/index.ts index 42c407a6..3fd3a90a 100644 --- a/tools/registrar-scraper/src/instructor-scraper/index.ts +++ b/tools/registrar-scraper/src/instructor-scraper/index.ts @@ -25,9 +25,9 @@ type InstructorsData = { log: InstructorsLog; }; -type InstructorsInfo = { - [ucinetid: string]: Omit; -}; +type ScrapedInstructor = Omit; + +type InstructorsInfo = Record; type InstructorsLog = { [key: string]: @@ -139,7 +139,7 @@ export async function getInstructors( } }); }); - const instructorPromises: Promise<[string, Omit]>[] = []; + const instructorPromises: Promise<[string, ScrapedInstructor]>[] = []; Object.keys(instructorsDict).forEach((name) => { const schools = instructorsDict[name].schools; const related_departments = Array.from(instructorsDict[name].courses); @@ -214,9 +214,9 @@ async function getInstructor( relatedDepartments: string[], attempts: number, year_threshold: number, -): Promise<[string, Omit]> { +): Promise<[string, ScrapedInstructor]> { logger.info(`Scraping data for ${instructorName}`); - const instructorObject: Omit = { + const instructorObject: ScrapedInstructor = { name: instructorName, ucinetid: "", title: "", From a905f6e4fd37b4da4b1b58ed6506297697888e11 Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Tue, 16 Jan 2024 12:12:37 -0800 Subject: [PATCH 8/8] =?UTF-8?q?refactor:=20=E2=99=BB=EF=B8=8F=20use=20notN?= =?UTF-8?q?ull=20predicate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/registrar-scraper/src/index.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/registrar-scraper/src/index.ts b/tools/registrar-scraper/src/index.ts index c13be550..dafefd1d 100644 --- a/tools/registrar-scraper/src/index.ts +++ b/tools/registrar-scraper/src/index.ts @@ -9,6 +9,8 @@ import { getPrereqs } from "./prereq-scraper"; const prisma = new PrismaClient(); +const notNull = (x: T): x is NonNullable => x !== null && x !== undefined; + async function main() { const courseInfo = await getCourses(); const instructorInfo = Object.fromEntries( @@ -61,11 +63,11 @@ async function main() { prerequisiteTree: course.prerequisiteTree as object, instructors: course.instructorHistory .map((x) => instructors[x]) - .filter((x) => x) + .filter(notNull) .map(({ ucinetid, name, shortenedName }) => ({ ucinetid, name, shortenedName })), prerequisites: course.prerequisiteList .map((x) => courses[x.replace(/ /g, "")]) - .filter((x) => x) + .filter(notNull) .map(({ id, title, department, courseNumber }) => ({ id, title, @@ -74,7 +76,7 @@ async function main() { })), dependencies: course.prerequisiteFor .map((x) => courses[x.replace(/ /g, "")]) - .filter((x) => x) + .filter(notNull) .map(({ id, title, department, courseNumber }) => ({ id, title, @@ -87,7 +89,7 @@ async function main() { courseHistory: instructor.courseHistory as object, courses: Object.keys(instructor.courseHistory!) .map((x) => courses[x.replace(/ /g, "")]) - .filter((x) => x) + .filter(notNull) .map(({ id, title, department, courseNumber }) => ({ id, title,