From cccc382838e8fe414fda75cd721798864343ac5c Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Sun, 3 Sep 2023 21:10:44 -0700 Subject: [PATCH] =?UTF-8?q?feat!:=20=F0=9F=92=A5=20=E2=9C=A8=20normalize?= =?UTF-8?q?=20section=20meeting=20and=20final=20times=20(WIP)=20(#68)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Aponia --- apps/api/v1/graphql/src/graphql/enum.graphql | 8 +- .../api/v1/graphql/src/graphql/websoc.graphql | 50 ++- apps/api/v1/rest/websoc/ant.config.ts | 1 + apps/api/v1/rest/websoc/package.json | 2 + .../api/v1/rest/websoc/src/APILambdaClient.ts | 72 ++++ apps/api/v1/rest/websoc/src/index.ts | 74 ++-- apps/api/v1/rest/websoc/src/lib.ts | 203 ----------- .../rest-api/reference/websoc.md | 293 ++++++++++++++- libs/websoc-api-next/src/index.ts | 345 +++++++++++++++++- libs/websoc-utils/package.json | 8 + libs/websoc-utils/src/index.ts | 260 +++++++++++++ .../peterportal-api-next-types/package.json | 2 +- .../types/constants.ts | 4 + .../types/websoc.ts | 80 +++- pnpm-lock.yaml | 67 +++- services/websoc-proxy/package.json | 6 +- services/websoc-proxy/src/index.ts | 19 +- services/websoc-scraper-v2/index.ts | 6 +- tools/grades-updater/src/sanitize-data.ts | 5 +- 19 files changed, 1181 insertions(+), 324 deletions(-) create mode 100644 apps/api/v1/rest/websoc/src/APILambdaClient.ts create mode 100644 libs/websoc-utils/package.json create mode 100644 libs/websoc-utils/src/index.ts diff --git a/apps/api/v1/graphql/src/graphql/enum.graphql b/apps/api/v1/graphql/src/graphql/enum.graphql index 39f08d3c..2bbeecfa 100644 --- a/apps/api/v1/graphql/src/graphql/enum.graphql +++ b/apps/api/v1/graphql/src/graphql/enum.graphql @@ -55,10 +55,16 @@ enum CancelledCourses { Include Only } -"The set of valid options for filtering based on enrollment status." +"The set of valid enrollment statuses." enum EnrollmentStatus { OPEN Waitl FULL NewOnly } +"The set of valid final exam statuses for a section." +enum WebsocSectionFinalExamStatus { + NO_FINAL + TBA_FINAL + SCHEDULED_FINAL +} diff --git a/apps/api/v1/graphql/src/graphql/websoc.graphql b/apps/api/v1/graphql/src/graphql/websoc.graphql index c850a0a0..bc211dad 100644 --- a/apps/api/v1/graphql/src/graphql/websoc.graphql +++ b/apps/api/v1/graphql/src/graphql/websoc.graphql @@ -1,11 +1,27 @@ -"The meeting time for a section." +"A type that represents the hour and minute parts of a time." +type HourMinute { + "The hour (0-23)." + hour: Int! + "The minute (0-59)." + minute: Int! +} +"The meeting information for a section." type WebsocSectionMeeting { - "What day(s) the section meets on (e.g. ``MWF``)." - days: String! - "What time the section meets at." - time: String! - "The building(s) the section meets in." + """ + Whether the meeting time is TBA. + + If this field is `false`, then `days`, `startTime`, and `endTime` + are **guaranteed** to be non-null; otherwise, they are **guaranteed** to be null. + """ + timeIsTBA: Boolean! + "The classroom(s) the section meets in." bldg: [String!]! + "What day(s) the section meets on (e.g. ``MWF``)." + days: String + "The time at which the section begins." + startTime: HourMinute + "The time at which the section concludes." + endTime: HourMinute } "The enrollment statistics for a section." type WebsocSectionEnrollment { @@ -18,6 +34,26 @@ type WebsocSectionEnrollment { """ sectionEnrolled: String! } +"The final exam data for a section." +type WebsocSectionFinalExam { + """ + The status of the exam. + + If this field is `SCHEDULED_FINAL`, then all other fields are + **guaranteed** to be non-null; otherwise, they are **guaranteed** to be null. + """ + examStatus: WebsocSectionFinalExamStatus! + "The month in which the final exam takes place." + month: Int + "The day of the month in which the final exam takes place." + day: Int + "When the final exam starts." + startTime: HourMinute + "When the final exam ends." + endTime: HourMinute + "Where the final exam takes place." + bldg: String +} "A WebSoc section object." type WebsocSection { "The section code." @@ -33,7 +69,7 @@ type WebsocSection { "The meeting time(s) of this section." meetings: [WebsocSectionMeeting!]! "The date and time of the final exam for this section." - finalExam: String! + finalExam: WebsocSectionFinalExam! "The maximum capacity of this section." maxCapacity: String! """ diff --git a/apps/api/v1/rest/websoc/ant.config.ts b/apps/api/v1/rest/websoc/ant.config.ts index 819b501b..d4fc3d16 100644 --- a/apps/api/v1/rest/websoc/ant.config.ts +++ b/apps/api/v1/rest/websoc/ant.config.ts @@ -18,6 +18,7 @@ const outDir = resolve(cwd, "./dist"); const config: AntConfigStub = { esbuild: { + external: process.env.NODE_ENV === "development" ? [] : ["@services/websoc-proxy"], plugins: [ cleanCopy({ cwd, outDir, prismaClientDir, prismaSchema }), selectDelete(env.NODE_ENV, outDir), diff --git a/apps/api/v1/rest/websoc/package.json b/apps/api/v1/rest/websoc/package.json index 12cd69d1..1007b01a 100644 --- a/apps/api/v1/rest/websoc/package.json +++ b/apps/api/v1/rest/websoc/package.json @@ -9,11 +9,13 @@ "dependencies": { "@aws-sdk/client-lambda": "3.398.0", "@libs/db": "workspace:*", + "@libs/websoc-utils": "workspace:*", "ant-stack": "workspace:*", "zod": "3.22.2" }, "devDependencies": { "@libs/build-tools": "workspace:*", + "@services/websoc-proxy": "workspace:*", "peterportal-api-next-types": "workspace:*" } } diff --git a/apps/api/v1/rest/websoc/src/APILambdaClient.ts b/apps/api/v1/rest/websoc/src/APILambdaClient.ts new file mode 100644 index 00000000..0e37c777 --- /dev/null +++ b/apps/api/v1/rest/websoc/src/APILambdaClient.ts @@ -0,0 +1,72 @@ +import { InvokeCommand, LambdaClient, LambdaClientConfig } from "@aws-sdk/client-lambda"; +import type { WebsocAPIResponse } from "@libs/websoc-api-next"; +import { zeroUUID } from "ant-stack"; +import type { APIGatewayProxyEvent, APIGatewayProxyResult, Context } from "aws-lambda"; +import type { Department, TermData } from "peterportal-api-next-types"; + +/** + * A {@link `LambdaClient`} wrapper for the API to interface with the WebSoc proxy service. + * + * This class behaves slightly differently depending on the environment. + * In the development environment, instead of talking to the proxy service in AWS, it takes the code + * of the proxy service and executes it directly. + * + * Since the import process is asynchronous, instances of this class have to be created using + * the static {@link `new`} method, rather than invoking the constructor. For the same reason, + * the constructor is private, and doesn't actually do anything other than return an empty instance. + */ +export class APILambdaClient { + private client!: LambdaClient; + + private service?: ( + event: APIGatewayProxyEvent, + context: Context, + ) => Promise; + + private constructor() {} + + static async new(configuration: LambdaClientConfig = {}) { + const client = new APILambdaClient(); + client.client = new LambdaClient(configuration); + if (process.env.NODE_ENV === "development") { + const { handler } = await import("@services/websoc-proxy"); + client.service = handler; + } else { + client.service = undefined; + } + return client; + } + + private async invoke(body: Record) { + if (this.service) { + const payload = await this.service( + { body: JSON.stringify(body) } as APIGatewayProxyEvent, + { awsRequestId: zeroUUID } as Context, + ); + return JSON.parse(payload.body); + } + const res = await this.client.send( + new InvokeCommand({ + FunctionName: "peterportal-api-next-services-prod-websoc-proxy-function", + Payload: new TextEncoder().encode(JSON.stringify({ body: JSON.stringify(body) })), + }), + ); + const payload = JSON.parse(Buffer.from(res.Payload ?? []).toString()); + return JSON.parse(payload.body); + } + + async getDepts(body: Record): Promise { + const invocationResponse = await this.invoke(body); + return invocationResponse.payload; + } + + async getTerms(body: Record): Promise { + const invocationResponse = await this.invoke(body); + return invocationResponse.payload; + } + + async getWebsoc(body: Record): Promise { + const invocationResponse = await this.invoke(body); + return invocationResponse.payload; + } +} diff --git a/apps/api/v1/rest/websoc/src/index.ts b/apps/api/v1/rest/websoc/src/index.ts index 40a1665e..15a5a05d 100644 --- a/apps/api/v1/rest/websoc/src/index.ts +++ b/apps/api/v1/rest/websoc/src/index.ts @@ -1,38 +1,34 @@ -import { LambdaClient } from "@aws-sdk/client-lambda"; import { PrismaClient } from "@libs/db"; +import type { WebsocAPIResponse } from "@libs/websoc-api-next"; +import { combineAndNormalizeResponses, notNull, sortResponse } from "@libs/websoc-utils"; import { createErrorResult, createOKResult, type InternalHandler } from "ant-stack"; -import type { WebsocAPIResponse } from "peterportal-api-next-types"; import { ZodError } from "zod"; -import { - combineResponses, - constructPrismaQuery, - normalizeQuery, - notNull, - PeterPortalApiLambdaClient, - sortResponse, -} from "./lib"; +import { APILambdaClient } from "./APILambdaClient"; +import { constructPrismaQuery, normalizeQuery } from "./lib"; import { QuerySchema } from "./schema"; -let prisma: PrismaClient; - -const lambda = new LambdaClient({}); - -const lambdaClient = new PeterPortalApiLambdaClient(lambda); - const quarterOrder = ["Winter", "Spring", "Summer1", "Summer10wk", "Summer2", "Fall"]; +let prisma: PrismaClient; +let connected = false; +let lambdaClient: APILambdaClient; + export const GET: InternalHandler = async (request) => { const { headers, params, query, requestId } = request; prisma ??= new PrismaClient(); + lambdaClient ??= await APILambdaClient.new(); - if (request.isWarmerRequest) { + if (!connected) { try { await prisma.$connect(); - return createOKResult("Warmed", headers, requestId); - } catch (error) { - createErrorResult(500, error, requestId); + connected = true; + if (request.isWarmerRequest) { + return createOKResult("Warmed", headers, requestId); + } + } catch { + // no-op } } @@ -40,14 +36,16 @@ export const GET: InternalHandler = async (request) => { switch (params?.id) { case "terms": { const [gradesTerms, webSocTerms] = await Promise.all([ - prisma.gradesSection.findMany({ - distinct: ["year", "quarter"], - select: { - year: true, - quarter: true, - }, - orderBy: [{ year: "desc" }, { quarter: "desc" }], - }), + connected + ? prisma.gradesSection.findMany({ + distinct: ["year", "quarter"], + select: { + year: true, + quarter: true, + }, + orderBy: [{ year: "desc" }, { quarter: "desc" }], + }) + : [], lambdaClient.getTerms({ function: "terms" }), ]); @@ -92,12 +90,14 @@ export const GET: InternalHandler = async (request) => { case "depts": { const [gradesDepts, webSocDepts] = await Promise.all([ - prisma.gradesSection.findMany({ - distinct: ["department"], - select: { - department: true, - }, - }), + connected + ? prisma.gradesSection.findMany({ + distinct: ["department"], + select: { + department: true, + }, + }) + : [], lambdaClient.getDepts({ function: "depts" }), ]); @@ -126,7 +126,7 @@ export const GET: InternalHandler = async (request) => { const parsedQuery = QuerySchema.parse(query); - if (parsedQuery.cache) { + if (connected && parsedQuery.cache) { const websocSections = await prisma.websocSection.findMany({ where: constructPrismaQuery(parsedQuery), select: { department: true, courseNumber: true, data: true }, @@ -181,7 +181,7 @@ export const GET: InternalHandler = async (request) => { .map((x) => x.data) .filter(notNull) as WebsocAPIResponse[]; - const combinedResponses = combineResponses(...responses); + const combinedResponses = combineAndNormalizeResponses(...responses); return createOKResult(sortResponse(combinedResponses), headers, requestId); } @@ -190,7 +190,7 @@ export const GET: InternalHandler = async (request) => { .map((x) => x.data) .filter(notNull) as WebsocAPIResponse[]; - const combinedResponses = combineResponses(...websocApiResponses); + const combinedResponses = combineAndNormalizeResponses(...websocApiResponses); return createOKResult(sortResponse(combinedResponses), headers, requestId); } diff --git a/apps/api/v1/rest/websoc/src/lib.ts b/apps/api/v1/rest/websoc/src/lib.ts index cf77e756..625f1f56 100644 --- a/apps/api/v1/rest/websoc/src/lib.ts +++ b/apps/api/v1/rest/websoc/src/lib.ts @@ -1,47 +1,8 @@ -import { InvokeCommand, LambdaClient } from "@aws-sdk/client-lambda"; import { Prisma } from "@libs/db"; import { WebsocAPIOptions } from "@libs/websoc-api-next"; -import type { - Department, - TermData, - WebsocAPIResponse, - WebsocCourse, - WebsocDepartment, - WebsocSchool, - WebsocSection, - WebsocSectionMeeting, -} from "peterportal-api-next-types"; import type { Query } from "./schema"; -/** - * Section that also contains all relevant Websoc metadata. - */ -type EnhancedSection = { - school: WebsocSchool; - department: WebsocDepartment; - course: WebsocCourse; - section: WebsocSection; -}; - -/** - * Returns the lexicographical ordering of two elements. - * @param a The left hand side of the comparison. - * @param b The right hand side of the comparison. - */ -const lexOrd = (a: string, b: string): number => (a === b ? 0 : a > b ? 1 : -1); - -/** - * Get unique array of meetings. - */ -const getUniqueMeetings = (meetings: WebsocSectionMeeting[]) => - meetings.reduce((acc, meeting) => { - if (!acc.find((m) => m.days === meeting.days && m.time === meeting.time)) { - acc.push(meeting); - } - return acc; - }, [] as WebsocSectionMeeting[]); - /** * type guard that asserts that the settled promise was fulfilled */ @@ -53,101 +14,6 @@ export const fulfilled = (value: PromiseSettledResult): value is PromiseFu */ export const notNull = (x: T): x is NonNullable => x != null; -/** - * Sleep for the given number of milliseconds. - * @param duration Duration in ms. - */ -export const sleep = async (duration: number) => - new Promise((resolve) => setTimeout(resolve, duration)); - -/** - * Given all parent data about a section, isolate relevant data. - * @returns ``EnhancedSection`` with all deduped, relevant metadata. - */ -function isolateSection(data: EnhancedSection) { - const section = { - ...data.section, - meetings: getUniqueMeetings(data.section.meetings), - }; - - const course = { - ...data.course, - sections: [section], - }; - - const department = { - ...data.department, - courses: [course], - }; - - const school = { - ...data.school, - departments: [department], - }; - - return { school, department, course, section }; -} - -/** - * Combines all given response objects into a single response object, - * eliminating duplicates and merging substructures. - * @param responses The responses to combine. - */ -export function combineResponses(...responses: WebsocAPIResponse[]): WebsocAPIResponse { - const allSections = responses.flatMap((response) => - response.schools.flatMap((school) => - school.departments.flatMap((department) => - department.courses.flatMap((course) => - course.sections.map((section) => isolateSection({ school, department, course, section })), - ), - ), - ), - ); - - /** - * for each section: - * if one of its parent structures hasn't been declared, - * append the corresponding structure of the section - */ - const schools = allSections.reduce((acc, section) => { - const foundSchool = acc.find((s) => s.schoolName === section.school.schoolName); - if (!foundSchool) { - acc.push(section.school); - return acc; - } - - const foundDept = foundSchool.departments.find( - (d) => d.deptCode === section.department.deptCode, - ); - if (!foundDept) { - foundSchool.departments.push(section.department); - return acc; - } - - const foundCourse = foundDept.courses.find( - (c) => - c.courseNumber === section.course.courseNumber && - c.courseTitle === section.course.courseTitle, - ); - if (!foundCourse) { - foundDept.courses.push(section.course); - return acc; - } - - const foundSection = foundCourse.sections.find( - (s) => s.sectionCode === section.section.sectionCode, - ); - if (!foundSection) { - foundCourse.sections.push(section.section); - return acc; - } - - return acc; - }, [] as WebsocSchool[]); - - return { schools }; -} - /** * Converts a 12-hour time string into number of minutes since midnight. * @param time The time string to parse. @@ -390,72 +256,3 @@ export function normalizeQuery(query: Query): WebsocAPIOptions[] { } return queries; } - -/** - * Deeply sorts the provided response and returns the sorted response. - * - * Schools are sorted in lexicographical order of their name, departments are - * sorted in lexicographical order of their code, courses are sorted in - * numerical order of their number (with lexicographical tiebreaks), - * and sections are sorted in numerical order of their code. - * @param response The response to sort. - */ -export function sortResponse(response: WebsocAPIResponse): WebsocAPIResponse { - response.schools.forEach((schools) => { - schools.departments.forEach((department) => { - department.courses.forEach((course) => - course.sections.sort((a, b) => parseInt(a.sectionCode, 10) - parseInt(b.sectionCode, 10)), - ); - department.courses.sort((a, b) => { - const numOrd = - parseInt(a.courseNumber.replace(/\D/g, ""), 10) - - parseInt(b.courseNumber.replace(/\D/g, ""), 10); - return numOrd ? numOrd : lexOrd(a.courseNumber, b.courseNumber); - }); - }); - schools.departments.sort((a, b) => lexOrd(a.deptCode, b.deptCode)); - }); - - response.schools.sort((a, b) => lexOrd(a.schoolName, b.schoolName)); - - return response; -} - -/** - * Wraps the Lambda client to invoke the WebSoc proxy service. - * @param client The Lambda Client to use. - * @param body The body to send to the proxy service. - */ -export async function invokeProxyService(client: LambdaClient, body: Record) { - const res = await client.send( - new InvokeCommand({ - FunctionName: "peterportal-api-next-services-prod-websoc-proxy-function", - Payload: new TextEncoder().encode(JSON.stringify({ body: JSON.stringify(body) })), - }), - ); - const payload = JSON.parse(Buffer.from(res.Payload ?? []).toString()); - return JSON.parse(payload.body); -} - -export class PeterPortalApiLambdaClient { - constructor(private readonly client: LambdaClient) {} - - private async invoke(body: Record) { - return invokeProxyService(this.client, body); - } - - async getDepts(body: Record): Promise { - const invocationResponse = await this.invoke(body); - return invocationResponse.payload; - } - - async getTerms(body: Record): Promise { - const invocationResponse = await this.invoke(body); - return invocationResponse.payload; - } - - async getWebsoc(body: Record): Promise { - const invocationResponse = await this.invoke(body); - return invocationResponse.payload; - } -} diff --git a/apps/docs/docs/developers-guide/rest-api/reference/websoc.md b/apps/docs/docs/developers-guide/rest-api/reference/websoc.md index 1eb390fc..30579cf2 100644 --- a/apps/docs/docs/developers-guide/rest-api/reference/websoc.md +++ b/apps/docs/docs/developers-guide/rest-api/reference/websoc.md @@ -118,7 +118,7 @@ Which sections to exclude based on their cancellation status. Case-sensitive. De ```bash -curl "https://api-next.peterportal.org/v1/rest/websoc?year=2023&quarter=Spring§ionCodes=34270" +curl "https://api-next.peterportal.org/v1/rest/websoc" ``` @@ -127,7 +127,14 @@ curl "https://api-next.peterportal.org/v1/rest/websoc?year=2023&quarter=Spring&s ### Response - + + +
+Section with no final exam + +```bash +curl "https://api-next.peterportal.org/v1/rest/websoc?year=2023&quarter=Spring§ionCodes=34271" +``` ```json { @@ -139,7 +146,158 @@ curl "https://api-next.peterportal.org/v1/rest/websoc?year=2023&quarter=Spring&s { "deptComment": "...", "sectionCodeRangeComments": [], - "courseNumberRangeComments": [], + "courseNumberRangeComments": ["..."], + "deptCode": "COMPSCI", + "deptName": "Computer Science", + "courses": [ + { + "deptCode": "COMPSCI", + "courseComment": "", + "prerequisiteLink": "https://www.reg.uci.edu/cob/prrqcgi?term=202314&dept=COMPSCI&action=view_by_term#162", + "courseNumber": "162", + "courseTitle": "FORMAL LANG & AUTM", + "sections": [ + { + "sectionCode": "34271", + "sectionType": "Dis", + "sectionNum": "A", + "units": "0", + "instructors": ["DEES, M.", "CHIU, A.", "SHINDLER, M."], + "meetings": [ + { + "timeIsTBA": false, + "bldg": ["SSH 100"], + "days": "Tu", + "startTime": { "hour": 19, "minute": 0 }, + "endTime": { "hour": 19, "minute": 50 } + } + ], + "finalExam": { + "examStatus": "NO_FINAL", + "dayOfWeek": null, + "month": null, + "day": null, + "startTime": null, + "endTime": null, + "bldg": null + }, + "maxCapacity": "249", + "numCurrentlyEnrolled": { + "totalEnrolled": "170", + "sectionEnrolled": "169" + }, + "numOnWaitlist": "", + "numWaitlistCap": "", + "numRequested": "219", + "numNewOnlyReserved": "", + "restrictions": "A", + "status": "OPEN", + "sectionComment": "\n\t\t\t

Same as 65131 (LSCI 102, Dis A).

\n\t\t\t " + } + ] + } + ] + } + ] + } + ] +} +``` + +
+ +
+Section with TBA meeting time and final exam + +```bash +curl "https://api-next.peterportal.org/v1/rest/websoc?year=2023&quarter=Spring§ionCodes=34160" +``` + +```json +{ + "schools": [ + { + "schoolName": "Donald Bren School of Information and Computer Sciences", + "schoolComment": "...", + "departments": [ + { + "deptComment": "...", + "sectionCodeRangeComments": [], + "courseNumberRangeComments": ["..."], + "deptCode": "COMPSCI", + "deptName": "Computer Science", + "courses": [ + { + "deptCode": "COMPSCI", + "courseComment": "", + "prerequisiteLink": "https://www.reg.uci.edu/cob/prrqcgi?term=202314&dept=COMPSCI&action=view_by_term#143A", + "courseNumber": "143A", + "courseTitle": "PRNCPLS OPERTNG SYS", + "sections": [ + { + "sectionCode": "34160", + "sectionType": "Lec", + "sectionNum": "A", + "units": "4", + "instructors": ["BIC, L.", "GIYAHCHI, T.", "YI, S."], + "meetings": [ + { + "timeIsTBA": true, + "bldg": ["ON LINE"], + "days": null, + "startTime": null, + "endTime": null + } + ], + "finalExam": { + "examStatus": "TBA_FINAL", + "dayOfWeek": null, + "month": null, + "day": null, + "startTime": null, + "endTime": null, + "bldg": null + }, + "maxCapacity": "125", + "numCurrentlyEnrolled": { "totalEnrolled": "124", "sectionEnrolled": "" }, + "numOnWaitlist": "", + "numWaitlistCap": "", + "numRequested": "186", + "numNewOnlyReserved": "", + "restrictions": "A", + "status": "OPEN", + "sectionComment": "" + } + ] + } + ] + } + ] + } + ] +} +``` + +
+ +
+Section with a final exam held in the same location as the lecture + +```bash +curl "https://api-next.peterportal.org/v1/rest/websoc?year=2023&quarter=Spring§ionCodes=34270" +``` + +```json +{ + "schools": [ + { + "schoolName": "Donald Bren School of Information and Computer Sciences", + "schoolComment": "...", + "departments": [ + { + "deptComment": "...", + "sectionCodeRangeComments": [], + "courseNumberRangeComments": ["..."], "deptCode": "COMPSCI", "deptName": "Computer Science", "courses": [ @@ -158,22 +316,29 @@ curl "https://api-next.peterportal.org/v1/rest/websoc?year=2023&quarter=Spring&s "instructors": ["SHINDLER, M."], "meetings": [ { + "timeIsTBA": false, + "bldg": ["ALP 2300"], "days": "MWF", - "time": "10:00-10:50 ", - "bldg": ["ALP 2300"] + "startTime": { "hour": 10, "minute": 0 }, + "endTime": { "hour": 10, "minute": 50 } } ], - "finalExam": "Mon Jun 12 10:30-12:30pm", - "maxCapacity": "250", - "numCurrentlyEnrolled": { - "totalEnrolled": "168", - "sectionEnrolled": "166" + "finalExam": { + "examStatus": "SCHEDULED_FINAL", + "dayOfWeek": "Mon", + "month": 5, + "day": 12, + "startTime": { "hour": 10, "minute": 30 }, + "endTime": { "hour": 12, "minute": 30 }, + "bldg": ["ALP 2300"] }, - "numOnWaitlist": "0", - "numWaitlistCap": "38", - "numRequested": "191", - "numNewOnlyReserved": "0", - "restrictions": "J", + "maxCapacity": "249", + "numCurrentlyEnrolled": { "totalEnrolled": "170", "sectionEnrolled": "169" }, + "numOnWaitlist": "", + "numWaitlistCap": "", + "numRequested": "246", + "numNewOnlyReserved": "", + "restrictions": "A", "status": "OPEN", "sectionComment": "\n\t\t\t

Same as 65130 (LSCI 102, Lec A).

\n\t\t\t " } @@ -187,6 +352,89 @@ curl "https://api-next.peterportal.org/v1/rest/websoc?year=2023&quarter=Spring&s } ``` +
+ +
+Section with multiple meetings and final exam held in a different location + +```bash +curl "https://api-next.peterportal.org/v1/rest/websoc?year=2023&quarter=Spring§ionCodes=44000" +``` + +```json +{ + "schools": [ + { + "schoolName": "School of Physical Sciences", + "schoolComment": "...", + "departments": [ + { + "deptComment": "...", + "sectionCodeRangeComments": [], + "courseNumberRangeComments": [], + "deptCode": "MATH", + "deptName": "Mathematics", + "courses": [ + { + "deptCode": "MATH", + "courseComment": "...", + "prerequisiteLink": "https://www.reg.uci.edu/cob/prrqcgi?term=202314&dept=MATH&action=view_by_term#2A", + "courseNumber": "2A", + "courseTitle": "CALCULUS I", + "sections": [ + { + "sectionCode": "44000", + "sectionType": "Lec", + "sectionNum": "A", + "units": "4", + "instructors": ["HUBER, K."], + "meetings": [ + { + "timeIsTBA": false, + "bldg": ["ON LINE"], + "days": "WF", + "startTime": { "hour": 10, "minute": 0 }, + "endTime": { "hour": 10, "minute": 50 } + }, + { + "timeIsTBA": false, + "bldg": ["ALP 1300"], + "days": "M", + "startTime": { "hour": 10, "minute": 0 }, + "endTime": { "hour": 10, "minute": 50 } + } + ], + "finalExam": { + "examStatus": "SCHEDULED_FINAL", + "dayOfWeek": "Sat", + "month": 5, + "day": 10, + "startTime": { "hour": 13, "minute": 30 }, + "endTime": { "hour": 15, "minute": 30 }, + "bldg": ["HIB 100"] + }, + "maxCapacity": "295", + "numCurrentlyEnrolled": { "totalEnrolled": "282", "sectionEnrolled": "" }, + "numOnWaitlist": "", + "numWaitlistCap": "", + "numRequested": "593", + "numNewOnlyReserved": "", + "restrictions": "A", + "status": "OPEN", + "sectionComment": "" + } + ] + } + ] + } + ] + } + ] +} +``` + +
+ @@ -213,11 +461,20 @@ type WebsocAPIResponse = { units: string; instructors: string[]; meetings: { - days: string; - time: string; + timeIsTBA: boolean; bldg: string[]; + days: string | null; + startTime: { hour: number; minute: number } | null; + endTime: { hour: number; minute: number } | null; }[]; - finalExam: string; + finalExam: { + examStatus: "NO_FINAL" | "TBA_FINAL" | "SCHEDULED_FINAL"; + month: number | null; + day: number | null; + startTime: { hour: number; minute: number } | null; + endTime: { hour: number; minute: number } | null; + bldg: string[] | null; + }; maxCapacity: string; numCurrentlyEnrolled: { totalEnrolled: string; diff --git a/libs/websoc-api-next/src/index.ts b/libs/websoc-api-next/src/index.ts index 9f186948..df795735 100644 --- a/libs/websoc-api-next/src/index.ts +++ b/libs/websoc-api-next/src/index.ts @@ -1,18 +1,7 @@ import { transform } from "@ap0nia/camaro"; import { load } from "cheerio"; import fetch from "cross-fetch"; -import type { - CancelledCourses, - Department, - Division, - FullCourses, - GE, - SectionType, - Term, - TermData, - WebsocAPIResponse, - WebsocSectionMeeting, -} from "peterportal-api-next-types"; +import type { Department, TermData } from "peterportal-api-next-types"; /* region Constants */ @@ -79,6 +68,8 @@ const template = { ], }; +/* endregion */ + /* region Internal type declarations */ type RequireAtLeastOne = Omit & @@ -124,7 +115,325 @@ type OptionalOptions = { /* region Exported type declarations */ /** - * The type alias for the options object accepted by `callWebSocAPI`. + * The list of quarters in an academic year. + */ +export const quarters = ["Fall", "Winter", "Spring", "Summer1", "Summer10wk", "Summer2"] as const; +/** + * The list of all section types. + */ +export const sectionTypes = [ + "Act", + "Col", + "Dis", + "Fld", + "Lab", + "Lec", + "Qiz", + "Res", + "Sem", + "Stu", + "Tap", + "Tut", +] as const; +/** + * The list of options for filtering full courses. + */ +export const fullCoursesOptions = [ + "SkipFull", + "SkipFullWaitlist", + "FullOnly", + "OverEnrolled", +] as const; +/** + * The list of options for filtering cancelled courses. + */ +export const cancelledCoursesOptions = ["Exclude", "Include", "Only"] as const; +/** + * The list of GE category codes. + */ +export const geCodes = [ + "GE-1A", + "GE-1B", + "GE-2", + "GE-3", + "GE-4", + "GE-5A", + "GE-5B", + "GE-6", + "GE-7", + "GE-8", +] as const; +/** + * The list of GE category names. + */ +export const geCategories = [ + "GE Ia: Lower Division Writing", + "GE Ib: Upper Division Writing", + "GE II: Science and Technology", + "GE III: Social & Behavioral Sciences", + "GE IV: Arts and Humanities", + "GE Va: Quantitative Literacy", + "GE Vb: Formal Reasoning", + "GE VI: Language Other Than English", + "GE VII: Multicultural Studies", + "GE VIII: International/Global Issues", +] as const; +/** + * The list of division codes. + */ +export const divisionCodes = ["LowerDiv", "UpperDiv", "Graduate"] as const; +/** + * The list of course level (division) names. + */ +export const courseLevels = [ + "Lower Division (1-99)", + "Upper Division (100-199)", + "Graduate/Professional Only (200+)", +] as const; + +/** + * Represents the absence of a particular value to filter for. + */ +export const anyArray = ["ANY"] as const; +export type Any = (typeof anyArray)[number]; +/** + * The quarter in an academic year. + */ +export type Quarter = (typeof quarters)[number]; +/** + * The type of the section. + */ +export type SectionType = Any | (typeof sectionTypes)[number]; +/** + * The option to filter full courses by. + */ +export type FullCourses = Any | (typeof fullCoursesOptions)[number]; +/** + * The option to filter cancelled courses by. + */ +export type CancelledCourses = (typeof cancelledCoursesOptions)[number]; +/** + * The GE category code. + */ +export type GE = Any | (typeof geCodes)[number]; +/** + * The division code. + */ +export type Division = Any | (typeof divisionCodes)[number]; +/** + * The course level name. + */ +export type CourseLevel = (typeof courseLevels)[number]; + +/** + * The meeting time for a section. + */ +export type WebsocSectionMeeting = { + /** + * What day(s) the section meets on (e.g. ``MWF``). + */ + days: string; + /** + * What time the section meets at. + */ + time: string; + /** + * The building(s) the section meets in. + */ + bldg: string[]; +}; + +/** + * The enrollment statistics for a section. + */ +export type WebsocSectionEnrollment = { + /** + * The total number of students enrolled in this section. + */ + totalEnrolled: string; + /** + * The number of students enrolled in the section referred to by this section + * code, if the section is cross-listed. If the section is not cross-listed, + * this field is the empty string. + */ + sectionEnrolled: string; +}; + +/** + * A WebSoc section object. + */ +export type WebsocSection = { + /** + * The section code. + */ + sectionCode: string; + /** + * The section type (e.g. ``Lec``, ``Dis``, ``Lab``, etc.) + */ + sectionType: string; + /** + * The section number (e.g. ``A1``). + */ + sectionNum: string; + /** + * The number of units afforded by taking this section. + */ + units: string; + /** + * The name(s) of the instructor(s) teaching this section. + */ + instructors: string[]; + /** + * The meeting time(s) of this section. + */ + meetings: WebsocSectionMeeting[]; + /** + * The date and time of the final exam for this section. + */ + finalExam: string; + /** + * The maximum capacity of this section. + */ + maxCapacity: string; + /** + * The number of students currently enrolled (cross-listed or otherwise) in + * this section. + */ + numCurrentlyEnrolled: WebsocSectionEnrollment; + /** + * The number of students currently on the waitlist for this section. + */ + numOnWaitlist: string; + /** + * The maximum number of students that can be on the waitlist for this section. + */ + numWaitlistCap: string; + /** + * The number of students who have requested to be enrolled in this section. + */ + numRequested: string; + /** + * The number of seats in this section reserved for new students. + */ + numNewOnlyReserved: string; + /** + * The restriction code(s) for this section. + */ + restrictions: string; + /** + * The enrollment status. + */ + status: "OPEN" | "Waitl" | "FULL" | "NewOnly"; + /** + * Any comments for the section. + */ + sectionComment: string; +}; + +/** + * A WebSoc course object. + */ +export type WebsocCourse = { + /** + * The code of the department the course belongs to. + */ + deptCode: string; + /** + * The course number. + */ + courseNumber: string; + /** + * The title of the course. + */ + courseTitle: string; + /** + * Any comments for the course. + */ + courseComment: string; + /** + * The link to the WebReg Course Prerequisites page for this course. + */ + prerequisiteLink: string; + /** + * All sections of the course. + */ + sections: WebsocSection[]; +}; + +/** + * A WebSoc department object. + */ +export type WebsocDepartment = { + /** + * The name of the department. + */ + deptName: string; + /** + * The department code. + */ + deptCode: string; + /** + * Any comments from the department. + */ + deptComment: string; + /** + * All courses of the department. + */ + courses: WebsocCourse[]; + /** + * Any comments for section code(s) under the department. + */ + sectionCodeRangeComments: string[]; + /** + * Any comments for course number(s) under the department. + */ + courseNumberRangeComments: string[]; +}; + +/** + * A WebSoc school object. + */ +export type WebsocSchool = { + /** + * The name of the school. + */ + schoolName: string; + /** + * Any comments from the school. + */ + schoolComment: string; + /** + * All departments of the school. + */ + departments: WebsocDepartment[]; +}; + +/** + * An object that represents a specific term. + */ +export type Term = { + /** + * The year of the term. + */ + year: string; + /** + * The quarter of the term. + */ + quarter: Quarter; +}; + +/** + * The type alias for the response from {@link `callWebSocAPI`}. + */ +export type WebsocAPIResponse = { + /** + * All schools matched by the query. + */ + schools: WebsocSchool[]; +}; + +/** + * The type alias for the options object accepted by {@link `callWebSocAPI`}. * * If your editor supports intelligent code completion, the fully expanded * initial type will probably look horrifying. But it's really not that bad. @@ -177,7 +486,7 @@ const getCodedDiv = (div: Division): string => { export const callWebSocAPI = async ( term: Term, - options: WebsocAPIOptions + options: WebsocAPIOptions, ): Promise => { const { ge = "ANY", @@ -238,9 +547,9 @@ export const callWebSocAPI = async ( meeting.bldg = [meeting.bldg].flat(); }); section.meetings = getUniqueMeetings(section.meetings); - }) - ) - ) + }), + ), + ), ); return json; }; @@ -329,7 +638,7 @@ export const getDepts = async (): Promise => { x .split(".") .filter((y) => y != " ") - .map((y) => y.trim()) + .map((y) => y.trim()), ) .filter((x) => x[0].length) .map((x): Department => { diff --git a/libs/websoc-utils/package.json b/libs/websoc-utils/package.json new file mode 100644 index 00000000..e9f203c4 --- /dev/null +++ b/libs/websoc-utils/package.json @@ -0,0 +1,8 @@ +{ + "name": "@libs/websoc-utils", + "version": "0.0.0", + "private": true, + "type": "module", + "main": "src/index.ts", + "types": "src/index.ts" +} diff --git a/libs/websoc-utils/src/index.ts b/libs/websoc-utils/src/index.ts new file mode 100644 index 00000000..03c2dc6c --- /dev/null +++ b/libs/websoc-utils/src/index.ts @@ -0,0 +1,260 @@ +import type { + WebsocAPIResponse, + WebsocCourse, + WebsocDepartment, + WebsocSchool, + WebsocSection, + WebsocSectionMeeting, +} from "@libs/websoc-api-next"; +import type { + DayOfWeek, + WebsocAPIResponse as NormalizedResponse, + WebsocCourse as NormalizedCourse, + WebsocDepartment as NormalizedDepartment, + WebsocSectionFinalExam as NormalizedFinalExam, + WebsocSchool as NormalizedSchool, + WebsocSection as NormalizedSection, + WebsocSectionMeeting as NormalizedMeeting, +} from "peterportal-api-next-types"; + +export type EnhancedSection = { + school: WebsocSchool; + department: WebsocDepartment; + course: WebsocCourse; + section: WebsocSection; +}; + +/** + * Normalized section that also contains all relevant WebSoc metadata. + */ +export type EnhancedNormalizedSection = { + school: NormalizedSchool; + department: NormalizedDepartment; + course: NormalizedCourse; + section: NormalizedSection; +}; + +/** + * type guard that asserts that the settled promise was fulfilled + */ +export const fulfilled = (value: PromiseSettledResult): value is PromiseFulfilledResult => + value.status === "fulfilled"; + +/** + * type guard that asserts input is defined + */ +export const notNull = (x: T): x is NonNullable => x != null; + +/** + * Sleep for the given number of milliseconds. + * @param duration Duration in ms. + */ +export const sleep = async (duration: number) => + new Promise((resolve) => setTimeout(resolve, duration)); + +const months = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; + +/** + * Returns the lexicographical ordering of two elements. + * @param a The left hand side of the comparison. + * @param b The right hand side of the comparison. + */ +const lexOrd = (a: string, b: string): number => (a === b ? 0 : a > b ? 1 : -1); + +/** + * Get unique array of meetings. + */ +const getUniqueMeetings = (meetings: WebsocSectionMeeting[]) => + meetings.reduce((acc, meeting) => { + if (!acc.find((m) => m.days === meeting.days && m.time === meeting.time)) { + acc.push(meeting); + } + return acc; + }, [] as WebsocSectionMeeting[]); + +/** + * Combines all given response objects into a single response object, + * eliminating duplicates and merging substructures. + * @param responses The responses to combine. + */ +export function combineAndNormalizeResponses( + ...responses: WebsocAPIResponse[] +): NormalizedResponse { + const allSections = responses.flatMap((response) => + response.schools.flatMap((school) => + school.departments.flatMap((department) => + department.courses.flatMap((course) => + course.sections.map((section) => isolateSection({ school, department, course, section })), + ), + ), + ), + ); + + /** + * for each section: + * if one of its parent structures hasn't been declared, + * append the corresponding structure of the section + */ + const schools = allSections.reduce((acc, section) => { + const foundSchool = acc.find((s) => s.schoolName === section.school.schoolName); + if (!foundSchool) { + acc.push(section.school); + return acc; + } + + const foundDept = foundSchool.departments.find( + (d) => d.deptCode === section.department.deptCode, + ); + if (!foundDept) { + foundSchool.departments.push(section.department); + return acc; + } + + const foundCourse = foundDept.courses.find( + (c) => + c.courseNumber === section.course.courseNumber && + c.courseTitle === section.course.courseTitle, + ); + if (!foundCourse) { + foundDept.courses.push(section.course); + return acc; + } + + const foundSection = foundCourse.sections.find( + (s) => s.sectionCode === section.section.sectionCode, + ); + if (!foundSection) { + foundCourse.sections.push(section.section); + return acc; + } + + return acc; + }, [] as NormalizedSchool[]); + + return { schools }; +} + +function parseNonTBAStartAndEndTimes(time: string) { + let startTime, endTime; + const [startTimeString, endTimeString] = time + .trim() + .split("-") + .map((x) => x.trim()); + const [startTimeHour, startTimeMinute] = startTimeString.split(":"); + startTime = (parseInt(startTimeHour, 10) % 12) * 60 + parseInt(startTimeMinute, 10); + const [endTimeHour, endTimeMinute] = endTimeString.split(":"); + endTime = (parseInt(endTimeHour, 10) % 12) * 60 + parseInt(endTimeMinute, 10); + if (endTimeMinute.includes("p")) { + startTime += 12 * 60; + endTime += 12 * 60; + } + if (startTime > endTime) startTime -= 12 * 60; + return { + startTime: { hour: Math.floor(startTime / 60), minute: startTime % 60 }, + endTime: { hour: Math.floor(endTime / 60), minute: endTime % 60 }, + }; +} + +function parseFinalExamString(section: WebsocSection): NormalizedFinalExam { + if (section.finalExam === "") + return { + examStatus: "NO_FINAL", + dayOfWeek: null, + month: null, + day: null, + startTime: null, + endTime: null, + bldg: null, + }; + if (section.finalExam === "TBA") + return { + examStatus: "TBA_FINAL", + dayOfWeek: null, + month: null, + day: null, + startTime: null, + endTime: null, + bldg: null, + }; + const [dateTime, locations] = section.finalExam.split("@").map((x) => x?.trim()); + const [dayOfWeek, month, day, time] = dateTime.split(" "); + const { startTime, endTime } = parseNonTBAStartAndEndTimes(time); + return { + examStatus: "SCHEDULED_FINAL", + dayOfWeek: dayOfWeek as DayOfWeek, + month: months.indexOf(month), + day: parseInt(day, 10), + startTime, + endTime, + bldg: locations ? locations.split(",").map((x) => x?.trim()) : [section.meetings[0].bldg[0]], + }; +} + +/** + * Given all parent data about a section, isolate relevant data. + * @returns ``EnhancedNormalizedSection`` with all deduped, relevant metadata. + */ +function isolateSection(data: EnhancedSection): EnhancedNormalizedSection { + const section = { + ...data.section, + finalExam: parseFinalExamString(data.section), + meetings: getUniqueMeetings(data.section.meetings).map((meeting): NormalizedMeeting => { + const { bldg, days, time } = meeting; + const timeIsTBA = meeting.time === "TBA"; + return { + timeIsTBA, + bldg, + ...(timeIsTBA + ? { days: null, startTime: null, endTime: null } + : { days, ...parseNonTBAStartAndEndTimes(time) }), + }; + }), + }; + + const course = { + ...data.course, + sections: [section], + }; + + const department = { + ...data.department, + courses: [course], + }; + + const school = { + ...data.school, + departments: [department], + }; + + return { school, department, course, section }; +} + +/** + * Deeply sorts the provided response and returns the sorted response. + * + * Schools are sorted in lexicographical order of their name, departments are + * sorted in lexicographical order of their code, courses are sorted in + * numerical order of their number (with lexicographical tiebreaks), + * and sections are sorted in numerical order of their code. + * @param response The response to sort. + */ +export function sortResponse(response: T): T { + response.schools.forEach((schools) => { + schools.departments.forEach((department) => { + department.courses.forEach((course) => + course.sections.sort((a, b) => parseInt(a.sectionCode, 10) - parseInt(b.sectionCode, 10)), + ); + department.courses.sort((a, b) => { + const numOrd = + parseInt(a.courseNumber.replace(/\D/g, ""), 10) - + parseInt(b.courseNumber.replace(/\D/g, ""), 10); + return numOrd ? numOrd : lexOrd(a.courseNumber, b.courseNumber); + }); + }); + schools.departments.sort((a, b) => lexOrd(a.deptCode, b.deptCode)); + }); + + response.schools.sort((a, b) => lexOrd(a.schoolName, b.schoolName)); + + return response; +} diff --git a/packages/peterportal-api-next-types/package.json b/packages/peterportal-api-next-types/package.json index 433a7360..a39886ee 100644 --- a/packages/peterportal-api-next-types/package.json +++ b/packages/peterportal-api-next-types/package.json @@ -1,6 +1,6 @@ { "name": "peterportal-api-next-types", - "version": "1.0.0-rc.2", + "version": "1.0.0-rc.2.68.2", "license": "MIT", "type": "module", "main": "index.ts", diff --git a/packages/peterportal-api-next-types/types/constants.ts b/packages/peterportal-api-next-types/types/constants.ts index be43eb43..fe4e6c00 100644 --- a/packages/peterportal-api-next-types/types/constants.ts +++ b/packages/peterportal-api-next-types/types/constants.ts @@ -75,6 +75,8 @@ export const courseLevels = [ "Graduate/Professional Only (200+)", ] as const; +export const daysOfWeek = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"] as const; + /** * Represents the absence of a particular value to filter for. */ @@ -112,3 +114,5 @@ export type Division = Any | (typeof divisionCodes)[number]; * The course level name. */ export type CourseLevel = (typeof courseLevels)[number]; + +export type DayOfWeek = (typeof daysOfWeek)[number]; diff --git a/packages/peterportal-api-next-types/types/websoc.ts b/packages/peterportal-api-next-types/types/websoc.ts index 64df07a3..61962c3c 100644 --- a/packages/peterportal-api-next-types/types/websoc.ts +++ b/packages/peterportal-api-next-types/types/websoc.ts @@ -1,21 +1,46 @@ -import { Quarter } from "./constants"; +import { DayOfWeek, Quarter } from "./constants"; /** - * The meeting time for a section. + * A type that represents the hour and minute parts of a time. + */ +export type HourMinute = { + /** + * The hour (0-23). + */ + hour: number; + /** + * The minute (0-59). + */ + minute: number; +}; + +/** + * The meeting information for a section. */ export type WebsocSectionMeeting = { + /** + * Whether the meeting time is TBA. + * + * If this field is `false`, then `days`, `startTime`, and `endTime` + * are **guaranteed** to be non-null; otherwise, they are **guaranteed** to be null. + */ + timeIsTBA: boolean; + /** + * The classroom(s) the section meets in. + */ + bldg: string[]; /** * What day(s) the section meets on (e.g. ``MWF``). */ - days: string; + days: string | null; /** - * What time the section meets at. + * The time at which the section begins. */ - time: string; + startTime: HourMinute | null; /** - * The building(s) the section meets in. + * The time at which the section concludes. */ - bldg: string[]; + endTime: HourMinute | null; }; /** @@ -34,6 +59,43 @@ export type WebsocSectionEnrollment = { sectionEnrolled: string; }; +/** + * The final exam data for a section. + */ +export type WebsocSectionFinalExam = { + /** + * The status of the exam. + * + * If this field is `SCHEDULED_FINAL`, then all other fields are + * **guaranteed** to be non-null; otherwise, they are **guaranteed** to be null. + */ + examStatus: "NO_FINAL" | "TBA_FINAL" | "SCHEDULED_FINAL"; + /** + * The day of the week in which the final exam takes place. + */ + dayOfWeek: DayOfWeek | null; + /** + * The zero-indexed month (e.g. January = 0) in which the final exam takes place. + */ + month: number | null; + /** + * The day of the month in which the final exam takes place. + */ + day: number | null; + /** + * When the final exam starts. + */ + startTime: HourMinute | null; + /** + * When the final exam ends. + */ + endTime: HourMinute | null; + /** + * Where the final exam takes place. + */ + bldg: string[] | null; +}; + /** * A WebSoc section object. */ @@ -63,9 +125,9 @@ export type WebsocSection = { */ meetings: WebsocSectionMeeting[]; /** - * The date and time of the final exam for this section. + * The details for the final exam for this section. */ - finalExam: string; + finalExam: WebsocSectionFinalExam; /** * The maximum capacity of this section. */ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 385b64dc..921f9bab 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -205,6 +205,9 @@ importers: '@libs/db': specifier: workspace:* version: link:../../../../../libs/db + '@libs/websoc-utils': + specifier: workspace:* + version: link:../../../../../libs/websoc-utils ant-stack: specifier: workspace:* version: link:../../../../../packages/ant-stack @@ -215,6 +218,9 @@ importers: '@libs/build-tools': specifier: workspace:* version: link:../../../../../libs/build-tools + '@services/websoc-proxy': + specifier: workspace:* + version: link:../../../../../services/websoc-proxy peterportal-api-next-types: specifier: workspace:* version: link:../../../../../packages/peterportal-api-next-types @@ -357,7 +363,7 @@ importers: version: 0.34.3(vitest@0.34.3) peterportal-api-next-types: specifier: '*' - version: 1.0.0-rc.2.68.2 + version: link:../../packages/peterportal-api-next-types tsup: specifier: 7.2.0 version: 7.2.0(ts-node@10.9.1)(typescript@5.2.2) @@ -378,7 +384,7 @@ importers: version: 4.0.0 peterportal-api-next-types: specifier: '*' - version: 1.0.0-rc.2.68.2 + version: link:../../packages/peterportal-api-next-types devDependencies: '@vitest/coverage-istanbul': specifier: 0.34.3 @@ -390,6 +396,8 @@ importers: specifier: 0.34.3 version: 0.34.3 + libs/websoc-utils: {} + packages/ant-stack: dependencies: arktype: @@ -452,7 +460,7 @@ importers: version: 4.17.17 peterportal-api-next-types: specifier: '*' - version: 1.0.0-rc.2.68.2 + version: link:../peterportal-api-next-types tsup: specifier: 7.2.0 version: 7.2.0(ts-node@10.9.1)(typescript@5.2.2) @@ -572,9 +580,6 @@ importers: ant-stack: specifier: workspace:* version: link:../../packages/ant-stack - api-v1-rest-websoc: - specifier: workspace:* - version: link:../../apps/api/v1/rest/websoc devDependencies: '@types/aws-lambda': specifier: 8.10.119 @@ -2821,6 +2826,9 @@ packages: '@babel/helper-validator-identifier': 7.19.1 to-fast-properties: 2.0.0 + /@balena/dockerignore@1.0.2: + resolution: {integrity: sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==} + /@colors/colors@1.5.0: resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} engines: {node: '>=0.1.90'} @@ -6263,6 +6271,10 @@ packages: resolution: {integrity: sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==} dev: true + /astral-regex@2.0.0: + resolution: {integrity: sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==} + engines: {node: '>=8'} + /async-retry@1.3.3: resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} dependencies: @@ -6311,7 +6323,17 @@ packages: '@aws-cdk/asset-awscli-v1': 2.2.200 '@aws-cdk/asset-kubectl-v20': 2.1.2 '@aws-cdk/asset-node-proxy-agent-v6': 2.0.1 + '@balena/dockerignore': 1.0.2 + case: 1.6.3 constructs: 10.2.69 + fs-extra: 11.1.1 + ignore: 5.2.4 + jsonschema: 1.4.1 + minimatch: 3.1.2 + punycode: 2.3.0 + semver: 7.5.4 + table: 6.8.1 + yaml: 1.10.2 bundledDependencies: - '@balena/dockerignore' - case @@ -6688,6 +6710,10 @@ packages: /caniuse-lite@1.0.30001489: resolution: {integrity: sha512-x1mgZEXK8jHIfAxm+xgdpHpk50IN3z3q3zP261/WS+uvePxW8izXuCu6AHz0lkuYTlATDehiZ/tNyYBdSQsOUQ==} + /case@1.6.3: + resolution: {integrity: sha512-mzDSXIPaFwVDvZAHqZ9VlbyF4yyXRuX6IvB06WvPYkqJVO24kX1PPhv9bfpKNFZyxYFmmgo03HUiD8iklmJYRQ==} + engines: {node: '>= 0.8.0'} + /ccount@1.1.0: resolution: {integrity: sha512-vlNK021QdI7PNeiUh/lKkC/mNHHfV0m/Ad5JoI0TYtlBnJAslM/JIkm/tGC88bkLIwO6OQ5uV6ztS6kVAtCDlg==} dev: false @@ -8765,7 +8791,6 @@ packages: graceful-fs: 4.2.11 jsonfile: 6.1.0 universalify: 2.0.0 - dev: true /fs-extra@9.1.0: resolution: {integrity: sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==} @@ -10013,6 +10038,9 @@ packages: engines: {'0': node >= 0.2.0} dev: true + /jsonschema@1.4.1: + resolution: {integrity: sha512-S6cATIPVv1z0IlxdN+zUk5EPjkGCdnhN4wVSBlvoUO1tOLJootbo9CquNJmbIh4yikWHiUedhRYrNPn1arpEmQ==} + /keyv@3.1.0: resolution: {integrity: sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==} dependencies: @@ -10211,6 +10239,9 @@ packages: resolution: {integrity: sha512-+WKqsK294HMSc2jEbNgpHpd0JfIBhp7rEV4aqXWqFr6AlXov+SlcgB1Fv01y2kGe3Gc8nMW7VA0SrGuSkRfIEg==} dev: true + /lodash.truncate@4.4.2: + resolution: {integrity: sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==} + /lodash.uniq@4.5.0: resolution: {integrity: sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==} @@ -11034,9 +11065,6 @@ packages: resolution: {integrity: sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==} dev: true - /peterportal-api-next-types@1.0.0-rc.2.68.2: - resolution: {integrity: sha512-1nKzPLreyCAaFOjUCG9vii7CseInNbrGXDBhBd725EKZMGu2NEAvGiYKGnVPH35OMbtCk0LtYDF3LujoD1WM2A==} - /picocolors@1.0.0: resolution: {integrity: sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==} @@ -12534,6 +12562,14 @@ packages: resolution: {integrity: sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==} engines: {node: '>=12'} + /slice-ansi@4.0.0: + resolution: {integrity: sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==} + engines: {node: '>=10'} + dependencies: + ansi-styles: 4.3.0 + astral-regex: 2.0.0 + is-fullwidth-code-point: 3.0.0 + /slice-ansi@5.0.0: resolution: {integrity: sha512-FC+lgizVPfie0kkhqUScwRu1O/lF6NOgJmlCgK+/LYxDCTk8sGelYaHDhFcDN+Sn3Cv+3VSa4Byeo+IMCzpMgQ==} engines: {node: '>=12'} @@ -12923,6 +12959,16 @@ packages: tslib: 2.5.2 dev: true + /table@6.8.1: + resolution: {integrity: sha512-Y4X9zqrCftUhMeH2EptSSERdVKt/nEdijTOacGD/97EKjhQ/Qs8RTlEGABSJNNN8lac9kheH+af7yAkEWlgneA==} + engines: {node: '>=10.0.0'} + dependencies: + ajv: 8.12.0 + lodash.truncate: 4.4.2 + slice-ansi: 4.0.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + /tapable@1.1.3: resolution: {integrity: sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==} engines: {node: '>=6'} @@ -14213,7 +14259,6 @@ packages: /yaml@1.10.2: resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} engines: {node: '>= 6'} - dev: false /yaml@2.3.1: resolution: {integrity: sha512-2eHWfjaoXgTBC2jNM1LRef62VQa0umtvRiDSk6HSzW7RvS5YtkabJrwYLLEKWBc8a5U2PTSCs+dJjUTJdlHsWQ==} diff --git a/services/websoc-proxy/package.json b/services/websoc-proxy/package.json index 1f32153a..62374641 100644 --- a/services/websoc-proxy/package.json +++ b/services/websoc-proxy/package.json @@ -4,14 +4,14 @@ "private": true, "license": "MIT", "type": "module", - "main": "index.ts", + "main": "src/index.ts", + "types": "src/index.ts", "scripts": { "build": "node build.mjs" }, "dependencies": { "@libs/websoc-api-next": "workspace:*", - "ant-stack": "workspace:*", - "api-v1-rest-websoc": "workspace:*" + "ant-stack": "workspace:*" }, "devDependencies": { "@types/aws-lambda": "8.10.119", diff --git a/services/websoc-proxy/src/index.ts b/services/websoc-proxy/src/index.ts index a654e611..73630368 100644 --- a/services/websoc-proxy/src/index.ts +++ b/services/websoc-proxy/src/index.ts @@ -1,8 +1,8 @@ -import { callWebSocAPI, getDepts, getTerms, WebsocAPIOptions } from "@libs/websoc-api-next"; +import { callWebSocAPI, getDepts, getTerms } from "@libs/websoc-api-next"; +import type { WebsocAPIResponse, WebsocAPIOptions } from "@libs/websoc-api-next"; +import { combineAndNormalizeResponses, fulfilled, sleep, sortResponse } from "@libs/websoc-utils"; import { createErrorResult, createOKResult, logger } from "ant-stack"; -import { combineResponses, fulfilled, sleep, sortResponse } from "api-v1-rest-websoc/src/lib"; import { APIGatewayProxyEvent, APIGatewayProxyResult, Context } from "aws-lambda"; -import { WebsocAPIResponse } from "peterportal-api-next-types"; export const handler = async ( event: APIGatewayProxyEvent, @@ -26,8 +26,6 @@ export const handler = async ( let successes: PromiseFulfilledResult[] = []; const failed: WebsocAPIOptions[] = []; - let websocResponseData: WebsocAPIResponse = { schools: [] }; - while (queries.length && retries < 3) { responses = await Promise.allSettled( queries.map((options) => callWebSocAPI(parsedQuery, options)), @@ -44,11 +42,6 @@ export const handler = async ( }); successes = responses.filter(fulfilled); - websocResponseData = successes.reduce( - (acc, curr) => combineResponses(acc, curr.value), - websocResponseData, - ); - queries = failed; if (queries.length) await sleep(1000 * 2 ** retries++); } @@ -62,7 +55,11 @@ export const handler = async ( ); // Do not compress responses. - return createOKResult(sortResponse(websocResponseData), { "accept-encoding": "" }, requestId); + return createOKResult( + sortResponse(combineAndNormalizeResponses(...successes.map((x) => x.value))), + { "accept-encoding": "" }, + requestId, + ); } default: return createOKResult({}, {}, requestId); diff --git a/services/websoc-scraper-v2/index.ts b/services/websoc-scraper-v2/index.ts index baedd545..fcbae322 100644 --- a/services/websoc-scraper-v2/index.ts +++ b/services/websoc-scraper-v2/index.ts @@ -1,6 +1,5 @@ import { PrismaClient } from "@libs/db"; import { getTermDateData } from "@libs/registrar-api"; -import { callWebSocAPI, getDepts, getTerms } from "@libs/websoc-api-next"; import type { GE, Quarter, @@ -11,8 +10,8 @@ import type { WebsocSchool, WebsocSection, WebsocSectionMeeting, -} from "peterportal-api-next-types"; -import { geCodes, sectionTypes } from "peterportal-api-next-types"; +} from "@libs/websoc-api-next"; +import { callWebSocAPI, getDepts, getTerms, geCodes, sectionTypes } from "@libs/websoc-api-next"; import { createLogger, format, transports } from "winston"; /** @@ -245,6 +244,7 @@ function parseStartAndEndTimes(time: string) { startTime += 12 * 60; endTime += 12 * 60; } + if (startTime > endTime) startTime -= 12 * 60; } return { startTime, endTime }; } diff --git a/tools/grades-updater/src/sanitize-data.ts b/tools/grades-updater/src/sanitize-data.ts index 7665f3bf..f2f58cf6 100644 --- a/tools/grades-updater/src/sanitize-data.ts +++ b/tools/grades-updater/src/sanitize-data.ts @@ -1,12 +1,13 @@ import fs from "fs"; import { EOL } from "os"; -import { basename, resolve } from "path"; +import { basename, resolve } from "node:path"; import { callWebSocAPI } from "@libs/websoc-api-next"; +import type { WebsocAPIResponse, WebsocSection } from "@libs/websoc-api-next"; import type { CastingContext, Parser } from "csv-parse"; import { parse } from "csv-parse"; import { stringify } from "csv-stringify/sync"; -import type { Quarter, WebsocAPIResponse, WebsocSection } from "peterportal-api-next-types"; +import type { Quarter } from "peterportal-api-next-types"; import { __dirname, dataColumns, type Grade, handleError, logger } from "./lib";