diff --git a/apps/api/v1/graphql/src/graphql/enum.graphql b/apps/api/v1/graphql/src/graphql/enum.graphql index 2bbeecfa..38820ffc 100644 --- a/apps/api/v1/graphql/src/graphql/enum.graphql +++ b/apps/api/v1/graphql/src/graphql/enum.graphql @@ -9,6 +9,7 @@ enum Quarter { } "The set of valid GE category codes." enum GE { + ANY GE_1A GE_1B GE_2 diff --git a/apps/api/v1/graphql/src/graphql/grades.graphql b/apps/api/v1/graphql/src/graphql/grades.graphql index 4f6089a6..c93c9697 100644 --- a/apps/api/v1/graphql/src/graphql/grades.graphql +++ b/apps/api/v1/graphql/src/graphql/grades.graphql @@ -12,6 +12,8 @@ type GradesSection { courseNumber: String! "The numeric part of the course number." courseNumeric: Int! + "What GE categor(y/ies) this section satisfies (if any)." + geCategories: [GE!]! "The shortened name(s) of the instructor(s) who taught the section." instructors: [String!]! } @@ -50,6 +52,8 @@ type RawGrade { courseNumber: String! "The numeric part of the course number." courseNumeric: Int! + "What GE categor(y/ies) this section satisfies (if any)." + geCategories: [GE!]! "How many students attained an A+/A/A-." gradeACount: Int! "How many students attained a B+/B/B-." @@ -91,6 +95,58 @@ type GradesOptions { "The list of instructors that matched the given filters." instructors: [String!]! } +"An object that represents aggregate grades statistics for a course taught by an instructor." +type AggregateGradeByCourse { + "The department code." + department: String! + "The course number the section belongs to." + courseNumber: String! + "How many students attained an A+/A/A-." + gradeACount: Int! + "How many students attained a B+/B/B-." + gradeBCount: Int! + "How many students attained a C+/C/C-." + gradeCCount: Int! + "How many students attained a D+/D/D-." + gradeDCount: Int! + "How many students attained an F." + gradeFCount: Int! + "How many students attained a P." + gradePCount: Int! + "How many students attained an NP." + gradeNPCount: Int! + "How many students attained a W." + gradeWCount: Int! + "The average GPA of all assigned grades in the course." + averageGPA: Float! +} +"An object that represents aggregate grades statistics for a course taught by an instructor." +type AggregateGradeByOffering { + "The department code." + department: String! + "The course number the section belongs to." + courseNumber: String! + "The shortened name of the instructor who taught the section." + instructor: String! + "How many students attained an A+/A/A-." + gradeACount: Int! + "How many students attained a B+/B/B-." + gradeBCount: Int! + "How many students attained a C+/C/C-." + gradeCCount: Int! + "How many students attained a D+/D/D-." + gradeDCount: Int! + "How many students attained an F." + gradeFCount: Int! + "How many students attained a P." + gradePCount: Int! + "How many students attained an NP." + gradeNPCount: Int! + "How many students attained a W." + gradeWCount: Int! + "The average GPA of all assigned grades in the course." + averageGPA: Float! +} extend type Query { "Get the raw grade info for the given parameters." @@ -113,6 +169,7 @@ extend type Query { courseNumber: String sectionCode: String division: Division + ge: GE excludePNP: Boolean ): AggregateGrades! "Get the available options for the given constraints." @@ -124,6 +181,34 @@ extend type Query { courseNumber: String sectionCode: String division: Division + ge: GE excludePNP: Boolean ): GradesOptions! + "Get the aggregate grade info, grouped by course, for the given parameters." + aggregateByCourse( + year: String + quarter: Quarter + instructor: String + department: String + courseNumber: String + sectionCode: String + division: Division + ge: GE + excludePNP: Boolean + ): [AggregateGradeByCourse!]! + """ + Get the aggregate grade info, grouped by offering (course and instructor), + for the given parameters. + """ + aggregateByOffering( + year: String + quarter: Quarter + instructor: String + department: String + courseNumber: String + sectionCode: String + division: Division + ge: GE + excludePNP: Boolean + ): [AggregateGradeByOffering!]! } diff --git a/apps/api/v1/graphql/src/resolvers.ts b/apps/api/v1/graphql/src/resolvers.ts index 96fcd1cf..61a6bdf0 100644 --- a/apps/api/v1/graphql/src/resolvers.ts +++ b/apps/api/v1/graphql/src/resolvers.ts @@ -15,6 +15,12 @@ export const resolvers: ApolloServerOptions["resolvers"] = { rawGrades: proxyRestApi("v1/rest/grades/raw"), aggregateGrades: proxyRestApi("v1/rest/grades/aggregate"), gradesOptions: proxyRestApi("v1/rest/grades/options"), + aggregateByCourse: proxyRestApi("v1/rest/grades/aggregateByCourse", { + argsTransform: geTransform, + }), + aggregateByOffering: proxyRestApi("v1/rest/grades/aggregateByOffering", { + argsTransform: geTransform, + }), instructor: proxyRestApi("v1/rest/instructors", { pathArg: "ucinetid" }), instructors: proxyRestApi("v1/rest/instructors"), allInstructors: proxyRestApi("v1/rest/instructors/all"), diff --git a/apps/api/v1/rest/grades/src/index.ts b/apps/api/v1/rest/grades/src/index.ts index 6b7ce8af..02c26694 100644 --- a/apps/api/v1/rest/grades/src/index.ts +++ b/apps/api/v1/rest/grades/src/index.ts @@ -1,10 +1,22 @@ import { PrismaClient } from "@libs/db"; import { createErrorResult, createOKResult, logger } from "ant-stack"; import type { InternalHandler } from "ant-stack"; -import type { GradesOptions, GradesRaw } from "peterportal-api-next-types"; +import type { + AggregateGradesByCourse, + AggregateGradesByOffering, + GradesOptions, + RawGrades, +} from "peterportal-api-next-types"; import { ZodError } from "zod"; -import { aggregateGrades, constructPrismaQuery, lexOrd } from "./lib"; +import { + aggregateGrades, + aggregateByOffering, + constructPrismaQuery, + lexOrd, + transformRow, + aggregateByCourse, +} from "./lib"; import { QuerySchema } from "./schema"; let prisma: PrismaClient; @@ -33,13 +45,10 @@ export const GET: InternalHandler = async (request) => { where: constructPrismaQuery(parsedQuery), include: { instructors: true }, }) - ).map((section) => ({ - ...section, - instructors: section.instructors.map((instructor) => instructor.name), - })); + ).map(transformRow); switch (params.id) { case "raw": - return createOKResult(res, headers, requestId); + return createOKResult(res, headers, requestId); case "aggregate": return createOKResult(aggregateGrades(res), headers, requestId); } @@ -97,6 +106,34 @@ export const GET: InternalHandler = async (request) => { requestId, ); } + case "aggregateByCourse": { + return createOKResult( + aggregateByCourse( + ( + await prisma.gradesSection.findMany({ + where: constructPrismaQuery(parsedQuery), + include: { instructors: true }, + }) + ).map(transformRow), + ), + headers, + requestId, + ); + } + case "aggregateByOffering": { + return createOKResult( + aggregateByOffering( + ( + await prisma.gradesSection.findMany({ + where: constructPrismaQuery(parsedQuery), + include: { instructors: true }, + }) + ).map(transformRow), + ), + headers, + requestId, + ); + } } return createErrorResult( 400, diff --git a/apps/api/v1/rest/grades/src/lib.ts b/apps/api/v1/rest/grades/src/lib.ts index 578f267e..05e02da0 100644 --- a/apps/api/v1/rest/grades/src/lib.ts +++ b/apps/api/v1/rest/grades/src/lib.ts @@ -1,8 +1,25 @@ -import { Prisma } from "@libs/db"; -import type { GradeDistribution, GradesAggregate, GradesRaw } from "peterportal-api-next-types"; +import { GradesSection, Prisma } from "@libs/db"; +import type { + AggregateGrades, + AggregateGradesByOffering, + AggregateGradeByOfferingHeader, + GE, + GradeDistribution, + Quarter, + RawGrade, + RawGrades, + AggregateGradesByCourse, + AggregateGradeByCourseHeader, +} from "peterportal-api-next-types"; +import { geCodes } from "peterportal-api-next-types"; import { Query } from "./schema"; +/** + * type guard that asserts input is defined + */ +export const notNull = (x: T): x is NonNullable => x != null; + /** * Returns the lexicographical ordering of two elements. * @param a The left hand side of the comparison. @@ -10,6 +27,21 @@ import { Query } from "./schema"; */ export const lexOrd = (a: string, b: string): number => (a === b ? 0 : a > b ? 1 : -1); +const headerKeys = ["department", "courseNumber", "instructor"]; + +const geKeys = [ + "isGE1A", + "isGE1B", + "isGE2", + "isGE3", + "isGE4", + "isGE5A", + "isGE5B", + "isGE6", + "isGE7", + "isGE8", +] as const; + const isNotPNPOnly = ({ gradeACount, gradeBCount, @@ -18,9 +50,71 @@ const isNotPNPOnly = ({ gradeFCount, }: GradeDistribution) => gradeACount || gradeBCount || gradeCCount || gradeDCount || gradeFCount; +const geToKey = (ge: Exclude) => geKeys[geCodes.indexOf(ge)]; + +export const transformRow = ({ + year, + quarter, + sectionCode, + department, + courseNumber, + courseNumeric, + isGE1A, + isGE1B, + isGE2, + isGE3, + isGE4, + isGE5A, + isGE5B, + isGE6, + isGE7, + isGE8, + gradeACount, + gradeBCount, + gradeCCount, + gradeDCount, + gradeFCount, + gradePCount, + gradeNPCount, + gradeWCount, + averageGPA, + instructors, +}: GradesSection & { + instructors: { year: string; quarter: Quarter; sectionCode: string; name: string }[]; +}): RawGrade => ({ + year, + quarter, + sectionCode, + department, + courseNumber, + courseNumeric, + geCategories: [isGE1A, isGE1B, isGE2, isGE3, isGE4, isGE5A, isGE5B, isGE6, isGE7, isGE8] + .map((x, i) => (x ? geCodes[i] : null)) + .filter(notNull), + instructors: instructors.map((x) => x.name), + gradeACount, + gradeBCount, + gradeCCount, + gradeDCount, + gradeFCount, + gradePCount, + gradeNPCount, + gradeWCount, + averageGPA, +}); + export function constructPrismaQuery(parsedQuery: Query): Prisma.GradesSectionWhereInput { - const { year, quarter, instructor, department, courseNumber, sectionCode, division, excludePNP } = - parsedQuery; + const { + year, + quarter, + instructor, + department, + courseNumber, + sectionCode, + division, + excludePNP, + ge, + } = parsedQuery; const courseNumeric: Prisma.IntFilter = {}; switch (division) { case "LowerDiv": @@ -43,6 +137,10 @@ export function constructPrismaQuery(parsedQuery: Query): Prisma.GradesSectionWh excludePNPFilters.gradeDCount = 0; excludePNPFilters.gradeFCount = 0; } + const geFilter: Record = {}; + if (ge && ge !== "ANY") { + geFilter[geToKey(ge)] = true; + } return { year, quarter, @@ -51,6 +149,7 @@ export function constructPrismaQuery(parsedQuery: Query): Prisma.GradesSectionWh courseNumber, courseNumeric, sectionCode, + ...geFilter, NOT: excludePNP ? { ...excludePNPFilters } : undefined, }; } @@ -63,10 +162,10 @@ export function constructPrismaQuery(parsedQuery: Query): Prisma.GradesSectionWh * mean of each section's average GPA in the given dataset. * @param grades The array of grades to aggregate. */ -export function aggregateGrades(grades: GradesRaw): GradesAggregate { +export function aggregateGrades(grades: RawGrades): AggregateGrades { return { sectionList: grades.map( - ({ year, quarter, sectionCode, department, courseNumber, courseNumeric, instructors }) => ({ + ({ year, quarter, sectionCode, @@ -74,6 +173,16 @@ export function aggregateGrades(grades: GradesRaw): GradesAggregate { courseNumber, courseNumeric, instructors, + geCategories, + }) => ({ + year, + quarter, + sectionCode, + department, + courseNumber, + courseNumeric, + geCategories, + instructors, }), ), gradeDistribution: { @@ -96,3 +205,61 @@ export function aggregateGrades(grades: GradesRaw): GradesAggregate { }, }; } + +/** + * Given an array of sections and their grades distributions, aggregate the sections with the same + * department and course number. + * @param grades The array of grades to aggregate. + */ +export function aggregateByCourse(grades: RawGrades): AggregateGradesByCourse { + const courses = new Map(); + for (const grade of grades) { + const { department, courseNumber } = grade; + const key = JSON.stringify([department, courseNumber]); + if (courses.has(key)) { + courses.get(key)?.push(grade); + } else { + courses.set(key, [grade]); + } + } + return Array.from(courses) + .map(([k, v]) => ({ + ...(Object.fromEntries( + (JSON.parse(k) as string[]).map((x, i) => [headerKeys[i], x]), + ) as AggregateGradeByCourseHeader), + ...aggregateGrades(v).gradeDistribution, + })) + .sort((a, b) => lexOrd(a.department, b.department) || lexOrd(a.courseNumber, b.courseNumber)); +} + +/** + * Same as the above but also factors into the instructor of the section. + * @param grades The array of grades to aggregate. + */ +export function aggregateByOffering(grades: RawGrades): AggregateGradesByOffering { + const courses = new Map(); + for (const grade of grades) { + for (const instructor of grade.instructors) { + const { department, courseNumber } = grade; + const key = JSON.stringify([department, courseNumber, instructor]); + if (courses.has(key)) { + courses.get(key)?.push(grade); + } else { + courses.set(key, [grade]); + } + } + } + return Array.from(courses) + .map(([k, v]) => ({ + ...(Object.fromEntries( + (JSON.parse(k) as string[]).map((x, i) => [headerKeys[i], x]), + ) as AggregateGradeByOfferingHeader), + ...aggregateGrades(v).gradeDistribution, + })) + .sort( + (a, b) => + lexOrd(a.department, b.department) || + lexOrd(a.courseNumber, b.courseNumber) || + lexOrd(a.instructor, b.instructor), + ); +} diff --git a/apps/api/v1/rest/grades/src/schema.ts b/apps/api/v1/rest/grades/src/schema.ts index 7f656657..05b3c581 100644 --- a/apps/api/v1/rest/grades/src/schema.ts +++ b/apps/api/v1/rest/grades/src/schema.ts @@ -1,4 +1,4 @@ -import { anyArray, divisionCodes, quarters } from "peterportal-api-next-types"; +import { anyArray, divisionCodes, geCodes, quarters } from "peterportal-api-next-types"; import { z } from "zod"; export const QuerySchema = z.object({ @@ -15,6 +15,7 @@ export const QuerySchema = z.object({ .regex(/^\d{5}$/, { message: "Invalid sectionCode provided" }) .optional(), division: z.enum(anyArray).or(z.enum(divisionCodes)).optional(), + ge: z.enum(anyArray).or(z.enum(geCodes)).optional(), excludePNP: z .string() .optional() diff --git a/apps/docs/docs/developers-guide/rest-api/reference/grades.md b/apps/docs/docs/developers-guide/rest-api/reference/grades.md index 58677ed8..2b9e140c 100644 --- a/apps/docs/docs/developers-guide/rest-api/reference/grades.md +++ b/apps/docs/docs/developers-guide/rest-api/reference/grades.md @@ -42,6 +42,10 @@ The five-digit section code to include. The course level/division code to include. Case-sensitive. +#### `ge` GE-1A | GE-1B | GE-2 | GE-3 | GE-4 | GE-5A | GE-5B | GE-6 | GE-7 | GE-8 + +Which GE category to include. Case-sensitive. + #### `excludePNP` boolean Whether to exclude sections that only reported Pass/No-Pass grades. @@ -74,6 +78,7 @@ curl "https://api-next.peterportal.org/v1/rest/grades/raw?year=2022&quarter=Fall "department": "I&C SCI", "courseNumber": "46", "courseNumeric": 46, + "geCategories": ["GE-5B"], "gradeACount": 34, "gradeBCount": 19, "gradeCCount": 40, @@ -93,12 +98,14 @@ curl "https://api-next.peterportal.org/v1/rest/grades/raw?year=2022&quarter=Fall ```typescript // https://github.com/icssc/peterportal-api-next/blob/main/packages/peterportal-api-next-types/types/grades.ts -type GradesRaw = { +type RawGrades = { year: string; quarter: string; + sectionCode: string; department: string; courseNumber: string; - sectionCode: string; + courseNumeric: number; + geCategories: GE[]; instructors: string[]; gradeACount: number; gradeBCount: number; @@ -144,6 +151,7 @@ curl "https://api-next.peterportal.org/v1/rest/grades/aggregate?year=2022&quarte "department": "I&C SCI", "courseNumber": "46", "courseNumeric": 46, + "geCategories": ["GE-5B"], "instructors": ["GARZA RODRIGUE, A.", "GILA, O.", "SHINDLER, M."] }, { @@ -153,6 +161,7 @@ curl "https://api-next.peterportal.org/v1/rest/grades/aggregate?year=2022&quarte "department": "I&C SCI", "courseNumber": "46", "courseNumeric": 46, + "geCategories": ["GE-5B"], "instructors": ["DICKERSON, M.", "SHINDLER, M."] } ], @@ -175,13 +184,14 @@ curl "https://api-next.peterportal.org/v1/rest/grades/aggregate?year=2022&quarte ```typescript // https://github.com/icssc/peterportal-api-next/blob/main/packages/peterportal-api-next-types/types/grades.ts -type GradesAggregate = { +type AggregateGrades = { sectionList: { year: string; quarter: string; department: string; courseNumber: string; sectionCode: string; + geCategories: GE[]; instructors: string[]; }[]; gradeDistribution: { @@ -269,3 +279,189 @@ type GradesOptions = { + +## Get grade statistics aggregated by course for certain sections + +Formally, if two sections have the same department code and course number, then they will be aggregated together for the purposes of this endpoint. For queries that involve an entire department, this is equivalent to running an aggregate query for each course number, but much faster. + +Note that graduate students who are listed as instructors on WebSoc may also be included. + +### Code sample + + + + +```bash +curl "https://api-next.peterportal.org/v1/rest/grades/aggregateByCourse?year=2023&department=COMPSCI" +``` + + + + +### Response + + + + +```json +[ + { + "department": "COMPSCI", + "courseNumber": "103", + "gradeACount": 80, + "gradeBCount": 11, + "gradeCCount": 10, + "gradeDCount": 4, + "gradeFCount": 5, + "gradePCount": 11, + "gradeNPCount": 8, + "gradeWCount": 0, + "averageGPA": 3.415 + }, + { + "department": "COMPSCI", + "courseNumber": "111", + "gradeACount": 112, + "gradeBCount": 100, + "gradeCCount": 35, + "gradeDCount": 13, + "gradeFCount": 13, + "gradePCount": 13, + "gradeNPCount": 2, + "gradeWCount": 2, + "averageGPA": 3.11625 + }, + "..." +] +``` + + + + +```typescript +// https://github.com/icssc/peterportal-api-next/blob/main/packages/peterportal-api-next-types/types/grades.ts +type AggregateGradesByCourse = { + department: string; + courseNumber: string; + gradeACount: number; + gradeBCount: number; + gradeCCount: number; + gradeDCount: number; + gradeFCount: number; + gradePCount: number; + gradeNPCount: number; + gradeWCount: number; + averageGPA: number; +}[]; +``` + + + + +## Get grade statistics aggregated by course/instructor for certain sections + +Formally, if two sections have the same department code, course number, and instructor name, then they will be aggregated together for the purposes of this endpoint. For queries that involve an entire department, this is equivalent to running an aggregate query for each course number-instructor pair, but much faster. + +Note that graduate students who are listed as instructors on WebSoc may also be included. + +### Code sample + + + + +```bash +curl "https://api-next.peterportal.org/v1/rest/grades/aggregateByOffering?year=2023&department=COMPSCI&courseNumber=161" +``` + + + + +### Response + + + + +```json +[ + { + "department": "COMPSCI", + "courseNumber": "161", + "instructor": "FRISHBERG, D.", + "gradeACount": 165, + "gradeBCount": 42, + "gradeCCount": 59, + "gradeDCount": 0, + "gradeFCount": 14, + "gradePCount": 0, + "gradeNPCount": 0, + "gradeWCount": 2, + "averageGPA": 3.23 + }, + { + "department": "COMPSCI", + "courseNumber": "161", + "instructor": "KALOGIANNIS, F.", + "gradeACount": 165, + "gradeBCount": 42, + "gradeCCount": 59, + "gradeDCount": 0, + "gradeFCount": 14, + "gradePCount": 0, + "gradeNPCount": 0, + "gradeWCount": 2, + "averageGPA": 3.23 + }, + { + "department": "COMPSCI", + "courseNumber": "161", + "instructor": "PANAGEAS, I.", + "gradeACount": 101, + "gradeBCount": 115, + "gradeCCount": 48, + "gradeDCount": 15, + "gradeFCount": 12, + "gradePCount": 0, + "gradeNPCount": 0, + "gradeWCount": 2, + "averageGPA": 2.935 + }, + { + "department": "COMPSCI", + "courseNumber": "161", + "instructor": "SHINDLER, M.", + "gradeACount": 165, + "gradeBCount": 42, + "gradeCCount": 59, + "gradeDCount": 0, + "gradeFCount": 14, + "gradePCount": 0, + "gradeNPCount": 0, + "gradeWCount": 2, + "averageGPA": 3.23 + } +] +``` + + + + +```typescript +// https://github.com/icssc/peterportal-api-next/blob/main/packages/peterportal-api-next-types/types/grades.ts +type AggregateGradesByOffering = { + department: string; + courseNumber: string; + instructor: string; + gradeACount: number; + gradeBCount: number; + gradeCCount: number; + gradeDCount: number; + gradeFCount: number; + gradePCount: number; + gradeNPCount: number; + gradeWCount: number; + averageGPA: number; +}[]; +``` + + + diff --git a/libs/db/prisma/schema.prisma b/libs/db/prisma/schema.prisma index 718f4a25..cf917d71 100644 --- a/libs/db/prisma/schema.prisma +++ b/libs/db/prisma/schema.prisma @@ -133,6 +133,17 @@ model GradesSection { department String courseNumber String courseNumeric Int + hasGEData Boolean @default(false) + isGE1A Boolean? + isGE1B Boolean? + isGE2 Boolean? + isGE3 Boolean? + isGE4 Boolean? + isGE5A Boolean? + isGE5B Boolean? + isGE6 Boolean? + isGE7 Boolean? + isGE8 Boolean? instructors GradesInstructor[] gradeACount Int gradeBCount Int diff --git a/packages/peterportal-api-next-types/types/grades.ts b/packages/peterportal-api-next-types/types/grades.ts index 47f5da4d..e1e5a6ef 100644 --- a/packages/peterportal-api-next-types/types/grades.ts +++ b/packages/peterportal-api-next-types/types/grades.ts @@ -1,9 +1,9 @@ -import { Quarter } from "./constants"; +import type { GE, Quarter } from "./constants"; /** * A section which has grades data associated with it. */ -export type GradeSection = { +export type GradesSection = { /** * The year the section was offered. */ @@ -28,6 +28,10 @@ export type GradeSection = { * The numeric part of the course number. */ courseNumeric: number; + /** + * What GE categor(y/ies) the section fulfills (if any). + */ + geCategories: GE[]; /** * The shortened name(s) of the instructor(s) who taught the section. */ @@ -76,22 +80,24 @@ export type GradeDistribution = { averageGPA: number; }; +export type RawGrade = GradesSection & GradeDistribution; + /** * The type of the payload returned on a successful response from querying * ``/v1/rest/grades/raw``. */ -export type GradesRaw = (GradeSection & GradeDistribution)[]; +export type RawGrades = RawGrade[]; /** * An object that represents aggregate grades statistics for a given query. * The type of the payload returned on a successful response from querying * ``/v1/rest/grades/aggregate``. */ -export type GradesAggregate = { +export type AggregateGrades = { /** * The list of sections in the query. */ - sectionList: GradeSection[]; + sectionList: GradesSection[]; /** * The combined grades distribution of all sections in the query. */ @@ -124,3 +130,22 @@ export type GradesOptions = { */ instructors: string[]; }; + +export type AggregateGradeByCourseHeader = { + department: string; + courseNumber: string; +}; + +export type AggregateGradeByCourse = AggregateGradeByCourseHeader & GradeDistribution; + +export type AggregateGradesByCourse = AggregateGradeByCourse[]; + +export type AggregateGradeByOfferingHeader = { + department: string; + courseNumber: string; + instructor: string; +}; + +export type AggregateGradeByOffering = AggregateGradeByOfferingHeader & GradeDistribution; + +export type AggregateGradesByOffering = AggregateGradeByOffering[]; diff --git a/tools/grades-updater/README.md b/tools/grades-updater/README.md index 4d1bed93..96cb699e 100644 --- a/tools/grades-updater/README.md +++ b/tools/grades-updater/README.md @@ -4,7 +4,7 @@ This directory contains the code for updating the grades cache for PeterPortal A ## Sanitizing the Data -1. Make sure all dependencies are up-to-date by running `npm i` in the project root. +1. Make sure all dependencies are up-to-date by running `pnpm i` in the project root. 2. Create the `inputData` and `outputData` directories if they do not already exist. 3. Using a spreadsheet editor, edit the grades spreadsheet you obtained from the [UCI Public Records Office](https://pro.uci.edu/) so that it matches the following format, and then save it as a CSV in `inputData`. @@ -23,3 +23,7 @@ This directory contains the code for updating the grades cache for PeterPortal A 2. Add the `.env.grades` file to the project root. Note that this is only available to members of the ICSSC Projects Committee, since it grants write access to the production database. 3. Run `pnpm upload` in this directory. 4. The logs should be present under `/logs`. + +## Populating GE data + +During the sanitization process, the data that encodes what GE categor(y/ies) is not fetched. This can be remedied by running `pnpm populate` after uploading the data. diff --git a/tools/grades-updater/package.json b/tools/grades-updater/package.json index 48c6bc96..bcff0e13 100644 --- a/tools/grades-updater/package.json +++ b/tools/grades-updater/package.json @@ -4,8 +4,9 @@ "private": true, "type": "module", "scripts": { - "sanitize": "tsx sanitize-data.ts", - "upload": "dotenv -e ../../.env.grades -- tsx upload-data.ts" + "populate": "dotenv -e ../../.env.grades -- tsx src/populate-ge.ts", + "sanitize": "tsx src/sanitize-data.ts", + "upload": "dotenv -e ../../.env.grades -- tsx src/upload-data.ts" }, "dependencies": { "@libs/db": "workspace:*", diff --git a/tools/grades-updater/src/populate-ge.ts b/tools/grades-updater/src/populate-ge.ts new file mode 100644 index 00000000..e6b2ff7d --- /dev/null +++ b/tools/grades-updater/src/populate-ge.ts @@ -0,0 +1,127 @@ +import { PrismaClient } from "@libs/db"; +import { callWebSocAPI, GE, geCodes, Quarter } from "@libs/websoc-api-next"; + +const prisma = new PrismaClient({ + log: [ + { emit: "event", level: "query" }, + { emit: "stdout", level: "error" }, + { emit: "stdout", level: "info" }, + { emit: "stdout", level: "warn" }, + ], +}); + +prisma.$on("query", (e) => { + console.log("Query: " + e.query); + console.log("Params: " + e.params); + console.log("Duration: " + e.duration + "ms"); +}); + +const geKeys = [ + "isGE1A", + "isGE1B", + "isGE2", + "isGE3", + "isGE4", + "isGE5A", + "isGE5B", + "isGE6", + "isGE7", + "isGE8", +] as const; + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +const log = (msg: string) => console.log(`[${new Date().toUTCString()}] ${msg}`); + +function categorySetToMask(geCategories: Set) { + let ret = 0; + for (const [idx, ge] of Object.entries(geCodes)) { + if (geCategories.has(ge)) { + ret |= 2 ** Number.parseInt(idx, 10); + } + } + return ret; +} + +function maskToCategoryMap(mask: number) { + const ret: Record<(typeof geKeys)[number], boolean> = { + isGE1A: false, + isGE1B: false, + isGE2: false, + isGE3: false, + isGE4: false, + isGE5A: false, + isGE5B: false, + isGE6: false, + isGE7: false, + isGE8: false, + }; + for (const i of Object.keys(geCodes)) { + const idx = Number.parseInt(i, 10); + ret[geKeys[idx]] = !!(mask & (2 ** idx)); + } + return ret; +} + +async function main() { + const sections = await prisma.gradesSection.findMany({ + where: { hasGEData: false }, + select: { year: true, quarter: true, sectionCode: true }, + }); + log(`Found ${sections.length} sections without GE data`); + const terms = new Set(sections.map(({ year, quarter }) => `${year}-${quarter}`)); + const geData = new Map>(); + for (const term of terms) { + const [year, quarter] = term.split("-") as [string, Quarter]; + for (const ge of geCodes) { + log(`Getting set of section codes for (year=${year}, quarter=${quarter}, ge=${ge})`); + const res = await callWebSocAPI({ year, quarter }, { ge }); + await sleep(1000); + geData.set( + `${year}-${quarter}-${ge}`, + new Set( + res.schools + .flatMap((x) => x.departments) + .flatMap((x) => x.courses) + .flatMap((x) => x.sections) + .map((x) => x.sectionCode), + ), + ); + } + } + const updates = new Map>(); + for (const { year, quarter, sectionCode } of sections) { + const key = `${year}-${quarter}`; + const geCategories = new Set(); + for (const ge of geCodes) { + if (geData.get(`${year}-${quarter}-${ge}`)?.has(sectionCode)) { + geCategories.add(ge); + } + } + const mask = categorySetToMask(geCategories); + if (updates.has(mask)) { + if (updates.get(mask)!.has(key)) { + updates.get(mask)!.get(key)!.push(sectionCode); + } else { + updates.get(mask)!.set(key, [sectionCode]); + } + } else { + updates.set(mask, new Map([[key, [sectionCode]]])); + } + } + const txn = []; + for (const [mask, mapping] of updates) { + for (const [term, sectionCodes] of mapping) { + const [year, quarter] = term.split("-") as [string, Quarter]; + txn.push( + prisma.gradesSection.updateMany({ + where: { year, quarter, sectionCode: { in: sectionCodes } }, + data: { hasGEData: true, ...maskToCategoryMap(mask) }, + }), + ); + } + } + await prisma.$transaction(txn); +} + +main().then();