From 2743fdf08246e674cb3f36f809c6784b58568897 Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Sun, 27 Aug 2023 13:56:32 -0700 Subject: [PATCH 01/19] feat: :sparkles: add aggregateGrouped REST operation --- apps/api/v1/rest/grades/src/index.ts | 23 +++++++-- apps/api/v1/rest/grades/src/lib.ts | 47 ++++++++++++++++++- .../types/grades.ts | 20 ++++++-- 3 files changed, 81 insertions(+), 9 deletions(-) diff --git a/apps/api/v1/rest/grades/src/index.ts b/apps/api/v1/rest/grades/src/index.ts index 6b7ce8af..fe821124 100644 --- a/apps/api/v1/rest/grades/src/index.ts +++ b/apps/api/v1/rest/grades/src/index.ts @@ -1,10 +1,10 @@ import { PrismaClient } from "@libs/db"; import { createErrorResult, createOKResult, logger } from "ant-stack"; import type { InternalHandler } from "ant-stack"; -import type { GradesOptions, GradesRaw } from "peterportal-api-next-types"; +import type { AggregateGroupedGrades, GradesOptions, RawGrades } from "peterportal-api-next-types"; import { ZodError } from "zod"; -import { aggregateGrades, constructPrismaQuery, lexOrd } from "./lib"; +import { aggregateGrades, aggregateGroupedGrades, constructPrismaQuery, lexOrd } from "./lib"; import { QuerySchema } from "./schema"; let prisma: PrismaClient; @@ -39,7 +39,7 @@ export const GET: InternalHandler = async (request) => { })); switch (params.id) { case "raw": - return createOKResult(res, headers, requestId); + return createOKResult(res, headers, requestId); case "aggregate": return createOKResult(aggregateGrades(res), headers, requestId); } @@ -97,6 +97,23 @@ export const GET: InternalHandler = async (request) => { requestId, ); } + case "aggregateGrouped": { + return createOKResult( + aggregateGroupedGrades( + ( + await prisma.gradesSection.findMany({ + where: constructPrismaQuery(parsedQuery), + include: { instructors: true }, + }) + ).map((section) => ({ + ...section, + instructors: section.instructors.map((instructor) => instructor.name), + })), + ), + headers, + requestId, + ); + } } return createErrorResult( 400, diff --git a/apps/api/v1/rest/grades/src/lib.ts b/apps/api/v1/rest/grades/src/lib.ts index 578f267e..59389af9 100644 --- a/apps/api/v1/rest/grades/src/lib.ts +++ b/apps/api/v1/rest/grades/src/lib.ts @@ -1,5 +1,11 @@ import { Prisma } from "@libs/db"; -import type { GradeDistribution, GradesAggregate, GradesRaw } from "peterportal-api-next-types"; +import type { + AggregateGrades, + AggregateGroupedGradeHeader, + AggregateGroupedGrades, + GradeDistribution, + RawGrades, +} from "peterportal-api-next-types"; import { Query } from "./schema"; @@ -10,6 +16,8 @@ import { Query } from "./schema"; */ export const lexOrd = (a: string, b: string): number => (a === b ? 0 : a > b ? 1 : -1); +const headerKeys = ["department", "courseNumber", "instructor"]; + const isNotPNPOnly = ({ gradeACount, gradeBCount, @@ -63,7 +71,7 @@ export function constructPrismaQuery(parsedQuery: Query): Prisma.GradesSectionWh * mean of each section's average GPA in the given dataset. * @param grades The array of grades to aggregate. */ -export function aggregateGrades(grades: GradesRaw): GradesAggregate { +export function aggregateGrades(grades: RawGrades): AggregateGrades { return { sectionList: grades.map( ({ year, quarter, sectionCode, department, courseNumber, courseNumeric, instructors }) => ({ @@ -96,3 +104,38 @@ export function aggregateGrades(grades: GradesRaw): GradesAggregate { }, }; } + +/** + * Given an array of sections and their grades distributions, aggregate them into + * an array of objects, such that if two sections have the same department, course number, + * and instructor, they would be grouped together and aggregated into the same object by + * {@link `aggregateGrades`}. + * @param grades The array of grades to aggregate. + */ +export function aggregateGroupedGrades(grades: RawGrades): AggregateGroupedGrades { + const courses = new Map(); + for (const grade of grades) { + for (const instructor of grade.instructors) { + const { department, courseNumber } = grade; + const key = JSON.stringify([department, courseNumber, instructor]); + if (courses.has(key)) { + courses.get(key)?.push(grade); + } else { + courses.set(key, [grade]); + } + } + } + return Array.from(courses) + .map(([k, v]) => ({ + ...(Object.fromEntries( + (JSON.parse(k) as string[]).map((x, i) => [headerKeys[i], x]), + ) as AggregateGroupedGradeHeader), + ...aggregateGrades(v).gradeDistribution, + })) + .sort( + (a, b) => + lexOrd(a.department, b.department) || + lexOrd(a.courseNumber, b.courseNumber) || + lexOrd(a.instructor, b.instructor), + ); +} diff --git a/packages/peterportal-api-next-types/types/grades.ts b/packages/peterportal-api-next-types/types/grades.ts index 47f5da4d..671a7718 100644 --- a/packages/peterportal-api-next-types/types/grades.ts +++ b/packages/peterportal-api-next-types/types/grades.ts @@ -3,7 +3,7 @@ import { Quarter } from "./constants"; /** * A section which has grades data associated with it. */ -export type GradeSection = { +export type GradesSection = { /** * The year the section was offered. */ @@ -76,22 +76,24 @@ export type GradeDistribution = { averageGPA: number; }; +export type RawGrade = GradesSection & GradeDistribution; + /** * The type of the payload returned on a successful response from querying * ``/v1/rest/grades/raw``. */ -export type GradesRaw = (GradeSection & GradeDistribution)[]; +export type RawGrades = RawGrade[]; /** * An object that represents aggregate grades statistics for a given query. * The type of the payload returned on a successful response from querying * ``/v1/rest/grades/aggregate``. */ -export type GradesAggregate = { +export type AggregateGrades = { /** * The list of sections in the query. */ - sectionList: GradeSection[]; + sectionList: GradesSection[]; /** * The combined grades distribution of all sections in the query. */ @@ -124,3 +126,13 @@ export type GradesOptions = { */ instructors: string[]; }; + +export type AggregateGroupedGradeHeader = { + department: string; + courseNumber: string; + instructor: string; +}; + +export type AggregateGroupedGrade = AggregateGroupedGradeHeader & GradeDistribution; + +export type AggregateGroupedGrades = AggregateGroupedGrade[]; From de61485eca1cd6868c65b2d5680eb51fac0c3948 Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Sun, 27 Aug 2023 14:03:59 -0700 Subject: [PATCH 02/19] feat: :sparkles: add GraphQL support --- .../api/v1/graphql/src/graphql/grades.graphql | 38 +++++++++++++++++++ apps/api/v1/graphql/src/resolvers.ts | 1 + 2 files changed, 39 insertions(+) diff --git a/apps/api/v1/graphql/src/graphql/grades.graphql b/apps/api/v1/graphql/src/graphql/grades.graphql index 4f6089a6..c81a7ea5 100644 --- a/apps/api/v1/graphql/src/graphql/grades.graphql +++ b/apps/api/v1/graphql/src/graphql/grades.graphql @@ -91,6 +91,33 @@ type GradesOptions { "The list of instructors that matched the given filters." instructors: [String!]! } +"An object that represents aggregate grades statistics for a course taught by an instructor." +type AggregateGroupedGrade { + "The department code." + department: String! + "The course number the section belongs to." + courseNumber: String! + "The shortened name of the instructor who taught the section." + instructors: String! + "How many students attained an A+/A/A-." + gradeACount: Int! + "How many students attained a B+/B/B-." + gradeBCount: Int! + "How many students attained a C+/C/C-." + gradeCCount: Int! + "How many students attained a D+/D/D-." + gradeDCount: Int! + "How many students attained an F." + gradeFCount: Int! + "How many students attained a P." + gradePCount: Int! + "How many students attained an NP." + gradeNPCount: Int! + "How many students attained a W." + gradeWCount: Int! + "The average GPA of all assigned grades in the course." + averageGPA: Float! +} extend type Query { "Get the raw grade info for the given parameters." @@ -126,4 +153,15 @@ extend type Query { division: Division excludePNP: Boolean ): GradesOptions! + "Get the aggregate grade info, grouped by course and instructor, for the given parameters." + aggregateGroupedGrades( + year: String + quarter: Quarter + instructor: String + department: String + courseNumber: String + sectionCode: String + division: Division + excludePNP: Boolean + ): [AggregateGroupedGrade!]! } diff --git a/apps/api/v1/graphql/src/resolvers.ts b/apps/api/v1/graphql/src/resolvers.ts index 38a7f3a3..3184605c 100644 --- a/apps/api/v1/graphql/src/resolvers.ts +++ b/apps/api/v1/graphql/src/resolvers.ts @@ -15,6 +15,7 @@ export const resolvers: ApolloServerOptions["resolvers"] = { rawGrades: proxyRestApi("v1/rest/grades/raw"), aggregateGrades: proxyRestApi("v1/rest/grades/aggregate"), gradesOptions: proxyRestApi("v1/rest/grades/options"), + aggregateGroupedGrades: proxyRestApi("v1/rest/grades/aggregateGrouped"), instructor: proxyRestApi("v1/rest/instructors", { pathArg: "courseId" }), instructors: proxyRestApi("v1/rest/instructors"), allInstructors: proxyRestApi("v1/rest/instructors/all"), From bb0100dca71a1dcd3a86fd180d72de849f39e239 Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Wed, 30 Aug 2023 21:47:45 -0700 Subject: [PATCH 03/19] docs: :books: add docs for new operation --- .../rest-api/reference/grades.md | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/apps/docs/docs/developers-guide/rest-api/reference/grades.md b/apps/docs/docs/developers-guide/rest-api/reference/grades.md index 58677ed8..78bdb593 100644 --- a/apps/docs/docs/developers-guide/rest-api/reference/grades.md +++ b/apps/docs/docs/developers-guide/rest-api/reference/grades.md @@ -269,3 +269,111 @@ type GradesOptions = { + +## Get grade statistics aggregated by course/instructor for certain sections + +Formally, if two sections have the same department code, course number, and instructor name, then they will be aggregated together for the purposes of this endpoint. For queries that involve an entire department, this is equivalent to running an aggregate query for each course number-instructor pair, but much faster. + +Note that graduate students who are listed as instructors on WebSoc may also be included. + +### Code sample + + + + +```bash +curl "https://api-next.peterportal.org/v1/rest/grades/aggregateGrouped?year=2023&department=COMPSCI&courseNumber=161" +``` + + + + +### Response + + + + +```json +[ + { + "department": "COMPSCI", + "courseNumber": "161", + "instructor": "FRISHBERG, D.", + "gradeACount": 165, + "gradeBCount": 42, + "gradeCCount": 59, + "gradeDCount": 0, + "gradeFCount": 14, + "gradePCount": 0, + "gradeNPCount": 0, + "gradeWCount": 2, + "averageGPA": 3.23 + }, + { + "department": "COMPSCI", + "courseNumber": "161", + "instructor": "KALOGIANNIS, F.", + "gradeACount": 165, + "gradeBCount": 42, + "gradeCCount": 59, + "gradeDCount": 0, + "gradeFCount": 14, + "gradePCount": 0, + "gradeNPCount": 0, + "gradeWCount": 2, + "averageGPA": 3.23 + }, + { + "department": "COMPSCI", + "courseNumber": "161", + "instructor": "PANAGEAS, I.", + "gradeACount": 101, + "gradeBCount": 115, + "gradeCCount": 48, + "gradeDCount": 15, + "gradeFCount": 12, + "gradePCount": 0, + "gradeNPCount": 0, + "gradeWCount": 2, + "averageGPA": 2.935 + }, + { + "department": "COMPSCI", + "courseNumber": "161", + "instructor": "SHINDLER, M.", + "gradeACount": 165, + "gradeBCount": 42, + "gradeCCount": 59, + "gradeDCount": 0, + "gradeFCount": 14, + "gradePCount": 0, + "gradeNPCount": 0, + "gradeWCount": 2, + "averageGPA": 3.23 + } +] +``` + + + + +```typescript +// https://github.com/icssc/peterportal-api-next/blob/main/packages/peterportal-api-next-types/types/grades.ts +type AggregateGroupedGrades = { + department: string; + courseNumber: string; + instructor: string; + gradeACount: number; + gradeBCount: number; + gradeCCount: number; + gradeDCount: number; + gradeFCount: number; + gradePCount: number; + gradeNPCount: number; + gradeWCount: number; + averageGPA: number; +}[]; +``` + + + From f1e5ac5ca15e5523928c96f7d6eb8f2b0d5021c3 Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Wed, 30 Aug 2023 21:53:22 -0700 Subject: [PATCH 04/19] =?UTF-8?q?chore:=20=F0=9F=94=A7=20suppress=20wfs=20?= =?UTF-8?q?index=20errors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/websoc-fuzzy-search/src/helpers.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/websoc-fuzzy-search/src/helpers.ts b/packages/websoc-fuzzy-search/src/helpers.ts index 88ffb3e0..67164e63 100644 --- a/packages/websoc-fuzzy-search/src/helpers.ts +++ b/packages/websoc-fuzzy-search/src/helpers.ts @@ -1,3 +1,5 @@ +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore import { index } from "../output"; import { types, fieldNames, courseFieldNames, instructorFieldNames } from "./constants"; From d9d2bc1c0919a1271d78b44216395b7e7b4b5ace Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Wed, 30 Aug 2023 22:02:25 -0700 Subject: [PATCH 05/19] =?UTF-8?q?fix(graphql):=20=F0=9F=90=9B=20use=20corr?= =?UTF-8?q?ect=20field=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/v1/graphql/src/graphql/grades.graphql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/v1/graphql/src/graphql/grades.graphql b/apps/api/v1/graphql/src/graphql/grades.graphql index c81a7ea5..d80e274c 100644 --- a/apps/api/v1/graphql/src/graphql/grades.graphql +++ b/apps/api/v1/graphql/src/graphql/grades.graphql @@ -98,7 +98,7 @@ type AggregateGroupedGrade { "The course number the section belongs to." courseNumber: String! "The shortened name of the instructor who taught the section." - instructors: String! + instructor: String! "How many students attained an A+/A/A-." gradeACount: Int! "How many students attained a B+/B/B-." From 87868e9fceee5ddf026d43acc484b1dc07884537 Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Sun, 3 Sep 2023 22:10:14 -0700 Subject: [PATCH 06/19] =?UTF-8?q?feat:=20=E2=9C=A8=20add=20geCategories=20?= =?UTF-8?q?to=20GradesSection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- libs/db/prisma/schema.prisma | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/db/prisma/schema.prisma b/libs/db/prisma/schema.prisma index 718f4a25..eee9c67e 100644 --- a/libs/db/prisma/schema.prisma +++ b/libs/db/prisma/schema.prisma @@ -133,6 +133,7 @@ model GradesSection { department String courseNumber String courseNumeric Int + geCategories Json? instructors GradesInstructor[] gradeACount Int gradeBCount Int From fa63f38bfbd2b9a3d8b7228a3474d58bf0909043 Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Mon, 4 Sep 2023 23:46:56 -0700 Subject: [PATCH 07/19] =?UTF-8?q?feat(grades):=20=E2=9C=A8=20add=20script?= =?UTF-8?q?=20for=20populating=20GE=20data?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/grades-updater/src/populate-ge.ts | 104 ++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 tools/grades-updater/src/populate-ge.ts diff --git a/tools/grades-updater/src/populate-ge.ts b/tools/grades-updater/src/populate-ge.ts new file mode 100644 index 00000000..c34084ef --- /dev/null +++ b/tools/grades-updater/src/populate-ge.ts @@ -0,0 +1,104 @@ +import { Prisma, PrismaClient } from "@libs/db"; +import { callWebSocAPI, GE, geCodes, Quarter } from "@libs/websoc-api-next"; + +const prisma = new PrismaClient({ + log: [ + { emit: "event", level: "query" }, + { emit: "stdout", level: "error" }, + { emit: "stdout", level: "info" }, + { emit: "stdout", level: "warn" }, + ], +}); + +prisma.$on("query", (e) => { + console.log("Query: " + e.query); + console.log("Params: " + e.params); + console.log("Duration: " + e.duration + "ms"); +}); + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +const log = (msg: string) => console.log(`[${new Date().toUTCString()}] ${msg}`); + +function categorySetToMask(geCategories: Set) { + let ret = 0; + for (const [idx, ge] of Object.entries(geCodes)) { + if (geCategories.has(ge)) { + ret |= 2 ** Number.parseInt(idx, 10); + } + } + return ret; +} + +function maskToCategorySet(mask: number) { + const ret = new Set(); + for (const [idx, ge] of Object.entries(geCodes)) { + if (mask & (2 ** Number.parseInt(idx, 10))) { + ret.add(ge); + } + } + return ret; +} + +async function main() { + const sections = await prisma.gradesSection.findMany({ + where: { geCategories: { equals: Prisma.AnyNull } }, + select: { year: true, quarter: true, sectionCode: true }, + }); + log(`Found ${sections.length} sections without GE data`); + const terms = new Set(sections.map(({ year, quarter }) => `${year}-${quarter}`)); + const geData = new Map>(); + for (const term of terms) { + const [year, quarter] = term.split("-") as [string, Quarter]; + for (const ge of geCodes) { + log(`Getting set of section codes for (year=${year}, quarter=${quarter}, ge=${ge})`); + const res = await callWebSocAPI({ year, quarter }, { ge }); + await sleep(1000); + geData.set( + `${year}-${quarter}-${ge}`, + new Set( + res.schools + .flatMap((x) => x.departments) + .flatMap((x) => x.courses) + .flatMap((x) => x.sections) + .map((x) => x.sectionCode), + ), + ); + } + } + const updates = new Map>(); + for (const { year, quarter, sectionCode } of sections) { + const key = `${year}-${quarter}`; + const geCategories = new Set(); + for (const ge of geCodes) { + if (geData.get(`${year}-${quarter}-${ge}`)?.has(sectionCode)) { + geCategories.add(ge); + } + } + const mask = categorySetToMask(geCategories); + if (updates.has(mask)) { + if (updates.get(mask)!.has(key)) { + updates.get(mask)!.get(key)!.push(sectionCode); + } else { + updates.get(mask)!.set(key, [sectionCode]); + } + } else { + updates.set(mask, new Map([[key, [sectionCode]]])); + } + } + const txn = []; + for (const [mask, mapping] of updates) { + for (const [term, sectionCodes] of mapping) { + const [year, quarter] = term.split("-") as [string, Quarter]; + txn.push( + prisma.gradesSection.updateMany({ + where: { year, quarter, sectionCode: { in: sectionCodes } }, + data: { geCategories: Array.from(maskToCategorySet(mask)).sort() }, + }), + ); + } + } + await prisma.$transaction(txn); +} + +main().then(); From b5c3f4f3906f818fa266dfa2e862edf0aff30d7c Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Tue, 5 Sep 2023 11:51:57 -0700 Subject: [PATCH 08/19] =?UTF-8?q?feat:=20=E2=9C=A8=20add=20GE=20param=20to?= =?UTF-8?q?=20schema?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/v1/rest/grades/src/lib.ts | 14 ++++++++++++-- apps/api/v1/rest/grades/src/schema.ts | 3 ++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/apps/api/v1/rest/grades/src/lib.ts b/apps/api/v1/rest/grades/src/lib.ts index 59389af9..53820903 100644 --- a/apps/api/v1/rest/grades/src/lib.ts +++ b/apps/api/v1/rest/grades/src/lib.ts @@ -27,8 +27,17 @@ const isNotPNPOnly = ({ }: GradeDistribution) => gradeACount || gradeBCount || gradeCCount || gradeDCount || gradeFCount; export function constructPrismaQuery(parsedQuery: Query): Prisma.GradesSectionWhereInput { - const { year, quarter, instructor, department, courseNumber, sectionCode, division, excludePNP } = - parsedQuery; + const { + year, + quarter, + instructor, + department, + courseNumber, + sectionCode, + division, + excludePNP, + ge, + } = parsedQuery; const courseNumeric: Prisma.IntFilter = {}; switch (division) { case "LowerDiv": @@ -59,6 +68,7 @@ export function constructPrismaQuery(parsedQuery: Query): Prisma.GradesSectionWh courseNumber, courseNumeric, sectionCode, + geCategories: { array_contains: ge }, NOT: excludePNP ? { ...excludePNPFilters } : undefined, }; } diff --git a/apps/api/v1/rest/grades/src/schema.ts b/apps/api/v1/rest/grades/src/schema.ts index 7f656657..05b3c581 100644 --- a/apps/api/v1/rest/grades/src/schema.ts +++ b/apps/api/v1/rest/grades/src/schema.ts @@ -1,4 +1,4 @@ -import { anyArray, divisionCodes, quarters } from "peterportal-api-next-types"; +import { anyArray, divisionCodes, geCodes, quarters } from "peterportal-api-next-types"; import { z } from "zod"; export const QuerySchema = z.object({ @@ -15,6 +15,7 @@ export const QuerySchema = z.object({ .regex(/^\d{5}$/, { message: "Invalid sectionCode provided" }) .optional(), division: z.enum(anyArray).or(z.enum(divisionCodes)).optional(), + ge: z.enum(anyArray).or(z.enum(geCodes)).optional(), excludePNP: z .string() .optional() From 24ae0d9e60c86a40859f79c839c0ce0f09bc3c99 Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Tue, 5 Sep 2023 12:10:45 -0700 Subject: [PATCH 09/19] =?UTF-8?q?fix:=20=F0=9F=90=9B=20add=20geCategories?= =?UTF-8?q?=20to=20aggregate=20endpoint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/v1/rest/grades/src/lib.ts | 12 +++++++++++- packages/peterportal-api-next-types/types/grades.ts | 6 +++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/apps/api/v1/rest/grades/src/lib.ts b/apps/api/v1/rest/grades/src/lib.ts index 53820903..ecd692e9 100644 --- a/apps/api/v1/rest/grades/src/lib.ts +++ b/apps/api/v1/rest/grades/src/lib.ts @@ -84,7 +84,7 @@ export function constructPrismaQuery(parsedQuery: Query): Prisma.GradesSectionWh export function aggregateGrades(grades: RawGrades): AggregateGrades { return { sectionList: grades.map( - ({ year, quarter, sectionCode, department, courseNumber, courseNumeric, instructors }) => ({ + ({ year, quarter, sectionCode, @@ -92,6 +92,16 @@ export function aggregateGrades(grades: RawGrades): AggregateGrades { courseNumber, courseNumeric, instructors, + geCategories, + }) => ({ + year, + quarter, + sectionCode, + department, + courseNumber, + courseNumeric, + geCategories, + instructors, }), ), gradeDistribution: { diff --git a/packages/peterportal-api-next-types/types/grades.ts b/packages/peterportal-api-next-types/types/grades.ts index 671a7718..b49e6c98 100644 --- a/packages/peterportal-api-next-types/types/grades.ts +++ b/packages/peterportal-api-next-types/types/grades.ts @@ -1,4 +1,4 @@ -import { Quarter } from "./constants"; +import type { GE, Quarter } from "./constants"; /** * A section which has grades data associated with it. @@ -28,6 +28,10 @@ export type GradesSection = { * The numeric part of the course number. */ courseNumeric: number; + /** + * What GE categor(y/ies) the section fulfills (if any). + */ + geCategories: GE[]; /** * The shortened name(s) of the instructor(s) who taught the section. */ From 4e9eabd2a7e8b40125ba1f5f7e8e25a2c9f7b69f Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Tue, 5 Sep 2023 12:17:46 -0700 Subject: [PATCH 10/19] =?UTF-8?q?docs:=20=F0=9F=93=9A=EF=B8=8F=20update=20?= =?UTF-8?q?params?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../developers-guide/rest-api/reference/grades.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/docs/docs/developers-guide/rest-api/reference/grades.md b/apps/docs/docs/developers-guide/rest-api/reference/grades.md index 78bdb593..fc873677 100644 --- a/apps/docs/docs/developers-guide/rest-api/reference/grades.md +++ b/apps/docs/docs/developers-guide/rest-api/reference/grades.md @@ -42,6 +42,10 @@ The five-digit section code to include. The course level/division code to include. Case-sensitive. +#### `ge` GE-1A | GE-1B | GE-2 | GE-3 | GE-4 | GE-5A | GE-5B | GE-6 | GE-7 | GE-8 + +Which GE category to include. Case-sensitive. + #### `excludePNP` boolean Whether to exclude sections that only reported Pass/No-Pass grades. @@ -74,6 +78,7 @@ curl "https://api-next.peterportal.org/v1/rest/grades/raw?year=2022&quarter=Fall "department": "I&C SCI", "courseNumber": "46", "courseNumeric": 46, + "geCategories": ["GE-5B"], "gradeACount": 34, "gradeBCount": 19, "gradeCCount": 40, @@ -96,9 +101,11 @@ curl "https://api-next.peterportal.org/v1/rest/grades/raw?year=2022&quarter=Fall type GradesRaw = { year: string; quarter: string; + sectionCode: string; department: string; courseNumber: string; - sectionCode: string; + courseNumeric: number; + geCategories: GE[]; instructors: string[]; gradeACount: number; gradeBCount: number; @@ -144,6 +151,7 @@ curl "https://api-next.peterportal.org/v1/rest/grades/aggregate?year=2022&quarte "department": "I&C SCI", "courseNumber": "46", "courseNumeric": 46, + "geCategories": ["GE-5B"], "instructors": ["GARZA RODRIGUE, A.", "GILA, O.", "SHINDLER, M."] }, { @@ -153,6 +161,7 @@ curl "https://api-next.peterportal.org/v1/rest/grades/aggregate?year=2022&quarte "department": "I&C SCI", "courseNumber": "46", "courseNumeric": 46, + "geCategories": ["GE-5B"], "instructors": ["DICKERSON, M.", "SHINDLER, M."] } ], @@ -182,6 +191,7 @@ type GradesAggregate = { department: string; courseNumber: string; sectionCode: string; + geCategories: GE[]; instructors: string[]; }[]; gradeDistribution: { From 98e00347ffe001167b80eeb6bfef7a9f5311fc7d Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Tue, 5 Sep 2023 16:00:42 -0700 Subject: [PATCH 11/19] =?UTF-8?q?fix:=20=F0=9F=90=9B=20cast=20GE=20categor?= =?UTF-8?q?y=20arrays?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/v1/rest/grades/src/index.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/api/v1/rest/grades/src/index.ts b/apps/api/v1/rest/grades/src/index.ts index fe821124..acd3ff6e 100644 --- a/apps/api/v1/rest/grades/src/index.ts +++ b/apps/api/v1/rest/grades/src/index.ts @@ -1,7 +1,12 @@ import { PrismaClient } from "@libs/db"; import { createErrorResult, createOKResult, logger } from "ant-stack"; import type { InternalHandler } from "ant-stack"; -import type { AggregateGroupedGrades, GradesOptions, RawGrades } from "peterportal-api-next-types"; +import type { + AggregateGroupedGrades, + GE, + GradesOptions, + RawGrades, +} from "peterportal-api-next-types"; import { ZodError } from "zod"; import { aggregateGrades, aggregateGroupedGrades, constructPrismaQuery, lexOrd } from "./lib"; @@ -35,6 +40,7 @@ export const GET: InternalHandler = async (request) => { }) ).map((section) => ({ ...section, + geCategories: section.geCategories as GE[], instructors: section.instructors.map((instructor) => instructor.name), })); switch (params.id) { @@ -107,6 +113,7 @@ export const GET: InternalHandler = async (request) => { }) ).map((section) => ({ ...section, + geCategories: section.geCategories as GE[], instructors: section.instructors.map((instructor) => instructor.name), })), ), From 0020a7d47d526b646d127d8079f41693bb1dd6ab Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Wed, 6 Sep 2023 21:35:26 -0700 Subject: [PATCH 12/19] =?UTF-8?q?fix:=20=F0=9F=90=9B=20add=20params=20to?= =?UTF-8?q?=20graphql=20queries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/v1/graphql/src/graphql/grades.graphql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/api/v1/graphql/src/graphql/grades.graphql b/apps/api/v1/graphql/src/graphql/grades.graphql index d80e274c..2d1c358d 100644 --- a/apps/api/v1/graphql/src/graphql/grades.graphql +++ b/apps/api/v1/graphql/src/graphql/grades.graphql @@ -140,6 +140,7 @@ extend type Query { courseNumber: String sectionCode: String division: Division + ge: GE excludePNP: Boolean ): AggregateGrades! "Get the available options for the given constraints." @@ -151,6 +152,7 @@ extend type Query { courseNumber: String sectionCode: String division: Division + ge: GE excludePNP: Boolean ): GradesOptions! "Get the aggregate grade info, grouped by course and instructor, for the given parameters." @@ -162,6 +164,7 @@ extend type Query { courseNumber: String sectionCode: String division: Division + ge: GE excludePNP: Boolean ): [AggregateGroupedGrade!]! } From 9770c6dd237dd2886817834c7fea8666d0cb988e Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Wed, 6 Sep 2023 22:02:02 -0700 Subject: [PATCH 13/19] =?UTF-8?q?fix:=20=F0=9F=90=9B=20add=20GE=20transfor?= =?UTF-8?q?m=20to=20graphql=20queries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/v1/graphql/src/resolvers.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/api/v1/graphql/src/resolvers.ts b/apps/api/v1/graphql/src/resolvers.ts index 3184605c..962e3470 100644 --- a/apps/api/v1/graphql/src/resolvers.ts +++ b/apps/api/v1/graphql/src/resolvers.ts @@ -12,10 +12,12 @@ export const resolvers: ApolloServerOptions["resolvers"] = { course: proxyRestApi("v1/rest/courses", { pathArg: "courseId" }), courses: proxyRestApi("v1/rest/courses", { argsTransform: geTransform }), allCourses: proxyRestApi("v1/rest/courses/all"), - rawGrades: proxyRestApi("v1/rest/grades/raw"), - aggregateGrades: proxyRestApi("v1/rest/grades/aggregate"), - gradesOptions: proxyRestApi("v1/rest/grades/options"), - aggregateGroupedGrades: proxyRestApi("v1/rest/grades/aggregateGrouped"), + rawGrades: proxyRestApi("v1/rest/grades/raw", { argsTransform: geTransform }), + aggregateGrades: proxyRestApi("v1/rest/grades/aggregate", { argsTransform: geTransform }), + gradesOptions: proxyRestApi("v1/rest/grades/options", { argsTransform: geTransform }), + aggregateGroupedGrades: proxyRestApi("v1/rest/grades/aggregateGrouped", { + argsTransform: geTransform, + }), instructor: proxyRestApi("v1/rest/instructors", { pathArg: "courseId" }), instructors: proxyRestApi("v1/rest/instructors"), allInstructors: proxyRestApi("v1/rest/instructors/all"), From 028721cbe66db45aa158e08a034eb92875225163 Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Thu, 7 Sep 2023 09:30:06 -0700 Subject: [PATCH 14/19] =?UTF-8?q?perf:=20=E2=9A=A1=EF=B8=8F=20test=20wheth?= =?UTF-8?q?er=20local=20filtering=20improves=20response=20time?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/v1/rest/grades/src/index.ts | 28 ++++++++++++++++++---------- apps/api/v1/rest/grades/src/lib.ts | 14 ++------------ 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/apps/api/v1/rest/grades/src/index.ts b/apps/api/v1/rest/grades/src/index.ts index acd3ff6e..13ed5f61 100644 --- a/apps/api/v1/rest/grades/src/index.ts +++ b/apps/api/v1/rest/grades/src/index.ts @@ -38,11 +38,15 @@ export const GET: InternalHandler = async (request) => { where: constructPrismaQuery(parsedQuery), include: { instructors: true }, }) - ).map((section) => ({ - ...section, - geCategories: section.geCategories as GE[], - instructors: section.instructors.map((instructor) => instructor.name), - })); + ) + .map((section) => ({ + ...section, + geCategories: section.geCategories as GE[], + instructors: section.instructors.map((instructor) => instructor.name), + })) + .filter((section) => + parsedQuery.ge ? section.geCategories.includes(parsedQuery.ge) : section, + ); switch (params.id) { case "raw": return createOKResult(res, headers, requestId); @@ -111,11 +115,15 @@ export const GET: InternalHandler = async (request) => { where: constructPrismaQuery(parsedQuery), include: { instructors: true }, }) - ).map((section) => ({ - ...section, - geCategories: section.geCategories as GE[], - instructors: section.instructors.map((instructor) => instructor.name), - })), + ) + .map((section) => ({ + ...section, + geCategories: section.geCategories as GE[], + instructors: section.instructors.map((instructor) => instructor.name), + })) + .filter((section) => + parsedQuery.ge ? section.geCategories.includes(parsedQuery.ge) : section, + ), ), headers, requestId, diff --git a/apps/api/v1/rest/grades/src/lib.ts b/apps/api/v1/rest/grades/src/lib.ts index ecd692e9..9b95b190 100644 --- a/apps/api/v1/rest/grades/src/lib.ts +++ b/apps/api/v1/rest/grades/src/lib.ts @@ -27,17 +27,8 @@ const isNotPNPOnly = ({ }: GradeDistribution) => gradeACount || gradeBCount || gradeCCount || gradeDCount || gradeFCount; export function constructPrismaQuery(parsedQuery: Query): Prisma.GradesSectionWhereInput { - const { - year, - quarter, - instructor, - department, - courseNumber, - sectionCode, - division, - excludePNP, - ge, - } = parsedQuery; + const { year, quarter, instructor, department, courseNumber, sectionCode, division, excludePNP } = + parsedQuery; const courseNumeric: Prisma.IntFilter = {}; switch (division) { case "LowerDiv": @@ -68,7 +59,6 @@ export function constructPrismaQuery(parsedQuery: Query): Prisma.GradesSectionWh courseNumber, courseNumeric, sectionCode, - geCategories: { array_contains: ge }, NOT: excludePNP ? { ...excludePNPFilters } : undefined, }; } From 3a770ddb0f89a1c7237cb5bc003b738fcb6ea3b4 Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Thu, 7 Sep 2023 09:32:43 -0700 Subject: [PATCH 15/19] =?UTF-8?q?docs:=20=F0=9F=93=9A=EF=B8=8F=20update=20?= =?UTF-8?q?grades=20updater=20docs=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/grades-updater/README.md | 6 +++++- tools/grades-updater/package.json | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/grades-updater/README.md b/tools/grades-updater/README.md index 4d1bed93..96cb699e 100644 --- a/tools/grades-updater/README.md +++ b/tools/grades-updater/README.md @@ -4,7 +4,7 @@ This directory contains the code for updating the grades cache for PeterPortal A ## Sanitizing the Data -1. Make sure all dependencies are up-to-date by running `npm i` in the project root. +1. Make sure all dependencies are up-to-date by running `pnpm i` in the project root. 2. Create the `inputData` and `outputData` directories if they do not already exist. 3. Using a spreadsheet editor, edit the grades spreadsheet you obtained from the [UCI Public Records Office](https://pro.uci.edu/) so that it matches the following format, and then save it as a CSV in `inputData`. @@ -23,3 +23,7 @@ This directory contains the code for updating the grades cache for PeterPortal A 2. Add the `.env.grades` file to the project root. Note that this is only available to members of the ICSSC Projects Committee, since it grants write access to the production database. 3. Run `pnpm upload` in this directory. 4. The logs should be present under `/logs`. + +## Populating GE data + +During the sanitization process, the data that encodes what GE categor(y/ies) is not fetched. This can be remedied by running `pnpm populate` after uploading the data. diff --git a/tools/grades-updater/package.json b/tools/grades-updater/package.json index 48c6bc96..bd13eced 100644 --- a/tools/grades-updater/package.json +++ b/tools/grades-updater/package.json @@ -4,6 +4,7 @@ "private": true, "type": "module", "scripts": { + "populate": "dotenv -e ../../.env.grades -- tsx populate-ge.ts", "sanitize": "tsx sanitize-data.ts", "upload": "dotenv -e ../../.env.grades -- tsx upload-data.ts" }, From 3fec2dfca5d70a35323e44f73e154a8a68ac43c6 Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Thu, 7 Sep 2023 11:03:36 -0700 Subject: [PATCH 16/19] =?UTF-8?q?perf:=20=E2=9A=A1=EF=B8=8F=20use=20one=20?= =?UTF-8?q?column=20per=20GE=20category?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/v1/graphql/src/graphql/enum.graphql | 1 + .../api/v1/graphql/src/graphql/grades.graphql | 4 + apps/api/v1/rest/grades/src/index.ts | 35 ++----- apps/api/v1/rest/grades/src/lib.ts | 95 ++++++++++++++++++- libs/db/prisma/schema.prisma | 12 ++- tools/grades-updater/package.json | 6 +- tools/grades-updater/src/populate-ge.ts | 41 ++++++-- 7 files changed, 153 insertions(+), 41 deletions(-) diff --git a/apps/api/v1/graphql/src/graphql/enum.graphql b/apps/api/v1/graphql/src/graphql/enum.graphql index 2bbeecfa..38820ffc 100644 --- a/apps/api/v1/graphql/src/graphql/enum.graphql +++ b/apps/api/v1/graphql/src/graphql/enum.graphql @@ -9,6 +9,7 @@ enum Quarter { } "The set of valid GE category codes." enum GE { + ANY GE_1A GE_1B GE_2 diff --git a/apps/api/v1/graphql/src/graphql/grades.graphql b/apps/api/v1/graphql/src/graphql/grades.graphql index 2d1c358d..8a9ddef2 100644 --- a/apps/api/v1/graphql/src/graphql/grades.graphql +++ b/apps/api/v1/graphql/src/graphql/grades.graphql @@ -12,6 +12,8 @@ type GradesSection { courseNumber: String! "The numeric part of the course number." courseNumeric: Int! + "What GE categor(y/ies) this section satisfies (if any)." + geCategories: [GE!]! "The shortened name(s) of the instructor(s) who taught the section." instructors: [String!]! } @@ -50,6 +52,8 @@ type RawGrade { courseNumber: String! "The numeric part of the course number." courseNumeric: Int! + "What GE categor(y/ies) this section satisfies (if any)." + geCategories: [GE!]! "How many students attained an A+/A/A-." gradeACount: Int! "How many students attained a B+/B/B-." diff --git a/apps/api/v1/rest/grades/src/index.ts b/apps/api/v1/rest/grades/src/index.ts index 13ed5f61..1bbd0149 100644 --- a/apps/api/v1/rest/grades/src/index.ts +++ b/apps/api/v1/rest/grades/src/index.ts @@ -1,15 +1,16 @@ import { PrismaClient } from "@libs/db"; import { createErrorResult, createOKResult, logger } from "ant-stack"; import type { InternalHandler } from "ant-stack"; -import type { - AggregateGroupedGrades, - GE, - GradesOptions, - RawGrades, -} from "peterportal-api-next-types"; +import type { AggregateGroupedGrades, GradesOptions, RawGrades } from "peterportal-api-next-types"; import { ZodError } from "zod"; -import { aggregateGrades, aggregateGroupedGrades, constructPrismaQuery, lexOrd } from "./lib"; +import { + aggregateGrades, + aggregateGroupedGrades, + constructPrismaQuery, + lexOrd, + transformRow, +} from "./lib"; import { QuerySchema } from "./schema"; let prisma: PrismaClient; @@ -38,15 +39,7 @@ export const GET: InternalHandler = async (request) => { where: constructPrismaQuery(parsedQuery), include: { instructors: true }, }) - ) - .map((section) => ({ - ...section, - geCategories: section.geCategories as GE[], - instructors: section.instructors.map((instructor) => instructor.name), - })) - .filter((section) => - parsedQuery.ge ? section.geCategories.includes(parsedQuery.ge) : section, - ); + ).map(transformRow); switch (params.id) { case "raw": return createOKResult(res, headers, requestId); @@ -115,15 +108,7 @@ export const GET: InternalHandler = async (request) => { where: constructPrismaQuery(parsedQuery), include: { instructors: true }, }) - ) - .map((section) => ({ - ...section, - geCategories: section.geCategories as GE[], - instructors: section.instructors.map((instructor) => instructor.name), - })) - .filter((section) => - parsedQuery.ge ? section.geCategories.includes(parsedQuery.ge) : section, - ), + ).map(transformRow), ), headers, requestId, diff --git a/apps/api/v1/rest/grades/src/lib.ts b/apps/api/v1/rest/grades/src/lib.ts index 9b95b190..9d8ba2cd 100644 --- a/apps/api/v1/rest/grades/src/lib.ts +++ b/apps/api/v1/rest/grades/src/lib.ts @@ -1,14 +1,23 @@ -import { Prisma } from "@libs/db"; +import { GradesSection, Prisma } from "@libs/db"; import type { AggregateGrades, AggregateGroupedGradeHeader, AggregateGroupedGrades, + GE, GradeDistribution, + Quarter, + RawGrade, RawGrades, } from "peterportal-api-next-types"; +import { geCodes } from "peterportal-api-next-types"; import { Query } from "./schema"; +/** + * type guard that asserts input is defined + */ +export const notNull = (x: T): x is NonNullable => x != null; + /** * Returns the lexicographical ordering of two elements. * @param a The left hand side of the comparison. @@ -18,6 +27,19 @@ export const lexOrd = (a: string, b: string): number => (a === b ? 0 : a > b ? 1 const headerKeys = ["department", "courseNumber", "instructor"]; +const geKeys = [ + "isGE1A", + "isGE1B", + "isGE2", + "isGE3", + "isGE4", + "isGE5A", + "isGE5B", + "isGE6", + "isGE7", + "isGE8", +] as const; + const isNotPNPOnly = ({ gradeACount, gradeBCount, @@ -26,9 +48,71 @@ const isNotPNPOnly = ({ gradeFCount, }: GradeDistribution) => gradeACount || gradeBCount || gradeCCount || gradeDCount || gradeFCount; +const geToKey = (ge: Exclude) => geKeys[geCodes.indexOf(ge)]; + +export const transformRow = ({ + year, + quarter, + sectionCode, + department, + courseNumber, + courseNumeric, + isGE1A, + isGE1B, + isGE2, + isGE3, + isGE4, + isGE5A, + isGE5B, + isGE6, + isGE7, + isGE8, + gradeACount, + gradeBCount, + gradeCCount, + gradeDCount, + gradeFCount, + gradePCount, + gradeNPCount, + gradeWCount, + averageGPA, + instructors, +}: GradesSection & { + instructors: { year: string; quarter: Quarter; sectionCode: string; name: string }[]; +}): RawGrade => ({ + year, + quarter, + sectionCode, + department, + courseNumber, + courseNumeric, + geCategories: [isGE1A, isGE1B, isGE2, isGE3, isGE4, isGE5A, isGE5B, isGE6, isGE7, isGE8] + .map((x, i) => (x ? geCodes[i] : null)) + .filter(notNull), + instructors: instructors.map((x) => x.name), + gradeACount, + gradeBCount, + gradeCCount, + gradeDCount, + gradeFCount, + gradePCount, + gradeNPCount, + gradeWCount, + averageGPA, +}); + export function constructPrismaQuery(parsedQuery: Query): Prisma.GradesSectionWhereInput { - const { year, quarter, instructor, department, courseNumber, sectionCode, division, excludePNP } = - parsedQuery; + const { + year, + quarter, + instructor, + department, + courseNumber, + sectionCode, + division, + excludePNP, + ge, + } = parsedQuery; const courseNumeric: Prisma.IntFilter = {}; switch (division) { case "LowerDiv": @@ -51,6 +135,10 @@ export function constructPrismaQuery(parsedQuery: Query): Prisma.GradesSectionWh excludePNPFilters.gradeDCount = 0; excludePNPFilters.gradeFCount = 0; } + const geFilter: Record = {}; + if (ge && ge !== "ANY") { + geFilter[geToKey(ge)] = true; + } return { year, quarter, @@ -59,6 +147,7 @@ export function constructPrismaQuery(parsedQuery: Query): Prisma.GradesSectionWh courseNumber, courseNumeric, sectionCode, + ...geFilter, NOT: excludePNP ? { ...excludePNPFilters } : undefined, }; } diff --git a/libs/db/prisma/schema.prisma b/libs/db/prisma/schema.prisma index eee9c67e..cf917d71 100644 --- a/libs/db/prisma/schema.prisma +++ b/libs/db/prisma/schema.prisma @@ -133,7 +133,17 @@ model GradesSection { department String courseNumber String courseNumeric Int - geCategories Json? + hasGEData Boolean @default(false) + isGE1A Boolean? + isGE1B Boolean? + isGE2 Boolean? + isGE3 Boolean? + isGE4 Boolean? + isGE5A Boolean? + isGE5B Boolean? + isGE6 Boolean? + isGE7 Boolean? + isGE8 Boolean? instructors GradesInstructor[] gradeACount Int gradeBCount Int diff --git a/tools/grades-updater/package.json b/tools/grades-updater/package.json index bd13eced..bcff0e13 100644 --- a/tools/grades-updater/package.json +++ b/tools/grades-updater/package.json @@ -4,9 +4,9 @@ "private": true, "type": "module", "scripts": { - "populate": "dotenv -e ../../.env.grades -- tsx populate-ge.ts", - "sanitize": "tsx sanitize-data.ts", - "upload": "dotenv -e ../../.env.grades -- tsx upload-data.ts" + "populate": "dotenv -e ../../.env.grades -- tsx src/populate-ge.ts", + "sanitize": "tsx src/sanitize-data.ts", + "upload": "dotenv -e ../../.env.grades -- tsx src/upload-data.ts" }, "dependencies": { "@libs/db": "workspace:*", diff --git a/tools/grades-updater/src/populate-ge.ts b/tools/grades-updater/src/populate-ge.ts index c34084ef..e6b2ff7d 100644 --- a/tools/grades-updater/src/populate-ge.ts +++ b/tools/grades-updater/src/populate-ge.ts @@ -1,4 +1,4 @@ -import { Prisma, PrismaClient } from "@libs/db"; +import { PrismaClient } from "@libs/db"; import { callWebSocAPI, GE, geCodes, Quarter } from "@libs/websoc-api-next"; const prisma = new PrismaClient({ @@ -16,6 +16,19 @@ prisma.$on("query", (e) => { console.log("Duration: " + e.duration + "ms"); }); +const geKeys = [ + "isGE1A", + "isGE1B", + "isGE2", + "isGE3", + "isGE4", + "isGE5A", + "isGE5B", + "isGE6", + "isGE7", + "isGE8", +] as const; + const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); const log = (msg: string) => console.log(`[${new Date().toUTCString()}] ${msg}`); @@ -30,19 +43,29 @@ function categorySetToMask(geCategories: Set) { return ret; } -function maskToCategorySet(mask: number) { - const ret = new Set(); - for (const [idx, ge] of Object.entries(geCodes)) { - if (mask & (2 ** Number.parseInt(idx, 10))) { - ret.add(ge); - } +function maskToCategoryMap(mask: number) { + const ret: Record<(typeof geKeys)[number], boolean> = { + isGE1A: false, + isGE1B: false, + isGE2: false, + isGE3: false, + isGE4: false, + isGE5A: false, + isGE5B: false, + isGE6: false, + isGE7: false, + isGE8: false, + }; + for (const i of Object.keys(geCodes)) { + const idx = Number.parseInt(i, 10); + ret[geKeys[idx]] = !!(mask & (2 ** idx)); } return ret; } async function main() { const sections = await prisma.gradesSection.findMany({ - where: { geCategories: { equals: Prisma.AnyNull } }, + where: { hasGEData: false }, select: { year: true, quarter: true, sectionCode: true }, }); log(`Found ${sections.length} sections without GE data`); @@ -93,7 +116,7 @@ async function main() { txn.push( prisma.gradesSection.updateMany({ where: { year, quarter, sectionCode: { in: sectionCodes } }, - data: { geCategories: Array.from(maskToCategorySet(mask)).sort() }, + data: { hasGEData: true, ...maskToCategoryMap(mask) }, }), ); } From f2b0d089567262d9c83632b8e3e71ed609fa7bd1 Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Thu, 7 Sep 2023 15:37:47 -0700 Subject: [PATCH 17/19] =?UTF-8?q?feat!:=20=F0=9F=92=A5=20=E2=9C=A8=20add?= =?UTF-8?q?=20aggregateByCourse=20endpoint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/v1/graphql/src/graphql/grades.graphql | 50 ++++++++++- apps/api/v1/graphql/src/resolvers.ts | 5 +- apps/api/v1/rest/grades/src/index.ts | 30 +++++-- apps/api/v1/rest/grades/src/lib.ts | 41 ++++++++-- .../rest-api/reference/grades.md | 82 ++++++++++++++++++- .../types/grades.ts | 15 +++- 6 files changed, 200 insertions(+), 23 deletions(-) diff --git a/apps/api/v1/graphql/src/graphql/grades.graphql b/apps/api/v1/graphql/src/graphql/grades.graphql index 8a9ddef2..beb6087b 100644 --- a/apps/api/v1/graphql/src/graphql/grades.graphql +++ b/apps/api/v1/graphql/src/graphql/grades.graphql @@ -96,7 +96,34 @@ type GradesOptions { instructors: [String!]! } "An object that represents aggregate grades statistics for a course taught by an instructor." -type AggregateGroupedGrade { +type AggregateGradeByCourse { + "The department code." + department: String! + "The course number the section belongs to." + courseNumber: String! + "The shortened name of the instructor who taught the section." + instructor: String! + "How many students attained an A+/A/A-." + gradeACount: Int! + "How many students attained a B+/B/B-." + gradeBCount: Int! + "How many students attained a C+/C/C-." + gradeCCount: Int! + "How many students attained a D+/D/D-." + gradeDCount: Int! + "How many students attained an F." + gradeFCount: Int! + "How many students attained a P." + gradePCount: Int! + "How many students attained an NP." + gradeNPCount: Int! + "How many students attained a W." + gradeWCount: Int! + "The average GPA of all assigned grades in the course." + averageGPA: Float! +} +"An object that represents aggregate grades statistics for a course taught by an instructor." +type AggregateGradeByOffering { "The department code." department: String! "The course number the section belongs to." @@ -159,8 +186,23 @@ extend type Query { ge: GE excludePNP: Boolean ): GradesOptions! - "Get the aggregate grade info, grouped by course and instructor, for the given parameters." - aggregateGroupedGrades( + "Get the aggregate grade info, grouped by course, for the given parameters." + aggregateByCourse( + year: String + quarter: Quarter + instructor: String + department: String + courseNumber: String + sectionCode: String + division: Division + ge: GE + excludePNP: Boolean + ): [AggregateGradeByOffering!]! + """ + Get the aggregate grade info, grouped by offering (course and instructor), + for the given parameters. + """ + aggregateByOffering( year: String quarter: Quarter instructor: String @@ -170,5 +212,5 @@ extend type Query { division: Division ge: GE excludePNP: Boolean - ): [AggregateGroupedGrade!]! + ): [AggregateGradeByOffering!]! } diff --git a/apps/api/v1/graphql/src/resolvers.ts b/apps/api/v1/graphql/src/resolvers.ts index 962e3470..b499f393 100644 --- a/apps/api/v1/graphql/src/resolvers.ts +++ b/apps/api/v1/graphql/src/resolvers.ts @@ -15,7 +15,10 @@ export const resolvers: ApolloServerOptions["resolvers"] = { rawGrades: proxyRestApi("v1/rest/grades/raw", { argsTransform: geTransform }), aggregateGrades: proxyRestApi("v1/rest/grades/aggregate", { argsTransform: geTransform }), gradesOptions: proxyRestApi("v1/rest/grades/options", { argsTransform: geTransform }), - aggregateGroupedGrades: proxyRestApi("v1/rest/grades/aggregateGrouped", { + aggregateByCourse: proxyRestApi("v1/rest/grades/aggregateByCourse", { + argsTransform: geTransform, + }), + aggregateByOffering: proxyRestApi("v1/rest/grades/aggregateByOffering", { argsTransform: geTransform, }), instructor: proxyRestApi("v1/rest/instructors", { pathArg: "courseId" }), diff --git a/apps/api/v1/rest/grades/src/index.ts b/apps/api/v1/rest/grades/src/index.ts index 1bbd0149..02c26694 100644 --- a/apps/api/v1/rest/grades/src/index.ts +++ b/apps/api/v1/rest/grades/src/index.ts @@ -1,15 +1,21 @@ import { PrismaClient } from "@libs/db"; import { createErrorResult, createOKResult, logger } from "ant-stack"; import type { InternalHandler } from "ant-stack"; -import type { AggregateGroupedGrades, GradesOptions, RawGrades } from "peterportal-api-next-types"; +import type { + AggregateGradesByCourse, + AggregateGradesByOffering, + GradesOptions, + RawGrades, +} from "peterportal-api-next-types"; import { ZodError } from "zod"; import { aggregateGrades, - aggregateGroupedGrades, + aggregateByOffering, constructPrismaQuery, lexOrd, transformRow, + aggregateByCourse, } from "./lib"; import { QuerySchema } from "./schema"; @@ -100,9 +106,23 @@ export const GET: InternalHandler = async (request) => { requestId, ); } - case "aggregateGrouped": { - return createOKResult( - aggregateGroupedGrades( + case "aggregateByCourse": { + return createOKResult( + aggregateByCourse( + ( + await prisma.gradesSection.findMany({ + where: constructPrismaQuery(parsedQuery), + include: { instructors: true }, + }) + ).map(transformRow), + ), + headers, + requestId, + ); + } + case "aggregateByOffering": { + return createOKResult( + aggregateByOffering( ( await prisma.gradesSection.findMany({ where: constructPrismaQuery(parsedQuery), diff --git a/apps/api/v1/rest/grades/src/lib.ts b/apps/api/v1/rest/grades/src/lib.ts index 9d8ba2cd..05e02da0 100644 --- a/apps/api/v1/rest/grades/src/lib.ts +++ b/apps/api/v1/rest/grades/src/lib.ts @@ -1,13 +1,15 @@ import { GradesSection, Prisma } from "@libs/db"; import type { AggregateGrades, - AggregateGroupedGradeHeader, - AggregateGroupedGrades, + AggregateGradesByOffering, + AggregateGradeByOfferingHeader, GE, GradeDistribution, Quarter, RawGrade, RawGrades, + AggregateGradesByCourse, + AggregateGradeByCourseHeader, } from "peterportal-api-next-types"; import { geCodes } from "peterportal-api-next-types"; @@ -205,13 +207,36 @@ export function aggregateGrades(grades: RawGrades): AggregateGrades { } /** - * Given an array of sections and their grades distributions, aggregate them into - * an array of objects, such that if two sections have the same department, course number, - * and instructor, they would be grouped together and aggregated into the same object by - * {@link `aggregateGrades`}. + * Given an array of sections and their grades distributions, aggregate the sections with the same + * department and course number. * @param grades The array of grades to aggregate. */ -export function aggregateGroupedGrades(grades: RawGrades): AggregateGroupedGrades { +export function aggregateByCourse(grades: RawGrades): AggregateGradesByCourse { + const courses = new Map(); + for (const grade of grades) { + const { department, courseNumber } = grade; + const key = JSON.stringify([department, courseNumber]); + if (courses.has(key)) { + courses.get(key)?.push(grade); + } else { + courses.set(key, [grade]); + } + } + return Array.from(courses) + .map(([k, v]) => ({ + ...(Object.fromEntries( + (JSON.parse(k) as string[]).map((x, i) => [headerKeys[i], x]), + ) as AggregateGradeByCourseHeader), + ...aggregateGrades(v).gradeDistribution, + })) + .sort((a, b) => lexOrd(a.department, b.department) || lexOrd(a.courseNumber, b.courseNumber)); +} + +/** + * Same as the above but also factors into the instructor of the section. + * @param grades The array of grades to aggregate. + */ +export function aggregateByOffering(grades: RawGrades): AggregateGradesByOffering { const courses = new Map(); for (const grade of grades) { for (const instructor of grade.instructors) { @@ -228,7 +253,7 @@ export function aggregateGroupedGrades(grades: RawGrades): AggregateGroupedGrade .map(([k, v]) => ({ ...(Object.fromEntries( (JSON.parse(k) as string[]).map((x, i) => [headerKeys[i], x]), - ) as AggregateGroupedGradeHeader), + ) as AggregateGradeByOfferingHeader), ...aggregateGrades(v).gradeDistribution, })) .sort( diff --git a/apps/docs/docs/developers-guide/rest-api/reference/grades.md b/apps/docs/docs/developers-guide/rest-api/reference/grades.md index fc873677..9ad9c7e6 100644 --- a/apps/docs/docs/developers-guide/rest-api/reference/grades.md +++ b/apps/docs/docs/developers-guide/rest-api/reference/grades.md @@ -280,6 +280,84 @@ type GradesOptions = { +## Get grade statistics aggregated by course for certain sections + +Formally, if two sections have the same department code and course number, then they will be aggregated together for the purposes of this endpoint. For queries that involve an entire department, this is equivalent to running an aggregate query for each course number, but much faster. + +Note that graduate students who are listed as instructors on WebSoc may also be included. + +### Code sample + + + + +```bash +curl "https://api-next.peterportal.org/v1/rest/grades/aggregateByCourse?year=2023&department=COMPSCI" +``` + + + + +### Response + + + + +```json +[ + { + "department": "COMPSCI", + "courseNumber": "103", + "gradeACount": 80, + "gradeBCount": 11, + "gradeCCount": 10, + "gradeDCount": 4, + "gradeFCount": 5, + "gradePCount": 11, + "gradeNPCount": 8, + "gradeWCount": 0, + "averageGPA": 3.415 + }, + { + "department": "COMPSCI", + "courseNumber": "111", + "gradeACount": 112, + "gradeBCount": 100, + "gradeCCount": 35, + "gradeDCount": 13, + "gradeFCount": 13, + "gradePCount": 13, + "gradeNPCount": 2, + "gradeWCount": 2, + "averageGPA": 3.11625 + }, + "..." +] +``` + + + + +```typescript +// https://github.com/icssc/peterportal-api-next/blob/main/packages/peterportal-api-next-types/types/grades.ts +type AggregateGradesByCourse = { + department: string; + courseNumber: string; + gradeACount: number; + gradeBCount: number; + gradeCCount: number; + gradeDCount: number; + gradeFCount: number; + gradePCount: number; + gradeNPCount: number; + gradeWCount: number; + averageGPA: number; +}[]; +``` + + + + ## Get grade statistics aggregated by course/instructor for certain sections Formally, if two sections have the same department code, course number, and instructor name, then they will be aggregated together for the purposes of this endpoint. For queries that involve an entire department, this is equivalent to running an aggregate query for each course number-instructor pair, but much faster. @@ -292,7 +370,7 @@ Note that graduate students who are listed as instructors on WebSoc may also be ```bash -curl "https://api-next.peterportal.org/v1/rest/grades/aggregateGrouped?year=2023&department=COMPSCI&courseNumber=161" +curl "https://api-next.peterportal.org/v1/rest/grades/aggregateByOffering?year=2023&department=COMPSCI&courseNumber=161" ``` @@ -369,7 +447,7 @@ curl "https://api-next.peterportal.org/v1/rest/grades/aggregateGrouped?year=2023 ```typescript // https://github.com/icssc/peterportal-api-next/blob/main/packages/peterportal-api-next-types/types/grades.ts -type AggregateGroupedGrades = { +type AggregateGradesByOffering = { department: string; courseNumber: string; instructor: string; diff --git a/packages/peterportal-api-next-types/types/grades.ts b/packages/peterportal-api-next-types/types/grades.ts index b49e6c98..e1e5a6ef 100644 --- a/packages/peterportal-api-next-types/types/grades.ts +++ b/packages/peterportal-api-next-types/types/grades.ts @@ -131,12 +131,21 @@ export type GradesOptions = { instructors: string[]; }; -export type AggregateGroupedGradeHeader = { +export type AggregateGradeByCourseHeader = { + department: string; + courseNumber: string; +}; + +export type AggregateGradeByCourse = AggregateGradeByCourseHeader & GradeDistribution; + +export type AggregateGradesByCourse = AggregateGradeByCourse[]; + +export type AggregateGradeByOfferingHeader = { department: string; courseNumber: string; instructor: string; }; -export type AggregateGroupedGrade = AggregateGroupedGradeHeader & GradeDistribution; +export type AggregateGradeByOffering = AggregateGradeByOfferingHeader & GradeDistribution; -export type AggregateGroupedGrades = AggregateGroupedGrade[]; +export type AggregateGradesByOffering = AggregateGradeByOffering[]; From 6a5b147dcaa1b9ace92f63a633731380656df88a Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Thu, 7 Sep 2023 15:38:09 -0700 Subject: [PATCH 18/19] =?UTF-8?q?docs:=20=F0=9F=93=9A=EF=B8=8F=20update=20?= =?UTF-8?q?type=20names?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/docs/docs/developers-guide/rest-api/reference/grades.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/docs/docs/developers-guide/rest-api/reference/grades.md b/apps/docs/docs/developers-guide/rest-api/reference/grades.md index 9ad9c7e6..2b9e140c 100644 --- a/apps/docs/docs/developers-guide/rest-api/reference/grades.md +++ b/apps/docs/docs/developers-guide/rest-api/reference/grades.md @@ -98,7 +98,7 @@ curl "https://api-next.peterportal.org/v1/rest/grades/raw?year=2022&quarter=Fall ```typescript // https://github.com/icssc/peterportal-api-next/blob/main/packages/peterportal-api-next-types/types/grades.ts -type GradesRaw = { +type RawGrades = { year: string; quarter: string; sectionCode: string; @@ -184,7 +184,7 @@ curl "https://api-next.peterportal.org/v1/rest/grades/aggregate?year=2022&quarte ```typescript // https://github.com/icssc/peterportal-api-next/blob/main/packages/peterportal-api-next-types/types/grades.ts -type GradesAggregate = { +type AggregateGrades = { sectionList: { year: string; quarter: string; From e2b4d87c2d11f63290f1013c96fb3bca789a0f49 Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Sun, 17 Sep 2023 18:50:39 -0700 Subject: [PATCH 19/19] =?UTF-8?q?fix:=20=F0=9F=90=9B=20use=20correct=20typ?= =?UTF-8?q?e=20for=20aggregateByCourse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/v1/graphql/src/graphql/grades.graphql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/api/v1/graphql/src/graphql/grades.graphql b/apps/api/v1/graphql/src/graphql/grades.graphql index beb6087b..c93c9697 100644 --- a/apps/api/v1/graphql/src/graphql/grades.graphql +++ b/apps/api/v1/graphql/src/graphql/grades.graphql @@ -101,8 +101,6 @@ type AggregateGradeByCourse { department: String! "The course number the section belongs to." courseNumber: String! - "The shortened name of the instructor who taught the section." - instructor: String! "How many students attained an A+/A/A-." gradeACount: Int! "How many students attained a B+/B/B-." @@ -197,7 +195,7 @@ extend type Query { division: Division ge: GE excludePNP: Boolean - ): [AggregateGradeByOffering!]! + ): [AggregateGradeByCourse!]! """ Get the aggregate grade info, grouped by offering (course and instructor), for the given parameters.