Skip to content
This repository has been archived by the owner on Oct 18, 2024. It is now read-only.

feat: ✨ implement degreeworks scraper and requirements endpoint #140

Merged
merged 57 commits into from
May 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
57 commits
Select commit Hold shift + click to select a range
1bb9d82
feat: :sparkles: add degreeworks scraper skeleton
ecxyzzy Aug 14, 2023
f2697e0
feat: :sparkles: more degreeworks stuff
ecxyzzy Aug 14, 2023
77a560c
feat: :sparkles: parse student ID from auth cookie
ecxyzzy Aug 17, 2023
c3a6067
chore: :wrench: update ProgramId type
ecxyzzy Aug 17, 2023
a42e7ad
feat: :sparkles: normalize course numbers and remove excluded courses
ecxyzzy Aug 17, 2023
14344fe
fix: :bug: ratelimit when dogfooding
ecxyzzy Aug 17, 2023
9b76eae
feat: :sparkles: add separate class for degreeworks requests
ecxyzzy Aug 20, 2023
998afe1
chore(deps): :link: update lockfile
ecxyzzy Aug 20, 2023
ee235ba
chore: :wrench: merge branch 'main' into degreeworks-scraper
ecxyzzy Aug 20, 2023
a46d357
feat: :sparkles: add cache API courses locally when scraping
ecxyzzy Aug 20, 2023
20a7057
fix: :bug: parallelize now that we're no longer ratelimited
ecxyzzy Aug 20, 2023
3ea98d1
fix: :bug: improve course number ordering
ecxyzzy Aug 20, 2023
523e189
feat: :sparkles: add majors, misc refactoring
ecxyzzy Aug 20, 2023
2a3db67
refactor: :recycle: use static async factory & encapsulate functions
ecxyzzy Aug 20, 2023
c8be841
fix: :bug: address method binding issues
ecxyzzy Aug 20, 2023
3f95048
feat: :sparkles: scrape specializations
ecxyzzy Aug 20, 2023
2f356be
chore: :wrench: =
ecxyzzy Aug 20, 2023
d703fa7
refactor: :recycle: add readonly modifiers
ecxyzzy Aug 20, 2023
b291eee
feat: :sparkles: key by program name and dedupe
ecxyzzy Aug 21, 2023
353c877
fix: :bug: don't parse blockId for specs
ecxyzzy Aug 21, 2023
2c41268
fix: :bug: actually scrape grad programs
ecxyzzy Aug 21, 2023
92592f5
feat: :sparkles: parse specializations from stringified block
ecxyzzy Aug 21, 2023
137da4d
feat: :sparkles: determine program equality by audit title
ecxyzzy Aug 21, 2023
820c1ca
chore: :wrench: use non-null assert instead of ts-ignore
ecxyzzy Aug 21, 2023
c5386de
style: :art: clean up non-interpolated strings
ecxyzzy Aug 21, 2023
06910a4
feat: :sparkles: more encapsulation
ecxyzzy Aug 21, 2023
dc4ccf8
refactor: :recycle: separate components dir
ecxyzzy Aug 21, 2023
2e48af2
chore: :wrench: add json ext to output
ecxyzzy Aug 22, 2023
3527a8a
feat: :sparkles: handle Subset rule type
ecxyzzy Aug 22, 2023
5d39f85
chore(deps): :link: remove deep-equal
ecxyzzy Aug 22, 2023
2721723
feat: :sparkles: account for 'specs' of type other
ecxyzzy Aug 22, 2023
bdd75f7
feat: :sparkles: more separation of concerns
ecxyzzy Aug 23, 2023
924e9fc
feat: :sparkles: clean up programs with 'other' blocks
ecxyzzy Aug 23, 2023
876e493
fix: :bug: encapsulated a little too hard
ecxyzzy Aug 23, 2023
f1ff93c
feat: :sparkles: merge specs of all programs with empty reqs
ecxyzzy Aug 23, 2023
5b7a469
chore: :wrench: shrimplify spec/other predicate
ecxyzzy Aug 23, 2023
d3308e2
chore: :wrench: merge main into degreeworks-scraper
ecxyzzy Aug 25, 2023
3abc6c4
docs(types): :books: update comments
ecxyzzy Sep 9, 2023
d1a1be6
chore: 🔧 merge main into degreeworks-scraper
ecxyzzy Oct 16, 2023
ee1f26a
chore: 🔧 merge main into degreeworks-scraper
ecxyzzy Nov 17, 2023
0252f42
fix: 🐛 import correct jwtDecode
ecxyzzy Nov 17, 2023
f1cb7e8
chore: 🔧 stop worrying and use the null-assert
ecxyzzy Nov 17, 2023
21b9b08
chore: 🔧 add .env to ignore
ecxyzzy Nov 17, 2023
da5eec9
feat: ✨ force GC before enrollment history, add logging
ecxyzzy Nov 20, 2023
35f42bc
chore: merge main into degreeworks-scraper
ecxyzzy Nov 20, 2023
4a41af2
chore: 🔧 merge main
ecxyzzy Dec 28, 2023
91ef3f6
chore: 🔧 merge main into degreeworks-scraper
ecxyzzy Jan 31, 2024
258eb11
chore(deps): 🔗 update package.json
ecxyzzy Jan 31, 2024
7db875c
chore: 🔧 merge main
ecxyzzy Mar 1, 2024
1e18912
fix(api-client): 🐛 specify gzip encoding
ecxyzzy Mar 1, 2024
c3683e4
feat: ✨ add degree schemas
ecxyzzy Mar 2, 2024
1f6dfe8
feat: ✨ rework schema, get scraper to upload
ecxyzzy Mar 6, 2024
e24d7f1
feat: ✨ implement rest/gql endpoints for degree data
ecxyzzy Mar 7, 2024
93c4cdb
chore: merge main
ecxyzzy Mar 9, 2024
be32515
chore(deps): 🔗 fix broken lockfile
ecxyzzy Mar 9, 2024
01f5751
refactor(api): ♻️ degrees endpoint (#147)
ap0nia May 19, 2024
d81b2cc
fix: 🐛 address requested changes
ecxyzzy May 19, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions apps/api/src/routes/v1/graphql/resolvers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,14 @@ export const resolvers: ApolloServerOptions<BaseContext>["resolvers"] = {
course: proxyRestApi("/v1/rest/courses", { pathArg: "courseId" }),
courses: proxyRestApi("/v1/rest/courses", { argsTransform: geTransform }),
allCourses: proxyRestApi("/v1/rest/courses/all"),
major: proxyRestApi("/v1/rest/degrees/majors"),
majors: proxyRestApi("/v1/rest/degrees/majors"),
minor: proxyRestApi("/v1/rest/degrees/minors"),
minors: proxyRestApi("/v1/rest/degrees/minors"),
specialization: proxyRestApi("/v1/rest/degrees/specializations"),
specializations: proxyRestApi("/v1/rest/degrees/specializations"),
specializationsByMajorId: proxyRestApi("/v1/rest/degrees/specializations"),
allDegrees: proxyRestApi("/v1/rest/degrees/all"),
enrollmentHistory: proxyRestApi("/v1/rest/enrollmentHistory"),
rawGrades: proxyRestApi("/v1/rest/grades/raw"),
aggregateGrades: proxyRestApi("/v1/rest/grades/aggregate"),
Expand Down
39 changes: 39 additions & 0 deletions apps/api/src/routes/v1/graphql/schema/degrees.graphql
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
type Specialization {
id: String!
majorId: String!
name: String!
requirements: JSON!
}

type Major {
id: String!
degreeId: String!
code: String!
name: String!
requirements: JSON!
specializations: [Specialization!]!
}

type Minor {
id: String!
name: String!
requirements: JSON!
}

type Degree {
id: String!
name: String!
division: DegreeDivision!
majors: [Major!]!
}

extend type Query {
major(id: String!): Major!
majors(degreeId: String, nameContains: String): [Major!]!
minor(id: String!): Minor!
minors(nameContains: String): [Minor!]!
specialization(id: String!): Specialization!
specializations(nameContains: String): [Specialization!]!
specializationsByMajorId(majorId: String!): [Specialization!]!
allDegrees: [Degree!]!
}
5 changes: 5 additions & 0 deletions apps/api/src/routes/v1/graphql/schema/enum.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -69,3 +69,8 @@ enum WebsocSectionFinalExamStatus {
TBA_FINAL
SCHEDULED_FINAL
}
"The set of valid degree divisions."
enum DegreeDivision {
Undergraduate
Graduate
}
146 changes: 146 additions & 0 deletions apps/api/src/routes/v1/rest/degrees/{id}/+endpoint.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
import { PrismaClient } from "@libs/db";
import { createHandler } from "@libs/lambda";

import { ProgramSchema, SpecializationSchema } from "./schema";

const prisma = new PrismaClient();

async function onWarm() {
await prisma.$connect();
}

const degreeRepository = {
majors: {
findMany: async () => {
return await prisma.major.findMany({ include: { specializations: true } });
},
findFirstById: async (id: string) => {
return await prisma.major.findFirst({ where: { id }, include: { specializations: true } });
},
findManyNameContains: async (degreeId: string, contains?: string) => {
return await prisma.major.findMany({
where: {
degreeId,
name: { contains, mode: "insensitive" },
},
include: { specializations: true },
});
},
},
minors: {
findMany: async () => {
return await prisma.minor.findMany({});
},
findFirstById: async (id: string) => {
return await prisma.minor.findFirst({ where: { id } });
},
},
};

export const GET = createHandler(async (event, context, res) => {
const headers = event.headers;
const params = event.pathParameters ?? {};
const query = event.queryStringParameters ?? {};
const requestId = context.awsRequestId;

switch (params?.id) {
case "all":
return res.createOKResult(
await prisma.degree.findMany({
include: { majors: { include: { specializations: true } } },
}),
headers,
requestId,
);

case "majors": // falls through
case "minors": {
const parsedQuery = ProgramSchema.safeParse(query);

if (!parsedQuery.success) {
return res.createErrorResult(
400,
parsedQuery.error.issues.map((issue) => issue.message).join("; "),
requestId,
);
}

switch (parsedQuery.data.type) {
case "id": {
const result = await degreeRepository[params.id].findFirstById(parsedQuery.data.id);
return result
? res.createOKResult(result, headers, requestId)
: res.createErrorResult(
404,
`${params.id === "majors" ? "Major" : "Minor"} with ID ${parsedQuery.data.id} not found`,
requestId,
);
}

case "degreeOrName": {
const { degreeId, nameContains } = parsedQuery.data;

if (params.id === "minors" && degreeId != null) {
return res.createErrorResult(400, "Invalid input", requestId);
}

const result = await degreeRepository.majors.findManyNameContains(degreeId, nameContains);
return res.createOKResult(result, headers, requestId);
}

case "empty": {
const result = await degreeRepository[params.id].findMany();
return res.createOKResult(result, headers, requestId);
}
}
break;
}

case "specializations": {
const parsedQuery = SpecializationSchema.safeParse(query);

if (!parsedQuery.success) {
return res.createErrorResult(
400,
parsedQuery.error.issues.map((issue) => issue.message).join("; "),
requestId,
);
}

switch (parsedQuery.data.type) {
case "id": {
const row = await prisma.specialization.findFirst({ where: { id: parsedQuery.data.id } });

return row
? res.createOKResult(row, headers, requestId)
: res.createErrorResult(
404,
`Specialization with ID ${parsedQuery.data.id} not found`,
requestId,
);
}

case "major": {
const result = await prisma.specialization.findMany({
where: { majorId: parsedQuery.data.majorId },
});
return res.createOKResult(result, headers, requestId);
}

case "name": {
const result = await prisma.specialization.findMany({
where: { name: { contains: parsedQuery.data.nameContains, mode: "insensitive" } },
});
return res.createOKResult(result, headers, requestId);
}

case "empty": {
const result = await prisma.specialization.findMany();
return res.createOKResult(result, headers, requestId);
}
}
}
}

return res.createErrorResult(400, "Invalid endpoint", requestId);
}, onWarm);
42 changes: 42 additions & 0 deletions apps/api/src/routes/v1/rest/degrees/{id}/schema.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
import { z } from "zod";

export const ProgramSchema = z
.union([
z.object({ id: z.string() }),
z.object({ degreeId: z.string().optional(), nameContains: z.string().optional() }),
z.object({}),
])
.transform((data) => {
if ("id" in data) {
return { type: "id" as const, ...data };
}

if ("degreeId" in data && data.degreeId != null) {
return { type: "degreeOrName" as const, degreeId: data.degreeId, ...data };
}

return { type: "empty" as const, ...data };
});

export const SpecializationSchema = z
.union([
z.object({ id: z.string() }),
z.object({ majorId: z.string() }),
z.object({ nameContains: z.string() }),
z.object({}),
])
.transform((data) => {
if ("id" in data) {
return { type: "id" as const, ...data };
}

if ("majorId" in data) {
return { type: "major" as const, ...data };
}

if ("nameContains" in data) {
return { type: "name" as const, ...data };
}

return { type: "empty" as const, ...data };
});
58 changes: 47 additions & 11 deletions libs/db/prisma/schema.prisma
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,11 @@ enum CourseLevel {
Graduate
}

enum Division {
Undergraduate
Graduate
}

enum Quarter {
Fall
Winter
Expand Down Expand Up @@ -91,17 +96,11 @@ model Course {
terms String[]
}

model Instructor {
ucinetid String @id
name String
shortenedName String
title String
email String
department String
schools String[]
relatedDepartments String[]
courseHistory Json
courses Json @default("[]")
model Degree {
id String @id
name String
division Division
majors Major[]
}

model GradesInstructor {
Expand Down Expand Up @@ -148,6 +147,43 @@ model GradesSection {
@@unique([year, quarter, sectionCode], name: "idx")
}

model Instructor {
ucinetid String @id
name String
shortenedName String
title String
email String
department String
schools String[]
relatedDepartments String[]
courseHistory Json
courses Json @default("[]")
}

model Major {
id String @id
degreeId String
degree Degree @relation(fields: [degreeId], references: [id])
code String
name String
requirements Json
specializations Specialization[]
}

model Minor {
id String @id
name String
requirements Json
}

model Specialization {
id String @id
majorId String
major Major @relation(fields: [majorId], references: [id])
name String
requirements Json
}

model WebsocEnrollmentHistoryEntry {
year String
quarter Quarter
Expand Down
Loading
Loading