From 1bb9d8293248af28537ccc601e8dc3c1f2aa9ead Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Mon, 14 Aug 2023 11:52:46 -0700
Subject: [PATCH 01/48] feat: :sparkles: add degreeworks scraper skeleton

---
 pnpm-lock.yaml                         |   6 +
 tools/degreeworks-scraper/.gitignore   |   1 +
 tools/degreeworks-scraper/package.json |  10 ++
 tools/degreeworks-scraper/src/index.ts | 231 +++++++++++++++++++++++++
 4 files changed, 248 insertions(+)
 create mode 100644 tools/degreeworks-scraper/.gitignore
 create mode 100644 tools/degreeworks-scraper/package.json
 create mode 100644 tools/degreeworks-scraper/src/index.ts

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 2acd3f4e..93ac1e50 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -604,6 +604,12 @@ importers:
         specifier: 4.0.0
         version: 4.0.0
 
+  tools/degreeworks-scraper:
+    dependencies:
+      cross-fetch:
+        specifier: 4.0.0
+        version: 4.0.0
+
   tools/grades-updater:
     dependencies:
       '@libs/db':
diff --git a/tools/degreeworks-scraper/.gitignore b/tools/degreeworks-scraper/.gitignore
new file mode 100644
index 00000000..ea1472ec
--- /dev/null
+++ b/tools/degreeworks-scraper/.gitignore
@@ -0,0 +1 @@
+output/
diff --git a/tools/degreeworks-scraper/package.json b/tools/degreeworks-scraper/package.json
new file mode 100644
index 00000000..ed564456
--- /dev/null
+++ b/tools/degreeworks-scraper/package.json
@@ -0,0 +1,10 @@
+{
+  "name": "degreeworks-scraper",
+  "version": "0.0.0",
+  "private": true,
+  "type": "module",
+  "main": "src/index.ts",
+  "dependencies": {
+    "cross-fetch": "4.0.0"
+  }
+}
diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
new file mode 100644
index 00000000..5ddd502a
--- /dev/null
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -0,0 +1,231 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import fetch from "cross-fetch";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+/**
+ * The base type for all `Rule` objects.
+ */
+type RuleBase = { label: string };
+/**
+ * A group of `numberOfRules` rules,
+ * of which `numberOfGroups` must be satisfied
+ * in order to fulfill this rule.
+ */
+type RuleGroup = {
+  ruleType: "Group";
+  requirement: { numberOfGroups: string; numberOfRules: string };
+  ruleArray: Rule[];
+};
+/**
+ * An object that represents a (range of) course(s).
+ */
+type Course = { discipline: string; number: string; numberEnd?: string };
+/**
+ * A rule that is fulfilled by taking `creditsBegin` units
+ * and/or `classesBegin` courses from the `courseArray`.
+ */
+type RuleCourse = {
+  ruleType: "Course";
+  requirement: { creditsBegin?: string; classesBegin?: string; courseArray: Course[] };
+};
+/**
+ * A rule that has different requirements depending on some boolean condition.
+ * This seems to be used to denote all specializations that can be applied to a major.
+ */
+type RuleIfStmt = {
+  ruleType: "IfStmt";
+  requirement: { ifPart: { ruleArray: Rule[] }; elsePart?: { ruleArray: Rule[] } };
+};
+/**
+ * A rule that refers to another block (typically a specialization).
+ */
+type RuleBlock = {
+  ruleType: "Block";
+  requirement: { numBlocks: string; type: string; value: string };
+};
+/**
+ * A rule that is not a course.
+ * This seems to be only used by Engineering majors
+ * that have a design unit requirement.
+ */
+type RuleNoncourse = {
+  ruleType: "Noncourse";
+  requirement: { numNoncourses: string; code: string };
+};
+type Rule = RuleBase & (RuleGroup | RuleCourse | RuleIfStmt | RuleBlock | RuleNoncourse);
+type Block = {
+  requirementType: string;
+  requirementValue: string;
+  title: string;
+  ruleArray: Rule[];
+};
+type DWAuditOKResponse = { blockArray: Block[] };
+type DWAuditErrorResponse = { error: never };
+/**
+ * The type of the DegreeWorks audit response.
+ */
+type DWAuditResponse = DWAuditOKResponse | DWAuditErrorResponse;
+
+type DWMappingResponse<T extends string> = {
+  _embedded: { [P in T]: { key: string; description: string }[] };
+};
+
+const DW_API_URL = "https://reg.uci.edu/RespDashboard/api";
+const AUDIT_URL = `${DW_API_URL}/audit`;
+const HEADERS = {
+  "Content-Type": "application/json",
+  Cookie: `X-AUTH-TOKEN=${process.env["X_AUTH_TOKEN"]}`,
+  Origin: "https://reg.uci.edu",
+};
+const DELAY = 1000;
+
+async function getMajorAudit(
+  catalogYear: string,
+  degree: string,
+  school: string,
+  majorCode: string,
+): Promise<Block | undefined> {
+  const res = await fetch(AUDIT_URL, {
+    method: "POST",
+    body: JSON.stringify({
+      catalogYear,
+      degree,
+      school,
+      classes: [],
+      goals: [{ code: "MAJOR", value: majorCode }],
+      studentId: process.env["STUDENT_ID"],
+    }),
+    headers: HEADERS,
+  });
+  await sleep(DELAY);
+  const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
+  return "error" in json
+    ? undefined
+    : json.blockArray.find(
+        (x) => x.requirementType === "MAJOR" && x.requirementValue === majorCode,
+      );
+}
+
+async function getMinorAudit(catalogYear: string, minorCode: string): Promise<Block | undefined> {
+  const res = await fetch(AUDIT_URL, {
+    method: "POST",
+    body: JSON.stringify({
+      catalogYear,
+      degree: "BA",
+      school: "U",
+      classes: [],
+      goals: [
+        { code: "MAJOR", value: "000" },
+        { code: "MINOR", value: minorCode },
+      ],
+      studentId: process.env["STUDENT_ID"],
+    }),
+    headers: HEADERS,
+  });
+  await sleep(DELAY);
+  const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
+  return "error" in json
+    ? undefined
+    : json.blockArray.find(
+        (x) => x.requirementType === "MINOR" && x.requirementValue === minorCode,
+      );
+}
+
+// async function getSpecAudit(
+//   catalogYear: string,
+//   degree: string,
+//   school: string,
+//   majorCode: string,
+//   specCode: string,
+// ): Promise<Block | undefined> {
+//   const res = await fetch(AUDIT_URL, {
+//     method: "POST",
+//     body: JSON.stringify({
+//       catalogYear,
+//       degree,
+//       school,
+//       classes: [],
+//       goals: [
+//         { code: "MAJOR", value: majorCode },
+//         { code: "SPEC", value: specCode },
+//       ],
+//       studentId: process.env["STUDENT_ID"],
+//     }),
+//     headers: HEADERS,
+//   });
+//   await sleep(DELAY);
+//   const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
+//   return "error" in json
+//     ? undefined
+//     : json.blockArray.find((x) => x.requirementType === "SPEC" && x.requirementValue === specCode);
+// }
+
+async function getMapping<T extends string>(path: T): Promise<Map<string, string>> {
+  const res = await fetch(`${DW_API_URL}/${path}`, { headers: HEADERS });
+  await sleep(DELAY);
+  const json: DWMappingResponse<T> = await res.json();
+  return new Map(json._embedded[path].map((x) => [x.key, x.description]));
+}
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+async function main() {
+  if (!process.env["STUDENT_ID"]) throw new Error("Student ID not set.");
+  if (!process.env["X_AUTH_TOKEN"]) throw new Error("Auth cookie not set.");
+  console.log("degreeworks-scraper starting");
+  const currentYear = new Date().getUTCFullYear();
+  /**
+   * The current catalog year.
+   *
+   * Depending on when we are scraping, this may be the academic year that started
+   * the previous calendar year, or the one that will start this calendar year.
+   *
+   * We determine the catalog year by seeing if we can fetch the major data for the
+   * B.S. in Computer Science for the latter. If it is available, then we use that
+   * as the catalog year. Otherwise, we use the former.
+   */
+  const catalogYear = (await getMajorAudit(`${currentYear}${currentYear + 1}`, "BS", "U", "201"))
+    ? `${currentYear}${currentYear + 1}`
+    : `${currentYear - 1}${currentYear}`;
+  console.log(`Set catalogYear to ${catalogYear}`);
+
+  const degrees = await getMapping("degrees");
+  console.log(`Fetched ${degrees.size} degrees`);
+  const majorPrograms = await getMapping("majors");
+  console.log(`Fetched ${majorPrograms.size} major programs`);
+  const minorPrograms = await getMapping("minors");
+  console.log(`Fetched ${minorPrograms.size} minor programs`);
+
+  // const undergraduateDegrees = new Map(
+  //   [...degrees.entries()].filter(([k, _]) => k.startsWith("B")),
+  // );
+  // const graduateDegrees = new Map([...degrees.entries()].filter(([k, _]) => !k.startsWith("B")));
+
+  const minorProgramRequirements = new Map<string, Block>();
+  console.log("Scraping minor program requirements");
+  for (const minorCode of minorPrograms.keys()) {
+    const audit = await getMinorAudit(catalogYear, minorCode);
+    if (!audit) {
+      console.log(`Minor program not found for code ${minorCode}`);
+      continue;
+    }
+    console.log(`Requirements block for "${audit.title}" found for code ${minorCode}`);
+    minorProgramRequirements.set(minorCode, {
+      requirementType: audit.requirementType,
+      requirementValue: audit.requirementValue,
+      title: audit.title,
+      ruleArray: [...audit.ruleArray],
+    });
+  }
+  await mkdir(join(__dirname, "../output"), { recursive: true });
+  await writeFile(
+    join(__dirname, "../output/minorProgramRequirements.json"),
+    JSON.stringify(Object.fromEntries(minorProgramRequirements.entries())),
+  );
+}
+
+main().then(() => []);

From f2697e0cc35f35f5d7c435c5c07c4c0279e2d4da Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Mon, 14 Aug 2023 15:59:37 -0700
Subject: [PATCH 02/48] feat: :sparkles: more degreeworks stuff

---
 tools/degreeworks-scraper/package.json |   3 +
 tools/degreeworks-scraper/src/index.ts |  84 ++--------------
 tools/degreeworks-scraper/src/parse.ts | 107 ++++++++++++++++++++
 tools/degreeworks-scraper/src/types.ts | 132 +++++++++++++++++++++++++
 4 files changed, 251 insertions(+), 75 deletions(-)
 create mode 100644 tools/degreeworks-scraper/src/parse.ts
 create mode 100644 tools/degreeworks-scraper/src/types.ts

diff --git a/tools/degreeworks-scraper/package.json b/tools/degreeworks-scraper/package.json
index ed564456..df5d8bb8 100644
--- a/tools/degreeworks-scraper/package.json
+++ b/tools/degreeworks-scraper/package.json
@@ -6,5 +6,8 @@
   "main": "src/index.ts",
   "dependencies": {
     "cross-fetch": "4.0.0"
+  },
+  "devDependencies": {
+    "peterportal-api-next-types": "workspace:*"
   }
 }
diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index 5ddd502a..dd792fd5 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -4,75 +4,9 @@ import { fileURLToPath } from "node:url";
 
 import fetch from "cross-fetch";
 
-const __dirname = dirname(fileURLToPath(import.meta.url));
-
-/**
- * The base type for all `Rule` objects.
- */
-type RuleBase = { label: string };
-/**
- * A group of `numberOfRules` rules,
- * of which `numberOfGroups` must be satisfied
- * in order to fulfill this rule.
- */
-type RuleGroup = {
-  ruleType: "Group";
-  requirement: { numberOfGroups: string; numberOfRules: string };
-  ruleArray: Rule[];
-};
-/**
- * An object that represents a (range of) course(s).
- */
-type Course = { discipline: string; number: string; numberEnd?: string };
-/**
- * A rule that is fulfilled by taking `creditsBegin` units
- * and/or `classesBegin` courses from the `courseArray`.
- */
-type RuleCourse = {
-  ruleType: "Course";
-  requirement: { creditsBegin?: string; classesBegin?: string; courseArray: Course[] };
-};
-/**
- * A rule that has different requirements depending on some boolean condition.
- * This seems to be used to denote all specializations that can be applied to a major.
- */
-type RuleIfStmt = {
-  ruleType: "IfStmt";
-  requirement: { ifPart: { ruleArray: Rule[] }; elsePart?: { ruleArray: Rule[] } };
-};
-/**
- * A rule that refers to another block (typically a specialization).
- */
-type RuleBlock = {
-  ruleType: "Block";
-  requirement: { numBlocks: string; type: string; value: string };
-};
-/**
- * A rule that is not a course.
- * This seems to be only used by Engineering majors
- * that have a design unit requirement.
- */
-type RuleNoncourse = {
-  ruleType: "Noncourse";
-  requirement: { numNoncourses: string; code: string };
-};
-type Rule = RuleBase & (RuleGroup | RuleCourse | RuleIfStmt | RuleBlock | RuleNoncourse);
-type Block = {
-  requirementType: string;
-  requirementValue: string;
-  title: string;
-  ruleArray: Rule[];
-};
-type DWAuditOKResponse = { blockArray: Block[] };
-type DWAuditErrorResponse = { error: never };
-/**
- * The type of the DegreeWorks audit response.
- */
-type DWAuditResponse = DWAuditOKResponse | DWAuditErrorResponse;
+import type { Block, DWAuditResponse, DWMappingResponse } from "./types";
 
-type DWMappingResponse<T extends string> = {
-  _embedded: { [P in T]: { key: string; description: string }[] };
-};
+const __dirname = dirname(fileURLToPath(import.meta.url));
 
 const DW_API_URL = "https://reg.uci.edu/RespDashboard/api";
 const AUDIT_URL = `${DW_API_URL}/audit`;
@@ -83,6 +17,8 @@ const HEADERS = {
 };
 const DELAY = 1000;
 
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
 async function getMajorAudit(
   catalogYear: string,
   degree: string,
@@ -171,8 +107,6 @@ async function getMapping<T extends string>(path: T): Promise<Map<string, string
   return new Map(json._embedded[path].map((x) => [x.key, x.description]));
 }
 
-const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
-
 async function main() {
   if (!process.env["STUDENT_ID"]) throw new Error("Student ID not set.");
   if (!process.env["X_AUTH_TOKEN"]) throw new Error("Auth cookie not set.");
@@ -200,10 +134,10 @@ async function main() {
   const minorPrograms = await getMapping("minors");
   console.log(`Fetched ${minorPrograms.size} minor programs`);
 
-  // const undergraduateDegrees = new Map(
-  //   [...degrees.entries()].filter(([k, _]) => k.startsWith("B")),
-  // );
-  // const graduateDegrees = new Map([...degrees.entries()].filter(([k, _]) => !k.startsWith("B")));
+  const undergraduateDegrees = new Set<string>();
+  const graduateDegrees = new Set<string>();
+  for (const degree of degrees.keys())
+    (degree.startsWith("B") ? undergraduateDegrees : graduateDegrees).add(degree);
 
   const minorProgramRequirements = new Map<string, Block>();
   console.log("Scraping minor program requirements");
@@ -214,7 +148,7 @@ async function main() {
       continue;
     }
     console.log(`Requirements block for "${audit.title}" found for code ${minorCode}`);
-    minorProgramRequirements.set(minorCode, {
+    minorProgramRequirements.set(`U-MINOR-${minorCode}`, {
       requirementType: audit.requirementType,
       requirementValue: audit.requirementValue,
       title: audit.title,
diff --git a/tools/degreeworks-scraper/src/parse.ts b/tools/degreeworks-scraper/src/parse.ts
new file mode 100644
index 00000000..339857e7
--- /dev/null
+++ b/tools/degreeworks-scraper/src/parse.ts
@@ -0,0 +1,107 @@
+import type { Block, Program, ProgramId, Requirement, Rule } from "./types";
+
+const parseSpecs = (ruleArray: Rule[]) =>
+  ruleArray
+    .filter((x) => x.ruleType === "IfStmt")
+    .flatMap((x) => ifStmtToSpecArray([x]))
+    .sort();
+
+function ifStmtToSpecArray(ruleArray: Rule[]): string[] {
+  const ret = [];
+  for (const rule of ruleArray) {
+    switch (rule.ruleType) {
+      case "IfStmt":
+        ret.push(
+          ...ifStmtToSpecArray(rule.requirement.ifPart.ruleArray),
+          ...ifStmtToSpecArray(rule.requirement.elsePart?.ruleArray ?? []),
+        );
+        break;
+      case "Block":
+        ret.push(rule.requirement.value);
+        break;
+    }
+  }
+  return ret;
+}
+
+function flattenIfStmt(ruleArray: Rule[]): Rule[] {
+  const ret = [];
+  for (const rule of ruleArray) {
+    switch (rule.ruleType) {
+      case "IfStmt":
+        ret.push(
+          ...flattenIfStmt(rule.requirement.ifPart.ruleArray),
+          ...flattenIfStmt(rule.requirement.elsePart?.ruleArray ?? []),
+        );
+        break;
+      default:
+        ret.push(rule);
+    }
+  }
+  return ret;
+}
+
+function ruleArrayToRequirements(ruleArray: Rule[]) {
+  const ret: Record<string, Requirement> = {};
+  for (const rule of ruleArray) {
+    switch (rule.ruleType) {
+      case "Course": {
+        const courses = {
+          include: rule.requirement.courseArray.map(
+            (x) => `${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`,
+          ),
+          ...(rule.requirement.except?.courseArray && {
+            exclude: rule.requirement.except.courseArray.map(
+              (x) => `${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`,
+            ),
+          }),
+        };
+        if (rule.requirement.classesBegin) {
+          ret[rule.label] = {
+            requirementType: "Course",
+            courseCount: Number.parseInt(rule.requirement.classesBegin, 10),
+            courses,
+          };
+        } else if (rule.requirement.creditsBegin) {
+          ret[rule.label] = {
+            requirementType: "Unit",
+            unitCount: Number.parseInt(rule.requirement.creditsBegin, 10),
+            courses,
+          };
+        }
+        break;
+      }
+      case "Group":
+        ret[rule.label] = {
+          requirementType: "Group",
+          requirementCount: Number.parseInt(rule.requirement.numberOfGroups),
+          requirements: ruleArrayToRequirements(rule.ruleArray),
+        };
+        break;
+      case "IfStmt": {
+        const rules = flattenIfStmt([rule]);
+        if (rules.length > 1 && !rules.find((x) => x.ruleType === "Block")) {
+          ret["Select 1 of the following"] = {
+            requirementType: "Group",
+            requirementCount: 1,
+            requirements: ruleArrayToRequirements(rules),
+          };
+        }
+        break;
+      }
+    }
+  }
+  return ret;
+}
+
+function parseBlockId(blockId: string) {
+  const [school, programType, code, degreeType] = blockId.split("-");
+  return { school, programType, code, degreeType } as ProgramId;
+}
+
+export const parseBlock = (blockId: string, block: Block): Program => ({
+  ...parseBlockId(blockId),
+  name: block.title,
+  requirements: ruleArrayToRequirements(block.ruleArray),
+  specs: parseSpecs(block.ruleArray),
+});
diff --git a/tools/degreeworks-scraper/src/types.ts b/tools/degreeworks-scraper/src/types.ts
new file mode 100644
index 00000000..331fa311
--- /dev/null
+++ b/tools/degreeworks-scraper/src/types.ts
@@ -0,0 +1,132 @@
+// region DegreeWorks response types
+/**
+ * The base type for all `Rule` objects.
+ */
+export type RuleBase = { label: string };
+/**
+ * A group of `numberOfRules` rules,
+ * of which `numberOfGroups` must be satisfied
+ * in order to fulfill this rule.
+ */
+export type RuleGroup = {
+  ruleType: "Group";
+  requirement: { numberOfGroups: string; numberOfRules: string };
+  ruleArray: Rule[];
+};
+/**
+ * An object that represents a (range of) course(s).
+ */
+export type Course = { discipline: string; number: string; numberEnd?: string };
+/**
+ * A rule that is fulfilled by taking `creditsBegin` units
+ * and/or `classesBegin` courses from the `courseArray`.
+ */
+export type RuleCourse = {
+  ruleType: "Course";
+  requirement: {
+    creditsBegin?: string;
+    classesBegin?: string;
+    courseArray: Course[];
+    except?: { courseArray: Course[] };
+  };
+};
+/**
+ * A rule that has different requirements depending on some boolean condition.
+ * This seems to be used to denote all specializations that can be applied to a major.
+ */
+export type RuleIfStmt = {
+  ruleType: "IfStmt";
+  requirement: { ifPart: { ruleArray: Rule[] }; elsePart?: { ruleArray: Rule[] } };
+};
+/**
+ * A rule that refers to another block (typically a specialization).
+ */
+export type RuleBlock = {
+  ruleType: "Block";
+  requirement: { numBlocks: string; type: string; value: string };
+};
+/**
+ * A rule that is not a course.
+ * This seems to be only used by Engineering majors
+ * that have a design unit requirement.
+ */
+export type RuleNoncourse = {
+  ruleType: "Noncourse";
+  requirement: { numNoncourses: string; code: string };
+};
+export type Rule = RuleBase & (RuleGroup | RuleCourse | RuleIfStmt | RuleBlock | RuleNoncourse);
+export type Block = {
+  requirementType: string;
+  requirementValue: string;
+  title: string;
+  ruleArray: Rule[];
+};
+export type DWAuditOKResponse = { blockArray: Block[] };
+export type DWAuditErrorResponse = { error: never };
+/**
+ * The type of the DegreeWorks audit response.
+ */
+export type DWAuditResponse = DWAuditOKResponse | DWAuditErrorResponse;
+
+export type DWMappingResponse<T extends string> = {
+  _embedded: { [P in T]: { key: string; description: string }[] };
+};
+// endregion
+
+// region Processed types
+
+export type ProgramId = {
+  school: "U" | "G";
+  programType: "MAJOR" | "MINOR" | "SPEC";
+  code: string;
+};
+
+export type Program = ProgramId & {
+  /**
+   * The display name of the program.
+   * @example "Major in Computer Science"
+   * @example "Minor in Mathematics"
+   * @example "Specialization in Digital Signal Processing"
+   */
+  name: string;
+  requirements: Record<string, Requirement>;
+  specs: string[];
+};
+
+export type CourseRequirement = {
+  requirementType: "Course";
+  /**
+   * The number of `courses` required to fulfill this requirement.
+   */
+  courseCount: number;
+  courses: { include: string[]; exclude?: string[] };
+};
+
+export type UnitRequirement = {
+  requirementType: "Unit";
+  /**
+   * The number of units earned from taking `courses` that are required to fulfill this requirement.
+   */
+  unitCount: number;
+  courses: { include: string[]; exclude?: string[] };
+};
+
+export type GroupRequirement = {
+  requirementType: "Group";
+  /**
+   * The number of `requirements` that must be fulfilled to fulfill this requirement.
+   */
+  requirementCount: number;
+  requirements: Record<string, Requirement>;
+};
+
+export type IfStmtRequirement = {
+  requirementType: "IfStmt";
+  rules: Rule[];
+};
+
+export type Requirement =
+  | CourseRequirement
+  | UnitRequirement
+  | GroupRequirement
+  | IfStmtRequirement;

From 77a560c7fe29372653ea00a4fa61f2e4f250e0ac Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Thu, 17 Aug 2023 12:10:21 -0700
Subject: [PATCH 03/48] feat: :sparkles: parse student ID from auth cookie

---
 tools/degreeworks-scraper/package.json |  10 +-
 tools/degreeworks-scraper/src/index.ts | 132 +++++--------------------
 tools/degreeworks-scraper/src/lib.ts   | 109 ++++++++++++++++++++
 3 files changed, 142 insertions(+), 109 deletions(-)
 create mode 100644 tools/degreeworks-scraper/src/lib.ts

diff --git a/tools/degreeworks-scraper/package.json b/tools/degreeworks-scraper/package.json
index df5d8bb8..4b1a4211 100644
--- a/tools/degreeworks-scraper/package.json
+++ b/tools/degreeworks-scraper/package.json
@@ -4,10 +4,16 @@
   "private": true,
   "type": "module",
   "main": "src/index.ts",
+  "scripts": {
+    "start": "tsx src/index.ts"
+  },
   "dependencies": {
-    "cross-fetch": "4.0.0"
+    "cross-fetch": "4.0.0",
+    "dotenv": "16.3.1",
+    "jwt-decode": "3.1.2"
   },
   "devDependencies": {
-    "peterportal-api-next-types": "workspace:*"
+    "peterportal-api-next-types": "workspace:*",
+    "tsx": "3.12.7"
   }
 }
diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index dd792fd5..b40d59a5 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -2,114 +2,25 @@ import { mkdir, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 
-import fetch from "cross-fetch";
+import jwtDecode from "jwt-decode";
+import type { JwtPayload } from "jwt-decode";
 
-import type { Block, DWAuditResponse, DWMappingResponse } from "./types";
+import { getMajorAudit, getMapping, getMinorAudit } from "./lib";
+import type { Block } from "./types";
 
-const __dirname = dirname(fileURLToPath(import.meta.url));
-
-const DW_API_URL = "https://reg.uci.edu/RespDashboard/api";
-const AUDIT_URL = `${DW_API_URL}/audit`;
-const HEADERS = {
-  "Content-Type": "application/json",
-  Cookie: `X-AUTH-TOKEN=${process.env["X_AUTH_TOKEN"]}`,
-  Origin: "https://reg.uci.edu",
-};
-const DELAY = 1000;
-
-const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
-
-async function getMajorAudit(
-  catalogYear: string,
-  degree: string,
-  school: string,
-  majorCode: string,
-): Promise<Block | undefined> {
-  const res = await fetch(AUDIT_URL, {
-    method: "POST",
-    body: JSON.stringify({
-      catalogYear,
-      degree,
-      school,
-      classes: [],
-      goals: [{ code: "MAJOR", value: majorCode }],
-      studentId: process.env["STUDENT_ID"],
-    }),
-    headers: HEADERS,
-  });
-  await sleep(DELAY);
-  const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
-  return "error" in json
-    ? undefined
-    : json.blockArray.find(
-        (x) => x.requirementType === "MAJOR" && x.requirementValue === majorCode,
-      );
-}
+import "dotenv/config";
 
-async function getMinorAudit(catalogYear: string, minorCode: string): Promise<Block | undefined> {
-  const res = await fetch(AUDIT_URL, {
-    method: "POST",
-    body: JSON.stringify({
-      catalogYear,
-      degree: "BA",
-      school: "U",
-      classes: [],
-      goals: [
-        { code: "MAJOR", value: "000" },
-        { code: "MINOR", value: minorCode },
-      ],
-      studentId: process.env["STUDENT_ID"],
-    }),
-    headers: HEADERS,
-  });
-  await sleep(DELAY);
-  const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
-  return "error" in json
-    ? undefined
-    : json.blockArray.find(
-        (x) => x.requirementType === "MINOR" && x.requirementValue === minorCode,
-      );
-}
-
-// async function getSpecAudit(
-//   catalogYear: string,
-//   degree: string,
-//   school: string,
-//   majorCode: string,
-//   specCode: string,
-// ): Promise<Block | undefined> {
-//   const res = await fetch(AUDIT_URL, {
-//     method: "POST",
-//     body: JSON.stringify({
-//       catalogYear,
-//       degree,
-//       school,
-//       classes: [],
-//       goals: [
-//         { code: "MAJOR", value: majorCode },
-//         { code: "SPEC", value: specCode },
-//       ],
-//       studentId: process.env["STUDENT_ID"],
-//     }),
-//     headers: HEADERS,
-//   });
-//   await sleep(DELAY);
-//   const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
-//   return "error" in json
-//     ? undefined
-//     : json.blockArray.find((x) => x.requirementType === "SPEC" && x.requirementValue === specCode);
-// }
-
-async function getMapping<T extends string>(path: T): Promise<Map<string, string>> {
-  const res = await fetch(`${DW_API_URL}/${path}`, { headers: HEADERS });
-  await sleep(DELAY);
-  const json: DWMappingResponse<T> = await res.json();
-  return new Map(json._embedded[path].map((x) => [x.key, x.description]));
-}
+const __dirname = dirname(fileURLToPath(import.meta.url));
 
 async function main() {
-  if (!process.env["STUDENT_ID"]) throw new Error("Student ID not set.");
   if (!process.env["X_AUTH_TOKEN"]) throw new Error("Auth cookie not set.");
+  const studentId = jwtDecode<JwtPayload>(process.env["X_AUTH_TOKEN"].slice(7))?.sub;
+  if (!studentId) throw new Error("Could not parse student ID from auth cookie.");
+  const headers = {
+    "Content-Type": "application/json",
+    Cookie: `X-AUTH-TOKEN=${process.env["X_AUTH_TOKEN"]}`,
+    Origin: "https://reg.uci.edu",
+  };
   console.log("degreeworks-scraper starting");
   const currentYear = new Date().getUTCFullYear();
   /**
@@ -122,16 +33,23 @@ async function main() {
    * B.S. in Computer Science for the latter. If it is available, then we use that
    * as the catalog year. Otherwise, we use the former.
    */
-  const catalogYear = (await getMajorAudit(`${currentYear}${currentYear + 1}`, "BS", "U", "201"))
+  const catalogYear = (await getMajorAudit(
+    `${currentYear}${currentYear + 1}`,
+    "BS",
+    "U",
+    "201",
+    studentId,
+    headers,
+  ))
     ? `${currentYear}${currentYear + 1}`
     : `${currentYear - 1}${currentYear}`;
   console.log(`Set catalogYear to ${catalogYear}`);
 
-  const degrees = await getMapping("degrees");
+  const degrees = await getMapping("degrees", headers);
   console.log(`Fetched ${degrees.size} degrees`);
-  const majorPrograms = await getMapping("majors");
+  const majorPrograms = await getMapping("majors", headers);
   console.log(`Fetched ${majorPrograms.size} major programs`);
-  const minorPrograms = await getMapping("minors");
+  const minorPrograms = await getMapping("minors", headers);
   console.log(`Fetched ${minorPrograms.size} minor programs`);
 
   const undergraduateDegrees = new Set<string>();
@@ -142,7 +60,7 @@ async function main() {
   const minorProgramRequirements = new Map<string, Block>();
   console.log("Scraping minor program requirements");
   for (const minorCode of minorPrograms.keys()) {
-    const audit = await getMinorAudit(catalogYear, minorCode);
+    const audit = await getMinorAudit(catalogYear, minorCode, studentId, headers);
     if (!audit) {
       console.log(`Minor program not found for code ${minorCode}`);
       continue;
diff --git a/tools/degreeworks-scraper/src/lib.ts b/tools/degreeworks-scraper/src/lib.ts
new file mode 100644
index 00000000..92ffaace
--- /dev/null
+++ b/tools/degreeworks-scraper/src/lib.ts
@@ -0,0 +1,109 @@
+import fetch from "cross-fetch";
+
+import type { Block, DWAuditResponse, DWMappingResponse } from "./types";
+
+const DW_API_URL = "https://reg.uci.edu/RespDashboard/api";
+const AUDIT_URL = `${DW_API_URL}/audit`;
+const DELAY = 1000;
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+export async function getMajorAudit(
+  catalogYear: string,
+  degree: string,
+  school: string,
+  majorCode: string,
+  studentId: string,
+  headers: HeadersInit,
+): Promise<Block | undefined> {
+  const res = await fetch(AUDIT_URL, {
+    method: "POST",
+    body: JSON.stringify({
+      catalogYear,
+      degree,
+      school,
+      studentId,
+      classes: [],
+      goals: [{ code: "MAJOR", value: majorCode }],
+    }),
+    headers,
+  });
+  await sleep(DELAY);
+  const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
+  return "error" in json
+    ? undefined
+    : json.blockArray.find(
+        (x) => x.requirementType === "MAJOR" && x.requirementValue === majorCode,
+      );
+}
+
+export async function getMinorAudit(
+  catalogYear: string,
+  minorCode: string,
+  studentId: string,
+  headers: HeadersInit,
+): Promise<Block | undefined> {
+  const res = await fetch(AUDIT_URL, {
+    method: "POST",
+    body: JSON.stringify({
+      catalogYear,
+      studentId,
+      degree: "BA",
+      school: "U",
+      classes: [],
+      goals: [
+        { code: "MAJOR", value: "000" },
+        { code: "MINOR", value: minorCode },
+      ],
+    }),
+    headers,
+  });
+  await sleep(DELAY);
+  const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
+  return "error" in json
+    ? undefined
+    : json.blockArray.find(
+        (x) => x.requirementType === "MINOR" && x.requirementValue === minorCode,
+      );
+}
+
+export async function getSpecAudit(
+  catalogYear: string,
+  degree: string,
+  school: string,
+  majorCode: string,
+  specCode: string,
+  studentId: string,
+  headers: HeadersInit,
+): Promise<Block | undefined> {
+  const res = await fetch(AUDIT_URL, {
+    method: "POST",
+    body: JSON.stringify({
+      catalogYear,
+      degree,
+      school,
+      studentId,
+      classes: [],
+      goals: [
+        { code: "MAJOR", value: majorCode },
+        { code: "SPEC", value: specCode },
+      ],
+    }),
+    headers,
+  });
+  await sleep(DELAY);
+  const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
+  return "error" in json
+    ? undefined
+    : json.blockArray.find((x) => x.requirementType === "SPEC" && x.requirementValue === specCode);
+}
+
+export async function getMapping<T extends string>(
+  path: T,
+  headers: HeadersInit,
+): Promise<Map<string, string>> {
+  const res = await fetch(`${DW_API_URL}/${path}`, { headers });
+  await sleep(DELAY);
+  const json: DWMappingResponse<T> = await res.json();
+  return new Map(json._embedded[path].map((x) => [x.key, x.description]));
+}

From c3a606719257a328af06f4ec99191ce8ec707d8c Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Thu, 17 Aug 2023 13:31:03 -0700
Subject: [PATCH 04/48] chore: :wrench: update ProgramId type

---
 tools/degreeworks-scraper/src/types.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/degreeworks-scraper/src/types.ts b/tools/degreeworks-scraper/src/types.ts
index 331fa311..f3df1a60 100644
--- a/tools/degreeworks-scraper/src/types.ts
+++ b/tools/degreeworks-scraper/src/types.ts
@@ -79,6 +79,7 @@ export type ProgramId = {
   school: "U" | "G";
   programType: "MAJOR" | "MINOR" | "SPEC";
   code: string;
+  degreeType?: string;
 };
 
 export type Program = ProgramId & {

From a42e7ad17100bf8062a3bca08b5ec08f433ac178 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Thu, 17 Aug 2023 14:36:58 -0700
Subject: [PATCH 05/48] feat: :sparkles: normalize course numbers and remove
 excluded courses

---
 tools/degreeworks-scraper/package.json |   4 +-
 tools/degreeworks-scraper/src/index.ts |  30 ++---
 tools/degreeworks-scraper/src/lib.ts   | 169 ++++++++++++++++++++++++-
 tools/degreeworks-scraper/src/parse.ts | 107 ----------------
 tools/degreeworks-scraper/src/types.ts |   6 +-
 5 files changed, 186 insertions(+), 130 deletions(-)
 delete mode 100644 tools/degreeworks-scraper/src/parse.ts

diff --git a/tools/degreeworks-scraper/package.json b/tools/degreeworks-scraper/package.json
index 4b1a4211..ca02fc7a 100644
--- a/tools/degreeworks-scraper/package.json
+++ b/tools/degreeworks-scraper/package.json
@@ -10,10 +10,10 @@
   "dependencies": {
     "cross-fetch": "4.0.0",
     "dotenv": "16.3.1",
-    "jwt-decode": "3.1.2"
+    "jwt-decode": "3.1.2",
+    "peterportal-api-next-types": "workspace:*"
   },
   "devDependencies": {
-    "peterportal-api-next-types": "workspace:*",
     "tsx": "3.12.7"
   }
 }
diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index b40d59a5..9bb6db3d 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -5,8 +5,8 @@ import { fileURLToPath } from "node:url";
 import jwtDecode from "jwt-decode";
 import type { JwtPayload } from "jwt-decode";
 
-import { getMajorAudit, getMapping, getMinorAudit } from "./lib";
-import type { Block } from "./types";
+import { getMajorAudit, getMapping, getMinorAudit, parseBlock } from "./lib";
+import type { Program } from "./types";
 
 import "dotenv/config";
 
@@ -14,8 +14,9 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
 
 async function main() {
   if (!process.env["X_AUTH_TOKEN"]) throw new Error("Auth cookie not set.");
-  const studentId = jwtDecode<JwtPayload>(process.env["X_AUTH_TOKEN"].slice(7))?.sub;
-  if (!studentId) throw new Error("Could not parse student ID from auth cookie.");
+  const studentId = jwtDecode<JwtPayload>(process.env["X_AUTH_TOKEN"].slice("Bearer+".length))?.sub;
+  if (!studentId || studentId.length !== 8)
+    throw new Error("Could not parse student ID from auth cookie.");
   const headers = {
     "Content-Type": "application/json",
     Cookie: `X-AUTH-TOKEN=${process.env["X_AUTH_TOKEN"]}`,
@@ -47,9 +48,9 @@ async function main() {
 
   const degrees = await getMapping("degrees", headers);
   console.log(`Fetched ${degrees.size} degrees`);
-  const majorPrograms = await getMapping("majors", headers);
+  const majorPrograms = new Set((await getMapping("majors", headers)).keys());
   console.log(`Fetched ${majorPrograms.size} major programs`);
-  const minorPrograms = await getMapping("minors", headers);
+  const minorPrograms = new Set((await getMapping("minors", headers)).keys());
   console.log(`Fetched ${minorPrograms.size} minor programs`);
 
   const undergraduateDegrees = new Set<string>();
@@ -57,27 +58,22 @@ async function main() {
   for (const degree of degrees.keys())
     (degree.startsWith("B") ? undergraduateDegrees : graduateDegrees).add(degree);
 
-  const minorProgramRequirements = new Map<string, Block>();
+  const parsedMinorPrograms = new Map<string, Program>();
   console.log("Scraping minor program requirements");
-  for (const minorCode of minorPrograms.keys()) {
+  for (const minorCode of minorPrograms) {
     const audit = await getMinorAudit(catalogYear, minorCode, studentId, headers);
     if (!audit) {
       console.log(`Minor program not found for code ${minorCode}`);
       continue;
     }
     console.log(`Requirements block for "${audit.title}" found for code ${minorCode}`);
-    minorProgramRequirements.set(`U-MINOR-${minorCode}`, {
-      requirementType: audit.requirementType,
-      requirementValue: audit.requirementValue,
-      title: audit.title,
-      ruleArray: [...audit.ruleArray],
-    });
+    parsedMinorPrograms.set(`U-MINOR-${minorCode}`, await parseBlock(audit));
   }
   await mkdir(join(__dirname, "../output"), { recursive: true });
   await writeFile(
-    join(__dirname, "../output/minorProgramRequirements.json"),
-    JSON.stringify(Object.fromEntries(minorProgramRequirements.entries())),
+    join(__dirname, "../output/parsedMinorPrograms.json"),
+    JSON.stringify(Object.fromEntries(parsedMinorPrograms.entries())),
   );
 }
 
-main().then(() => []);
+main().then();
diff --git a/tools/degreeworks-scraper/src/lib.ts b/tools/degreeworks-scraper/src/lib.ts
index 92ffaace..7a737c94 100644
--- a/tools/degreeworks-scraper/src/lib.ts
+++ b/tools/degreeworks-scraper/src/lib.ts
@@ -1,13 +1,175 @@
 import fetch from "cross-fetch";
+import { isErrorResponse } from "peterportal-api-next-types";
+import type { Course, RawResponse } from "peterportal-api-next-types";
 
-import type { Block, DWAuditResponse, DWMappingResponse } from "./types";
+import type {
+  Block,
+  DWAuditResponse,
+  DWMappingResponse,
+  Program,
+  ProgramId,
+  Requirement,
+  Rule,
+} from "./types";
 
+const PPAPI_REST_URL = "https://api-next.peterportal.org/v1/rest";
 const DW_API_URL = "https://reg.uci.edu/RespDashboard/api";
 const AUDIT_URL = `${DW_API_URL}/audit`;
 const DELAY = 1000;
 
+const electiveMatcher = /ELECTIVE @+/;
+const wildcardMatcher = /\d+@+/;
+const rangeMatcher = /\d+-\d+/;
+
 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
 
+const parseSpecs = (ruleArray: Rule[]) =>
+  ruleArray
+    .filter((x) => x.ruleType === "IfStmt")
+    .flatMap((x) => ifStmtToSpecArray([x]))
+    .sort();
+
+function ifStmtToSpecArray(ruleArray: Rule[]): string[] {
+  const ret = [];
+  for (const rule of ruleArray) {
+    switch (rule.ruleType) {
+      case "IfStmt":
+        ret.push(
+          ...ifStmtToSpecArray(rule.requirement.ifPart.ruleArray),
+          ...ifStmtToSpecArray(rule.requirement.elsePart?.ruleArray ?? []),
+        );
+        break;
+      case "Block":
+        ret.push(rule.requirement.value);
+        break;
+    }
+  }
+  return ret;
+}
+
+function flattenIfStmt(ruleArray: Rule[]): Rule[] {
+  const ret = [];
+  for (const rule of ruleArray) {
+    switch (rule.ruleType) {
+      case "IfStmt":
+        ret.push(
+          ...flattenIfStmt(rule.requirement.ifPart.ruleArray),
+          ...flattenIfStmt(rule.requirement.elsePart?.ruleArray ?? []),
+        );
+        break;
+      default:
+        ret.push(rule);
+    }
+  }
+  return ret;
+}
+
+async function getCourseId(courseNumber: string): Promise<string | undefined> {
+  const res = await fetch(`${PPAPI_REST_URL}/courses/${courseNumber}`);
+  const json: RawResponse<Course> = await res.json();
+  return isErrorResponse(json) ? undefined : json.payload.id;
+}
+
+async function getCourseIds(
+  department: string,
+  predicate: (x: Course) => boolean,
+): Promise<string[] | undefined> {
+  const res = await fetch(`${PPAPI_REST_URL}/courses/?department=${department}`);
+  const json: RawResponse<Course[]> = await res.json();
+  return isErrorResponse(json) ? undefined : json.payload.filter(predicate).map((x) => x.id);
+}
+
+async function normalizeCourseId(courseIdLike: string): Promise<string[]> {
+  // "ELECTIVE @" is typically used as a pseudo-course and can be safely ignored.
+  if (courseIdLike.match(electiveMatcher)) return [];
+  const [department, courseNumber] = courseIdLike.split(" ");
+  if (courseNumber.match(wildcardMatcher)) {
+    // Wildcard course numbers.
+    const courseIds = await getCourseIds(
+      department,
+      (x) => !!x.courseNumber.match(new RegExp(courseNumber.replace(/@/g, "."))),
+    );
+    return courseIds ? courseIds : [];
+  }
+  if (courseNumber.match(rangeMatcher)) {
+    // Course number ranges.
+    const [minCourseNumber, maxCourseNumber] = courseNumber.split("-");
+    const courseIds = await getCourseIds(
+      department,
+      (x) =>
+        x.courseNumeric >= Number.parseInt(minCourseNumber, 10) &&
+        x.courseNumeric <= Number.parseInt(maxCourseNumber, 10),
+    );
+    return courseIds ? courseIds : [];
+  }
+  // Probably a normal course, just make sure that it exists.
+  const courseId = await getCourseId(`${department}${courseNumber}`);
+  return courseId ? [courseId] : [];
+}
+
+async function ruleArrayToRequirements(ruleArray: Rule[]) {
+  const ret: Record<string, Requirement> = {};
+  for (const rule of ruleArray) {
+    switch (rule.ruleType) {
+      case "Course": {
+        const includedCourses = await Promise.all(
+          rule.requirement.courseArray.map((x) =>
+            normalizeCourseId(`${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`),
+          ),
+        );
+        const excludedCourses = await Promise.all(
+          rule.requirement.except?.courseArray.map((x) =>
+            normalizeCourseId(`${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`),
+          ) ?? [],
+        );
+        const toExclude = new Set(excludedCourses.flat());
+        const courses = Array.from(includedCourses.flat())
+          .filter((x) => !toExclude.has(x))
+          .sort();
+        if (rule.requirement.classesBegin) {
+          ret[rule.label] = {
+            requirementType: "Course",
+            courseCount: Number.parseInt(rule.requirement.classesBegin, 10),
+            courses,
+          };
+        } else if (rule.requirement.creditsBegin) {
+          ret[rule.label] = {
+            requirementType: "Unit",
+            unitCount: Number.parseInt(rule.requirement.creditsBegin, 10),
+            courses,
+          };
+        }
+        break;
+      }
+      case "Group":
+        ret[rule.label] = {
+          requirementType: "Group",
+          requirementCount: Number.parseInt(rule.requirement.numberOfGroups),
+          requirements: await ruleArrayToRequirements(rule.ruleArray),
+        };
+        break;
+      case "IfStmt": {
+        const rules = flattenIfStmt([rule]);
+        if (rules.length > 1 && !rules.find((x) => x.ruleType === "Block")) {
+          ret["Select 1 of the following"] = {
+            requirementType: "Group",
+            requirementCount: 1,
+            requirements: await ruleArrayToRequirements(rules),
+          };
+        }
+        break;
+      }
+    }
+  }
+  return ret;
+}
+
+export const parseBlock = async (block: Block): Promise<Program> => ({
+  name: block.title,
+  requirements: await ruleArrayToRequirements(block.ruleArray),
+  specs: parseSpecs(block.ruleArray),
+});
+
 export async function getMajorAudit(
   catalogYear: string,
   degree: string,
@@ -107,3 +269,8 @@ export async function getMapping<T extends string>(
   const json: DWMappingResponse<T> = await res.json();
   return new Map(json._embedded[path].map((x) => [x.key, x.description]));
 }
+
+export function parseBlockId(blockId: string) {
+  const [school, programType, code, degreeType] = blockId.split("-");
+  return { school, programType, code, degreeType } as ProgramId;
+}
diff --git a/tools/degreeworks-scraper/src/parse.ts b/tools/degreeworks-scraper/src/parse.ts
deleted file mode 100644
index 339857e7..00000000
--- a/tools/degreeworks-scraper/src/parse.ts
+++ /dev/null
@@ -1,107 +0,0 @@
-import type { Block, Program, ProgramId, Requirement, Rule } from "./types";
-
-const parseSpecs = (ruleArray: Rule[]) =>
-  ruleArray
-    .filter((x) => x.ruleType === "IfStmt")
-    .flatMap((x) => ifStmtToSpecArray([x]))
-    .sort();
-
-function ifStmtToSpecArray(ruleArray: Rule[]): string[] {
-  const ret = [];
-  for (const rule of ruleArray) {
-    switch (rule.ruleType) {
-      case "IfStmt":
-        ret.push(
-          ...ifStmtToSpecArray(rule.requirement.ifPart.ruleArray),
-          ...ifStmtToSpecArray(rule.requirement.elsePart?.ruleArray ?? []),
-        );
-        break;
-      case "Block":
-        ret.push(rule.requirement.value);
-        break;
-    }
-  }
-  return ret;
-}
-
-function flattenIfStmt(ruleArray: Rule[]): Rule[] {
-  const ret = [];
-  for (const rule of ruleArray) {
-    switch (rule.ruleType) {
-      case "IfStmt":
-        ret.push(
-          ...flattenIfStmt(rule.requirement.ifPart.ruleArray),
-          ...flattenIfStmt(rule.requirement.elsePart?.ruleArray ?? []),
-        );
-        break;
-      default:
-        ret.push(rule);
-    }
-  }
-  return ret;
-}
-
-function ruleArrayToRequirements(ruleArray: Rule[]) {
-  const ret: Record<string, Requirement> = {};
-  for (const rule of ruleArray) {
-    switch (rule.ruleType) {
-      case "Course": {
-        const courses = {
-          include: rule.requirement.courseArray.map(
-            (x) => `${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`,
-          ),
-          ...(rule.requirement.except?.courseArray && {
-            exclude: rule.requirement.except.courseArray.map(
-              (x) => `${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`,
-            ),
-          }),
-        };
-        if (rule.requirement.classesBegin) {
-          ret[rule.label] = {
-            requirementType: "Course",
-            courseCount: Number.parseInt(rule.requirement.classesBegin, 10),
-            courses,
-          };
-        } else if (rule.requirement.creditsBegin) {
-          ret[rule.label] = {
-            requirementType: "Unit",
-            unitCount: Number.parseInt(rule.requirement.creditsBegin, 10),
-            courses,
-          };
-        }
-        break;
-      }
-      case "Group":
-        ret[rule.label] = {
-          requirementType: "Group",
-          requirementCount: Number.parseInt(rule.requirement.numberOfGroups),
-          requirements: ruleArrayToRequirements(rule.ruleArray),
-        };
-        break;
-      case "IfStmt": {
-        const rules = flattenIfStmt([rule]);
-        if (rules.length > 1 && !rules.find((x) => x.ruleType === "Block")) {
-          ret["Select 1 of the following"] = {
-            requirementType: "Group",
-            requirementCount: 1,
-            requirements: ruleArrayToRequirements(rules),
-          };
-        }
-        break;
-      }
-    }
-  }
-  return ret;
-}
-
-function parseBlockId(blockId: string) {
-  const [school, programType, code, degreeType] = blockId.split("-");
-  return { school, programType, code, degreeType } as ProgramId;
-}
-
-export const parseBlock = (blockId: string, block: Block): Program => ({
-  ...parseBlockId(blockId),
-  name: block.title,
-  requirements: ruleArrayToRequirements(block.ruleArray),
-  specs: parseSpecs(block.ruleArray),
-});
diff --git a/tools/degreeworks-scraper/src/types.ts b/tools/degreeworks-scraper/src/types.ts
index f3df1a60..c5caadd9 100644
--- a/tools/degreeworks-scraper/src/types.ts
+++ b/tools/degreeworks-scraper/src/types.ts
@@ -82,7 +82,7 @@ export type ProgramId = {
   degreeType?: string;
 };
 
-export type Program = ProgramId & {
+export type Program = {
   /**
    * The display name of the program.
    * @example "Major in Computer Science"
@@ -100,7 +100,7 @@ export type CourseRequirement = {
    * The number of `courses` required to fulfill this requirement.
    */
   courseCount: number;
-  courses: { include: string[]; exclude?: string[] };
+  courses: string[];
 };
 
 export type UnitRequirement = {
@@ -109,7 +109,7 @@ export type UnitRequirement = {
    * The number of units earned from taking `courses` that are required to fulfill this requirement.
    */
   unitCount: number;
-  courses: { include: string[]; exclude?: string[] };
+  courses: string[];
 };
 
 export type GroupRequirement = {

From 14344fea03c37cae75f39de85e3db809677d6019 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Thu, 17 Aug 2023 15:56:05 -0700
Subject: [PATCH 06/48] fix: :bug: ratelimit when dogfooding

---
 tools/degreeworks-scraper/src/index.ts |  4 +-
 tools/degreeworks-scraper/src/lib.ts   | 57 ++++++++++++++++----------
 2 files changed, 37 insertions(+), 24 deletions(-)

diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index 9bb6db3d..650832bb 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -63,11 +63,11 @@ async function main() {
   for (const minorCode of minorPrograms) {
     const audit = await getMinorAudit(catalogYear, minorCode, studentId, headers);
     if (!audit) {
-      console.log(`Minor program not found for code ${minorCode}`);
+      console.log(`Requirements block not found for minor program with code ${minorCode}`);
       continue;
     }
-    console.log(`Requirements block for "${audit.title}" found for code ${minorCode}`);
     parsedMinorPrograms.set(`U-MINOR-${minorCode}`, await parseBlock(audit));
+    console.log(`Requirements block found and parsed for "${audit.title}" (code ${minorCode})`);
   }
   await mkdir(join(__dirname, "../output"), { recursive: true });
   await writeFile(
diff --git a/tools/degreeworks-scraper/src/lib.ts b/tools/degreeworks-scraper/src/lib.ts
index 7a737c94..e2cf8340 100644
--- a/tools/degreeworks-scraper/src/lib.ts
+++ b/tools/degreeworks-scraper/src/lib.ts
@@ -21,6 +21,8 @@ const electiveMatcher = /ELECTIVE @+/;
 const wildcardMatcher = /\d+@+/;
 const rangeMatcher = /\d+-\d+/;
 
+const lexOrd = new Intl.Collator().compare;
+
 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
 
 const parseSpecs = (ruleArray: Rule[]) =>
@@ -64,28 +66,30 @@ function flattenIfStmt(ruleArray: Rule[]): Rule[] {
   return ret;
 }
 
-async function getCourseId(courseNumber: string): Promise<string | undefined> {
+async function getCourse(courseNumber: string): Promise<Course | undefined> {
   const res = await fetch(`${PPAPI_REST_URL}/courses/${courseNumber}`);
+  await sleep(DELAY);
   const json: RawResponse<Course> = await res.json();
-  return isErrorResponse(json) ? undefined : json.payload.id;
+  return isErrorResponse(json) ? undefined : json.payload;
 }
 
-async function getCourseIds(
+async function getCourses(
   department: string,
   predicate: (x: Course) => boolean,
-): Promise<string[] | undefined> {
+): Promise<Course[] | undefined> {
   const res = await fetch(`${PPAPI_REST_URL}/courses/?department=${department}`);
+  await sleep(DELAY);
   const json: RawResponse<Course[]> = await res.json();
-  return isErrorResponse(json) ? undefined : json.payload.filter(predicate).map((x) => x.id);
+  return isErrorResponse(json) ? undefined : json.payload.filter(predicate);
 }
 
-async function normalizeCourseId(courseIdLike: string): Promise<string[]> {
+async function normalizeCourseId(courseIdLike: string): Promise<Course[]> {
   // "ELECTIVE @" is typically used as a pseudo-course and can be safely ignored.
   if (courseIdLike.match(electiveMatcher)) return [];
   const [department, courseNumber] = courseIdLike.split(" ");
   if (courseNumber.match(wildcardMatcher)) {
     // Wildcard course numbers.
-    const courseIds = await getCourseIds(
+    const courseIds = await getCourses(
       department,
       (x) => !!x.courseNumber.match(new RegExp(courseNumber.replace(/@/g, "."))),
     );
@@ -94,7 +98,7 @@ async function normalizeCourseId(courseIdLike: string): Promise<string[]> {
   if (courseNumber.match(rangeMatcher)) {
     // Course number ranges.
     const [minCourseNumber, maxCourseNumber] = courseNumber.split("-");
-    const courseIds = await getCourseIds(
+    const courseIds = await getCourses(
       department,
       (x) =>
         x.courseNumeric >= Number.parseInt(minCourseNumber, 10) &&
@@ -103,7 +107,7 @@ async function normalizeCourseId(courseIdLike: string): Promise<string[]> {
     return courseIds ? courseIds : [];
   }
   // Probably a normal course, just make sure that it exists.
-  const courseId = await getCourseId(`${department}${courseNumber}`);
+  const courseId = await getCourse(`${department}${courseNumber}`);
   return courseId ? [courseId] : [];
 }
 
@@ -112,20 +116,29 @@ async function ruleArrayToRequirements(ruleArray: Rule[]) {
   for (const rule of ruleArray) {
     switch (rule.ruleType) {
       case "Course": {
-        const includedCourses = await Promise.all(
-          rule.requirement.courseArray.map((x) =>
-            normalizeCourseId(`${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`),
-          ),
+        const includedCourses = rule.requirement.courseArray.map(
+          (x) => `${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`,
         );
-        const excludedCourses = await Promise.all(
-          rule.requirement.except?.courseArray.map((x) =>
-            normalizeCourseId(`${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`),
-          ) ?? [],
-        );
-        const toExclude = new Set(excludedCourses.flat());
-        const courses = Array.from(includedCourses.flat())
-          .filter((x) => !toExclude.has(x))
-          .sort();
+        const toInclude = new Set<Course>();
+        for (const id of includedCourses) {
+          (await normalizeCourseId(id)).forEach((x) => toInclude.add(x));
+        }
+        const excludedCourses =
+          rule.requirement.except?.courseArray.map(
+            (x) => `${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`,
+          ) ?? [];
+        const toExclude = new Set<string>();
+        for (const id of excludedCourses) {
+          (await normalizeCourseId(id)).map((x) => x.id).forEach((x) => toExclude.add(x));
+        }
+        const courses = Array.from(toInclude)
+          .filter((x) => !toExclude.has(x.id))
+          .sort((a, b) =>
+            a.department === b.department
+              ? a.courseNumeric - b.courseNumeric
+              : lexOrd(a.department, b.department),
+          )
+          .map((x) => x.id);
         if (rule.requirement.classesBegin) {
           ret[rule.label] = {
             requirementType: "Course",

From 9b76eae695e734a070eaf5d6735073162fd41dcc Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Sat, 19 Aug 2023 18:09:05 -0700
Subject: [PATCH 07/48] feat: :sparkles: add separate class for degreeworks
 requests

---
 .../src/DegreeworksClient.ts                  | 115 +++++++++++++
 tools/degreeworks-scraper/src/index.ts        |  42 ++---
 tools/degreeworks-scraper/src/lib.ts          | 161 ++++--------------
 tools/degreeworks-scraper/src/types.ts        |  11 +-
 4 files changed, 158 insertions(+), 171 deletions(-)
 create mode 100644 tools/degreeworks-scraper/src/DegreeworksClient.ts

diff --git a/tools/degreeworks-scraper/src/DegreeworksClient.ts b/tools/degreeworks-scraper/src/DegreeworksClient.ts
new file mode 100644
index 00000000..dac4adc2
--- /dev/null
+++ b/tools/degreeworks-scraper/src/DegreeworksClient.ts
@@ -0,0 +1,115 @@
+import fetch from "cross-fetch";
+
+import { sleep } from "./lib";
+import type { Block, DWAuditResponse, DWMappingResponse } from "./types";
+
+export class DegreeworksClient {
+  private static readonly API_URL = "https://reg.uci.edu/RespDashboard/api";
+  private static readonly AUDIT_URL = `${DegreeworksClient.API_URL}/audit`;
+  private catalogYear: string = "";
+  constructor(
+    private readonly studentId: string,
+    private readonly headers: HeadersInit,
+    private readonly delay: number = 1000,
+  ) {
+    /**
+     * Depending on when we are scraping, this may be the academic year that started
+     * the previous calendar year, or the one that will start this calendar year.
+     *
+     * We determine the catalog year by seeing if we can fetch the major data for the
+     * B.S. in Computer Science for the latter. If it is available, then we use that
+     * as the catalog year. Otherwise, we use the former.
+     */
+    const currentYear = new Date().getUTCFullYear();
+    this.getMajorAudit("BS", "U", "201").then((x) => {
+      this.catalogYear = x
+        ? `${currentYear}${currentYear + 1}`
+        : `${currentYear - 1}${currentYear}`;
+      console.log(`[DegreeworksClient] Set catalogYear to ${this.catalogYear}`);
+    });
+  }
+  async getMajorAudit(
+    degree: string,
+    school: string,
+    majorCode: string,
+  ): Promise<Block | undefined> {
+    const res = await fetch(DegreeworksClient.AUDIT_URL, {
+      method: "POST",
+      body: JSON.stringify({
+        catalogYear: this.catalogYear,
+        degree,
+        school,
+        studentId: this.studentId,
+        classes: [],
+        goals: [{ code: "MAJOR", value: majorCode }],
+      }),
+      headers: this.headers,
+    });
+    const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
+    return "error" in json
+      ? undefined
+      : json.blockArray.find(
+          (x) => x.requirementType === "MAJOR" && x.requirementValue === majorCode,
+        );
+  }
+
+  async getMinorAudit(minorCode: string): Promise<Block | undefined> {
+    const res = await fetch(DegreeworksClient.AUDIT_URL, {
+      method: "POST",
+      body: JSON.stringify({
+        catalogYear: this.catalogYear,
+        studentId: this.studentId,
+        degree: "BA",
+        school: "U",
+        classes: [],
+        goals: [
+          { code: "MAJOR", value: "000" },
+          { code: "MINOR", value: minorCode },
+        ],
+      }),
+      headers: this.headers,
+    });
+    const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
+    return "error" in json
+      ? undefined
+      : json.blockArray.find(
+          (x) => x.requirementType === "MINOR" && x.requirementValue === minorCode,
+        );
+  }
+
+  async getSpecAudit(
+    degree: string,
+    school: string,
+    majorCode: string,
+    specCode: string,
+  ): Promise<Block | undefined> {
+    const res = await fetch(DegreeworksClient.AUDIT_URL, {
+      method: "POST",
+      body: JSON.stringify({
+        catalogYear: this.catalogYear,
+        degree,
+        school,
+        studentId: this.studentId,
+        classes: [],
+        goals: [
+          { code: "MAJOR", value: majorCode },
+          { code: "SPEC", value: specCode },
+        ],
+      }),
+      headers: this.headers,
+    });
+    const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
+    return "error" in json
+      ? undefined
+      : json.blockArray.find(
+          (x) => x.requirementType === "SPEC" && x.requirementValue === specCode,
+        );
+  }
+
+  async getMapping<T extends string>(path: T): Promise<Map<string, string>> {
+    const res = await fetch(`${DegreeworksClient.API_URL}/${path}`, { headers: this.headers });
+    await sleep(this.delay);
+    const json: DWMappingResponse<T> = await res.json();
+    return new Map(json._embedded[path].map((x) => [x.key, x.description]));
+  }
+}
diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index 650832bb..15140635 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -5,7 +5,8 @@ import { fileURLToPath } from "node:url";
 import jwtDecode from "jwt-decode";
 import type { JwtPayload } from "jwt-decode";
 
-import { getMajorAudit, getMapping, getMinorAudit, parseBlock } from "./lib";
+import { DegreeworksClient } from "./DegreeworksClient";
+import { parseBlock, sleep } from "./lib";
 import type { Program } from "./types";
 
 import "dotenv/config";
@@ -23,34 +24,12 @@ async function main() {
     Origin: "https://reg.uci.edu",
   };
   console.log("degreeworks-scraper starting");
-  const currentYear = new Date().getUTCFullYear();
-  /**
-   * The current catalog year.
-   *
-   * Depending on when we are scraping, this may be the academic year that started
-   * the previous calendar year, or the one that will start this calendar year.
-   *
-   * We determine the catalog year by seeing if we can fetch the major data for the
-   * B.S. in Computer Science for the latter. If it is available, then we use that
-   * as the catalog year. Otherwise, we use the former.
-   */
-  const catalogYear = (await getMajorAudit(
-    `${currentYear}${currentYear + 1}`,
-    "BS",
-    "U",
-    "201",
-    studentId,
-    headers,
-  ))
-    ? `${currentYear}${currentYear + 1}`
-    : `${currentYear - 1}${currentYear}`;
-  console.log(`Set catalogYear to ${catalogYear}`);
-
-  const degrees = await getMapping("degrees", headers);
+  const dw = new DegreeworksClient(studentId, headers);
+  const degrees = await dw.getMapping("degrees");
   console.log(`Fetched ${degrees.size} degrees`);
-  const majorPrograms = new Set((await getMapping("majors", headers)).keys());
+  const majorPrograms = new Set((await dw.getMapping("majors")).keys());
   console.log(`Fetched ${majorPrograms.size} major programs`);
-  const minorPrograms = new Set((await getMapping("minors", headers)).keys());
+  const minorPrograms = new Set((await dw.getMapping("minors")).keys());
   console.log(`Fetched ${minorPrograms.size} minor programs`);
 
   const undergraduateDegrees = new Set<string>();
@@ -61,13 +40,16 @@ async function main() {
   const parsedMinorPrograms = new Map<string, Program>();
   console.log("Scraping minor program requirements");
   for (const minorCode of minorPrograms) {
-    const audit = await getMinorAudit(catalogYear, minorCode, studentId, headers);
+    const audit = await dw.getMinorAudit(minorCode);
     if (!audit) {
-      console.log(`Requirements block not found for minor program with code ${minorCode}`);
+      console.log(`Requirements block not found (minorCode = ${minorCode})`);
+      await sleep(1000);
       continue;
     }
     parsedMinorPrograms.set(`U-MINOR-${minorCode}`, await parseBlock(audit));
-    console.log(`Requirements block found and parsed for "${audit.title}" (code ${minorCode})`);
+    console.log(
+      `Requirements block found and parsed for "${audit.title}" (minorCode = ${minorCode})`,
+    );
   }
   await mkdir(join(__dirname, "../output"), { recursive: true });
   await writeFile(
diff --git a/tools/degreeworks-scraper/src/lib.ts b/tools/degreeworks-scraper/src/lib.ts
index e2cf8340..eb1ad20a 100644
--- a/tools/degreeworks-scraper/src/lib.ts
+++ b/tools/degreeworks-scraper/src/lib.ts
@@ -2,28 +2,24 @@ import fetch from "cross-fetch";
 import { isErrorResponse } from "peterportal-api-next-types";
 import type { Course, RawResponse } from "peterportal-api-next-types";
 
-import type {
-  Block,
-  DWAuditResponse,
-  DWMappingResponse,
-  Program,
-  ProgramId,
-  Requirement,
-  Rule,
-} from "./types";
+import type { Block, Program, Requirement, Rule } from "./types";
+import { ProgramId } from "./types";
 
 const PPAPI_REST_URL = "https://api-next.peterportal.org/v1/rest";
-const DW_API_URL = "https://reg.uci.edu/RespDashboard/api";
-const AUDIT_URL = `${DW_API_URL}/audit`;
-const DELAY = 1000;
 
 const electiveMatcher = /ELECTIVE @+/;
 const wildcardMatcher = /\d+@+/;
 const rangeMatcher = /\d+-\d+/;
 
-const lexOrd = new Intl.Collator().compare;
+export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+export const parseBlock = async (block: Block): Promise<Program> => ({
+  name: block.title,
+  requirements: await ruleArrayToRequirements(block.ruleArray),
+  specs: parseSpecs(block.ruleArray),
+});
 
-const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+const lexOrd = new Intl.Collator().compare;
 
 const parseSpecs = (ruleArray: Rule[]) =>
   ruleArray
@@ -68,17 +64,17 @@ function flattenIfStmt(ruleArray: Rule[]): Rule[] {
 
 async function getCourse(courseNumber: string): Promise<Course | undefined> {
   const res = await fetch(`${PPAPI_REST_URL}/courses/${courseNumber}`);
-  await sleep(DELAY);
+  await sleep(1000);
   const json: RawResponse<Course> = await res.json();
   return isErrorResponse(json) ? undefined : json.payload;
 }
 
 async function getCourses(
   department: string,
-  predicate: (x: Course) => boolean,
+  predicate: (x: Course) => boolean = () => true,
 ): Promise<Course[] | undefined> {
   const res = await fetch(`${PPAPI_REST_URL}/courses/?department=${department}`);
-  await sleep(DELAY);
+  await sleep(1000);
   const json: RawResponse<Course[]> = await res.json();
   return isErrorResponse(json) ? undefined : json.payload.filter(predicate);
 }
@@ -89,26 +85,35 @@ async function normalizeCourseId(courseIdLike: string): Promise<Course[]> {
   const [department, courseNumber] = courseIdLike.split(" ");
   if (courseNumber.match(wildcardMatcher)) {
     // Wildcard course numbers.
-    const courseIds = await getCourses(
+    const courses = await getCourses(
       department,
-      (x) => !!x.courseNumber.match(new RegExp(courseNumber.replace(/@/g, "."))),
+      (x) =>
+        !!x.courseNumber.match(
+          new RegExp(
+            "^" +
+              courseNumber.replace(
+                /@+/g,
+                `.{${[...courseNumber].filter((y) => y === "@").length},}`,
+              ),
+          ),
+        ),
     );
-    return courseIds ? courseIds : [];
+    return courses ?? [];
   }
   if (courseNumber.match(rangeMatcher)) {
     // Course number ranges.
     const [minCourseNumber, maxCourseNumber] = courseNumber.split("-");
-    const courseIds = await getCourses(
+    const courses = await getCourses(
       department,
       (x) =>
         x.courseNumeric >= Number.parseInt(minCourseNumber, 10) &&
         x.courseNumeric <= Number.parseInt(maxCourseNumber, 10),
     );
-    return courseIds ? courseIds : [];
+    return courses ?? [];
   }
   // Probably a normal course, just make sure that it exists.
-  const courseId = await getCourse(`${department}${courseNumber}`);
-  return courseId ? [courseId] : [];
+  const course = await getCourse(`${department}${courseNumber}`);
+  return course ? [course] : [];
 }
 
 async function ruleArrayToRequirements(ruleArray: Rule[]) {
@@ -163,7 +168,7 @@ async function ruleArrayToRequirements(ruleArray: Rule[]) {
         break;
       case "IfStmt": {
         const rules = flattenIfStmt([rule]);
-        if (rules.length > 1 && !rules.find((x) => x.ruleType === "Block")) {
+        if (rules.length > 1 && !rules.some((x) => x.ruleType === "Block")) {
           ret["Select 1 of the following"] = {
             requirementType: "Group",
             requirementCount: 1,
@@ -177,112 +182,6 @@ async function ruleArrayToRequirements(ruleArray: Rule[]) {
   return ret;
 }
 
-export const parseBlock = async (block: Block): Promise<Program> => ({
-  name: block.title,
-  requirements: await ruleArrayToRequirements(block.ruleArray),
-  specs: parseSpecs(block.ruleArray),
-});
-
-export async function getMajorAudit(
-  catalogYear: string,
-  degree: string,
-  school: string,
-  majorCode: string,
-  studentId: string,
-  headers: HeadersInit,
-): Promise<Block | undefined> {
-  const res = await fetch(AUDIT_URL, {
-    method: "POST",
-    body: JSON.stringify({
-      catalogYear,
-      degree,
-      school,
-      studentId,
-      classes: [],
-      goals: [{ code: "MAJOR", value: majorCode }],
-    }),
-    headers,
-  });
-  await sleep(DELAY);
-  const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
-  return "error" in json
-    ? undefined
-    : json.blockArray.find(
-        (x) => x.requirementType === "MAJOR" && x.requirementValue === majorCode,
-      );
-}
-
-export async function getMinorAudit(
-  catalogYear: string,
-  minorCode: string,
-  studentId: string,
-  headers: HeadersInit,
-): Promise<Block | undefined> {
-  const res = await fetch(AUDIT_URL, {
-    method: "POST",
-    body: JSON.stringify({
-      catalogYear,
-      studentId,
-      degree: "BA",
-      school: "U",
-      classes: [],
-      goals: [
-        { code: "MAJOR", value: "000" },
-        { code: "MINOR", value: minorCode },
-      ],
-    }),
-    headers,
-  });
-  await sleep(DELAY);
-  const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
-  return "error" in json
-    ? undefined
-    : json.blockArray.find(
-        (x) => x.requirementType === "MINOR" && x.requirementValue === minorCode,
-      );
-}
-
-export async function getSpecAudit(
-  catalogYear: string,
-  degree: string,
-  school: string,
-  majorCode: string,
-  specCode: string,
-  studentId: string,
-  headers: HeadersInit,
-): Promise<Block | undefined> {
-  const res = await fetch(AUDIT_URL, {
-    method: "POST",
-    body: JSON.stringify({
-      catalogYear,
-      degree,
-      school,
-      studentId,
-      classes: [],
-      goals: [
-        { code: "MAJOR", value: majorCode },
-        { code: "SPEC", value: specCode },
-      ],
-    }),
-    headers,
-  });
-  await sleep(DELAY);
-  const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
-  return "error" in json
-    ? undefined
-    : json.blockArray.find((x) => x.requirementType === "SPEC" && x.requirementValue === specCode);
-}
-
-export async function getMapping<T extends string>(
-  path: T,
-  headers: HeadersInit,
-): Promise<Map<string, string>> {
-  const res = await fetch(`${DW_API_URL}/${path}`, { headers });
-  await sleep(DELAY);
-  const json: DWMappingResponse<T> = await res.json();
-  return new Map(json._embedded[path].map((x) => [x.key, x.description]));
-}
-
 export function parseBlockId(blockId: string) {
   const [school, programType, code, degreeType] = blockId.split("-");
   return { school, programType, code, degreeType } as ProgramId;
diff --git a/tools/degreeworks-scraper/src/types.ts b/tools/degreeworks-scraper/src/types.ts
index c5caadd9..7ef048bd 100644
--- a/tools/degreeworks-scraper/src/types.ts
+++ b/tools/degreeworks-scraper/src/types.ts
@@ -121,13 +121,4 @@ export type GroupRequirement = {
   requirements: Record<string, Requirement>;
 };
 
-export type IfStmtRequirement = {
-  requirementType: "IfStmt";
-  rules: Rule[];
-};
-
-export type Requirement =
-  | CourseRequirement
-  | UnitRequirement
-  | GroupRequirement
-  | IfStmtRequirement;
+export type Requirement = CourseRequirement | UnitRequirement | GroupRequirement;

From 998afe17b713f53fafb3883811198bfba0129ea6 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Sat, 19 Aug 2023 19:09:46 -0700
Subject: [PATCH 08/48] chore(deps): :link: update lockfile

---
 pnpm-lock.yaml | 944 +++++++++++++++++++++++++------------------------
 1 file changed, 474 insertions(+), 470 deletions(-)

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 93ac1e50..ac2a26d1 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -609,6 +609,19 @@ importers:
       cross-fetch:
         specifier: 4.0.0
         version: 4.0.0
+      dotenv:
+        specifier: 16.3.1
+        version: 16.3.1
+      jwt-decode:
+        specifier: 3.1.2
+        version: 3.1.2
+      peterportal-api-next-types:
+        specifier: workspace:*
+        version: link:../../packages/peterportal-api-next-types
+    devDependencies:
+      tsx:
+        specifier: 3.12.7
+        version: 3.12.7
 
   tools/grades-updater:
     dependencies:
@@ -844,8 +857,8 @@ packages:
       '@jridgewell/gen-mapping': 0.3.3
       '@jridgewell/trace-mapping': 0.3.18
 
-  /@antfu/utils@0.7.3:
-    resolution: {integrity: sha512-sAPXKvlJFVQB4cvmdGoUa9IAavzRrm7N2ctxdD1GuAEIOZu8BRrv2SUzquGXTpRDUa0sY7OkkVHqhi6ySMnMTg==}
+  /@antfu/utils@0.7.6:
+    resolution: {integrity: sha512-pvFiLP2BeOKA/ZOS6jxx4XhKzdVLHDhGlFEaZ2flWWYf2xOqVniqpk38I04DFRyz+L0ASggl7SkItTc+ZLju4w==}
     dev: false
 
   /@ap0nia/camaro@6.2.5:
@@ -1057,20 +1070,20 @@ packages:
     engines: {node: '>=14'}
     dev: false
 
-  /@aws-cdk/asset-awscli-v1@2.2.184:
-    resolution: {integrity: sha512-03q3Pm/IFEJEA4QS1GH87LwU4YhN1nuvA986k7KtaMIMPTOt/YXpUsriw/Sx2XcTpUk419sPGewr5N0D2slDCg==}
+  /@aws-cdk/asset-awscli-v1@2.2.200:
+    resolution: {integrity: sha512-Kf5J8DfJK4wZFWT2Myca0lhwke7LwHcHBo+4TvWOGJrFVVKVuuiLCkzPPRBQQVDj0Vtn2NBokZAz8pfMpAqAKg==}
 
-  /@aws-cdk/asset-kubectl-v20@2.1.1:
-    resolution: {integrity: sha512-U1ntiX8XiMRRRH5J1IdC+1t5CE89015cwyt5U63Cpk0GnMlN5+h9WsWMlKlPXZR4rdq/m806JRlBMRpBUB2Dhw==}
+  /@aws-cdk/asset-kubectl-v20@2.1.2:
+    resolution: {integrity: sha512-3M2tELJOxQv0apCIiuKQ4pAbncz9GuLwnKFqxifWfe77wuMxyTRPmxssYHs42ePqzap1LT6GDcPygGs+hHstLg==}
 
-  /@aws-cdk/asset-node-proxy-agent-v5@2.0.155:
-    resolution: {integrity: sha512-Q+Ny25hUPINlBbS6lmbUr4m6Tr6ToEJBla7sXA3FO3JUD0Z69ddcgbhuEBF8Rh1a2xmPONm89eX77kwK2fb4vQ==}
+  /@aws-cdk/asset-node-proxy-agent-v5@2.0.166:
+    resolution: {integrity: sha512-j0xnccpUQHXJKPgCwQcGGNu4lRiC1PptYfdxBIH1L4dRK91iBxtSQHESRQX+yB47oGLaF/WfNN/aF3WXwlhikg==}
 
   /@aws-crypto/crc32@3.0.0:
     resolution: {integrity: sha512-IzSgsrxUcsrejQbPVilIKy16kAT52EwB6zSaI+M3xxIhKh5+aldEyvI+z6erM7TCLB2BJsFrtHjp6/4/sr+3dA==}
     dependencies:
       '@aws-crypto/util': 3.0.0
-      '@aws-sdk/types': 3.370.0
+      '@aws-sdk/types': 3.387.0
       tslib: 1.14.1
     dev: false
 
@@ -1087,7 +1100,7 @@ packages:
       '@aws-crypto/sha256-js': 3.0.0
       '@aws-crypto/supports-web-crypto': 3.0.0
       '@aws-crypto/util': 3.0.0
-      '@aws-sdk/types': 3.370.0
+      '@aws-sdk/types': 3.387.0
       '@aws-sdk/util-locate-window': 3.310.0
       '@aws-sdk/util-utf8-browser': 3.259.0
       tslib: 1.14.1
@@ -1097,7 +1110,7 @@ packages:
     resolution: {integrity: sha512-PnNN7os0+yd1XvXAy23CFOmTbMaDxgxXtTKHybrJ39Y8kGzBATgBFibWJKH6BhytLI/Zyszs87xCOBNyBig6vQ==}
     dependencies:
       '@aws-crypto/util': 3.0.0
-      '@aws-sdk/types': 3.370.0
+      '@aws-sdk/types': 3.387.0
       tslib: 1.14.1
     dev: false
 
@@ -1110,7 +1123,7 @@ packages:
   /@aws-crypto/util@3.0.0:
     resolution: {integrity: sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==}
     dependencies:
-      '@aws-sdk/types': 3.370.0
+      '@aws-sdk/types': 3.387.0
       '@aws-sdk/util-utf8-browser': 3.259.0
       tslib: 1.14.1
     dev: false
@@ -1132,33 +1145,33 @@ packages:
       '@aws-sdk/util-endpoints': 3.370.0
       '@aws-sdk/util-user-agent-browser': 3.370.0
       '@aws-sdk/util-user-agent-node': 3.370.0
-      '@smithy/config-resolver': 1.0.2
-      '@smithy/eventstream-serde-browser': 1.0.2
-      '@smithy/eventstream-serde-config-resolver': 1.0.2
-      '@smithy/eventstream-serde-node': 1.0.2
-      '@smithy/fetch-http-handler': 1.0.2
-      '@smithy/hash-node': 1.0.2
-      '@smithy/invalid-dependency': 1.0.2
-      '@smithy/middleware-content-length': 1.0.2
-      '@smithy/middleware-endpoint': 1.0.3
-      '@smithy/middleware-retry': 1.0.4
-      '@smithy/middleware-serde': 1.0.2
-      '@smithy/middleware-stack': 1.0.2
-      '@smithy/node-config-provider': 1.0.2
-      '@smithy/node-http-handler': 1.0.3
-      '@smithy/protocol-http': 1.1.1
-      '@smithy/smithy-client': 1.0.4
-      '@smithy/types': 1.1.1
-      '@smithy/url-parser': 1.0.2
-      '@smithy/util-base64': 1.0.2
-      '@smithy/util-body-length-browser': 1.0.2
-      '@smithy/util-body-length-node': 1.0.2
-      '@smithy/util-defaults-mode-browser': 1.0.2
-      '@smithy/util-defaults-mode-node': 1.0.2
-      '@smithy/util-retry': 1.0.4
-      '@smithy/util-stream': 1.0.2
-      '@smithy/util-utf8': 1.0.2
-      '@smithy/util-waiter': 1.0.2
+      '@smithy/config-resolver': 1.1.0
+      '@smithy/eventstream-serde-browser': 1.1.0
+      '@smithy/eventstream-serde-config-resolver': 1.1.0
+      '@smithy/eventstream-serde-node': 1.1.0
+      '@smithy/fetch-http-handler': 1.1.0
+      '@smithy/hash-node': 1.1.0
+      '@smithy/invalid-dependency': 1.1.0
+      '@smithy/middleware-content-length': 1.1.0
+      '@smithy/middleware-endpoint': 1.1.0
+      '@smithy/middleware-retry': 1.1.0
+      '@smithy/middleware-serde': 1.1.0
+      '@smithy/middleware-stack': 1.1.0
+      '@smithy/node-config-provider': 1.1.0
+      '@smithy/node-http-handler': 1.1.0
+      '@smithy/protocol-http': 1.2.0
+      '@smithy/smithy-client': 1.1.0
+      '@smithy/types': 1.2.0
+      '@smithy/url-parser': 1.1.0
+      '@smithy/util-base64': 1.1.0
+      '@smithy/util-body-length-browser': 1.1.0
+      '@smithy/util-body-length-node': 1.1.0
+      '@smithy/util-defaults-mode-browser': 1.1.0
+      '@smithy/util-defaults-mode-node': 1.1.0
+      '@smithy/util-retry': 1.1.0
+      '@smithy/util-stream': 1.1.0
+      '@smithy/util-utf8': 1.1.0
+      '@smithy/util-waiter': 1.1.0
       tslib: 2.5.2
     transitivePeerDependencies:
       - aws-crt
@@ -1178,28 +1191,28 @@ packages:
       '@aws-sdk/util-endpoints': 3.370.0
       '@aws-sdk/util-user-agent-browser': 3.370.0
       '@aws-sdk/util-user-agent-node': 3.370.0
-      '@smithy/config-resolver': 1.0.2
-      '@smithy/fetch-http-handler': 1.0.2
-      '@smithy/hash-node': 1.0.2
-      '@smithy/invalid-dependency': 1.0.2
-      '@smithy/middleware-content-length': 1.0.2
-      '@smithy/middleware-endpoint': 1.0.3
-      '@smithy/middleware-retry': 1.0.4
-      '@smithy/middleware-serde': 1.0.2
-      '@smithy/middleware-stack': 1.0.2
-      '@smithy/node-config-provider': 1.0.2
-      '@smithy/node-http-handler': 1.0.3
-      '@smithy/protocol-http': 1.1.1
-      '@smithy/smithy-client': 1.0.4
-      '@smithy/types': 1.1.1
-      '@smithy/url-parser': 1.0.2
-      '@smithy/util-base64': 1.0.2
-      '@smithy/util-body-length-browser': 1.0.2
-      '@smithy/util-body-length-node': 1.0.2
-      '@smithy/util-defaults-mode-browser': 1.0.2
-      '@smithy/util-defaults-mode-node': 1.0.2
-      '@smithy/util-retry': 1.0.4
-      '@smithy/util-utf8': 1.0.2
+      '@smithy/config-resolver': 1.1.0
+      '@smithy/fetch-http-handler': 1.1.0
+      '@smithy/hash-node': 1.1.0
+      '@smithy/invalid-dependency': 1.1.0
+      '@smithy/middleware-content-length': 1.1.0
+      '@smithy/middleware-endpoint': 1.1.0
+      '@smithy/middleware-retry': 1.1.0
+      '@smithy/middleware-serde': 1.1.0
+      '@smithy/middleware-stack': 1.1.0
+      '@smithy/node-config-provider': 1.1.0
+      '@smithy/node-http-handler': 1.1.0
+      '@smithy/protocol-http': 1.2.0
+      '@smithy/smithy-client': 1.1.0
+      '@smithy/types': 1.2.0
+      '@smithy/url-parser': 1.1.0
+      '@smithy/util-base64': 1.1.0
+      '@smithy/util-body-length-browser': 1.1.0
+      '@smithy/util-body-length-node': 1.1.0
+      '@smithy/util-defaults-mode-browser': 1.1.0
+      '@smithy/util-defaults-mode-node': 1.1.0
+      '@smithy/util-retry': 1.1.0
+      '@smithy/util-utf8': 1.1.0
       tslib: 2.5.2
     transitivePeerDependencies:
       - aws-crt
@@ -1219,28 +1232,28 @@ packages:
       '@aws-sdk/util-endpoints': 3.370.0
       '@aws-sdk/util-user-agent-browser': 3.370.0
       '@aws-sdk/util-user-agent-node': 3.370.0
-      '@smithy/config-resolver': 1.0.2
-      '@smithy/fetch-http-handler': 1.0.2
-      '@smithy/hash-node': 1.0.2
-      '@smithy/invalid-dependency': 1.0.2
-      '@smithy/middleware-content-length': 1.0.2
-      '@smithy/middleware-endpoint': 1.0.3
-      '@smithy/middleware-retry': 1.0.4
-      '@smithy/middleware-serde': 1.0.2
-      '@smithy/middleware-stack': 1.0.2
-      '@smithy/node-config-provider': 1.0.2
-      '@smithy/node-http-handler': 1.0.3
-      '@smithy/protocol-http': 1.1.1
-      '@smithy/smithy-client': 1.0.4
-      '@smithy/types': 1.1.1
-      '@smithy/url-parser': 1.0.2
-      '@smithy/util-base64': 1.0.2
-      '@smithy/util-body-length-browser': 1.0.2
-      '@smithy/util-body-length-node': 1.0.2
-      '@smithy/util-defaults-mode-browser': 1.0.2
-      '@smithy/util-defaults-mode-node': 1.0.2
-      '@smithy/util-retry': 1.0.4
-      '@smithy/util-utf8': 1.0.2
+      '@smithy/config-resolver': 1.1.0
+      '@smithy/fetch-http-handler': 1.1.0
+      '@smithy/hash-node': 1.1.0
+      '@smithy/invalid-dependency': 1.1.0
+      '@smithy/middleware-content-length': 1.1.0
+      '@smithy/middleware-endpoint': 1.1.0
+      '@smithy/middleware-retry': 1.1.0
+      '@smithy/middleware-serde': 1.1.0
+      '@smithy/middleware-stack': 1.1.0
+      '@smithy/node-config-provider': 1.1.0
+      '@smithy/node-http-handler': 1.1.0
+      '@smithy/protocol-http': 1.2.0
+      '@smithy/smithy-client': 1.1.0
+      '@smithy/types': 1.2.0
+      '@smithy/url-parser': 1.1.0
+      '@smithy/util-base64': 1.1.0
+      '@smithy/util-body-length-browser': 1.1.0
+      '@smithy/util-body-length-node': 1.1.0
+      '@smithy/util-defaults-mode-browser': 1.1.0
+      '@smithy/util-defaults-mode-node': 1.1.0
+      '@smithy/util-retry': 1.1.0
+      '@smithy/util-utf8': 1.1.0
       tslib: 2.5.2
     transitivePeerDependencies:
       - aws-crt
@@ -1263,28 +1276,28 @@ packages:
       '@aws-sdk/util-endpoints': 3.370.0
       '@aws-sdk/util-user-agent-browser': 3.370.0
       '@aws-sdk/util-user-agent-node': 3.370.0
-      '@smithy/config-resolver': 1.0.2
-      '@smithy/fetch-http-handler': 1.0.2
-      '@smithy/hash-node': 1.0.2
-      '@smithy/invalid-dependency': 1.0.2
-      '@smithy/middleware-content-length': 1.0.2
-      '@smithy/middleware-endpoint': 1.0.3
-      '@smithy/middleware-retry': 1.0.4
-      '@smithy/middleware-serde': 1.0.2
-      '@smithy/middleware-stack': 1.0.2
-      '@smithy/node-config-provider': 1.0.2
-      '@smithy/node-http-handler': 1.0.3
-      '@smithy/protocol-http': 1.1.1
-      '@smithy/smithy-client': 1.0.4
-      '@smithy/types': 1.1.1
-      '@smithy/url-parser': 1.0.2
-      '@smithy/util-base64': 1.0.2
-      '@smithy/util-body-length-browser': 1.0.2
-      '@smithy/util-body-length-node': 1.0.2
-      '@smithy/util-defaults-mode-browser': 1.0.2
-      '@smithy/util-defaults-mode-node': 1.0.2
-      '@smithy/util-retry': 1.0.4
-      '@smithy/util-utf8': 1.0.2
+      '@smithy/config-resolver': 1.1.0
+      '@smithy/fetch-http-handler': 1.1.0
+      '@smithy/hash-node': 1.1.0
+      '@smithy/invalid-dependency': 1.1.0
+      '@smithy/middleware-content-length': 1.1.0
+      '@smithy/middleware-endpoint': 1.1.0
+      '@smithy/middleware-retry': 1.1.0
+      '@smithy/middleware-serde': 1.1.0
+      '@smithy/middleware-stack': 1.1.0
+      '@smithy/node-config-provider': 1.1.0
+      '@smithy/node-http-handler': 1.1.0
+      '@smithy/protocol-http': 1.2.0
+      '@smithy/smithy-client': 1.1.0
+      '@smithy/types': 1.2.0
+      '@smithy/url-parser': 1.1.0
+      '@smithy/util-base64': 1.1.0
+      '@smithy/util-body-length-browser': 1.1.0
+      '@smithy/util-body-length-node': 1.1.0
+      '@smithy/util-defaults-mode-browser': 1.1.0
+      '@smithy/util-defaults-mode-node': 1.1.0
+      '@smithy/util-retry': 1.1.0
+      '@smithy/util-utf8': 1.1.0
       fast-xml-parser: 4.2.5
       tslib: 2.5.2
     transitivePeerDependencies:
@@ -1296,8 +1309,8 @@ packages:
     engines: {node: '>=14.0.0'}
     dependencies:
       '@aws-sdk/types': 3.370.0
-      '@smithy/property-provider': 1.0.2
-      '@smithy/types': 1.1.1
+      '@smithy/property-provider': 1.2.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
@@ -1310,10 +1323,10 @@ packages:
       '@aws-sdk/credential-provider-sso': 3.370.0
       '@aws-sdk/credential-provider-web-identity': 3.370.0
       '@aws-sdk/types': 3.370.0
-      '@smithy/credential-provider-imds': 1.0.2
-      '@smithy/property-provider': 1.0.2
-      '@smithy/shared-ini-file-loader': 1.0.2
-      '@smithy/types': 1.1.1
+      '@smithy/credential-provider-imds': 1.1.0
+      '@smithy/property-provider': 1.2.0
+      '@smithy/shared-ini-file-loader': 1.1.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     transitivePeerDependencies:
       - aws-crt
@@ -1329,10 +1342,10 @@ packages:
       '@aws-sdk/credential-provider-sso': 3.370.0
       '@aws-sdk/credential-provider-web-identity': 3.370.0
       '@aws-sdk/types': 3.370.0
-      '@smithy/credential-provider-imds': 1.0.2
-      '@smithy/property-provider': 1.0.2
-      '@smithy/shared-ini-file-loader': 1.0.2
-      '@smithy/types': 1.1.1
+      '@smithy/credential-provider-imds': 1.1.0
+      '@smithy/property-provider': 1.2.0
+      '@smithy/shared-ini-file-loader': 1.1.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     transitivePeerDependencies:
       - aws-crt
@@ -1343,9 +1356,9 @@ packages:
     engines: {node: '>=14.0.0'}
     dependencies:
       '@aws-sdk/types': 3.370.0
-      '@smithy/property-provider': 1.0.2
-      '@smithy/shared-ini-file-loader': 1.0.2
-      '@smithy/types': 1.1.1
+      '@smithy/property-provider': 1.2.0
+      '@smithy/shared-ini-file-loader': 1.1.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
@@ -1356,9 +1369,9 @@ packages:
       '@aws-sdk/client-sso': 3.370.0
       '@aws-sdk/token-providers': 3.370.0
       '@aws-sdk/types': 3.370.0
-      '@smithy/property-provider': 1.0.2
-      '@smithy/shared-ini-file-loader': 1.0.2
-      '@smithy/types': 1.1.1
+      '@smithy/property-provider': 1.2.0
+      '@smithy/shared-ini-file-loader': 1.1.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     transitivePeerDependencies:
       - aws-crt
@@ -1369,8 +1382,8 @@ packages:
     engines: {node: '>=14.0.0'}
     dependencies:
       '@aws-sdk/types': 3.370.0
-      '@smithy/property-provider': 1.0.2
-      '@smithy/types': 1.1.1
+      '@smithy/property-provider': 1.2.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
@@ -1379,8 +1392,8 @@ packages:
     engines: {node: '>=14.0.0'}
     dependencies:
       '@aws-sdk/types': 3.370.0
-      '@smithy/protocol-http': 1.1.1
-      '@smithy/types': 1.1.1
+      '@smithy/protocol-http': 1.2.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
@@ -1389,7 +1402,7 @@ packages:
     engines: {node: '>=14.0.0'}
     dependencies:
       '@aws-sdk/types': 3.370.0
-      '@smithy/types': 1.1.1
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
@@ -1398,8 +1411,8 @@ packages:
     engines: {node: '>=14.0.0'}
     dependencies:
       '@aws-sdk/types': 3.370.0
-      '@smithy/protocol-http': 1.1.1
-      '@smithy/types': 1.1.1
+      '@smithy/protocol-http': 1.2.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
@@ -1409,7 +1422,7 @@ packages:
     dependencies:
       '@aws-sdk/middleware-signing': 3.370.0
       '@aws-sdk/types': 3.370.0
-      '@smithy/types': 1.1.1
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
@@ -1418,11 +1431,11 @@ packages:
     engines: {node: '>=14.0.0'}
     dependencies:
       '@aws-sdk/types': 3.370.0
-      '@smithy/property-provider': 1.0.2
-      '@smithy/protocol-http': 1.1.1
-      '@smithy/signature-v4': 1.0.2
-      '@smithy/types': 1.1.1
-      '@smithy/util-middleware': 1.0.2
+      '@smithy/property-provider': 1.2.0
+      '@smithy/protocol-http': 1.2.0
+      '@smithy/signature-v4': 1.1.0
+      '@smithy/types': 1.2.0
+      '@smithy/util-middleware': 1.1.0
       tslib: 2.5.2
     dev: false
 
@@ -1432,8 +1445,8 @@ packages:
     dependencies:
       '@aws-sdk/types': 3.370.0
       '@aws-sdk/util-endpoints': 3.370.0
-      '@smithy/protocol-http': 1.1.1
-      '@smithy/types': 1.1.1
+      '@smithy/protocol-http': 1.2.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
@@ -1443,9 +1456,9 @@ packages:
     dependencies:
       '@aws-sdk/client-sso-oidc': 3.370.0
       '@aws-sdk/types': 3.370.0
-      '@smithy/property-provider': 1.0.2
-      '@smithy/shared-ini-file-loader': 1.0.2
-      '@smithy/types': 1.1.1
+      '@smithy/property-provider': 1.2.0
+      '@smithy/shared-ini-file-loader': 1.1.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     transitivePeerDependencies:
       - aws-crt
@@ -1455,7 +1468,15 @@ packages:
     resolution: {integrity: sha512-8PGMKklSkRKjunFhzM2y5Jm0H2TBu7YRNISdYzXLUHKSP9zlMEYagseKVdmox0zKHf1LXVNuSlUV2b6SRrieCQ==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/types': 1.1.1
+      '@smithy/types': 1.2.0
+      tslib: 2.5.2
+    dev: false
+
+  /@aws-sdk/types@3.387.0:
+    resolution: {integrity: sha512-YTjFabNwjTF+6yl88f0/tWff018qmmgMmjlw45s6sdVKueWxdxV68U7gepNLF2nhaQPZa6FDOBoA51NaviVs0Q==}
+    engines: {node: '>=14.0.0'}
+    dependencies:
+      '@smithy/types': 2.2.1
       tslib: 2.5.2
     dev: false
 
@@ -1478,7 +1499,7 @@ packages:
     resolution: {integrity: sha512-028LxYZMQ0DANKhW+AKFQslkScZUeYlPmSphrCIXgdIItRZh6ZJHGzE7J/jDsEntZOrZJsjI4z0zZ5W2idj04w==}
     dependencies:
       '@aws-sdk/types': 3.370.0
-      '@smithy/types': 1.1.1
+      '@smithy/types': 1.2.0
       bowser: 2.11.0
       tslib: 2.5.2
     dev: false
@@ -1493,8 +1514,8 @@ packages:
         optional: true
     dependencies:
       '@aws-sdk/types': 3.370.0
-      '@smithy/node-config-provider': 1.0.2
-      '@smithy/types': 1.1.1
+      '@smithy/node-config-provider': 1.1.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
@@ -1531,7 +1552,7 @@ packages:
       gensync: 1.0.0-beta.2
       json5: 2.2.3
       lodash: 4.17.21
-      resolve: 1.22.2
+      resolve: 1.22.4
       semver: 5.7.1
       source-map: 0.5.7
     transitivePeerDependencies:
@@ -1556,7 +1577,7 @@ packages:
       debug: 4.3.4
       gensync: 1.0.0-beta.2
       json5: 2.2.3
-      semver: 6.3.0
+      semver: 6.3.1
     transitivePeerDependencies:
       - supports-color
 
@@ -1594,7 +1615,7 @@ packages:
       '@babel/helper-validator-option': 7.21.0
       browserslist: 4.21.5
       lru-cache: 5.1.1
-      semver: 6.3.0
+      semver: 6.3.1
 
   /@babel/helper-create-class-features-plugin@7.22.1(@babel/core@7.22.1):
     resolution: {integrity: sha512-SowrZ9BWzYFgzUMwUmowbPSGu6CXL5MSuuCkG3bejahSpSymioPmuLdhPxNOc9MjuNGjy7M/HaXvJ8G82Lywlw==}
@@ -1611,7 +1632,7 @@ packages:
       '@babel/helper-replace-supers': 7.22.1
       '@babel/helper-skip-transparent-expression-wrappers': 7.20.0
       '@babel/helper-split-export-declaration': 7.18.6
-      semver: 6.3.0
+      semver: 6.3.1
     transitivePeerDependencies:
       - supports-color
     dev: false
@@ -1625,7 +1646,7 @@ packages:
       '@babel/core': 7.22.1
       '@babel/helper-annotate-as-pure': 7.18.6
       regexpu-core: 5.3.2
-      semver: 6.3.0
+      semver: 6.3.1
     dev: false
 
   /@babel/helper-define-polyfill-provider@0.4.0(@babel/core@7.22.1):
@@ -1639,7 +1660,7 @@ packages:
       debug: 4.3.4
       lodash.debounce: 4.0.8
       resolve: 1.22.2
-      semver: 6.3.0
+      semver: 6.3.1
     transitivePeerDependencies:
       - supports-color
     dev: false
@@ -2607,7 +2628,7 @@ packages:
       babel-plugin-polyfill-corejs2: 0.4.3(@babel/core@7.22.1)
       babel-plugin-polyfill-corejs3: 0.8.1(@babel/core@7.22.1)
       babel-plugin-polyfill-regenerator: 0.5.0(@babel/core@7.22.1)
-      semver: 6.3.0
+      semver: 6.3.1
     transitivePeerDependencies:
       - supports-color
     dev: false
@@ -2807,7 +2828,7 @@ packages:
       babel-plugin-polyfill-corejs3: 0.8.1(@babel/core@7.22.1)
       babel-plugin-polyfill-regenerator: 0.5.0(@babel/core@7.22.1)
       core-js-compat: 3.30.2
-      semver: 6.3.0
+      semver: 6.3.1
     transitivePeerDependencies:
       - supports-color
     dev: false
@@ -2907,9 +2928,6 @@ packages:
       '@babel/helper-validator-identifier': 7.19.1
       to-fast-properties: 2.0.0
 
-  /@balena/dockerignore@1.0.2:
-    resolution: {integrity: sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==}
-
   /@colors/colors@1.5.0:
     resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==}
     engines: {node: '>=0.1.90'}
@@ -2922,8 +2940,8 @@ packages:
     hasBin: true
     dependencies:
       '@commitlint/format': 17.4.4
-      '@commitlint/lint': 17.6.6
-      '@commitlint/load': 17.5.0
+      '@commitlint/lint': 17.7.0
+      '@commitlint/load': 17.7.1
       '@commitlint/read': 17.5.1
       '@commitlint/types': 17.4.4
       execa: 5.1.1
@@ -2943,17 +2961,16 @@ packages:
       conventional-changelog-conventionalcommits: 5.0.0
     dev: true
 
-  /@commitlint/config-validator@17.4.4:
-    resolution: {integrity: sha512-bi0+TstqMiqoBAQDvdEP4AFh0GaKyLFlPPEObgI29utoKEYoPQTvF0EYqIwYYLEoJYhj5GfMIhPHJkTJhagfeg==}
+  /@commitlint/config-validator@17.6.7:
+    resolution: {integrity: sha512-vJSncmnzwMvpr3lIcm0I8YVVDJTzyjy7NZAeXbTXy+MPUdAr9pKyyg7Tx/ebOQ9kqzE6O9WT6jg2164br5UdsQ==}
     engines: {node: '>=v14'}
-    requiresBuild: true
     dependencies:
       '@commitlint/types': 17.4.4
       ajv: 8.12.0
     dev: true
 
-  /@commitlint/ensure@17.4.4:
-    resolution: {integrity: sha512-AHsFCNh8hbhJiuZ2qHv/m59W/GRE9UeOXbkOqxYMNNg9pJ7qELnFcwj5oYpa6vzTSHtPGKf3C2yUFNy1GGHq6g==}
+  /@commitlint/ensure@17.6.7:
+    resolution: {integrity: sha512-mfDJOd1/O/eIb/h4qwXzUxkmskXDL9vNPnZ4AKYKiZALz4vHzwMxBSYtyL2mUIDeU9DRSpEUins8SeKtFkYHSw==}
     engines: {node: '>=v14'}
     dependencies:
       '@commitlint/types': 17.4.4
@@ -2978,41 +2995,41 @@ packages:
       chalk: 4.1.2
     dev: true
 
-  /@commitlint/is-ignored@17.6.6:
-    resolution: {integrity: sha512-4Fw875faAKO+2nILC04yW/2Vy/wlV3BOYCSQ4CEFzriPEprc1Td2LILmqmft6PDEK5Sr14dT9tEzeaZj0V56Gg==}
+  /@commitlint/is-ignored@17.7.0:
+    resolution: {integrity: sha512-043rA7m45tyEfW7Zv2vZHF++176MLHH9h70fnPoYlB1slKBeKl8BwNIlnPg4xBdRBVNPaCqvXxWswx2GR4c9Hw==}
     engines: {node: '>=v14'}
     dependencies:
       '@commitlint/types': 17.4.4
-      semver: 7.5.2
+      semver: 7.5.4
     dev: true
 
-  /@commitlint/lint@17.6.6:
-    resolution: {integrity: sha512-5bN+dnHcRLkTvwCHYMS7Xpbr+9uNi0Kq5NR3v4+oPNx6pYXt8ACuw9luhM/yMgHYwW0ajIR20wkPAFkZLEMGmg==}
+  /@commitlint/lint@17.7.0:
+    resolution: {integrity: sha512-TCQihm7/uszA5z1Ux1vw+Nf3yHTgicus/+9HiUQk+kRSQawByxZNESeQoX9ujfVd3r4Sa+3fn0JQAguG4xvvbA==}
     engines: {node: '>=v14'}
     dependencies:
-      '@commitlint/is-ignored': 17.6.6
-      '@commitlint/parse': 17.6.5
-      '@commitlint/rules': 17.6.5
+      '@commitlint/is-ignored': 17.7.0
+      '@commitlint/parse': 17.7.0
+      '@commitlint/rules': 17.7.0
       '@commitlint/types': 17.4.4
     dev: true
 
-  /@commitlint/load@17.5.0:
-    resolution: {integrity: sha512-l+4W8Sx4CD5rYFsrhHH8HP01/8jEP7kKf33Xlx2Uk2out/UKoKPYMOIRcDH5ppT8UXLMV+x6Wm5osdRKKgaD1Q==}
+  /@commitlint/load@17.7.1:
+    resolution: {integrity: sha512-S/QSOjE1ztdogYj61p6n3UbkUvweR17FQ0zDbNtoTLc+Hz7vvfS7ehoTMQ27hPSjVBpp7SzEcOQu081RLjKHJQ==}
     engines: {node: '>=v14'}
     dependencies:
-      '@commitlint/config-validator': 17.4.4
+      '@commitlint/config-validator': 17.6.7
       '@commitlint/execute-rule': 17.4.0
-      '@commitlint/resolve-extends': 17.4.4
+      '@commitlint/resolve-extends': 17.6.7
       '@commitlint/types': 17.4.4
-      '@types/node': 18.16.19
+      '@types/node': 20.4.7
       chalk: 4.1.2
       cosmiconfig: 8.1.3
-      cosmiconfig-typescript-loader: 4.3.0(@types/node@18.16.19)(cosmiconfig@8.1.3)(ts-node@10.9.1)(typescript@5.1.6)
+      cosmiconfig-typescript-loader: 4.3.0(@types/node@20.4.7)(cosmiconfig@8.1.3)(ts-node@10.9.1)(typescript@5.1.6)
       lodash.isplainobject: 4.0.6
       lodash.merge: 4.6.2
       lodash.uniq: 4.5.0
       resolve-from: 5.0.0
-      ts-node: 10.9.1(@types/node@18.16.19)(typescript@5.1.6)
+      ts-node: 10.9.1(@types/node@20.4.7)(typescript@5.1.6)
       typescript: 5.1.6
     transitivePeerDependencies:
       - '@swc/core'
@@ -3024,13 +3041,13 @@ packages:
     engines: {node: '>=v14'}
     dev: true
 
-  /@commitlint/parse@17.6.5:
-    resolution: {integrity: sha512-0zle3bcn1Hevw5Jqpz/FzEWNo2KIzUbc1XyGg6WrWEoa6GH3A1pbqNF6MvE6rjuy6OY23c8stWnb4ETRZyN+Yw==}
+  /@commitlint/parse@17.7.0:
+    resolution: {integrity: sha512-dIvFNUMCUHqq5Abv80mIEjLVfw8QNuA4DS7OWip4pcK/3h5wggmjVnlwGCDvDChkw2TjK1K6O+tAEV78oxjxag==}
     engines: {node: '>=v14'}
     dependencies:
       '@commitlint/types': 17.4.4
-      conventional-changelog-angular: 5.0.13
-      conventional-commits-parser: 3.2.4
+      conventional-changelog-angular: 6.0.0
+      conventional-commits-parser: 4.0.0
     dev: true
 
   /@commitlint/read@17.5.1:
@@ -3044,12 +3061,11 @@ packages:
       minimist: 1.2.8
     dev: true
 
-  /@commitlint/resolve-extends@17.4.4:
-    resolution: {integrity: sha512-znXr1S0Rr8adInptHw0JeLgumS11lWbk5xAWFVno+HUFVN45875kUtqjrI6AppmD3JI+4s0uZlqqlkepjJd99A==}
+  /@commitlint/resolve-extends@17.6.7:
+    resolution: {integrity: sha512-PfeoAwLHtbOaC9bGn/FADN156CqkFz6ZKiVDMjuC2N5N0740Ke56rKU7Wxdwya8R8xzLK9vZzHgNbuGhaOVKIg==}
     engines: {node: '>=v14'}
-    requiresBuild: true
     dependencies:
-      '@commitlint/config-validator': 17.4.4
+      '@commitlint/config-validator': 17.6.7
       '@commitlint/types': 17.4.4
       import-fresh: 3.3.0
       lodash.mergewith: 4.6.2
@@ -3057,11 +3073,11 @@ packages:
       resolve-global: 1.0.0
     dev: true
 
-  /@commitlint/rules@17.6.5:
-    resolution: {integrity: sha512-uTB3zSmnPyW2qQQH+Dbq2rekjlWRtyrjDo4aLFe63uteandgkI+cc0NhhbBAzcXShzVk0qqp8SlkQMu0mgHg/A==}
+  /@commitlint/rules@17.7.0:
+    resolution: {integrity: sha512-J3qTh0+ilUE5folSaoK91ByOb8XeQjiGcdIdiB/8UT1/Rd1itKo0ju/eQVGyFzgTMYt8HrDJnGTmNWwcMR1rmA==}
     engines: {node: '>=v14'}
     dependencies:
-      '@commitlint/ensure': 17.4.4
+      '@commitlint/ensure': 17.6.7
       '@commitlint/message': 17.4.2
       '@commitlint/to-lines': 17.4.0
       '@commitlint/types': 17.4.4
@@ -4296,14 +4312,14 @@ packages:
       eslint: ^6.0.0 || ^7.0.0 || >=8.0.0
     dependencies:
       eslint: 8.45.0
-      eslint-visitor-keys: 3.4.1
+      eslint-visitor-keys: 3.4.3
 
-  /@eslint-community/regexpp@4.5.1:
-    resolution: {integrity: sha512-Z5ba73P98O1KUYCCJTUeVpja9RcGoMdncZ6T49FCUl2lN38JtCJ+3WgIDBv0AuY4WChU5PmtJmOCTlN6FZTFKQ==}
+  /@eslint-community/regexpp@4.6.2:
+    resolution: {integrity: sha512-pPTNuaAG3QMH+buKyBIGJs3g/S5y0caxw0ygM3YyE6yJFySwiGGSzA+mM3KJ8QQvzeLh3blwgSonkFjgQdxzMw==}
     engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0}
 
-  /@eslint/eslintrc@2.1.0:
-    resolution: {integrity: sha512-Lj7DECXqIVCqnqjjHMPna4vn6GJcMgul/wuS0je9OZ9gsL0zzDpKPVtcG1HaDVc+9y+qgXneTeUMbCqXJNpH1A==}
+  /@eslint/eslintrc@2.1.2:
+    resolution: {integrity: sha512-+wvgpDsrB1YqAMdEUCcnTlpfVBH7Vqn6A/NT3D8WVXFIaKMlErPIZT3oCIAVCOtarRpMtelZLqJeU3t7WY6X6g==}
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
     dependencies:
       ajv: 6.12.6
@@ -4427,25 +4443,17 @@ packages:
     engines: {node: '>=8'}
     dev: true
 
-  /@jest/schemas@29.4.3:
-    resolution: {integrity: sha512-VLYKXQmtmuEz6IxJsrZwzG9NvtkQsWNnWMsKxqWNu3+CnfzJQhp0WDDKWLVV9hLKr0l3SLLFRqcYHjhtyuDVxg==}
-    engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
-    dependencies:
-      '@sinclair/typebox': 0.25.24
-    dev: false
-
   /@jest/schemas@29.6.0:
     resolution: {integrity: sha512-rxLjXyJBTL4LQeJW3aKo0M/+GkCOXsO+8i9Iu7eDb6KwtP65ayoDsitrdPBtujxQ88k4wI2FNYfa6TOGwSn6cQ==}
     engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
     dependencies:
       '@sinclair/typebox': 0.27.8
-    dev: true
 
   /@jest/types@29.5.0:
     resolution: {integrity: sha512-qbu7kN6czmVRc3xWFQcAN03RAUamgppVUdXrvl1Wr3jlNF93o9mJbGcDWrwGB6ht44u7efB1qCFgVQmca24Uog==}
     engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
     dependencies:
-      '@jest/schemas': 29.4.3
+      '@jest/schemas': 29.6.0
       '@types/istanbul-lib-coverage': 2.0.4
       '@types/istanbul-reports': 3.0.1
       '@types/node': 18.16.19
@@ -4661,13 +4669,8 @@ packages:
   /@sideway/pinpoint@2.0.0:
     resolution: {integrity: sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==}
 
-  /@sinclair/typebox@0.25.24:
-    resolution: {integrity: sha512-XJfwUVUKDHF5ugKwIcxEgc9k8b7HbznCp6eUfWgu710hMPNIO4aw4/zB5RogDQz8nd6gyCDpU9O/m6qYEWY6yQ==}
-    dev: false
-
   /@sinclair/typebox@0.27.8:
     resolution: {integrity: sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==}
-    dev: true
 
   /@sindresorhus/is@0.14.0:
     resolution: {integrity: sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==}
@@ -4683,382 +4686,389 @@ packages:
       webpack-sources: 3.2.3
     dev: false
 
-  /@smithy/abort-controller@1.0.2:
-    resolution: {integrity: sha512-tb2h0b+JvMee+eAxTmhnyqyNk51UXIK949HnE14lFeezKsVJTB30maan+CO2IMwnig2wVYQH84B5qk6ylmKCuA==}
+  /@smithy/abort-controller@1.1.0:
+    resolution: {integrity: sha512-5imgGUlZL4dW4YWdMYAKLmal9ny/tlenM81QZY7xYyb76z9Z/QOg7oM5Ak9HQl8QfFTlGVWwcMXl+54jroRgEQ==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/types': 1.1.1
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/config-resolver@1.0.2:
-    resolution: {integrity: sha512-8Bk7CgnVKg1dn5TgnjwPz2ebhxeR7CjGs5yhVYH3S8x0q8yPZZVWwpRIglwXaf5AZBzJlNO1lh+lUhMf2e73zQ==}
+  /@smithy/config-resolver@1.1.0:
+    resolution: {integrity: sha512-7WD9eZHp46BxAjNGHJLmxhhyeiNWkBdVStd7SUJPUZqQGeIO/REtIrcIfKUfdiHTQ9jyu2SYoqvzqqaFc6987w==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/types': 1.1.1
-      '@smithy/util-config-provider': 1.0.2
-      '@smithy/util-middleware': 1.0.2
+      '@smithy/types': 1.2.0
+      '@smithy/util-config-provider': 1.1.0
+      '@smithy/util-middleware': 1.1.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/credential-provider-imds@1.0.2:
-    resolution: {integrity: sha512-fLjCya+JOu2gPJpCiwSUyoLvT8JdNJmOaTOkKYBZoGf7CzqR6lluSyI+eboZnl/V0xqcfcqBG4tgqCISmWS3/w==}
+  /@smithy/credential-provider-imds@1.1.0:
+    resolution: {integrity: sha512-kUMOdEu3RP6ozH0Ga8OeMP8gSkBsK1UqZZKyPLFnpZHrtZuHSSt7M7gsHYB/bYQBZAo3o7qrGmRty3BubYtYxQ==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/node-config-provider': 1.0.2
-      '@smithy/property-provider': 1.0.2
-      '@smithy/types': 1.1.1
-      '@smithy/url-parser': 1.0.2
+      '@smithy/node-config-provider': 1.1.0
+      '@smithy/property-provider': 1.2.0
+      '@smithy/types': 1.2.0
+      '@smithy/url-parser': 1.1.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/eventstream-codec@1.0.2:
-    resolution: {integrity: sha512-eW/XPiLauR1VAgHKxhVvgvHzLROUgTtqat2lgljztbH8uIYWugv7Nz+SgCavB+hWRazv2iYgqrSy74GvxXq/rg==}
+  /@smithy/eventstream-codec@1.1.0:
+    resolution: {integrity: sha512-3tEbUb8t8an226jKB6V/Q2XU/J53lCwCzULuBPEaF4JjSh+FlCMp7TmogE/Aij5J9DwlsZ4VAD/IRDuQ/0ZtMw==}
     dependencies:
       '@aws-crypto/crc32': 3.0.0
-      '@smithy/types': 1.1.1
-      '@smithy/util-hex-encoding': 1.0.2
+      '@smithy/types': 1.2.0
+      '@smithy/util-hex-encoding': 1.1.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/eventstream-serde-browser@1.0.2:
-    resolution: {integrity: sha512-8bDImzBewLQrIF6hqxMz3eoYwEus2E5JrEwKnhpkSFkkoj8fDSKiLeP/26xfcaoVJgZXB8M1c6jSEZiY3cUMsw==}
+  /@smithy/eventstream-serde-browser@1.1.0:
+    resolution: {integrity: sha512-qUov6SYlcCeubwTQgaSBuNO0J31UdwgGRSZvmHhc3CCYOywoVSsA5vahcNuhoZDzZkhWTpol3Pm7+6OUuHF0aA==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/eventstream-serde-universal': 1.0.2
-      '@smithy/types': 1.1.1
+      '@smithy/eventstream-serde-universal': 1.1.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/eventstream-serde-config-resolver@1.0.2:
-    resolution: {integrity: sha512-SeiJ5pfrXzkGP4WCt9V3Pimfr3OM85Nyh9u/V4J6E0O2dLOYuqvSuKdVnktV0Tcmuu1ZYbt78Th0vfetnSEcdQ==}
+  /@smithy/eventstream-serde-config-resolver@1.1.0:
+    resolution: {integrity: sha512-vtPnp8FJkrNibWZCXvJN6rijTAEAzrmEKNfCUJOHAeBScY25hc6NjYlEJfdSmhW1qaA179oXeqHobcUNzvFkmw==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/types': 1.1.1
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/eventstream-serde-node@1.0.2:
-    resolution: {integrity: sha512-jqSfi7bpOBHqgd5OgUtCX0wAVhPqxlVdqcj2c4gHaRRXcbpCmK0DRDg7P+Df0h4JJVvTqI6dy2c0YhHk5ehPCw==}
+  /@smithy/eventstream-serde-node@1.1.0:
+    resolution: {integrity: sha512-r8kUOPsJMolBGi/eU2gKfw5spfAhQjJXLe4bjjTzkapsqL654JZ+8G9iS1TprYUcCoCHDbwnH1of3kjrYKk7CQ==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/eventstream-serde-universal': 1.0.2
-      '@smithy/types': 1.1.1
+      '@smithy/eventstream-serde-universal': 1.1.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/eventstream-serde-universal@1.0.2:
-    resolution: {integrity: sha512-cQ9bT0j0x49cp8TQ1yZSnn4+9qU0WQSTkoucl3jKRoTZMzNYHg62LQao6HTQ3Jgd77nAXo00c7hqUEjHXwNA+A==}
+  /@smithy/eventstream-serde-universal@1.1.0:
+    resolution: {integrity: sha512-8nQttgdbefJbLfz7Mao0FtkdRUlc92fCiHV3vClAl1N/qetm/I6Lsu5mLt9CzG7TGFkFb5t3qzAV2FaeAqF+ag==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/eventstream-codec': 1.0.2
-      '@smithy/types': 1.1.1
+      '@smithy/eventstream-codec': 1.1.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/fetch-http-handler@1.0.2:
-    resolution: {integrity: sha512-kynyofLf62LvR8yYphPPdyHb8fWG3LepFinM/vWUTG2Q1pVpmPCM530ppagp3+q2p+7Ox0UvSqldbKqV/d1BpA==}
+  /@smithy/fetch-http-handler@1.1.0:
+    resolution: {integrity: sha512-N22C9R44u5WGlcY+Wuv8EXmCAq62wWwriRAuoczMEwAIjPbvHSthyPSLqI4S7kAST1j6niWg8kwpeJ3ReAv3xg==}
     dependencies:
-      '@smithy/protocol-http': 1.1.1
-      '@smithy/querystring-builder': 1.0.2
-      '@smithy/types': 1.1.1
-      '@smithy/util-base64': 1.0.2
+      '@smithy/protocol-http': 1.2.0
+      '@smithy/querystring-builder': 1.1.0
+      '@smithy/types': 1.2.0
+      '@smithy/util-base64': 1.1.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/hash-node@1.0.2:
-    resolution: {integrity: sha512-K6PKhcUNrJXtcesyzhIvNlU7drfIU7u+EMQuGmPw6RQDAg/ufUcfKHz4EcUhFAodUmN+rrejhRG9U6wxjeBOQA==}
+  /@smithy/hash-node@1.1.0:
+    resolution: {integrity: sha512-yiNKDGMzrQjnpnbLfkYKo+HwIxmBAsv0AI++QIJwvhfkLpUTBylelkv6oo78/YqZZS6h+bGfl0gILJsKE2wAKQ==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/types': 1.1.1
-      '@smithy/util-buffer-from': 1.0.2
-      '@smithy/util-utf8': 1.0.2
+      '@smithy/types': 1.2.0
+      '@smithy/util-buffer-from': 1.1.0
+      '@smithy/util-utf8': 1.1.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/invalid-dependency@1.0.2:
-    resolution: {integrity: sha512-B1Y3Tsa6dfC+Vvb+BJMhTHOfFieeYzY9jWQSTR1vMwKkxsymD0OIAnEw8rD/RiDj/4E4RPGFdx9Mdgnyd6Bv5Q==}
+  /@smithy/invalid-dependency@1.1.0:
+    resolution: {integrity: sha512-h2rXn68ClTwzPXYzEUNkz+0B/A0Hz8YdFNTiEwlxkwzkETGKMxmsrQGFXwYm3jd736R5vkXcClXz1ddKrsaBEQ==}
     dependencies:
-      '@smithy/types': 1.1.1
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/is-array-buffer@1.0.2:
-    resolution: {integrity: sha512-pkyBnsBRpe+c/6ASavqIMRBdRtZNJEVJOEzhpxZ9JoAXiZYbkfaSMRA/O1dUxGdJ653GHONunnZ4xMo/LJ7utQ==}
+  /@smithy/is-array-buffer@1.1.0:
+    resolution: {integrity: sha512-twpQ/n+3OWZJ7Z+xu43MJErmhB/WO/mMTnqR6PwWQShvSJ/emx5d1N59LQZk6ZpTAeuRWrc+eHhkzTp9NFjNRQ==}
     engines: {node: '>=14.0.0'}
     dependencies:
       tslib: 2.5.2
     dev: false
 
-  /@smithy/middleware-content-length@1.0.2:
-    resolution: {integrity: sha512-pa1/SgGIrSmnEr2c9Apw7CdU4l/HW0fK3+LKFCPDYJrzM0JdYpqjQzgxi31P00eAkL0EFBccpus/p1n2GF9urw==}
+  /@smithy/middleware-content-length@1.1.0:
+    resolution: {integrity: sha512-iNxwhZ7Xc5+LjeDElEOi/Nh8fFsc9Dw9+5w7h7/GLFIU0RgAwBJuJtcP1vNTOwzW4B3hG+gRu8sQLqA9OEaTwA==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/protocol-http': 1.1.1
-      '@smithy/types': 1.1.1
+      '@smithy/protocol-http': 1.2.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/middleware-endpoint@1.0.3:
-    resolution: {integrity: sha512-GsWvTXMFjSgl617PCE2km//kIjjtvMRrR2GAuRDIS9sHiLwmkS46VWaVYy+XE7ubEsEtzZ5yK2e8TKDR6Qr5Lw==}
+  /@smithy/middleware-endpoint@1.1.0:
+    resolution: {integrity: sha512-PvpazNjVpxX2ICrzoFYCpFnjB39DKCpZds8lRpAB3p6HGrx6QHBaNvOzVhJGBf0jcAbfCdc5/W0n9z8VWaSSww==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/middleware-serde': 1.0.2
-      '@smithy/types': 1.1.1
-      '@smithy/url-parser': 1.0.2
-      '@smithy/util-middleware': 1.0.2
+      '@smithy/middleware-serde': 1.1.0
+      '@smithy/types': 1.2.0
+      '@smithy/url-parser': 1.1.0
+      '@smithy/util-middleware': 1.1.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/middleware-retry@1.0.4:
-    resolution: {integrity: sha512-G7uRXGFL8c3F7APnoIMTtNAHH8vT4F2qVnAWGAZaervjupaUQuRRHYBLYubK0dWzOZz86BtAXKieJ5p+Ni2Xpg==}
+  /@smithy/middleware-retry@1.1.0:
+    resolution: {integrity: sha512-lINKYxIvT+W20YFOtHBKeGm7npuJg0/YCoShttU7fVpsmU+a2rdb9zrJn1MHqWfUL6DhTAWGa0tH2O7l4XrDcw==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/protocol-http': 1.1.1
-      '@smithy/service-error-classification': 1.0.3
-      '@smithy/types': 1.1.1
-      '@smithy/util-middleware': 1.0.2
-      '@smithy/util-retry': 1.0.4
+      '@smithy/protocol-http': 1.2.0
+      '@smithy/service-error-classification': 1.1.0
+      '@smithy/types': 1.2.0
+      '@smithy/util-middleware': 1.1.0
+      '@smithy/util-retry': 1.1.0
       tslib: 2.5.2
       uuid: 8.3.2
     dev: false
 
-  /@smithy/middleware-serde@1.0.2:
-    resolution: {integrity: sha512-T4PcdMZF4xme6koUNfjmSZ1MLi7eoFeYCtodQNQpBNsS77TuJt1A6kt5kP/qxrTvfZHyFlj0AubACoaUqgzPeg==}
+  /@smithy/middleware-serde@1.1.0:
+    resolution: {integrity: sha512-RiBMxhxuO9VTjHsjJvhzViyceoLhU6gtrnJGpAXY43wE49IstXIGEQz8MT50/hOq5EumX16FCpup0r5DVyfqNQ==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/types': 1.1.1
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/middleware-stack@1.0.2:
-    resolution: {integrity: sha512-H7/uAQEcmO+eDqweEFMJ5YrIpsBwmrXSP6HIIbtxKJSQpAcMGY7KrR2FZgZBi1FMnSUOh+rQrbOyj5HQmSeUBA==}
+  /@smithy/middleware-stack@1.1.0:
+    resolution: {integrity: sha512-XynYiIvXNea2BbLcppvpNK0zu8o2woJqgnmxqYTn4FWagH/Hr2QIk8LOsUz7BIJ4tooFhmx8urHKCdlPbbPDCA==}
     engines: {node: '>=14.0.0'}
     dependencies:
       tslib: 2.5.2
     dev: false
 
-  /@smithy/node-config-provider@1.0.2:
-    resolution: {integrity: sha512-HU7afWpTToU0wL6KseGDR2zojeyjECQfr8LpjAIeHCYIW7r360ABFf4EaplaJRMVoC3hD9FeltgI3/NtShOqCg==}
+  /@smithy/node-config-provider@1.1.0:
+    resolution: {integrity: sha512-2G4TlzUnmTrUY26VKTonQqydwb+gtM/mcl+TqDP8CnWtJKVL8ElPpKgLGScP04bPIRY9x2/10lDdoaRXDqPuCw==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/property-provider': 1.0.2
-      '@smithy/shared-ini-file-loader': 1.0.2
-      '@smithy/types': 1.1.1
+      '@smithy/property-provider': 1.2.0
+      '@smithy/shared-ini-file-loader': 1.1.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/node-http-handler@1.0.3:
-    resolution: {integrity: sha512-PcPUSzTbIb60VCJCiH0PU0E6bwIekttsIEf5Aoo/M0oTfiqsxHTn0Rcij6QoH6qJy6piGKXzLSegspXg5+Kq6g==}
+  /@smithy/node-http-handler@1.1.0:
+    resolution: {integrity: sha512-d3kRriEgaIiGXLziAM8bjnaLn1fthCJeTLZIwEIpzQqe6yPX0a+yQoLCTyjb2fvdLwkMoG4p7THIIB5cj5lkbg==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/abort-controller': 1.0.2
-      '@smithy/protocol-http': 1.1.1
-      '@smithy/querystring-builder': 1.0.2
-      '@smithy/types': 1.1.1
+      '@smithy/abort-controller': 1.1.0
+      '@smithy/protocol-http': 1.2.0
+      '@smithy/querystring-builder': 1.1.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/property-provider@1.0.2:
-    resolution: {integrity: sha512-pXDPyzKX8opzt38B205kDgaxda6LHcTfPvTYQZnwP6BAPp1o9puiCPjeUtkKck7Z6IbpXCPUmUQnzkUzWTA42Q==}
+  /@smithy/property-provider@1.2.0:
+    resolution: {integrity: sha512-qlJd9gT751i4T0t/hJAyNGfESfi08Fek8QiLcysoKPgR05qHhG0OYhlaCJHhpXy4ECW0lHyjvFM1smrCLIXVfw==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/types': 1.1.1
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/protocol-http@1.1.1:
-    resolution: {integrity: sha512-mFLFa2sSvlUxm55U7B4YCIsJJIMkA6lHxwwqOaBkral1qxFz97rGffP/mmd4JDuin1EnygiO5eNJGgudiUgmDQ==}
+  /@smithy/protocol-http@1.2.0:
+    resolution: {integrity: sha512-GfGfruksi3nXdFok5RhgtOnWe5f6BndzYfmEXISD+5gAGdayFGpjWu5pIqIweTudMtse20bGbc+7MFZXT1Tb8Q==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/types': 1.1.1
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/querystring-builder@1.0.2:
-    resolution: {integrity: sha512-6P/xANWrtJhMzTPUR87AbXwSBuz1SDHIfL44TFd/GT3hj6rA+IEv7rftEpPjayUiWRocaNnrCPLvmP31mobOyA==}
+  /@smithy/querystring-builder@1.1.0:
+    resolution: {integrity: sha512-gDEi4LxIGLbdfjrjiY45QNbuDmpkwh9DX4xzrR2AzjjXpxwGyfSpbJaYhXARw9p17VH0h9UewnNQXNwaQyYMDA==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/types': 1.1.1
-      '@smithy/util-uri-escape': 1.0.2
+      '@smithy/types': 1.2.0
+      '@smithy/util-uri-escape': 1.1.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/querystring-parser@1.0.2:
-    resolution: {integrity: sha512-IWxwxjn+KHWRRRB+K2Ngl+plTwo2WSgc2w+DvLy0DQZJh9UGOpw40d6q97/63GBlXIt4TEt5NbcFrO30CKlrsA==}
+  /@smithy/querystring-parser@1.1.0:
+    resolution: {integrity: sha512-Lm/FZu2qW3XX+kZ4WPwr+7aAeHf1Lm84UjNkKyBu16XbmEV7ukfhXni2aIwS2rcVf8Yv5E7wchGGpOFldj9V4Q==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/types': 1.1.1
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/service-error-classification@1.0.3:
-    resolution: {integrity: sha512-2eglIYqrtcUnuI71yweu7rSfCgt6kVvRVf0C72VUqrd0LrV1M0BM0eYN+nitp2CHPSdmMI96pi+dU9U/UqAMSA==}
+  /@smithy/service-error-classification@1.1.0:
+    resolution: {integrity: sha512-OCTEeJ1igatd5kFrS2VDlYbainNNpf7Lj1siFOxnRWqYOP9oNvC5HOJBd3t+Z8MbrmehBtuDJ2QqeBsfeiNkww==}
+    engines: {node: '>=14.0.0'}
+    dev: false
+
+  /@smithy/shared-ini-file-loader@1.1.0:
+    resolution: {integrity: sha512-S/v33zvCWzFyGZGlsEF0XsZtNNR281UhR7byk3nRfsgw5lGpg51rK/zjMgulM+h6NSuXaFILaYrw1I1v4kMcuA==}
     engines: {node: '>=14.0.0'}
+    dependencies:
+      '@smithy/types': 1.2.0
+      tslib: 2.5.2
     dev: false
 
-  /@smithy/shared-ini-file-loader@1.0.2:
-    resolution: {integrity: sha512-bdQj95VN+lCXki+P3EsDyrkpeLn8xDYiOISBGnUG/AGPYJXN8dmp4EhRRR7XOoLoSs8anZHR4UcGEOzFv2jwGw==}
+  /@smithy/signature-v4@1.1.0:
+    resolution: {integrity: sha512-fDo3m7YqXBs7neciOePPd/X9LPm5QLlDMdIC4m1H6dgNLnXfLMFNIxEfPyohGA8VW9Wn4X8lygnPSGxDZSmp0Q==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/types': 1.1.1
+      '@smithy/eventstream-codec': 1.1.0
+      '@smithy/is-array-buffer': 1.1.0
+      '@smithy/types': 1.2.0
+      '@smithy/util-hex-encoding': 1.1.0
+      '@smithy/util-middleware': 1.1.0
+      '@smithy/util-uri-escape': 1.1.0
+      '@smithy/util-utf8': 1.1.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/signature-v4@1.0.2:
-    resolution: {integrity: sha512-rpKUhmCuPmpV5dloUkOb9w1oBnJatvKQEjIHGmkjRGZnC3437MTdzWej9TxkagcZ8NRRJavYnEUixzxM1amFig==}
+  /@smithy/smithy-client@1.1.0:
+    resolution: {integrity: sha512-j32SGgVhv2G9nBTmel9u3OXux8KG20ssxuFakJrEeDug3kqbl1qrGzVLCe+Eib402UDtA0Sp1a4NZ2SEXDBxag==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/eventstream-codec': 1.0.2
-      '@smithy/is-array-buffer': 1.0.2
-      '@smithy/types': 1.1.1
-      '@smithy/util-hex-encoding': 1.0.2
-      '@smithy/util-middleware': 1.0.2
-      '@smithy/util-uri-escape': 1.0.2
-      '@smithy/util-utf8': 1.0.2
+      '@smithy/middleware-stack': 1.1.0
+      '@smithy/types': 1.2.0
+      '@smithy/util-stream': 1.1.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/smithy-client@1.0.4:
-    resolution: {integrity: sha512-gpo0Xl5Nyp9sgymEfpt7oa9P2q/GlM3VmQIdm+FeH0QEdYOQx3OtvwVmBYAMv2FIPWxkMZlsPYRTnEiBTK5TYg==}
+  /@smithy/types@1.2.0:
+    resolution: {integrity: sha512-z1r00TvBqF3dh4aHhya7nz1HhvCg4TRmw51fjMrh5do3h+ngSstt/yKlNbHeb9QxJmFbmN8KEVSWgb1bRvfEoA==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/middleware-stack': 1.0.2
-      '@smithy/types': 1.1.1
-      '@smithy/util-stream': 1.0.2
       tslib: 2.5.2
     dev: false
 
-  /@smithy/types@1.1.1:
-    resolution: {integrity: sha512-tMpkreknl2gRrniHeBtdgQwaOlo39df8RxSrwsHVNIGXULy5XP6KqgScUw2m12D15wnJCKWxVhCX+wbrBW/y7g==}
+  /@smithy/types@2.2.1:
+    resolution: {integrity: sha512-6nyDOf027ZeJiQVm6PXmLm7dR+hR2YJUkr4VwUniXA8xZUGAu5Mk0zfx2BPFrt+e5YauvlIqQoH0CsrM4tLkfg==}
     engines: {node: '>=14.0.0'}
     dependencies:
       tslib: 2.5.2
     dev: false
 
-  /@smithy/url-parser@1.0.2:
-    resolution: {integrity: sha512-0JRsDMQe53F6EHRWksdcavKDRjyqp8vrjakg8EcCUOa7PaFRRB1SO/xGZdzSlW1RSTWQDEksFMTCEcVEKmAoqA==}
+  /@smithy/url-parser@1.1.0:
+    resolution: {integrity: sha512-tpvi761kzboiLNGEWczuybMPCJh6WHB3cz9gWAG95mSyaKXmmX8ZcMxoV+irZfxDqLwZVJ22XTumu32S7Ow8aQ==}
     dependencies:
-      '@smithy/querystring-parser': 1.0.2
-      '@smithy/types': 1.1.1
+      '@smithy/querystring-parser': 1.1.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/util-base64@1.0.2:
-    resolution: {integrity: sha512-BCm15WILJ3SL93nusoxvJGMVfAMWHZhdeDZPtpAaskozuexd0eF6szdz4kbXaKp38bFCSenA6bkUHqaE3KK0dA==}
+  /@smithy/util-base64@1.1.0:
+    resolution: {integrity: sha512-FpYmDmVbOXAxqvoVCwqehUN0zXS+lN8V7VS9O7I8MKeVHdSTsZzlwiMEvGoyTNOXWn8luF4CTDYgNHnZViR30g==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/util-buffer-from': 1.0.2
+      '@smithy/util-buffer-from': 1.1.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/util-body-length-browser@1.0.2:
-    resolution: {integrity: sha512-Xh8L06H2anF5BHjSYTg8hx+Itcbf4SQZnVMl4PIkCOsKtneMJoGjPRLy17lEzfoh/GOaa0QxgCP6lRMQWzNl4w==}
+  /@smithy/util-body-length-browser@1.1.0:
+    resolution: {integrity: sha512-cep3ioRxzRZ2Jbp3Kly7gy6iNVefYXiT6ETt8W01RQr3uwi1YMkrbU1p3lMR4KhX/91Nrk6UOgX1RH+oIt48RQ==}
     dependencies:
       tslib: 2.5.2
     dev: false
 
-  /@smithy/util-body-length-node@1.0.2:
-    resolution: {integrity: sha512-nXHbZsUtvZeyfL4Ceds9nmy2Uh2AhWXohG4vWHyjSdmT8cXZlJdmJgnH6SJKDjyUecbu+BpKeVvSrA4cWPSOPA==}
+  /@smithy/util-body-length-node@1.1.0:
+    resolution: {integrity: sha512-fRHRjkUuT5em4HZoshySXmB1n3HAU7IS232s+qU4TicexhyGJpXMK/2+c56ePOIa1FOK2tV1Q3J/7Mae35QVSw==}
     engines: {node: '>=14.0.0'}
     dependencies:
       tslib: 2.5.2
     dev: false
 
-  /@smithy/util-buffer-from@1.0.2:
-    resolution: {integrity: sha512-lHAYIyrBO9RANrPvccnPjU03MJnWZ66wWuC5GjWWQVfsmPwU6m00aakZkzHdUT6tGCkGacXSgArP5wgTgA+oCw==}
+  /@smithy/util-buffer-from@1.1.0:
+    resolution: {integrity: sha512-9m6NXE0ww+ra5HKHCHig20T+FAwxBAm7DIdwc/767uGWbRcY720ybgPacQNB96JMOI7xVr/CDa3oMzKmW4a+kw==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/is-array-buffer': 1.0.2
+      '@smithy/is-array-buffer': 1.1.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/util-config-provider@1.0.2:
-    resolution: {integrity: sha512-HOdmDm+3HUbuYPBABLLHtn8ittuRyy+BSjKOA169H+EMc+IozipvXDydf+gKBRAxUa4dtKQkLraypwppzi+PRw==}
+  /@smithy/util-config-provider@1.1.0:
+    resolution: {integrity: sha512-rQ47YpNmF6Is4I9GiE3T3+0xQ+r7RKRKbmHYyGSbyep/0cSf9kteKcI0ssJTvveJ1K4QvwrxXj1tEFp/G2UqxQ==}
     engines: {node: '>=14.0.0'}
     dependencies:
       tslib: 2.5.2
     dev: false
 
-  /@smithy/util-defaults-mode-browser@1.0.2:
-    resolution: {integrity: sha512-J1u2PO235zxY7dg0+ZqaG96tFg4ehJZ7isGK1pCBEA072qxNPwIpDzUVGnLJkHZvjWEGA8rxIauDtXfB0qxeAg==}
+  /@smithy/util-defaults-mode-browser@1.1.0:
+    resolution: {integrity: sha512-0bWhs1e412bfC5gwPCMe8Zbz0J8UoZ/meEQdo6MYj8Ne+c+QZ+KxVjx0a1dFYOclvM33SslL9dP0odn8kfblkg==}
     engines: {node: '>= 10.0.0'}
     dependencies:
-      '@smithy/property-provider': 1.0.2
-      '@smithy/types': 1.1.1
+      '@smithy/property-provider': 1.2.0
+      '@smithy/types': 1.2.0
       bowser: 2.11.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/util-defaults-mode-node@1.0.2:
-    resolution: {integrity: sha512-9/BN63rlIsFStvI+AvljMh873Xw6bbI6b19b+PVYXyycQ2DDQImWcjnzRlHW7eP65CCUNGQ6otDLNdBQCgMXqg==}
+  /@smithy/util-defaults-mode-node@1.1.0:
+    resolution: {integrity: sha512-440e25TUH2b+TeK5CwsjYFrI9ShVOgA31CoxCKiv4ncSK4ZM68XW5opYxQmzMbRWARGEMu2XEUeBmOgMU2RLsw==}
     engines: {node: '>= 10.0.0'}
     dependencies:
-      '@smithy/config-resolver': 1.0.2
-      '@smithy/credential-provider-imds': 1.0.2
-      '@smithy/node-config-provider': 1.0.2
-      '@smithy/property-provider': 1.0.2
-      '@smithy/types': 1.1.1
+      '@smithy/config-resolver': 1.1.0
+      '@smithy/credential-provider-imds': 1.1.0
+      '@smithy/node-config-provider': 1.1.0
+      '@smithy/property-provider': 1.2.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/util-hex-encoding@1.0.2:
-    resolution: {integrity: sha512-Bxydb5rMJorMV6AuDDMOxro3BMDdIwtbQKHpwvQFASkmr52BnpDsWlxgpJi8Iq7nk1Bt4E40oE1Isy/7ubHGzg==}
+  /@smithy/util-hex-encoding@1.1.0:
+    resolution: {integrity: sha512-7UtIE9eH0u41zpB60Jzr0oNCQ3hMJUabMcKRUVjmyHTXiWDE4vjSqN6qlih7rCNeKGbioS7f/y2Jgym4QZcKFg==}
     engines: {node: '>=14.0.0'}
     dependencies:
       tslib: 2.5.2
     dev: false
 
-  /@smithy/util-middleware@1.0.2:
-    resolution: {integrity: sha512-vtXK7GOR2BoseCX8NCGe9SaiZrm9M2lm/RVexFGyPuafTtry9Vyv7hq/vw8ifd/G/pSJ+msByfJVb1642oQHKw==}
+  /@smithy/util-middleware@1.1.0:
+    resolution: {integrity: sha512-6hhckcBqVgjWAqLy2vqlPZ3rfxLDhFWEmM7oLh2POGvsi7j0tHkbN7w4DFhuBExVJAbJ/qqxqZdRY6Fu7/OezQ==}
     engines: {node: '>=14.0.0'}
     dependencies:
       tslib: 2.5.2
     dev: false
 
-  /@smithy/util-retry@1.0.4:
-    resolution: {integrity: sha512-RnZPVFvRoqdj2EbroDo3OsnnQU8eQ4AlnZTOGusbYKybH3269CFdrZfZJloe60AQjX7di3J6t/79PjwCLO5Khw==}
+  /@smithy/util-retry@1.1.0:
+    resolution: {integrity: sha512-ygQW5HBqYXpR3ua09UciS0sL7UGJzGiktrKkOuEJwARoUuzz40yaEGU6xd9Gs7KBmAaFC8gMfnghHtwZ2nyBCQ==}
     engines: {node: '>= 14.0.0'}
     dependencies:
-      '@smithy/service-error-classification': 1.0.3
+      '@smithy/service-error-classification': 1.1.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/util-stream@1.0.2:
-    resolution: {integrity: sha512-qyN2M9QFMTz4UCHi6GnBfLOGYKxQZD01Ga6nzaXFFC51HP/QmArU72e4kY50Z/EtW8binPxspP2TAsGbwy9l3A==}
+  /@smithy/util-stream@1.1.0:
+    resolution: {integrity: sha512-w3lsdGsntaLQIrwDWJkIFKrFscgZXwU/oxsse09aSTNv5TckPhDeYea3LhsDrU5MGAG3vprhVZAKr33S45coVA==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/fetch-http-handler': 1.0.2
-      '@smithy/node-http-handler': 1.0.3
-      '@smithy/types': 1.1.1
-      '@smithy/util-base64': 1.0.2
-      '@smithy/util-buffer-from': 1.0.2
-      '@smithy/util-hex-encoding': 1.0.2
-      '@smithy/util-utf8': 1.0.2
+      '@smithy/fetch-http-handler': 1.1.0
+      '@smithy/node-http-handler': 1.1.0
+      '@smithy/types': 1.2.0
+      '@smithy/util-base64': 1.1.0
+      '@smithy/util-buffer-from': 1.1.0
+      '@smithy/util-hex-encoding': 1.1.0
+      '@smithy/util-utf8': 1.1.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/util-uri-escape@1.0.2:
-    resolution: {integrity: sha512-k8C0BFNS9HpBMHSgUDnWb1JlCQcFG+PPlVBq9keP4Nfwv6a9Q0yAfASWqUCtzjuMj1hXeLhn/5ADP6JxnID1Pg==}
+  /@smithy/util-uri-escape@1.1.0:
+    resolution: {integrity: sha512-/jL/V1xdVRt5XppwiaEU8Etp5WHZj609n0xMTuehmCqdoOFbId1M+aEeDWZsQ+8JbEB/BJ6ynY2SlYmOaKtt8w==}
     engines: {node: '>=14.0.0'}
     dependencies:
       tslib: 2.5.2
     dev: false
 
-  /@smithy/util-utf8@1.0.2:
-    resolution: {integrity: sha512-V4cyjKfJlARui0dMBfWJMQAmJzoW77i4N3EjkH/bwnE2Ngbl4tqD2Y0C/xzpzY/J1BdxeCKxAebVFk8aFCaSCw==}
+  /@smithy/util-utf8@1.1.0:
+    resolution: {integrity: sha512-p/MYV+JmqmPyjdgyN2UxAeYDj9cBqCjp0C/NsTWnnjoZUVqoeZ6IrW915L9CAKWVECgv9lVQGc4u/yz26/bI1A==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/util-buffer-from': 1.0.2
+      '@smithy/util-buffer-from': 1.1.0
       tslib: 2.5.2
     dev: false
 
-  /@smithy/util-waiter@1.0.2:
-    resolution: {integrity: sha512-+jq4/Vd9ejPzR45qwYSePyjQbqYP9QqtyZYsFVyfzRnbGGC0AjswOh7txcxroafuEBExK4qE+L/QZA8wWXsJYw==}
+  /@smithy/util-waiter@1.1.0:
+    resolution: {integrity: sha512-S6FNIB3UJT+5Efd/0DeziO5Rs82QAMODHW4v2V3oNRrwaBigY/7Yx3SiLudZuF9WpVsV08Ih3BjIH34nzZiinQ==}
     engines: {node: '>=14.0.0'}
     dependencies:
-      '@smithy/abort-controller': 1.0.2
-      '@smithy/types': 1.1.1
+      '@smithy/abort-controller': 1.1.0
+      '@smithy/types': 1.2.0
       tslib: 2.5.2
     dev: false
 
@@ -5422,6 +5432,10 @@ packages:
   /@types/node@18.16.19:
     resolution: {integrity: sha512-IXl7o+R9iti9eBW4Wg2hx1xQDig183jj7YLn8F7udNceyfkbn1ZxmzZXuak20gR40D7pIkIY1kYGx5VIGbaHKA==}
 
+  /@types/node@20.4.7:
+    resolution: {integrity: sha512-bUBrPjEry2QUTsnuEjzjbS7voGWCc30W0qzgMf90GPeDGFRakvrz47ju+oqDAKCXLUCe39u57/ORMl/O/04/9g==}
+    dev: true
+
   /@types/normalize-package-data@2.4.1:
     resolution: {integrity: sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==}
     dev: true
@@ -5564,7 +5578,7 @@ packages:
       typescript:
         optional: true
     dependencies:
-      '@eslint-community/regexpp': 4.5.1
+      '@eslint-community/regexpp': 4.6.2
       '@typescript-eslint/parser': 6.1.0(eslint@8.45.0)(typescript@5.1.6)
       '@typescript-eslint/scope-manager': 6.1.0
       '@typescript-eslint/type-utils': 6.1.0(eslint@8.45.0)(typescript@5.1.6)
@@ -5682,7 +5696,7 @@ packages:
     engines: {node: ^16.0.0 || >=18.0.0}
     dependencies:
       '@typescript-eslint/types': 6.1.0
-      eslint-visitor-keys: 3.4.1
+      eslint-visitor-keys: 3.4.3
     dev: true
 
   /@vitest/coverage-istanbul@0.33.0(vitest@0.33.0):
@@ -5692,7 +5706,7 @@ packages:
     dependencies:
       istanbul-lib-coverage: 3.2.0
       istanbul-lib-instrument: 5.2.1
-      istanbul-lib-report: 3.0.0
+      istanbul-lib-report: 3.0.1
       istanbul-lib-source-maps: 4.0.1
       istanbul-reports: 3.1.5
       test-exclude: 6.0.0
@@ -6110,6 +6124,7 @@ packages:
   /astral-regex@2.0.0:
     resolution: {integrity: sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==}
     engines: {node: '>=8'}
+    dev: true
 
   /async-retry@1.3.3:
     resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==}
@@ -6156,20 +6171,10 @@ packages:
     peerDependencies:
       constructs: ^10.0.0
     dependencies:
-      '@aws-cdk/asset-awscli-v1': 2.2.184
-      '@aws-cdk/asset-kubectl-v20': 2.1.1
-      '@aws-cdk/asset-node-proxy-agent-v5': 2.0.155
-      '@balena/dockerignore': 1.0.2
-      case: 1.6.3
+      '@aws-cdk/asset-awscli-v1': 2.2.200
+      '@aws-cdk/asset-kubectl-v20': 2.1.2
+      '@aws-cdk/asset-node-proxy-agent-v5': 2.0.166
       constructs: 10.2.69
-      fs-extra: 11.1.1
-      ignore: 5.2.4
-      jsonschema: 1.4.1
-      minimatch: 3.1.2
-      punycode: 2.3.0
-      semver: 7.5.1
-      table: 6.8.1
-      yaml: 1.10.2
     bundledDependencies:
       - '@balena/dockerignore'
       - case
@@ -6242,7 +6247,7 @@ packages:
       '@babel/compat-data': 7.22.3
       '@babel/core': 7.22.1
       '@babel/helper-define-polyfill-provider': 0.4.0(@babel/core@7.22.1)
-      semver: 6.3.0
+      semver: 6.3.1
     transitivePeerDependencies:
       - supports-color
     dev: false
@@ -6546,10 +6551,6 @@ packages:
   /caniuse-lite@1.0.30001489:
     resolution: {integrity: sha512-x1mgZEXK8jHIfAxm+xgdpHpk50IN3z3q3zP261/WS+uvePxW8izXuCu6AHz0lkuYTlATDehiZ/tNyYBdSQsOUQ==}
 
-  /case@1.6.3:
-    resolution: {integrity: sha512-mzDSXIPaFwVDvZAHqZ9VlbyF4yyXRuX6IvB06WvPYkqJVO24kX1PPhv9bfpKNFZyxYFmmgo03HUiD8iklmJYRQ==}
-    engines: {node: '>= 0.8.0'}
-
   /ccount@1.1.0:
     resolution: {integrity: sha512-vlNK021QdI7PNeiUh/lKkC/mNHHfV0m/Ad5JoI0TYtlBnJAslM/JIkm/tGC88bkLIwO6OQ5uV6ztS6kVAtCDlg==}
     dev: false
@@ -6964,12 +6965,11 @@ packages:
     engines: {node: '>= 0.6'}
     dev: false
 
-  /conventional-changelog-angular@5.0.13:
-    resolution: {integrity: sha512-i/gipMxs7s8L/QeuavPF2hLnJgH6pEZAttySB6aiQLWcX3puWDL3ACVmvBhJGxnAy52Qc15ua26BufY6KpmrVA==}
-    engines: {node: '>=10'}
+  /conventional-changelog-angular@6.0.0:
+    resolution: {integrity: sha512-6qLgrBF4gueoC7AFVHu51nHL9pF9FRjXrH+ceVf7WmAfH3gs+gEYOkvxhjMPjZu57I4AGUGoNTY8V7Hrgf1uqg==}
+    engines: {node: '>=14'}
     dependencies:
       compare-func: 2.0.0
-      q: 1.5.1
     dev: true
 
   /conventional-changelog-conventionalcommits@5.0.0:
@@ -6985,17 +6985,15 @@ packages:
     resolution: {integrity: sha512-SmmCYnOniSsAa9GqWOeLqc179lfr5TRu5b4QFDkbsrJ5TZjPJx85wtOr3zn+1dbeNiXDKGPbZ72IKbPhLXh/Lg==}
     dev: true
 
-  /conventional-commits-parser@3.2.4:
-    resolution: {integrity: sha512-nK7sAtfi+QXbxHCYfhpZsfRtaitZLIA6889kFIouLvz6repszQDgxBu7wf2WbU+Dco7sAnNCJYERCwt54WPC2Q==}
-    engines: {node: '>=10'}
+  /conventional-commits-parser@4.0.0:
+    resolution: {integrity: sha512-WRv5j1FsVM5FISJkoYMR6tPk07fkKT0UodruX4je86V4owk451yjXAKzKAPOs9l7y59E2viHUS9eQ+dfUA9NSg==}
+    engines: {node: '>=14'}
     hasBin: true
     dependencies:
       JSONStream: 1.3.5
       is-text-path: 1.0.1
-      lodash: 4.17.21
       meow: 8.1.2
       split2: 3.2.2
-      through2: 4.0.2
     dev: true
 
   /convert-source-map@1.9.0:
@@ -7058,7 +7056,7 @@ packages:
       vary: 1.1.2
     dev: false
 
-  /cosmiconfig-typescript-loader@4.3.0(@types/node@18.16.19)(cosmiconfig@8.1.3)(ts-node@10.9.1)(typescript@5.1.6):
+  /cosmiconfig-typescript-loader@4.3.0(@types/node@20.4.7)(cosmiconfig@8.1.3)(ts-node@10.9.1)(typescript@5.1.6):
     resolution: {integrity: sha512-NTxV1MFfZDLPiBMjxbHRwSh5LaLcPMwNdCutmnHJCKoVnlvldPWlllonKwrsRJ5pYZBIBGRWWU2tfvzxgeSW5Q==}
     engines: {node: '>=12', npm: '>=6'}
     requiresBuild: true
@@ -7068,9 +7066,9 @@ packages:
       ts-node: '>=10'
       typescript: '>=3'
     dependencies:
-      '@types/node': 18.16.19
+      '@types/node': 20.4.7
       cosmiconfig: 8.1.3
-      ts-node: 10.9.1(@types/node@18.16.19)(typescript@5.1.6)
+      ts-node: 10.9.1(@types/node@20.4.7)(typescript@5.1.6)
       typescript: 5.1.6
     dev: true
 
@@ -7342,7 +7340,7 @@ packages:
       longest: 2.0.1
       word-wrap: 1.2.3
     optionalDependencies:
-      '@commitlint/load': 17.5.0
+      '@commitlint/load': 17.7.1
     transitivePeerDependencies:
       - '@swc/core'
       - '@swc/wasm'
@@ -7986,8 +7984,8 @@ packages:
     resolution: {integrity: sha512-gozW2blMLJCeFpBwugLTGyvVjNoeo1knonXAcatC6bjPBZitotxdWf7Gimr25N4c0AAOo4eOUfaG82IJPDpqCA==}
     dependencies:
       debug: 3.2.7
-      is-core-module: 2.12.1
-      resolve: 1.22.2
+      is-core-module: 2.13.0
+      resolve: 1.22.4
     transitivePeerDependencies:
       - supports-color
     dev: true
@@ -8041,12 +8039,12 @@ packages:
       eslint-import-resolver-node: 0.3.7
       eslint-module-utils: 2.8.0(@typescript-eslint/parser@6.1.0)(eslint-import-resolver-node@0.3.7)(eslint@8.45.0)
       has: 1.0.3
-      is-core-module: 2.12.1
+      is-core-module: 2.13.0
       is-glob: 4.0.3
       minimatch: 3.1.2
       object.values: 1.1.6
-      resolve: 1.22.2
-      semver: 6.3.0
+      resolve: 1.22.4
+      semver: 6.3.1
       tsconfig-paths: 3.14.2
     transitivePeerDependencies:
       - eslint-import-resolver-typescript
@@ -8070,15 +8068,15 @@ packages:
       esrecurse: 4.3.0
       estraverse: 4.3.0
 
-  /eslint-scope@7.2.0:
-    resolution: {integrity: sha512-DYj5deGlHBfMt15J7rdtyKNq/Nqlv5KfU4iodrQ019XESsRnwXH9KAE0y3cwtUHDo2ob7CypAnCqefh6vioWRw==}
+  /eslint-scope@7.2.2:
+    resolution: {integrity: sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==}
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
     dependencies:
       esrecurse: 4.3.0
       estraverse: 5.3.0
 
-  /eslint-visitor-keys@3.4.1:
-    resolution: {integrity: sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA==}
+  /eslint-visitor-keys@3.4.3:
+    resolution: {integrity: sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==}
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
 
   /eslint@8.45.0:
@@ -8087,8 +8085,8 @@ packages:
     hasBin: true
     dependencies:
       '@eslint-community/eslint-utils': 4.4.0(eslint@8.45.0)
-      '@eslint-community/regexpp': 4.5.1
-      '@eslint/eslintrc': 2.1.0
+      '@eslint-community/regexpp': 4.6.2
+      '@eslint/eslintrc': 2.1.2
       '@eslint/js': 8.44.0
       '@humanwhocodes/config-array': 0.11.10
       '@humanwhocodes/module-importer': 1.0.1
@@ -8099,8 +8097,8 @@ packages:
       debug: 4.3.4
       doctrine: 3.0.0
       escape-string-regexp: 4.0.0
-      eslint-scope: 7.2.0
-      eslint-visitor-keys: 3.4.1
+      eslint-scope: 7.2.2
+      eslint-visitor-keys: 3.4.3
       espree: 9.6.1
       esquery: 1.5.0
       esutils: 2.0.3
@@ -8132,7 +8130,7 @@ packages:
     dependencies:
       acorn: 8.10.0
       acorn-jsx: 5.3.2(acorn@8.10.0)
-      eslint-visitor-keys: 3.4.1
+      eslint-visitor-keys: 3.4.3
 
   /esprima@4.0.1:
     resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==}
@@ -8588,6 +8586,7 @@ packages:
       graceful-fs: 4.2.11
       jsonfile: 6.1.0
       universalify: 2.0.0
+    dev: true
 
   /fs-extra@9.1.0:
     resolution: {integrity: sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==}
@@ -9401,6 +9400,12 @@ packages:
     resolution: {integrity: sha512-Q4ZuBAe2FUsKtyQJoQHlvP8OvBERxO3jEmy1I7hcRXcJBGGHFh/aJBswbXuS9sgrDH2QUO8ilkwNPHvHMd8clg==}
     dependencies:
       has: 1.0.3
+    dev: false
+
+  /is-core-module@2.13.0:
+    resolution: {integrity: sha512-Z7dk6Qo8pOCp3l4tsX2C5ZVas4V+UxwQodwZhLopL91TX8UyyHEXafPcyoeeWuLrwzHcr3igO78wNLwHJHsMCQ==}
+    dependencies:
+      has: 1.0.3
 
   /is-date-object@1.0.5:
     resolution: {integrity: sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==}
@@ -9673,17 +9678,17 @@ packages:
       '@babel/parser': 7.22.3
       '@istanbuljs/schema': 0.1.3
       istanbul-lib-coverage: 3.2.0
-      semver: 6.3.0
+      semver: 6.3.1
     transitivePeerDependencies:
       - supports-color
     dev: true
 
-  /istanbul-lib-report@3.0.0:
-    resolution: {integrity: sha512-wcdi+uAKzfiGT2abPpKZ0hSU1rGQjUQnLvtY5MpQ7QCTahD3VODhcu4wcfY1YtkGaDD5yuydOLINXsfbus9ROw==}
-    engines: {node: '>=8'}
+  /istanbul-lib-report@3.0.1:
+    resolution: {integrity: sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==}
+    engines: {node: '>=10'}
     dependencies:
       istanbul-lib-coverage: 3.2.0
-      make-dir: 3.1.0
+      make-dir: 4.0.0
       supports-color: 7.2.0
     dev: true
 
@@ -9703,7 +9708,7 @@ packages:
     engines: {node: '>=8'}
     dependencies:
       html-escaper: 2.0.2
-      istanbul-lib-report: 3.0.0
+      istanbul-lib-report: 3.0.1
     dev: true
 
   /iterall@1.3.0:
@@ -9745,6 +9750,11 @@ packages:
     hasBin: true
     dev: false
 
+  /jiti@1.19.1:
+    resolution: {integrity: sha512-oVhqoRDaBXf7sjkll95LHVS6Myyyb1zaunVwk4Z0+WPSW4gjS0pl01zYKHScTuyEhQsFxV5L4DR5r+YqSyqyyg==}
+    hasBin: true
+    dev: false
+
   /joi@17.9.2:
     resolution: {integrity: sha512-Itk/r+V4Dx0V3c7RLFdRh12IOjySm2/WGPMubBT92cQvRfYZhPM2W0hZlctjj72iES8jsRCwp7S/cRmWBnJ4nw==}
     dependencies:
@@ -9798,6 +9808,7 @@ packages:
 
   /json-schema-traverse@1.0.0:
     resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==}
+    requiresBuild: true
 
   /json-stable-stringify-without-jsonify@1.0.1:
     resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==}
@@ -9830,8 +9841,9 @@ packages:
     engines: {'0': node >= 0.2.0}
     dev: true
 
-  /jsonschema@1.4.1:
-    resolution: {integrity: sha512-S6cATIPVv1z0IlxdN+zUk5EPjkGCdnhN4wVSBlvoUO1tOLJootbo9CquNJmbIh4yikWHiUedhRYrNPn1arpEmQ==}
+  /jwt-decode@3.1.2:
+    resolution: {integrity: sha512-UfpWE/VZn0iP50d8cz9NrZLM9lSWhcJ+0Gt/nm4by88UL+J1SiKN8/5dkjMmbEzwL2CAe+67GsegCbIKtbp75A==}
+    dev: false
 
   /keyv@3.1.0:
     resolution: {integrity: sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==}
@@ -10036,9 +10048,6 @@ packages:
     resolution: {integrity: sha512-+WKqsK294HMSc2jEbNgpHpd0JfIBhp7rEV4aqXWqFr6AlXov+SlcgB1Fv01y2kGe3Gc8nMW7VA0SrGuSkRfIEg==}
     dev: true
 
-  /lodash.truncate@4.4.2:
-    resolution: {integrity: sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==}
-
   /lodash.uniq@4.5.0:
     resolution: {integrity: sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==}
 
@@ -10147,7 +10156,15 @@ packages:
     resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==}
     engines: {node: '>=8'}
     dependencies:
-      semver: 6.3.0
+      semver: 6.3.1
+    dev: false
+
+  /make-dir@4.0.0:
+    resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==}
+    engines: {node: '>=10'}
+    dependencies:
+      semver: 7.5.4
+    dev: true
 
   /make-error@1.3.6:
     resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==}
@@ -10448,7 +10465,7 @@ packages:
     resolution: {integrity: sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==}
     dependencies:
       hosted-git-info: 2.8.9
-      resolve: 1.22.2
+      resolve: 1.22.4
       semver: 5.7.1
       validate-npm-package-license: 3.0.4
     dev: true
@@ -10458,7 +10475,7 @@ packages:
     engines: {node: '>=10'}
     dependencies:
       hosted-git-info: 4.1.0
-      is-core-module: 2.12.1
+      is-core-module: 2.13.0
       semver: 7.5.4
       validate-npm-package-license: 3.0.4
     dev: true
@@ -10708,7 +10725,7 @@ packages:
       got: 9.6.0
       registry-auth-token: 4.2.2
       registry-url: 5.1.0
-      semver: 6.3.0
+      semver: 6.3.1
     dev: false
 
   /pako@2.1.0:
@@ -10979,7 +10996,7 @@ packages:
         optional: true
     dependencies:
       lilconfig: 2.1.0
-      ts-node: 10.9.1(@types/node@18.16.19)(typescript@5.1.6)
+      ts-node: 10.9.1(@types/node@20.4.7)(typescript@5.1.6)
       yaml: 2.3.1
     dev: true
 
@@ -11932,6 +11949,7 @@ packages:
   /require-from-string@2.0.2:
     resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==}
     engines: {node: '>=0.10.0'}
+    requiresBuild: true
 
   /require-like@0.1.2:
     resolution: {integrity: sha512-oyrU88skkMtDdauHDuKVrgR+zuItqr6/c//FXzvmxRGMexSDc6hNvJInGW3LL46n+8b50RykrvwSUIIQH2LQ5A==}
@@ -11976,6 +11994,15 @@ packages:
       is-core-module: 2.12.1
       path-parse: 1.0.7
       supports-preserve-symlinks-flag: 1.0.0
+    dev: false
+
+  /resolve@1.22.4:
+    resolution: {integrity: sha512-PXNdCiPqDqeUou+w1C2eTQbNfxKSuMxqTCuvlmmMsk1NWHL5fRrhY6Pl0qEYYc6+QqGClco1Qj8XnjPego4wfg==}
+    hasBin: true
+    dependencies:
+      is-core-module: 2.13.0
+      path-parse: 1.0.7
+      supports-preserve-symlinks-flag: 1.0.0
 
   /responselike@1.0.2:
     resolution: {integrity: sha512-/Fpe5guzJk1gPqdJLJR5u7eG/gNY4nImjbRDaVWVMRhne55TCmj2i9Q+54PBRfatRC8v/rIiv9BN0pMd9OV5EQ==}
@@ -12146,31 +12173,16 @@ packages:
     resolution: {integrity: sha512-GX0Ix/CJcHyB8c4ykpHGIAvLyOwOobtM/8d+TQkAd81/bEjgPHrfba41Vpesr7jX/t8Uh+R3EX9eAS5be+jQYg==}
     engines: {node: '>=8'}
     dependencies:
-      semver: 6.3.0
+      semver: 6.3.1
     dev: false
 
   /semver@5.7.1:
     resolution: {integrity: sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==}
     hasBin: true
 
-  /semver@6.3.0:
-    resolution: {integrity: sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==}
-    hasBin: true
-
-  /semver@7.5.1:
-    resolution: {integrity: sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw==}
-    engines: {node: '>=10'}
-    hasBin: true
-    dependencies:
-      lru-cache: 6.0.0
-
-  /semver@7.5.2:
-    resolution: {integrity: sha512-SoftuTROv/cRjCze/scjGyiDtcUyxw1rgYQSZY7XTmtR5hX+dm76iDbTH8TkLPHCQmlbQVSSbNZCPM2hb0knnQ==}
-    engines: {node: '>=10'}
+  /semver@6.3.1:
+    resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==}
     hasBin: true
-    dependencies:
-      lru-cache: 6.0.0
-    dev: true
 
   /semver@7.5.4:
     resolution: {integrity: sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==}
@@ -12366,6 +12378,7 @@ packages:
       ansi-styles: 4.3.0
       astral-regex: 2.0.0
       is-fullwidth-code-point: 3.0.0
+    dev: true
 
   /slice-ansi@5.0.0:
     resolution: {integrity: sha512-FC+lgizVPfie0kkhqUScwRu1O/lF6NOgJmlCgK+/LYxDCTk8sGelYaHDhFcDN+Sn3Cv+3VSa4Byeo+IMCzpMgQ==}
@@ -12756,16 +12769,6 @@ packages:
       tslib: 2.5.2
     dev: true
 
-  /table@6.8.1:
-    resolution: {integrity: sha512-Y4X9zqrCftUhMeH2EptSSERdVKt/nEdijTOacGD/97EKjhQ/Qs8RTlEGABSJNNN8lac9kheH+af7yAkEWlgneA==}
-    engines: {node: '>=10.0.0'}
-    dependencies:
-      ajv: 8.12.0
-      lodash.truncate: 4.4.2
-      slice-ansi: 4.0.0
-      string-width: 4.2.3
-      strip-ansi: 6.0.1
-
   /tapable@1.1.3:
     resolution: {integrity: sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==}
     engines: {node: '>=6'}
@@ -12973,7 +12976,7 @@ packages:
     resolution: {integrity: sha512-yUeWbFBDiwPodNqrqpvQpGWheL6PvNu2/pVAb9yy2vzdkkflCgwVA4U2akByPCXzYTum3/5/nB92yKuiLpSo/Q==}
     dev: true
 
-  /ts-node@10.9.1(@types/node@18.16.19)(typescript@5.1.6):
+  /ts-node@10.9.1(@types/node@20.4.7)(typescript@5.1.6):
     resolution: {integrity: sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==}
     hasBin: true
     requiresBuild: true
@@ -12993,7 +12996,7 @@ packages:
       '@tsconfig/node12': 1.0.11
       '@tsconfig/node14': 1.0.3
       '@tsconfig/node16': 1.0.4
-      '@types/node': 18.16.19
+      '@types/node': 20.4.7
       acorn: 8.10.0
       acorn-walk: 8.2.0
       arg: 4.1.3
@@ -13220,9 +13223,9 @@ packages:
   /unconfig@0.3.9:
     resolution: {integrity: sha512-8yhetFd48M641mxrkWA+C/lZU4N0rCOdlo3dFsyFPnBHBjMJfjT/3eAZBRT2RxCRqeBMAKBVgikejdS6yeBjMw==}
     dependencies:
-      '@antfu/utils': 0.7.3
+      '@antfu/utils': 0.7.6
       defu: 6.1.2
-      jiti: 1.18.2
+      jiti: 1.19.1
     dev: false
 
   /unherit@1.1.3:
@@ -14065,6 +14068,7 @@ packages:
   /yaml@1.10.2:
     resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==}
     engines: {node: '>= 6'}
+    dev: false
 
   /yaml@2.3.1:
     resolution: {integrity: sha512-2eHWfjaoXgTBC2jNM1LRef62VQa0umtvRiDSk6HSzW7RvS5YtkabJrwYLLEKWBc8a5U2PTSCs+dJjUTJdlHsWQ==}

From a46d3572c4f673aa8abca2d65647665b6b60673d Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Sat, 19 Aug 2023 20:56:50 -0700
Subject: [PATCH 09/48] feat: :sparkles: add cache API courses locally when
 scraping

---
 .../src/DegreeworksClient.ts                  |  8 ++-
 .../src/PPAPIOfflineClient.ts                 | 29 ++++++++
 tools/degreeworks-scraper/src/index.ts        |  5 +-
 tools/degreeworks-scraper/src/lib.ts          | 66 ++++++++-----------
 4 files changed, 63 insertions(+), 45 deletions(-)
 create mode 100644 tools/degreeworks-scraper/src/PPAPIOfflineClient.ts

diff --git a/tools/degreeworks-scraper/src/DegreeworksClient.ts b/tools/degreeworks-scraper/src/DegreeworksClient.ts
index dac4adc2..c8c297eb 100644
--- a/tools/degreeworks-scraper/src/DegreeworksClient.ts
+++ b/tools/degreeworks-scraper/src/DegreeworksClient.ts
@@ -21,10 +21,9 @@ export class DegreeworksClient {
      * as the catalog year. Otherwise, we use the former.
      */
     const currentYear = new Date().getUTCFullYear();
+    this.catalogYear = `${currentYear}${currentYear + 1}`;
     this.getMajorAudit("BS", "U", "201").then((x) => {
-      this.catalogYear = x
-        ? `${currentYear}${currentYear + 1}`
-        : `${currentYear - 1}${currentYear}`;
+      if (!x) this.catalogYear = `${currentYear - 1}${currentYear}`;
       console.log(`[DegreeworksClient] Set catalogYear to ${this.catalogYear}`);
     });
   }
@@ -45,6 +44,7 @@ export class DegreeworksClient {
       }),
       headers: this.headers,
     });
+    await sleep(this.delay);
     const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
     return "error" in json
       ? undefined
@@ -69,6 +69,7 @@ export class DegreeworksClient {
       }),
       headers: this.headers,
     });
+    await sleep(this.delay);
     const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
     return "error" in json
       ? undefined
@@ -98,6 +99,7 @@ export class DegreeworksClient {
       }),
       headers: this.headers,
     });
+    await sleep(this.delay);
     const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
     return "error" in json
       ? undefined
diff --git a/tools/degreeworks-scraper/src/PPAPIOfflineClient.ts b/tools/degreeworks-scraper/src/PPAPIOfflineClient.ts
new file mode 100644
index 00000000..a0720182
--- /dev/null
+++ b/tools/degreeworks-scraper/src/PPAPIOfflineClient.ts
@@ -0,0 +1,29 @@
+import fetch from "cross-fetch";
+import { isErrorResponse } from "peterportal-api-next-types";
+import type { Course, RawResponse } from "peterportal-api-next-types";
+
+export class PPAPIOfflineClient {
+  private cache: Map<string, Course> = new Map();
+  constructor() {
+    fetch("https://api-next.peterportal.org/v1/rest/courses/all")
+      .then((x) => x.json() as Promise<RawResponse<Course[]>>)
+      .then((x) => {
+        if (isErrorResponse(x))
+          throw new Error("Could not fetch courses cache from PeterPortal API");
+        x.payload.forEach((y) => this.cache.set(y.id, y));
+        console.log(
+          `[PPAPIOfflineClient] Fetched and stored ${x.payload.length} courses from PeterPortal API`,
+        );
+      });
+  }
+
+  getCourse = (courseNumber: string): Course | undefined => this.cache.get(courseNumber);
+
+  getCourses = (
+    department: string,
+    predicate: (x: Course) => boolean = () => true,
+  ): Course[] | undefined =>
+    Array.from(this.cache.values())
+      .filter((x) => x.id.startsWith(department))
+      .filter(predicate);
+}
diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index 15140635..20ffc67d 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -6,7 +6,7 @@ import jwtDecode from "jwt-decode";
 import type { JwtPayload } from "jwt-decode";
 
 import { DegreeworksClient } from "./DegreeworksClient";
-import { parseBlock, sleep } from "./lib";
+import { parseBlock } from "./lib";
 import type { Program } from "./types";
 
 import "dotenv/config";
@@ -43,10 +43,9 @@ async function main() {
     const audit = await dw.getMinorAudit(minorCode);
     if (!audit) {
       console.log(`Requirements block not found (minorCode = ${minorCode})`);
-      await sleep(1000);
       continue;
     }
-    parsedMinorPrograms.set(`U-MINOR-${minorCode}`, await parseBlock(audit));
+    parsedMinorPrograms.set(`U-MINOR-${minorCode}`, parseBlock(audit));
     console.log(
       `Requirements block found and parsed for "${audit.title}" (minorCode = ${minorCode})`,
     );
diff --git a/tools/degreeworks-scraper/src/lib.ts b/tools/degreeworks-scraper/src/lib.ts
index eb1ad20a..25906591 100644
--- a/tools/degreeworks-scraper/src/lib.ts
+++ b/tools/degreeworks-scraper/src/lib.ts
@@ -1,21 +1,20 @@
-import fetch from "cross-fetch";
-import { isErrorResponse } from "peterportal-api-next-types";
-import type { Course, RawResponse } from "peterportal-api-next-types";
+import type { Course } from "peterportal-api-next-types";
 
+import { PPAPIOfflineClient } from "./PPAPIOfflineClient";
 import type { Block, Program, Requirement, Rule } from "./types";
 import { ProgramId } from "./types";
 
-const PPAPI_REST_URL = "https://api-next.peterportal.org/v1/rest";
-
 const electiveMatcher = /ELECTIVE @+/;
-const wildcardMatcher = /\d+@+/;
-const rangeMatcher = /\d+-\d+/;
+const wildcardMatcher = /\w@/;
+const rangeMatcher = /-\w+/;
+
+const ppapi = new PPAPIOfflineClient();
 
 export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
 
-export const parseBlock = async (block: Block): Promise<Program> => ({
+export const parseBlock = (block: Block): Program => ({
   name: block.title,
-  requirements: await ruleArrayToRequirements(block.ruleArray),
+  requirements: ruleArrayToRequirements(block.ruleArray),
   specs: parseSpecs(block.ruleArray),
 });
 
@@ -62,30 +61,13 @@ function flattenIfStmt(ruleArray: Rule[]): Rule[] {
   return ret;
 }
 
-async function getCourse(courseNumber: string): Promise<Course | undefined> {
-  const res = await fetch(`${PPAPI_REST_URL}/courses/${courseNumber}`);
-  await sleep(1000);
-  const json: RawResponse<Course> = await res.json();
-  return isErrorResponse(json) ? undefined : json.payload;
-}
-
-async function getCourses(
-  department: string,
-  predicate: (x: Course) => boolean = () => true,
-): Promise<Course[] | undefined> {
-  const res = await fetch(`${PPAPI_REST_URL}/courses/?department=${department}`);
-  await sleep(1000);
-  const json: RawResponse<Course[]> = await res.json();
-  return isErrorResponse(json) ? undefined : json.payload.filter(predicate);
-}
-
-async function normalizeCourseId(courseIdLike: string): Promise<Course[]> {
+function normalizeCourseId(courseIdLike: string): Course[] {
   // "ELECTIVE @" is typically used as a pseudo-course and can be safely ignored.
   if (courseIdLike.match(electiveMatcher)) return [];
   const [department, courseNumber] = courseIdLike.split(" ");
   if (courseNumber.match(wildcardMatcher)) {
     // Wildcard course numbers.
-    const courses = await getCourses(
+    const courses = ppapi.getCourses(
       department,
       (x) =>
         !!x.courseNumber.match(
@@ -103,7 +85,7 @@ async function normalizeCourseId(courseIdLike: string): Promise<Course[]> {
   if (courseNumber.match(rangeMatcher)) {
     // Course number ranges.
     const [minCourseNumber, maxCourseNumber] = courseNumber.split("-");
-    const courses = await getCourses(
+    const courses = ppapi.getCourses(
       department,
       (x) =>
         x.courseNumeric >= Number.parseInt(minCourseNumber, 10) &&
@@ -112,21 +94,25 @@ async function normalizeCourseId(courseIdLike: string): Promise<Course[]> {
     return courses ?? [];
   }
   // Probably a normal course, just make sure that it exists.
-  const course = await getCourse(`${department}${courseNumber}`);
+  const course = ppapi.getCourse(`${department}${courseNumber}`);
   return course ? [course] : [];
 }
 
-async function ruleArrayToRequirements(ruleArray: Rule[]) {
+function ruleArrayToRequirements(ruleArray: Rule[]) {
   const ret: Record<string, Requirement> = {};
   for (const rule of ruleArray) {
     switch (rule.ruleType) {
+      case "Block":
+        break;
+      case "Noncourse":
+        break;
       case "Course": {
         const includedCourses = rule.requirement.courseArray.map(
           (x) => `${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`,
         );
-        const toInclude = new Set<Course>();
+        const toInclude = new Map<string, Course>();
         for (const id of includedCourses) {
-          (await normalizeCourseId(id)).forEach((x) => toInclude.add(x));
+          normalizeCourseId(id).forEach((x) => toInclude.set(x.id, x));
         }
         const excludedCourses =
           rule.requirement.except?.courseArray.map(
@@ -134,16 +120,18 @@ async function ruleArrayToRequirements(ruleArray: Rule[]) {
           ) ?? [];
         const toExclude = new Set<string>();
         for (const id of excludedCourses) {
-          (await normalizeCourseId(id)).map((x) => x.id).forEach((x) => toExclude.add(x));
+          normalizeCourseId(id)
+            .map((x) => x.id)
+            .forEach((x) => toExclude.add(x));
         }
         const courses = Array.from(toInclude)
-          .filter((x) => !toExclude.has(x.id))
-          .sort((a, b) =>
+          .filter(([x]) => !toExclude.has(x))
+          .sort(([, a], [, b]) =>
             a.department === b.department
               ? a.courseNumeric - b.courseNumeric
               : lexOrd(a.department, b.department),
           )
-          .map((x) => x.id);
+          .map(([x]) => x);
         if (rule.requirement.classesBegin) {
           ret[rule.label] = {
             requirementType: "Course",
@@ -163,7 +151,7 @@ async function ruleArrayToRequirements(ruleArray: Rule[]) {
         ret[rule.label] = {
           requirementType: "Group",
           requirementCount: Number.parseInt(rule.requirement.numberOfGroups),
-          requirements: await ruleArrayToRequirements(rule.ruleArray),
+          requirements: ruleArrayToRequirements(rule.ruleArray),
         };
         break;
       case "IfStmt": {
@@ -172,7 +160,7 @@ async function ruleArrayToRequirements(ruleArray: Rule[]) {
           ret["Select 1 of the following"] = {
             requirementType: "Group",
             requirementCount: 1,
-            requirements: await ruleArrayToRequirements(rules),
+            requirements: ruleArrayToRequirements(rules),
           };
         }
         break;

From 20a7057c33f5dda7ad1846d694291d3cc7e4c645 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Sat, 19 Aug 2023 21:02:47 -0700
Subject: [PATCH 10/48] fix: :bug: parallelize now that we're no longer
 ratelimited

---
 tools/degreeworks-scraper/src/lib.ts | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/tools/degreeworks-scraper/src/lib.ts b/tools/degreeworks-scraper/src/lib.ts
index 25906591..8f62e187 100644
--- a/tools/degreeworks-scraper/src/lib.ts
+++ b/tools/degreeworks-scraper/src/lib.ts
@@ -110,20 +110,16 @@ function ruleArrayToRequirements(ruleArray: Rule[]) {
         const includedCourses = rule.requirement.courseArray.map(
           (x) => `${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`,
         );
-        const toInclude = new Map<string, Course>();
-        for (const id of includedCourses) {
-          normalizeCourseId(id).forEach((x) => toInclude.set(x.id, x));
-        }
+        const toInclude = new Map<string, Course>(
+          includedCourses.flatMap(normalizeCourseId).map((x) => [x.id, x]),
+        );
         const excludedCourses =
           rule.requirement.except?.courseArray.map(
             (x) => `${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`,
           ) ?? [];
-        const toExclude = new Set<string>();
-        for (const id of excludedCourses) {
-          normalizeCourseId(id)
-            .map((x) => x.id)
-            .forEach((x) => toExclude.add(x));
-        }
+        const toExclude = new Set<string>(
+          excludedCourses.flatMap(normalizeCourseId).map((x) => x.id),
+        );
         const courses = Array.from(toInclude)
           .filter(([x]) => !toExclude.has(x))
           .sort(([, a], [, b]) =>

From 3ea98d1a081486b82b35573d0037e7e30b96efc5 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Sat, 19 Aug 2023 21:12:00 -0700
Subject: [PATCH 11/48] fix: :bug: improve course number ordering

---
 tools/degreeworks-scraper/src/lib.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/degreeworks-scraper/src/lib.ts b/tools/degreeworks-scraper/src/lib.ts
index 8f62e187..386ed6c7 100644
--- a/tools/degreeworks-scraper/src/lib.ts
+++ b/tools/degreeworks-scraper/src/lib.ts
@@ -124,7 +124,7 @@ function ruleArrayToRequirements(ruleArray: Rule[]) {
           .filter(([x]) => !toExclude.has(x))
           .sort(([, a], [, b]) =>
             a.department === b.department
-              ? a.courseNumeric - b.courseNumeric
+              ? a.courseNumeric - b.courseNumeric || lexOrd(a.courseNumber, b.courseNumber)
               : lexOrd(a.department, b.department),
           )
           .map(([x]) => x);

From 523e189f889127733e02b9974f040a878fea31c1 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Sun, 20 Aug 2023 11:12:41 -0700
Subject: [PATCH 12/48] feat: :sparkles: add majors, misc refactoring

---
 .../src/PPAPIOfflineClient.ts                 |  6 +-
 tools/degreeworks-scraper/src/index.ts        | 55 +++++++++++++++++--
 tools/degreeworks-scraper/src/lib.ts          |  6 +-
 3 files changed, 56 insertions(+), 11 deletions(-)

diff --git a/tools/degreeworks-scraper/src/PPAPIOfflineClient.ts b/tools/degreeworks-scraper/src/PPAPIOfflineClient.ts
index a0720182..6fbb5280 100644
--- a/tools/degreeworks-scraper/src/PPAPIOfflineClient.ts
+++ b/tools/degreeworks-scraper/src/PPAPIOfflineClient.ts
@@ -3,7 +3,7 @@ import { isErrorResponse } from "peterportal-api-next-types";
 import type { Course, RawResponse } from "peterportal-api-next-types";
 
 export class PPAPIOfflineClient {
-  private cache: Map<string, Course> = new Map();
+  private cache = new Map<string, Course>();
   constructor() {
     fetch("https://api-next.peterportal.org/v1/rest/courses/all")
       .then((x) => x.json() as Promise<RawResponse<Course[]>>)
@@ -19,10 +19,10 @@ export class PPAPIOfflineClient {
 
   getCourse = (courseNumber: string): Course | undefined => this.cache.get(courseNumber);
 
-  getCourses = (
+  getCoursesByDepartment = (
     department: string,
     predicate: (x: Course) => boolean = () => true,
-  ): Course[] | undefined =>
+  ): Course[] =>
     Array.from(this.cache.values())
       .filter((x) => x.id.startsWith(department))
       .filter(predicate);
diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index 20ffc67d..5c9c3bd0 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -50,11 +50,58 @@ async function main() {
       `Requirements block found and parsed for "${audit.title}" (minorCode = ${minorCode})`,
     );
   }
+  const parsedUgradPrograms = new Map<string, Program>();
+  const degreesAwarded = new Map<string, string>();
+  console.log("Scraping undergraduate program requirements");
+  for (const degree of undergraduateDegrees) {
+    for (const majorCode of majorPrograms) {
+      const audit = await dw.getMajorAudit(degree, "U", majorCode);
+      if (!audit) {
+        console.log(`Requirements block not found (majorCode = ${majorCode}, degree = ${degree})`);
+        continue;
+      }
+      degreesAwarded.set(degree, degrees.get(degree) ?? "");
+      parsedUgradPrograms.set(`U-MAJOR-${majorCode}-${degree}`, parseBlock(audit));
+      console.log(
+        `Requirements block found and parsed for "${audit.title}" (majorCode = ${majorCode}, degree = ${degree})`,
+      );
+    }
+  }
+  const parsedGradPrograms = new Map<string, Program>();
+  console.log("Scraping graduate program requirements");
+  for (const degree of graduateDegrees) {
+    for (const majorCode of majorPrograms) {
+      const audit = await dw.getMajorAudit(degree, "G", majorCode);
+      if (!audit) {
+        console.log(`Requirements block not found (majorCode = ${majorCode}, degree = ${degree})`);
+        continue;
+      }
+      degreesAwarded.set(degree, degrees.get(degree) ?? "");
+      parsedGradPrograms.set(`G-MAJOR-${majorCode}-${degree}`, parseBlock(audit));
+      console.log(
+        `Requirements block found and parsed for "${audit.title}" (majorCode = ${majorCode}, degree = ${degree})`,
+      );
+    }
+  }
   await mkdir(join(__dirname, "../output"), { recursive: true });
-  await writeFile(
-    join(__dirname, "../output/parsedMinorPrograms.json"),
-    JSON.stringify(Object.fromEntries(parsedMinorPrograms.entries())),
-  );
+  await Promise.all([
+    writeFile(
+      join(__dirname, "../output/parsedMinorPrograms.json"),
+      JSON.stringify(Object.fromEntries(parsedMinorPrograms.entries())),
+    ),
+    writeFile(
+      join(__dirname, "../output/parsedUgradPrograms.json"),
+      JSON.stringify(Object.fromEntries(parsedUgradPrograms.entries())),
+    ),
+    writeFile(
+      join(__dirname, "../output/parsedGradPrograms.json"),
+      JSON.stringify(Object.fromEntries(parsedGradPrograms.entries())),
+    ),
+    writeFile(
+      join(__dirname, "../output/degreesAwarded.json"),
+      JSON.stringify(Object.fromEntries(degreesAwarded.entries())),
+    ),
+  ]);
 }
 
 main().then();
diff --git a/tools/degreeworks-scraper/src/lib.ts b/tools/degreeworks-scraper/src/lib.ts
index 386ed6c7..b8b98e98 100644
--- a/tools/degreeworks-scraper/src/lib.ts
+++ b/tools/degreeworks-scraper/src/lib.ts
@@ -67,7 +67,7 @@ function normalizeCourseId(courseIdLike: string): Course[] {
   const [department, courseNumber] = courseIdLike.split(" ");
   if (courseNumber.match(wildcardMatcher)) {
     // Wildcard course numbers.
-    const courses = ppapi.getCourses(
+    return ppapi.getCoursesByDepartment(
       department,
       (x) =>
         !!x.courseNumber.match(
@@ -80,18 +80,16 @@ function normalizeCourseId(courseIdLike: string): Course[] {
           ),
         ),
     );
-    return courses ?? [];
   }
   if (courseNumber.match(rangeMatcher)) {
     // Course number ranges.
     const [minCourseNumber, maxCourseNumber] = courseNumber.split("-");
-    const courses = ppapi.getCourses(
+    return ppapi.getCoursesByDepartment(
       department,
       (x) =>
         x.courseNumeric >= Number.parseInt(minCourseNumber, 10) &&
         x.courseNumeric <= Number.parseInt(maxCourseNumber, 10),
     );
-    return courses ?? [];
   }
   // Probably a normal course, just make sure that it exists.
   const course = ppapi.getCourse(`${department}${courseNumber}`);

From 2a3db6716bb750e28d9c1b712e9a1253ca59c76f Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Sun, 20 Aug 2023 12:58:59 -0700
Subject: [PATCH 13/48] refactor: :recycle: use static async factory &
 encapsulate functions

- Use static async factories for the DegreeWorks and PPAPI clients. This
  prevents logging weirdness and also ensures that all required fields
  are actually initialized before instance methods are called.
- Encapsulate all audit-related functions into an AuditParser class.
  Like the other clients this also uses a static async factory, since it
  requires the PPAPI client to normalize course numbers.
---
 tools/degreeworks-scraper/src/AuditParser.ts  | 183 ++++++++++++++++++
 .../src/DegreeworksClient.ts                  |  40 ++--
 .../src/PPAPIOfflineClient.ts                 |  24 +--
 tools/degreeworks-scraper/src/index.ts        |  11 +-
 tools/degreeworks-scraper/src/lib.ts          | 170 ----------------
 5 files changed, 227 insertions(+), 201 deletions(-)
 create mode 100644 tools/degreeworks-scraper/src/AuditParser.ts
 delete mode 100644 tools/degreeworks-scraper/src/lib.ts

diff --git a/tools/degreeworks-scraper/src/AuditParser.ts b/tools/degreeworks-scraper/src/AuditParser.ts
new file mode 100644
index 00000000..45783eee
--- /dev/null
+++ b/tools/degreeworks-scraper/src/AuditParser.ts
@@ -0,0 +1,183 @@
+import type { Course } from "peterportal-api-next-types";
+
+import { PPAPIOfflineClient } from "./PPAPIOfflineClient";
+import type { Block, Program, ProgramId, Requirement, Rule } from "./types";
+
+export class AuditParser {
+  private static electiveMatcher = /ELECTIVE @+/;
+  private static wildcardMatcher = /\w@/;
+  private static rangeMatcher = /-\w+/;
+
+  /**
+   * This will always be properly initialized using the static async factory function (`new()`).
+   */
+  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+  // @ts-ignore TS2564
+  private ppapi: PPAPIOfflineClient;
+
+  private constructor() {}
+
+  static async new(): Promise<AuditParser> {
+    const ap = new AuditParser();
+    ap.ppapi = await PPAPIOfflineClient.new();
+    console.log("[AuditParser.new] AuditParser initialized");
+    return ap;
+  }
+
+  parseBlock = (block: Block): Program => ({
+    name: block.title,
+    requirements: this.ruleArrayToRequirements(block.ruleArray),
+    specs: this.parseSpecs(block.ruleArray),
+  });
+
+  lexOrd = new Intl.Collator().compare;
+
+  parseSpecs = (ruleArray: Rule[]) =>
+    ruleArray
+      .filter((x) => x.ruleType === "IfStmt")
+      .flatMap((x) => this.ifStmtToSpecArray([x]))
+      .sort();
+
+  ifStmtToSpecArray(ruleArray: Rule[]): string[] {
+    const ret = [];
+    for (const rule of ruleArray) {
+      switch (rule.ruleType) {
+        case "IfStmt":
+          ret.push(
+            ...this.ifStmtToSpecArray(rule.requirement.ifPart.ruleArray),
+            ...this.ifStmtToSpecArray(rule.requirement.elsePart?.ruleArray ?? []),
+          );
+          break;
+        case "Block":
+          ret.push(rule.requirement.value);
+          break;
+      }
+    }
+    return ret;
+  }
+
+  flattenIfStmt(ruleArray: Rule[]): Rule[] {
+    const ret = [];
+    for (const rule of ruleArray) {
+      switch (rule.ruleType) {
+        case "IfStmt":
+          ret.push(
+            ...this.flattenIfStmt(rule.requirement.ifPart.ruleArray),
+            ...this.flattenIfStmt(rule.requirement.elsePart?.ruleArray ?? []),
+          );
+          break;
+        default:
+          ret.push(rule);
+      }
+    }
+    return ret;
+  }
+
+  normalizeCourseId(courseIdLike: string): Course[] {
+    // "ELECTIVE @" is typically used as a pseudo-course and can be safely ignored.
+    if (courseIdLike.match(AuditParser.electiveMatcher)) return [];
+    const [department, courseNumber] = courseIdLike.split(" ");
+    if (courseNumber.match(AuditParser.wildcardMatcher)) {
+      // Wildcard course numbers.
+      return this.ppapi.getCoursesByDepartment(
+        department,
+        (x) =>
+          !!x.courseNumber.match(
+            new RegExp(
+              "^" +
+                courseNumber.replace(
+                  /@+/g,
+                  `.{${[...courseNumber].filter((y) => y === "@").length},}`,
+                ),
+            ),
+          ),
+      );
+    }
+    if (courseNumber.match(AuditParser.rangeMatcher)) {
+      // Course number ranges.
+      const [minCourseNumber, maxCourseNumber] = courseNumber.split("-");
+      return this.ppapi.getCoursesByDepartment(
+        department,
+        (x) =>
+          x.courseNumeric >= Number.parseInt(minCourseNumber, 10) &&
+          x.courseNumeric <= Number.parseInt(maxCourseNumber, 10),
+      );
+    }
+    // Probably a normal course, just make sure that it exists.
+    const course = this.ppapi.getCourse(`${department}${courseNumber}`);
+    return course ? [course] : [];
+  }
+
+  ruleArrayToRequirements(ruleArray: Rule[]) {
+    const ret: Record<string, Requirement> = {};
+    for (const rule of ruleArray) {
+      switch (rule.ruleType) {
+        case "Block":
+          break;
+        case "Noncourse":
+          break;
+        case "Course": {
+          const includedCourses = rule.requirement.courseArray.map(
+            (x) => `${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`,
+          );
+          const toInclude = new Map<string, Course>(
+            includedCourses.flatMap(this.normalizeCourseId).map((x) => [x.id, x]),
+          );
+          const excludedCourses =
+            rule.requirement.except?.courseArray.map(
+              (x) => `${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`,
+            ) ?? [];
+          const toExclude = new Set<string>(
+            excludedCourses.flatMap(this.normalizeCourseId).map((x) => x.id),
+          );
+          const courses = Array.from(toInclude)
+            .filter(([x]) => !toExclude.has(x))
+            .sort(([, a], [, b]) =>
+              a.department === b.department
+                ? a.courseNumeric - b.courseNumeric || this.lexOrd(a.courseNumber, b.courseNumber)
+                : this.lexOrd(a.department, b.department),
+            )
+            .map(([x]) => x);
+          if (rule.requirement.classesBegin) {
+            ret[rule.label] = {
+              requirementType: "Course",
+              courseCount: Number.parseInt(rule.requirement.classesBegin, 10),
+              courses,
+            };
+          } else if (rule.requirement.creditsBegin) {
+            ret[rule.label] = {
+              requirementType: "Unit",
+              unitCount: Number.parseInt(rule.requirement.creditsBegin, 10),
+              courses,
+            };
+          }
+          break;
+        }
+        case "Group":
+          ret[rule.label] = {
+            requirementType: "Group",
+            requirementCount: Number.parseInt(rule.requirement.numberOfGroups),
+            requirements: this.ruleArrayToRequirements(rule.ruleArray),
+          };
+          break;
+        case "IfStmt": {
+          const rules = this.flattenIfStmt([rule]);
+          if (rules.length > 1 && !rules.some((x) => x.ruleType === "Block")) {
+            ret["Select 1 of the following"] = {
+              requirementType: "Group",
+              requirementCount: 1,
+              requirements: this.ruleArrayToRequirements(rules),
+            };
+          }
+          break;
+        }
+      }
+    }
+    return ret;
+  }
+
+  parseBlockId(blockId: string) {
+    const [school, programType, code, degreeType] = blockId.split("-");
+    return { school, programType, code, degreeType } as ProgramId;
+  }
+}
diff --git a/tools/degreeworks-scraper/src/DegreeworksClient.ts b/tools/degreeworks-scraper/src/DegreeworksClient.ts
index c8c297eb..d3dc9f69 100644
--- a/tools/degreeworks-scraper/src/DegreeworksClient.ts
+++ b/tools/degreeworks-scraper/src/DegreeworksClient.ts
@@ -1,32 +1,42 @@
 import fetch from "cross-fetch";
 
-import { sleep } from "./lib";
 import type { Block, DWAuditResponse, DWMappingResponse } from "./types";
 
 export class DegreeworksClient {
   private static readonly API_URL = "https://reg.uci.edu/RespDashboard/api";
   private static readonly AUDIT_URL = `${DegreeworksClient.API_URL}/audit`;
   private catalogYear: string = "";
-  constructor(
+  private constructor(
     private readonly studentId: string,
     private readonly headers: HeadersInit,
-    private readonly delay: number = 1000,
-  ) {
+    private readonly delay: number,
+  ) {}
+
+  static async new(
+    studentId: string,
+    headers: HeadersInit,
+    delay: number = 1000,
+  ): Promise<DegreeworksClient> {
+    const dw = new DegreeworksClient(studentId, headers, delay);
     /**
-     * Depending on when we are scraping, this may be the academic year that started
-     * the previous calendar year, or the one that will start this calendar year.
+     * Depending on when we are scraping, the catalog year may be the academic year that
+     * started the previous calendar year, or the one that will start this calendar year.
      *
      * We determine the catalog year by seeing if we can fetch the major data for the
      * B.S. in Computer Science for the latter. If it is available, then we use that
      * as the catalog year. Otherwise, we use the former.
      */
     const currentYear = new Date().getUTCFullYear();
-    this.catalogYear = `${currentYear}${currentYear + 1}`;
-    this.getMajorAudit("BS", "U", "201").then((x) => {
-      if (!x) this.catalogYear = `${currentYear - 1}${currentYear}`;
-      console.log(`[DegreeworksClient] Set catalogYear to ${this.catalogYear}`);
-    });
+    dw.catalogYear = `${currentYear}${currentYear + 1}`;
+    if (!(await dw.getMajorAudit("BS", "U", "201"))) {
+      dw.catalogYear = `${currentYear - 1}${currentYear}`;
+    }
+    console.log(`[DegreeworksClient.new] Set catalogYear to ${dw.catalogYear}`);
+    return dw;
   }
+
+  sleep = (ms: number = this.delay) => new Promise((r) => setTimeout(r, ms));
+
   async getMajorAudit(
     degree: string,
     school: string,
@@ -44,7 +54,7 @@ export class DegreeworksClient {
       }),
       headers: this.headers,
     });
-    await sleep(this.delay);
+    await this.sleep();
     const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
     return "error" in json
       ? undefined
@@ -69,7 +79,7 @@ export class DegreeworksClient {
       }),
       headers: this.headers,
     });
-    await sleep(this.delay);
+    await this.sleep();
     const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
     return "error" in json
       ? undefined
@@ -99,7 +109,7 @@ export class DegreeworksClient {
       }),
       headers: this.headers,
     });
-    await sleep(this.delay);
+    await this.sleep();
     const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
     return "error" in json
       ? undefined
@@ -110,7 +120,7 @@ export class DegreeworksClient {
 
   async getMapping<T extends string>(path: T): Promise<Map<string, string>> {
     const res = await fetch(`${DegreeworksClient.API_URL}/${path}`, { headers: this.headers });
-    await sleep(this.delay);
+    await this.sleep();
     const json: DWMappingResponse<T> = await res.json();
     return new Map(json._embedded[path].map((x) => [x.key, x.description]));
   }
diff --git a/tools/degreeworks-scraper/src/PPAPIOfflineClient.ts b/tools/degreeworks-scraper/src/PPAPIOfflineClient.ts
index 6fbb5280..c19809f6 100644
--- a/tools/degreeworks-scraper/src/PPAPIOfflineClient.ts
+++ b/tools/degreeworks-scraper/src/PPAPIOfflineClient.ts
@@ -4,17 +4,19 @@ import type { Course, RawResponse } from "peterportal-api-next-types";
 
 export class PPAPIOfflineClient {
   private cache = new Map<string, Course>();
-  constructor() {
-    fetch("https://api-next.peterportal.org/v1/rest/courses/all")
-      .then((x) => x.json() as Promise<RawResponse<Course[]>>)
-      .then((x) => {
-        if (isErrorResponse(x))
-          throw new Error("Could not fetch courses cache from PeterPortal API");
-        x.payload.forEach((y) => this.cache.set(y.id, y));
-        console.log(
-          `[PPAPIOfflineClient] Fetched and stored ${x.payload.length} courses from PeterPortal API`,
-        );
-      });
+  private constructor() {}
+
+  static async new(): Promise<PPAPIOfflineClient> {
+    const ppapi = new PPAPIOfflineClient();
+    const res = await fetch("https://api-next.peterportal.org/v1/rest/courses/all");
+    const json: RawResponse<Course[]> = await res.json();
+    if (isErrorResponse(json))
+      throw new Error("Could not fetch courses cache from PeterPortal API");
+    json.payload.forEach((y) => ppapi.cache.set(y.id, y));
+    console.log(
+      `[PPAPIOfflineClient.new] Fetched and stored ${json.payload.length} courses from PeterPortal API`,
+    );
+    return ppapi;
   }
 
   getCourse = (courseNumber: string): Course | undefined => this.cache.get(courseNumber);
diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index 5c9c3bd0..52c2a4d1 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -5,8 +5,8 @@ import { fileURLToPath } from "node:url";
 import jwtDecode from "jwt-decode";
 import type { JwtPayload } from "jwt-decode";
 
+import { AuditParser } from "./AuditParser";
 import { DegreeworksClient } from "./DegreeworksClient";
-import { parseBlock } from "./lib";
 import type { Program } from "./types";
 
 import "dotenv/config";
@@ -24,7 +24,7 @@ async function main() {
     Origin: "https://reg.uci.edu",
   };
   console.log("degreeworks-scraper starting");
-  const dw = new DegreeworksClient(studentId, headers);
+  const dw = await DegreeworksClient.new(studentId, headers);
   const degrees = await dw.getMapping("degrees");
   console.log(`Fetched ${degrees.size} degrees`);
   const majorPrograms = new Set((await dw.getMapping("majors")).keys());
@@ -37,6 +37,7 @@ async function main() {
   for (const degree of degrees.keys())
     (degree.startsWith("B") ? undergraduateDegrees : graduateDegrees).add(degree);
 
+  const ap = await AuditParser.new();
   const parsedMinorPrograms = new Map<string, Program>();
   console.log("Scraping minor program requirements");
   for (const minorCode of minorPrograms) {
@@ -45,7 +46,7 @@ async function main() {
       console.log(`Requirements block not found (minorCode = ${minorCode})`);
       continue;
     }
-    parsedMinorPrograms.set(`U-MINOR-${minorCode}`, parseBlock(audit));
+    parsedMinorPrograms.set(`U-MINOR-${minorCode}`, ap.parseBlock(audit));
     console.log(
       `Requirements block found and parsed for "${audit.title}" (minorCode = ${minorCode})`,
     );
@@ -61,7 +62,7 @@ async function main() {
         continue;
       }
       degreesAwarded.set(degree, degrees.get(degree) ?? "");
-      parsedUgradPrograms.set(`U-MAJOR-${majorCode}-${degree}`, parseBlock(audit));
+      parsedUgradPrograms.set(`U-MAJOR-${majorCode}-${degree}`, ap.parseBlock(audit));
       console.log(
         `Requirements block found and parsed for "${audit.title}" (majorCode = ${majorCode}, degree = ${degree})`,
       );
@@ -77,7 +78,7 @@ async function main() {
         continue;
       }
       degreesAwarded.set(degree, degrees.get(degree) ?? "");
-      parsedGradPrograms.set(`G-MAJOR-${majorCode}-${degree}`, parseBlock(audit));
+      parsedGradPrograms.set(`G-MAJOR-${majorCode}-${degree}`, ap.parseBlock(audit));
       console.log(
         `Requirements block found and parsed for "${audit.title}" (majorCode = ${majorCode}, degree = ${degree})`,
       );
diff --git a/tools/degreeworks-scraper/src/lib.ts b/tools/degreeworks-scraper/src/lib.ts
deleted file mode 100644
index b8b98e98..00000000
--- a/tools/degreeworks-scraper/src/lib.ts
+++ /dev/null
@@ -1,170 +0,0 @@
-import type { Course } from "peterportal-api-next-types";
-
-import { PPAPIOfflineClient } from "./PPAPIOfflineClient";
-import type { Block, Program, Requirement, Rule } from "./types";
-import { ProgramId } from "./types";
-
-const electiveMatcher = /ELECTIVE @+/;
-const wildcardMatcher = /\w@/;
-const rangeMatcher = /-\w+/;
-
-const ppapi = new PPAPIOfflineClient();
-
-export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
-
-export const parseBlock = (block: Block): Program => ({
-  name: block.title,
-  requirements: ruleArrayToRequirements(block.ruleArray),
-  specs: parseSpecs(block.ruleArray),
-});
-
-const lexOrd = new Intl.Collator().compare;
-
-const parseSpecs = (ruleArray: Rule[]) =>
-  ruleArray
-    .filter((x) => x.ruleType === "IfStmt")
-    .flatMap((x) => ifStmtToSpecArray([x]))
-    .sort();
-
-function ifStmtToSpecArray(ruleArray: Rule[]): string[] {
-  const ret = [];
-  for (const rule of ruleArray) {
-    switch (rule.ruleType) {
-      case "IfStmt":
-        ret.push(
-          ...ifStmtToSpecArray(rule.requirement.ifPart.ruleArray),
-          ...ifStmtToSpecArray(rule.requirement.elsePart?.ruleArray ?? []),
-        );
-        break;
-      case "Block":
-        ret.push(rule.requirement.value);
-        break;
-    }
-  }
-  return ret;
-}
-
-function flattenIfStmt(ruleArray: Rule[]): Rule[] {
-  const ret = [];
-  for (const rule of ruleArray) {
-    switch (rule.ruleType) {
-      case "IfStmt":
-        ret.push(
-          ...flattenIfStmt(rule.requirement.ifPart.ruleArray),
-          ...flattenIfStmt(rule.requirement.elsePart?.ruleArray ?? []),
-        );
-        break;
-      default:
-        ret.push(rule);
-    }
-  }
-  return ret;
-}
-
-function normalizeCourseId(courseIdLike: string): Course[] {
-  // "ELECTIVE @" is typically used as a pseudo-course and can be safely ignored.
-  if (courseIdLike.match(electiveMatcher)) return [];
-  const [department, courseNumber] = courseIdLike.split(" ");
-  if (courseNumber.match(wildcardMatcher)) {
-    // Wildcard course numbers.
-    return ppapi.getCoursesByDepartment(
-      department,
-      (x) =>
-        !!x.courseNumber.match(
-          new RegExp(
-            "^" +
-              courseNumber.replace(
-                /@+/g,
-                `.{${[...courseNumber].filter((y) => y === "@").length},}`,
-              ),
-          ),
-        ),
-    );
-  }
-  if (courseNumber.match(rangeMatcher)) {
-    // Course number ranges.
-    const [minCourseNumber, maxCourseNumber] = courseNumber.split("-");
-    return ppapi.getCoursesByDepartment(
-      department,
-      (x) =>
-        x.courseNumeric >= Number.parseInt(minCourseNumber, 10) &&
-        x.courseNumeric <= Number.parseInt(maxCourseNumber, 10),
-    );
-  }
-  // Probably a normal course, just make sure that it exists.
-  const course = ppapi.getCourse(`${department}${courseNumber}`);
-  return course ? [course] : [];
-}
-
-function ruleArrayToRequirements(ruleArray: Rule[]) {
-  const ret: Record<string, Requirement> = {};
-  for (const rule of ruleArray) {
-    switch (rule.ruleType) {
-      case "Block":
-        break;
-      case "Noncourse":
-        break;
-      case "Course": {
-        const includedCourses = rule.requirement.courseArray.map(
-          (x) => `${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`,
-        );
-        const toInclude = new Map<string, Course>(
-          includedCourses.flatMap(normalizeCourseId).map((x) => [x.id, x]),
-        );
-        const excludedCourses =
-          rule.requirement.except?.courseArray.map(
-            (x) => `${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`,
-          ) ?? [];
-        const toExclude = new Set<string>(
-          excludedCourses.flatMap(normalizeCourseId).map((x) => x.id),
-        );
-        const courses = Array.from(toInclude)
-          .filter(([x]) => !toExclude.has(x))
-          .sort(([, a], [, b]) =>
-            a.department === b.department
-              ? a.courseNumeric - b.courseNumeric || lexOrd(a.courseNumber, b.courseNumber)
-              : lexOrd(a.department, b.department),
-          )
-          .map(([x]) => x);
-        if (rule.requirement.classesBegin) {
-          ret[rule.label] = {
-            requirementType: "Course",
-            courseCount: Number.parseInt(rule.requirement.classesBegin, 10),
-            courses,
-          };
-        } else if (rule.requirement.creditsBegin) {
-          ret[rule.label] = {
-            requirementType: "Unit",
-            unitCount: Number.parseInt(rule.requirement.creditsBegin, 10),
-            courses,
-          };
-        }
-        break;
-      }
-      case "Group":
-        ret[rule.label] = {
-          requirementType: "Group",
-          requirementCount: Number.parseInt(rule.requirement.numberOfGroups),
-          requirements: ruleArrayToRequirements(rule.ruleArray),
-        };
-        break;
-      case "IfStmt": {
-        const rules = flattenIfStmt([rule]);
-        if (rules.length > 1 && !rules.some((x) => x.ruleType === "Block")) {
-          ret["Select 1 of the following"] = {
-            requirementType: "Group",
-            requirementCount: 1,
-            requirements: ruleArrayToRequirements(rules),
-          };
-        }
-        break;
-      }
-    }
-  }
-  return ret;
-}
-
-export function parseBlockId(blockId: string) {
-  const [school, programType, code, degreeType] = blockId.split("-");
-  return { school, programType, code, degreeType } as ProgramId;
-}

From c8be84193459257ca062c63eaa230faa0eaa195e Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Sun, 20 Aug 2023 14:01:36 -0700
Subject: [PATCH 14/48] fix: :bug: address method binding issues

---
 tools/degreeworks-scraper/src/AuditParser.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/degreeworks-scraper/src/AuditParser.ts b/tools/degreeworks-scraper/src/AuditParser.ts
index 45783eee..e39c61ba 100644
--- a/tools/degreeworks-scraper/src/AuditParser.ts
+++ b/tools/degreeworks-scraper/src/AuditParser.ts
@@ -103,7 +103,7 @@ export class AuditParser {
           x.courseNumeric <= Number.parseInt(maxCourseNumber, 10),
       );
     }
-    // Probably a normal course, just make sure that it exists.
+    // Probably a normal course, just make sure that it exists.=
     const course = this.ppapi.getCourse(`${department}${courseNumber}`);
     return course ? [course] : [];
   }
@@ -121,14 +121,14 @@ export class AuditParser {
             (x) => `${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`,
           );
           const toInclude = new Map<string, Course>(
-            includedCourses.flatMap(this.normalizeCourseId).map((x) => [x.id, x]),
+            includedCourses.flatMap(this.normalizeCourseId.bind(this)).map((x) => [x.id, x]),
           );
           const excludedCourses =
             rule.requirement.except?.courseArray.map(
               (x) => `${x.discipline} ${x.number}${x.numberEnd ? `-${x.numberEnd}` : ""}`,
             ) ?? [];
           const toExclude = new Set<string>(
-            excludedCourses.flatMap(this.normalizeCourseId).map((x) => x.id),
+            excludedCourses.flatMap(this.normalizeCourseId.bind(this)).map((x) => x.id),
           );
           const courses = Array.from(toInclude)
             .filter(([x]) => !toExclude.has(x))

From 3f9504846e364f90c22bd1ec581d1f2170825f52 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Sun, 20 Aug 2023 14:02:33 -0700
Subject: [PATCH 15/48] feat: :sparkles: scrape specializations

---
 tools/degreeworks-scraper/src/index.ts | 33 ++++++++++++++++++--------
 1 file changed, 23 insertions(+), 10 deletions(-)

diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index 52c2a4d1..19ab35ed 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -51,7 +51,7 @@ async function main() {
       `Requirements block found and parsed for "${audit.title}" (minorCode = ${minorCode})`,
     );
   }
-  const parsedUgradPrograms = new Map<string, Program>();
+  const parsedMajorsAndSpecs = new Map<string, Program>();
   const degreesAwarded = new Map<string, string>();
   console.log("Scraping undergraduate program requirements");
   for (const degree of undergraduateDegrees) {
@@ -62,13 +62,12 @@ async function main() {
         continue;
       }
       degreesAwarded.set(degree, degrees.get(degree) ?? "");
-      parsedUgradPrograms.set(`U-MAJOR-${majorCode}-${degree}`, ap.parseBlock(audit));
+      parsedMajorsAndSpecs.set(`U-MAJOR-${majorCode}-${degree}`, ap.parseBlock(audit));
       console.log(
         `Requirements block found and parsed for "${audit.title}" (majorCode = ${majorCode}, degree = ${degree})`,
       );
     }
   }
-  const parsedGradPrograms = new Map<string, Program>();
   console.log("Scraping graduate program requirements");
   for (const degree of graduateDegrees) {
     for (const majorCode of majorPrograms) {
@@ -78,12 +77,30 @@ async function main() {
         continue;
       }
       degreesAwarded.set(degree, degrees.get(degree) ?? "");
-      parsedGradPrograms.set(`G-MAJOR-${majorCode}-${degree}`, ap.parseBlock(audit));
+      parsedMajorsAndSpecs.set(`G-MAJOR-${majorCode}-${degree}`, ap.parseBlock(audit));
       console.log(
         `Requirements block found and parsed for "${audit.title}" (majorCode = ${majorCode}, degree = ${degree})`,
       );
     }
   }
+  console.log("Scraping all specialization requirements");
+  for (const [blockId, { specs }] of parsedMajorsAndSpecs) {
+    const { school, code: majorCode, degreeType: degree } = ap.parseBlockId(blockId);
+    if (!degree) throw new Error(`Could not parse degree type from malformed blockId "${blockId}"`);
+    for (const specCode of specs) {
+      const audit = await dw.getSpecAudit(degree, school, majorCode, specCode);
+      if (!audit) {
+        console.log(
+          `Requirements block not found (school = ${school}, majorCode = ${majorCode}, specCode = ${specCode}, degree = ${degree})`,
+        );
+        continue;
+      }
+      parsedMajorsAndSpecs.set(`${school}-SPEC-${specCode}-${degree}`, ap.parseBlock(audit));
+      console.log(
+        `Requirements block found and parsed for "${audit.title}" (school = ${school}, majorCode = ${majorCode}, specCode = ${specCode}, degree = ${degree})`,
+      );
+    }
+  }
   await mkdir(join(__dirname, "../output"), { recursive: true });
   await Promise.all([
     writeFile(
@@ -91,12 +108,8 @@ async function main() {
       JSON.stringify(Object.fromEntries(parsedMinorPrograms.entries())),
     ),
     writeFile(
-      join(__dirname, "../output/parsedUgradPrograms.json"),
-      JSON.stringify(Object.fromEntries(parsedUgradPrograms.entries())),
-    ),
-    writeFile(
-      join(__dirname, "../output/parsedGradPrograms.json"),
-      JSON.stringify(Object.fromEntries(parsedGradPrograms.entries())),
+      join(__dirname, "../output/parsedMajorsAndSpecs.json"),
+      JSON.stringify(Object.fromEntries(parsedMajorsAndSpecs.entries())),
     ),
     writeFile(
       join(__dirname, "../output/degreesAwarded.json"),

From 2f356beef4222c777fbdf9a3d2ca8565d4b47349 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Sun, 20 Aug 2023 15:52:07 -0700
Subject: [PATCH 16/48] chore: :wrench: =

---
 tools/degreeworks-scraper/src/AuditParser.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/degreeworks-scraper/src/AuditParser.ts b/tools/degreeworks-scraper/src/AuditParser.ts
index e39c61ba..8ecbeaad 100644
--- a/tools/degreeworks-scraper/src/AuditParser.ts
+++ b/tools/degreeworks-scraper/src/AuditParser.ts
@@ -103,7 +103,7 @@ export class AuditParser {
           x.courseNumeric <= Number.parseInt(maxCourseNumber, 10),
       );
     }
-    // Probably a normal course, just make sure that it exists.=
+    // Probably a normal course, just make sure that it exists.
     const course = this.ppapi.getCourse(`${department}${courseNumber}`);
     return course ? [course] : [];
   }

From d703fa76e9d61f638c7b01c6d390751858708091 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Sun, 20 Aug 2023 16:37:57 -0700
Subject: [PATCH 17/48] refactor: :recycle: add readonly modifiers

---
 tools/degreeworks-scraper/src/AuditParser.ts        | 6 +++---
 tools/degreeworks-scraper/src/DegreeworksClient.ts  | 1 +
 tools/degreeworks-scraper/src/PPAPIOfflineClient.ts | 1 +
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/tools/degreeworks-scraper/src/AuditParser.ts b/tools/degreeworks-scraper/src/AuditParser.ts
index 8ecbeaad..0983189e 100644
--- a/tools/degreeworks-scraper/src/AuditParser.ts
+++ b/tools/degreeworks-scraper/src/AuditParser.ts
@@ -4,9 +4,9 @@ import { PPAPIOfflineClient } from "./PPAPIOfflineClient";
 import type { Block, Program, ProgramId, Requirement, Rule } from "./types";
 
 export class AuditParser {
-  private static electiveMatcher = /ELECTIVE @+/;
-  private static wildcardMatcher = /\w@/;
-  private static rangeMatcher = /-\w+/;
+  private static readonly electiveMatcher = /ELECTIVE @+/;
+  private static readonly wildcardMatcher = /\w@/;
+  private static readonly rangeMatcher = /-\w+/;
 
   /**
    * This will always be properly initialized using the static async factory function (`new()`).
diff --git a/tools/degreeworks-scraper/src/DegreeworksClient.ts b/tools/degreeworks-scraper/src/DegreeworksClient.ts
index d3dc9f69..68abab7d 100644
--- a/tools/degreeworks-scraper/src/DegreeworksClient.ts
+++ b/tools/degreeworks-scraper/src/DegreeworksClient.ts
@@ -6,6 +6,7 @@ export class DegreeworksClient {
   private static readonly API_URL = "https://reg.uci.edu/RespDashboard/api";
   private static readonly AUDIT_URL = `${DegreeworksClient.API_URL}/audit`;
   private catalogYear: string = "";
+
   private constructor(
     private readonly studentId: string,
     private readonly headers: HeadersInit,
diff --git a/tools/degreeworks-scraper/src/PPAPIOfflineClient.ts b/tools/degreeworks-scraper/src/PPAPIOfflineClient.ts
index c19809f6..ae8ef36f 100644
--- a/tools/degreeworks-scraper/src/PPAPIOfflineClient.ts
+++ b/tools/degreeworks-scraper/src/PPAPIOfflineClient.ts
@@ -4,6 +4,7 @@ import type { Course, RawResponse } from "peterportal-api-next-types";
 
 export class PPAPIOfflineClient {
   private cache = new Map<string, Course>();
+
   private constructor() {}
 
   static async new(): Promise<PPAPIOfflineClient> {

From b291eee5caf498443c77d5b049c0dc32f7c3a365 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Sun, 20 Aug 2023 17:06:17 -0700
Subject: [PATCH 18/48] feat: :sparkles: key by program name and dedupe

---
 pnpm-lock.yaml                               | 164 ++++++++++++++++---
 tools/degreeworks-scraper/package.json       |   2 +
 tools/degreeworks-scraper/src/AuditParser.ts |   3 +-
 tools/degreeworks-scraper/src/index.ts       | 103 ++++++++----
 tools/degreeworks-scraper/src/types.ts       |   2 +-
 5 files changed, 210 insertions(+), 64 deletions(-)

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 7b82c957..f6a9fc47 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -606,6 +606,9 @@ importers:
       cross-fetch:
         specifier: 4.0.0
         version: 4.0.0
+      deep-equal:
+        specifier: ^2.2.2
+        version: 2.2.2
       dotenv:
         specifier: 16.3.1
         version: 16.3.1
@@ -616,6 +619,9 @@ importers:
         specifier: workspace:*
         version: link:../../packages/peterportal-api-next-types
     devDependencies:
+      '@types/deep-equal':
+        specifier: ^1.0.1
+        version: 1.0.1
       tsx:
         specifier: 3.12.7
         version: 3.12.7
@@ -2905,6 +2911,9 @@ packages:
       '@babel/helper-validator-identifier': 7.19.1
       to-fast-properties: 2.0.0
 
+  /@balena/dockerignore@1.0.2:
+    resolution: {integrity: sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==}
+
   /@colors/colors@1.5.0:
     resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==}
     engines: {node: '>=0.1.90'}
@@ -2941,6 +2950,7 @@ packages:
   /@commitlint/config-validator@17.6.7:
     resolution: {integrity: sha512-vJSncmnzwMvpr3lIcm0I8YVVDJTzyjy7NZAeXbTXy+MPUdAr9pKyyg7Tx/ebOQ9kqzE6O9WT6jg2164br5UdsQ==}
     engines: {node: '>=v14'}
+    requiresBuild: true
     dependencies:
       '@commitlint/types': 17.4.4
       ajv: 8.12.0
@@ -3041,6 +3051,7 @@ packages:
   /@commitlint/resolve-extends@17.6.7:
     resolution: {integrity: sha512-PfeoAwLHtbOaC9bGn/FADN156CqkFz6ZKiVDMjuC2N5N0740Ke56rKU7Wxdwya8R8xzLK9vZzHgNbuGhaOVKIg==}
     engines: {node: '>=v14'}
+    requiresBuild: true
     dependencies:
       '@commitlint/config-validator': 17.6.7
       '@commitlint/types': 17.4.4
@@ -5479,6 +5490,10 @@ packages:
       '@types/node': 18.17.5
     dev: true
 
+  /@types/deep-equal@1.0.1:
+    resolution: {integrity: sha512-mMUu4nWHLBlHtxXY17Fg6+ucS/MnndyOWyOe7MmwkoMYxvfQU2ajtRaEvqSUv+aVkMqH/C0NCI8UoVfRNQ10yg==}
+    dev: true
+
   /@types/eslint-scope@3.7.4:
     resolution: {integrity: sha512-9K4zoImiZc3HlIp6AVUDE4CWYx22a+lhSZMYNpbjW04+YF0KWj4pJXnEMjdnFTiQibFFmElcsasJXDbdI/EPhA==}
     dependencies:
@@ -5602,6 +5617,7 @@ packages:
 
   /@types/node@20.4.7:
     resolution: {integrity: sha512-bUBrPjEry2QUTsnuEjzjbS7voGWCc30W0qzgMf90GPeDGFRakvrz47ju+oqDAKCXLUCe39u57/ORMl/O/04/9g==}
+    requiresBuild: true
     dev: true
 
   /@types/normalize-package-data@2.4.1:
@@ -6227,7 +6243,6 @@ packages:
     dependencies:
       call-bind: 1.0.2
       is-array-buffer: 3.0.2
-    dev: true
 
   /array-flatten@1.1.1:
     resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==}
@@ -6303,7 +6318,6 @@ packages:
   /astral-regex@2.0.0:
     resolution: {integrity: sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==}
     engines: {node: '>=8'}
-    dev: true
 
   /async-retry@1.3.3:
     resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==}
@@ -6342,7 +6356,6 @@ packages:
   /available-typed-arrays@1.0.5:
     resolution: {integrity: sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==}
     engines: {node: '>= 0.4'}
-    dev: true
 
   /aws-cdk-lib@2.91.0(constructs@10.2.69):
     resolution: {integrity: sha512-sxXVUlb9OOjwakEssppty7QMTcMX9F6/cNA980JMmQpKeVALXvT60jWdCeAeKeZcGz1Y4whLoXLdU2/bJzh07w==}
@@ -6353,7 +6366,17 @@ packages:
       '@aws-cdk/asset-awscli-v1': 2.2.200
       '@aws-cdk/asset-kubectl-v20': 2.1.2
       '@aws-cdk/asset-node-proxy-agent-v5': 2.0.166
+      '@balena/dockerignore': 1.0.2
+      case: 1.6.3
       constructs: 10.2.69
+      fs-extra: 11.1.1
+      ignore: 5.2.4
+      jsonschema: 1.4.1
+      minimatch: 3.1.2
+      punycode: 2.3.0
+      semver: 7.5.4
+      table: 6.8.1
+      yaml: 1.10.2
     bundledDependencies:
       - '@balena/dockerignore'
       - case
@@ -6730,6 +6753,10 @@ packages:
   /caniuse-lite@1.0.30001489:
     resolution: {integrity: sha512-x1mgZEXK8jHIfAxm+xgdpHpk50IN3z3q3zP261/WS+uvePxW8izXuCu6AHz0lkuYTlATDehiZ/tNyYBdSQsOUQ==}
 
+  /case@1.6.3:
+    resolution: {integrity: sha512-mzDSXIPaFwVDvZAHqZ9VlbyF4yyXRuX6IvB06WvPYkqJVO24kX1PPhv9bfpKNFZyxYFmmgo03HUiD8iklmJYRQ==}
+    engines: {node: '>= 0.8.0'}
+
   /ccount@1.1.0:
     resolution: {integrity: sha512-vlNK021QdI7PNeiUh/lKkC/mNHHfV0m/Ad5JoI0TYtlBnJAslM/JIkm/tGC88bkLIwO6OQ5uV6ztS6kVAtCDlg==}
     dev: false
@@ -7591,6 +7618,29 @@ packages:
       type-detect: 4.0.8
     dev: true
 
+  /deep-equal@2.2.2:
+    resolution: {integrity: sha512-xjVyBf0w5vH0I42jdAZzOKVldmPgSulmiyPRywoyq7HXC9qdgo17kxJE+rdnif5Tz6+pIrpJI8dCpMNLIGkUiA==}
+    dependencies:
+      array-buffer-byte-length: 1.0.0
+      call-bind: 1.0.2
+      es-get-iterator: 1.1.3
+      get-intrinsic: 1.2.1
+      is-arguments: 1.1.1
+      is-array-buffer: 3.0.2
+      is-date-object: 1.0.5
+      is-regex: 1.1.4
+      is-shared-array-buffer: 1.0.2
+      isarray: 2.0.5
+      object-is: 1.1.5
+      object-keys: 1.1.1
+      object.assign: 4.1.4
+      regexp.prototype.flags: 1.5.0
+      side-channel: 1.0.4
+      which-boxed-primitive: 1.0.2
+      which-collection: 1.0.1
+      which-typed-array: 1.1.9
+    dev: false
+
   /deep-extend@0.6.0:
     resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==}
     engines: {node: '>=4.0.0'}
@@ -8031,6 +8081,20 @@ packages:
       which-typed-array: 1.1.9
     dev: true
 
+  /es-get-iterator@1.1.3:
+    resolution: {integrity: sha512-sPZmqHBe6JIiTfN5q2pEi//TwxmAFHwj/XEuYjTuse78i8KxaqMTTzxPoFKuzRpDpTJ+0NAbpfenkmH2rePtuw==}
+    dependencies:
+      call-bind: 1.0.2
+      get-intrinsic: 1.2.1
+      has-symbols: 1.0.3
+      is-arguments: 1.1.1
+      is-map: 2.0.2
+      is-set: 2.0.2
+      is-string: 1.0.7
+      isarray: 2.0.5
+      stop-iteration-iterator: 1.0.0
+    dev: false
+
   /es-module-lexer@1.2.1:
     resolution: {integrity: sha512-9978wrXM50Y4rTMmW5kXIC09ZdXQZqkE4mxhwkd8VbzsGkXGPgV4zWuqQJgCEzYngdo2dYDa0l8xhX4fkSwJSg==}
 
@@ -8722,7 +8786,6 @@ packages:
     resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==}
     dependencies:
       is-callable: 1.2.7
-    dev: true
 
   /fork-ts-checker-webpack-plugin@6.5.3(eslint@8.47.0)(typescript@5.1.6)(webpack@5.84.1):
     resolution: {integrity: sha512-SbH/l9ikmMWycd5puHJKTkZJKddF4iRLyW3DeZ08HTI7NGyLS38MXd/KGgeWumQO7YNQbW2u/NtPT2YowbPaGQ==}
@@ -8795,7 +8858,6 @@ packages:
       graceful-fs: 4.2.11
       jsonfile: 6.1.0
       universalify: 2.0.0
-    dev: true
 
   /fs-extra@9.1.0:
     resolution: {integrity: sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==}
@@ -8835,7 +8897,6 @@ packages:
 
   /functions-have-names@1.2.3:
     resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==}
-    dev: true
 
   /gensync@1.0.0-beta.2:
     resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==}
@@ -9045,7 +9106,6 @@ packages:
     resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==}
     dependencies:
       get-intrinsic: 1.2.1
-    dev: true
 
   /got@9.6.0:
     resolution: {integrity: sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q==}
@@ -9114,7 +9174,6 @@ packages:
 
   /has-bigints@1.0.2:
     resolution: {integrity: sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==}
-    dev: true
 
   /has-flag@3.0.0:
     resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==}
@@ -9142,7 +9201,6 @@ packages:
     engines: {node: '>= 0.4'}
     dependencies:
       has-symbols: 1.0.3
-    dev: true
 
   /has-yarn@2.1.0:
     resolution: {integrity: sha512-UqBRqi4ju7T+TqGNdqAO0PaSVGsDGJUBQvk9eUWNGRY1CFGDzYhLWoM7JQEemnlvVcv/YEmc2wNW8BC24EnUsw==}
@@ -9520,7 +9578,6 @@ packages:
       get-intrinsic: 1.2.1
       has: 1.0.3
       side-channel: 1.0.4
-    dev: true
 
   /interpret@1.4.0:
     resolution: {integrity: sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==}
@@ -9553,13 +9610,20 @@ packages:
       is-decimal: 1.0.4
     dev: false
 
+  /is-arguments@1.1.1:
+    resolution: {integrity: sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      call-bind: 1.0.2
+      has-tostringtag: 1.0.0
+    dev: false
+
   /is-array-buffer@3.0.2:
     resolution: {integrity: sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==}
     dependencies:
       call-bind: 1.0.2
       get-intrinsic: 1.2.1
       is-typed-array: 1.1.10
-    dev: true
 
   /is-arrayish@0.2.1:
     resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==}
@@ -9572,7 +9636,6 @@ packages:
     resolution: {integrity: sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==}
     dependencies:
       has-bigints: 1.0.2
-    dev: true
 
   /is-binary-path@2.1.0:
     resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==}
@@ -9586,7 +9649,6 @@ packages:
     dependencies:
       call-bind: 1.0.2
       has-tostringtag: 1.0.0
-    dev: true
 
   /is-buffer@2.0.5:
     resolution: {integrity: sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==}
@@ -9596,7 +9658,6 @@ packages:
   /is-callable@1.2.7:
     resolution: {integrity: sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==}
     engines: {node: '>= 0.4'}
-    dev: true
 
   /is-ci@2.0.0:
     resolution: {integrity: sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==}
@@ -9621,7 +9682,6 @@ packages:
     engines: {node: '>= 0.4'}
     dependencies:
       has-tostringtag: 1.0.0
-    dev: true
 
   /is-decimal@1.0.4:
     resolution: {integrity: sha512-RGdriMmQQvZ2aqaQq3awNA6dCGtKpiDFcOzrTWrDAT2MiWrKQVPmxLGHl7Y2nNu6led0kEyoX0enY0qXYsv9zw==}
@@ -9687,6 +9747,10 @@ packages:
     engines: {node: '>=8'}
     dev: true
 
+  /is-map@2.0.2:
+    resolution: {integrity: sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg==}
+    dev: false
+
   /is-negative-zero@2.0.2:
     resolution: {integrity: sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==}
     engines: {node: '>= 0.4'}
@@ -9702,7 +9766,6 @@ packages:
     engines: {node: '>= 0.4'}
     dependencies:
       has-tostringtag: 1.0.0
-    dev: true
 
   /is-number@7.0.0:
     resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==}
@@ -9758,7 +9821,6 @@ packages:
     dependencies:
       call-bind: 1.0.2
       has-tostringtag: 1.0.0
-    dev: true
 
   /is-regexp@1.0.0:
     resolution: {integrity: sha512-7zjFAPO4/gwyQAAgRRmqeEeyIICSdmCqa3tsVHMdBzaXXRiqopZL4Cyghg/XulGWrtABTpbnYYzzIRffLkP4oA==}
@@ -9770,11 +9832,14 @@ packages:
     engines: {node: '>=6'}
     dev: false
 
+  /is-set@2.0.2:
+    resolution: {integrity: sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g==}
+    dev: false
+
   /is-shared-array-buffer@1.0.2:
     resolution: {integrity: sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==}
     dependencies:
       call-bind: 1.0.2
-    dev: true
 
   /is-stream@2.0.1:
     resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==}
@@ -9790,14 +9855,12 @@ packages:
     engines: {node: '>= 0.4'}
     dependencies:
       has-tostringtag: 1.0.0
-    dev: true
 
   /is-symbol@1.0.4:
     resolution: {integrity: sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==}
     engines: {node: '>= 0.4'}
     dependencies:
       has-symbols: 1.0.3
-    dev: true
 
   /is-text-path@1.0.1:
     resolution: {integrity: sha512-xFuJpne9oFz5qDaodwmmG08e3CawH/2ZV8Qqza1Ko7Sk8POWbkRdwIoAWVhqvq0XeUzANEhKo2n0IXUGBm7A/w==}
@@ -9815,7 +9878,6 @@ packages:
       for-each: 0.3.3
       gopd: 1.0.1
       has-tostringtag: 1.0.0
-    dev: true
 
   /is-typedarray@1.0.0:
     resolution: {integrity: sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==}
@@ -9830,12 +9892,23 @@ packages:
     resolution: {integrity: sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q==}
     dev: true
 
+  /is-weakmap@2.0.1:
+    resolution: {integrity: sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA==}
+    dev: false
+
   /is-weakref@1.0.2:
     resolution: {integrity: sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==}
     dependencies:
       call-bind: 1.0.2
     dev: true
 
+  /is-weakset@2.0.2:
+    resolution: {integrity: sha512-t2yVvttHkQktwnNNmBQ98AhENLdPUTDTE21uPqAQ0ARwQfGeQKRVS0NNurH7bTf7RrvcVn1OOge45CnBeHCSmg==}
+    dependencies:
+      call-bind: 1.0.2
+      get-intrinsic: 1.2.1
+    dev: false
+
   /is-whitespace-character@1.0.4:
     resolution: {integrity: sha512-SDweEzfIZM0SJV0EUga669UTKlmL0Pq8Lno0QDQsPnvECB3IM2aP0gdx5TrU0A01MAPfViaZiI2V1QMZLaKK5w==}
     dev: false
@@ -9867,6 +9940,10 @@ packages:
     resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==}
     dev: false
 
+  /isarray@2.0.5:
+    resolution: {integrity: sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==}
+    dev: false
+
   /isexe@2.0.0:
     resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
 
@@ -10049,6 +10126,9 @@ packages:
     engines: {'0': node >= 0.2.0}
     dev: true
 
+  /jsonschema@1.4.1:
+    resolution: {integrity: sha512-S6cATIPVv1z0IlxdN+zUk5EPjkGCdnhN4wVSBlvoUO1tOLJootbo9CquNJmbIh4yikWHiUedhRYrNPn1arpEmQ==}
+
   /jwt-decode@3.1.2:
     resolution: {integrity: sha512-UfpWE/VZn0iP50d8cz9NrZLM9lSWhcJ+0Gt/nm4by88UL+J1SiKN8/5dkjMmbEzwL2CAe+67GsegCbIKtbp75A==}
     dev: false
@@ -10256,6 +10336,9 @@ packages:
     resolution: {integrity: sha512-+WKqsK294HMSc2jEbNgpHpd0JfIBhp7rEV4aqXWqFr6AlXov+SlcgB1Fv01y2kGe3Gc8nMW7VA0SrGuSkRfIEg==}
     dev: true
 
+  /lodash.truncate@4.4.2:
+    resolution: {integrity: sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==}
+
   /lodash.uniq@4.5.0:
     resolution: {integrity: sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==}
 
@@ -10747,6 +10830,14 @@ packages:
   /object-inspect@1.12.3:
     resolution: {integrity: sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==}
 
+  /object-is@1.1.5:
+    resolution: {integrity: sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      call-bind: 1.0.2
+      define-properties: 1.2.0
+    dev: false
+
   /object-keys@1.1.1:
     resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==}
     engines: {node: '>= 0.4'}
@@ -12045,7 +12136,6 @@ packages:
       call-bind: 1.0.2
       define-properties: 1.2.0
       functions-have-names: 1.2.3
-    dev: true
 
   /regexpu-core@5.3.2:
     resolution: {integrity: sha512-RAM5FlZz+Lhmo7db9L298p2vHP5ZywrVXmVXpmAD9GuL5MPH6t9ROw1iA/wfHkQ76Qe7AaPF0nGuim96/IrQMQ==}
@@ -12595,7 +12685,6 @@ packages:
       ansi-styles: 4.3.0
       astral-regex: 2.0.0
       is-fullwidth-code-point: 3.0.0
-    dev: true
 
   /slice-ansi@5.0.0:
     resolution: {integrity: sha512-FC+lgizVPfie0kkhqUScwRu1O/lF6NOgJmlCgK+/LYxDCTk8sGelYaHDhFcDN+Sn3Cv+3VSa4Byeo+IMCzpMgQ==}
@@ -12753,6 +12842,13 @@ packages:
   /std-env@3.3.3:
     resolution: {integrity: sha512-Rz6yejtVyWnVjC1RFvNmYL10kgjC49EOghxWn0RFqlCHGFpQx+Xe7yW3I4ceK1SGrWIGMjD5Kbue8W/udkbMJg==}
 
+  /stop-iteration-iterator@1.0.0:
+    resolution: {integrity: sha512-iCGQj+0l0HOdZ2AEeBADlsRC+vsnDsZsbdSiH1yNSjcfKM7fdpCMfqAL/dwF5BLiw/XhRft/Wax6zQbhq2BcjQ==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      internal-slot: 1.0.5
+    dev: false
+
   /string-argv@0.3.2:
     resolution: {integrity: sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==}
     engines: {node: '>=0.6.19'}
@@ -12986,6 +13082,16 @@ packages:
       tslib: 2.5.2
     dev: true
 
+  /table@6.8.1:
+    resolution: {integrity: sha512-Y4X9zqrCftUhMeH2EptSSERdVKt/nEdijTOacGD/97EKjhQ/Qs8RTlEGABSJNNN8lac9kheH+af7yAkEWlgneA==}
+    engines: {node: '>=10.0.0'}
+    dependencies:
+      ajv: 8.12.0
+      lodash.truncate: 4.4.2
+      slice-ansi: 4.0.0
+      string-width: 4.2.3
+      strip-ansi: 6.0.1
+
   /tapable@1.1.3:
     resolution: {integrity: sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==}
     engines: {node: '>=6'}
@@ -14103,7 +14209,15 @@ packages:
       is-number-object: 1.0.7
       is-string: 1.0.7
       is-symbol: 1.0.4
-    dev: true
+
+  /which-collection@1.0.1:
+    resolution: {integrity: sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A==}
+    dependencies:
+      is-map: 2.0.2
+      is-set: 2.0.2
+      is-weakmap: 2.0.1
+      is-weakset: 2.0.2
+    dev: false
 
   /which-typed-array@1.1.9:
     resolution: {integrity: sha512-w9c4xkx6mPidwp7180ckYWfMmvxpjlZuIudNtDf4N/tTAUB8VJbX25qZoAsrtGuYNnGw3pa0AXgbGKRB8/EceA==}
@@ -14115,7 +14229,6 @@ packages:
       gopd: 1.0.1
       has-tostringtag: 1.0.0
       is-typed-array: 1.1.10
-    dev: true
 
   /which@1.3.1:
     resolution: {integrity: sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==}
@@ -14282,7 +14395,6 @@ packages:
   /yaml@1.10.2:
     resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==}
     engines: {node: '>= 6'}
-    dev: false
 
   /yaml@2.3.1:
     resolution: {integrity: sha512-2eHWfjaoXgTBC2jNM1LRef62VQa0umtvRiDSk6HSzW7RvS5YtkabJrwYLLEKWBc8a5U2PTSCs+dJjUTJdlHsWQ==}
diff --git a/tools/degreeworks-scraper/package.json b/tools/degreeworks-scraper/package.json
index ca02fc7a..84207a20 100644
--- a/tools/degreeworks-scraper/package.json
+++ b/tools/degreeworks-scraper/package.json
@@ -9,11 +9,13 @@
   },
   "dependencies": {
     "cross-fetch": "4.0.0",
+    "deep-equal": "^2.2.2",
     "dotenv": "16.3.1",
     "jwt-decode": "3.1.2",
     "peterportal-api-next-types": "workspace:*"
   },
   "devDependencies": {
+    "@types/deep-equal": "^1.0.1",
     "tsx": "3.12.7"
   }
 }
diff --git a/tools/degreeworks-scraper/src/AuditParser.ts b/tools/degreeworks-scraper/src/AuditParser.ts
index 0983189e..0068212d 100644
--- a/tools/degreeworks-scraper/src/AuditParser.ts
+++ b/tools/degreeworks-scraper/src/AuditParser.ts
@@ -24,7 +24,8 @@ export class AuditParser {
     return ap;
   }
 
-  parseBlock = (block: Block): Program => ({
+  parseBlock = (blockId: string, block: Block): Program => ({
+    ...this.parseBlockId(blockId),
     name: block.title,
     requirements: this.ruleArrayToRequirements(block.ruleArray),
     specs: this.parseSpecs(block.ruleArray),
diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index 19ab35ed..af8823ab 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -2,6 +2,7 @@ import { mkdir, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 
+import deepEqual from "deep-equal";
 import jwtDecode from "jwt-decode";
 import type { JwtPayload } from "jwt-decode";
 
@@ -13,6 +14,37 @@ import "dotenv/config";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
+async function scrapePrograms(
+  ap: AuditParser,
+  dw: DegreeworksClient,
+  degrees: Set<string>,
+  majorPrograms: Set<string>,
+  school: string,
+) {
+  const ret = new Map<string, Program>();
+  for (const degree of degrees) {
+    for (const majorCode of majorPrograms) {
+      const audit = await dw.getMajorAudit(degree, school, majorCode);
+      if (!audit) {
+        console.log(`Requirements block not found (majorCode = ${majorCode}, degree = ${degree})`);
+        continue;
+      }
+      const parsedBlock = ap.parseBlock(`${school}-MAJOR-${majorCode}-${degree}`, audit);
+      const program = ret.get(audit.title);
+      if (deepEqual(parsedBlock, program)) {
+        console.log(
+          `Requirements block already exists for "${audit.title}" (majorCode = ${majorCode}, degree = ${degree})`,
+        );
+      }
+      ret.set(audit.title, parsedBlock);
+      console.log(
+        `Requirements block found and parsed for "${audit.title}" (majorCode = ${majorCode}, degree = ${degree})`,
+      );
+    }
+  }
+  return ret;
+}
+
 async function main() {
   if (!process.env["X_AUTH_TOKEN"]) throw new Error("Auth cookie not set.");
   const studentId = jwtDecode<JwtPayload>(process.env["X_AUTH_TOKEN"].slice("Bearer+".length))?.sub;
@@ -46,45 +78,24 @@ async function main() {
       console.log(`Requirements block not found (minorCode = ${minorCode})`);
       continue;
     }
-    parsedMinorPrograms.set(`U-MINOR-${minorCode}`, ap.parseBlock(audit));
+    parsedMinorPrograms.set(audit.title, ap.parseBlock(`U-MINOR-${minorCode}`, audit));
     console.log(
       `Requirements block found and parsed for "${audit.title}" (minorCode = ${minorCode})`,
     );
   }
-  const parsedMajorsAndSpecs = new Map<string, Program>();
-  const degreesAwarded = new Map<string, string>();
   console.log("Scraping undergraduate program requirements");
-  for (const degree of undergraduateDegrees) {
-    for (const majorCode of majorPrograms) {
-      const audit = await dw.getMajorAudit(degree, "U", majorCode);
-      if (!audit) {
-        console.log(`Requirements block not found (majorCode = ${majorCode}, degree = ${degree})`);
-        continue;
-      }
-      degreesAwarded.set(degree, degrees.get(degree) ?? "");
-      parsedMajorsAndSpecs.set(`U-MAJOR-${majorCode}-${degree}`, ap.parseBlock(audit));
-      console.log(
-        `Requirements block found and parsed for "${audit.title}" (majorCode = ${majorCode}, degree = ${degree})`,
-      );
-    }
-  }
+  const parsedUgradPrograms = await scrapePrograms(
+    ap,
+    dw,
+    undergraduateDegrees,
+    majorPrograms,
+    "U",
+  );
   console.log("Scraping graduate program requirements");
-  for (const degree of graduateDegrees) {
-    for (const majorCode of majorPrograms) {
-      const audit = await dw.getMajorAudit(degree, "G", majorCode);
-      if (!audit) {
-        console.log(`Requirements block not found (majorCode = ${majorCode}, degree = ${degree})`);
-        continue;
-      }
-      degreesAwarded.set(degree, degrees.get(degree) ?? "");
-      parsedMajorsAndSpecs.set(`G-MAJOR-${majorCode}-${degree}`, ap.parseBlock(audit));
-      console.log(
-        `Requirements block found and parsed for "${audit.title}" (majorCode = ${majorCode}, degree = ${degree})`,
-      );
-    }
-  }
+  const parsedGradPrograms = await scrapePrograms(ap, dw, undergraduateDegrees, majorPrograms, "U");
+  const parsedSpecializations = new Map<string, Program>();
   console.log("Scraping all specialization requirements");
-  for (const [blockId, { specs }] of parsedMajorsAndSpecs) {
+  for (const [blockId, { specs }] of [...parsedUgradPrograms, ...parsedGradPrograms]) {
     const { school, code: majorCode, degreeType: degree } = ap.parseBlockId(blockId);
     if (!degree) throw new Error(`Could not parse degree type from malformed blockId "${blockId}"`);
     for (const specCode of specs) {
@@ -95,12 +106,24 @@ async function main() {
         );
         continue;
       }
-      parsedMajorsAndSpecs.set(`${school}-SPEC-${specCode}-${degree}`, ap.parseBlock(audit));
+      parsedSpecializations.set(
+        specCode,
+        ap.parseBlock(`${school}-SPEC-${specCode}-${degree}`, audit),
+      );
       console.log(
-        `Requirements block found and parsed for "${audit.title}" (school = ${school}, majorCode = ${majorCode}, specCode = ${specCode}, degree = ${degree})`,
+        `Requirements block found and parsed for "${audit.title}" (specCode = ${specCode})`,
       );
     }
   }
+  const degreesAwarded = new Map(
+    Array.from(
+      new Set(
+        [...parsedUgradPrograms, ...parsedGradPrograms]
+          .map(([, x]) => x.degreeType)
+          .filter((x) => x) as string[],
+      ),
+    ).map((x) => [x, degrees.get(x) as string]),
+  );
   await mkdir(join(__dirname, "../output"), { recursive: true });
   await Promise.all([
     writeFile(
@@ -108,8 +131,16 @@ async function main() {
       JSON.stringify(Object.fromEntries(parsedMinorPrograms.entries())),
     ),
     writeFile(
-      join(__dirname, "../output/parsedMajorsAndSpecs.json"),
-      JSON.stringify(Object.fromEntries(parsedMajorsAndSpecs.entries())),
+      join(__dirname, "../output/parsedUgradPrograms.json"),
+      JSON.stringify(Object.fromEntries(parsedUgradPrograms.entries())),
+    ),
+    writeFile(
+      join(__dirname, "../output/parsedGradPrograms.json"),
+      JSON.stringify(Object.fromEntries(parsedGradPrograms.entries())),
+    ),
+    writeFile(
+      join(__dirname, "../output/parsedSpecializations.json"),
+      JSON.stringify(Object.fromEntries(parsedSpecializations.entries())),
     ),
     writeFile(
       join(__dirname, "../output/degreesAwarded.json"),
diff --git a/tools/degreeworks-scraper/src/types.ts b/tools/degreeworks-scraper/src/types.ts
index 7ef048bd..ba945297 100644
--- a/tools/degreeworks-scraper/src/types.ts
+++ b/tools/degreeworks-scraper/src/types.ts
@@ -82,7 +82,7 @@ export type ProgramId = {
   degreeType?: string;
 };
 
-export type Program = {
+export type Program = ProgramId & {
   /**
    * The display name of the program.
    * @example "Major in Computer Science"

From 353c877324214d6938c2890d8481caf10de53992 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Sun, 20 Aug 2023 18:14:36 -0700
Subject: [PATCH 19/48] fix: :bug: don't parse blockId for specs

---
 tools/degreeworks-scraper/src/index.ts | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index af8823ab..22b78e86 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -95,9 +95,11 @@ async function main() {
   const parsedGradPrograms = await scrapePrograms(ap, dw, undergraduateDegrees, majorPrograms, "U");
   const parsedSpecializations = new Map<string, Program>();
   console.log("Scraping all specialization requirements");
-  for (const [blockId, { specs }] of [...parsedUgradPrograms, ...parsedGradPrograms]) {
-    const { school, code: majorCode, degreeType: degree } = ap.parseBlockId(blockId);
-    if (!degree) throw new Error(`Could not parse degree type from malformed blockId "${blockId}"`);
+  for (const [, { specs, school, code: majorCode, degreeType: degree }] of [
+    ...parsedUgradPrograms,
+    ...parsedGradPrograms,
+  ]) {
+    if (!degree) throw new Error(`Degree type is undefined`);
     for (const specCode of specs) {
       const audit = await dw.getSpecAudit(degree, school, majorCode, specCode);
       if (!audit) {

From 2c412687f126c244df16299f3012be9abf036dba Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Mon, 21 Aug 2023 10:30:28 -0700
Subject: [PATCH 20/48] fix: :bug: actually scrape grad programs

---
 tools/degreeworks-scraper/src/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index 22b78e86..f7b7def5 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -92,7 +92,7 @@ async function main() {
     "U",
   );
   console.log("Scraping graduate program requirements");
-  const parsedGradPrograms = await scrapePrograms(ap, dw, undergraduateDegrees, majorPrograms, "U");
+  const parsedGradPrograms = await scrapePrograms(ap, dw, graduateDegrees, majorPrograms, "G");
   const parsedSpecializations = new Map<string, Program>();
   console.log("Scraping all specialization requirements");
   for (const [, { specs, school, code: majorCode, degreeType: degree }] of [

From 92592f588dab5b2c750801c62ee2875f411b52aa Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Mon, 21 Aug 2023 13:58:58 -0700
Subject: [PATCH 21/48] feat: :sparkles: parse specializations from stringified
 block

---
 tools/degreeworks-scraper/src/AuditParser.ts | 28 ++++----------------
 1 file changed, 5 insertions(+), 23 deletions(-)

diff --git a/tools/degreeworks-scraper/src/AuditParser.ts b/tools/degreeworks-scraper/src/AuditParser.ts
index 0068212d..4fea4520 100644
--- a/tools/degreeworks-scraper/src/AuditParser.ts
+++ b/tools/degreeworks-scraper/src/AuditParser.ts
@@ -4,6 +4,7 @@ import { PPAPIOfflineClient } from "./PPAPIOfflineClient";
 import type { Block, Program, ProgramId, Requirement, Rule } from "./types";
 
 export class AuditParser {
+  private static readonly specMatcher = /"type":"SPEC","value":"\w+"/g;
   private static readonly electiveMatcher = /ELECTIVE @+/;
   private static readonly wildcardMatcher = /\w@/;
   private static readonly rangeMatcher = /-\w+/;
@@ -28,35 +29,16 @@ export class AuditParser {
     ...this.parseBlockId(blockId),
     name: block.title,
     requirements: this.ruleArrayToRequirements(block.ruleArray),
-    specs: this.parseSpecs(block.ruleArray),
+    specs: this.parseSpecs(block),
   });
 
   lexOrd = new Intl.Collator().compare;
 
-  parseSpecs = (ruleArray: Rule[]) =>
-    ruleArray
-      .filter((x) => x.ruleType === "IfStmt")
-      .flatMap((x) => this.ifStmtToSpecArray([x]))
+  parseSpecs = (block: Block): string[] =>
+    Array.from(JSON.stringify(block).matchAll(AuditParser.specMatcher))
+      .map((x) => JSON.parse(`{${x[0]}}`).value)
       .sort();
 
-  ifStmtToSpecArray(ruleArray: Rule[]): string[] {
-    const ret = [];
-    for (const rule of ruleArray) {
-      switch (rule.ruleType) {
-        case "IfStmt":
-          ret.push(
-            ...this.ifStmtToSpecArray(rule.requirement.ifPart.ruleArray),
-            ...this.ifStmtToSpecArray(rule.requirement.elsePart?.ruleArray ?? []),
-          );
-          break;
-        case "Block":
-          ret.push(rule.requirement.value);
-          break;
-      }
-    }
-    return ret;
-  }
-
   flattenIfStmt(ruleArray: Rule[]): Rule[] {
     const ret = [];
     for (const rule of ruleArray) {

From 137da4d14d9fb03a4ffed8637ac3e8d2c82f8965 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Mon, 21 Aug 2023 13:59:19 -0700
Subject: [PATCH 22/48] feat: :sparkles: determine program equality by audit
 title

---
 tools/degreeworks-scraper/src/index.ts | 24 ++++++++----------------
 1 file changed, 8 insertions(+), 16 deletions(-)

diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index f7b7def5..0780f2ba 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -2,7 +2,6 @@ import { mkdir, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 
-import deepEqual from "deep-equal";
 import jwtDecode from "jwt-decode";
 import type { JwtPayload } from "jwt-decode";
 
@@ -29,14 +28,13 @@ async function scrapePrograms(
         console.log(`Requirements block not found (majorCode = ${majorCode}, degree = ${degree})`);
         continue;
       }
-      const parsedBlock = ap.parseBlock(`${school}-MAJOR-${majorCode}-${degree}`, audit);
-      const program = ret.get(audit.title);
-      if (deepEqual(parsedBlock, program)) {
+      if (ret.has(audit.title)) {
         console.log(
           `Requirements block already exists for "${audit.title}" (majorCode = ${majorCode}, degree = ${degree})`,
         );
+        continue;
       }
-      ret.set(audit.title, parsedBlock);
+      ret.set(audit.title, ap.parseBlock(`${school}-MAJOR-${majorCode}-${degree}`, audit));
       console.log(
         `Requirements block found and parsed for "${audit.title}" (majorCode = ${majorCode}, degree = ${degree})`,
       );
@@ -64,10 +62,10 @@ async function main() {
   const minorPrograms = new Set((await dw.getMapping("minors")).keys());
   console.log(`Fetched ${minorPrograms.size} minor programs`);
 
-  const undergraduateDegrees = new Set<string>();
-  const graduateDegrees = new Set<string>();
+  const ugradDegrees = new Set<string>();
+  const gradDegrees = new Set<string>();
   for (const degree of degrees.keys())
-    (degree.startsWith("B") ? undergraduateDegrees : graduateDegrees).add(degree);
+    (degree.startsWith("B") ? ugradDegrees : gradDegrees).add(degree);
 
   const ap = await AuditParser.new();
   const parsedMinorPrograms = new Map<string, Program>();
@@ -84,15 +82,9 @@ async function main() {
     );
   }
   console.log("Scraping undergraduate program requirements");
-  const parsedUgradPrograms = await scrapePrograms(
-    ap,
-    dw,
-    undergraduateDegrees,
-    majorPrograms,
-    "U",
-  );
+  const parsedUgradPrograms = await scrapePrograms(ap, dw, ugradDegrees, majorPrograms, "U");
   console.log("Scraping graduate program requirements");
-  const parsedGradPrograms = await scrapePrograms(ap, dw, graduateDegrees, majorPrograms, "G");
+  const parsedGradPrograms = await scrapePrograms(ap, dw, gradDegrees, majorPrograms, "G");
   const parsedSpecializations = new Map<string, Program>();
   console.log("Scraping all specialization requirements");
   for (const [, { specs, school, code: majorCode, degreeType: degree }] of [

From 820c1caf89559f27c426acc90abb23ce224e106c Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Mon, 21 Aug 2023 14:03:19 -0700
Subject: [PATCH 23/48] chore: :wrench: use non-null assert instead of
 ts-ignore

---
 tools/degreeworks-scraper/src/AuditParser.ts | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/tools/degreeworks-scraper/src/AuditParser.ts b/tools/degreeworks-scraper/src/AuditParser.ts
index 4fea4520..cffe452b 100644
--- a/tools/degreeworks-scraper/src/AuditParser.ts
+++ b/tools/degreeworks-scraper/src/AuditParser.ts
@@ -9,12 +9,7 @@ export class AuditParser {
   private static readonly wildcardMatcher = /\w@/;
   private static readonly rangeMatcher = /-\w+/;
 
-  /**
-   * This will always be properly initialized using the static async factory function (`new()`).
-   */
-  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-  // @ts-ignore TS2564
-  private ppapi: PPAPIOfflineClient;
+  private ppapi!: PPAPIOfflineClient;
 
   private constructor() {}
 

From c5386de114af5e7c8361f7c57b26b2b395cbbe12 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Mon, 21 Aug 2023 14:03:57 -0700
Subject: [PATCH 24/48] style: :art: clean up non-interpolated strings

---
 tools/degreeworks-scraper/src/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index 0780f2ba..e8473c21 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -91,7 +91,7 @@ async function main() {
     ...parsedUgradPrograms,
     ...parsedGradPrograms,
   ]) {
-    if (!degree) throw new Error(`Degree type is undefined`);
+    if (!degree) throw new Error("Degree type is undefined");
     for (const specCode of specs) {
       const audit = await dw.getSpecAudit(degree, school, majorCode, specCode);
       if (!audit) {

From 06910a490c33480bbe084bd333e303ed825c6be6 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Mon, 21 Aug 2023 14:24:37 -0700
Subject: [PATCH 25/48] feat: :sparkles: more encapsulation

---
 tools/degreeworks-scraper/src/Scraper.ts | 130 +++++++++++++++++++++++
 tools/degreeworks-scraper/src/index.ts   | 128 ++--------------------
 2 files changed, 138 insertions(+), 120 deletions(-)
 create mode 100644 tools/degreeworks-scraper/src/Scraper.ts

diff --git a/tools/degreeworks-scraper/src/Scraper.ts b/tools/degreeworks-scraper/src/Scraper.ts
new file mode 100644
index 00000000..c0b146a4
--- /dev/null
+++ b/tools/degreeworks-scraper/src/Scraper.ts
@@ -0,0 +1,130 @@
+import { AuditParser } from "./AuditParser";
+import { DegreeworksClient } from "./DegreeworksClient";
+import { Program } from "./types";
+
+export class Scraper {
+  private ap!: AuditParser;
+  private dw!: DegreeworksClient;
+
+  private degrees: Map<string, string> | undefined = undefined;
+  private majorPrograms: Set<string> | undefined = undefined;
+  private minorPrograms: Set<string> | undefined = undefined;
+
+  private done = false;
+  private parsedMinorPrograms: Map<string, Program> | undefined = undefined;
+  private parsedUgradPrograms: Map<string, Program> | undefined = undefined;
+  private parsedGradPrograms: Map<string, Program> | undefined = undefined;
+  private parsedSpecializations: Map<string, Program> | undefined = undefined;
+  private degreesAwarded: Map<string, string> | undefined = undefined;
+
+  private constructor() {}
+
+  private async scrapePrograms(school: string, degrees: Set<string>) {
+    if (!this.majorPrograms) throw new Error("majorPrograms has not yet been initialized.");
+    const ret = new Map<string, Program>();
+    for (const degree of degrees) {
+      for (const majorCode of this.majorPrograms) {
+        const audit = await this.dw.getMajorAudit(degree, school, majorCode);
+        if (!audit) {
+          console.log(
+            `Requirements block not found (majorCode = ${majorCode}, degree = ${degree})`,
+          );
+          continue;
+        }
+        if (ret.has(audit.title)) {
+          console.log(
+            `Requirements block already exists for "${audit.title}" (majorCode = ${majorCode}, degree = ${degree})`,
+          );
+          continue;
+        }
+        ret.set(audit.title, this.ap.parseBlock(`${school}-MAJOR-${majorCode}-${degree}`, audit));
+        console.log(
+          `Requirements block found and parsed for "${audit.title}" (majorCode = ${majorCode}, degree = ${degree})`,
+        );
+      }
+    }
+    return ret;
+  }
+  async run() {
+    if (this.done) throw new Error("This scraper instance has already finished its run.");
+    this.degrees = await this.dw.getMapping("degrees");
+    console.log(`Fetched ${this.degrees.size} degrees`);
+    this.majorPrograms = new Set((await this.dw.getMapping("majors")).keys());
+    console.log(`Fetched ${this.majorPrograms.size} major programs`);
+    this.minorPrograms = new Set((await this.dw.getMapping("minors")).keys());
+    console.log(`Fetched ${this.minorPrograms.size} minor programs`);
+    const ugradDegrees = new Set<string>();
+    const gradDegrees = new Set<string>();
+    for (const degree of this.degrees.keys())
+      (degree.startsWith("B") ? ugradDegrees : gradDegrees).add(degree);
+    this.parsedMinorPrograms = new Map<string, Program>();
+    console.log("Scraping minor program requirements");
+    for (const minorCode of this.minorPrograms) {
+      const audit = await this.dw.getMinorAudit(minorCode);
+      if (!audit) {
+        console.log(`Requirements block not found (minorCode = ${minorCode})`);
+        continue;
+      }
+      this.parsedMinorPrograms.set(audit.title, this.ap.parseBlock(`U-MINOR-${minorCode}`, audit));
+      console.log(
+        `Requirements block found and parsed for "${audit.title}" (minorCode = ${minorCode})`,
+      );
+    }
+    console.log("Scraping undergraduate program requirements");
+    this.parsedUgradPrograms = await this.scrapePrograms("U", ugradDegrees);
+    console.log("Scraping graduate program requirements");
+    this.parsedGradPrograms = await this.scrapePrograms("G", gradDegrees);
+    this.parsedSpecializations = new Map<string, Program>();
+    console.log("Scraping all specialization requirements");
+    for (const [, { specs, school, code: majorCode, degreeType: degree }] of [
+      ...this.parsedUgradPrograms,
+      ...this.parsedGradPrograms,
+    ]) {
+      if (!degree) throw new Error("Degree type is undefined");
+      for (const specCode of specs) {
+        const audit = await this.dw.getSpecAudit(degree, school, majorCode, specCode);
+        if (!audit) {
+          console.log(
+            `Requirements block not found (school = ${school}, majorCode = ${majorCode}, specCode = ${specCode}, degree = ${degree})`,
+          );
+          continue;
+        }
+        this.parsedSpecializations.set(
+          specCode,
+          this.ap.parseBlock(`${school}-SPEC-${specCode}-${degree}`, audit),
+        );
+        console.log(
+          `Requirements block found and parsed for "${audit.title}" (specCode = ${specCode})`,
+        );
+      }
+    }
+    this.degreesAwarded = new Map(
+      Array.from(
+        new Set(
+          [...this.parsedUgradPrograms, ...this.parsedGradPrograms]
+            .map(([, x]) => x.degreeType)
+            .filter((x) => x) as string[],
+        ),
+      ).map((x) => [x, this.degrees?.get(x) as string]),
+    );
+    this.done = true;
+  }
+  get() {
+    if (!this.done) throw new Error("This scraper instance has not yet finished its run.");
+    return new Map(
+      Object.entries({
+        parsedMinorPrograms: this.parsedMinorPrograms,
+        parsedUgradPrograms: this.parsedUgradPrograms,
+        parsedGradPrograms: this.parsedGradPrograms,
+        parsedSpecializations: this.parsedSpecializations,
+        degreesAwarded: this.degreesAwarded,
+      }),
+    ) as Map<string, Map<string, Program>>;
+  }
+  static async new(studentId: string, headers: HeadersInit): Promise<Scraper> {
+    const scraper = new Scraper();
+    scraper.ap = await AuditParser.new();
+    scraper.dw = await DegreeworksClient.new(studentId, headers);
+    return scraper;
+  }
+}
diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index e8473c21..10f80639 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -5,44 +5,11 @@ import { fileURLToPath } from "node:url";
 import jwtDecode from "jwt-decode";
 import type { JwtPayload } from "jwt-decode";
 
-import { AuditParser } from "./AuditParser";
-import { DegreeworksClient } from "./DegreeworksClient";
-import type { Program } from "./types";
-
 import "dotenv/config";
+import { Scraper } from "./Scraper";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
-async function scrapePrograms(
-  ap: AuditParser,
-  dw: DegreeworksClient,
-  degrees: Set<string>,
-  majorPrograms: Set<string>,
-  school: string,
-) {
-  const ret = new Map<string, Program>();
-  for (const degree of degrees) {
-    for (const majorCode of majorPrograms) {
-      const audit = await dw.getMajorAudit(degree, school, majorCode);
-      if (!audit) {
-        console.log(`Requirements block not found (majorCode = ${majorCode}, degree = ${degree})`);
-        continue;
-      }
-      if (ret.has(audit.title)) {
-        console.log(
-          `Requirements block already exists for "${audit.title}" (majorCode = ${majorCode}, degree = ${degree})`,
-        );
-        continue;
-      }
-      ret.set(audit.title, ap.parseBlock(`${school}-MAJOR-${majorCode}-${degree}`, audit));
-      console.log(
-        `Requirements block found and parsed for "${audit.title}" (majorCode = ${majorCode}, degree = ${degree})`,
-      );
-    }
-  }
-  return ret;
-}
-
 async function main() {
   if (!process.env["X_AUTH_TOKEN"]) throw new Error("Auth cookie not set.");
   const studentId = jwtDecode<JwtPayload>(process.env["X_AUTH_TOKEN"].slice("Bearer+".length))?.sub;
@@ -54,93 +21,14 @@ async function main() {
     Origin: "https://reg.uci.edu",
   };
   console.log("degreeworks-scraper starting");
-  const dw = await DegreeworksClient.new(studentId, headers);
-  const degrees = await dw.getMapping("degrees");
-  console.log(`Fetched ${degrees.size} degrees`);
-  const majorPrograms = new Set((await dw.getMapping("majors")).keys());
-  console.log(`Fetched ${majorPrograms.size} major programs`);
-  const minorPrograms = new Set((await dw.getMapping("minors")).keys());
-  console.log(`Fetched ${minorPrograms.size} minor programs`);
-
-  const ugradDegrees = new Set<string>();
-  const gradDegrees = new Set<string>();
-  for (const degree of degrees.keys())
-    (degree.startsWith("B") ? ugradDegrees : gradDegrees).add(degree);
-
-  const ap = await AuditParser.new();
-  const parsedMinorPrograms = new Map<string, Program>();
-  console.log("Scraping minor program requirements");
-  for (const minorCode of minorPrograms) {
-    const audit = await dw.getMinorAudit(minorCode);
-    if (!audit) {
-      console.log(`Requirements block not found (minorCode = ${minorCode})`);
-      continue;
-    }
-    parsedMinorPrograms.set(audit.title, ap.parseBlock(`U-MINOR-${minorCode}`, audit));
-    console.log(
-      `Requirements block found and parsed for "${audit.title}" (minorCode = ${minorCode})`,
-    );
-  }
-  console.log("Scraping undergraduate program requirements");
-  const parsedUgradPrograms = await scrapePrograms(ap, dw, ugradDegrees, majorPrograms, "U");
-  console.log("Scraping graduate program requirements");
-  const parsedGradPrograms = await scrapePrograms(ap, dw, gradDegrees, majorPrograms, "G");
-  const parsedSpecializations = new Map<string, Program>();
-  console.log("Scraping all specialization requirements");
-  for (const [, { specs, school, code: majorCode, degreeType: degree }] of [
-    ...parsedUgradPrograms,
-    ...parsedGradPrograms,
-  ]) {
-    if (!degree) throw new Error("Degree type is undefined");
-    for (const specCode of specs) {
-      const audit = await dw.getSpecAudit(degree, school, majorCode, specCode);
-      if (!audit) {
-        console.log(
-          `Requirements block not found (school = ${school}, majorCode = ${majorCode}, specCode = ${specCode}, degree = ${degree})`,
-        );
-        continue;
-      }
-      parsedSpecializations.set(
-        specCode,
-        ap.parseBlock(`${school}-SPEC-${specCode}-${degree}`, audit),
-      );
-      console.log(
-        `Requirements block found and parsed for "${audit.title}" (specCode = ${specCode})`,
-      );
-    }
-  }
-  const degreesAwarded = new Map(
-    Array.from(
-      new Set(
-        [...parsedUgradPrograms, ...parsedGradPrograms]
-          .map(([, x]) => x.degreeType)
-          .filter((x) => x) as string[],
-      ),
-    ).map((x) => [x, degrees.get(x) as string]),
-  );
+  const scraper = await Scraper.new(studentId, headers);
+  await scraper.run();
   await mkdir(join(__dirname, "../output"), { recursive: true });
-  await Promise.all([
-    writeFile(
-      join(__dirname, "../output/parsedMinorPrograms.json"),
-      JSON.stringify(Object.fromEntries(parsedMinorPrograms.entries())),
-    ),
-    writeFile(
-      join(__dirname, "../output/parsedUgradPrograms.json"),
-      JSON.stringify(Object.fromEntries(parsedUgradPrograms.entries())),
-    ),
-    writeFile(
-      join(__dirname, "../output/parsedGradPrograms.json"),
-      JSON.stringify(Object.fromEntries(parsedGradPrograms.entries())),
-    ),
-    writeFile(
-      join(__dirname, "../output/parsedSpecializations.json"),
-      JSON.stringify(Object.fromEntries(parsedSpecializations.entries())),
-    ),
-    writeFile(
-      join(__dirname, "../output/degreesAwarded.json"),
-      JSON.stringify(Object.fromEntries(degreesAwarded.entries())),
-    ),
-  ]);
+  for (const [fileName, contents] of scraper.get())
+    await writeFile(
+      join(__dirname, `../output/${fileName}`),
+      JSON.stringify(Object.fromEntries(contents)),
+    );
 }
 
 main().then();

From dc4ccf89363bd7c4ad5a8cbd5945fa3d1243fe3e Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Mon, 21 Aug 2023 14:26:10 -0700
Subject: [PATCH 26/48] refactor: :recycle: separate components dir

---
 tools/degreeworks-scraper/src/{ => components}/AuditParser.ts  | 3 ++-
 .../src/{ => components}/DegreeworksClient.ts                  | 2 +-
 .../src/{ => components}/PPAPIOfflineClient.ts                 | 0
 tools/degreeworks-scraper/src/{ => components}/Scraper.ts      | 3 ++-
 tools/degreeworks-scraper/src/index.ts                         | 3 ++-
 5 files changed, 7 insertions(+), 4 deletions(-)
 rename tools/degreeworks-scraper/src/{ => components}/AuditParser.ts (99%)
 rename tools/degreeworks-scraper/src/{ => components}/DegreeworksClient.ts (99%)
 rename tools/degreeworks-scraper/src/{ => components}/PPAPIOfflineClient.ts (100%)
 rename tools/degreeworks-scraper/src/{ => components}/Scraper.ts (99%)

diff --git a/tools/degreeworks-scraper/src/AuditParser.ts b/tools/degreeworks-scraper/src/components/AuditParser.ts
similarity index 99%
rename from tools/degreeworks-scraper/src/AuditParser.ts
rename to tools/degreeworks-scraper/src/components/AuditParser.ts
index cffe452b..f1218ff5 100644
--- a/tools/degreeworks-scraper/src/AuditParser.ts
+++ b/tools/degreeworks-scraper/src/components/AuditParser.ts
@@ -1,7 +1,8 @@
 import type { Course } from "peterportal-api-next-types";
 
+import type { Block, Program, ProgramId, Requirement, Rule } from "../types";
+
 import { PPAPIOfflineClient } from "./PPAPIOfflineClient";
-import type { Block, Program, ProgramId, Requirement, Rule } from "./types";
 
 export class AuditParser {
   private static readonly specMatcher = /"type":"SPEC","value":"\w+"/g;
diff --git a/tools/degreeworks-scraper/src/DegreeworksClient.ts b/tools/degreeworks-scraper/src/components/DegreeworksClient.ts
similarity index 99%
rename from tools/degreeworks-scraper/src/DegreeworksClient.ts
rename to tools/degreeworks-scraper/src/components/DegreeworksClient.ts
index 68abab7d..8ee876d5 100644
--- a/tools/degreeworks-scraper/src/DegreeworksClient.ts
+++ b/tools/degreeworks-scraper/src/components/DegreeworksClient.ts
@@ -1,6 +1,6 @@
 import fetch from "cross-fetch";
 
-import type { Block, DWAuditResponse, DWMappingResponse } from "./types";
+import type { Block, DWAuditResponse, DWMappingResponse } from "../types";
 
 export class DegreeworksClient {
   private static readonly API_URL = "https://reg.uci.edu/RespDashboard/api";
diff --git a/tools/degreeworks-scraper/src/PPAPIOfflineClient.ts b/tools/degreeworks-scraper/src/components/PPAPIOfflineClient.ts
similarity index 100%
rename from tools/degreeworks-scraper/src/PPAPIOfflineClient.ts
rename to tools/degreeworks-scraper/src/components/PPAPIOfflineClient.ts
diff --git a/tools/degreeworks-scraper/src/Scraper.ts b/tools/degreeworks-scraper/src/components/Scraper.ts
similarity index 99%
rename from tools/degreeworks-scraper/src/Scraper.ts
rename to tools/degreeworks-scraper/src/components/Scraper.ts
index c0b146a4..eca2d25b 100644
--- a/tools/degreeworks-scraper/src/Scraper.ts
+++ b/tools/degreeworks-scraper/src/components/Scraper.ts
@@ -1,6 +1,7 @@
+import { Program } from "../types";
+
 import { AuditParser } from "./AuditParser";
 import { DegreeworksClient } from "./DegreeworksClient";
-import { Program } from "./types";
 
 export class Scraper {
   private ap!: AuditParser;
diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index 10f80639..ebb47624 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -5,8 +5,9 @@ import { fileURLToPath } from "node:url";
 import jwtDecode from "jwt-decode";
 import type { JwtPayload } from "jwt-decode";
 
+import { Scraper } from "./components/Scraper";
+
 import "dotenv/config";
-import { Scraper } from "./Scraper";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 

From 2e48af23654590cbd2db80658711c3865ff64b1e Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Mon, 21 Aug 2023 21:09:54 -0700
Subject: [PATCH 27/48] chore: :wrench: add json ext to output

---
 tools/degreeworks-scraper/src/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index ebb47624..9c9f93bd 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -27,7 +27,7 @@ async function main() {
   await mkdir(join(__dirname, "../output"), { recursive: true });
   for (const [fileName, contents] of scraper.get())
     await writeFile(
-      join(__dirname, `../output/${fileName}`),
+      join(__dirname, `../output/${fileName}.json`),
       JSON.stringify(Object.fromEntries(contents)),
     );
 }

From 3527a8a97328749e43bc93cdf9452df96a9ce0f3 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Mon, 21 Aug 2023 21:23:58 -0700
Subject: [PATCH 28/48] feat: :sparkles: handle Subset rule type

---
 .../src/components/AuditParser.ts                    | 12 ++++++++++--
 tools/degreeworks-scraper/src/types.ts               |  7 ++++++-
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/tools/degreeworks-scraper/src/components/AuditParser.ts b/tools/degreeworks-scraper/src/components/AuditParser.ts
index f1218ff5..b575665b 100644
--- a/tools/degreeworks-scraper/src/components/AuditParser.ts
+++ b/tools/degreeworks-scraper/src/components/AuditParser.ts
@@ -92,7 +92,6 @@ export class AuditParser {
     for (const rule of ruleArray) {
       switch (rule.ruleType) {
         case "Block":
-          break;
         case "Noncourse":
           break;
         case "Course": {
@@ -132,13 +131,14 @@ export class AuditParser {
           }
           break;
         }
-        case "Group":
+        case "Group": {
           ret[rule.label] = {
             requirementType: "Group",
             requirementCount: Number.parseInt(rule.requirement.numberOfGroups),
             requirements: this.ruleArrayToRequirements(rule.ruleArray),
           };
           break;
+        }
         case "IfStmt": {
           const rules = this.flattenIfStmt([rule]);
           if (rules.length > 1 && !rules.some((x) => x.ruleType === "Block")) {
@@ -150,6 +150,14 @@ export class AuditParser {
           }
           break;
         }
+        case "Subset": {
+          const requirements = this.ruleArrayToRequirements(rule.ruleArray);
+          ret[rule.label] = {
+            requirementType: "Group",
+            requirementCount: Object.keys(requirements).length,
+            requirements,
+          };
+        }
       }
     }
     return ret;
diff --git a/tools/degreeworks-scraper/src/types.ts b/tools/degreeworks-scraper/src/types.ts
index ba945297..60057634 100644
--- a/tools/degreeworks-scraper/src/types.ts
+++ b/tools/degreeworks-scraper/src/types.ts
@@ -54,7 +54,12 @@ export type RuleNoncourse = {
   ruleType: "Noncourse";
   requirement: { numNoncourses: string; code: string };
 };
-export type Rule = RuleBase & (RuleGroup | RuleCourse | RuleIfStmt | RuleBlock | RuleNoncourse);
+export type RuleSubset = {
+  ruleType: "Subset";
+  ruleArray: Rule[];
+};
+export type Rule = RuleBase &
+  (RuleGroup | RuleCourse | RuleIfStmt | RuleBlock | RuleNoncourse | RuleSubset);
 export type Block = {
   requirementType: string;
   requirementValue: string;

From 5d39f853a5a7a71b34be7899336b60a50a3443b1 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Mon, 21 Aug 2023 21:28:25 -0700
Subject: [PATCH 29/48] chore(deps): :link: remove deep-equal

---
 tools/degreeworks-scraper/package.json | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tools/degreeworks-scraper/package.json b/tools/degreeworks-scraper/package.json
index 84207a20..ca02fc7a 100644
--- a/tools/degreeworks-scraper/package.json
+++ b/tools/degreeworks-scraper/package.json
@@ -9,13 +9,11 @@
   },
   "dependencies": {
     "cross-fetch": "4.0.0",
-    "deep-equal": "^2.2.2",
     "dotenv": "16.3.1",
     "jwt-decode": "3.1.2",
     "peterportal-api-next-types": "workspace:*"
   },
   "devDependencies": {
-    "@types/deep-equal": "^1.0.1",
     "tsx": "3.12.7"
   }
 }

From 2721723fe33a2a05948d508786497278cfbcf267 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Tue, 22 Aug 2023 16:07:44 -0700
Subject: [PATCH 30/48] feat: :sparkles: account for 'specs' of type other

---
 tools/degreeworks-scraper/src/components/AuditParser.ts | 7 +++++++
 tools/degreeworks-scraper/src/types.ts                  | 1 +
 2 files changed, 8 insertions(+)

diff --git a/tools/degreeworks-scraper/src/components/AuditParser.ts b/tools/degreeworks-scraper/src/components/AuditParser.ts
index b575665b..5519ab36 100644
--- a/tools/degreeworks-scraper/src/components/AuditParser.ts
+++ b/tools/degreeworks-scraper/src/components/AuditParser.ts
@@ -6,6 +6,7 @@ import { PPAPIOfflineClient } from "./PPAPIOfflineClient";
 
 export class AuditParser {
   private static readonly specMatcher = /"type":"SPEC","value":"\w+"/g;
+  private static readonly otherMatcher = /"type":"OTHER","value":"\w+"/g;
   private static readonly electiveMatcher = /ELECTIVE @+/;
   private static readonly wildcardMatcher = /\w@/;
   private static readonly rangeMatcher = /-\w+/;
@@ -26,6 +27,7 @@ export class AuditParser {
     name: block.title,
     requirements: this.ruleArrayToRequirements(block.ruleArray),
     specs: this.parseSpecs(block),
+    otherBlocks: this.parseOtherBlocks(block),
   });
 
   lexOrd = new Intl.Collator().compare;
@@ -35,6 +37,11 @@ export class AuditParser {
       .map((x) => JSON.parse(`{${x[0]}}`).value)
       .sort();
 
+  parseOtherBlocks = (block: Block): string[] =>
+    Array.from(JSON.stringify(block).matchAll(AuditParser.otherMatcher))
+      .map((x) => JSON.parse(`{${x[0]}}`).value)
+      .sort();
+
   flattenIfStmt(ruleArray: Rule[]): Rule[] {
     const ret = [];
     for (const rule of ruleArray) {
diff --git a/tools/degreeworks-scraper/src/types.ts b/tools/degreeworks-scraper/src/types.ts
index 60057634..af29e59a 100644
--- a/tools/degreeworks-scraper/src/types.ts
+++ b/tools/degreeworks-scraper/src/types.ts
@@ -97,6 +97,7 @@ export type Program = ProgramId & {
   name: string;
   requirements: Record<string, Requirement>;
   specs: string[];
+  otherBlocks: string[];
 };
 
 export type CourseRequirement = {

From bdd75f725e6cb1757b237667c91e01497d6816da Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Tue, 22 Aug 2023 17:27:44 -0700
Subject: [PATCH 31/48] feat: :sparkles: more separation of concerns

---
 .../src/components/Scraper.ts                 | 19 +++++++++++++++++--
 tools/degreeworks-scraper/src/index.ts        | 15 +--------------
 2 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/tools/degreeworks-scraper/src/components/Scraper.ts b/tools/degreeworks-scraper/src/components/Scraper.ts
index eca2d25b..6db968d6 100644
--- a/tools/degreeworks-scraper/src/components/Scraper.ts
+++ b/tools/degreeworks-scraper/src/components/Scraper.ts
@@ -1,9 +1,18 @@
+import jwtDecode from "jwt-decode";
+import type { JwtPayload } from "jwt-decode";
+
 import { Program } from "../types";
 
 import { AuditParser } from "./AuditParser";
 import { DegreeworksClient } from "./DegreeworksClient";
 
 export class Scraper {
+  private static readonly headers = {
+    "Content-Type": "application/json",
+    Cookie: `X-AUTH-TOKEN=${process.env["X_AUTH_TOKEN"]}`,
+    Origin: "https://reg.uci.edu",
+  };
+
   private ap!: AuditParser;
   private dw!: DegreeworksClient;
 
@@ -47,6 +56,7 @@ export class Scraper {
     return ret;
   }
   async run() {
+    console.log("[Scraper] degreeworks-scraper starting");
     if (this.done) throw new Error("This scraper instance has already finished its run.");
     this.degrees = await this.dw.getMapping("degrees");
     console.log(`Fetched ${this.degrees.size} degrees`);
@@ -122,10 +132,15 @@ export class Scraper {
       }),
     ) as Map<string, Map<string, Program>>;
   }
-  static async new(studentId: string, headers: HeadersInit): Promise<Scraper> {
+  static async new(): Promise<Scraper> {
+    if (!process.env["X_AUTH_TOKEN"]) throw new Error("Auth cookie not set.");
+    const studentId = jwtDecode<JwtPayload>(process.env["X_AUTH_TOKEN"].slice("Bearer+".length))
+      ?.sub;
+    if (!studentId || studentId.length !== 8)
+      throw new Error("Could not parse student ID from auth cookie.");
     const scraper = new Scraper();
     scraper.ap = await AuditParser.new();
-    scraper.dw = await DegreeworksClient.new(studentId, headers);
+    scraper.dw = await DegreeworksClient.new(studentId, Scraper.headers);
     return scraper;
   }
 }
diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index 9c9f93bd..fcfa3fd9 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -2,9 +2,6 @@ import { mkdir, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 
-import jwtDecode from "jwt-decode";
-import type { JwtPayload } from "jwt-decode";
-
 import { Scraper } from "./components/Scraper";
 
 import "dotenv/config";
@@ -12,17 +9,7 @@ import "dotenv/config";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
 async function main() {
-  if (!process.env["X_AUTH_TOKEN"]) throw new Error("Auth cookie not set.");
-  const studentId = jwtDecode<JwtPayload>(process.env["X_AUTH_TOKEN"].slice("Bearer+".length))?.sub;
-  if (!studentId || studentId.length !== 8)
-    throw new Error("Could not parse student ID from auth cookie.");
-  const headers = {
-    "Content-Type": "application/json",
-    Cookie: `X-AUTH-TOKEN=${process.env["X_AUTH_TOKEN"]}`,
-    Origin: "https://reg.uci.edu",
-  };
-  console.log("degreeworks-scraper starting");
-  const scraper = await Scraper.new(studentId, headers);
+  const scraper = await Scraper.new();
   await scraper.run();
   await mkdir(join(__dirname, "../output"), { recursive: true });
   for (const [fileName, contents] of scraper.get())

From 924e9fcbc7fd2998318049482f6de27fda07c64b Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Tue, 22 Aug 2023 21:04:38 -0700
Subject: [PATCH 32/48] feat: :sparkles: clean up programs with 'other' blocks

---
 .../src/components/AuditParser.ts             | 11 +----
 .../src/components/DegreeworksClient.ts       |  6 ++-
 .../src/components/Scraper.ts                 | 47 +++++++++++++++++++
 tools/degreeworks-scraper/src/types.ts        |  1 -
 4 files changed, 54 insertions(+), 11 deletions(-)

diff --git a/tools/degreeworks-scraper/src/components/AuditParser.ts b/tools/degreeworks-scraper/src/components/AuditParser.ts
index 5519ab36..ab9065aa 100644
--- a/tools/degreeworks-scraper/src/components/AuditParser.ts
+++ b/tools/degreeworks-scraper/src/components/AuditParser.ts
@@ -5,8 +5,7 @@ import type { Block, Program, ProgramId, Requirement, Rule } from "../types";
 import { PPAPIOfflineClient } from "./PPAPIOfflineClient";
 
 export class AuditParser {
-  private static readonly specMatcher = /"type":"SPEC","value":"\w+"/g;
-  private static readonly otherMatcher = /"type":"OTHER","value":"\w+"/g;
+  private static readonly specOrOtherMatcher = /"type":"(?:SPEC|OTHER)","value":"\w+"/g;
   private static readonly electiveMatcher = /ELECTIVE @+/;
   private static readonly wildcardMatcher = /\w@/;
   private static readonly rangeMatcher = /-\w+/;
@@ -27,18 +26,12 @@ export class AuditParser {
     name: block.title,
     requirements: this.ruleArrayToRequirements(block.ruleArray),
     specs: this.parseSpecs(block),
-    otherBlocks: this.parseOtherBlocks(block),
   });
 
   lexOrd = new Intl.Collator().compare;
 
   parseSpecs = (block: Block): string[] =>
-    Array.from(JSON.stringify(block).matchAll(AuditParser.specMatcher))
-      .map((x) => JSON.parse(`{${x[0]}}`).value)
-      .sort();
-
-  parseOtherBlocks = (block: Block): string[] =>
-    Array.from(JSON.stringify(block).matchAll(AuditParser.otherMatcher))
+    Array.from(JSON.stringify(block).matchAll(AuditParser.specOrOtherMatcher))
       .map((x) => JSON.parse(`{${x[0]}}`).value)
       .sort();
 
diff --git a/tools/degreeworks-scraper/src/components/DegreeworksClient.ts b/tools/degreeworks-scraper/src/components/DegreeworksClient.ts
index 8ee876d5..cb16f781 100644
--- a/tools/degreeworks-scraper/src/components/DegreeworksClient.ts
+++ b/tools/degreeworks-scraper/src/components/DegreeworksClient.ts
@@ -106,6 +106,7 @@ export class DegreeworksClient {
         goals: [
           { code: "MAJOR", value: majorCode },
           { code: "SPEC", value: specCode },
+          { code: "OTHER", value: specCode },
         ],
       }),
       headers: this.headers,
@@ -116,7 +117,10 @@ export class DegreeworksClient {
       ? undefined
       : json.blockArray.find(
           (x) => x.requirementType === "SPEC" && x.requirementValue === specCode,
-        );
+        ) ||
+          json.blockArray.find(
+            (x) => x.requirementType === "OTHER" && x.requirementValue === specCode,
+          );
   }
 
   async getMapping<T extends string>(path: T): Promise<Map<string, string>> {
diff --git a/tools/degreeworks-scraper/src/components/Scraper.ts b/tools/degreeworks-scraper/src/components/Scraper.ts
index 6db968d6..ac5bf39d 100644
--- a/tools/degreeworks-scraper/src/components/Scraper.ts
+++ b/tools/degreeworks-scraper/src/components/Scraper.ts
@@ -55,6 +55,27 @@ export class Scraper {
     }
     return ret;
   }
+  private cleanUpPrograms(programs: Map<string, Program>) {
+    const ret = new Map<string, Program>();
+    for (const [name, program] of programs) {
+      if (!Object.keys(program.requirements).length && program.specs.length === 2) {
+        program.requirements = {
+          "Select 1 of the following": {
+            requirementType: "Group",
+            requirementCount: 1,
+            requirements: Object.fromEntries(
+              program.specs.map((x) => [
+                this.parsedSpecializations?.get(x)?.name,
+                this.parsedSpecializations?.get(x)?.requirements,
+              ]),
+            ),
+          },
+        };
+      }
+      ret.set(name, program);
+    }
+    return ret;
+  }
   async run() {
     console.log("[Scraper] degreeworks-scraper starting");
     if (this.done) throw new Error("This scraper instance has already finished its run.");
@@ -118,6 +139,32 @@ export class Scraper {
         ),
       ).map((x) => [x, this.degrees?.get(x) as string]),
     );
+
+    // Post-processing steps.
+
+    // As of this commit, the only program which seems to require both of
+    // its "specializations" is the B.A. in Art History. There's probably a
+    // cleaner way to address this, but this is such an insanely niche case
+    // that it's probably not worth the effort to write a general solution.
+
+    let x, y, z;
+    if (
+      (x = this.parsedUgradPrograms.get("Major in Art History") as Program) &&
+      (y = this.parsedSpecializations.get("AHGEO") as Program) &&
+      (z = this.parsedSpecializations.get("AHPER") as Program)
+    ) {
+      x.specs = [];
+      x.requirements = { ...x.requirements, ...y.requirements, ...z.requirements };
+      this.parsedSpecializations.delete("AHGEO");
+      this.parsedSpecializations.delete("AHPER");
+      this.parsedUgradPrograms.set("Major in Art History", x);
+    }
+
+    // Some programs have an empty requirements block and exactly two specializations.
+    // They can be simplified into a "Select 1 of the following" group requirement.
+    this.parsedUgradPrograms = this.cleanUpPrograms(this.parsedUgradPrograms);
+    this.parsedGradPrograms = this.cleanUpPrograms(this.parsedGradPrograms);
+
     this.done = true;
   }
   get() {
diff --git a/tools/degreeworks-scraper/src/types.ts b/tools/degreeworks-scraper/src/types.ts
index af29e59a..60057634 100644
--- a/tools/degreeworks-scraper/src/types.ts
+++ b/tools/degreeworks-scraper/src/types.ts
@@ -97,7 +97,6 @@ export type Program = ProgramId & {
   name: string;
   requirements: Record<string, Requirement>;
   specs: string[];
-  otherBlocks: string[];
 };
 
 export type CourseRequirement = {

From 876e49358663e2454f0a0a993f7ba4d0704c5ad9 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Tue, 22 Aug 2023 21:15:43 -0700
Subject: [PATCH 33/48] fix: :bug: encapsulated a little too hard

---
 .../src/components/Scraper.ts                 | 19 ++++++++-----------
 tools/degreeworks-scraper/src/index.ts        |  3 ++-
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/tools/degreeworks-scraper/src/components/Scraper.ts b/tools/degreeworks-scraper/src/components/Scraper.ts
index ac5bf39d..1f1f5a04 100644
--- a/tools/degreeworks-scraper/src/components/Scraper.ts
+++ b/tools/degreeworks-scraper/src/components/Scraper.ts
@@ -7,12 +7,6 @@ import { AuditParser } from "./AuditParser";
 import { DegreeworksClient } from "./DegreeworksClient";
 
 export class Scraper {
-  private static readonly headers = {
-    "Content-Type": "application/json",
-    Cookie: `X-AUTH-TOKEN=${process.env["X_AUTH_TOKEN"]}`,
-    Origin: "https://reg.uci.edu",
-  };
-
   private ap!: AuditParser;
   private dw!: DegreeworksClient;
 
@@ -179,15 +173,18 @@ export class Scraper {
       }),
     ) as Map<string, Map<string, Program>>;
   }
-  static async new(): Promise<Scraper> {
-    if (!process.env["X_AUTH_TOKEN"]) throw new Error("Auth cookie not set.");
-    const studentId = jwtDecode<JwtPayload>(process.env["X_AUTH_TOKEN"].slice("Bearer+".length))
-      ?.sub;
+  static async new(authCookie: string): Promise<Scraper> {
+    const studentId = jwtDecode<JwtPayload>(authCookie.slice("Bearer+".length))?.sub;
     if (!studentId || studentId.length !== 8)
       throw new Error("Could not parse student ID from auth cookie.");
+    const headers = {
+      "Content-Type": "application/json",
+      Cookie: `X-AUTH-TOKEN=${authCookie}`,
+      Origin: "https://reg.uci.edu",
+    };
     const scraper = new Scraper();
     scraper.ap = await AuditParser.new();
-    scraper.dw = await DegreeworksClient.new(studentId, Scraper.headers);
+    scraper.dw = await DegreeworksClient.new(studentId, headers);
     return scraper;
   }
 }
diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index fcfa3fd9..d3b68ef4 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -9,7 +9,8 @@ import "dotenv/config";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
 async function main() {
-  const scraper = await Scraper.new();
+  if (!process.env["X_AUTH_TOKEN"]) throw new Error("Auth cookie not set.");
+  const scraper = await Scraper.new(process.env["X_AUTH_TOKEN"]);
   await scraper.run();
   await mkdir(join(__dirname, "../output"), { recursive: true });
   for (const [fileName, contents] of scraper.get())

From f1ff93cb3198f0ff712dc1c097fc3046e3d38b44 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Wed, 23 Aug 2023 10:38:42 -0700
Subject: [PATCH 34/48] feat: :sparkles: merge specs of all programs with empty
 reqs

---
 .../src/components/Scraper.ts                 | 35 +++++++++++--------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/tools/degreeworks-scraper/src/components/Scraper.ts b/tools/degreeworks-scraper/src/components/Scraper.ts
index 1f1f5a04..c82e8b0f 100644
--- a/tools/degreeworks-scraper/src/components/Scraper.ts
+++ b/tools/degreeworks-scraper/src/components/Scraper.ts
@@ -52,27 +52,32 @@ export class Scraper {
   private cleanUpPrograms(programs: Map<string, Program>) {
     const ret = new Map<string, Program>();
     for (const [name, program] of programs) {
-      if (!Object.keys(program.requirements).length && program.specs.length === 2) {
-        program.requirements = {
-          "Select 1 of the following": {
-            requirementType: "Group",
-            requirementCount: 1,
-            requirements: Object.fromEntries(
-              program.specs.map((x) => [
-                this.parsedSpecializations?.get(x)?.name,
-                this.parsedSpecializations?.get(x)?.requirements,
-              ]),
-            ),
-          },
-        };
+      if (!Object.keys(program.requirements).length) {
+        if (program.specs.length === 1) {
+          program.requirements = this.parsedSpecializations!.get(program.specs[0])!.requirements;
+        } else {
+          program.requirements = {
+            "Select 1 of the following": {
+              requirementType: "Group",
+              requirementCount: 1,
+              requirements: Object.fromEntries(
+                program.specs.map((x) => [
+                  this.parsedSpecializations?.get(x)?.name,
+                  this.parsedSpecializations?.get(x)?.requirements,
+                ]),
+              ),
+            },
+          };
+        }
+        program.specs = [];
       }
       ret.set(name, program);
     }
     return ret;
   }
   async run() {
-    console.log("[Scraper] degreeworks-scraper starting");
     if (this.done) throw new Error("This scraper instance has already finished its run.");
+    console.log("[Scraper] degreeworks-scraper starting");
     this.degrees = await this.dw.getMapping("degrees");
     console.log(`Fetched ${this.degrees.size} degrees`);
     this.majorPrograms = new Set((await this.dw.getMapping("majors")).keys());
@@ -154,7 +159,7 @@ export class Scraper {
       this.parsedUgradPrograms.set("Major in Art History", x);
     }
 
-    // Some programs have an empty requirements block and exactly two specializations.
+    // Some programs have an empty requirements block and more than one specialization.
     // They can be simplified into a "Select 1 of the following" group requirement.
     this.parsedUgradPrograms = this.cleanUpPrograms(this.parsedUgradPrograms);
     this.parsedGradPrograms = this.cleanUpPrograms(this.parsedGradPrograms);

From 5b7a469498cef8830a2c59ea66da2b504cf11939 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Wed, 23 Aug 2023 10:39:37 -0700
Subject: [PATCH 35/48] chore: :wrench: shrimplify spec/other predicate

---
 .../src/components/DegreeworksClient.ts                  | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/tools/degreeworks-scraper/src/components/DegreeworksClient.ts b/tools/degreeworks-scraper/src/components/DegreeworksClient.ts
index cb16f781..d828fd77 100644
--- a/tools/degreeworks-scraper/src/components/DegreeworksClient.ts
+++ b/tools/degreeworks-scraper/src/components/DegreeworksClient.ts
@@ -116,11 +116,10 @@ export class DegreeworksClient {
     return "error" in json
       ? undefined
       : json.blockArray.find(
-          (x) => x.requirementType === "SPEC" && x.requirementValue === specCode,
-        ) ||
-          json.blockArray.find(
-            (x) => x.requirementType === "OTHER" && x.requirementValue === specCode,
-          );
+          (x) =>
+            (x.requirementType === "SPEC" || x.requirementType === "OTHER") &&
+            x.requirementValue === specCode,
+        );
   }
 
   async getMapping<T extends string>(path: T): Promise<Map<string, string>> {

From 3abc6c4cf2536ce6bd905cf6e061d33461f75c5c Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Fri, 8 Sep 2023 21:51:31 -0700
Subject: [PATCH 36/48] docs(types): :books: update comments

---
 tools/degreeworks-scraper/src/types.ts | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/tools/degreeworks-scraper/src/types.ts b/tools/degreeworks-scraper/src/types.ts
index 60057634..c96c9a75 100644
--- a/tools/degreeworks-scraper/src/types.ts
+++ b/tools/degreeworks-scraper/src/types.ts
@@ -95,34 +95,53 @@ export type Program = ProgramId & {
    * @example "Specialization in Digital Signal Processing"
    */
   name: string;
+  /**
+   * The mapping of requirement names to requirement nodes.
+   */
   requirements: Record<string, Requirement>;
+  /**
+   * The set of specializations (if any) that this program has.
+   * If this array is not empty, then exactly one specialization must be selected
+   * to fulfill the requirements of the program.
+   */
   specs: string[];
 };
 
 export type CourseRequirement = {
   requirementType: "Course";
   /**
-   * The number of `courses` required to fulfill this requirement.
+   * The number of courses required to fulfill this requirement.
    */
   courseCount: number;
+  /**
+   * The set of courses that can be taken to fulfill this requirement.
+   */
   courses: string[];
 };
 
 export type UnitRequirement = {
   requirementType: "Unit";
   /**
-   * The number of units earned from taking `courses` that are required to fulfill this requirement.
+   * The number of units earned from the following list of courses
+   * that are required to fulfill this requirement.
    */
   unitCount: number;
+  /**
+   * The set of courses which units count towards this requirement.
+   */
   courses: string[];
 };
 
 export type GroupRequirement = {
   requirementType: "Group";
   /**
-   * The number of `requirements` that must be fulfilled to fulfill this requirement.
+   * The number of requirement from the mapping below
+   * that must be fulfilled to fulfill this requirement.
    */
   requirementCount: number;
+  /**
+   * The mapping of requirement names to requirement nodes.
+   */
   requirements: Record<string, Requirement>;
 };
 

From 0252f422e7822513953c7ea6dd8a86c635172c01 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Fri, 17 Nov 2023 13:35:59 -0800
Subject: [PATCH 37/48] =?UTF-8?q?fix:=20=F0=9F=90=9B=20import=20correct=20?=
 =?UTF-8?q?jwtDecode?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 tools/degreeworks-scraper/src/components/Scraper.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/degreeworks-scraper/src/components/Scraper.ts b/tools/degreeworks-scraper/src/components/Scraper.ts
index c82e8b0f..41a48ff3 100644
--- a/tools/degreeworks-scraper/src/components/Scraper.ts
+++ b/tools/degreeworks-scraper/src/components/Scraper.ts
@@ -1,4 +1,4 @@
-import jwtDecode from "jwt-decode";
+import { jwtDecode } from "jwt-decode";
 import type { JwtPayload } from "jwt-decode";
 
 import { Program } from "../types";

From f1cb7e83e1a4be9fe3725ab14b43c06075381eaf Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Fri, 17 Nov 2023 13:43:06 -0800
Subject: [PATCH 38/48] =?UTF-8?q?chore:=20=F0=9F=94=A7=20stop=20worrying?=
 =?UTF-8?q?=20and=20use=20the=20null-assert?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../src/components/Scraper.ts                 | 24 +++++++++----------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/tools/degreeworks-scraper/src/components/Scraper.ts b/tools/degreeworks-scraper/src/components/Scraper.ts
index 41a48ff3..0125daac 100644
--- a/tools/degreeworks-scraper/src/components/Scraper.ts
+++ b/tools/degreeworks-scraper/src/components/Scraper.ts
@@ -132,11 +132,9 @@ export class Scraper {
     this.degreesAwarded = new Map(
       Array.from(
         new Set(
-          [...this.parsedUgradPrograms, ...this.parsedGradPrograms]
-            .map(([, x]) => x.degreeType)
-            .filter((x) => x) as string[],
+          [...this.parsedUgradPrograms, ...this.parsedGradPrograms].map(([, x]) => x.degreeType!),
         ),
-      ).map((x) => [x, this.degrees?.get(x) as string]),
+      ).map((x) => [x, this.degrees!.get(x)!]),
     );
 
     // Post-processing steps.
@@ -148,9 +146,9 @@ export class Scraper {
 
     let x, y, z;
     if (
-      (x = this.parsedUgradPrograms.get("Major in Art History") as Program) &&
-      (y = this.parsedSpecializations.get("AHGEO") as Program) &&
-      (z = this.parsedSpecializations.get("AHPER") as Program)
+      (x = this.parsedUgradPrograms.get("Major in Art History")!) &&
+      (y = this.parsedSpecializations.get("AHGEO")!) &&
+      (z = this.parsedSpecializations.get("AHPER")!)
     ) {
       x.specs = [];
       x.requirements = { ...x.requirements, ...y.requirements, ...z.requirements };
@@ -170,13 +168,13 @@ export class Scraper {
     if (!this.done) throw new Error("This scraper instance has not yet finished its run.");
     return new Map(
       Object.entries({
-        parsedMinorPrograms: this.parsedMinorPrograms,
-        parsedUgradPrograms: this.parsedUgradPrograms,
-        parsedGradPrograms: this.parsedGradPrograms,
-        parsedSpecializations: this.parsedSpecializations,
-        degreesAwarded: this.degreesAwarded,
+        parsedMinorPrograms: this.parsedMinorPrograms!,
+        parsedUgradPrograms: this.parsedUgradPrograms!,
+        parsedGradPrograms: this.parsedGradPrograms!,
+        parsedSpecializations: this.parsedSpecializations!,
+        degreesAwarded: this.degreesAwarded!,
       }),
-    ) as Map<string, Map<string, Program>>;
+    );
   }
   static async new(authCookie: string): Promise<Scraper> {
     const studentId = jwtDecode<JwtPayload>(authCookie.slice("Bearer+".length))?.sub;

From 21b9b08077621821c75013e5ec1e766c24918e43 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Fri, 17 Nov 2023 13:44:27 -0800
Subject: [PATCH 39/48] =?UTF-8?q?chore:=20=F0=9F=94=A7=20add=20.env=20to?=
 =?UTF-8?q?=20ignore?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 tools/degreeworks-scraper/.gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/degreeworks-scraper/.gitignore b/tools/degreeworks-scraper/.gitignore
index ea1472ec..2132813f 100644
--- a/tools/degreeworks-scraper/.gitignore
+++ b/tools/degreeworks-scraper/.gitignore
@@ -1 +1,2 @@
 output/
+.env

From da5eec9a36d05d469c1ce0b87f651ac97b2d7629 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Mon, 20 Nov 2023 12:25:06 -0800
Subject: [PATCH 40/48] =?UTF-8?q?feat:=20=E2=9C=A8=20force=20GC=20before?=
 =?UTF-8?q?=20enrollment=20history,=20add=20logging?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 services/websoc-scraper-v2/index.ts | 33 +++++++++++++++++------------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/services/websoc-scraper-v2/index.ts b/services/websoc-scraper-v2/index.ts
index ddc933d1..a0696c0a 100644
--- a/services/websoc-scraper-v2/index.ts
+++ b/services/websoc-scraper-v2/index.ts
@@ -473,6 +473,21 @@ async function scrape(name: string, term: Term) {
     },
   };
 
+  const [instructorsDeleted, buildingsDeleted, meetingsDeleted, sectionsDeleted] =
+    await prisma.$transaction([
+      prisma.websocSectionInstructor.deleteMany(params),
+      prisma.websocSectionMeetingBuilding.deleteMany(params),
+      prisma.websocSectionMeeting.deleteMany(params),
+      prisma.websocSection.deleteMany(params),
+    ]);
+
+  logger.info(`Removed ${instructorsDeleted.count} instructors`);
+  logger.info(`Removed ${buildingsDeleted.count} buildings`);
+  logger.info(`Removed ${meetingsDeleted.count} meetings`);
+  logger.info(`Removed ${sectionsDeleted.count} sections`);
+
+  forceGC();
+
   const enrollmentHistory = Object.fromEntries(
     (await prisma.websocEnrollmentHistory.findMany(params)).map((x) => [
       `${x.year}-${x.quarter}-${x.sectionCode}`,
@@ -480,6 +495,8 @@ async function scrape(name: string, term: Term) {
     ]),
   );
 
+  logger.info(`Retrieved ${enrollmentHistory.length} enrollment history entries`);
+
   for (const { data } of Object.values(res)) {
     const key = `${data.year}-${data.quarter}-${data.sectionCode}`;
     if (key in enrollmentHistory) {
@@ -550,7 +567,7 @@ async function scrape(name: string, term: Term) {
     }
   }
 
-  await prisma.$transaction([
+  const [enrollmentInserted, enrollmentRemoved] = await prisma.$transaction([
     prisma.websocEnrollmentHistory.createMany({
       data: Object.values(enrollmentHistory) as Prisma.WebsocEnrollmentHistoryCreateManyInput[],
       skipDuplicates: true,
@@ -558,18 +575,8 @@ async function scrape(name: string, term: Term) {
     prisma.websocEnrollmentHistory.deleteMany(params),
   ]);
 
-  const [instructorsDeleted, buildingsDeleted, meetingsDeleted, sectionsDeleted] =
-    await prisma.$transaction([
-      prisma.websocSectionInstructor.deleteMany(params),
-      prisma.websocSectionMeetingBuilding.deleteMany(params),
-      prisma.websocSectionMeeting.deleteMany(params),
-      prisma.websocSection.deleteMany(params),
-    ]);
-
-  logger.info(`Removed ${instructorsDeleted.count} instructors`);
-  logger.info(`Removed ${buildingsDeleted.count} buildings`);
-  logger.info(`Removed ${meetingsDeleted.count} meetings`);
-  logger.info(`Removed ${sectionsDeleted.count} sections`);
+  logger.info(`Inserted ${enrollmentInserted.count} enrollment history entries`);
+  logger.info(`Removed ${enrollmentRemoved.count} enrollment history entries`);
   logger.info("Sleeping for 3 minutes");
 
   await sleep(SLEEP_DURATION);

From 258eb1169cb8e1c9cd18dcdb2e5babc5d0ea9a27 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Wed, 31 Jan 2024 15:13:40 -0800
Subject: [PATCH 41/48] =?UTF-8?q?chore(deps):=20=F0=9F=94=97=20update=20pa?=
 =?UTF-8?q?ckage.json?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 tools/degreeworks-scraper/package.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/degreeworks-scraper/package.json b/tools/degreeworks-scraper/package.json
index ecd5767e..ec6607b9 100644
--- a/tools/degreeworks-scraper/package.json
+++ b/tools/degreeworks-scraper/package.json
@@ -1,5 +1,5 @@
 {
-  "name": "degreeworks-scraper",
+  "name": "@tools/degreeworks-scraper",
   "version": "0.0.0",
   "private": true,
   "type": "module",
@@ -10,10 +10,10 @@
   "dependencies": {
     "@peterportal-api/types": "workspace:^",
     "cross-fetch": "4.0.0",
-    "dotenv": "16.3.1",
+    "dotenv": "16.4.1",
     "jwt-decode": "4.0.0"
   },
   "devDependencies": {
-    "tsx": "4.1.1"
+    "tsx": "4.7.0"
   }
 }

From 1e18912d75ab8d327bbb70d122de79a5c70f0ee0 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Fri, 1 Mar 2024 11:32:56 -0800
Subject: [PATCH 42/48] =?UTF-8?q?fix(api-client):=20=F0=9F=90=9B=20specify?=
 =?UTF-8?q?=20gzip=20encoding?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../degreeworks-scraper/src/components/PPAPIOfflineClient.ts  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tools/degreeworks-scraper/src/components/PPAPIOfflineClient.ts b/tools/degreeworks-scraper/src/components/PPAPIOfflineClient.ts
index 964a5f22..22fc065b 100644
--- a/tools/degreeworks-scraper/src/components/PPAPIOfflineClient.ts
+++ b/tools/degreeworks-scraper/src/components/PPAPIOfflineClient.ts
@@ -9,7 +9,9 @@ export class PPAPIOfflineClient {
 
   static async new(): Promise<PPAPIOfflineClient> {
     const ppapi = new PPAPIOfflineClient();
-    const res = await fetch("https://api-next.peterportal.org/v1/rest/courses/all");
+    const res = await fetch("https://api-next.peterportal.org/v1/rest/courses/all", {
+      headers: { "accept-encoding": "gzip" },
+    });
     const json: RawResponse<Course[]> = await res.json();
     if (isErrorResponse(json))
       throw new Error("Could not fetch courses cache from PeterPortal API");

From c3683e475f8fcadec968daf5d1dd7a238b413648 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Sat, 2 Mar 2024 14:35:58 -0800
Subject: [PATCH 43/48] =?UTF-8?q?feat:=20=E2=9C=A8=20add=20degree=20schema?=
 =?UTF-8?q?s?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 libs/db/prisma/schema.prisma | 64 +++++++++++++++++++++++++++++-------
 1 file changed, 53 insertions(+), 11 deletions(-)

diff --git a/libs/db/prisma/schema.prisma b/libs/db/prisma/schema.prisma
index d648c063..c15a6057 100644
--- a/libs/db/prisma/schema.prisma
+++ b/libs/db/prisma/schema.prisma
@@ -20,6 +20,11 @@ enum CourseLevel {
   Graduate
 }
 
+enum Division {
+  Undergraduate
+  Graduate
+}
+
 enum Quarter {
   Fall
   Winter
@@ -91,17 +96,11 @@ model Course {
   terms             String[]
 }
 
-model Instructor {
-  ucinetid           String   @id
-  name               String
-  shortenedName      String
-  title              String
-  email              String
-  department         String
-  schools            String[]
-  relatedDepartments String[]
-  courseHistory      Json
-  courses            Json     @default("[]")
+model Degree {
+  id       String   @id
+  name     String
+  division Division
+  Major    Major[]
 }
 
 model GradesInstructor {
@@ -148,6 +147,49 @@ model GradesSection {
   @@unique([year, quarter, sectionCode], name: "idx")
 }
 
+model Instructor {
+  ucinetid           String   @id
+  name               String
+  shortenedName      String
+  title              String
+  email              String
+  department         String
+  schools            String[]
+  relatedDepartments String[]
+  courseHistory      Json
+  courses            Json     @default("[]")
+}
+
+model Major {
+  id           String           @id
+  division     Division
+  code         String
+  degreeId     String
+  degree       Degree           @relation(fields: [degreeId], references: [id])
+  name         String
+  requirements Json
+  specs        Specialization[]
+
+  @@unique([division, code], name: "idx")
+}
+
+model Minor {
+  id           String @id
+  code         String
+  name         String
+  requirements Json
+}
+
+model Specialization {
+  id           String @id
+  code         String
+  degreeType   String
+  name         String
+  requirements Json
+  majorId      String
+  major        Major  @relation(fields: [majorId], references: [id])
+}
+
 model WebsocEnrollmentHistoryEntry {
   year                    String
   quarter                 Quarter

From 1f6dfe815c823cad01768a19bcb5c5a9c4e5e340 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Wed, 6 Mar 2024 14:07:47 -0800
Subject: [PATCH 44/48] =?UTF-8?q?feat:=20=E2=9C=A8=20rework=20schema,=20ge?=
 =?UTF-8?q?t=20scraper=20to=20upload?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 libs/db/prisma/schema.prisma                  | 24 +++-----
 pnpm-lock.yaml                                |  3 +
 tools/degreeworks-scraper/package.json        |  1 +
 .../src/components/AuditParser.ts             |  2 +-
 .../src/components/Scraper.ts                 | 21 +++----
 .../src/components/index.ts                   |  4 ++
 tools/degreeworks-scraper/src/index.ts        | 55 +++++++++++++++----
 7 files changed, 71 insertions(+), 39 deletions(-)
 create mode 100644 tools/degreeworks-scraper/src/components/index.ts

diff --git a/libs/db/prisma/schema.prisma b/libs/db/prisma/schema.prisma
index c15a6057..6ff90b81 100644
--- a/libs/db/prisma/schema.prisma
+++ b/libs/db/prisma/schema.prisma
@@ -161,33 +161,27 @@ model Instructor {
 }
 
 model Major {
-  id           String           @id
-  division     Division
-  code         String
-  degreeId     String
-  degree       Degree           @relation(fields: [degreeId], references: [id])
-  name         String
-  requirements Json
-  specs        Specialization[]
-
-  @@unique([division, code], name: "idx")
+  id              String           @id
+  degreeId        String
+  degree          Degree           @relation(fields: [degreeId], references: [id])
+  code            String
+  name            String
+  requirements    Json
+  specializations Specialization[]
 }
 
 model Minor {
   id           String @id
-  code         String
   name         String
   requirements Json
 }
 
 model Specialization {
   id           String @id
-  code         String
-  degreeType   String
-  name         String
-  requirements Json
   majorId      String
   major        Major  @relation(fields: [majorId], references: [id])
+  name         String
+  requirements Json
 }
 
 model WebsocEnrollmentHistoryEntry {
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 8c9b5660..812d39b6 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -399,6 +399,9 @@ importers:
 
   tools/degreeworks-scraper:
     dependencies:
+      '@libs/db':
+        specifier: workspace:^
+        version: link:../../libs/db
       '@peterportal-api/types':
         specifier: workspace:^
         version: link:../../packages/types
diff --git a/tools/degreeworks-scraper/package.json b/tools/degreeworks-scraper/package.json
index ec6607b9..b370177b 100644
--- a/tools/degreeworks-scraper/package.json
+++ b/tools/degreeworks-scraper/package.json
@@ -8,6 +8,7 @@
     "start": "tsx src/index.ts"
   },
   "dependencies": {
+    "@libs/db": "workspace:^",
     "@peterportal-api/types": "workspace:^",
     "cross-fetch": "4.0.0",
     "dotenv": "16.4.1",
diff --git a/tools/degreeworks-scraper/src/components/AuditParser.ts b/tools/degreeworks-scraper/src/components/AuditParser.ts
index d991eaf0..47316c8f 100644
--- a/tools/degreeworks-scraper/src/components/AuditParser.ts
+++ b/tools/degreeworks-scraper/src/components/AuditParser.ts
@@ -2,7 +2,7 @@ import type { Course } from "@peterportal-api/types";
 
 import type { Block, Program, ProgramId, Requirement, Rule } from "../types";
 
-import { PPAPIOfflineClient } from "./PPAPIOfflineClient";
+import { PPAPIOfflineClient } from ".";
 
 export class AuditParser {
   private static readonly specOrOtherMatcher = /"type":"(?:SPEC|OTHER)","value":"\w+"/g;
diff --git a/tools/degreeworks-scraper/src/components/Scraper.ts b/tools/degreeworks-scraper/src/components/Scraper.ts
index 0125daac..6493e996 100644
--- a/tools/degreeworks-scraper/src/components/Scraper.ts
+++ b/tools/degreeworks-scraper/src/components/Scraper.ts
@@ -1,10 +1,9 @@
 import { jwtDecode } from "jwt-decode";
 import type { JwtPayload } from "jwt-decode";
 
-import { Program } from "../types";
+import type { Program } from "../types";
 
-import { AuditParser } from "./AuditParser";
-import { DegreeworksClient } from "./DegreeworksClient";
+import { AuditParser, DegreeworksClient } from ".";
 
 export class Scraper {
   private ap!: AuditParser;
@@ -166,15 +165,13 @@ export class Scraper {
   }
   get() {
     if (!this.done) throw new Error("This scraper instance has not yet finished its run.");
-    return new Map(
-      Object.entries({
-        parsedMinorPrograms: this.parsedMinorPrograms!,
-        parsedUgradPrograms: this.parsedUgradPrograms!,
-        parsedGradPrograms: this.parsedGradPrograms!,
-        parsedSpecializations: this.parsedSpecializations!,
-        degreesAwarded: this.degreesAwarded!,
-      }),
-    );
+    return {
+      parsedMinorPrograms: this.parsedMinorPrograms!,
+      parsedUgradPrograms: this.parsedUgradPrograms!,
+      parsedGradPrograms: this.parsedGradPrograms!,
+      parsedSpecializations: this.parsedSpecializations!,
+      degreesAwarded: this.degreesAwarded!,
+    };
   }
   static async new(authCookie: string): Promise<Scraper> {
     const studentId = jwtDecode<JwtPayload>(authCookie.slice("Bearer+".length))?.sub;
diff --git a/tools/degreeworks-scraper/src/components/index.ts b/tools/degreeworks-scraper/src/components/index.ts
new file mode 100644
index 00000000..e8ce9349
--- /dev/null
+++ b/tools/degreeworks-scraper/src/components/index.ts
@@ -0,0 +1,4 @@
+export { AuditParser } from "./AuditParser";
+export { DegreeworksClient } from "./DegreeworksClient";
+export { PPAPIOfflineClient } from "./PPAPIOfflineClient";
+export { Scraper } from "./Scraper";
diff --git a/tools/degreeworks-scraper/src/index.ts b/tools/degreeworks-scraper/src/index.ts
index d3b68ef4..ecc46a90 100644
--- a/tools/degreeworks-scraper/src/index.ts
+++ b/tools/degreeworks-scraper/src/index.ts
@@ -1,23 +1,56 @@
-import { mkdir, writeFile } from "node:fs/promises";
-import { dirname, join } from "node:path";
-import { fileURLToPath } from "node:url";
+import { PrismaClient } from "@libs/db";
 
-import { Scraper } from "./components/Scraper";
+import { Scraper } from "./components";
 
 import "dotenv/config";
 
-const __dirname = dirname(fileURLToPath(import.meta.url));
+const prisma = new PrismaClient();
+
+type Division = "Undergraduate" | "Graduate";
 
 async function main() {
   if (!process.env["X_AUTH_TOKEN"]) throw new Error("Auth cookie not set.");
   const scraper = await Scraper.new(process.env["X_AUTH_TOKEN"]);
   await scraper.run();
-  await mkdir(join(__dirname, "../output"), { recursive: true });
-  for (const [fileName, contents] of scraper.get())
-    await writeFile(
-      join(__dirname, `../output/${fileName}.json`),
-      JSON.stringify(Object.fromEntries(contents)),
-    );
+  const {
+    degreesAwarded,
+    parsedSpecializations,
+    parsedGradPrograms,
+    parsedMinorPrograms,
+    parsedUgradPrograms,
+  } = scraper.get();
+  const degreeData = Array.from(degreesAwarded.entries()).map(([id, name]) => ({
+    id,
+    name,
+    division: (id.startsWith("B") ? "Undergraduate" : "Graduate") as Division,
+  }));
+  const majorData = [
+    ...Array.from(parsedUgradPrograms.values()),
+    ...Array.from(parsedGradPrograms.values()),
+  ].map(({ name, degreeType, code, requirements }) => ({
+    id: `${degreeType}-${code}`,
+    degreeId: degreeType!,
+    code,
+    name,
+    requirements,
+  }));
+  const minorData = Array.from(parsedMinorPrograms.values()).map(
+    ({ name, code: id, requirements }) => ({ id, name, requirements }),
+  );
+  const specData = Array.from(parsedSpecializations.values()).map(
+    ({ name, degreeType, code, requirements }) => ({
+      id: `${degreeType}-${code}`,
+      majorId: `${degreeType}-${code.slice(0, code.length - 1)}`,
+      name,
+      requirements,
+    }),
+  );
+  await prisma.$transaction([
+    prisma.degree.createMany({ data: degreeData, skipDuplicates: true }),
+    prisma.major.createMany({ data: majorData, skipDuplicates: true }),
+    prisma.minor.createMany({ data: minorData, skipDuplicates: true }),
+    prisma.specialization.createMany({ data: specData, skipDuplicates: true }),
+  ]);
 }
 
 main().then();

From e24d7f15365973e5b4970b72cfb8f138d4c75791 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Thu, 7 Mar 2024 12:20:49 -0800
Subject: [PATCH 45/48] =?UTF-8?q?feat:=20=E2=9C=A8=20implement=20rest/gql?=
 =?UTF-8?q?=20endpoints=20for=20degree=20data?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 apps/api/src/routes/v1/graphql/resolvers.ts   |   8 ++
 .../routes/v1/graphql/schema/degrees.graphql  |  39 ++++++
 .../src/routes/v1/graphql/schema/enum.graphql |   5 +
 .../routes/v1/rest/degrees/{id}/+endpoint.ts  | 119 ++++++++++++++++++
 .../src/routes/v1/rest/degrees/{id}/schema.ts |  14 +++
 libs/db/prisma/schema.prisma                  |   2 +-
 6 files changed, 186 insertions(+), 1 deletion(-)
 create mode 100644 apps/api/src/routes/v1/graphql/schema/degrees.graphql
 create mode 100644 apps/api/src/routes/v1/rest/degrees/{id}/+endpoint.ts
 create mode 100644 apps/api/src/routes/v1/rest/degrees/{id}/schema.ts

diff --git a/apps/api/src/routes/v1/graphql/resolvers.ts b/apps/api/src/routes/v1/graphql/resolvers.ts
index 52e56058..e67dcf8e 100644
--- a/apps/api/src/routes/v1/graphql/resolvers.ts
+++ b/apps/api/src/routes/v1/graphql/resolvers.ts
@@ -9,6 +9,14 @@ export const resolvers: ApolloServerOptions<BaseContext>["resolvers"] = {
     course: proxyRestApi("/v1/rest/courses", { pathArg: "courseId" }),
     courses: proxyRestApi("/v1/rest/courses", { argsTransform: geTransform }),
     allCourses: proxyRestApi("/v1/rest/courses/all"),
+    major: proxyRestApi("/v1/rest/degrees/majors"),
+    majors: proxyRestApi("/v1/rest/degrees/majors"),
+    minor: proxyRestApi("/v1/rest/degrees/minors"),
+    minors: proxyRestApi("/v1/rest/degrees/minors"),
+    specialization: proxyRestApi("/v1/rest/degrees/specializations"),
+    specializations: proxyRestApi("/v1/rest/degrees/specializations"),
+    specializationsByMajorId: proxyRestApi("/v1/rest/degrees/specializations"),
+    allDegrees: proxyRestApi("/v1/rest/degrees/all"),
     enrollmentHistory: proxyRestApi("/v1/rest/enrollmentHistory"),
     rawGrades: proxyRestApi("/v1/rest/grades/raw"),
     aggregateGrades: proxyRestApi("/v1/rest/grades/aggregate"),
diff --git a/apps/api/src/routes/v1/graphql/schema/degrees.graphql b/apps/api/src/routes/v1/graphql/schema/degrees.graphql
new file mode 100644
index 00000000..609dbb03
--- /dev/null
+++ b/apps/api/src/routes/v1/graphql/schema/degrees.graphql
@@ -0,0 +1,39 @@
+type Specialization {
+  id: String!
+  majorId: String!
+  name: String!
+  requirements: JSON!
+}
+
+type Major {
+  id: String!
+  degreeId: String!
+  code: String!
+  name: String!
+  requirements: JSON!
+  specializations: [Specialization!]!
+}
+
+type Minor {
+  id: String!
+  name: String!
+  requirements: JSON!
+}
+
+type Degree {
+  id: String!
+  name: String!
+  division: DegreeDivision!
+  majors: [Major!]!
+}
+
+extend type Query {
+  major(id: String!): Major!
+  majors(degreeId: String, nameContains: String): [Major!]!
+  minor(id: String!): Minor!
+  minors(nameContains: String): [Minor!]!
+  specialization(id: String!): Specialization!
+  specializations(nameContains: String): [Specialization!]!
+  specializationsByMajorId(majorId: String!): [Specialization!]!
+  allDegrees: [Degree!]!
+}
diff --git a/apps/api/src/routes/v1/graphql/schema/enum.graphql b/apps/api/src/routes/v1/graphql/schema/enum.graphql
index 38820ffc..b33d4cf6 100644
--- a/apps/api/src/routes/v1/graphql/schema/enum.graphql
+++ b/apps/api/src/routes/v1/graphql/schema/enum.graphql
@@ -69,3 +69,8 @@ enum WebsocSectionFinalExamStatus {
   TBA_FINAL
   SCHEDULED_FINAL
 }
+"The set of valid degree divisions."
+enum DegreeDivision {
+  Undergraduate
+  Graduate
+}
diff --git a/apps/api/src/routes/v1/rest/degrees/{id}/+endpoint.ts b/apps/api/src/routes/v1/rest/degrees/{id}/+endpoint.ts
new file mode 100644
index 00000000..cd5caca1
--- /dev/null
+++ b/apps/api/src/routes/v1/rest/degrees/{id}/+endpoint.ts
@@ -0,0 +1,119 @@
+import { PrismaClient } from "@libs/db";
+import { createHandler } from "@libs/lambda";
+
+import { ProgramSchema, SpecializationSchema } from "./schema";
+
+const prisma = new PrismaClient();
+
+async function onWarm() {
+  await prisma.$connect();
+}
+
+export const GET = createHandler(async (event, context, res) => {
+  const headers = event.headers;
+  const params = event.pathParameters ?? {};
+  const query = event.queryStringParameters ?? {};
+  const requestId = context.awsRequestId;
+
+  switch (params?.id) {
+    case "all":
+      return res.createOKResult(
+        await prisma.degree.findMany({
+          include: { majors: { include: { specializations: true } } },
+        }),
+        headers,
+        requestId,
+      );
+    case "majors":
+    case "minors": {
+      const maybeParsed = ProgramSchema.safeParse(query);
+      if (maybeParsed.success) {
+        const { data } = maybeParsed;
+        console.log(data);
+        // Ugly TypeScript kludge that lets this union typecheck properly when calling the find methods.
+        // This is real code, written by real software engineers.
+        const table =
+          params.id === "majors" ? prisma.major : (prisma.minor as unknown as typeof prisma.major);
+        if (!Object.keys(data).length)
+          return res.createOKResult(
+            await table.findMany({
+              include: { specializations: params.id === "majors" ? true : undefined },
+            }),
+            headers,
+            requestId,
+          );
+        if ("id" in data) {
+          const row = await table.findFirst({
+            where: { id: data.id },
+            include: { specializations: params.id === "majors" ? true : undefined },
+          });
+          return row
+            ? res.createOKResult(row, headers, requestId)
+            : res.createErrorResult(
+                404,
+                `${params.id === "majors" ? "Major" : "Minor"} with ID ${data.id} not found`,
+                requestId,
+              );
+        }
+        if ("degreeId" in data || "nameContains" in data) {
+          if (params.id === "minors" && data.degreeId)
+            return res.createErrorResult(400, "Invalid input", requestId);
+          return res.createOKResult(
+            await table.findMany({
+              where: {
+                degreeId: data.degreeId,
+                name: { contains: data.nameContains, mode: "insensitive" },
+              },
+              include: { specializations: params.id === "majors" ? true : undefined },
+            }),
+            headers,
+            requestId,
+          );
+        }
+      } else {
+        return res.createErrorResult(
+          400,
+          maybeParsed.error.issues.map((issue) => issue.message).join("; "),
+          requestId,
+        );
+      }
+      break;
+    }
+    case "specializations": {
+      const maybeParsed = SpecializationSchema.safeParse(query);
+      if (maybeParsed.success) {
+        const { data } = maybeParsed;
+        if (!Object.keys(data).length)
+          return res.createOKResult(await prisma.specialization.findMany(), headers, requestId);
+        if ("id" in data) {
+          const row = await prisma.specialization.findFirst({ where: { id: data.id } });
+          return row
+            ? res.createOKResult(row, headers, requestId)
+            : res.createErrorResult(404, `Specialization with ID ${data.id} not found`, requestId);
+        }
+        if ("majorId" in data)
+          return res.createOKResult(
+            await prisma.specialization.findMany({ where: { majorId: data.majorId } }),
+            headers,
+            requestId,
+          );
+        if ("nameContains" in data) {
+          return res.createOKResult(
+            await prisma.specialization.findMany({
+              where: { name: { contains: data.nameContains, mode: "insensitive" } },
+            }),
+            headers,
+            requestId,
+          );
+        }
+      } else {
+        return res.createErrorResult(
+          400,
+          maybeParsed.error.issues.map((issue) => issue.message).join("; "),
+          requestId,
+        );
+      }
+    }
+  }
+  return res.createErrorResult(400, "Invalid endpoint", requestId);
+}, onWarm);
diff --git a/apps/api/src/routes/v1/rest/degrees/{id}/schema.ts b/apps/api/src/routes/v1/rest/degrees/{id}/schema.ts
new file mode 100644
index 00000000..af8ddb4a
--- /dev/null
+++ b/apps/api/src/routes/v1/rest/degrees/{id}/schema.ts
@@ -0,0 +1,14 @@
+import { z } from "zod";
+
+export const ProgramSchema = z.union([
+  z.object({ id: z.string() }),
+  z.object({ degreeId: z.string().optional(), nameContains: z.string().optional() }),
+  z.object({}),
+]);
+
+export const SpecializationSchema = z.union([
+  z.object({ id: z.string() }),
+  z.object({ majorId: z.string() }),
+  z.object({ nameContains: z.string() }),
+  z.object({}),
+]);
diff --git a/libs/db/prisma/schema.prisma b/libs/db/prisma/schema.prisma
index 6ff90b81..109e7414 100644
--- a/libs/db/prisma/schema.prisma
+++ b/libs/db/prisma/schema.prisma
@@ -100,7 +100,7 @@ model Degree {
   id       String   @id
   name     String
   division Division
-  Major    Major[]
+  majors   Major[]
 }
 
 model GradesInstructor {

From be32515f83ceb8fbd96395254de9b486cf29a449 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Sat, 9 Mar 2024 13:40:01 -0800
Subject: [PATCH 46/48] =?UTF-8?q?chore(deps):=20=F0=9F=94=97=20fix=20broke?=
 =?UTF-8?q?n=20lockfile?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pnpm-lock.yaml | 98 +++++++++-----------------------------------------
 1 file changed, 17 insertions(+), 81 deletions(-)

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 89a6270b..a4363a11 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -2898,16 +2898,6 @@ packages:
       conventional-changelog-conventionalcommits: 7.0.2
     dev: false
 
-  /@commitlint/config-validator@18.6.0:
-    resolution: {integrity: sha512-Ptfa865arNozlkjxrYG3qt6wT9AlhNUHeuDyKEZiTL/l0ftncFhK/KN0t/EAMV2tec+0Mwxo0FmhbESj/bI+1g==}
-    engines: {node: '>=v18'}
-    requiresBuild: true
-    dependencies:
-      '@commitlint/types': 18.6.0
-      ajv: 8.12.0
-    dev: false
-    optional: true
-
   /@commitlint/config-validator@19.0.3:
     resolution: {integrity: sha512-2D3r4PKjoo59zBc2auodrSCaUnCSALCx54yveOFwwP/i2kfEAQrygwOleFWswLqK0UL/F9r07MFi5ev2ohyM4Q==}
     engines: {node: '>=v18'}
@@ -2928,13 +2918,6 @@ packages:
       lodash.upperfirst: 4.3.1
     dev: false
 
-  /@commitlint/execute-rule@18.4.4:
-    resolution: {integrity: sha512-a37Nd3bDQydtg9PCLLWM9ZC+GO7X5i4zJvrggJv5jBhaHsXeQ9ZWdO6ODYR+f0LxBXXNYK3geYXJrCWUCP8JEg==}
-    engines: {node: '>=v18'}
-    requiresBuild: true
-    dev: false
-    optional: true
-
   /@commitlint/execute-rule@19.0.0:
     resolution: {integrity: sha512-mtsdpY1qyWgAO/iOK0L6gSGeR7GFcdW7tIjcNFxcWkfLDF5qVbPHKuGATFqRMsxcO8OUKNj0+3WOHB7EHm4Jdw==}
     engines: {node: '>=v18'}
@@ -2966,28 +2949,6 @@ packages:
       '@commitlint/types': 19.0.3
     dev: false
 
-  /@commitlint/load@18.6.0(@types/node@20.11.24)(typescript@5.3.3):
-    resolution: {integrity: sha512-RRssj7TmzT0bowoEKlgwg8uQ7ORXWkw7lYLsZZBMi9aInsJuGNLNWcMxJxRZbwxG3jkCidGUg85WmqJvRjsaDA==}
-    engines: {node: '>=v18'}
-    requiresBuild: true
-    dependencies:
-      '@commitlint/config-validator': 18.6.0
-      '@commitlint/execute-rule': 18.4.4
-      '@commitlint/resolve-extends': 18.6.0
-      '@commitlint/types': 18.6.0
-      chalk: 4.1.2
-      cosmiconfig: 8.3.6(typescript@5.3.3)
-      cosmiconfig-typescript-loader: 5.0.0(@types/node@20.11.24)(cosmiconfig@8.3.6)(typescript@5.3.3)
-      lodash.isplainobject: 4.0.6
-      lodash.merge: 4.6.2
-      lodash.uniq: 4.5.0
-      resolve-from: 5.0.0
-    transitivePeerDependencies:
-      - '@types/node'
-      - typescript
-    dev: false
-    optional: true
-
   /@commitlint/load@19.0.3(@types/node@20.11.24)(typescript@5.3.3):
     resolution: {integrity: sha512-18Tk/ZcDFRKIoKfEcl7kC+bYkEQ055iyKmGsYDoYWpKf6FUvBrP9bIWapuy/MB+kYiltmP9ITiUx6UXtqC9IRw==}
     engines: {node: '>=v18'}
@@ -3031,20 +2992,6 @@ packages:
       minimist: 1.2.8
     dev: false
 
-  /@commitlint/resolve-extends@18.6.0:
-    resolution: {integrity: sha512-k2Xp+Fxeggki2i90vGrbiLDMefPius3zGSTFFlRAPKce/SWLbZtI+uqE9Mne23mHO5lmcSV8z5m6ziiJwGpOcg==}
-    engines: {node: '>=v18'}
-    requiresBuild: true
-    dependencies:
-      '@commitlint/config-validator': 18.6.0
-      '@commitlint/types': 18.6.0
-      import-fresh: 3.3.0
-      lodash.mergewith: 4.6.2
-      resolve-from: 5.0.0
-      resolve-global: 1.0.0
-    dev: false
-    optional: true
-
   /@commitlint/resolve-extends@19.0.3:
     resolution: {integrity: sha512-18BKmta8OC8+Ub+Q3QGM9l27VjQaXobloVXOrMvu8CpEwJYv62vC/t7Ka5kJnsW0tU9q1eMqJFZ/nN9T/cOaIA==}
     engines: {node: '>=v18'}
@@ -3080,15 +3027,6 @@ packages:
       find-up: 7.0.0
     dev: false
 
-  /@commitlint/types@18.6.0:
-    resolution: {integrity: sha512-oavoKLML/eJa2rJeyYSbyGAYzTxQ6voG5oeX3OrxpfrkRWhJfm4ACnhoRf5tgiybx2MZ+EVFqC1Lw3W8/uwpZA==}
-    engines: {node: '>=v18'}
-    requiresBuild: true
-    dependencies:
-      chalk: 4.1.2
-    dev: false
-    optional: true
-
   /@commitlint/types@19.0.3:
     resolution: {integrity: sha512-tpyc+7i6bPG9mvaBbtKUeghfyZSDgWquIDfMgqYtTbmZ9Y9VzEm2je9EYcQ0aoz5o7NvGS+rcDec93yO08MHYA==}
     engines: {node: '>=v18'}
@@ -7395,7 +7333,7 @@ packages:
       longest: 2.0.1
       word-wrap: 1.2.3
     optionalDependencies:
-      '@commitlint/load': 18.6.0(@types/node@20.11.24)(typescript@5.3.3)
+      '@commitlint/load': 19.0.3(@types/node@20.11.24)(typescript@5.3.3)
     transitivePeerDependencies:
       - '@types/node'
       - typescript
@@ -7740,6 +7678,11 @@ packages:
       webpack: 5.84.1
     dev: false
 
+  /dotenv@16.4.1:
+    resolution: {integrity: sha512-CjA3y+Dr3FyFDOAMnxZEGtnW9KBR2M0JvvUtXNW+dYJL5ROWxP9DUHCwgFqpMk0OXCc0ljhaNTr2w/kutYIcHQ==}
+    engines: {node: '>=12'}
+    dev: false
+
   /dotenv@16.4.5:
     resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==}
     engines: {node: '>=12'}
@@ -8803,15 +8746,6 @@ packages:
       ini: 4.1.1
     dev: false
 
-  /global-dirs@0.1.1:
-    resolution: {integrity: sha512-NknMLn7F2J7aflwFOlGdNIuCDpN3VGoSoB+aap3KABFWbHVn1TCgFC+np23J8W2BiZbjfEw3BFBycSMv1AFblg==}
-    engines: {node: '>=4'}
-    requiresBuild: true
-    dependencies:
-      ini: 1.3.8
-    dev: false
-    optional: true
-
   /global-dirs@3.0.1:
     resolution: {integrity: sha512-NBcGGFbBA9s1VzD41QXDG+3++t9Mn5t1FpLdhESY6oKY4gYTFpX4wO3sqGUa0Srjtbfj3szX0RnemmrVRUdULA==}
     engines: {node: '>=10'}
@@ -11938,15 +11872,6 @@ packages:
     resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==}
     engines: {node: '>=8'}
 
-  /resolve-global@1.0.0:
-    resolution: {integrity: sha512-zFa12V4OLtT5XUX/Q4VLvTfBf+Ok0SPc1FNGM/z9ctUdiU618qwKpWnd0CHs3+RqROfyEg/DhuHbMWYqcgljEw==}
-    engines: {node: '>=8'}
-    requiresBuild: true
-    dependencies:
-      global-dirs: 0.1.1
-    dev: false
-    optional: true
-
   /resolve-pathname@3.0.0:
     resolution: {integrity: sha512-C7rARubxI8bXFNB/hqcp/4iUeIXJhJZvFPFPiSPRnhU5UPxzMFIl+2E6yY6c4k9giDJAhtV+enfA+G89N6Csng==}
     dev: false
@@ -12989,6 +12914,17 @@ packages:
       - ts-node
     dev: true
 
+  /tsx@4.7.0:
+    resolution: {integrity: sha512-I+t79RYPlEYlHn9a+KzwrvEwhJg35h/1zHsLC2JXvhC2mdynMv6Zxzvhv5EMV6VF5qJlLlkSnMVvdZV3PSIGcg==}
+    engines: {node: '>=18.0.0'}
+    hasBin: true
+    dependencies:
+      esbuild: 0.19.11
+      get-tsconfig: 4.7.2
+    optionalDependencies:
+      fsevents: 2.3.3
+    dev: true
+
   /tsx@4.7.1:
     resolution: {integrity: sha512-8d6VuibXHtlN5E3zFkgY8u4DX7Y3Z27zvvPKVmLon/D4AjuKzarkUBTLDBgj9iTQ0hg5xM7c/mYiRVM+HETf0g==}
     engines: {node: '>=18.0.0'}

From 01f5751cd4ed49f0e553e51d57833e206282c5cf Mon Sep 17 00:00:00 2001
From: Aponia <brianmv@uci.edu>
Date: Sun, 19 May 2024 12:56:58 -0700
Subject: [PATCH 47/48] =?UTF-8?q?refactor(api):=20=E2=99=BB=EF=B8=8F=20deg?=
 =?UTF-8?q?rees=20endpoint=20(#147)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../routes/v1/rest/degrees/{id}/+endpoint.ts  | 169 ++++++++++--------
 .../src/routes/v1/rest/degrees/{id}/schema.ts |  52 ++++--
 2 files changed, 138 insertions(+), 83 deletions(-)

diff --git a/apps/api/src/routes/v1/rest/degrees/{id}/+endpoint.ts b/apps/api/src/routes/v1/rest/degrees/{id}/+endpoint.ts
index cd5caca1..a1cc3655 100644
--- a/apps/api/src/routes/v1/rest/degrees/{id}/+endpoint.ts
+++ b/apps/api/src/routes/v1/rest/degrees/{id}/+endpoint.ts
@@ -9,6 +9,34 @@ async function onWarm() {
   await prisma.$connect();
 }
 
+const degreeRepository = {
+  majors: {
+    findMany: async () => {
+      return await prisma.major.findMany({ include: { specializations: true } });
+    },
+    findFirstById: async (id: string) => {
+      return await prisma.major.findFirst({ where: { id }, include: { specializations: true } });
+    },
+    findManyNameContains: async (degreeId: string, contains?: string) => {
+      return await prisma.major.findMany({
+        where: {
+          degreeId,
+          name: { contains, mode: "insensitive" },
+        },
+        include: { specializations: true },
+      });
+    },
+  },
+  minors: {
+    findMany: async () => {
+      return await prisma.minor.findMany({});
+    },
+    findFirstById: async (id: string) => {
+      return await prisma.minor.findFirst({ where: { id } });
+    },
+  },
+};
+
 export const GET = createHandler(async (event, context, res) => {
   const headers = event.headers;
   const params = event.pathParameters ?? {};
@@ -24,96 +52,95 @@ export const GET = createHandler(async (event, context, res) => {
         headers,
         requestId,
       );
-    case "majors":
+
+    case "majors": // falls through
     case "minors": {
-      const maybeParsed = ProgramSchema.safeParse(query);
-      if (maybeParsed.success) {
-        const { data } = maybeParsed;
-        console.log(data);
-        // Ugly TypeScript kludge that lets this union typecheck properly when calling the find methods.
-        // This is real code, written by real software engineers.
-        const table =
-          params.id === "majors" ? prisma.major : (prisma.minor as unknown as typeof prisma.major);
-        if (!Object.keys(data).length)
-          return res.createOKResult(
-            await table.findMany({
-              include: { specializations: params.id === "majors" ? true : undefined },
-            }),
-            headers,
-            requestId,
-          );
-        if ("id" in data) {
-          const row = await table.findFirst({
-            where: { id: data.id },
-            include: { specializations: params.id === "majors" ? true : undefined },
-          });
-          return row
-            ? res.createOKResult(row, headers, requestId)
+      const parsedQuery = ProgramSchema.safeParse(query);
+
+      if (!parsedQuery.success) {
+        return res.createErrorResult(
+          400,
+          parsedQuery.error.issues.map((issue) => issue.message).join("; "),
+          requestId,
+        );
+      }
+
+      switch (parsedQuery.data.type) {
+        case "id": {
+          const result = await degreeRepository[params.id].findFirstById(parsedQuery.data.id);
+          return result
+            ? res.createOKResult(result, headers, requestId)
             : res.createErrorResult(
                 404,
-                `${params.id === "majors" ? "Major" : "Minor"} with ID ${data.id} not found`,
+                `${params.id === "majors" ? "Major" : "Minor"} with ID ${parsedQuery.data.id} not found`,
                 requestId,
               );
         }
-        if ("degreeId" in data || "nameContains" in data) {
-          if (params.id === "minors" && data.degreeId)
+
+        case "degreeOrName": {
+          const { degreeId, nameContains } = parsedQuery.data;
+
+          if (params.id === "minors" && degreeId != null) {
             return res.createErrorResult(400, "Invalid input", requestId);
-          return res.createOKResult(
-            await table.findMany({
-              where: {
-                degreeId: data.degreeId,
-                name: { contains: data.nameContains, mode: "insensitive" },
-              },
-              include: { specializations: params.id === "majors" ? true : undefined },
-            }),
-            headers,
-            requestId,
-          );
+          }
+
+          const result = await degreeRepository.majors.findManyNameContains(degreeId, nameContains);
+          return res.createOKResult(result, headers, requestId);
+        }
+
+        case "empty": {
+          const result = await degreeRepository[params.id].findMany();
+          return res.createOKResult(result, headers, requestId);
         }
-      } else {
+      }
+      break;
+    }
+
+    case "specializations": {
+      const parsedQuery = SpecializationSchema.safeParse(query);
+
+      if (!parsedQuery.success) {
         return res.createErrorResult(
           400,
-          maybeParsed.error.issues.map((issue) => issue.message).join("; "),
+          parsedQuery.error.issues.map((issue) => issue.message).join("; "),
           requestId,
         );
       }
-      break;
-    }
-    case "specializations": {
-      const maybeParsed = SpecializationSchema.safeParse(query);
-      if (maybeParsed.success) {
-        const { data } = maybeParsed;
-        if (!Object.keys(data).length)
-          return res.createOKResult(await prisma.specialization.findMany(), headers, requestId);
-        if ("id" in data) {
-          const row = await prisma.specialization.findFirst({ where: { id: data.id } });
+
+      switch (parsedQuery.data.type) {
+        case "id": {
+          const row = await prisma.specialization.findFirst({ where: { id: parsedQuery.data.id } });
+
           return row
             ? res.createOKResult(row, headers, requestId)
-            : res.createErrorResult(404, `Specialization with ID ${data.id} not found`, requestId);
+            : res.createErrorResult(
+                404,
+                `Specialization with ID ${parsedQuery.data.id} not found`,
+                requestId,
+              );
+        }
+
+        case "major": {
+          const result = await prisma.specialization.findMany({
+            where: { majorId: parsedQuery.data.majorId },
+          });
+          return res.createOKResult(result, headers, requestId);
         }
-        if ("majorId" in data)
-          return res.createOKResult(
-            await prisma.specialization.findMany({ where: { majorId: data.majorId } }),
-            headers,
-            requestId,
-          );
-        if ("nameContains" in data) {
-          return res.createOKResult(
-            await prisma.specialization.findMany({
-              where: { name: { contains: data.nameContains, mode: "insensitive" } },
-            }),
-            headers,
-            requestId,
-          );
+
+        case "name": {
+          const result = await prisma.specialization.findMany({
+            where: { name: { contains: parsedQuery.data.nameContains, mode: "insensitive" } },
+          });
+          return res.createOKResult(result, headers, requestId);
+        }
+
+        case "empty": {
+          const result = await prisma.specialization.findMany();
+          return res.createOKResult(result, headers, requestId);
         }
-      } else {
-        return res.createErrorResult(
-          400,
-          maybeParsed.error.issues.map((issue) => issue.message).join("; "),
-          requestId,
-        );
       }
     }
   }
+
   return res.createErrorResult(400, "Invalid endpoint", requestId);
 }, onWarm);
diff --git a/apps/api/src/routes/v1/rest/degrees/{id}/schema.ts b/apps/api/src/routes/v1/rest/degrees/{id}/schema.ts
index af8ddb4a..04dc5e69 100644
--- a/apps/api/src/routes/v1/rest/degrees/{id}/schema.ts
+++ b/apps/api/src/routes/v1/rest/degrees/{id}/schema.ts
@@ -1,14 +1,42 @@
 import { z } from "zod";
 
-export const ProgramSchema = z.union([
-  z.object({ id: z.string() }),
-  z.object({ degreeId: z.string().optional(), nameContains: z.string().optional() }),
-  z.object({}),
-]);
-
-export const SpecializationSchema = z.union([
-  z.object({ id: z.string() }),
-  z.object({ majorId: z.string() }),
-  z.object({ nameContains: z.string() }),
-  z.object({}),
-]);
+export const ProgramSchema = z
+  .union([
+    z.object({ id: z.string() }),
+    z.object({ degreeId: z.string().optional(), nameContains: z.string().optional() }),
+    z.object({}),
+  ])
+  .transform((data) => {
+    if ("id" in data) {
+      return { type: "id" as const, ...data };
+    }
+
+    if ("degreeId" in data && data.degreeId != null) {
+      return { type: "degreeOrName" as const, degreeId: data.degreeId, ...data };
+    }
+
+    return { type: "empty" as const, ...data };
+  });
+
+export const SpecializationSchema = z
+  .union([
+    z.object({ id: z.string() }),
+    z.object({ majorId: z.string() }),
+    z.object({ nameContains: z.string() }),
+    z.object({}),
+  ])
+  .transform((data) => {
+    if ("id" in data) {
+      return { type: "id" as const, ...data };
+    }
+
+    if ("majorId" in data) {
+      return { type: "major" as const, ...data };
+    }
+
+    if ("nameContains" in data) {
+      return { type: "name" as const, ...data };
+    }
+
+    return { type: "empty" as const, ...data };
+  });

From d81b2cc0a23ba7b94381d0d99e666571afbeca25 Mon Sep 17 00:00:00 2001
From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com>
Date: Sun, 19 May 2024 13:15:44 -0700
Subject: [PATCH 48/48] =?UTF-8?q?fix:=20=F0=9F=90=9B=20address=20requested?=
 =?UTF-8?q?=20changes?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../src/components/DegreeworksClient.ts         |  8 ++++----
 .../src/components/PPAPIOfflineClient.ts        | 17 ++++++++++-------
 .../src/components/Scraper.ts                   |  7 ++++---
 3 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/tools/degreeworks-scraper/src/components/DegreeworksClient.ts b/tools/degreeworks-scraper/src/components/DegreeworksClient.ts
index d828fd77..6f475885 100644
--- a/tools/degreeworks-scraper/src/components/DegreeworksClient.ts
+++ b/tools/degreeworks-scraper/src/components/DegreeworksClient.ts
@@ -28,10 +28,10 @@ export class DegreeworksClient {
      * as the catalog year. Otherwise, we use the former.
      */
     const currentYear = new Date().getUTCFullYear();
-    dw.catalogYear = `${currentYear}${currentYear + 1}`;
-    if (!(await dw.getMajorAudit("BS", "U", "201"))) {
-      dw.catalogYear = `${currentYear - 1}${currentYear}`;
-    }
+    const dataThisYear = await dw.getMajorAudit("BS", "U", "201");
+    dw.catalogYear = dataThisYear
+      ? `${currentYear}${currentYear + 1}`
+      : `${currentYear - 1}${currentYear}`;
     console.log(`[DegreeworksClient.new] Set catalogYear to ${dw.catalogYear}`);
     return dw;
   }
diff --git a/tools/degreeworks-scraper/src/components/PPAPIOfflineClient.ts b/tools/degreeworks-scraper/src/components/PPAPIOfflineClient.ts
index 22fc065b..a1c9729a 100644
--- a/tools/degreeworks-scraper/src/components/PPAPIOfflineClient.ts
+++ b/tools/degreeworks-scraper/src/components/PPAPIOfflineClient.ts
@@ -2,6 +2,8 @@ import { isErrorResponse } from "@peterportal-api/types";
 import type { Course, RawResponse } from "@peterportal-api/types";
 import fetch from "cross-fetch";
 
+const ENDPOINT = "https://api-next.peterportal.org/v1/rest/courses/all";
+
 export class PPAPIOfflineClient {
   private cache = new Map<string, Course>();
 
@@ -9,9 +11,7 @@ export class PPAPIOfflineClient {
 
   static async new(): Promise<PPAPIOfflineClient> {
     const ppapi = new PPAPIOfflineClient();
-    const res = await fetch("https://api-next.peterportal.org/v1/rest/courses/all", {
-      headers: { "accept-encoding": "gzip" },
-    });
+    const res = await fetch(ENDPOINT, { headers: { "accept-encoding": "gzip" } });
     const json: RawResponse<Course[]> = await res.json();
     if (isErrorResponse(json))
       throw new Error("Could not fetch courses cache from PeterPortal API");
@@ -22,13 +22,16 @@ export class PPAPIOfflineClient {
     return ppapi;
   }
 
-  getCourse = (courseNumber: string): Course | undefined => this.cache.get(courseNumber);
+  getCourse(courseNumber: string): Course | undefined {
+    return this.cache.get(courseNumber);
+  }
 
-  getCoursesByDepartment = (
+  getCoursesByDepartment(
     department: string,
     predicate: (x: Course) => boolean = () => true,
-  ): Course[] =>
-    Array.from(this.cache.values())
+  ): Course[] {
+    return Array.from(this.cache.values())
       .filter((x) => x.id.startsWith(department))
       .filter(predicate);
+  }
 }
diff --git a/tools/degreeworks-scraper/src/components/Scraper.ts b/tools/degreeworks-scraper/src/components/Scraper.ts
index 6493e996..bef2a812 100644
--- a/tools/degreeworks-scraper/src/components/Scraper.ts
+++ b/tools/degreeworks-scraper/src/components/Scraper.ts
@@ -5,6 +5,8 @@ import type { Program } from "../types";
 
 import { AuditParser, DegreeworksClient } from ".";
 
+const JWT_HEADER_PREFIX_LENGTH = 7;
+
 export class Scraper {
   private ap!: AuditParser;
   private dw!: DegreeworksClient;
@@ -174,9 +176,8 @@ export class Scraper {
     };
   }
   static async new(authCookie: string): Promise<Scraper> {
-    const studentId = jwtDecode<JwtPayload>(authCookie.slice("Bearer+".length))?.sub;
-    if (!studentId || studentId.length !== 8)
-      throw new Error("Could not parse student ID from auth cookie.");
+    const studentId = jwtDecode<JwtPayload>(authCookie.slice(JWT_HEADER_PREFIX_LENGTH))?.sub;
+    if (studentId?.length !== 8) throw new Error("Could not parse student ID from auth cookie.");
     const headers = {
       "Content-Type": "application/json",
       Cookie: `X-AUTH-TOKEN=${authCookie}`,