From 635f842e3ff052d900ea61c081a88173071fa45f Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Sun, 10 Mar 2024 18:31:44 -0700 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20=E2=9C=A8=20fuzzy=20SaaS=20proof=20?= =?UTF-8?q?of=20concept?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/bronya.config.ts | 25 ++++++ apps/api/package.json | 1 + apps/api/src/global.d.ts | 7 ++ apps/api/src/routes/v1/rest/search/+config.ts | 11 +++ .../src/routes/v1/rest/search/+endpoint.ts | 36 ++++++++ apps/api/src/routes/v1/rest/search/schema.ts | 10 +++ libs/db/prisma/schema.prisma | 5 ++ pnpm-lock.yaml | 89 ++----------------- 8 files changed, 103 insertions(+), 81 deletions(-) create mode 100644 apps/api/src/routes/v1/rest/search/+config.ts create mode 100644 apps/api/src/routes/v1/rest/search/+endpoint.ts create mode 100644 apps/api/src/routes/v1/rest/search/schema.ts diff --git a/apps/api/bronya.config.ts b/apps/api/bronya.config.ts index 6059ef5c..8d2864d4 100644 --- a/apps/api/bronya.config.ts +++ b/apps/api/bronya.config.ts @@ -121,6 +121,7 @@ export const esbuildOptions: BuildOptions = { path: args.path, namespace, })); + build.onResolve({ filter: /virtual:search/ }, (args) => ({ path: args.path, namespace })); build.onLoad({ filter: /virtual:courses/, namespace }, async () => ({ contents: `export const courses = ${JSON.stringify( Object.fromEntries( @@ -133,6 +134,30 @@ export const esbuildOptions: BuildOptions = { Object.fromEntries((await prisma.instructor.findMany()).map((x) => [x.ucinetid, x])), )}`, })); + build.onLoad({ filter: /virtual:search/, namespace }, async () => { + const aliases = Object.fromEntries( + (await prisma.alias.findMany()).map((x) => [x.department, x.alias]), + ); + const courses = (await prisma.course.findMany()).map(normalizeCourse); + const instructors = await prisma.instructor.findMany(); + return { + contents: `export const haystack = ${JSON.stringify([ + ...courses.flatMap((x) => + [ + x.id, + x.title, + aliases[x.department] ? `${aliases[x.department]}${x.courseNumber}` : "", + ].filter((x) => x.length), + ), + ...instructors.flatMap((x) => [x.ucinetid, x.name]), + ])}; export const mapping = ${JSON.stringify([ + ...courses.flatMap((x) => + [x.id, x.id, aliases[x.department] ? x.id : ""].filter((x) => x.length), + ), + ...instructors.flatMap((x) => [x.ucinetid, x.ucinetid]), + ])}`, + }; + }); }, }, ], diff --git a/apps/api/package.json b/apps/api/package.json index 9304fd79..3a8ff0f1 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -25,6 +25,7 @@ "@graphql-tools/load-files": "7.0.0", "@graphql-tools/merge": "9.0.3", "@graphql-tools/utils": "10.1.0", + "@leeoniya/ufuzzy": "1.0.14", "@libs/db": "workspace:^", "@libs/lambda": "workspace:^", "@libs/uc-irvine-lib": "workspace:^", diff --git a/apps/api/src/global.d.ts b/apps/api/src/global.d.ts index ff85b418..66a952d2 100644 --- a/apps/api/src/global.d.ts +++ b/apps/api/src/global.d.ts @@ -19,3 +19,10 @@ declare module "virtual:instructors" { // eslint-disable-next-line @typescript-eslint/consistent-type-imports declare const instructors: Record; } +/** + * Virtual module for caching the haystack and mappings for fuzzy search during build time. + */ +declare module "virtual:search" { + declare const haystack: string[]; + declare const mapping: string[]; +} diff --git a/apps/api/src/routes/v1/rest/search/+config.ts b/apps/api/src/routes/v1/rest/search/+config.ts new file mode 100644 index 00000000..b50cd8bb --- /dev/null +++ b/apps/api/src/routes/v1/rest/search/+config.ts @@ -0,0 +1,11 @@ +import type { ApiPropsOverride } from "@bronya.js/api-construct"; + +import { esbuildOptions, constructs } from "../../../../../bronya.config"; + +export const overrides: ApiPropsOverride = { + esbuild: esbuildOptions, + constructs: { + functionPlugin: constructs.functionPlugin, + restApiProps: constructs.restApiProps, + }, +}; diff --git a/apps/api/src/routes/v1/rest/search/+endpoint.ts b/apps/api/src/routes/v1/rest/search/+endpoint.ts new file mode 100644 index 00000000..be8880c6 --- /dev/null +++ b/apps/api/src/routes/v1/rest/search/+endpoint.ts @@ -0,0 +1,36 @@ +import uFuzzy from "@leeoniya/ufuzzy"; +import { createHandler } from "@libs/lambda"; +import type { Course, Instructor } from "@peterportal-api/types"; +import { courses } from "virtual:courses"; +import { instructors } from "virtual:instructors"; +import { haystack, mapping } from "virtual:search"; + +import { QuerySchema } from "./schema"; + +const u = new uFuzzy({ intraMode: 1 /* IntraMode.SingleError */ }); + +export const GET = createHandler(async (event, context, res) => { + const headers = event.headers; + const requestId = context.awsRequestId; + const query = event.queryStringParameters ?? {}; + + const maybeParsed = QuerySchema.safeParse(query); + if (maybeParsed.success) { + const { data } = maybeParsed; + const keys = Array.from(new Set(u.search(haystack, data.q)[0]?.map((x) => mapping[x]))); + const results: Array = keys + .slice(data.offset, data.offset + data.limit) + .map((x) => courses[x] ?? instructors[x]); + return res.createOKResult( + { + count: keys.length, + results: results.filter((x) => + !data.resultType ? x : data.resultType === "course" ? "id" in x : "ucinetid" in x, + ), + }, + headers, + requestId, + ); + } + return res.createErrorResult(400, "Search query not provided", requestId); +}); diff --git a/apps/api/src/routes/v1/rest/search/schema.ts b/apps/api/src/routes/v1/rest/search/schema.ts new file mode 100644 index 00000000..fd56b2bd --- /dev/null +++ b/apps/api/src/routes/v1/rest/search/schema.ts @@ -0,0 +1,10 @@ +import { z } from "zod"; + +export const QuerySchema = z.object({ + q: z.string(), + resultType: z.enum(["course", "instructor"]).optional(), + limit: z.coerce.number().default(10), + offset: z.coerce.number().default(0), +}); + +export type Query = z.infer; diff --git a/libs/db/prisma/schema.prisma b/libs/db/prisma/schema.prisma index d648c063..114f11e5 100644 --- a/libs/db/prisma/schema.prisma +++ b/libs/db/prisma/schema.prisma @@ -46,6 +46,11 @@ enum WebsocSectionType { // Models +model Alias { + department String @id + alias String +} + model CalendarTerm { year String quarter Quarter diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 346fc602..47bcb711 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -101,6 +101,9 @@ importers: '@graphql-tools/utils': specifier: 10.1.0 version: 10.1.0(graphql@16.8.1) + '@leeoniya/ufuzzy': + specifier: 1.0.14 + version: 1.0.14 '@libs/db': specifier: workspace:^ version: link:../../libs/db @@ -2876,16 +2879,6 @@ packages: conventional-changelog-conventionalcommits: 7.0.2 dev: false - /@commitlint/config-validator@18.6.0: - resolution: {integrity: sha512-Ptfa865arNozlkjxrYG3qt6wT9AlhNUHeuDyKEZiTL/l0ftncFhK/KN0t/EAMV2tec+0Mwxo0FmhbESj/bI+1g==} - engines: {node: '>=v18'} - requiresBuild: true - dependencies: - '@commitlint/types': 18.6.0 - ajv: 8.12.0 - dev: false - optional: true - /@commitlint/config-validator@19.0.3: resolution: {integrity: sha512-2D3r4PKjoo59zBc2auodrSCaUnCSALCx54yveOFwwP/i2kfEAQrygwOleFWswLqK0UL/F9r07MFi5ev2ohyM4Q==} engines: {node: '>=v18'} @@ -2906,13 +2899,6 @@ packages: lodash.upperfirst: 4.3.1 dev: false - /@commitlint/execute-rule@18.4.4: - resolution: {integrity: sha512-a37Nd3bDQydtg9PCLLWM9ZC+GO7X5i4zJvrggJv5jBhaHsXeQ9ZWdO6ODYR+f0LxBXXNYK3geYXJrCWUCP8JEg==} - engines: {node: '>=v18'} - requiresBuild: true - dev: false - optional: true - /@commitlint/execute-rule@19.0.0: resolution: {integrity: sha512-mtsdpY1qyWgAO/iOK0L6gSGeR7GFcdW7tIjcNFxcWkfLDF5qVbPHKuGATFqRMsxcO8OUKNj0+3WOHB7EHm4Jdw==} engines: {node: '>=v18'} @@ -2944,28 +2930,6 @@ packages: '@commitlint/types': 19.0.3 dev: false - /@commitlint/load@18.6.0(@types/node@20.11.24)(typescript@5.3.3): - resolution: {integrity: sha512-RRssj7TmzT0bowoEKlgwg8uQ7ORXWkw7lYLsZZBMi9aInsJuGNLNWcMxJxRZbwxG3jkCidGUg85WmqJvRjsaDA==} - engines: {node: '>=v18'} - requiresBuild: true - dependencies: - '@commitlint/config-validator': 18.6.0 - '@commitlint/execute-rule': 18.4.4 - '@commitlint/resolve-extends': 18.6.0 - '@commitlint/types': 18.6.0 - chalk: 4.1.2 - cosmiconfig: 8.3.6(typescript@5.3.3) - cosmiconfig-typescript-loader: 5.0.0(@types/node@20.11.24)(cosmiconfig@8.3.6)(typescript@5.3.3) - lodash.isplainobject: 4.0.6 - lodash.merge: 4.6.2 - lodash.uniq: 4.5.0 - resolve-from: 5.0.0 - transitivePeerDependencies: - - '@types/node' - - typescript - dev: false - optional: true - /@commitlint/load@19.0.3(@types/node@20.11.24)(typescript@5.3.3): resolution: {integrity: sha512-18Tk/ZcDFRKIoKfEcl7kC+bYkEQ055iyKmGsYDoYWpKf6FUvBrP9bIWapuy/MB+kYiltmP9ITiUx6UXtqC9IRw==} engines: {node: '>=v18'} @@ -3009,20 +2973,6 @@ packages: minimist: 1.2.8 dev: false - /@commitlint/resolve-extends@18.6.0: - resolution: {integrity: sha512-k2Xp+Fxeggki2i90vGrbiLDMefPius3zGSTFFlRAPKce/SWLbZtI+uqE9Mne23mHO5lmcSV8z5m6ziiJwGpOcg==} - engines: {node: '>=v18'} - requiresBuild: true - dependencies: - '@commitlint/config-validator': 18.6.0 - '@commitlint/types': 18.6.0 - import-fresh: 3.3.0 - lodash.mergewith: 4.6.2 - resolve-from: 5.0.0 - resolve-global: 1.0.0 - dev: false - optional: true - /@commitlint/resolve-extends@19.0.3: resolution: {integrity: sha512-18BKmta8OC8+Ub+Q3QGM9l27VjQaXobloVXOrMvu8CpEwJYv62vC/t7Ka5kJnsW0tU9q1eMqJFZ/nN9T/cOaIA==} engines: {node: '>=v18'} @@ -3058,15 +3008,6 @@ packages: find-up: 7.0.0 dev: false - /@commitlint/types@18.6.0: - resolution: {integrity: sha512-oavoKLML/eJa2rJeyYSbyGAYzTxQ6voG5oeX3OrxpfrkRWhJfm4ACnhoRf5tgiybx2MZ+EVFqC1Lw3W8/uwpZA==} - engines: {node: '>=v18'} - requiresBuild: true - dependencies: - chalk: 4.1.2 - dev: false - optional: true - /@commitlint/types@19.0.3: resolution: {integrity: sha512-tpyc+7i6bPG9mvaBbtKUeghfyZSDgWquIDfMgqYtTbmZ9Y9VzEm2je9EYcQ0aoz5o7NvGS+rcDec93yO08MHYA==} engines: {node: '>=v18'} @@ -4486,6 +4427,10 @@ packages: '@jridgewell/resolve-uri': 3.1.0 '@jridgewell/sourcemap-codec': 1.4.14 + /@leeoniya/ufuzzy@1.0.14: + resolution: {integrity: sha512-/xF4baYuCQMo+L/fMSUrZnibcu0BquEGnbxfVPiZhs/NbJeKj4c/UmFpQzW9Us0w45ui/yYW3vyaqawhNYsTzA==} + dev: false + /@leichtgewicht/ip-codec@2.0.4: resolution: {integrity: sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A==} dev: false @@ -7373,7 +7318,7 @@ packages: longest: 2.0.1 word-wrap: 1.2.3 optionalDependencies: - '@commitlint/load': 18.6.0(@types/node@20.11.24)(typescript@5.3.3) + '@commitlint/load': 19.0.3(@types/node@20.11.24)(typescript@5.3.3) transitivePeerDependencies: - '@types/node' - typescript @@ -8781,15 +8726,6 @@ packages: ini: 4.1.1 dev: false - /global-dirs@0.1.1: - resolution: {integrity: sha512-NknMLn7F2J7aflwFOlGdNIuCDpN3VGoSoB+aap3KABFWbHVn1TCgFC+np23J8W2BiZbjfEw3BFBycSMv1AFblg==} - engines: {node: '>=4'} - requiresBuild: true - dependencies: - ini: 1.3.8 - dev: false - optional: true - /global-dirs@3.0.1: resolution: {integrity: sha512-NBcGGFbBA9s1VzD41QXDG+3++t9Mn5t1FpLdhESY6oKY4gYTFpX4wO3sqGUa0Srjtbfj3szX0RnemmrVRUdULA==} engines: {node: '>=10'} @@ -11911,15 +11847,6 @@ packages: resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} engines: {node: '>=8'} - /resolve-global@1.0.0: - resolution: {integrity: sha512-zFa12V4OLtT5XUX/Q4VLvTfBf+Ok0SPc1FNGM/z9ctUdiU618qwKpWnd0CHs3+RqROfyEg/DhuHbMWYqcgljEw==} - engines: {node: '>=8'} - requiresBuild: true - dependencies: - global-dirs: 0.1.1 - dev: false - optional: true - /resolve-pathname@3.0.0: resolution: {integrity: sha512-C7rARubxI8bXFNB/hqcp/4iUeIXJhJZvFPFPiSPRnhU5UPxzMFIl+2E6yY6c4k9giDJAhtV+enfA+G89N6Csng==} dev: false From 32d311064f71857a743ccfba33d1f4c87fe3563b Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Sun, 10 Mar 2024 21:22:50 -0700 Subject: [PATCH 2/3] =?UTF-8?q?chore:=20=F0=9F=94=A7=20slight=20enhancemen?= =?UTF-8?q?ts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/src/routes/v1/rest/search/+endpoint.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/apps/api/src/routes/v1/rest/search/+endpoint.ts b/apps/api/src/routes/v1/rest/search/+endpoint.ts index be8880c6..9438122e 100644 --- a/apps/api/src/routes/v1/rest/search/+endpoint.ts +++ b/apps/api/src/routes/v1/rest/search/+endpoint.ts @@ -19,18 +19,22 @@ export const GET = createHandler(async (event, context, res) => { const { data } = maybeParsed; const keys = Array.from(new Set(u.search(haystack, data.q)[0]?.map((x) => mapping[x]))); const results: Array = keys - .slice(data.offset, data.offset + data.limit) - .map((x) => courses[x] ?? instructors[x]); + .map((x) => courses[x] ?? instructors[x]) + .filter((x) => + !data.resultType ? x : data.resultType === "course" ? "id" in x : "ucinetid" in x, + ); return res.createOKResult( { - count: keys.length, - results: results.filter((x) => - !data.resultType ? x : data.resultType === "course" ? "id" in x : "ucinetid" in x, - ), + count: results.length, + results: results.slice(data.offset, data.offset + data.limit), }, headers, requestId, ); } - return res.createErrorResult(400, "Search query not provided", requestId); + return res.createErrorResult( + 400, + maybeParsed.error.issues.map((issue) => issue.message).join("; "), + requestId, + ); }); From 444004e78f546fc4dde8ff1be41e10eddf7bbbe9 Mon Sep 17 00:00:00 2001 From: Eddy Chen <89349085+ecxyzzy@users.noreply.github.com> Date: Sun, 10 Mar 2024 21:31:53 -0700 Subject: [PATCH 3/3] =?UTF-8?q?feat:=20=E2=9C=A8=20fixup=20schema,=20add?= =?UTF-8?q?=20gql=20support?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/src/routes/v1/graphql/resolvers.ts | 6 ++++++ .../src/routes/v1/graphql/schema/search.graphql | 14 ++++++++++++++ apps/api/src/routes/v1/rest/search/+endpoint.ts | 2 +- apps/api/src/routes/v1/rest/search/schema.ts | 6 +++--- 4 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 apps/api/src/routes/v1/graphql/schema/search.graphql diff --git a/apps/api/src/routes/v1/graphql/resolvers.ts b/apps/api/src/routes/v1/graphql/resolvers.ts index 52e56058..c261d6f1 100644 --- a/apps/api/src/routes/v1/graphql/resolvers.ts +++ b/apps/api/src/routes/v1/graphql/resolvers.ts @@ -23,6 +23,12 @@ export const resolvers: ApolloServerOptions["resolvers"] = { instructors: proxyRestApi("/v1/rest/instructors"), allInstructors: proxyRestApi("/v1/rest/instructors/all"), larc: proxyRestApi("/v1/rest/larc"), + searchCourses: proxyRestApi("/v1/rest/search", { + argsTransform: (args) => ({ ...args, resultType: "course" }), + }), + searchInstructors: proxyRestApi("/v1/rest/search", { + argsTransform: (args) => ({ ...args, resultType: "instructors" }), + }), websoc: proxyRestApi("/v1/rest/websoc", { argsTransform: geTransform }), depts: proxyRestApi("/v1/rest/websoc/depts"), terms: proxyRestApi("/v1/rest/websoc/terms"), diff --git a/apps/api/src/routes/v1/graphql/schema/search.graphql b/apps/api/src/routes/v1/graphql/schema/search.graphql new file mode 100644 index 00000000..e9d66a20 --- /dev/null +++ b/apps/api/src/routes/v1/graphql/schema/search.graphql @@ -0,0 +1,14 @@ +type CourseSearchResult { + count: Int! + results: [Course!]! +} + +type InstructorSearchResult { + count: Int! + results: [Instructor!]! +} + +extend type Query { + searchCourses(query: String!, limit: Int, offset: Int): CourseSearchResult! + searchInstructors(query: String!, limit: Int, offset: Int): InstructorSearchResult! +} diff --git a/apps/api/src/routes/v1/rest/search/+endpoint.ts b/apps/api/src/routes/v1/rest/search/+endpoint.ts index 9438122e..1ba3b374 100644 --- a/apps/api/src/routes/v1/rest/search/+endpoint.ts +++ b/apps/api/src/routes/v1/rest/search/+endpoint.ts @@ -17,7 +17,7 @@ export const GET = createHandler(async (event, context, res) => { const maybeParsed = QuerySchema.safeParse(query); if (maybeParsed.success) { const { data } = maybeParsed; - const keys = Array.from(new Set(u.search(haystack, data.q)[0]?.map((x) => mapping[x]))); + const keys = Array.from(new Set(u.search(haystack, data.query)[0]?.map((x) => mapping[x]))); const results: Array = keys .map((x) => courses[x] ?? instructors[x]) .filter((x) => diff --git a/apps/api/src/routes/v1/rest/search/schema.ts b/apps/api/src/routes/v1/rest/search/schema.ts index fd56b2bd..6b52bf31 100644 --- a/apps/api/src/routes/v1/rest/search/schema.ts +++ b/apps/api/src/routes/v1/rest/search/schema.ts @@ -1,10 +1,10 @@ import { z } from "zod"; export const QuerySchema = z.object({ - q: z.string(), + query: z.string(), resultType: z.enum(["course", "instructor"]).optional(), - limit: z.coerce.number().default(10), - offset: z.coerce.number().default(0), + limit: z.coerce.number().int().default(10), + offset: z.coerce.number().int().default(0), }); export type Query = z.infer;