From 3b360895daa2cfdd2c0791fa4cda7c8f6a0be2ce Mon Sep 17 00:00:00 2001 From: Alexis Kaufman Date: Mon, 27 Mar 2023 19:06:24 -0500 Subject: [PATCH 01/12] SKEDGE-16 #comment Patch for single professor with middle name for courses which do not exist on Nebula API --- src/data/builder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data/builder.ts b/src/data/builder.ts index a00f8ab..50808ae 100644 --- a/src/data/builder.ts +++ b/src/data/builder.ts @@ -34,7 +34,7 @@ export async function buildProfessorProfiles (payload: ShowCourseTabPayload) { return fetchNebulaProfessor({ firstName: firstName, lastName: lastName}) })) const nebulaSections = await Promise.all(nebulaProfessors.map(professor => { - if (professor?._id === undefined) return null + if (professor?._id === undefined || !nebulaCourse) return null return fetchNebulaSections({ courseReference: nebulaCourse._id, professorReference: professor._id }) })) const rmps = await requestProfessorsFromRmp({ professorNames: professors.map(prof => prof.split(' ')[0] + " " + prof.split(' ').at(-1)), schoolId: SCHOOL_ID }) From 76e542f75bfc30a81cc0893ff81ad4aa0151fcb7 Mon Sep 17 00:00:00 2001 From: Alexis Kaufman Date: Mon, 27 Mar 2023 19:24:35 -0500 Subject: [PATCH 02/12] SKEDGE-14 #comment convert to async/await instead of .then() logics, organizing awaits --- src/data/fetchFromRmp.ts | 63 ++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/src/data/fetchFromRmp.ts b/src/data/fetchFromRmp.ts index ffa848b..39b6c80 100644 --- a/src/data/fetchFromRmp.ts +++ b/src/data/fetchFromRmp.ts @@ -46,48 +46,49 @@ function getGraphQlUrlProps(professorIds: string[]) { return graphQlUrlProps } -function fetchWithGraphQl(graphQlUrlProps: any[], resolve) { +async function fetchWithGraphQl(graphQlUrlProps: any[]) { const graphqlUrl = "https://www.ratemyprofessors.com/graphql"; - - Promise.all(graphQlUrlProps.map(u=>fetch(graphqlUrl, u))) - .then(responses => Promise.all(responses.map(res => res.json()))) - .then(ratings => { - for (let i = 0; i < ratings.length; i++) { - if (ratings[i] != null && ratings[i].hasOwnProperty("data") && ratings[i]["data"].hasOwnProperty("node")) { - ratings[i] = ratings[i]["data"]["node"]; - } + let ratings: any[]; + try { + let responses = await Promise.all(graphQlUrlProps.map(u=>fetch(graphqlUrl, u))); + ratings = await Promise.all(responses.map(res => res.json())); + for (let i = 0; i < ratings.length; i++) { + if (ratings[i] != null && ratings[i].hasOwnProperty("data") && ratings[i]["data"].hasOwnProperty("node")) { + ratings[i] = ratings[i]["data"]["node"]; } - console.log(ratings) - resolve(ratings) - }) + } + + } + catch (err) { + console.error(err); + } + return ratings; } export interface RmpRequest { professorNames: string[], schoolId: string } -export function requestProfessorsFromRmp(request: RmpRequest): Promise { - return new Promise((resolve, reject) => { +export async function requestProfessorsFromRmp(request: RmpRequest): Promise { - // make a list of urls for promises - const professorUrls = getProfessorUrls(request.professorNames, request.schoolId) + // make a list of urls for promises + const professorUrls = getProfessorUrls(request.professorNames, request.schoolId) + let professors; + // fetch professor ids from each url + try { + let responses = await Promise.all(professorUrls.map(u=>fetch(u))); + let texts = await Promise.all(responses.map(res => res.text())); + const professorIds = getProfessorIds(texts, request.professorNames) - // fetch professor ids from each url - Promise.all(professorUrls.map(u=>fetch(u))) - .then(responses => Promise.all(responses.map(res => res.text()))) - .then(texts => { - const professorIds = getProfessorIds(texts, request.professorNames) + // create fetch objects for each professor id + const graphQlUrlProps = getGraphQlUrlProps(professorIds) - // create fetch objects for each professor id - const graphQlUrlProps = getGraphQlUrlProps(professorIds) - - // fetch professor info by id with graphQL - fetchWithGraphQl(graphQlUrlProps, resolve) - } - ).catch(error => { - reject(error); - }); - }) + // fetch professor info by id with graphQL + professors = await fetchWithGraphQl(graphQlUrlProps); + } catch (error) { + console.error(error); + }; + return professors; } interface RMPInterface { From 2fd774031e89887838ea1bc90234436dc1ca8e90 Mon Sep 17 00:00:00 2001 From: Alexis Kaufman Date: Mon, 27 Mar 2023 19:55:22 -0500 Subject: [PATCH 03/12] SKEDGE-14 logic to patch CS4348 for Scott Dollinger. If there are multiple listings for a professor on RMP, use the one with the most ratings. --- src/data/fetchFromRmp.ts | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/data/fetchFromRmp.ts b/src/data/fetchFromRmp.ts index 39b6c80..a4b5fe8 100644 --- a/src/data/fetchFromRmp.ts +++ b/src/data/fetchFromRmp.ts @@ -15,16 +15,22 @@ function getProfessorIds(texts: string[], professorNames: string[]): string[] { const professorIds = [] const lowerCaseProfessorNames = professorNames.map(name => name.toLowerCase()) texts.forEach(text => { - let matched = false; - const regex = /"legacyId":(\d+).*?"firstName":"(.*?)","lastName":"(.*?)"/g; - for (const match of text.matchAll(regex)) { - console.log(match[2].split(' ')[0].toLowerCase() + " " + match[3].toLowerCase()) - if (lowerCaseProfessorNames.includes(match[2].split(' ')[0].toLowerCase() + " " + match[3].toLowerCase())) { - professorIds.push(match[1]); - matched = true; + let pendingMatch = null; + const regex = /"legacyId":(\d+).*?"numRatings":(\d+).*?"firstName":"(.*?)","lastName":"(.*?)"/g; + let allMatches: string[] = text.match(regex); + let highestNumRatings = 0; + + for (const fullMatch of allMatches) { + for (const match of fullMatch.matchAll(regex)) { + console.log(match[3].split(' ')[0].toLowerCase() + " " + match[4].toLowerCase() + " ") + let numRatings = parseInt(match[2]); + if (lowerCaseProfessorNames.includes(match[3].split(' ')[0].toLowerCase() + " " + match[4].toLowerCase()) && numRatings >= highestNumRatings) { + pendingMatch = match[1]; + } } } - if (!matched) professorIds.push(null) + + professorIds.push(pendingMatch); }) return professorIds } From b5fcd34d913a09083c847fd5d1e4e16b64425fb3 Mon Sep 17 00:00:00 2001 From: Alexis Kaufman Date: Tue, 28 Mar 2023 18:10:00 -0500 Subject: [PATCH 04/12] SKEDGE-14 #comment add function to group properties, use this function for the RMP fetches to avoid 504 error. #comment address requests from Adam Szumski #comment add new interfaces for the Professor Rating object --- src/data/config.ts | 41 ++++++++++++++++++++++++++++++++++++++ src/data/fetchFromRmp.ts | 43 +++++++++++++--------------------------- src/data/interfaces.ts | 35 +++++++++++++++++++++++++++++++- 3 files changed, 89 insertions(+), 30 deletions(-) diff --git a/src/data/config.ts b/src/data/config.ts index e38d06d..0a000b4 100644 --- a/src/data/config.ts +++ b/src/data/config.ts @@ -1,3 +1,4 @@ +import type { RMPRatingInterface } from "~data/interfaces"; export const HEADERS = { "Authorization": "Basic dGVzdDp0ZXN0", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", @@ -29,3 +30,43 @@ function unRegister(key: string) { return newVar; } +function groupProps(props: any[], maxGroupSize: number): any[] { + let groupedProps = [[]]; + + for (const [key, value] of Object.entries(props)) { + let lastGroup = groupedProps[groupedProps.length - 1]; + if (lastGroup.length < maxGroupSize) { + lastGroup.push(value); + } + else { + groupedProps.push([value]); + } + }; + return groupedProps; +} + +export async function RMP_GRAPHQL_URL(graphQlUrlProps: any[]) { + const graphqlUrl = "https://www.ratemyprofessors.com/graphql"; + try { + let groupedProps = groupProps(graphQlUrlProps, 3); + let responses: any[] = []; + for (let i = 0; i < groupedProps.length; i++) { + // Our fetches need to be grouped in groups of 3 to avoid 504 errors. + let smallGroup = await Promise.all(groupedProps[i].map(u=>fetch(graphqlUrl, u))) + for (let j = 0; j < smallGroup.length; j++) { + responses.push(smallGroup[j]); + } + } + let ratings: RMPRatingInterface [] = await Promise.all(responses.map(res => res.json())); + for (let i = 0; i < ratings.length; i++) { + if (ratings[i] != null && ratings[i].hasOwnProperty("data") && ratings[i]["data"].hasOwnProperty("node")) { + ratings[i] = ratings[i]["data"]["node"]; + } + } + return ratings; + } + catch (err) { + console.error(err); + return []; + } +} \ No newline at end of file diff --git a/src/data/fetchFromRmp.ts b/src/data/fetchFromRmp.ts index a4b5fe8..62d4941 100644 --- a/src/data/fetchFromRmp.ts +++ b/src/data/fetchFromRmp.ts @@ -1,4 +1,5 @@ -import { HEADERS, PROFESSOR_QUERY } from "~data/config"; +import { HEADERS, PROFESSOR_QUERY, RMP_GRAPHQL_URL } from "~data/config"; +import type { RMPRatingInterface } from "~data/interfaces"; function getProfessorUrl(professorName: string, schoolId: string): string { return `https://www.ratemyprofessors.com/search/teachers?query=${encodeURIComponent(professorName)}&sid=${btoa(`School-${schoolId}`)}` @@ -20,12 +21,14 @@ function getProfessorIds(texts: string[], professorNames: string[]): string[] { let allMatches: string[] = text.match(regex); let highestNumRatings = 0; - for (const fullMatch of allMatches) { - for (const match of fullMatch.matchAll(regex)) { - console.log(match[3].split(' ')[0].toLowerCase() + " " + match[4].toLowerCase() + " ") - let numRatings = parseInt(match[2]); - if (lowerCaseProfessorNames.includes(match[3].split(' ')[0].toLowerCase() + " " + match[4].toLowerCase()) && numRatings >= highestNumRatings) { - pendingMatch = match[1]; + if (allMatches) { + for (const fullMatch of allMatches) { + for (const match of fullMatch.matchAll(regex)) { + console.log(match[3].split(' ')[0].toLowerCase() + " " + match[4].toLowerCase() + " ") + let numRatings = parseInt(match[2]); + if (lowerCaseProfessorNames.includes(match[3].split(' ')[0].toLowerCase() + " " + match[4].toLowerCase()) && numRatings >= highestNumRatings) { + pendingMatch = match[1]; + } } } } @@ -52,25 +55,6 @@ function getGraphQlUrlProps(professorIds: string[]) { return graphQlUrlProps } -async function fetchWithGraphQl(graphQlUrlProps: any[]) { - const graphqlUrl = "https://www.ratemyprofessors.com/graphql"; - let ratings: any[]; - try { - let responses = await Promise.all(graphQlUrlProps.map(u=>fetch(graphqlUrl, u))); - ratings = await Promise.all(responses.map(res => res.json())); - for (let i = 0; i < ratings.length; i++) { - if (ratings[i] != null && ratings[i].hasOwnProperty("data") && ratings[i]["data"].hasOwnProperty("node")) { - ratings[i] = ratings[i]["data"]["node"]; - } - } - - } - catch (err) { - console.error(err); - } - return ratings; -} - export interface RmpRequest { professorNames: string[], schoolId: string @@ -79,7 +63,7 @@ export async function requestProfessorsFromRmp(request: RmpRequest): Promisefetch(u))); @@ -90,11 +74,12 @@ export async function requestProfessorsFromRmp(request: RmpRequest): Promise Date: Tue, 28 Mar 2023 20:00:37 -0500 Subject: [PATCH 05/12] move functions, add comments --- src/data/config.ts | 42 +------------------------------ src/data/fetchFromRmp.ts | 53 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 42 deletions(-) diff --git a/src/data/config.ts b/src/data/config.ts index 0a000b4..aa1a3fd 100644 --- a/src/data/config.ts +++ b/src/data/config.ts @@ -19,6 +19,7 @@ export const NEBULA_FETCH_OPTIONS = { } export const SCHOOL_ID = "1273" +export const RMP_GRAPHQL_URL = "https://www.ratemyprofessors.com/graphql"; function unRegister(key: string) { let newVar = ""; @@ -28,45 +29,4 @@ function unRegister(key: string) { newVar = newVar.concat(String.fromCharCode(a)); } return newVar; -} - -function groupProps(props: any[], maxGroupSize: number): any[] { - let groupedProps = [[]]; - - for (const [key, value] of Object.entries(props)) { - let lastGroup = groupedProps[groupedProps.length - 1]; - if (lastGroup.length < maxGroupSize) { - lastGroup.push(value); - } - else { - groupedProps.push([value]); - } - }; - return groupedProps; -} - -export async function RMP_GRAPHQL_URL(graphQlUrlProps: any[]) { - const graphqlUrl = "https://www.ratemyprofessors.com/graphql"; - try { - let groupedProps = groupProps(graphQlUrlProps, 3); - let responses: any[] = []; - for (let i = 0; i < groupedProps.length; i++) { - // Our fetches need to be grouped in groups of 3 to avoid 504 errors. - let smallGroup = await Promise.all(groupedProps[i].map(u=>fetch(graphqlUrl, u))) - for (let j = 0; j < smallGroup.length; j++) { - responses.push(smallGroup[j]); - } - } - let ratings: RMPRatingInterface [] = await Promise.all(responses.map(res => res.json())); - for (let i = 0; i < ratings.length; i++) { - if (ratings[i] != null && ratings[i].hasOwnProperty("data") && ratings[i]["data"].hasOwnProperty("node")) { - ratings[i] = ratings[i]["data"]["node"]; - } - } - return ratings; - } - catch (err) { - console.error(err); - return []; - } } \ No newline at end of file diff --git a/src/data/fetchFromRmp.ts b/src/data/fetchFromRmp.ts index 62d4941..48fb7f7 100644 --- a/src/data/fetchFromRmp.ts +++ b/src/data/fetchFromRmp.ts @@ -55,6 +55,57 @@ function getGraphQlUrlProps(professorIds: string[]) { return graphQlUrlProps } +// Function to group a list of items into groups of a certain size. +// For example, groupProps([1,2,3,4,5,6,7,8,9], 3) returns [[1,2,3], [4,5,6], [7,8,9]] +function groupProps(props: any[], maxGroupSize: number): any[] { + let groupedProps = [[]]; + + // Iterate all entries in the props list. (Could be an array or an Object) + for (const [key, value] of Object.entries(props)) { + let lastGroup: any[] = groupedProps[groupedProps.length - 1]; + if (lastGroup.length < maxGroupSize) { + // If there are less than 3 items in the last group, add the current item to the last group. + lastGroup.push(value); + } + else { + // Otherwise, make a new group and add the current item to it. + groupedProps.push([value]); + } + }; + return groupedProps; + } + + export async function fetchWithGraphQl(graphQlUrlProps: any[]) { + try { + let groupedProps = groupProps(graphQlUrlProps, 3); // Group our fetches into groups of 3. + let responses: any[] = []; + + // Here, we iterate through each group of fetches, one at a time. + for (let i = 0; i < groupedProps.length; i++) { + // Wait for all fetches in the current group to finish. + let smallGroup = await Promise.all(groupedProps[i].map(u=>fetch(RMP_GRAPHQL_URL, u))); + + // Now that we have all the responses, we can iterate through them and add them to the responses array. + for (let j = 0; j < smallGroup.length; j++) { + responses.push(smallGroup[j]); + } + } + + // We now have all the responses. So, we consider all the responses, and collect the ratings. + let ratings: RMPRatingInterface [] = await Promise.all(responses.map(res => res.json())); + for (let i = 0; i < ratings.length; i++) { + if (ratings[i] != null && ratings[i].hasOwnProperty("data") && ratings[i]["data"].hasOwnProperty("node")) { + ratings[i] = ratings[i]["data"]["node"]; + } + } + return ratings; + } + catch (err) { + console.error(err); + return []; + } + } + export interface RmpRequest { professorNames: string[], schoolId: string @@ -74,7 +125,7 @@ export async function requestProfessorsFromRmp(request: RmpRequest): Promise Date: Tue, 28 Mar 2023 20:49:43 -0500 Subject: [PATCH 06/12] add reportError function, to be modified later; use fetchRetry for initial fetches of professors from RMP --- src/data/fetchFromRmp.ts | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/data/fetchFromRmp.ts b/src/data/fetchFromRmp.ts index 48fb7f7..afc8a6c 100644 --- a/src/data/fetchFromRmp.ts +++ b/src/data/fetchFromRmp.ts @@ -1,6 +1,10 @@ import { HEADERS, PROFESSOR_QUERY, RMP_GRAPHQL_URL } from "~data/config"; import type { RMPRatingInterface } from "~data/interfaces"; +function reportError(context, err) { + console.error("Error in " + context + ": " + err); +} + function getProfessorUrl(professorName: string, schoolId: string): string { return `https://www.ratemyprofessors.com/search/teachers?query=${encodeURIComponent(professorName)}&sid=${btoa(`School-${schoolId}`)}` } @@ -55,6 +59,21 @@ function getGraphQlUrlProps(professorIds: string[]) { return graphQlUrlProps } +function wait(delay){ + return new Promise((resolve) => setTimeout(resolve, delay)); +} + +function fetchRetry(url: string, delay: number, tries: number, fetchOptions) { + function onError(err){ + let triesLeft: number = tries - 1; + if(!triesLeft){ + throw err; + } + return wait(delay).then(() => fetchRetry(url, delay, triesLeft, fetchOptions)); + } + return fetch(url,fetchOptions).catch(onError); +} + // Function to group a list of items into groups of a certain size. // For example, groupProps([1,2,3,4,5,6,7,8,9], 3) returns [[1,2,3], [4,5,6], [7,8,9]] function groupProps(props: any[], maxGroupSize: number): any[] { @@ -101,7 +120,7 @@ function groupProps(props: any[], maxGroupSize: number): any[] { return ratings; } catch (err) { - console.error(err); + reportError("fetchWithGraphQl",err); return []; } } @@ -117,7 +136,15 @@ export async function requestProfessorsFromRmp(request: RmpRequest): Promisefetch(u))); + let responses = await Promise.all(professorUrls.map(u=>(fetch(u)))); + for (const [key, value] of Object.entries(responses)) { + let notOk = value?.status !== 200; + if (notOk && value && value.url) { + reportError("requestProfessorsFromRmp", "Status not OK for fetch request."); + responses[key] = await fetchRetry(value?.url, 200, 3, {}); + } + }; + // let responses = await Promise.all(professorUrls.map(u=>fetchRetry(u, 500, 3, {}))); let texts = await Promise.all(responses.map(res => res.text())); const professorIds = getProfessorIds(texts, request.professorNames) @@ -128,7 +155,7 @@ export async function requestProfessorsFromRmp(request: RmpRequest): Promise Date: Tue, 28 Mar 2023 20:57:46 -0500 Subject: [PATCH 07/12] SKEDGE-14 #comment modularize retry logic inside validateResponses function, instead of grouping fetches. (This way seems shorter and safer) --- src/data/fetchFromRmp.ts | 44 +++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/data/fetchFromRmp.ts b/src/data/fetchFromRmp.ts index afc8a6c..15a7d96 100644 --- a/src/data/fetchFromRmp.ts +++ b/src/data/fetchFromRmp.ts @@ -74,6 +74,23 @@ function fetchRetry(url: string, delay: number, tries: number, fetchOptions) { return fetch(url,fetchOptions).catch(onError); } +async function validateResponses(responses: any[]) { + for (const [key, value] of Object.entries(responses)) { + let notOk = value?.status !== 200; + if (notOk && value && value.url) { + let details = { + status: value.status, + statusText: value.statusText, + redirected: value.redirected, + url: value.url + } + reportError("validateResponses", "Status not OK for fetch request. Details are: "+JSON.stringify(details)); + responses[key] = await fetchRetry(value?.url, 200, 3, {}); + } + }; + return responses; +} + // Function to group a list of items into groups of a certain size. // For example, groupProps([1,2,3,4,5,6,7,8,9], 3) returns [[1,2,3], [4,5,6], [7,8,9]] function groupProps(props: any[], maxGroupSize: number): any[] { @@ -96,20 +113,7 @@ function groupProps(props: any[], maxGroupSize: number): any[] { export async function fetchWithGraphQl(graphQlUrlProps: any[]) { try { - let groupedProps = groupProps(graphQlUrlProps, 3); // Group our fetches into groups of 3. - let responses: any[] = []; - - // Here, we iterate through each group of fetches, one at a time. - for (let i = 0; i < groupedProps.length; i++) { - // Wait for all fetches in the current group to finish. - let smallGroup = await Promise.all(groupedProps[i].map(u=>fetch(RMP_GRAPHQL_URL, u))); - - // Now that we have all the responses, we can iterate through them and add them to the responses array. - for (let j = 0; j < smallGroup.length; j++) { - responses.push(smallGroup[j]); - } - } - + let responses = await validateResponses(await Promise.all(graphQlUrlProps.map(u=>fetch(RMP_GRAPHQL_URL, u)))); // We now have all the responses. So, we consider all the responses, and collect the ratings. let ratings: RMPRatingInterface [] = await Promise.all(responses.map(res => res.json())); for (let i = 0; i < ratings.length; i++) { @@ -136,14 +140,8 @@ export async function requestProfessorsFromRmp(request: RmpRequest): Promise(fetch(u)))); - for (const [key, value] of Object.entries(responses)) { - let notOk = value?.status !== 200; - if (notOk && value && value.url) { - reportError("requestProfessorsFromRmp", "Status not OK for fetch request."); - responses[key] = await fetchRetry(value?.url, 200, 3, {}); - } - }; + let responses = await validateResponses(await Promise.all(professorUrls.map(u=>(fetch(u))))); + // let responses = await Promise.all(professorUrls.map(u=>fetchRetry(u, 500, 3, {}))); let texts = await Promise.all(responses.map(res => res.text())); const professorIds = getProfessorIds(texts, request.professorNames) @@ -155,7 +153,7 @@ export async function requestProfessorsFromRmp(request: RmpRequest): Promise Date: Wed, 29 Mar 2023 12:23:01 -0500 Subject: [PATCH 08/12] SKEDGE-14 #comment remove groupProps --- src/data/fetchFromRmp.ts | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/src/data/fetchFromRmp.ts b/src/data/fetchFromRmp.ts index 15a7d96..5fe74d1 100644 --- a/src/data/fetchFromRmp.ts +++ b/src/data/fetchFromRmp.ts @@ -91,27 +91,7 @@ async function validateResponses(responses: any[]) { return responses; } -// Function to group a list of items into groups of a certain size. -// For example, groupProps([1,2,3,4,5,6,7,8,9], 3) returns [[1,2,3], [4,5,6], [7,8,9]] -function groupProps(props: any[], maxGroupSize: number): any[] { - let groupedProps = [[]]; - - // Iterate all entries in the props list. (Could be an array or an Object) - for (const [key, value] of Object.entries(props)) { - let lastGroup: any[] = groupedProps[groupedProps.length - 1]; - if (lastGroup.length < maxGroupSize) { - // If there are less than 3 items in the last group, add the current item to the last group. - lastGroup.push(value); - } - else { - // Otherwise, make a new group and add the current item to it. - groupedProps.push([value]); - } - }; - return groupedProps; - } - - export async function fetchWithGraphQl(graphQlUrlProps: any[]) { +export async function fetchWithGraphQl(graphQlUrlProps: any[]) { try { let responses = await validateResponses(await Promise.all(graphQlUrlProps.map(u=>fetch(RMP_GRAPHQL_URL, u)))); // We now have all the responses. So, we consider all the responses, and collect the ratings. @@ -127,7 +107,7 @@ function groupProps(props: any[], maxGroupSize: number): any[] { reportError("fetchWithGraphQl",err); return []; } - } +} export interface RmpRequest { professorNames: string[], From 517598e128edc8ea2765d165a5d0bd448ead7894 Mon Sep 17 00:00:00 2001 From: Alexis Kaufman Date: Fri, 31 Mar 2023 18:27:18 -0500 Subject: [PATCH 09/12] SKEDGE-14 #comment use fetchOptions if needed to re-fetch rmp graphql data --- src/data/fetchFromRmp.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/data/fetchFromRmp.ts b/src/data/fetchFromRmp.ts index 5fe74d1..c2df6de 100644 --- a/src/data/fetchFromRmp.ts +++ b/src/data/fetchFromRmp.ts @@ -74,7 +74,8 @@ function fetchRetry(url: string, delay: number, tries: number, fetchOptions) { return fetch(url,fetchOptions).catch(onError); } -async function validateResponses(responses: any[]) { +// If using orderedFetchOpts, make sure that it is an array and that the index of the fetch options corresponds to the index of the response in the responses array. +async function validateResponses(responses: any[], orderedFetchOpts: any[]) { for (const [key, value] of Object.entries(responses)) { let notOk = value?.status !== 200; if (notOk && value && value.url) { @@ -85,7 +86,8 @@ async function validateResponses(responses: any[]) { url: value.url } reportError("validateResponses", "Status not OK for fetch request. Details are: "+JSON.stringify(details)); - responses[key] = await fetchRetry(value?.url, 200, 3, {}); + let fetchOptions = orderedFetchOpts[key] || {}; // If we don't have fetch options, we just use an empty object. + responses[key] = await fetchRetry(value?.url, 200, 3, fetchOptions); } }; return responses; @@ -93,7 +95,7 @@ async function validateResponses(responses: any[]) { export async function fetchWithGraphQl(graphQlUrlProps: any[]) { try { - let responses = await validateResponses(await Promise.all(graphQlUrlProps.map(u=>fetch(RMP_GRAPHQL_URL, u)))); + let responses = await validateResponses(await Promise.all(graphQlUrlProps.map(u=>fetch(RMP_GRAPHQL_URL, u))),graphQlUrlProps); // We now have all the responses. So, we consider all the responses, and collect the ratings. let ratings: RMPRatingInterface [] = await Promise.all(responses.map(res => res.json())); for (let i = 0; i < ratings.length; i++) { @@ -120,9 +122,8 @@ export async function requestProfessorsFromRmp(request: RmpRequest): Promise(fetch(u))))); + let responses = await validateResponses(await Promise.all(professorUrls.map(u=>(fetch(u)))), []); - // let responses = await Promise.all(professorUrls.map(u=>fetchRetry(u, 500, 3, {}))); let texts = await Promise.all(responses.map(res => res.text())); const professorIds = getProfessorIds(texts, request.professorNames) From f96555c3388aead57425fb12f1b51d3795c5f5f1 Mon Sep 17 00:00:00 2001 From: Alexis Kaufman Date: Thu, 8 Jun 2023 19:12:05 -0500 Subject: [PATCH 10/12] Fix edge case resulting from missing grade distribution. --- src/data/builder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data/builder.ts b/src/data/builder.ts index a00f8ab..b184317 100644 --- a/src/data/builder.ts +++ b/src/data/builder.ts @@ -43,7 +43,7 @@ export async function buildProfessorProfiles (payload: ShowCourseTabPayload) { const sectionsWithGrades = [] for (let j = 0; j < nebulaSections[i]?.length; j++) { const section = nebulaSections[i][j]; - if (section.grade_distribution.length !== 0 && !compareArrays(section.grade_distribution, Array(14).fill(0))) + if (section.grade_distribution && section.grade_distribution.length !== 0 && !compareArrays(section.grade_distribution, Array(14).fill(0))) sectionsWithGrades.push(section) } professorProfiles.push({ From abafc61cbcf9052be07a5a0f9f2f8f3f68a38529 Mon Sep 17 00:00:00 2001 From: Alexis Kaufman Date: Thu, 8 Jun 2023 19:21:43 -0500 Subject: [PATCH 11/12] fix RMP fetch link; Our logic was wrong, they changed the way that their routing works. --- src/data/fetchFromRmp.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data/fetchFromRmp.ts b/src/data/fetchFromRmp.ts index ffa848b..0fe4e87 100644 --- a/src/data/fetchFromRmp.ts +++ b/src/data/fetchFromRmp.ts @@ -1,7 +1,7 @@ import { HEADERS, PROFESSOR_QUERY } from "~data/config"; function getProfessorUrl(professorName: string, schoolId: string): string { - return `https://www.ratemyprofessors.com/search/teachers?query=${encodeURIComponent(professorName)}&sid=${btoa(`School-${schoolId}`)}` + return `https://www.ratemyprofessors.com/search/professors/${schoolId}?q=${encodeURIComponent(professorName)}}` } function getProfessorUrls(professorNames: string[], schoolId: string): string[] { const professorUrls = [] From 4127c573b6ada5d967145db3ab5100fc363f9eb0 Mon Sep 17 00:00:00 2001 From: Tyler Hill Date: Sat, 27 Jan 2024 00:01:29 -0600 Subject: [PATCH 12/12] Lint fixes --- src/data/config.ts | 1 - src/data/fetchFromRmp.ts | 32 ++++++++++++++++---------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/data/config.ts b/src/data/config.ts index 22fd429..643e151 100644 --- a/src/data/config.ts +++ b/src/data/config.ts @@ -1,4 +1,3 @@ -import type { RMPRatingInterface } from '~data/interfaces'; export const HEADERS = { Authorization: 'Basic dGVzdDp0ZXN0', 'User-Agent': diff --git a/src/data/fetchFromRmp.ts b/src/data/fetchFromRmp.ts index 473486b..a6d6bbc 100644 --- a/src/data/fetchFromRmp.ts +++ b/src/data/fetchFromRmp.ts @@ -29,8 +29,8 @@ function getProfessorIds(texts: string[], professorNames: string[]): string[] { let pendingMatch = null; const regex = /"legacyId":(\d+).*?"numRatings":(\d+).*?"firstName":"(.*?)","lastName":"(.*?)"/g; - let allMatches: string[] = text.match(regex); - let highestNumRatings = 0; + const allMatches: string[] = text.match(regex); + const highestNumRatings = 0; if (allMatches) { for (const fullMatch of allMatches) { @@ -41,7 +41,7 @@ function getProfessorIds(texts: string[], professorNames: string[]): string[] { match[4].toLowerCase() + ' ', ); - let numRatings = parseInt(match[2]); + const numRatings = parseInt(match[2]); if ( lowerCaseProfessorNames.includes( match[3].split(' ')[0].toLowerCase() + @@ -86,7 +86,7 @@ function wait(delay) { function fetchRetry(url: string, delay: number, tries: number, fetchOptions) { function onError(err) { - let triesLeft: number = tries - 1; + const triesLeft: number = tries - 1; if (!triesLeft) { throw err; } @@ -98,11 +98,11 @@ function fetchRetry(url: string, delay: number, tries: number, fetchOptions) { } // If using orderedFetchOpts, make sure that it is an array and that the index of the fetch options corresponds to the index of the response in the responses array. -async function validateResponses(responses: any[], orderedFetchOpts: any[]) { +async function validateResponses(responses, orderedFetchOpts) { for (const [key, value] of Object.entries(responses)) { - let notOk = value?.status !== 200; + const notOk = value?.status !== 200; if (notOk && value && value.url) { - let details = { + const details = { status: value.status, statusText: value.statusText, redirected: value.redirected, @@ -113,28 +113,28 @@ async function validateResponses(responses: any[], orderedFetchOpts: any[]) { 'Status not OK for fetch request. Details are: ' + JSON.stringify(details), ); - let fetchOptions = orderedFetchOpts[key] || {}; // If we don't have fetch options, we just use an empty object. + const fetchOptions = orderedFetchOpts[key] || {}; // If we don't have fetch options, we just use an empty object. responses[key] = await fetchRetry(value?.url, 200, 3, fetchOptions); } } return responses; } -export async function fetchWithGraphQl(graphQlUrlProps: any[]) { +export async function fetchWithGraphQl(graphQlUrlProps) { try { - let responses = await validateResponses( + const responses = await validateResponses( await Promise.all(graphQlUrlProps.map((u) => fetch(RMP_GRAPHQL_URL, u))), graphQlUrlProps, ); // We now have all the responses. So, we consider all the responses, and collect the ratings. - let ratings: RMPRatingInterface[] = await Promise.all( + const ratings: RMPRatingInterface[] = await Promise.all( responses.map((res) => res.json()), ); for (let i = 0; i < ratings.length; i++) { if ( ratings[i] != null && - ratings[i].hasOwnProperty('data') && - ratings[i]['data'].hasOwnProperty('node') + Object.prototype.hasOwnProperty.call(ratings[i], 'data') && + Object.prototype.hasOwnProperty.call(ratings[i]['data'], 'node') ) { ratings[i] = ratings[i]['data']['node']; } @@ -162,19 +162,19 @@ export async function requestProfessorsFromRmp( // fetch professor ids from each url try { - let responses = await validateResponses( + const responses = await validateResponses( await Promise.all(professorUrls.map((u) => fetch(u))), [], ); - let texts = await Promise.all(responses.map((res) => res.text())); + const texts = await Promise.all(responses.map((res) => res.text())); const professorIds = getProfessorIds(texts, request.professorNames); // create fetch objects for each professor id const graphQlUrlProps = getGraphQlUrlProps(professorIds); // fetch professor info by id with graphQL - let professors = await fetchWithGraphQl(graphQlUrlProps); + const professors = await fetchWithGraphQl(graphQlUrlProps); return professors; } catch (error) { reportError('requestProfessorsFromRmp', error);