From 5b202452f0edd7e41719e5c9483bcf9edcc260ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl?= Date: Sun, 27 Mar 2022 04:50:24 +0200 Subject: [PATCH] Optimizing Oengus schedule Import - calling twitch and speedrun.com api asynchronously - removed the 1s sleep from searchForUserData - prevent speedrun.com api rate limiting - no longer call speedrun.com api for runners info if we won't use any of them - added caching to searchForTwitchGame --- src/extension/oengus-import.ts | 173 ++++++++++++++++++++------------- src/extension/srcom-api.ts | 31 +++++- 2 files changed, 132 insertions(+), 72 deletions(-) diff --git a/src/extension/oengus-import.ts b/src/extension/oengus-import.ts index de2898bd..24ae1c70 100644 --- a/src/extension/oengus-import.ts +++ b/src/extension/oengus-import.ts @@ -1,4 +1,4 @@ -import { OengusMarathon, OengusSchedule, RunData, RunDataPlayer, RunDataTeam } from '@nodecg-speedcontrol/types'; // eslint-disable-line object-curly-newline, max-len +import { OengusMarathon, OengusSchedule, OengusUser, RunData, RunDataPlayer, RunDataTeam } from '@nodecg-speedcontrol/types'; // eslint-disable-line object-curly-newline, max-len import { Duration, parse as isoParse, toSeconds } from 'iso8601-duration'; import { isObject } from 'lodash'; import needle, { NeedleResponse } from 'needle'; @@ -87,6 +87,7 @@ function resetImportStatus(): void { async function importSchedule(marathonShort: string, useJapanese: boolean): Promise { try { oengusImportStatus.value.importing = true; + const marathonResp = await get(`/marathons/${marathonShort}`); const scheduleResp = await get(`/marathons/${marathonShort}/schedule?withCustomData=true`); if (!isOengusMarathon(marathonResp.body)) { @@ -101,12 +102,18 @@ async function importSchedule(marathonShort: string, useJapanese: boolean): Prom // This is updated for every run so we can calculate a scheduled time for each one. let scheduledTime = Math.floor(Date.parse(marathonResp.body.startDate) / 1000); + var gameTwitchPromises: Promise[] = []; + var allRunnersPromises: Promise[] = []; + + var lines = oengusLines.filter((line) => (!checkGameAgainstIgnoreList(line.gameName, 'oengus'))); + oengusImportStatus.value.item = 0; + oengusImportStatus.value.total = oengusLines.length; + // Filtering out any games on the ignore list before processing them all. - const newRunDataArray = await mapSeries(oengusLines.filter((line) => ( - !checkGameAgainstIgnoreList(line.gameName, 'oengus') - )), async (line, index, arr) => { - oengusImportStatus.value.item = index + 1; - oengusImportStatus.value.total = arr.length; + const newRunDataArray = await mapSeries(lines, async (line) => { + + var gameTwitchPromise : Promise; + var runnersPromise: Promise[] = []; // If Oengus ID matches run already imported, re-use our UUID. const matchingOldRun = runDataArray.value @@ -129,33 +136,22 @@ async function importSchedule(marathonShort: string, useJapanese: boolean): Prom const parsedSetup = isoParse(line.setupTime); runData.setupTime = formatDuration(parsedSetup); runData.setupTimeS = toSeconds(parsedSetup); + if (line.setupBlock) { // Game name set to "Setup" if the line is a setup block. runData.game = line.setupBlockText || 'Setup'; runData.gameTwitch = 'Just Chatting'; + gameTwitchPromise = Promise.resolve(); + // Estimate for a setup block will be the setup time instead. runData.estimate = runData.setupTime; runData.estimateS = runData.setupTimeS; runData.setupTime = formatDuration({ seconds: 0 }); runData.setupTimeS = 0; - } else if (line.gameName) { - // Attempt to find Twitch directory on speedrun.com if setting is enabled. - let srcomGameTwitch; - if (!config.oengus.disableSpeedrunComLookup) { - [, srcomGameTwitch] = await to(searchForTwitchGame(line.gameName)); - } - let gameTwitch; - // Verify some game directory supplied exists on Twitch. - for (const str of [srcomGameTwitch, line.gameName]) { - if (str) { - gameTwitch = (await to(verifyTwitchDir(str)))[1]?.name; - if (gameTwitch) { - break; // If a directory was successfully found, stop loop early. - } - } - } - runData.gameTwitch = gameTwitch; + } else { + gameTwitchPromise = setTwitchGame(runData); } + gameTwitchPromises.push(gameTwitchPromise); // Custom Data if (line.customDataDTO) { @@ -175,57 +171,23 @@ async function importSchedule(marathonShort: string, useJapanese: boolean): Prom scheduledTime += runData.estimateS + runData.setupTimeS; // Team Data - runData.teams = await mapSeries(line.runners, async (runner) => { - const team: RunDataTeam = { - id: uuid(), - players: [], - }; - const playerTwitch = runner.connections - ?.find((c) => c.platform === 'TWITCH')?.username || runner.twitchName; - const playerPronouns = typeof runner.pronouns === 'string' - ? runner.pronouns.split(',') - : runner.pronouns; - const player: RunDataPlayer = { - name: (useJapanese && runner.usernameJapanese) - ? runner.usernameJapanese : runner.username, - id: uuid(), - teamID: team.id, - social: { - twitch: playerTwitch || undefined, - }, - country: runner.country?.toLowerCase() || undefined, - pronouns: playerPronouns?.join(', ') || undefined, - customData: {}, - }; - if (!config.oengus.disableSpeedrunComLookup) { - const playerTwitter = runner.connections - ?.find((c) => c.platform === 'TWITTER')?.username || runner.twitterName; - const playerSrcom = runner.connections - ?.find((c) => c.platform === 'SPEEDRUNCOM')?.username || runner.speedruncomName; - const data = await searchForUserDataMultiple( - { type: 'srcom', val: playerSrcom }, - { type: 'twitch', val: playerTwitch }, - { type: 'twitter', val: playerTwitter }, - { type: 'name', val: runner.username }, - ); - if (data) { - // Always favour the supplied Twitch username/country/pronouns - // from Oengus if available. - if (!playerTwitch) { - const tURL = data.twitch?.uri || undefined; - player.social.twitch = getTwitchUserFromURL(tURL); - } - if (!runner.country) player.country = data.location?.country.code || undefined; - if (!runner.pronouns?.length) { - player.pronouns = data.pronouns?.toLowerCase() || undefined; - } - } + runData.teams = []; + line.runners.forEach((runner, index, array) => { + var runnerPromise = setRunner(runData, runner, useJapanese,index); + runnersPromise.push(runnerPromise); + allRunnersPromises.push(runnerPromise); + }); + + Promise.allSettled(runnersPromise.concat(gameTwitchPromise)).then(()=>{ + if (oengusImportStatus.value.item != undefined) { + oengusImportStatus.value.item++; } - team.players.push(player); - return team; }); return runData; }); + + await Promise.allSettled(gameTwitchPromises); + await Promise.allSettled(allRunnersPromises); runDataArray.value = newRunDataArray; resetImportStatus(); } catch (err) { @@ -248,3 +210,74 @@ nodecg.listenFor('importOengusSchedule', async (data, ack) => { processAck(ack, err); } }); + +async function setTwitchGame(runData: RunData): Promise { + if (runData.game) { + // Attempt to find Twitch directory on speedrun.com if setting is enabled. + let srcomGameTwitch; + if (!config.oengus.disableSpeedrunComLookup) { + [, srcomGameTwitch] = await to(searchForTwitchGame(runData.game)); + } + let gameTwitch; + // Verify some game directory supplied exists on Twitch. + for (const str of [srcomGameTwitch, runData.game]) { + if (str) { + gameTwitch = (await to(verifyTwitchDir(str)))[1]?.name; + if (gameTwitch) { + break; // If a directory was successfully found, stop loop early. + } + } + } + runData.gameTwitch = gameTwitch; + } +} + +async function setRunner(runData: RunData, runner: OengusUser, useJapanese: boolean, index: number): Promise { + const team: RunDataTeam = { + id: uuid(), + players: [], + }; + const playerTwitch = runner.connections + ?.find((c) => c.platform === 'TWITCH')?.username || runner.twitchName; + const playerPronouns = typeof runner.pronouns === 'string' + ? runner.pronouns.split(',') + : runner.pronouns; + const player: RunDataPlayer = { + name: (useJapanese && runner.usernameJapanese) + ? runner.usernameJapanese : runner.username, + id: uuid(), + teamID: team.id, + social: { + twitch: playerTwitch || undefined, + }, + country: runner.country?.toLowerCase() || undefined, + pronouns: playerPronouns?.join(', ') || undefined, + customData: {}, + }; + if (!config.oengus.disableSpeedrunComLookup && !(playerTwitch && runner.country && runner.pronouns?.length)) { + const playerTwitter = runner.connections + ?.find((c) => c.platform === 'TWITTER')?.username || runner.twitterName; + const playerSrcom = runner.connections + ?.find((c) => c.platform === 'SPEEDRUNCOM')?.username || runner.speedruncomName; + const data = await searchForUserDataMultiple( + { type: 'srcom', val: playerSrcom }, + { type: 'twitch', val: playerTwitch }, + { type: 'twitter', val: playerTwitter }, + { type: 'name', val: runner.username }, + ); + if (data) { + // Always favour the supplied Twitch username/country/pronouns + // from Oengus if available. + if (!playerTwitch) { + const tURL = data.twitch?.uri || undefined; + player.social.twitch = getTwitchUserFromURL(tURL); + } + if (!runner.country) player.country = data.location?.country.code || undefined; + if (!runner.pronouns?.length) { + player.pronouns = data.pronouns?.toLowerCase() || undefined; + } + } + } + team.players.push(player); + runData.teams[index]=team; +} diff --git a/src/extension/srcom-api.ts b/src/extension/srcom-api.ts index 16bdb77a..42eab918 100644 --- a/src/extension/srcom-api.ts +++ b/src/extension/srcom-api.ts @@ -6,7 +6,11 @@ import { get as ncgGet } from './util/nodecg'; const nodecg = ncgGet(); const userDataCache: { [k: string]: Speedruncom.UserData } = {}; +const gameDataCache: { [k: string]: string } = {}; +const rateLimitNumber = 100; // 100 requests +const rateLimitTime = 60000; // per minute +var requestTimes: number[] = []; /** * Make a GET request to speedrun.com API. * @param url speedrun.com API endpoint you want to access. @@ -18,6 +22,18 @@ async function get(endpoint: string): Promise { ? `https://www.speedrun.com${endpoint}` : `https://www.speedrun.com/api/v1${endpoint}`; nodecg.log.debug(`[speedrun.com] API request processing on ${endpoint}`); + nodecg.log.debug(`[speedrun.com] API request number ${requestTimes.length}`); + + while (requestTimes.length >= rateLimitNumber) { + var now = Date.now(); + requestTimes = requestTimes.filter(time => time + rateLimitTime > now); + + if (requestTimes.length >= rateLimitNumber) { + nodecg.log.debug(`[speedrun.com] Waiting to not reach API limit rate`); + await sleep(requestTimes[0] + rateLimitTime - now); + } + } + requestTimes.push(Date.now()); const resp = await needle( 'get', url, @@ -53,13 +69,22 @@ async function get(endpoint: string): Promise { */ export async function searchForTwitchGame(query: string, abbr = false): Promise { try { + const cacheKey = `${query}_${abbr}`; + if (gameDataCache[cacheKey]) { + nodecg.log.debug( + `[speedrun.com] Game data found in cache for "${query}_${abbr}":`, + JSON.stringify(gameDataCache[cacheKey]), + ); + return gameDataCache[cacheKey]; + } + let result: Speedruncom.GameData | undefined; // Abbreviation is easy to find, plug in and receive result. if (abbr) { const resp = await get(`/games?abbreviation=${encodeURIComponent(query)}&max=1`); [result] = resp.body.data; - // Using a name is slightly more complicated to find an accurate result. + // Using a name is slightly more complicated to find an accurate result. } else { // First, try searching the regular API's top 10 and see if there's an exact match at all. const resp1 = await get(`/games?name=${encodeURIComponent(query)}&max=10`); @@ -96,6 +121,9 @@ export async function searchForTwitchGame(query: string, abbr = false): Promise< `[speedrun.com] Twitch game name found for "${query}":`, result.names.twitch, ); + + gameDataCache[cacheKey] = result.names.twitch; // Simple temp cache storage. + return result.names.twitch; } catch (err) { nodecg.log.debug(`[speedrun.com] Twitch game name lookup failed for "${query}":`, err); @@ -119,7 +147,6 @@ export async function searchForUserData( return userDataCache[cacheKey]; } try { - await sleep(1000); let data: Speedruncom.UserData | undefined; if (type === 'srcom') { const resp = await get(