Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimizing Oengus schedule Import #129

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Optimizing Oengus schedule Import #129

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 103 additions & 70 deletions src/extension/oengus-import.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { OengusMarathon, OengusSchedule, RunData, RunDataPlayer, RunDataTeam } from '@nodecg-speedcontrol/types'; // eslint-disable-line object-curly-newline, max-len
import { OengusMarathon, OengusSchedule, OengusUser, RunData, RunDataPlayer, RunDataTeam } from '@nodecg-speedcontrol/types'; // eslint-disable-line object-curly-newline, max-len
import { Duration, parse as isoParse, toSeconds } from 'iso8601-duration';
import { isObject } from 'lodash';
import needle, { NeedleResponse } from 'needle';
Expand Down Expand Up @@ -87,6 +87,7 @@ function resetImportStatus(): void {
async function importSchedule(marathonShort: string, useJapanese: boolean): Promise<void> {
try {
oengusImportStatus.value.importing = true;

const marathonResp = await get(`/marathons/${marathonShort}`);
const scheduleResp = await get(`/marathons/${marathonShort}/schedule?withCustomData=true`);
if (!isOengusMarathon(marathonResp.body)) {
Expand All @@ -101,12 +102,18 @@ async function importSchedule(marathonShort: string, useJapanese: boolean): Prom
// This is updated for every run so we can calculate a scheduled time for each one.
let scheduledTime = Math.floor(Date.parse(marathonResp.body.startDate) / 1000);

var gameTwitchPromises: Promise<void>[] = [];
var allRunnersPromises: Promise<void>[] = [];

var lines = oengusLines.filter((line) => (!checkGameAgainstIgnoreList(line.gameName, 'oengus')));
oengusImportStatus.value.item = 0;
oengusImportStatus.value.total = oengusLines.length;

// Filtering out any games on the ignore list before processing them all.
const newRunDataArray = await mapSeries(oengusLines.filter((line) => (
!checkGameAgainstIgnoreList(line.gameName, 'oengus')
)), async (line, index, arr) => {
oengusImportStatus.value.item = index + 1;
oengusImportStatus.value.total = arr.length;
const newRunDataArray = await mapSeries(lines, async (line) => {

var gameTwitchPromise : Promise<void>;
var runnersPromise: Promise<void>[] = [];

// If Oengus ID matches run already imported, re-use our UUID.
const matchingOldRun = runDataArray.value
Expand All @@ -129,33 +136,22 @@ async function importSchedule(marathonShort: string, useJapanese: boolean): Prom
const parsedSetup = isoParse(line.setupTime);
runData.setupTime = formatDuration(parsedSetup);
runData.setupTimeS = toSeconds(parsedSetup);

if (line.setupBlock) {
// Game name set to "Setup" if the line is a setup block.
runData.game = line.setupBlockText || 'Setup';
runData.gameTwitch = 'Just Chatting';
gameTwitchPromise = Promise.resolve();

// Estimate for a setup block will be the setup time instead.
runData.estimate = runData.setupTime;
runData.estimateS = runData.setupTimeS;
runData.setupTime = formatDuration({ seconds: 0 });
runData.setupTimeS = 0;
} else if (line.gameName) {
// Attempt to find Twitch directory on speedrun.com if setting is enabled.
let srcomGameTwitch;
if (!config.oengus.disableSpeedrunComLookup) {
[, srcomGameTwitch] = await to(searchForTwitchGame(line.gameName));
}
let gameTwitch;
// Verify some game directory supplied exists on Twitch.
for (const str of [srcomGameTwitch, line.gameName]) {
if (str) {
gameTwitch = (await to(verifyTwitchDir(str)))[1]?.name;
if (gameTwitch) {
break; // If a directory was successfully found, stop loop early.
}
}
}
runData.gameTwitch = gameTwitch;
} else {
gameTwitchPromise = setTwitchGame(runData);
}
gameTwitchPromises.push(gameTwitchPromise);

// Custom Data
if (line.customDataDTO) {
Expand All @@ -175,57 +171,23 @@ async function importSchedule(marathonShort: string, useJapanese: boolean): Prom
scheduledTime += runData.estimateS + runData.setupTimeS;

// Team Data
runData.teams = await mapSeries(line.runners, async (runner) => {
const team: RunDataTeam = {
id: uuid(),
players: [],
};
const playerTwitch = runner.connections
?.find((c) => c.platform === 'TWITCH')?.username || runner.twitchName;
const playerPronouns = typeof runner.pronouns === 'string'
? runner.pronouns.split(',')
: runner.pronouns;
const player: RunDataPlayer = {
name: (useJapanese && runner.usernameJapanese)
? runner.usernameJapanese : runner.username,
id: uuid(),
teamID: team.id,
social: {
twitch: playerTwitch || undefined,
},
country: runner.country?.toLowerCase() || undefined,
pronouns: playerPronouns?.join(', ') || undefined,
customData: {},
};
if (!config.oengus.disableSpeedrunComLookup) {
const playerTwitter = runner.connections
?.find((c) => c.platform === 'TWITTER')?.username || runner.twitterName;
const playerSrcom = runner.connections
?.find((c) => c.platform === 'SPEEDRUNCOM')?.username || runner.speedruncomName;
const data = await searchForUserDataMultiple(
{ type: 'srcom', val: playerSrcom },
{ type: 'twitch', val: playerTwitch },
{ type: 'twitter', val: playerTwitter },
{ type: 'name', val: runner.username },
);
if (data) {
// Always favour the supplied Twitch username/country/pronouns
// from Oengus if available.
if (!playerTwitch) {
const tURL = data.twitch?.uri || undefined;
player.social.twitch = getTwitchUserFromURL(tURL);
}
if (!runner.country) player.country = data.location?.country.code || undefined;
if (!runner.pronouns?.length) {
player.pronouns = data.pronouns?.toLowerCase() || undefined;
}
}
runData.teams = [];
line.runners.forEach((runner, index, array) => {
var runnerPromise = setRunner(runData, runner, useJapanese,index);
runnersPromise.push(runnerPromise);
allRunnersPromises.push(runnerPromise);
});

Promise.allSettled(runnersPromise.concat(gameTwitchPromise)).then(()=>{
if (oengusImportStatus.value.item != undefined) {
oengusImportStatus.value.item++;
}
team.players.push(player);
return team;
});
return runData;
});

await Promise.allSettled(gameTwitchPromises);
await Promise.allSettled(allRunnersPromises);
runDataArray.value = newRunDataArray;
resetImportStatus();
} catch (err) {
Expand All @@ -248,3 +210,74 @@ nodecg.listenFor('importOengusSchedule', async (data, ack) => {
processAck(ack, err);
}
});

async function setTwitchGame(runData: RunData): Promise<void> {
if (runData.game) {
// Attempt to find Twitch directory on speedrun.com if setting is enabled.
let srcomGameTwitch;
if (!config.oengus.disableSpeedrunComLookup) {
[, srcomGameTwitch] = await to(searchForTwitchGame(runData.game));
}
let gameTwitch;
// Verify some game directory supplied exists on Twitch.
for (const str of [srcomGameTwitch, runData.game]) {
if (str) {
gameTwitch = (await to(verifyTwitchDir(str)))[1]?.name;
if (gameTwitch) {
break; // If a directory was successfully found, stop loop early.
}
}
}
runData.gameTwitch = gameTwitch;
}
}

async function setRunner(runData: RunData, runner: OengusUser, useJapanese: boolean, index: number): Promise<void> {
const team: RunDataTeam = {
id: uuid(),
players: [],
};
const playerTwitch = runner.connections
?.find((c) => c.platform === 'TWITCH')?.username || runner.twitchName;
const playerPronouns = typeof runner.pronouns === 'string'
? runner.pronouns.split(',')
: runner.pronouns;
const player: RunDataPlayer = {
name: (useJapanese && runner.usernameJapanese)
? runner.usernameJapanese : runner.username,
id: uuid(),
teamID: team.id,
social: {
twitch: playerTwitch || undefined,
},
country: runner.country?.toLowerCase() || undefined,
pronouns: playerPronouns?.join(', ') || undefined,
customData: {},
};
if (!config.oengus.disableSpeedrunComLookup && !(playerTwitch && runner.country && runner.pronouns?.length)) {
const playerTwitter = runner.connections
?.find((c) => c.platform === 'TWITTER')?.username || runner.twitterName;
const playerSrcom = runner.connections
?.find((c) => c.platform === 'SPEEDRUNCOM')?.username || runner.speedruncomName;
const data = await searchForUserDataMultiple(
{ type: 'srcom', val: playerSrcom },
{ type: 'twitch', val: playerTwitch },
{ type: 'twitter', val: playerTwitter },
{ type: 'name', val: runner.username },
);
if (data) {
// Always favour the supplied Twitch username/country/pronouns
// from Oengus if available.
if (!playerTwitch) {
const tURL = data.twitch?.uri || undefined;
player.social.twitch = getTwitchUserFromURL(tURL);
}
if (!runner.country) player.country = data.location?.country.code || undefined;
if (!runner.pronouns?.length) {
player.pronouns = data.pronouns?.toLowerCase() || undefined;
}
}
}
team.players.push(player);
runData.teams[index]=team;
}
31 changes: 29 additions & 2 deletions src/extension/srcom-api.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@ import { get as ncgGet } from './util/nodecg';

const nodecg = ncgGet();
const userDataCache: { [k: string]: Speedruncom.UserData } = {};
const gameDataCache: { [k: string]: string } = {};
const rateLimitNumber = 100; // 100 requests
const rateLimitTime = 60000; // per minute

var requestTimes: number[] = [];
/**
* Make a GET request to speedrun.com API.
* @param url speedrun.com API endpoint you want to access.
Expand All @@ -18,6 +22,18 @@ async function get(endpoint: string): Promise<NeedleResponse> {
? `https://www.speedrun.com${endpoint}`
: `https://www.speedrun.com/api/v1${endpoint}`;
nodecg.log.debug(`[speedrun.com] API request processing on ${endpoint}`);
nodecg.log.debug(`[speedrun.com] API request number ${requestTimes.length}`);

while (requestTimes.length >= rateLimitNumber) {
var now = Date.now();
requestTimes = requestTimes.filter(time => time + rateLimitTime > now);

if (requestTimes.length >= rateLimitNumber) {
nodecg.log.debug(`[speedrun.com] Waiting to not reach API limit rate`);
await sleep(requestTimes[0] + rateLimitTime - now);
}
}
requestTimes.push(Date.now());
const resp = await needle(
'get',
url,
Expand Down Expand Up @@ -53,13 +69,22 @@ async function get(endpoint: string): Promise<NeedleResponse> {
*/
export async function searchForTwitchGame(query: string, abbr = false): Promise<string> {
try {
const cacheKey = `${query}_${abbr}`;
if (gameDataCache[cacheKey]) {
nodecg.log.debug(
`[speedrun.com] Game data found in cache for "${query}_${abbr}":`,
JSON.stringify(gameDataCache[cacheKey]),
);
return gameDataCache[cacheKey];
}

let result: Speedruncom.GameData | undefined;

// Abbreviation is easy to find, plug in and receive result.
if (abbr) {
const resp = await get(`/games?abbreviation=${encodeURIComponent(query)}&max=1`);
[result] = resp.body.data;
// Using a name is slightly more complicated to find an accurate result.
// Using a name is slightly more complicated to find an accurate result.
} else {
// First, try searching the regular API's top 10 and see if there's an exact match at all.
const resp1 = await get(`/games?name=${encodeURIComponent(query)}&max=10`);
Expand Down Expand Up @@ -96,6 +121,9 @@ export async function searchForTwitchGame(query: string, abbr = false): Promise<
`[speedrun.com] Twitch game name found for "${query}":`,
result.names.twitch,
);

gameDataCache[cacheKey] = result.names.twitch; // Simple temp cache storage.

return result.names.twitch;
} catch (err) {
nodecg.log.debug(`[speedrun.com] Twitch game name lookup failed for "${query}":`, err);
Expand All @@ -119,7 +147,6 @@ export async function searchForUserData(
return userDataCache[cacheKey];
}
try {
await sleep(1000);
let data: Speedruncom.UserData | undefined;
if (type === 'srcom') {
const resp = await get(
Expand Down