From 31f9b0c4289a6b8b8d3dc500f19b297b2b4c57c1 Mon Sep 17 00:00:00 2001 From: Michael Goodnow Date: Sun, 17 Dec 2023 01:36:42 -0500 Subject: [PATCH 1/2] perform assessments serially to avoid artificially long snatches --- src/pipeline.ts | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index d1127e978..ffd6f1267 100755 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -61,17 +61,27 @@ interface FoundOnOtherSites { matches: number; } +async function assessCandidates( + candidates: Candidate[], + searchee: Searchee, + hashesToExclude: string[] +) { + const assessments = []; + for (const result of candidates) { + const assessment = await assessCandidate( + result, + searchee, + hashesToExclude + ); + assessments.push({ assessment, tracker: result.tracker }); + } + return assessments; +} + async function findOnOtherSites( searchee: Searchee, hashesToExclude: string[] ): Promise { - const assessEach = async ( - result: Candidate - ): Promise => ({ - assessment: await assessCandidate(result, searchee, hashesToExclude), - tracker: result.tracker, - }); - // make sure searchee is in database await db("searchee") .insert({ name: searchee.name }) @@ -94,11 +104,13 @@ async function findOnOtherSites( })) ); - const assessed = await Promise.all( - results.map(assessEach) + const assessments = await assessCandidates( + results, + searchee, + hashesToExclude ); - const { rateLimited, notRateLimited } = assessed.reduce( + const { rateLimited, notRateLimited } = assessments.reduce( (acc, cur, idx) => { const candidate = results[idx]; if (cur.assessment.decision === Decision.RATE_LIMITED) { @@ -113,7 +125,7 @@ async function findOnOtherSites( } ); - const matches = assessed.filter( + const matches = assessments.filter( (e) => e.assessment.decision === Decision.MATCH || e.assessment.decision === Decision.MATCH_SIZE_ONLY From b1747540097eea8b5baceadbc5bac3749b3fb282 Mon Sep 17 00:00:00 2001 From: Michael Goodnow Date: Thu, 28 Dec 2023 23:10:30 -0500 Subject: [PATCH 2/2] add types --- src/pipeline.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index ffd6f1267..a154f4dfd 100755 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -65,8 +65,8 @@ async function assessCandidates( candidates: Candidate[], searchee: Searchee, hashesToExclude: string[] -) { - const assessments = []; +): Promise { + const assessments: AssessmentWithTracker[] = []; for (const result of candidates) { const assessment = await assessCandidate( result,