From 9761e59c8a0fd0058edcafc153fedf192b7770f3 Mon Sep 17 00:00:00 2001 From: bakerboy448 <55419169+bakerboy448@users.noreply.github.com> Date: Sat, 9 Sep 2023 14:41:30 -0500 Subject: [PATCH 1/6] (feat): Use Retry-After Header given HTTP 429 --- src/torznab.ts | 47 ++++++++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/src/torznab.ts b/src/torznab.ts index 27475f3dc..c474f9eee 100644 --- a/src/torznab.ts +++ b/src/torznab.ts @@ -179,11 +179,10 @@ export async function searchTorznab( const timestampCallout = " (filtered by timestamps)"; logger.info({ label: Label.TORZNAB, - message: `Searching ${indexersToUse.length} indexers for ${name}${ - indexersToUse.length < enabledIndexers.length - ? timestampCallout - : "" - }`, + message: `Searching ${indexersToUse.length} indexers for ${name}${indexersToUse.length < enabledIndexers.length + ? timestampCallout + : "" + }`, }); return makeRequests(name, indexersToUse, (indexer) => @@ -463,26 +462,32 @@ async function makeRequests( }) .then((response) => { if (!response.ok) { + let indexerStatusError = IndexerStatus.UNKNOWN_ERROR; + let retryInMs = 3600000; // Default 1 hour in milliseconds + if (response.status === 429) { - updateIndexerStatus( - IndexerStatus.RATE_LIMITED, - Date.now() + ms("1 hour"), - [indexers[i].id] - ); - } else { - updateIndexerStatus( - IndexerStatus.UNKNOWN_ERROR, - Date.now() + ms("1 hour"), - [indexers[i].id] - ); + indexerStatusError = IndexerStatus.RATE_LIMITED; + const retryAfter = response.headers.get('Retry-After'); + if (retryAfter) { + const retryAfterInSeconds = parseInt(retryAfter); + if (!isNaN(retryAfterInSeconds)) { + retryInMs = retryAfterInSeconds * 1000; + } else { + const retryAfterDate = new Date(retryAfter); + if (!isNaN(retryAfterDate.getTime())) { + retryInMs = retryAfterDate.getTime() - Date.now(); + } + } + } } - throw new Error( - `request failed with code: ${response.status}` - ); + + updateIndexerStatus(indexerStatusError, Date.now() + retryInMs, [indexers[i].id]); + + throw new Error(`request failed with code: ${response.status}`); } - return response; + + return response.text(); }) - .then((r) => r.text()) .then(xml2js.parseStringPromise) .then(parseTorznabResults) ) From c05a13546248c2352dddfc55c6bd6f340060c32e Mon Sep 17 00:00:00 2001 From: Michael Goodnow Date: Tue, 12 Sep 2023 01:15:31 -0400 Subject: [PATCH 2/6] change code style to match project --- src/torznab.ts | 62 +++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/src/torznab.ts b/src/torznab.ts index c474f9eee..0b4c42465 100644 --- a/src/torznab.ts +++ b/src/torznab.ts @@ -1,7 +1,7 @@ import ms from "ms"; import fetch from "node-fetch"; import xml2js from "xml2js"; -import { EP_REGEX, SEASON_REGEX } from "./constants.js"; +import { EP_REGEX, SEASON_REGEX, USER_AGENT } from "./constants.js"; import { db } from "./db.js"; import { CrossSeedError } from "./errors.js"; import { @@ -21,7 +21,6 @@ import { reformatTitleForSearching, stripExtension, } from "./utils.js"; -import { USER_AGENT } from "./constants.js"; interface TorznabParams { t: "caps" | "search" | "tvsearch" | "movie"; @@ -179,10 +178,11 @@ export async function searchTorznab( const timestampCallout = " (filtered by timestamps)"; logger.info({ label: Label.TORZNAB, - message: `Searching ${indexersToUse.length} indexers for ${name}${indexersToUse.length < enabledIndexers.length - ? timestampCallout - : "" - }`, + message: `Searching ${indexersToUse.length} indexers for ${name}${ + indexersToUse.length < enabledIndexers.length + ? timestampCallout + : "" + }`, }); return makeRequests(name, indexersToUse, (indexer) => @@ -462,32 +462,36 @@ async function makeRequests( }) .then((response) => { if (!response.ok) { - let indexerStatusError = IndexerStatus.UNKNOWN_ERROR; - let retryInMs = 3600000; // Default 1 hour in milliseconds - - if (response.status === 429) { - indexerStatusError = IndexerStatus.RATE_LIMITED; - const retryAfter = response.headers.get('Retry-After'); - if (retryAfter) { - const retryAfterInSeconds = parseInt(retryAfter); - if (!isNaN(retryAfterInSeconds)) { - retryInMs = retryAfterInSeconds * 1000; - } else { - const retryAfterDate = new Date(retryAfter); - if (!isNaN(retryAfterDate.getTime())) { - retryInMs = retryAfterDate.getTime() - Date.now(); - } - } - } + const retryAfterSeconds = Number( + response.headers.get("Retry-After") + ); + + if (!Number.isNaN(retryAfterSeconds)) { + updateIndexerStatus( + response.status === 429 + ? IndexerStatus.RATE_LIMITED + : IndexerStatus.UNKNOWN_ERROR, + Date.now() + ms(`${retryAfterSeconds} seconds`), + [indexers[i].id] + ); + } else { + updateIndexerStatus( + response.status === 429 + ? IndexerStatus.RATE_LIMITED + : IndexerStatus.UNKNOWN_ERROR, + response.status === 429 + ? Date.now() + ms("1 hour") + : Date.now() + ms("10 minutes"), + [indexers[i].id] + ); } - - updateIndexerStatus(indexerStatusError, Date.now() + retryInMs, [indexers[i].id]); - - throw new Error(`request failed with code: ${response.status}`); + throw new Error( + `request failed with code: ${response.status}` + ); } - - return response.text(); + return response; }) + .then((r) => r.text()) .then(xml2js.parseStringPromise) .then(parseTorznabResults) ) From 618c8538f66511a307ea12db4f545767e10f1dcf Mon Sep 17 00:00:00 2001 From: Michael Goodnow Date: Tue, 12 Sep 2023 01:17:16 -0400 Subject: [PATCH 3/6] response.text() --- src/torznab.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/torznab.ts b/src/torznab.ts index 0b4c42465..bed7aecec 100644 --- a/src/torznab.ts +++ b/src/torznab.ts @@ -489,9 +489,8 @@ async function makeRequests( `request failed with code: ${response.status}` ); } - return response; + return response.text(); }) - .then((r) => r.text()) .then(xml2js.parseStringPromise) .then(parseTorznabResults) ) From cbf18b6bec2b85bfd74d628a8e4ec33e378d4fff Mon Sep 17 00:00:00 2001 From: Michael Goodnow Date: Tue, 12 Sep 2023 11:30:45 -0400 Subject: [PATCH 4/6] 5.4.4 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index d7d40e94f..8fe6bbf23 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cross-seed", - "version": "5.4.3", + "version": "5.4.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "cross-seed", - "version": "5.4.3", + "version": "5.4.4", "license": "Apache-2.0", "dependencies": { "bencode": "^2.0.1", diff --git a/package.json b/package.json index 0b1979483..c7a63e621 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cross-seed", - "version": "5.4.3", + "version": "5.4.4", "description": "Query Jackett for cross-seedable torrents", "scripts": { "test": "true", From b9f840246d8eb331349b1406692b6dc3ea73e38b Mon Sep 17 00:00:00 2001 From: zakary <123845855+zakkarry@users.noreply.github.com> Date: Tue, 12 Sep 2023 16:54:01 -0500 Subject: [PATCH 5/6] fix(regex): movie and tv (pack and episode) regex The following are changes made to improve matching for TV (season pack and episode) as well as movies. All Regex's * Replaced literal space (' ') in expressions with \s MOVIE_REGEX * Added support for '_' in release name. SEASON_REGEX * Added support for '_' in release naming. * Capture multiple variations (including poorly named) of multiple season packs properly. * Fixed matching incorrectly when season has more than one digit EP_REGEX * Added more variations on multi-episode capturing **fix/update(regex): change title capture to lazy** * Modify capture group to match lazily, reducing the number of step proportional to the length of the release name. **fix(regex): corrected <seasonmax> matching** * <seasonmax> previously required formatting to be S## while not all releases contain the preceding S in multi-season packs. * Removed the '_' from separator between seasons in multi season packs. If you run some test cases and find any outliers or mismatching, please comment and note what release naming you used to demonstrate. --- src/constants.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index 187c17a57..3f270b312 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -6,11 +6,11 @@ export const PROGRAM_NAME = packageDotJson.name; export const PROGRAM_VERSION = packageDotJson.version; export const USER_AGENT = `CrossSeed/${PROGRAM_VERSION}`; -export const EP_REGEX = /^(?<title>.+)[. ](?<season>S\d+)(?<episode>E\d+)/i; +export const EP_REGEX = /^(?<title>.+?)[\s._](?<season>S\d+)?[_.\s]?(?<episode>E\d+(?:[-\s]?E?\d+)?)/i; export const SEASON_REGEX = - /^(?<title>.+)[. ](?<season>S\d+)(?:\s?-\s?(?<seasonmax>S?\d+))?(?!E\d+)/i; + /^(?<title>.+?)[_.\s](?<season>S\d+)(?:[.\-\s]*?(?<seasonmax>S?\d+))?(?=[_.\s](?!E\d+))/i; export const MOVIE_REGEX = - /^(?<title>.+)[. ][[(]?(?<year>\d{4})[)\]]?(?![pi])/i; + /^(?<title>.+?)[._\s][[(]?(?<year>\d{4})[)\]]?(?![pi])/i; export const VIDEO_EXTENSIONS = [".mkv", ".mp4", ".avi"]; From d168ac81481bcb35e9e10cd78685526123f7aa39 Mon Sep 17 00:00:00 2001 From: Michael Goodnow <mmgoodnow@gmail.com> Date: Tue, 12 Sep 2023 23:51:19 -0400 Subject: [PATCH 6/6] 5.4.5 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8fe6bbf23..5f935310f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cross-seed", - "version": "5.4.4", + "version": "5.4.5", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "cross-seed", - "version": "5.4.4", + "version": "5.4.5", "license": "Apache-2.0", "dependencies": { "bencode": "^2.0.1", diff --git a/package.json b/package.json index c7a63e621..ee46be51a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cross-seed", - "version": "5.4.4", + "version": "5.4.5", "description": "Query Jackett for cross-seedable torrents", "scripts": { "test": "true",