Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: tieredProxyUrls accept null for switching the proxy off #2743

Merged
merged 1 commit into from
Nov 12, 2024
Merged

feat: tieredProxyUrls accept null for switching the proxy off #2743

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions packages/core/src/proxy_configuration.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,14 @@ export interface ProxyConfigurationOptions {
* The crawler probes lower-level proxies at intervals to check if it can make the tier downshift.
*
* This feature is useful when you have a set of proxies with different performance characteristics (speed, price, antibot performance etc.) and you want to use the best one for each domain.
*
* Use `null` as a proxy URL to disable the proxy for the given tier.
*/
tieredProxyUrls?: string[][];
tieredProxyUrls?: (string | null)[][];
}

export interface TieredProxy {
proxyUrl: string;
proxyUrl: string | null;
proxyTier?: number;
}

Expand Down Expand Up @@ -123,7 +125,7 @@ class ProxyTierTracker {
private histogram: number[];
private currentTier: number;

constructor(tieredProxyUrls: string[][]) {
constructor(tieredProxyUrls: (string | null)[][]) {
this.histogram = tieredProxyUrls.map(() => 0);
this.currentTier = 0;
}
Expand Down Expand Up @@ -199,7 +201,7 @@ export class ProxyConfiguration {
isManInTheMiddle = false;
protected nextCustomUrlIndex = 0;
protected proxyUrls?: string[];
protected tieredProxyUrls?: string[][];
protected tieredProxyUrls?: (string | null)[][];
protected usedProxyUrls = new Map<string, string>();
protected newUrlFunction?: ProxyConfigurationFunction;
protected log = log.child({ prefix: 'ProxyConfiguration' });
Expand Down Expand Up @@ -232,7 +234,9 @@ export class ProxyConfiguration {
ow.object.exactShape({
proxyUrls: ow.optional.array.nonEmpty.ofType(ow.string.url),
newUrlFunction: ow.optional.function,
tieredProxyUrls: ow.optional.array.nonEmpty.ofType(ow.array.nonEmpty.ofType(ow.string.url)),
tieredProxyUrls: ow.optional.array.nonEmpty.ofType(
ow.array.nonEmpty.ofType(ow.any(ow.string.url, ow.null)),
),
}),
);

Expand Down Expand Up @@ -271,7 +275,7 @@ export class ProxyConfiguration {
let tier: number | undefined;
if (this.tieredProxyUrls) {
const { proxyUrl, proxyTier } = this._handleTieredUrl(sessionId ?? cryptoRandomObjectId(6), options);
url = proxyUrl;
url = proxyUrl ?? undefined;
tier = proxyTier;
} else {
url = await this.newUrl(sessionId, options);
Expand Down Expand Up @@ -381,7 +385,7 @@ export class ProxyConfiguration {
}

if (this.tieredProxyUrls) {
return this._handleTieredUrl(sessionId ?? cryptoRandomObjectId(6), options).proxyUrl;
return this._handleTieredUrl(sessionId ?? cryptoRandomObjectId(6), options).proxyUrl ?? undefined;
}

return this._handleCustomUrl(sessionId);
Expand Down