feat: add constraint relaxation by default & strict mode (#233)

apify · Sep 27, 2023 · f01f3f3 · f01f3f3
1 parent 7c372c8
commit f01f3f3
Show file tree

Hide file tree

Showing 4 changed files with 74 additions and 3 deletions.
diff --git a/packages/fingerprint-generator/src/fingerprint-generator.ts b/packages/fingerprint-generator/src/fingerprint-generator.ts
@@ -132,7 +132,13 @@ export class FingerprintGenerator extends HeaderGenerator {
                 })
                 : undefined;
 
-            return utils.getPossibleValues(this.fingerprintGeneratorNetwork, filteredValues);
+            try {
+                return utils.getPossibleValues(this.fingerprintGeneratorNetwork, filteredValues);
+            } catch (e) {
+                if (options?.strict) throw e;
+                delete filteredValues.screen;
+                return undefined;
+            }
         })();
 
         while (true) {

diff --git a/packages/header-generator/src/header-generator.ts b/packages/header-generator/src/header-generator.ts
@@ -35,6 +35,7 @@ export const headerGeneratorOptionsShape = {
     locales: ow.optional.array.ofType(ow.string),
     httpVersion: ow.optional.string.oneOf(SUPPORTED_HTTP_VERSIONS),
     browserListQuery: ow.optional.string,
+    strict: ow.optional.boolean,
 };
 
 /**
@@ -113,6 +114,10 @@ export interface HeaderGeneratorOptions {
     *  Can be either 1 or 2. Default value is 2.
     */
     httpVersion: HttpVersion;
+    /**
+     * If true, the generator will throw an error if it cannot generate headers based on the input.
+     */
+    strict: boolean;
 }
 
 /**
@@ -155,6 +160,14 @@ export class HeaderGenerator {
 
     private headersOrder: string[];
 
+    private relaxationOrder: (keyof typeof headerGeneratorOptionsShape)[] = [
+        'locales',
+        'devices',
+        'operatingSystems',
+        'browsers',
+        'browserListQuery',
+    ];
+
     /**
     * @param options Default header generation options used - unless overridden.
     */
@@ -168,6 +181,7 @@ export class HeaderGenerator {
             locales = ['en-US'],
             httpVersion = '2',
             browserListQuery = '',
+            strict = false,
         } = options;
         this.globalOptions = {
             browsers: this._prepareBrowsersConfig(browsers as BrowsersType, browserListQuery, httpVersion),
@@ -176,6 +190,7 @@ export class HeaderGenerator {
             locales,
             httpVersion,
             browserListQuery,
+            strict,
         };
         this.uniqueBrowsers = [];
 
@@ -251,7 +266,17 @@ export class HeaderGenerator {
 
                 return this.orderHeaders(converted2to1);
             }
-            throw new Error('No headers based on this input can be generated. Please relax or change some of the requirements you specified.');
+
+            const relaxationIndex = this.relaxationOrder.findIndex((key) => options[key] !== undefined);
+            if (options.strict || relaxationIndex === -1) {
+                throw new Error('No headers based on this input can be generated. Please relax or change some of the requirements you specified.');
+            }
+
+            // Relax the requirements and try again
+            const relaxedOptions = { ...options };
+            const relaxationKey = this.relaxationOrder[relaxationIndex];
+            relaxedOptions[relaxationKey] = undefined as never;
+            return this.getHeaders(relaxedOptions, requestDependentHeaders, userAgentValues);
         }
 
         // Generate the actual headers

diff --git a/test/fingerprint-generator/generation.test.ts b/test/fingerprint-generator/generation.test.ts
@@ -111,4 +111,36 @@ describe('Generate fingerprints with basic constraints', () => {
             },
         })).toBeDefined();
     });
+
+    test.only('[relaxation] header strict mode propagates', () => {
+        const fingerprintGenerator = new FingerprintGenerator();
+
+        expect(fingerprintGenerator.getFingerprint({
+            devices: ['mobile'],
+            operatingSystems: ['windows'],
+        })).toBeDefined();
+
+        expect(() => fingerprintGenerator.getFingerprint({
+            devices: ['mobile'],
+            operatingSystems: ['windows'],
+            strict: true,
+        })).toThrow();
+    });
+
+    test.only('[relaxation] strict mode works with fp-only features', () => {
+        const fingerprintGenerator = new FingerprintGenerator();
+
+        expect(fingerprintGenerator.getFingerprint({
+            screen: {
+                minHeight: 9999,
+            },
+        })).toBeDefined();
+
+        expect(() => fingerprintGenerator.getFingerprint({
+            screen: {
+                minHeight: 9999,
+            },
+            strict: true,
+        })).toThrow();
+    });
 });
diff --git a/test/header-generator/generation.test.ts b/test/header-generator/generation.test.ts
@@ -156,12 +156,13 @@ describe('Generation tests', () => {
         expect(/phone|android|mobile/i.test(headers['user-agent'])).toBeTruthy();
     });
 
-    test('Throws an error when nothing can be generated', () => {
+    test('Strict mode throws an error when nothing can be generated', () => {
         try {
             headerGenerator.getHeaders({
                 browsers: [{
                     name: 'non-existing-browser',
                 }],
+                strict: true,
             } as unknown as HeaderGeneratorOptions);
             fail("HeaderGenerator didn't throw an error when trying to generate headers for a nonexisting browser.");
         } catch (error) {
@@ -172,6 +173,13 @@ describe('Generation tests', () => {
         }
     });
 
+    test('Default mode generates an approximately good header', () => {
+        expect(() => headerGenerator.getHeaders({
+            devices: ['mobile'],
+            operatingSystems: ['windows'],
+        } as unknown as HeaderGeneratorOptions)).not.toThrow();
+    });
+
     test('Supports browserListQuery generation', () => {
         const headers = headerGenerator.getHeaders({
             browserListQuery: 'last 15 firefox versions',