feat: Integrate browserforge fingerprints

Makefile

@@ -10,6 +10,7 @@ install-dev:
 	poetry install --all-extras
 	poetry run pre-commit install
 	poetry run playwright install
+	poetry run python -m browserforge update
 
 build:
 	poetry build --no-interaction -vv

docs/examples/code/playwright_crawler_with_fingerprint_generator.py

@@ -0,0 +1,40 @@
+import asyncio
+
+from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
+from crawlee.fingerprint_suite import DefaultFingerprintGenerator, HeaderGeneratorOptions, ScreenOptions
+
+
+async def main() -> None:
+    # Use default fingerprint generator with desired fingerprint options.
+    # Generator will try to generate real looking browser fingerprint based on the options.
+    # Unspecified fingerprint options will be automatically selected by the generator.
+    fingerprint_generator = DefaultFingerprintGenerator(
+        header_options=HeaderGeneratorOptions(browsers=['chromium']),
+        screen_options=ScreenOptions(min_width=400),
+    )
+
+    crawler = PlaywrightCrawler(
+        # Limit the crawl to max requests. Remove or increase it for crawling all links.
+        max_requests_per_crawl=10,
+        # Headless mode, set to False to see the browser in action.
+        headless=False,
+        # Browser types supported by Playwright.
+        browser_type='chromium',
+        # Fingerprint generator to be used. By default no fingerprint generation is done.
+        fingerprint_generator=fingerprint_generator,
+    )
+
+    # Define the default request handler, which will be called for every request.
+    @crawler.router.default_handler
+    async def request_handler(context: PlaywrightCrawlingContext) -> None:
+        context.log.info(f'Processing {context.request.url} ...')
+
+        # Find a link to the next page and enqueue it if it exists.
+        await context.enqueue_links(selector='.morelink')
+
+    # Run the crawler with the initial list of URLs.
+    await crawler.run(['https://news.ycombinator.com/'])
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/examples/playwright_crawler_with_fingerprint_generator.mdx

@@ -0,0 +1,17 @@
+---
+id: playwright-crawler-with-fingeprint-generator
+title: Playwright crawler with fingerprint generator
+---
+
+import ApiLink from '@site/src/components/ApiLink';
+import CodeBlock from '@theme/CodeBlock';
+
+import PlaywrightCrawlerExample from '!!raw-loader!./code/playwright_crawler_with_fingerprint_generator.py';
+
+This example demonstrates how to use <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> together with <ApiLink to="class/FingerprintGenerator">`FingerprintGenerator`</ApiLink> that will populate several browser attributes to mimic real browser fingerprint. To read more about fingerprints please see: https://docs.apify.com/academy/anti-scraping/techniques/fingerprinting.
+
+You can implement your own fingerprint generator or use <ApiLink to="class/BrowserforgeFingerprintGenerator">`DefaultFingerprintGenerator`</ApiLink>. To use the generator initialize it with the desired fingerprint options. The generator will try to create fingerprint based on those options. Unspecified options will be automatically selected by the generator from the set of reasonable values. If some option is important for you, do not rely on the default and explicitly define it.
+
+<CodeBlock className="language-python">
+    {PlaywrightCrawlerExample}
+</CodeBlock>

pyproject.toml

@@ -44,6 +44,7 @@ keywords = [
 python = "^3.9"
 apify = { version = ">=2.0.0", optional = true }
 beautifulsoup4 = { version = ">=4.12.0", optional = true }
+browserforge = { version = ">=1.2.3", optional = true }
 colorama = ">=0.4.0"
 cookiecutter = ">=2.6.0"
 curl-cffi = { version = ">=0.7.2", optional = true }
@@ -100,10 +101,10 @@ types-python-dateutil = "~2.9.0.20240316"
 # Support for re-using groups in other groups https://peps.python.org/pep-0735/ in poetry:
 # https://github.com/python-poetry/poetry/issues/9751
 adaptive-playwright = ["jaro-winkler", "playwright", "scikit-learn"]
-all = ["beautifulsoup4", "curl-cffi", "html5lib", "jaro-winkler", "lxml", "parsel", "playwright", "scikit-learn"]
+all = ["beautifulsoup4", "browserforge", "curl-cffi", "html5lib", "jaro-winkler", "lxml", "parsel", "playwright", "scikit-learn"]
 beautifulsoup = ["beautifulsoup4", "lxml", "html5lib"]
 curl-impersonate = ["curl-cffi"]
-playwright = ["playwright"]
+playwright = ["browserforge", "playwright"]
 parsel = ["parsel"]
 
 [tool.poetry.scripts]

src/crawlee/browsers/_browser_pool.py

@@ -23,6 +23,7 @@
     from types import TracebackType
 
     from crawlee.browsers._base_browser_plugin import BaseBrowserPlugin
+    from crawlee.fingerprint_suite import FingerprintGenerator
     from crawlee.proxy_configuration import ProxyInfo
 
 logger = getLogger(__name__)
@@ -103,6 +104,7 @@ def with_default_plugin(
         browser_launch_options: Mapping[str, Any] | None = None,
         browser_new_context_options: Mapping[str, Any] | None = None,
         headless: bool | None = None,
+        fingerprint_generator: FingerprintGenerator | None = None,
         use_incognito_pages: bool | None = False,
         **kwargs: Any,
     ) -> BrowserPool:
@@ -117,6 +119,8 @@ def with_default_plugin(
                 are provided directly to Playwright's `browser.new_context` method. For more details, refer to the
                 Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context.
             headless: Whether to run the browser in headless mode.
+            fingerprint_generator: An optional instance of implementation of `FingerprintGenerator` that is used
+                to generate browser fingerprints together with consistent headers.
             use_incognito_pages: By default pages share the same browser context. If set to True each page uses its
                 own context that is destroyed once the page is closed or crashes.
             kwargs: Additional arguments for default constructor.
@@ -134,7 +138,10 @@ def with_default_plugin(
         if browser_type:
             plugin_options['browser_type'] = browser_type
 
-        plugin = PlaywrightBrowserPlugin(**plugin_options)
+        plugin = PlaywrightBrowserPlugin(
+            **plugin_options,
+            fingerprint_generator=fingerprint_generator,
+        )
         return cls(plugins=[plugin], **kwargs)
 
     @property