Skip to content

Commit

Permalink
feat: add better type hints for impersonate and session
Browse files Browse the repository at this point in the history
  • Loading branch information
vvanglro committed Jul 29, 2024
1 parent 0cf03fe commit bdb963f
Show file tree
Hide file tree
Showing 5 changed files with 89 additions and 50 deletions.
2 changes: 1 addition & 1 deletion curl_cffi/requests/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ def request(
referer: Optional[str] = None,
accept_encoding: Optional[str] = "gzip, deflate, br, zstd",
content_callback: Optional[Callable] = None,
impersonate: Optional[Union[str, BrowserType]] = None,
impersonate: Optional[BrowserType] = None,
ja3: Optional[str] = None,
akamai: Optional[str] = None,
extra_fp: Optional[Union[ExtraFingerprints, ExtraFpDict]] = None,
Expand Down
115 changes: 77 additions & 38 deletions curl_cffi/requests/impersonate.py
Original file line number Diff line number Diff line change
@@ -1,49 +1,61 @@
from dataclasses import dataclass
from typing import List, Literal, Optional, TypedDict
from typing import List, Literal, Optional, TypedDict, TypeAlias, Tuple, Union, Callable
import warnings
from enum import Enum

from ..const import CurlSslVersion, CurlOpt
from curl_cffi.requests import HeaderTypes, CookieTypes, ProxySpec

from curl_cffi.const import CurlSslVersion, CurlOpt, CurlHttpVersion

class BrowserType(str, Enum):
edge99 = "edge99"
edge101 = "edge101"
chrome99 = "chrome99"
chrome100 = "chrome100"
chrome101 = "chrome101"
chrome104 = "chrome104"
chrome107 = "chrome107"
chrome110 = "chrome110"
chrome116 = "chrome116"
chrome119 = "chrome119"
chrome120 = "chrome120"
chrome123 = "chrome123"
chrome124 = "chrome124"
chrome99_android = "chrome99_android"
safari15_3 = "safari15_3"
safari15_5 = "safari15_5"
safari17_0 = "safari17_0"
safari17_2_ios = "safari17_2_ios"
BrowserType: TypeAlias = Literal[
# Edge
"edge99",
"edge101",
# Chrome
"chrome99",
"chrome100",
"chrome101",
"chrome104",
"chrome107",
"chrome110",
"chrome116",
"chrome119",
"chrome120",
"chrome123",
"chrome124",
"chrome99_android",
# Safari
"safari15_3",
"safari15_5",
"safari17_0",
"safari17_2_ios",
# alias
"chrome",
"edge",
"safari",
"safari_ios",
"chrome_android",
]

chrome = "chrome124"
safari = "safari17_0"
safari_ios = "safari17_2_ios"
DefaultChrome = "chrome124"
DefaultEdge = "edge101"
DefaultSafari = "safari17_0"
DefaultSafariPhone = "safari17_2_ios"
DefaultChromePhone = "chrome99_android"

@classmethod
def has(cls, item):
return item in cls.__members__

@classmethod
def normalize(cls, item):
if item == "chrome": # noqa: SIM116
return cls.chrome
elif item == "safari":
return cls.safari
elif item == "safari_ios":
return cls.safari_ios
else:
return item
def normalize_browser_type(item):
if item == "chrome": # noqa: SIM116
return DefaultChrome
elif item == "edge":
return DefaultEdge
elif item == "safari":
return DefaultSafari
elif item == "safari_ios":
return DefaultSafariPhone
elif item == "chrome_android":
return DefaultChromePhone
else:
return item


@dataclass
Expand All @@ -67,6 +79,33 @@ class ExtraFpDict(TypedDict, total=False):
http2_stream_exclusive: int


class BaseSessionParams(TypedDict, total=False):
headers: Optional[HeaderTypes]
cookies: Optional[CookieTypes]
auth: Optional[Tuple[str, str]]
proxies: Optional[ProxySpec]
proxy: Optional[str]
proxy_auth: Optional[Tuple[str, str]]
base_url: Optional[str]
params: Optional[dict]
verify: bool
timeout: Union[float, Tuple[float, float]]
trust_env: bool
allow_redirects: bool
max_redirects: int
impersonate: Optional[BrowserType]
ja3: Optional[str]
akamai: Optional[str]
extra_fp: Optional[Union[ExtraFingerprints, ExtraFpDict]]
default_headers: bool
default_encoding: Union[str, Callable[[bytes], str]]
curl_options: Optional[dict]
curl_infos: Optional[list]
http_version: Optional[CurlHttpVersion]
debug: bool
interface: Optional[str]
cert: Optional[Union[str, Tuple[str, str]]]

# TLS version are in the format of 0xAABB, where AA is major version and BB is minor
# version. As of today, the major version is always 03.
TLS_VERSION_MAP = {
Expand Down
18 changes: 9 additions & 9 deletions curl_cffi/requests/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
Tuple,
TypedDict,
Union,
cast,
cast, Unpack,
)
from urllib.parse import ParseResult, parse_qsl, unquote, urlencode, urljoin, urlparse

Expand All @@ -37,7 +37,7 @@
BrowserType,
TLS_VERSION_MAP,
TLS_CIPHER_NAME_MAP,
TLS_EC_CURVES_MAP
TLS_EC_CURVES_MAP, normalize_browser_type, BaseSessionParams
)
from .models import Request, Response
from .websockets import WebSocket
Expand Down Expand Up @@ -177,7 +177,7 @@ def __init__(
trust_env: bool = True,
allow_redirects: bool = True,
max_redirects: int = 30,
impersonate: Optional[Union[str, BrowserType]] = None,
impersonate: Optional[BrowserType] = None,
ja3: Optional[str] = None,
akamai: Optional[str] = None,
extra_fp: Optional[Union[ExtraFingerprints, ExtraFpDict]] = None,
Expand Down Expand Up @@ -334,7 +334,7 @@ def _set_curl_options(
referer: Optional[str] = None,
accept_encoding: Optional[str] = "gzip, deflate, br, zstd",
content_callback: Optional[Callable] = None,
impersonate: Optional[Union[str, BrowserType]] = None,
impersonate: Optional[BrowserType] = None,
ja3: Optional[str] = None,
akamai: Optional[str] = None,
extra_fp: Optional[Union[ExtraFingerprints, ExtraFpDict]] = None,
Expand Down Expand Up @@ -577,7 +577,7 @@ def _set_curl_options(
impersonate = impersonate or self.impersonate
default_headers = self.default_headers if default_headers is None else default_headers
if impersonate:
impersonate = BrowserType.normalize(impersonate)
impersonate = normalize_browser_type(impersonate)
ret = c.impersonate(impersonate, default_headers=default_headers)
if ret != 0:
raise RequestsError(f"Impersonating {impersonate} is not supported")
Expand Down Expand Up @@ -716,7 +716,7 @@ def __init__(
curl: Optional[Curl] = None,
thread: Optional[ThreadType] = None,
use_thread_local_curl: bool = True,
**kwargs,
**kwargs: Unpack[BaseSessionParams],
):
"""
Parameters set in the init method will be override by the same parameter in request method.
Expand Down Expand Up @@ -875,7 +875,7 @@ def request(
referer: Optional[str] = None,
accept_encoding: Optional[str] = "gzip, deflate, br",
content_callback: Optional[Callable] = None,
impersonate: Optional[Union[str, BrowserType]] = None,
impersonate: Optional[BrowserType] = None,
ja3: Optional[str] = None,
akamai: Optional[str] = None,
extra_fp: Optional[Union[ExtraFingerprints, ExtraFpDict]] = None,
Expand Down Expand Up @@ -1013,7 +1013,7 @@ def __init__(
loop=None,
async_curl: Optional[AsyncCurl] = None,
max_clients: int = 10,
**kwargs,
**kwargs: Unpack[BaseSessionParams],
):
"""
Parameters set in the init method will be override by the same parameter in request method.
Expand Down Expand Up @@ -1167,7 +1167,7 @@ async def request(
referer: Optional[str] = None,
accept_encoding: Optional[str] = "gzip, deflate, br",
content_callback: Optional[Callable] = None,
impersonate: Optional[Union[str, BrowserType]] = None,
impersonate: Optional[BrowserType] = None,
ja3: Optional[str] = None,
akamai: Optional[str] = None,
extra_fp: Optional[Union[ExtraFingerprints, ExtraFpDict]] = None,
Expand Down
2 changes: 1 addition & 1 deletion tests/integration/test_fingerprints.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def test_impersonate():


def test_impersonate_edge():
r = requests.get(JA3_URL, impersonate=requests.BrowserType.edge101)
r = requests.get(JA3_URL, impersonate="edge101")
assert r.json()["ja3_hash"] == EDGE_JA3_HASH


Expand Down
2 changes: 1 addition & 1 deletion tests/unittest/test_smoke.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def test_without_impersonate():

def test_with_impersonate():
for url in URLS:
r = requests.get(url, impersonate=requests.BrowserType.chrome)
r = requests.get(url, impersonate="chrome")
assert r.status_code == 200


Expand Down

0 comments on commit bdb963f

Please sign in to comment.