Improve wheels and sdist (#236)

- Move non-package files out of curl_cffi directory - Consolidate library download and build to build.py - Store building targets to a json file --------- Co-authored-by: Yifei Kong <[email protected]>
lexiforest · Feb 5, 2024 · 28cf8ba · 28cf8ba
1 parent 0c7ad76
commit 28cf8ba
Show file tree

Hide file tree

Showing 6 changed files with 148 additions and 97 deletions.
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,4 +1,7 @@
+recursive-include tests *
+
 include ffi/*
-include curl_cffi/_wrapper.*
-include curl_cffi/include/curl/*
+include include/curl/*
 include scripts/build.py
+include Makefile
+include libs.json
diff --git a/Makefile b/Makefile
@@ -18,8 +18,8 @@ curl-impersonate-$(VERSION)/chrome/patches: $(CURL_VERSION)
 	for p in $</curl-*.patch; do patch -p1 < ../$$p; done
 	# Re-generate the configure script
 	autoreconf -fi
-	mkdir -p ../curl_cffi/include/curl
-	cp -R include/curl/* ../curl_cffi/include/curl/
+	mkdir -p ../include/curl
+	cp -R include/curl/* ../include/curl/
 	# Sentinel files: https://tech.davis-hansson.com/p/make/
 	touch .preprocessed
 
@@ -44,6 +44,6 @@ clean:
 	rm -rf build/ dist/ curl_cffi.egg-info/ $(CURL_VERSION)/ curl-impersonate-$(VERSION)/
 	rm -rf curl_cffi/*.o curl_cffi/*.so curl_cffi/_wrapper.c
 	rm -rf .preprocessed $(CURL_VERSION).tar.xz curl-impersonate-$(VERSION).tar.gz
-	rm -rf curl_cffi/include/
+	rm -rf include/
 
 .PHONY: clean build test install-editable preprocess gen-const
diff --git a/libs.json b/libs.json
@@ -0,0 +1,74 @@
+[
+    {
+        "system": "Windows",
+        "machine": "AMD64",
+        "pointer_size": 64,
+        "libdir": "./lib64",
+        "sysname": "win32",
+        "so_name": "libcurl.dll",
+        "so_arch": "x86_64"
+    },
+    {
+        "system": "Windows",
+        "machine": "AMD64",
+        "pointer_size": 32,
+        "libdir": "./lib32",
+        "sysname": "win32",
+        "so_name": "libcurl.dll",
+        "so_arch": "i686"
+    },
+    {
+        "system": "Darwin",
+        "machine": "x86_64",
+        "pointer_size": 64,
+        "libdir": "/Users/runner/work/_temp/install/lib",
+        "sysname": "macos",
+        "so_name": "libcurl-impersonate-chrome.4.dylib",
+        "so_arch": "x86_64"
+    },
+    {
+        "system": "Darwin",
+        "machine": "arm64",
+        "pointer_size": 64,
+        "libdir": "/Users/runner/work/_temp/install/lib",
+        "sysname": "macos",
+        "so_name": "libcurl-impersonate-chrome.4.dylib",
+        "so_arch": "arm64"
+    },
+    {
+        "system": "Linux",
+        "machine": "x86_64",
+        "pointer_size": 64,
+        "libdir": "~/.local/lib",
+        "sysname": "linux-gnu",
+        "so_name": "libcurl-impersonate-chrome.so",
+        "so_arch": "x86_64"
+    },
+    {
+        "system": "Linux",
+        "machine": "aarch64",
+        "pointer_size": 64,
+        "libdir": "~/.local/lib",
+        "sysname": "linux-gnu",
+        "so_name": "libcurl-impersonate-chrome.so",
+        "so_arch": "aarch64"
+    },
+    {
+        "system": "Linux",
+        "machine": "armv6l",
+        "pointer_size": 32,
+        "libdir": "~/.local/lib",
+        "sysname": "linux-gnu",
+        "so_name": "libcurl-impersonate-chrome.so",
+        "so_arch": "armv6l"
+    },
+    {
+        "system": "Linux",
+        "machine": "armv7l",
+        "pointer_size": 32,
+        "libdir": "~/.local/lib",
+        "sysname": "linux-gnu",
+        "so_name": "libcurl-impersonate-chrome.so",
+        "so_arch": "armv7l"
+    }
+]
diff --git a/pyproject.toml b/pyproject.toml
@@ -5,7 +5,7 @@ authors = [{ name = "Yifei Kong", email = "[email protected]" }]
 description = "libcurl ffi bindings for Python, with impersonation support"
 license = { file = "LICENSE" }
 dependencies = [
-    "cffi >=1.12.0",
+    "cffi>=1.12.0",
     "certifi",
 ]
 readme = "README.md"
@@ -27,14 +27,12 @@ classifiers = [
 [project.optional-dependencies]
 dev = [
     "autoflake==1.4",
-    "black==22.8.0",
     "coverage==6.4.1",
     "cryptography==38.0.3",
     "flake8==6.0.0",
     "flake8-bugbear==22.7.1",
     "flake8-pie==0.15.0",
     "httpx==0.23.1",
-    "isort==5.10.1",
     "mypy==0.971",
     "types-certifi==2021.10.8.2",
     "pytest==7.1.2",
@@ -49,7 +47,7 @@ dev = [
 ]
 build = [
     "cibuildwheel",
-    "wheel"
+    "wheel",
 ]
 test = [
     "cryptography==38.0.3",
@@ -68,20 +66,15 @@ test = [
     "fastapi==0.100.0",
 ]
 
-[tool.ruff.lint]
-# Enable the isort rules.
-extend-select = ["I"]
+
+[build-system]
+requires = ["wheel", "setuptools", "cffi>=1.12.0"]
+build-backend = "setuptools.build_meta"
 
 
 [tool.setuptools]
 packages = ["curl_cffi", "curl_cffi.requests"]
 package-data = { curl_cffi = ["libcurl.dll"] }
-# include-package-data = true
-
-
-[build-system]
-requires = ["wheel", "setuptools", "cffi>=1.12.0"]
-build-backend = "setuptools.build_meta"
 
 
 [tool.cibuildwheel]
@@ -103,6 +96,7 @@ test-command = "python -bb -m pytest {project}/tests/unittest"
 test-extras = ["test"]
 test-skip = "pp*"
 
+
 # configure cibuildwheel to build native archs ('auto'), and some emulated ones
 [tool.cibuildwheel.linux]
 archs = ["auto", "aarch64"]
@@ -117,3 +111,8 @@ before-all = "gmake preprocess"
 [tool.pytest.ini_options]
 # pythonpath = [ "." ]
 asyncio_mode = "auto"
+
+
+[tool.ruff.lint]
+# Enable the isort rules.
+extend-select = ["I"]
diff --git a/scripts/build.py b/scripts/build.py
@@ -1,46 +1,82 @@
+import json
 import os
-import struct
 import platform
+import shutil
+import struct
 from pathlib import Path
+from urllib.request import urlretrieve
 
 from cffi import FFI
 
-ffibuilder = FFI()
-# arch = "%s-%s" % (os.uname().sysname, os.uname().machine)
-uname = platform.uname()
+__version__ = "0.6.0b9"
+
+
+def detect_arch():
+    with open(Path(__file__).parent.parent / "libs.json") as f:
+        archs = json.loads(f.read())
+    uname = platform.uname()
+    pointer_size = struct.calcsize("P") * 8
+    for arch in archs:
+        if (
+            arch["system"] == uname.system
+            and arch["machine"] == uname.machine
+            and arch["pointer_size"] == pointer_size
+        ):
+            arch["libdir"] = os.path.expanduser(arch["libdir"])
+            return arch
+    raise Exception(f"Unsupported arch: {uname}")
+
+
+arch = detect_arch()
 
+def download_so():
+    if (Path(arch["libdir"]) / arch["so_name"]).exists():
+        print(".so files alreay downloaded.")
+        return
 
-if uname.system == "Windows":
-    if struct.calcsize("P") * 8 == 64:
-        libdir = "./lib64"
-    else:
-        libdir = "./lib32"
-elif uname.system == "Darwin":
-    libdir = "/Users/runner/work/_temp/install/lib"
-else:
-    libdir = os.path.expanduser("~/.local/lib")
+    file = "libcurl-impersonate.tar.gz"
+    url = (
+        f"https://github.com/yifeikong/curl-impersonate/releases/download/"
+        f"v{__version__}/libcurl-impersonate-v{__version__}"
+        f".{arch['so_arch']}-{arch['sysname']}.tar.gz"
+    )
 
+    print(f"Downloading libcurl-impersonate-chrome from {url}...")
+    urlretrieve(url, file)
+
+    print("Unpacking downloaded files...")
+    os.makedirs(arch["libdir"], exist_ok=True)
+    shutil.unpack_archive(file, arch["libdir"])
+
+    if arch["system"] == "Windows":
+        shutil.copy2(f"{arch['libdir']}/libcurl.dll", "curl_cffi")
+
+
+ffibuilder = FFI()
+system = platform.system()
 root_dir = Path(__file__).parent.parent
+download_so()
+
 
 ffibuilder.set_source(
     "curl_cffi._wrapper",
     """
         #include "shim.h"
     """,
-    libraries=["curl-impersonate-chrome"] if uname.system != "Windows" else ["libcurl"],
-    library_dirs=[libdir],
+    # FIXME from `curl-impersonate`
+    libraries=["curl-impersonate-chrome"] if system != "Windows" else ["libcurl"],
+    library_dirs=[arch["libdir"]],
     source_extension=".c",
     include_dirs=[
-        str(root_dir / "curl_cffi/include"),
+        str(root_dir / "include"),
         str(root_dir / "ffi"),
     ],
     sources=[
         str(root_dir / "ffi/shim.c"),
     ],
     extra_compile_args=(
-        ["-Wno-implicit-function-declaration"] if uname.system == "Darwin" else []
+        ["-Wno-implicit-function-declaration"] if system == "Darwin" else []
     ),
-    # extra_link_args=["-Wl,-rpath,$ORIGIN/../libcurl/" + arch],
 )
 
 with open(root_dir / "ffi/cdef.c") as f:

diff --git a/setup.py b/setup.py
@@ -1,15 +1,5 @@
-import os
-import platform
-import shutil
-import struct
-from pathlib import Path
 from setuptools import setup
-from urllib.request import urlretrieve
 from wheel.bdist_wheel import bdist_wheel
-from distutils.command.build import build
-
-
-__version__ = "0.6.0b9"
 
 
 class bdist_wheel_abi3(bdist_wheel):
@@ -23,61 +13,10 @@ def get_tag(self):
         return python, abi, plat
 
 
-def download_so():
-    uname = platform.uname()
-    machine = uname.machine
-
-    # do not download if target platfrom dll not found
-
-    if uname.system == "Windows":
-        sysname = "win32"
-        if struct.calcsize("P") * 8 == 64:
-            machine = "x86_64"
-            libdir = "./lib64"
-        else:
-            machine = "i686"
-            libdir = "./lib32"
-        so_name = "libcurl.dll"
-    elif uname.system == "Darwin":
-        sysname = "macos"
-        libdir = "/Users/runner/work/_temp/install/lib"
-        so_name = "libcurl-impersonate-chrome.4.dylib"
-    else:
-        sysname = "linux-gnu"
-        libdir = os.path.expanduser("~/.local/lib")
-        so_name = "libcurl-impersonate-chrome.so"
-
-    if (Path(libdir) / so_name).exists():
-        print(".so files alreay downloaded.")
-        return
-    file = "libcurl-impersonate.tar.gz"
-    url = (
-        f"https://github.com/yifeikong/curl-impersonate/releases/download/"
-        f"v{__version__}/libcurl-impersonate-v{__version__}"
-        f".{machine}-{sysname}.tar.gz"
-    )
-
-    print(f"Downloading libcurl-impersonate-chrome from {url}...")
-    urlretrieve(url, file)
-
-    print("Unpacking downloaded files...")
-    os.makedirs(libdir, exist_ok=True)
-    shutil.unpack_archive(file, libdir)
-    if uname.system == "Windows":
-        shutil.copy2(f"{libdir}/libcurl.dll", "curl_cffi")
-
-
-class my_build(build):
-    def run(self):
-        download_so()
-        super().run()
-
-
 setup(
     # this option is only valid in setup.py
     cffi_modules=["scripts/build.py:ffibuilder"],
     cmdclass={
         "bdist_wheel": bdist_wheel_abi3,  # type: ignore
-        "build": my_build,  # type: ignore
     },
 )