Merge pull request #12 from joemarshall/main

patched requests at lower level

.github/workflows/main.yml

@@ -11,7 +11,7 @@ on:
 
 jobs:
   # Build for pyodide 0.21.0
-  Pyodide_build:
+  Pyodide_test_and_build:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
@@ -22,9 +22,9 @@ jobs:
       with:
         version: 3.1.14
     - run: pip install pyodide-build==0.21.0
-    - run: pyodide build
+    - run: pyodide build    
     - uses: actions/upload-artifact@v3
       with:
         name: pyodide wheel
         path: dist
-
+    - run: bash ./setup_test_env.sh

.gitignore

@@ -1,2 +1,7 @@
 .*
 !.github
+tests/pyodide
+build
+dist
+__pycache__
+*.egg-info/

pyodide_http/_core.py

@@ -90,15 +90,16 @@ def send(request: Request, stream: bool = False) -> Response:
     # XMLHttpRequest https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/timeout
     if _IN_WORKER and request.timeout!=0:
         xhr.timeout=int(request.timeout*1000)
-    for name, value in request.headers.items():
-        xhr.setRequestHeader(name, value)
 
     if _IN_WORKER:
         xhr.responseType = "arraybuffer"
     else:
         xhr.overrideMimeType('text/plain; charset=ISO-8859-15')
 
     xhr.open(request.method, request.url, False)
+    for name, value in request.headers.items():
+        xhr.setRequestHeader(name, value)
+
     xhr.send(request.body)
 
     headers = dict(Parser().parsestr(xhr.getAllResponseHeaders()))

pyodide_http/_requests.py

@@ -1,6 +1,6 @@
 from io import BytesIO, IOBase
 
-import requests
+from requests.adapters import BaseAdapter
 from requests.utils import get_encoding_from_headers, CaseInsensitiveDict
 
 from ._core import Request, send
@@ -9,30 +9,46 @@
 _IS_PATCHED = False
 
 
-class Session:
-    def __enter__(self):
-        return self
+class PyodideHTTPAdapter(BaseAdapter):
+    """The Base Transport Adapter"""
 
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        pass
+    def __init__(self):
+        super().__init__()
 
-    @staticmethod
-    def request(method, url, **kwargs):
+    def send(
+        self, request, **kwargs
+    ):
+        """Sends PreparedRequest object. Returns Response object.
+        :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
+        :param stream: (optional) Whether to stream the request content.
+        :param timeout: (optional) How long to wait for the server to send
+            data before giving up, as a float, or a :ref:`(connect timeout,
+            read timeout) <timeouts>` tuple.
+        :type timeout: float or tuple
+        :param verify: (optional) Either a boolean, in which case it controls whether we verify
+            the server's TLS certificate, or a string, in which case it must be a path
+            to a CA bundle to use
+        :param cert: (optional) Any user-provided SSL certificate to be trusted.
+        :param proxies: (optional) The proxies dictionary to apply to the request.
+        """
         stream = kwargs.get('stream', False)
-        request = Request(method, url)
-        request.timeout=kwargs.get('timeout',0)
-        request.params=kwargs.get('params',None)
-        request.headers = kwargs.get('headers', {})
-        if 'json' in kwargs:
-            request.set_json(kwargs['json'])
+        pyodide_request = Request(request.method, request.url)
+        pyodide_request.timeout=kwargs.get('timeout',0)
+        if not pyodide_request.timeout:
+            pyodide_request.timeout=0
+        pyodide_request.params=None # this is done in preparing request now
+        pyodide_request.headers = dict(request.headers)
+        if request.body:
+            pyodide_request.set_body(request.body)
         try:
-            resp = send(request, stream)
+            resp = send(pyodide_request, stream)
         except _StreamingTimeout:
             from requests import ConnectTimeout
-            raise ConnectTimeout(request=request)
+            raise ConnectTimeout(request=pyodide_request)
         except _StreamingError:
             from requests import ConnectionError
-            raise ConnectionError(request=request)
+            raise ConnectionError(request=pyodide_request)
+        import requests
         response = requests.Response()
         # Fallback to None if there's no status_code, for whatever reason.
         response.status_code = getattr(resp, "status_code", None)
@@ -46,11 +62,26 @@ def request(method, url, **kwargs):
         else:
             # non-streaming response, make it look like a stream
             response.raw = BytesIO(resp.body)
+
+        def new_read(self,amt=None,decode_content=False,cache_content=False):
+            return self.old_read(amt)
+
+        # make the response stream look like a urllib3 stream
+        response.raw.old_read=response.raw.read
+        response.raw.read=new_read.__get__(response.raw,type(response.raw))
+
+
         response.reason = ''
-        response.url = url
+        response.url = request.url
         return response
 
 
+
+    def close(self):
+        """Cleans up adapter specific items."""
+        pass
+
+
 def patch():
     global _IS_PATCHED
     """
@@ -62,9 +93,14 @@ def patch():
     if _IS_PATCHED:
         return
 
-    class Sessions:
-        Session = Session
+    import requests
+    requests.sessions.Session._old_init=requests.sessions.Session.__init__
+    def new_init(self):
+        self._old_init()
+        self.mount("https://", PyodideHTTPAdapter())
+        self.mount("http://", PyodideHTTPAdapter())        
 
-    requests.api.sessions = Sessions()
+    requests.sessions.Session._old_init=requests.sessions.Session.__init__
+    requests.sessions.Session.__init__=new_init
 
     _IS_PATCHED = True

pyodide_http/_streaming.py

@@ -102,7 +102,7 @@
             // return the headers first via textencoder
             var headers = [];
             for (const pair of response.headers.entries()) {
-                headers.push([pair[0], pair[0]]);
+                headers.push([pair[0], pair[1]]);
             }
             headerObj = { headers: headers, status: response.status, connectionID };
             const headerText = JSON.stringify(headerObj);
@@ -201,19 +201,20 @@ def __init__(self):
 
     def send(self, request):
         from ._core import Response
-        headers = _obj_from_dict(request.headers)
+        headers = request.headers
         body = request.body
-        fetch_data = _obj_from_dict(
-            {"headers": headers, "body": body, "method": request.method})
+        fetch_data = {"headers": headers, "body": body, "method": request.method}
         # start the request off in the worker
-        timeout=int(1000*timeout) if request.timeout>0 else None
+        timeout=int(1000*request.timeout) if request.timeout>0 else None
         shared_buffer = js.SharedArrayBuffer.new(1048576)
         int_buffer = js.Int32Array.new(shared_buffer)
         byte_buffer = js.Uint8Array.new(shared_buffer, 8)
 
         js.Atomics.store(int_buffer, 0, 0)
         js.Atomics.notify(int_buffer, 0)
         absolute_url = js.URL.new(request.url, js.location).href
+        js.console.log(_obj_from_dict(
+            {"buffer": shared_buffer, "url": absolute_url, "fetchParams": fetch_data}))
         self._worker.postMessage(_obj_from_dict(
             {"buffer": shared_buffer, "url": absolute_url, "fetchParams": fetch_data}))
         # wait for the worker to send something

setup_test_env.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+
+
+# install chrome
+wget -nc https://dl-ssl.google.com/linux/linux_signing_key.pub 
+cat linux_signing_key.pub | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/linux_signing_key.gpg  >/dev/null 
+sudo sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/chrome.list' 
+sudo apt update 
+sudo apt install google-chrome-stable 
+
+
+# pyodide build and test
+pip install pytest 
+pip install pyodide-build pytest-pyodide 
+# install chromedriver stuff for selenium to control chrome
+pip install selenium webdriver-manager 
+pip install chromedriver-binary-auto
+# make sure chromedriver is on path
+export PATH=$PATH:`chromedriver-path`
+
+# run the tests
+cd tests
+# get pyodide to tests/pyodide
+wget https://github.com/pyodide/pyodide/releases/download/0.21.0/pyodide-build-0.21.0.tar.bz2
+tar xjf pyodide-build-0.21.0.tar.bz2
+pytest . --dist-dir ./pyodide --rt chrome -v 

tests/test_non_streaming.py

@@ -0,0 +1,73 @@
+from pathlib import Path
+import glob
+
+from pytest_pyodide import run_in_pyodide, spawn_web_server
+from pytest import fixture
+
+@fixture(scope="module")
+def dist_dir(request):
+    # return pyodide dist dir relative to top level of webserver
+    p=Path(request.config.getoption('--dist-dir')).resolve()
+    p=p.relative_to(Path(__file__).parent.parent)
+    return p
+
+@fixture(scope="module")
+def web_server_base():
+    server_folder=Path(__file__).parent.parent
+    with spawn_web_server(server_folder) as server:
+        server_hostname, server_port, _ = server
+        base_url=f"http://{server_hostname}:{server_port}/"
+        yield base_url
+
+def _install_package(selenium,base_url):
+    wheel_folder=Path(__file__).parent.parent / "dist"
+
+    selenium.run_js(
+        f"""
+        await pyodide.loadPackage("micropip");
+        """
+    )
+    selenium.run_async(f'import micropip\nawait micropip.install("requests")')
+
+    for wheel in wheel_folder.glob("*.whl"):
+        url = base_url +"dist/" + str(wheel.name)
+        selenium.run_async(f'await micropip.install("{url}")')
+
+        selenium.run(
+            """
+            import pyodide_http
+            pyodide_http.patch_all()
+            import requests
+            """
+        )
+
+
+def test_install_package(selenium_standalone,web_server_base):
+    _install_package(selenium_standalone,web_server_base)
+
+
+def test_requests_get(selenium_standalone,dist_dir,web_server_base):
+    _install_package(selenium_standalone,web_server_base)
+
+    @run_in_pyodide
+    def test_fn(selenium_standalone,base_url):
+        import requests
+        print("get:",base_url)
+        resp=requests.get(f"{base_url}/yt-4.0.4-cp310-cp310-emscripten_3_1_14_wasm32.whl")
+        data=resp.content
+        return len(data)
+
+    assert test_fn(selenium_standalone,f"{web_server_base}{dist_dir}/")==11373926
+
+def test_requests_404(selenium_standalone,dist_dir,web_server_base):
+    _install_package(selenium_standalone,web_server_base)
+
+    @run_in_pyodide
+    def test_fn(selenium_standalone,base_url):
+        import requests
+        print("get:",base_url)
+        resp=requests.get(f"{base_url}/surely_this_file_does_not_exist.hopefully.")
+        response=resp.status_code
+        return response
+
+    assert test_fn(selenium_standalone,f"{web_server_base}{dist_dir}/")==404