prepare 6.13.2 release (#146)

Author: LaunchDarklyCI

ldclient/config.py

@@ -37,6 +37,7 @@ def __init__(self,
           variable, this is used regardless of whether the target URI is HTTP or HTTPS (the actual LaunchDarkly
           service uses HTTPS, but a Relay Proxy instance could use HTTP). Setting this Config parameter will
           override any proxy specified by an environment variable, but only for LaunchDarkly SDK connections.
+          The URL may contain authentication parameters in the form http://username:password@host:port.
         :param string ca_certs: If using a custom certificate authority, set this to the file path of the
           certificate bundle.
         :param string cert_file: If using a custom client certificate, set this to the file path of the

ldclient/impl/http.py

@@ -54,11 +54,18 @@ def create_pool_manager(self, num_pools, target_base_uri):
                 ca_certs=ca_certs
                 )
         else:
+            # Get proxy authentication, if provided
+            url = urllib3.util.parse_url(proxy_url)
+            proxy_headers = None
+            if url.auth != None:
+                proxy_headers = urllib3.util.make_headers(proxy_basic_auth=url.auth)
+            # Create a proxied connection
             return urllib3.ProxyManager(
                 proxy_url,
                 num_pools=num_pools,
                 cert_reqs=cert_reqs,
-                ca_certs = ca_certs
+                ca_certs = ca_certs,
+                proxy_headers=proxy_headers
             )
 
 def _get_proxy_url(target_base_uri):

testing/http_util.py

@@ -72,7 +72,11 @@ def await_request(self):
 
     def require_request(self):
         return self.requests.get(block=False)
-    
+
+    def wait_until_request_received(self):
+        req = self.requests.get()
+        self.requests.put(req)
+
     def should_have_requests(self, count):
         if self.requests.qsize() != count:
             rs = []

testing/proxy_test_util.py

@@ -0,0 +1,55 @@
+from ldclient.config import Config, HTTPConfig
+from testing.http_util import start_server, BasicResponse, JsonResponse
+
+# Runs tests of all of our supported proxy server configurations: secure or insecure, configured
+# by Config.http_proxy or by an environment variable, with or without authentication. The action
+# parameter is a function that takes three parameters: server, config, secure; the expectation is
+# that it causes an HTTP/HTTPS request to be made via the configured proxy. The caller must pass
+# in the monkeypatch fixture from pytest.
+def do_proxy_tests(action, action_method, monkeypatch):
+    # We'll test each permutation of use_env_vars, secure, and use_auth, except that if secure is
+    # true then we'll only test with use_auth=false because we don't have a way to test proxy
+    # authorization over HTTPS (even though we believe it works).
+    for (use_env_vars, secure, use_auth) in [
+        (False, False, False),
+        (False, False, True),
+        (False, True, False),
+        (True, False, False),
+        (True, False, True),
+        (True, True, False)]:
+        test_desc = "%s, %s, %s" % (
+            "using env vars" if use_env_vars else "using Config",
+            "secure" if secure else "insecure",
+            "with auth" if use_auth else "no auth")
+        with start_server() as server:
+            proxy_uri = server.uri.replace('http://', 'http://user:pass@') if use_auth else server.uri
+            target_uri = 'https://not-real' if secure else 'http://not-real'
+            if use_env_vars:
+                monkeypatch.setenv('https_proxy' if secure else 'http_proxy', proxy_uri)
+            config = Config(
+                sdk_key = 'sdk_key',
+                base_uri = target_uri,
+                events_uri = target_uri,
+                stream_uri = target_uri,
+                http = None if use_env_vars else HTTPConfig(http_proxy=proxy_uri),
+                diagnostic_opt_out = True)
+            try:
+                action(server, config, secure)
+            except:
+                print("test action failed (%s)" % test_desc)
+                raise
+            # For an insecure proxy request, our stub server behaves enough like the real thing to satisfy the
+            # HTTP client, so we should be able to see the request go through. Note that the URI path will
+            # actually be an absolute URI for a proxy request.
+            try:
+                req = server.require_request()
+            except:
+                print("server did not receive a request (%s)" % test_desc)
+                raise
+            expected_method = 'CONNECT' if secure else action_method
+            assert req.method == expected_method, "method should be %s, was %s (%s)" % (expected_method, req.method, test_desc)
+            if use_auth:
+                expected_auth = 'Basic dXNlcjpwYXNz'
+                actual_auth = req.headers.get('Proxy-Authorization')
+                assert actual_auth == expected_auth, "auth header should be %s, was %s (%s)" % (expected_auth, actual_auth, test_desc)
+            print("do_proxy_tests succeeded for: %s" % test_desc)

testing/test_event_processor.py

@@ -9,6 +9,7 @@
 from ldclient.event_processor import DefaultEventProcessor
 from ldclient.util import log
 from testing.http_util import start_server, BasicResponse
+from testing.proxy_test_util import do_proxy_tests
 from testing.stub_util import MockResponse, MockHttp
 
 
@@ -558,52 +559,13 @@ def start_consuming_events():
         assert message1.param == event1
         assert had_no_more
 
-def test_can_use_http_proxy_via_environment_var(monkeypatch):
-    with start_server() as server:
-        monkeypatch.setenv('http_proxy', server.uri)
-        config = Config(sdk_key = 'sdk-key', events_uri = 'http://not-real', diagnostic_opt_out = True)
-        _verify_http_proxy_is_used(server, config)
-
-def test_can_use_https_proxy_via_environment_var(monkeypatch):
-    with start_server() as server:
-        monkeypatch.setenv('https_proxy', server.uri)
-        config = Config(sdk_key = 'sdk-key', events_uri = 'https://not-real', diagnostic_opt_out = True)
-        _verify_https_proxy_is_used(server, config)
-
-def test_can_use_http_proxy_via_config():
-    with start_server() as server:
-        config = Config(sdk_key = 'sdk-key', events_uri = 'http://not-real', http_proxy=server.uri, diagnostic_opt_out = True)
-        _verify_http_proxy_is_used(server, config)
-
-def test_can_use_https_proxy_via_config():
-    with start_server() as server:
-        config = Config(sdk_key = 'sdk-key', events_uri = 'https://not-real', http_proxy=server.uri, diagnostic_opt_out = True)
-        _verify_https_proxy_is_used(server, config)
-
-def _verify_http_proxy_is_used(server, config):
-    server.for_path(config.events_uri + '/bulk', BasicResponse(200))
-    with DefaultEventProcessor(config) as ep:
-        ep.send_event({ 'kind': 'identify', 'user': user })
-        ep.flush()
-        ep._wait_until_inactive()
-
-        # For an insecure proxy request, our stub server behaves enough like the real thing to satisfy the
-        # HTTP client, so we should be able to see the request go through. Note that the URI path will
-        # actually be an absolute URI for a proxy request.
-        req = server.require_request()
-        assert req.method == 'POST'
-
-def _verify_https_proxy_is_used(server, config):
-    server.for_path(config.events_uri + '/bulk', BasicResponse(200))
-    with DefaultEventProcessor(config) as ep:
-        ep.send_event({ 'kind': 'identify', 'user': user })
-        ep.flush()
-        ep._wait_until_inactive()
-
-        # Our simple stub server implementation can't really do HTTPS proxying, so the request will fail, but
-        # it can still record that it *got* the request, which proves that the request went to the proxy.
-        req = server.require_request()
-        assert req.method == 'CONNECT'
+def test_http_proxy(monkeypatch):
+    def _event_processor_proxy_test(server, config, secure):
+        with DefaultEventProcessor(config) as ep:
+            ep.send_event({ 'kind': 'identify', 'user': user })
+            ep.flush()
+            ep._wait_until_inactive()
+    do_proxy_tests(_event_processor_proxy_test, 'POST', monkeypatch)
 
 def verify_unrecoverable_http_error(status):
     with DefaultTestProcessor(sdk_key = 'SDK_KEY') as ep:

testing/test_feature_requester.py

@@ -6,7 +6,7 @@
 from ldclient.version import VERSION
 from ldclient.versioned_data_kind import FEATURES, SEGMENTS
 from testing.http_util import start_server, BasicResponse, JsonResponse
-
+from testing.proxy_test_util import do_proxy_tests
 
 def test_get_all_data_returns_data():
     with start_server() as server:
@@ -102,54 +102,18 @@ def test_get_all_data_can_use_cached_data():
         req = server.require_request()
         assert req.headers['If-None-Match'] == etag2
 
-def test_can_use_http_proxy_via_environment_var(monkeypatch):
-    with start_server() as server:
-        monkeypatch.setenv('http_proxy', server.uri)
-        config = Config(sdk_key = 'sdk-key', base_uri = 'http://not-real')
-        _verify_http_proxy_is_used(server, config)
-
-def test_can_use_https_proxy_via_environment_var(monkeypatch):
-    with start_server() as server:
-        monkeypatch.setenv('https_proxy', server.uri)
-        config = Config(sdk_key = 'sdk-key', base_uri = 'https://not-real')
-        _verify_https_proxy_is_used(server, config)
-
-def test_can_use_http_proxy_via_config():
-    with start_server() as server:
-        config = Config(sdk_key = 'sdk-key', base_uri = 'http://not-real', http_proxy = server.uri)
-        _verify_http_proxy_is_used(server, config)
-
-def test_can_use_https_proxy_via_config():
-    with start_server() as server:
-        config = Config(sdk_key = 'sdk-key', base_uri = 'https://not-real', http_proxy = server.uri)
-        _verify_https_proxy_is_used(server, config)
-
-def _verify_http_proxy_is_used(server, config):
-    fr = FeatureRequesterImpl(config)
-
-    resp_data = { 'flags': {}, 'segments': {} }
-    expected_data = { FEATURES: {}, SEGMENTS: {} }
-    server.for_path(config.base_uri + '/sdk/latest-all', JsonResponse(resp_data))
-
-    # For an insecure proxy request, our stub server behaves enough like the real thing to satisfy the
-    # HTTP client, so we should be able to see the request go through. Note that the URI path will
-    # actually be an absolute URI for a proxy request.
-    result = fr.get_all_data()
-    assert result == expected_data
-    req = server.require_request()
-    assert req.method == 'GET'
-
-def _verify_https_proxy_is_used(server, config):
-    fr = FeatureRequesterImpl(config)
-
-    resp_data = { 'flags': {}, 'segments': {} }
-    server.for_path(config.base_uri + '/sdk/latest-all', JsonResponse(resp_data))
-
-    # Our simple stub server implementation can't really do HTTPS proxying, so the request will fail, but
-    # it can still record that it *got* the request, which proves that the request went to the proxy.
-    try:
-        fr.get_all_data()
-    except:
-        pass
-    req = server.require_request()
-    assert req.method == 'CONNECT'
+def test_http_proxy(monkeypatch):
+    def _feature_requester_proxy_test(server, config, secure):
+        resp_data = { 'flags': {}, 'segments': {} }
+        expected_data = { FEATURES: {}, SEGMENTS: {} }
+        server.for_path(config.base_uri + '/sdk/latest-all', JsonResponse(resp_data))
+        fr = FeatureRequesterImpl(config)
+        if secure:
+            try:
+                fr.get_all_data()
+            except:
+                pass # we expect this to fail because we don't have a real HTTPS proxy server
+        else:
+            result = fr.get_all_data()
+            assert result == expected_data
+    do_proxy_tests(_feature_requester_proxy_test, 'GET', monkeypatch)

testing/test_streaming.py

@@ -10,6 +10,7 @@
 from ldclient.version import VERSION
 from ldclient.versioned_data_kind import FEATURES, SEGMENTS
 from testing.http_util import start_server, BasicResponse, CauseNetworkError, SequentialHandler
+from testing.proxy_test_util import do_proxy_tests
 from testing.stub_util import make_delete_event, make_patch_event, make_put_event, stream_content
 
 brief_delay = 0.001
@@ -210,54 +211,24 @@ def test_unrecoverable_http_error(status):
                 assert not sp.initialized()
                 server.should_have_requests(1)
 
-def test_can_use_http_proxy_via_environment_var(monkeypatch):
-    with start_server() as server:
-        config = Config(sdk_key = 'sdk-key', stream_uri = 'http://not-real')
-        monkeypatch.setenv('http_proxy', server.uri)
-        _verify_http_proxy_is_used(server, config)
-
-def test_can_use_https_proxy_via_environment_var(monkeypatch):
-    with start_server() as server:
-        config = Config(sdk_key = 'sdk-key', stream_uri = 'https://not-real')
-        monkeypatch.setenv('https_proxy', server.uri)
-        _verify_https_proxy_is_used(server, config)
-
-def test_can_use_http_proxy_via_config():
-    with start_server() as server:
-        config = Config(sdk_key = 'sdk-key', stream_uri = 'http://not-real', http_proxy=server.uri)
-        _verify_http_proxy_is_used(server, config)
-
-def test_can_use_https_proxy_via_config():
-    with start_server() as server:
-        config = Config(sdk_key = 'sdk-key', stream_uri = 'https://not-real', http_proxy=server.uri)
-        _verify_https_proxy_is_used(server, config)
-
-def _verify_http_proxy_is_used(server, config):
-    store = InMemoryFeatureStore()
-    ready = Event()
-    with stream_content(make_put_event()) as stream:
-        server.for_path(config.stream_base_uri + '/all', stream)
-        with StreamingUpdateProcessor(config, store, ready, None) as sp:
-            sp.start()
-            # For an insecure proxy request, our stub server behaves enough like the real thing to satisfy the
-            # HTTP client, so we should be able to see the request go through. Note that the URI path will
-            # actually be an absolute URI for a proxy request.
-            req = server.await_request()
-            assert req.method == 'GET'
-            ready.wait(start_wait)
-            assert sp.initialized()
-
-def _verify_https_proxy_is_used(server, config):
-    store = InMemoryFeatureStore()
-    ready = Event()
-    with stream_content(make_put_event()) as stream:
-        server.for_path(config.stream_base_uri + '/all', stream)
-        with StreamingUpdateProcessor(config, store, ready, None) as sp:
-            sp.start()
-            # Our simple stub server implementation can't really do HTTPS proxying, so the request will fail, but
-            # it can still record that it *got* the request, which proves that the request went to the proxy.
-            req = server.await_request()
-            assert req.method == 'CONNECT'
+def test_http_proxy(monkeypatch):
+    def _stream_processor_proxy_test(server, config, secure):
+        store = InMemoryFeatureStore()
+        ready = Event()
+        with stream_content(make_put_event()) as stream:
+            server.for_path(config.stream_base_uri + '/all', stream)
+            with StreamingUpdateProcessor(config, store, ready, None) as sp:
+                sp.start()
+                # Wait till the server has received a request. We need to do this even though do_proxy_tests also
+                # does it, because if we return too soon out of this block, the object returned by stream_content
+                # could be closed and the test server would no longer work.
+                server.wait_until_request_received()
+                if not secure:
+                    # We only do this part with HTTP, because with HTTPS we don't have a real enough proxy server
+                    # for the stream connection to work correctly - we can only detect the request.
+                    ready.wait(start_wait)
+                    assert sp.initialized()
+    do_proxy_tests(_stream_processor_proxy_test, 'GET', monkeypatch)
 
 def test_records_diagnostic_on_stream_init_success():
     store = InMemoryFeatureStore()