Merge pull request #583 from msqd/portless_endpoints

💬 [feat/0.8] portless endpoints (endpoints that does not listen to a port but are only there to handle subrequests)

docs/apps/proxy/examples/full.yml

@@ -7,7 +7,8 @@ proxy:
       # a human-readable description of the endpoint (optional)
       description: A very informative description
 
-      # the local listening port for this endpoint
+      # the local listening port for this endpoint, if not set the proxy will not listen on
+      # any port for this endpoint.
       port: 4000
 
       # the controller to use for this endpoint, "null" means default but you can specify

docs/apps/proxy/schema.json

@@ -4,7 +4,10 @@
       "additionalProperties": false,
       "properties": {
         "name": { "type": "string" },
-        "port": { "type": "integer" },
+        "port": {
+          "anyOf": [{ "type": "integer" }, { "type": "null" }],
+          "default": null
+        },
         "description": {
           "anyOf": [{ "type": "string" }, { "type": "null" }],
           "default": null
@@ -14,7 +17,7 @@
           "default": null
         }
       },
-      "required": ["name", "port"],
+      "required": ["name"],
       "title": "BaseEndpointSettings",
       "type": "object"
     },
@@ -107,7 +110,10 @@
       "description": "Configuration parser for ``proxy.endpoints[]`` settings.\n\n.. code-block:: yaml\n\n    name: my-endpoint\n    port: 8080\n    description: My endpoint\n    remote:\n      # see HttpRemote\n      ...\n\nA shorthand syntax is also available for cases where you only need to proxy to a single URL and do not require\nfine-tuning the endpoint settings:\n\n.. code-block:: yaml\n\n    name: my-endpoint\n    port: 8080\n    description: My endpoint\n    url: http://my-endpoint:8080",
       "properties": {
         "name": { "type": "string" },
-        "port": { "type": "integer" },
+        "port": {
+          "anyOf": [{ "type": "integer" }, { "type": "null" }],
+          "default": null
+        },
         "description": {
           "anyOf": [{ "type": "string" }, { "type": "null" }],
           "default": null
@@ -121,7 +127,7 @@
           "default": null
         }
       },
-      "required": ["name", "port"],
+      "required": ["name"],
       "title": "EndpointSettings",
       "type": "object"
     },

docs/changelogs/unreleased.rst

@@ -1,2 +1,7 @@
 Unreleased
 ==========
+
+Added
+:::::::
+
+* Proxy: it is now possible to add not exposed endpoints in the proxy configuration.

docs/features/proxy.rst

@@ -29,6 +29,7 @@ Typically, you'll define one port per external (or semi-external) API you want t
 .. literalinclude:: ../apps/proxy/examples/multiple.yml
     :language: yaml
 
+A port is not required to configure an endpoint for the proxy. This is for advanced use cases only such as using an endpoint from within another controller (internal subrequests).
 
 Remote pools
 ::::::::::::

harp/config/tests/__snapshots__/test_all_examples.ambr

@@ -642,6 +642,10 @@
       remote:
         endpoints:
         - url: http://httpbin.org/
+    - name: httpbin4
+      remote:
+        endpoints:
+        - url: http://httpbin.org/
   storage: {}
 
   '''

harp/controllers/resolvers.py

@@ -28,11 +28,13 @@ async def resolve(self, request: HttpRequest):
 class ProxyControllerResolver(DefaultControllerResolver):
     _endpoints: dict[str, Endpoint]
     _ports: dict[int, IAsyncController]
+    _name_to_controller: dict[str, IAsyncController]
 
     def __init__(self, *, default_controller=None):
         super().__init__(default_controller=default_controller)
         self._endpoints = {}
         self._ports = {}
+        self._name_to_controller = {}
 
     @property
     def endpoints(self) -> dict[str, Endpoint]:
@@ -67,7 +69,10 @@ def add(
             http_client=http_client,
             name=endpoint.settings.name,
         )
-        self._ports[endpoint.settings.port] = controller
+
+        self._name_to_controller[endpoint.settings.name] = controller
+        if endpoint.settings.port:
+            self._ports[endpoint.settings.port] = controller
         logger.info(f"🏭 Map: *:{endpoint.settings.port} -> {controller}")
 
     def add_controller(self, port: int, controller: IAsyncController):
@@ -77,3 +82,6 @@ def add_controller(self, port: int, controller: IAsyncController):
 
     async def resolve(self, request: HttpRequest):
         return self._ports.get(request.server_port, self.default_controller)
+
+    def resolve_from_endpoint_name(self, endpoint_name: str):
+        return self._name_to_controller.get(endpoint_name, self.default_controller)

harp/controllers/tests/test_resolvers.py

@@ -0,0 +1,46 @@
+from httpx import AsyncClient
+
+from harp.controllers import ProxyControllerResolver
+from harp.http import HttpRequest
+from harp_apps.proxy.controllers import HttpProxyController
+from harp_apps.proxy.settings.endpoint import Endpoint
+
+
+def test_add():
+    http_client = AsyncClient()
+    endpoint = Endpoint.from_kwargs(settings={"name": "test-endpoint", "port": 8080, "url": "http://example.com/"})
+    resolver = ProxyControllerResolver()
+
+    resolver.add(endpoint, http_client=http_client, ControllerType=HttpProxyController)
+    assert resolver.endpoints["test-endpoint"] == endpoint
+    assert resolver.ports == (8080,)
+
+
+async def test_resolve():
+    http_client = AsyncClient()
+    endpoint = Endpoint.from_kwargs(settings={"name": "test-endpoint", "port": 8080, "url": "http://example.com/"})
+    resolver = ProxyControllerResolver()
+    resolver.add(endpoint, http_client=http_client, ControllerType=HttpProxyController)
+
+    request = HttpRequest(server_port=8080)
+    controller = await resolver.resolve(request)
+    assert isinstance(controller, HttpProxyController)
+    assert controller.name == "test-endpoint"
+
+    request = HttpRequest(server_port=8081)
+    controller = await resolver.resolve(request)
+    assert controller is resolver.default_controller
+
+
+def test_resolve_from_endpoint_name():
+    http_client = AsyncClient()
+    endpoint = Endpoint.from_kwargs(settings={"name": "test-endpoint", "port": 8080, "url": "http://example.com/"})
+    resolver = ProxyControllerResolver()
+    resolver.add(endpoint, http_client=http_client, ControllerType=HttpProxyController)
+
+    controller = resolver.resolve_from_endpoint_name("test-endpoint")
+    assert isinstance(controller, HttpProxyController)
+    assert controller.name == "test-endpoint"
+
+    controller = resolver.resolve_from_endpoint_name("non-existent")
+    assert controller is resolver.default_controller

harp_apps/dashboard/frontend/src/Pages/System/Topology/TopologyTable.tsx

@@ -8,10 +8,19 @@ export function TopologyTable({ endpoints }: { endpoints: Apps.Proxy.Endpoint[]
   return (
     <>
       {endpoints.map((endpoint, i) => (
-        <Pane className="flex items-start" key={i}>
-          <span className="inline-flex items-center gap-x-1 px-2 py-0.5 text-sm font-medium text-gray-800">
+        <Pane className="flex items-start overflow-auto space-x-1" key={i}>
+          <span className="flex gap-x-2 px-2 py-0.5 text-sm font-medium text-gray-800 items-center ">
             <PuzzlePieceIcon className="size-4" />
-            {endpoint.settings.name} ({endpoint.settings.port})
+            <span className="flex flex-col text-center">
+              {endpoint.settings.name}
+              {endpoint.settings.port ? (
+                <span className="inline-flex items-center gap-x-1 rounded-full px-2 py-0.5 text-xs font-medium text-gray-500 ring-1 ring-inset ring-gray-200 mx-1">
+                  {endpoint.settings.port}
+                </span>
+              ) : (
+                <p className="text-xs font-medium text-gray-400 mx-1">not exposed</p>
+              )}
+            </span>
           </span>
           <TopologyRemoteTable endpointName={endpoint.settings.name} remote={endpoint.remote} />
         </Pane>

harp_apps/dashboard/frontend/src/Pages/System/__snapshots__/SystemTopologyTabPanel.test.tsx.snap

@@ -25,10 +25,10 @@ exports[`renders the title and data when the query is successful 1`] = `
           Topology
         </h2>
         <div
-          class="flex items-start css-14upriz"
+          class="flex items-start overflow-auto space-x-1 css-14upriz"
         >
           <span
-            class="inline-flex items-center gap-x-1 px-2 py-0.5 text-sm font-medium text-gray-800"
+            class="flex gap-x-2 px-2 py-0.5 text-sm font-medium text-gray-800 items-center "
           >
             <svg
               aria-hidden="true"
@@ -46,10 +46,16 @@ exports[`renders the title and data when the query is successful 1`] = `
                 stroke-linejoin="round"
               />
             </svg>
-            endpoint1
-             (
-            8080
-            )
+            <span
+              class="flex flex-col text-center"
+            >
+              endpoint1
+              <span
+                class="inline-flex items-center gap-x-1 rounded-full px-2 py-0.5 text-xs font-medium text-gray-500 ring-1 ring-inset ring-gray-200 mx-1"
+              >
+                8080
+              </span>
+            </span>
           </span>
           <div
             class="flex flex-col divide-y divide-gray-200 css-14upriz"

harp_apps/dashboard/frontend/types/harp_apps.proxy.d.ts

@@ -47,7 +47,7 @@ declare namespace Apps.Proxy {
    */
   export interface EndpointSettings {
     name: string;
-    port: number;
+    port?: number | null;
     description?: string | null;
     controller?: string | null;
     remote?: RemoteSettings | null;
@@ -141,7 +141,7 @@ declare namespace Apps.Proxy {
   }
   export interface BaseEndpointSettings {
     name: string;
-    port: number;
+    port?: number | null;
     description?: string | null;
     controller?: string | null;
   }

harp_apps/proxy/examples/httpbins.yml

@@ -8,3 +8,5 @@ proxy.endpoints:
   - port: 4002
     name: httpbin3
     url: http://httpbin.org
+  - name: httpbin4
+    url: http://httpbin.org

harp_apps/proxy/settings/endpoint.py

@@ -11,7 +11,7 @@ class BaseEndpointSettings(Configurable):
     name: str
 
     #: port to listen on
-    port: int
+    port: Optional[int] = None
 
     #: description, informative only
     description: Optional[str] = None