Merge pull request #8591 from OpenMined/rasswanth/veilid-misc-improv

Veilid Improvements v3
OpenMined · Mar 18, 2024 · 1ea7eb5 · 1ea7eb5
2 parents 8ac20c7 + 66d6b4c
commit 1ea7eb5
Show file tree

Hide file tree

Showing 6 changed files with 97 additions and 21 deletions.
diff --git a/notebooks/Testing/Veilid/Alice-Python-Server.ipynb b/notebooks/Testing/Veilid/Alice-Python-Server.ipynb
@@ -152,20 +152,58 @@
   },
   {
    "cell_type": "markdown",
-   "id": "fd824cca-2a7f-4ea9-9e67-1c06d1f8bec2",
+   "id": "ddba6e22-96ee-46d7-8251-fcaa4140253b",
    "metadata": {},
    "source": [
-    "### Send AppMessage using VLD Key to Peer"
+    "### Ping Peer "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3de4b843-f3a2-4d96-bd48-121ae2b6f197",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "peer_vld_key = str(input(\"Enter Peer VLD Key\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "575c3441-cd11-4a42-ab4e-0bde3e5d5c72",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "peer_vld_key"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "64d0b338-a439-4982-b739-24c056833be1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = requests.post(f\"http://{host}:{port}/ping/{peer_vld_key}\")"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "25cfb508-dd08-44b9-85c9-e6aa07e96a97",
+   "id": "3ce13553-dae5-442e-bd56-2dddb526c0f2",
    "metadata": {},
    "outputs": [],
    "source": [
-    "peer_vld_key = input(\"Enter Peer VLD Key\")"
+    "res.json()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fd824cca-2a7f-4ea9-9e67-1c06d1f8bec2",
+   "metadata": {},
+   "source": [
+    "### Send AppMessage using VLD Key to Peer"
    ]
   },
   {
@@ -235,7 +273,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.5"
+   "version": "3.11.8"
   }
  },
  "nbformat": 4,

diff --git a/packages/grid/veilid/server/constants.py b/packages/grid/veilid/server/constants.py
@@ -9,3 +9,5 @@
 DHT_KEY_CREDS = "syft-dht-key-creds"
 
 USE_DIRECT_CONNECTION = True
+
+TIMEOUT = 10  # in seconds
diff --git a/packages/grid/veilid/server/main.py b/packages/grid/veilid/server/main.py
@@ -20,6 +20,7 @@
 from .veilid_core import app_message
 from .veilid_core import generate_vld_key
 from .veilid_core import healthcheck
+from .veilid_core import ping
 from .veilid_core import retrieve_vld_key
 
 # Logging Configuration
@@ -63,6 +64,16 @@ async def retrieve_vld_key_endpoint() -> ResponseModel:
         raise HTTPException(status_code=500, detail=str(e))
 
 
+@app.post("/ping/{vld_key}", response_model=ResponseModel)
+async def ping_endpoint(request: Request, vld_key: str) -> ResponseModel:
+    try:
+        logger.info(f"Received ping request:{vld_key}")
+        res = await ping(vld_key)
+        return ResponseModel(message=res)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
 @app.post("/app_message", response_model=ResponseModel)
 async def app_message_endpoint(
     request: Request, vld_key: Annotated[str, Body()], message: Annotated[bytes, Body()]

diff --git a/packages/grid/veilid/server/veilid_core.py b/packages/grid/veilid/server/veilid_core.py
@@ -1,3 +1,7 @@
+# stdlib
+import asyncio
+from enum import Enum
+
 # third party
 from loguru import logger
 import veilid
@@ -11,6 +15,7 @@
 from veilid.types import RouteId
 
 # relative
+from .constants import TIMEOUT
 from .constants import USE_DIRECT_CONNECTION
 from .veilid_connection import get_routing_context
 from .veilid_connection import get_veilid_conn
@@ -19,6 +24,11 @@
 from .veilid_db import store_dht_key_creds
 
 
+class PingResponse(Enum):
+    SUCCESS = "SUCCESS"
+    FAIL = "FAIL"
+
+
 async def create_private_route(
     conn: _JsonVeilidAPI,
     stability: Stability = veilid.Stability.RELIABLE,
@@ -156,6 +166,16 @@ async def app_call(vld_key: str, message: bytes) -> bytes:
             return result
 
 
+async def ping(vld_key: str) -> str:
+    async with await get_veilid_conn() as conn:
+        try:
+            _ = await asyncio.wait_for(conn.debug(f"ping {vld_key}"), timeout=TIMEOUT)
+            return PingResponse.SUCCESS.value
+        except Exception as e:
+            logger.error(f"Failed to ping {vld_key} : {e}")
+            return PingResponse.FAIL.value
+
+
 # TODO: Modify healthcheck endpoint to check public internet ready
 async def healthcheck() -> bool:
     async with await get_veilid_conn() as conn:

diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py
@@ -407,12 +407,17 @@ def _make_post(
         rev_proxy_url = self.vld_reverse_proxy.with_path(path)
         forward_proxy_url = self.vld_forward_proxy.with_path(VEILID_PROXY_PATH)
 
+        # Since JSON expects strings, we need to encode the bytes to base64
+        # as some bytes may not be valid utf-8
+        # TODO: Can we optimize this?
+        data_base64 = base64.b64encode(data).decode() if data else None
+
         json_data = {
             "url": str(rev_proxy_url),
             "method": "POST",
             "vld_key": self.vld_key,
             "json": json,
-            "data": data,
+            "data": data_base64,
         }
 
         response = self.session.post(str(forward_proxy_url), json=json_data)

diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json
@@ -23,7 +23,7 @@
         },
         "3": {
           "version": 3,
-          "hash": "18785a4cce6f25f1900b82f30acb2298b4afeab92bd00d0be358cfbf5a93d97e",
+          "hash": "37bb8f0f87b1da2525da8f6873e6257dff4a732f2dba293b62931ad0b85ef9e2",
           "action": "add"
         }
       },
@@ -40,7 +40,7 @@
         },
         "3": {
           "version": 3,
-          "hash": "4fd4c5b29e395b7a1af3b820166e69af7f267b6e3234fb8329bd0d74adc6e828",
+          "hash": "7c55461e3c6ba36ff999c64eb1b97a65b5a1f27193a973b1355ee2675f14c313",
           "action": "add"
         }
       },
@@ -52,7 +52,7 @@
         },
         "2": {
           "version": 2,
-          "hash": "1b04f527fdabaf329786b6bb38209f6ca82d622fe691d33c47ed1addccaaac02",
+          "hash": "1ab941c7669572a41067a17e0e3f2d9c7056f7a4df8f899e87ae2358d9113b02",
           "action": "add"
         }
       },
@@ -148,7 +148,7 @@
         },
         "3": {
           "version": 3,
-          "hash": "5922c1253370861185c53161ad31e488319f46ea5faee2d1802ca94657c428dc",
+          "hash": "709dc84a946267444a3f9968acf4a5e9807d6aa5143626c3fb635c9282108cc1",
           "action": "add"
         }
       },
@@ -165,7 +165,7 @@
         },
         "3": {
           "version": 3,
-          "hash": "dbb72f43add3141d13a76e18a2a0903a6937966632f0def452ca264f3f70d81b",
+          "hash": "5e84c9905a1816d51c0dfb1eedbfb4d831095ca6c89956c6fe200c2a193cbb8f",
           "action": "add"
         }
       },
@@ -182,7 +182,7 @@
         },
         "3": {
           "version": 3,
-          "hash": "cf831130f66f9addf8f68a8c9df0b67775e53322c8a32e8babc7f21631845608",
+          "hash": "bf936c1923ceee4def4cded06d41766998ea472322b0738bade7b85298e469da",
           "action": "add"
         }
       },
@@ -199,7 +199,7 @@
         },
         "3": {
           "version": 3,
-          "hash": "78334b746e5230ac156e47960e91ce449543d1a77a62d9b8be141882e4b549aa",
+          "hash": "daf3629fb7d26f41f96cd7f9200d7327a4b74d800b3e02afa75454d11bd47d78",
           "action": "add"
         }
       },
@@ -216,7 +216,7 @@
         },
         "3": {
           "version": 3,
-          "hash": "0007e86c39ede0f5756ba348083f809c5b6e3bb3a0a9ed6b94570d808467041f",
+          "hash": "4747a220d1587e99e6ac076496a2aa7217e2700205ac80fc24fe4768a313da78",
           "action": "add"
         }
       },
@@ -300,7 +300,7 @@
         },
         "2": {
           "version": 2,
-          "hash": "9eaed0a784525dea0018d95de74d70ed212f20f6ead2b50c66e59467c42bbe68",
+          "hash": "b35897295822f061fbc70522ca8967cd2be53a5c01b19e24c587cd7b0c4aa3e8",
           "action": "add"
         }
       },
@@ -574,7 +574,7 @@
         },
         "4": {
           "version": 4,
-          "hash": "077987cfc94d617f746f27fb468210330c328bad06eee09a89226759e5745a5f",
+          "hash": "c37bc1c6303c467050ce4f8faa088a2f66ef1781437ffe34f15aadf5477ac25b",
           "action": "add"
         }
       },
@@ -608,7 +608,7 @@
         },
         "3": {
           "version": 3,
-          "hash": "8a8e721a4ca8aa9107403368851acbe59f8d7bdc1eeff0ff101a44e325a058ff",
+          "hash": "4159d6ea45bc82577828bc19d668196422ff29bb8cc298b84623e6f4f476aaf3",
           "action": "add"
         }
       },
@@ -630,7 +630,7 @@
         },
         "4": {
           "version": 4,
-          "hash": "9b0dd1a64d64b1e824746e93aae0ca14863d2430aea2e2a758945edbfcb79bc9",
+          "hash": "dae431b87cadacfd30613519b5dd25d2e4ff59d2a971e21a31d56901103b9420",
           "action": "add"
         }
       },
@@ -1225,7 +1225,7 @@
         },
         "2": {
           "version": 2,
-          "hash": "747c87b947346fb0fc0466a912e2dc743ee082ef6254079176349d6b63748c32",
+          "hash": "93c75b45b9b74c69243cc2f2ef2d661e11eef5c23ecf71692ffdbd467d11efe6",
           "action": "add"
         }
       },
@@ -1513,7 +1513,7 @@
         },
         "2": {
           "version": 2,
-          "hash": "ac452023b98534eb13cb99a86fa7e379c08316353fc0837d1b788e0050e13ab9",
+          "hash": "24b7c302f9821afe073534d4ed02c377bd4f7cb691f66ca92b94c38c92dc78c2",
           "action": "add"
         }
       },
@@ -1525,7 +1525,7 @@
         },
         "2": {
           "version": 2,
-          "hash": "c9fdefdc622131c3676243aafadc30b7e67ee155793791bf1000bf742c1a251a",
+          "hash": "6d2e2f64c00dcda74a2545c77abbcf1630c56c26014987038feab174d15bd9d7",
           "action": "add"
         }
       },
Original file line number	Diff line number	Diff line change
Expand Up		@@ -9,3 +9,5 @@
		DHT_KEY_CREDS = "syft-dht-key-creds"

		USE_DIRECT_CONNECTION = True

		TIMEOUT = 10 # in seconds