Mongo action store

.gitignore

@@ -70,3 +70,6 @@ js/node_modules/*
 
 #nohup
 nohup.out
+
+# notebook data
+notebooks/helm/scenario_data.jsonl

notebooks/_experimental/koen/server.ipynb

@@ -0,0 +1,228 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "67759970",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "kj/filesystem-disk-unix.c++:1703: warning: PWD environment variable doesn't match current directory; pwd = /Users/koen/workspace/helm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import syft as sy\n",
+    "from syft import ActionObject\n",
+    "from syft import syft_function, syft_function_single_use"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "f6218715",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!rm -rf ~/.syft/data/*"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "904821d4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Starting test-domain-helm server on 0.0.0.0:8800\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "kj/filesystem-disk-unix.c++:1703: warning: PWD environment variable doesn't match current directory; pwd = /Users/koen/workspace/helm\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Waiting for server to start..\n",
+      "WARNING: private key is based on node name: test-domain-helm in dev_mode. Don't run this in production.\n",
+      "SQLite Store Path:\n",
+      "!open file:///var/folders/q1/ryq93kwj055dlbpngxv1c7z40000gn/T/80dacf8537ea463ab2c8c4598a02e80c.sqlite\n",
+      "\n",
+      "initializing queue manager\n",
+      ".127.0.0.1\n",
+      "tcp://127.0.0.1:56008\n",
+      "tcp://127.0.0.1:56008\n",
+      "spawning thread\n",
+      "alive True\n",
+      "defaultdict(<class 'list'>, {'api_call': [<syft.service.queue.zmq_queue.ZMQConsumer object at 0x7fc050ca48b0>]})\n",
+      ".INFO:     127.0.0.1:56011 - \"GET /api/v2/metadata HTTP/1.1\" 200 OK\n",
+      " Done.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:     Started server process [30355]\n",
+      "INFO:     Waiting for application startup.\n",
+      "INFO:     Application startup complete.\n",
+      "INFO:     Uvicorn running on http://0.0.0.0:8800 (Press CTRL+C to quit)\n"
+     ]
+    }
+   ],
+   "source": [
+    "node = sy.orchestra.launch(name=\"test-domain-helm\", port=8800, dev_mode=True, reset=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "049f4f8a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# node = sy.orchestra.launch(name=\"test-domain-helm\", port=8800, dev_mode=True, reset=True, tail=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "cc2ec126",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:     127.0.0.1:56013 - \"GET /api/v2/metadata HTTP/1.1\" 200 OK\n",
+      "INFO:     127.0.0.1:56013 - \"GET /api/v2/metadata HTTP/1.1\" 200 OK\n",
+      "INFO:     127.0.0.1:56013 - \"POST /api/v2/login HTTP/1.1\" 200 OK\n",
+      "INFO:     127.0.0.1:56013 - \"GET /api/v2/api?verify_key=10de27abe352de6bfc1bdf19764246dea7f3383aea0fd767c0c6d80d6d55c804 HTTP/1.1\" 200 OK\n",
+      "user.get_current_user {}\n",
+      "INFO:     127.0.0.1:56015 - \"POST /api/v2/api_call HTTP/1.1\" 200 OK\n",
+      "INFO:     127.0.0.1:56013 - \"GET /api/v2/api?verify_key=10de27abe352de6bfc1bdf19764246dea7f3383aea0fd767c0c6d80d6d55c804 HTTP/1.1\" 200 OK\n",
+      "Logged into <test-domain-helm: High side Domain> as <[email protected]>\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"alert-warning\" style=\"padding:5px;\"><strong>SyftWarning</strong>: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.</div><br />"
+      ],
+      "text/plain": [
+       "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`."
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "client = node.login(email=\"[email protected]\", password=\"changethis\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "773ac860",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "data_subject.get_all {}\n",
+      "Message Queued Successfully !\n",
+      "INFO:     127.0.0.1:56021 - \"POST /api/v2/api_call HTTP/1.1\" 200 OK\n"
+     ]
+    }
+   ],
+   "source": [
+    "res = client.data_subject_registry.get_all(blocking=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "066b2abb",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "queue {'uid': <UID: 909275b96ada44d0893a3ff2ddb29306>}\n",
+      "INFO:     127.0.0.1:56037 - \"POST /api/v2/api_call HTTP/1.1\" 200 OK\n"
+     ]
+    },
+    {
+     "ename": "AttributeError",
+     "evalue": "'SyftNotReady' object has no attribute 'data'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[9], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mres\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresolve\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdata\u001b[49m\n",
+      "\u001b[0;31mAttributeError\u001b[0m: 'SyftNotReady' object has no attribute 'data'"
+     ]
+    }
+   ],
+   "source": [
+    "res.resolve.data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8510fe87",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  },
+  "toc": {
+   "base_numbering": 1,
+   "nav_menu": {},
+   "number_sections": true,
+   "sideBar": true,
+   "skip_h1_title": false,
+   "title_cell": "Table of Contents",
+   "title_sidebar": "Contents",
+   "toc_cell": false,
+   "toc_position": {},
+   "toc_section_display": true,
+   "toc_window_display": true
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/helm/Untitled.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/helm/consumer node.ipynb

@@ -0,0 +1,134 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "a196017f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "kj/filesystem-disk-unix.c++:1703: warning: PWD environment variable doesn't match current directory; pwd = /Users/koen/workspace/pysyft\n"
+     ]
+    }
+   ],
+   "source": [
+    "import syft as sy\n",
+    "from syft import ActionObject\n",
+    "from syft import syft_function, syft_function_single_use\n",
+    "from time import sleep\n",
+    "from syft.service.queue.zmq_queue import ZMQQueueConfig, ZMQClientConfig"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "830cb5cf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "worker_node = sy.orchestra.launch(name=\"worker-node-helm1\", dev_mode=True,\n",
+    "                                  reset=True,\n",
+    "                                  n_consumers=3,\n",
+    "                                  create_producer=False,\n",
+    "                                  queue_port=62249)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "cf3a5520",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# worker_node.python_node.queue_manager.producers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "88fb4211",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'tcp://localhost:62249'"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "worker_node.python_node.queue_manager.consumers[\"api_call\"][0].address"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d1f13284",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Heartbeat failure, worker can't reach queue, reconnecting in 1s\n",
+      "Heartbeat failure, worker can't reach queue, reconnecting in 1s\n",
+      "Heartbeat failure, worker can't reach queue, reconnecting in 1s\n"
+     ]
+    }
+   ],
+   "source": [
+    "#make sure to run this\n",
+    "sleep(1000000)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "146b67f5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  },
+  "toc": {
+   "base_numbering": 1,
+   "nav_menu": {},
+   "number_sections": true,
+   "sideBar": true,
+   "skip_h1_title": false,
+   "title_cell": "Table of Contents",
+   "title_sidebar": "Contents",
+   "toc_cell": false,
+   "toc_position": {},
+   "toc_section_display": true,
+   "toc_window_display": true
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}