diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml index 5cd61193455..59064cbda95 100644 --- a/.github/workflows/cd-syft.yml +++ b/.github/workflows/cd-syft.yml @@ -173,7 +173,24 @@ jobs: digest="${{ steps.grid-frontend-build.outputs.digest }}" touch "/tmp/digests/grid-frontend/${digest#sha256:}" - - name: Upload digest for grid-backend and grid-frontend + - name: Build and push `grid-seaweedfs` image to DockerHub + id: grid-seaweedfs-build + uses: docker/build-push-action@v5 + with: + context: ./packages/grid/seaweedfs + file: ./packages/grid/seaweedfs/seaweedfs.dockerfile + platforms: ${{ steps.release_metadata.outputs.release_platform }} + outputs: type=image,name=openmined/grid-seaweedfs,push-by-digest=true,name-canonical=true,push=true + cache-from: type=registry,ref=openmined/grid-seaweedfs:cache-${{ steps.release_metadata.outputs.short_release_platform }} + cache-to: type=registry,ref=openmined/grid-seaweedfs:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max + + - name: Export digest for grid-seaweedfs + run: | + mkdir -p /tmp/digests/grid-seaweedfs + digest="${{ steps.grid-seaweedfs-build.outputs.digest }}" + touch "/tmp/digests/grid-seaweedfs/${digest#sha256:}" + + - name: Upload digest for grid-backend, grid-frontend and grid-seaweedfs uses: actions/upload-artifact@v3 with: name: digests-${{ steps.release_metadata.outputs.grid_version }} @@ -221,6 +238,14 @@ jobs: -t openmined/grid-frontend:${{ needs.build-and-push-docker-images.outputs.release_tag }} \ $(printf 'openmined/grid-frontend@sha256:%s ' *) + - name: Create manifest list and push for grid-seaweedfs + working-directory: /tmp/digests/grid-seaweedfs + run: | + docker buildx imagetools create \ + -t openmined/grid-seaweedfs:${{ needs.build-and-push-docker-images.outputs.grid_version }} \ + -t openmined/grid-seaweedfs:${{ needs.build-and-push-docker-images.outputs.release_tag }} \ + $(printf 'openmined/grid-seaweedfs@sha256:%s ' *) + deploy-syft: needs: [merge-docker-images] if: always() && needs.merge-docker-images.result == 'success' diff --git a/notebooks/helm/docker-helm-syft.ipynb b/notebooks/helm/docker-helm-syft.ipynb index e6d32b32774..3ce0d4b6365 100644 --- a/notebooks/helm/docker-helm-syft.ipynb +++ b/notebooks/helm/docker-helm-syft.ipynb @@ -5,15 +5,7 @@ "execution_count": 1, "id": "3333ab14", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "kj/filesystem-disk-unix.c++:1703: warning: PWD environment variable doesn't match current directory; pwd = /Users/koen/workspace/pysyft\n" - ] - } - ], + "outputs": [], "source": [ "import syft as sy\n", "import os\n", @@ -41,10 +33,30 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "7cda8c72", + "execution_count": 2, + "id": "3fc952d5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logged into as \n" + ] + }, + { + "data": { + "text/html": [ + "
SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.

" + ], + "text/plain": [ + "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "client = sy.login(url=\"http://localhost:8080\", email=\"info@openmined.org\", password=\"changethis\")" ] @@ -59,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, "id": "8b93a69d", "metadata": {}, "outputs": [ @@ -72,7 +84,7 @@ "SyftSuccess: Mounting Azure Successful!" ] }, - "execution_count": 5, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -88,7 +100,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 4, "id": "fd89b14e", "metadata": {}, "outputs": [], @@ -98,7 +110,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 5, "id": "93f1f918", "metadata": {}, "outputs": [ @@ -308,7 +320,7 @@ " flex-grow: 0;\n", " }\n", "\n", - " .grid-tablebdbd8cdb41c949e380f610cfe408efd3 {\n", + " .grid-table67d3c3655c244210ac6987f00cc4b290 {\n", " display:grid;\n", " grid-template-columns: 1fr repeat(8, 1fr);\n", " grid-template-rows: repeat(2, 1fr);\n", @@ -480,25 +492,25 @@ "
\n", "
\n", "
\n", - "
\n", - "
\n", + "
\n", " \n", "
\n", - " \n", + " \n", "
\n", - " \n", "
\n", "\n", - "

0

\n", + "

0

\n", "
\n", - "
\n", + "
\n", " \n", "
\n", - "
\n", + "
\n", " \n", "
\n", "
\n", @@ -745,10 +757,44 @@ " syft.types.blob_storage.BlobFile,\n", " syft.types.blob_storage.BlobFile,\n", " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", + " syft.types.blob_storage.BlobFile,\n", " syft.types.blob_storage.BlobFile]" ] }, - "execution_count": 7, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -767,7 +813,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "id": "d84a897e", "metadata": {}, "outputs": [ @@ -780,7 +826,7 @@ "SyftSuccess: 3 workers added" ] }, - "execution_count": 8, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -791,7 +837,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 7, "id": "4cea5229", "metadata": {}, "outputs": [ @@ -1001,7 +1047,7 @@ " flex-grow: 0;\n", " }\n", "\n", - " .grid-table273757d5911c4388b8615af3100e40b1 {\n", + " .grid-table161debd8746d4ca98e6a94ee5b196795 {\n", " display:grid;\n", " grid-template-columns: 1fr repeat(12, 1fr);\n", " grid-template-rows: repeat(2, 1fr);\n", @@ -1173,25 +1219,25 @@ "
\n", "
\n", "
\n", - "
\n", - "
\n", + "
\n", " \n", "
\n", - " \n", + " \n", "
\n", - " \n", "
\n", "\n", - "

0

\n", + "

0

\n", "
\n", - "
\n", + "
\n", " \n", "
\n", - "
\n", + "
\n", " \n", "
\n", "
\n", @@ -1410,7 +1456,7 @@ " syft.service.worker.worker_service.DockerWorker]" ] }, - "execution_count": 9, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -1429,29 +1475,29 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 8, "id": "c7d90857", "metadata": {}, "outputs": [], "source": [ - "# train_file = sy.ActionObject.from_path(\"short_input.jsonl\").send(client).syft_action_data\n", - "# scenario_file = scenario_obj = sy.ActionObject.from_path(path=\"scenario_data.jsonl\").send(client).syft_action_data" + "train_file = sy.ActionObject.from_path(\"short_input.jsonl\").send(client).syft_action_data\n", + "scenario_file = scenario_obj = sy.ActionObject.from_path(path=\"scenario_data.jsonl\").send(client).syft_action_data" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "id": "740b3cf1", "metadata": {}, "outputs": [], "source": [ - "train_file = [f for f in blob_files if \"train-00\" in f.file_name][0]\n", - "scenario_file = [f for f in blob_files if \"scenario_data\" in f.file_name][0]" + "# train_file = [f for f in blob_files if \"train-00\" in f.file_name][0]\n", + "# scenario_file = [f for f in blob_files if \"scenario_data\" in f.file_name][0]" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 10, "id": "f0da9c8a", "metadata": {}, "outputs": [], @@ -1475,7 +1521,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 11, "id": "4400f06f", "metadata": {}, "outputs": [ @@ -1502,8 +1548,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "\r\n", - " 0%| | 0/2 [00:00SyftSuccess: Dataset uploaded to 'quizzical_pearl'. To see the datasets uploaded by a client on this node, use command `[your_client].datasets`

" + "
SyftSuccess: Dataset uploaded to 'determined_norvig'. To see the datasets uploaded by a client on this node, use command `[your_client].datasets`

" ], "text/plain": [ - "SyftSuccess: Dataset uploaded to 'quizzical_pearl'. To see the datasets uploaded by a client on this node, use command `[your_client].datasets`" + "SyftSuccess: Dataset uploaded to 'determined_norvig'. To see the datasets uploaded by a client on this node, use command `[your_client].datasets`" ] }, - "execution_count": 13, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -1554,7 +1601,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 12, "id": "842988d1", "metadata": {}, "outputs": [ @@ -1587,7 +1634,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 13, "id": "aa3a5c31", "metadata": {}, "outputs": [ @@ -1711,7 +1758,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 14, "id": "2f23c7ae", "metadata": {}, "outputs": [ @@ -1724,7 +1771,7 @@ "SyftSuccess: User Code Submitted" ] }, - "execution_count": 16, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -1735,7 +1782,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 15, "id": "27be4dc4", "metadata": {}, "outputs": [ @@ -1772,7 +1819,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 16, "id": "82d92df1", "metadata": {}, "outputs": [ @@ -1785,14 +1832,14 @@ " \n", "
\n", "

Request

\n", - "

Id: a64e3e9d68984e8f93f24e55a5f1d195

\n", - "

Request time: 2023-11-10 16:17:59

\n", + "

Id: e7f73ae2ab8e4f1eb0efb02d676e51c1

\n", + "

Request time: 2023-12-05 09:35:19

\n", " \n", " \n", - "

Changes: Request to change main_function to permission RequestStatus.APPROVED.

\n", "

Status: RequestStatus.PENDING

\n", - "

Requested on: Quizzical_pearl of type Domain owned by info@openmined.org

\n", + "

Requested on: Determined_norvig of type Domain

\n", "

Requested by: Jane Doe (info@openmined.org)

\n", + "

Changes: Request to change main_function to permission RequestStatus.APPROVED. Nested Requests not resolved.

\n", "
\n", "\n", " " @@ -1800,12 +1847,12 @@ "text/markdown": [ "```python\n", "class Request:\n", - " id: str = a64e3e9d68984e8f93f24e55a5f1d195\n", - " request_time: str = 2023-11-10 16:17:59\n", + " id: str = e7f73ae2ab8e4f1eb0efb02d676e51c1\n", + " request_time: str = 2023-12-05 09:35:19\n", " updated_at: str = None\n", " status: str = RequestStatus.PENDING\n", - " changes: str = ['Request to change main_function to permission RequestStatus.APPROVED']\n", - " requesting_user_verify_key: str = ea951c201322d4ff6002807caf7b6506d84ff54faa269130363c51eac35556a3\n", + " changes: str = ['Request to change main_function to permission RequestStatus.APPROVED. Nested Requests not resolved']\n", + " requesting_user_verify_key: str = 810847da4475205f7540b1823a72af4c85c327665bfdb463351fbbab2715a675\n", "\n", "```" ], @@ -1813,7 +1860,7 @@ "syft.service.request.request.Request" ] }, - "execution_count": 18, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -1824,7 +1871,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "id": "29ee2790", "metadata": {}, "outputs": [ @@ -1845,30 +1892,30 @@ "output_type": "stream", "text": [ "Would you like to proceed? [y/n]: y\n", - "Request approved for domain quizzical_pearl\n" + "Request approved for domain determined_norvig\n" ] }, { "data": { "text/html": [ - "
SyftSuccess: Request a64e3e9d68984e8f93f24e55a5f1d195 changes applied

" + "
SyftSuccess: Request e7f73ae2ab8e4f1eb0efb02d676e51c1 changes applied

" ], "text/plain": [ - "SyftSuccess: Request a64e3e9d68984e8f93f24e55a5f1d195 changes applied" + "SyftSuccess: Request e7f73ae2ab8e4f1eb0efb02d676e51c1 changes applied" ] }, - "execution_count": 19, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "client.requests[-1].approve()" + "client.requests[-1].approve(approve_nested=True)" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "id": "78b084c0", "metadata": {}, "outputs": [], @@ -1888,7 +1935,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "id": "55c3bee6", "metadata": {}, "outputs": [ @@ -1897,14 +1944,13 @@ "text/markdown": [ "```python\n", "class Job:\n", - " id: UID = 9ede1ee9194d476aac425ffa0c2caae7\n", - " status: completed\n", + " id: UID = 4d68a345f6f34892bc7d4d42ee1f004b\n", + " status: created\n", " has_parent: False\n", - " result: ActionDataEmpty UID: fb3b7674904448b6bb1d954a4ec255b6 \n", + " result: ActionDataEmpty \n", " logs:\n", "\n", "0 \n", - "JOB COMPLETED\n", " \n", "```" ], @@ -1912,7 +1958,7 @@ "syft.service.job.job_stash.Job" ] }, - "execution_count": 22, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -1923,7 +1969,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 22, "id": "cf89cf33", "metadata": {}, "outputs": [], @@ -1931,192 +1977,776 @@ "# job.subjobs" ] }, - { - "cell_type": "code", - "execution_count": 23, - "id": "4d567f04", - "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "```python\n", - "class Job:\n", - " id: UID = 81676edda69d420990dc8c2851e671a1\n", - " status: completed\n", - " has_parent: True\n", - " result: ActionDataEmpty UID: 64215c70129a4be78afb980b23aff68c \n", - " logs:\n", - "\n", - "0 starting overlap computation\n", - "1 preparing scenarios and creating indexes\n", - "2 scenario creation progress: 0.90%\n", - "3 Finished creating scenarios (0.34194445610046387s)\n", - "4 Creating indexes\n", - "5 n_gram indexing progress: 0.00%\n", - "6 Finished creating indexes (0.05307936668395996s)\n", - "7 computing overlap\n", - "8 computing overlap progress: 3.83%\n", - "9 Finished computing overlap (0.06248641014099121s)\n", - "10 done\n", - "JOB COMPLETED\n", - " \n", - "```" - ], - "text/plain": [ - "syft.service.job.job_stash.Job" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "job.subjobs[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "f63089d2", - "metadata": {}, - "outputs": [], - "source": [ - "# job.wait().get()" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "852360ec", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "starting overlap computation\n", - "preparing scenarios and creating indexes\n", - "scenario creation progress: 0.90%\n", - "Finished creating scenarios (0.34194445610046387s)\n", - "Creating indexes\n", - "n_gram indexing progress: 0.00%\n", - "Finished creating indexes (0.05307936668395996s)\n", - "computing overlap\n", - "computing overlap progress: 3.83%\n", - "Finished computing overlap (0.06248641014099121s)\n", - "done\n", - "\n" - ] - } - ], - "source": [ - "job.subjobs[0].logs()" - ] - }, { "cell_type": "code", "execution_count": 26, - "id": "c5de7233", - "metadata": {}, - "outputs": [], - "source": [ - "results = [j.wait().get() for j in job.subjobs]" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "id": "4a079df7", + "id": "4d567f04", "metadata": {}, "outputs": [ { "data": { "text/html": [ - "[(defaultdict(<class 'set'>, {"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_test_5": {'id328'}, "{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_valid_5": {'id12'}}), defaultdict(<class 'set'>, {}), defaultdict(<class 'int'>, {"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_train_5": 5, "{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_valid_5": 34, "{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_test_5": 311, "{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'anatomy'}}_train_5": 5, "{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'anatomy'}}_valid_5": 14, "{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'anatomy'}}_test_5": 135}))]" - ], - "text/plain": [ - "[(defaultdict(set,\n", - " {\"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_test_5\": {'id328'},\n", - " \"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_valid_5\": {'id12'}}),\n", - " defaultdict(set, {}),\n", - " defaultdict(int,\n", - " {\"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_train_5\": 5,\n", - " \"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_valid_5\": 34,\n", - " \"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_test_5\": 311,\n", - " \"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'anatomy'}}_train_5\": 5,\n", - " \"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'anatomy'}}_valid_5\": 14,\n", - " \"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'anatomy'}}_test_5\": 135}))]" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "#stats_key_to_input_ids, stats_key_to_reference_ids, stats_key_counts\n", - "results" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "id": "0dcd8d03", - "metadata": {}, - "outputs": [], - "source": [ - "# results[0]" - ] - }, - { - "cell_type": "markdown", - "id": "6fe4daea", - "metadata": {}, - "source": [ - "# Aggregate" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "id": "d5053b78", - "metadata": {}, - "outputs": [], - "source": [ - "stats_key_to_input_ids, stats_key_to_reference_ids, stats_key_counts = zip(*results)\n", - "\n", - "total_input_ids = defaultdict(set)\n", - "total_reference_ids = defaultdict(set)\n", - "total_stats_key_counts = defaultdict(int)\n", - "\n", - "for d in stats_key_counts:\n", - " for key, val in d.items():\n", - " total_stats_key_counts[key] += val\n", - "\n", - "\n", - "for d in stats_key_to_input_ids:\n", - " for key in d:\n", - " new_set = set()\n", - " if key in total_input_ids:\n", - " new_set = total_input_ids[key]\n", - " new_set = new_set.union(d[key])\n", - " total_input_ids[key] = new_set\n", - "\n", - "for d in stats_key_to_reference_ids:\n", - " for key in d:\n", - " new_set = set()\n", - " if key in total_reference_ids:\n", - " new_set = total_reference_ids[key]\n", - " new_set = total_reference_ids[key].union(d[key])\n", - " total_reference_ids[key] = new_set\n", - "\n", - "all_data_overlap_stats = []\n", - "for stats_key, count in total_stats_key_counts.items():\n", - " data_overlap_stats = {\n", - " 'data_overlap_stats_key': None,\n", - " 'num_instances': count,\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "

Job List

\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + "\n", + "

0

\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n" + ], + "text/plain": [ + "[syft.service.job.job_stash.Job]" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "job.subjobs" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "f63089d2", + "metadata": {}, + "outputs": [], + "source": [ + "# job.wait().get()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "852360ec", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "starting overlap computation\n", + "preparing scenarios and creating indexes\n", + "scenario creation progress: 0.90%\n", + "Finished creating scenarios (0.2474043369293213s)\n", + "Creating indexes\n", + "n_gram indexing progress: 0.00%\n", + "Finished creating indexes (0.033365488052368164s)\n", + "computing overlap\n", + "computing overlap progress: 3.83%\n", + "Finished computing overlap (0.07294917106628418s)\n", + "done\n", + "\n", + "\n" + ] + } + ], + "source": [ + "job.subjobs[0].logs()" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "c5de7233", + "metadata": {}, + "outputs": [], + "source": [ + "results = [j.wait().get() for j in job.subjobs]" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "4a079df7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "[(defaultdict(<class 'set'>, {"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_test_5": {'id328'}, "{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_valid_5": {'id12'}}), defaultdict(<class 'set'>, {}), defaultdict(<class 'int'>, {"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_train_5": 5, "{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_valid_5": 34, "{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_test_5": 311, "{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'anatomy'}}_train_5": 5, "{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'anatomy'}}_valid_5": 14, "{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'anatomy'}}_test_5": 135}))]" + ], + "text/plain": [ + "[(defaultdict(set,\n", + " {\"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_test_5\": {'id328'},\n", + " \"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_valid_5\": {'id12'}}),\n", + " defaultdict(set, {}),\n", + " defaultdict(int,\n", + " {\"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_train_5\": 5,\n", + " \"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_valid_5\": 34,\n", + " \"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'philosophy'}}_test_5\": 311,\n", + " \"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'anatomy'}}_train_5\": 5,\n", + " \"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'anatomy'}}_valid_5\": 14,\n", + " \"{'class_name': 'helm.benchmark.scenarios.mmlu_scenario.MMLUScenario', 'args': {'subject': 'anatomy'}}_test_5\": 135}))]" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#stats_key_to_input_ids, stats_key_to_reference_ids, stats_key_counts\n", + "results" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "0dcd8d03", + "metadata": {}, + "outputs": [], + "source": [ + "# results[0]" + ] + }, + { + "cell_type": "markdown", + "id": "6fe4daea", + "metadata": {}, + "source": [ + "# Aggregate" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "d5053b78", + "metadata": {}, + "outputs": [], + "source": [ + "stats_key_to_input_ids, stats_key_to_reference_ids, stats_key_counts = zip(*results)\n", + "\n", + "total_input_ids = defaultdict(set)\n", + "total_reference_ids = defaultdict(set)\n", + "total_stats_key_counts = defaultdict(int)\n", + "\n", + "for d in stats_key_counts:\n", + " for key, val in d.items():\n", + " total_stats_key_counts[key] += val\n", + "\n", + "\n", + "for d in stats_key_to_input_ids:\n", + " for key in d:\n", + " new_set = set()\n", + " if key in total_input_ids:\n", + " new_set = total_input_ids[key]\n", + " new_set = new_set.union(d[key])\n", + " total_input_ids[key] = new_set\n", + "\n", + "for d in stats_key_to_reference_ids:\n", + " for key in d:\n", + " new_set = set()\n", + " if key in total_reference_ids:\n", + " new_set = total_reference_ids[key]\n", + " new_set = total_reference_ids[key].union(d[key])\n", + " total_reference_ids[key] = new_set\n", + "\n", + "all_data_overlap_stats = []\n", + "for stats_key, count in total_stats_key_counts.items():\n", + " data_overlap_stats = {\n", + " 'data_overlap_stats_key': None,\n", + " 'num_instances': count,\n", " 'instance_ids_with_overlapping_input': sorted(total_input_ids[stats_key]),\n", " 'instance_ids_with_overlapping_reference': sorted(total_reference_ids[stats_key]),\n", " }\n", @@ -2130,7 +2760,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 33, "id": "9c53c3aa", "metadata": {}, "outputs": [ diff --git a/packages/grid/default.env b/packages/grid/default.env index 42a7092be24..28d881f8115 100644 --- a/packages/grid/default.env +++ b/packages/grid/default.env @@ -27,7 +27,7 @@ TRAEFIK_VERSION=v2.10 REDIS_VERSION=6.2 RABBITMQ_VERSION=3 SEAWEEDFS_VERSION="3.59" -DOCKER_IMAGE_SEAWEEDFS=chrislusf/seaweedfs +DOCKER_IMAGE_SEAWEEDFS=openmined/grid-seaweedfs VERSION=latest VERSION_HASH=unknown STACK_API_KEY="" diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 92562771d8c..ca55dd09081 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -44,6 +44,16 @@ images: tags: - dev-latest - "${VERSION}" + seaweedfs: + image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_SEAWEEDFS}" + buildKit: {} + buildArgs: + SEAWEEDFS_VERSION: ${SEAWEEDFS_VERSION} + dockerfile: ./seaweedfs/seaweedfs.dockerfile + context: ./seaweedfs + tags: + - dev-latest + - "${VERSION}" # This is a list of `deployments` that DevSpace can create for this project deployments: @@ -211,10 +221,7 @@ deployments: repo: https://charts.devspace.sh values: containers: - - image: "${DOCKER_IMAGE_SEAWEEDFS}:${SEAWEEDFS_VERSION}" - command: - - sh - - /etc/seaweedfs/start.sh + - image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_SEAWEEDFS}:${VERSION}" env: - name: S3_VOLUME_SIZE_MB value: "${S3_VOLUME_SIZE_MB}" @@ -224,6 +231,8 @@ deployments: value: "${S3_ROOT_PWD}" - name: S3_PORT value: "${S3_PORT}" + - name: SEAWEED_MOUNT_PORT + value: "${SEAWEED_MOUNT_PORT}" volumeMounts: - containerPath: /etc/seaweedfs/filer.toml volume: @@ -251,6 +260,7 @@ deployments: ports: - port: "8888" # filer - port: "8333" # S3 + - port: "4001" # mount azure frontend: helm: diff --git a/packages/grid/docker-compose.build.yml b/packages/grid/docker-compose.build.yml index cc967abc40b..2bc853212ab 100644 --- a/packages/grid/docker-compose.build.yml +++ b/packages/grid/docker-compose.build.yml @@ -15,10 +15,16 @@ services: target: "backend" profiles: - backend + seaweedfs: build: context: ${RELATIVE_PATH}./seaweedfs dockerfile: seaweedfs.dockerfile + args: + - SEAWEEDFS_VERSION=${SEAWEEDFS_VERSION} + profiles: + - blob-storage + worker: build: context: ${RELATIVE_PATH}../ diff --git a/packages/grid/docker-compose.pull.yml b/packages/grid/docker-compose.pull.yml index 40812975460..c556030d85d 100644 --- a/packages/grid/docker-compose.pull.yml +++ b/packages/grid/docker-compose.pull.yml @@ -7,7 +7,7 @@ services: # image: rabbitmq:${RABBITMQ_VERSION?Variable not Set}${RABBITMQ_MANAGEMENT:-} seaweedfs: - image: "${DOCKER_IMAGE_SEAWEEDFS?Variable not set}:${SEAWEEDFS_VERSION}" + image: "${DOCKER_IMAGE_SEAWEEDFS?Variable not set}:${VERSION-latest}" profiles: - blob-storage diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index 12f5ff56300..eee2c05ad19 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -236,6 +236,8 @@ services: - blob-storage depends_on: - proxy + env_file: + - .env image: "${DOCKER_IMAGE_SEAWEEDFS?Variable not set}:${VERSION-latest}" environment: - S3_VOLUME_SIZE_MB=${S3_VOLUME_SIZE_MB:-1024} @@ -243,10 +245,6 @@ services: - S3_ROOT_PWD=${S3_ROOT_PWD:-admin} - S3_PORT=${S3_PORT:-8888} - SEAWEED_MOUNT_PORT=${SEAWEED_MOUNT_PORT:-4001} - - STACK_API_KEY=$STACK_API_KEY - entrypoint: ["/bin/sh"] - command: - - "/start.sh" volumes: - seaweedfs-data:/data - ./seaweedfs/filer.toml:/etc/seaweedfs/filer.toml diff --git a/packages/grid/helm/manifests.yaml b/packages/grid/helm/manifests.yaml index 39e9c570a8b..a762b398d48 100644 --- a/packages/grid/helm/manifests.yaml +++ b/packages/grid/helm/manifests.yaml @@ -73,7 +73,7 @@ spec: - name: NODE_NAME value: default_node_name - name: NODE_SIDE_TYPE - value: ${NODE_SIDE_TYPE} + value: high - name: STACK_API_KEY value: changeme - name: PORT @@ -533,6 +533,10 @@ spec: port: 8333 protocol: TCP targetPort: 8333 + - name: port-2 + port: 4001 + protocol: TCP + targetPort: 4001 selector: app.kubernetes.io/component: seaweedfs app.kubernetes.io/name: devspace-app @@ -569,9 +573,7 @@ spec: affinity: null containers: - args: null - command: - - sh - - /etc/seaweedfs/start.sh + command: null env: - name: S3_VOLUME_SIZE_MB value: '1024' @@ -581,8 +583,10 @@ spec: value: admin - name: S3_PORT value: '8333' + - name: SEAWEED_MOUNT_PORT + value: '4001' envFrom: null - image: chrislusf/seaweedfs:3.59 + image: docker.io/openmined/grid-seaweedfs:0.8.3-beta.3 lifecycle: null livenessProbe: null name: container-0 diff --git a/packages/grid/helm/syft/templates/NOTES.txt b/packages/grid/helm/syft/templates/NOTES.txt new file mode 100644 index 00000000000..97025355b1d --- /dev/null +++ b/packages/grid/helm/syft/templates/NOTES.txt @@ -0,0 +1,908 @@ + + Thank you for installing {{ .Chart.Name }}. + Your release is named {{ .Release.Name }}. + To learn more about the release, try: + + $ helm status {{ .Release.Name }} -n {{ .Release.Namespace }} + $ helm get all {{ .Release.Name }} + + + Following class versions are either added/removed. + + { + "PartialSyftObject": { + "1": { + "version": 1, + "hash": "008917584d8e1c09015cdbef02f59c0622f48e0618877c1b44425c8846befc13", + "action": "add" + } + }, + "NodeMetadataUpdate": { + "1": { + "version": 1, + "hash": "569d124c23590360bda240c19b53314ccc6204c5d1ab0d2898976a028e002191", + "action": "add" + } + }, + "NodeMetadata": { + "1": { + "version": 1, + "hash": "6bee018894dfdf697ea624740d0bf051750e0b0d8470ced59646f6d8812068ac", + "action": "add" + }, + "2": { + "version": 2, + "hash": "f856169fea72486cd436875ce4411ef935da11eb7c5af48121adfa00d4c0cdb6", + "action": "add" + }, + "3": { + "version": 3, + "hash": "3cc67abf394a805066a88aef0bea15bde609b9ecbe7ec15172eac5e7a0b7ef7c", + "action": "add" + } + }, + "StoreConfig": { + "1": { + "version": 1, + "hash": "17de8875cf590311ddb042140347ffc79d4a85028e504dad178ca4e1237ec861", + "action": "add" + } + }, + "MongoDict": { + "1": { + "version": 1, + "hash": "640734396edae801e1601fe7777710e67685e552acb0244ad8b4f689599baca9", + "action": "add" + } + }, + "MongoStoreConfig": { + "1": { + "version": 1, + "hash": "e52aa382e300b0b69aaa2d80aadb4e3a9a3c02b3c741b71d56f959c4d3891ce5", + "action": "add" + } + }, + "LinkedObject": { + "1": { + "version": 1, + "hash": "824567c6933c095d0e2f6995c8de3581c0fbd2e9e4ead35c8159f7964709c28e", + "action": "add" + } + }, + "BaseConfig": { + "1": { + "version": 1, + "hash": "4e5257080ce615aa4122b02bad8487e4c7d6d0f171ff77abbc9e8cd3e33df89a", + "action": "add" + } + }, + "ServiceConfig": { + "1": { + "version": 1, + "hash": "ca91f59bf045d949d82860f7d52655bfbede4cf6bdc5bae8f847f08a16f05d74", + "action": "add" + } + }, + "LibConfig": { + "1": { + "version": 1, + "hash": "c6ff229aea16874c5d9ae4d1f9e500d13f5cf984bbcee7abd16c5841707a2f78", + "action": "add" + } + }, + "APIEndpoint": { + "1": { + "version": 1, + "hash": "c0e83867b107113e6fed06364ba364c24b2f4af35b15a3869b176318d3be7989", + "action": "add" + } + }, + "LibEndpoint": { + "1": { + "version": 1, + "hash": "153eac6d8990774eebfffaa75a9895e7c4e1a0e09465d5da0baf4c3a3b03369d", + "action": "add" + } + }, + "SignedSyftAPICall": { + "1": { + "version": 1, + "hash": "e66a116de2fa44ebdd0d4c2d7d5a047dedb555fd201a0f431cd8017d9d33a61d", + "action": "add" + } + }, + "SyftAPICall": { + "1": { + "version": 1, + "hash": "014bd1d0933f6070888a313edba239170759de24eae49bf2374c1be4dbe2b4d7", + "action": "add" + } + }, + "SyftAPIData": { + "1": { + "version": 1, + "hash": "db101a75227e34750d7056785a1e87bb2e8ad6604f19c372d0cb6aa437243bf5", + "action": "add" + } + }, + "SyftAPI": { + "1": { + "version": 1, + "hash": "2bba1d9fcf677a58e35bf903de3da22ee4913af138aa3012af9c46b3609579cd", + "action": "add" + } + }, + "User": { + "1": { + "version": 1, + "hash": "078636e64f737e60245b39cf348d30fb006531e80c12b70aa7cf98254e1bb37a", + "action": "add" + } + }, + "UserUpdate": { + "1": { + "version": 1, + "hash": "839dd90aeb611e1dc471c8fd6daf230e913465c0625c6a297079cb7f0a271195", + "action": "add" + } + }, + "UserCreate": { + "1": { + "version": 1, + "hash": "dab78b63544ae91c09f9843c323cb237c0a6fcfeb71c1acf5f738e2fcf5c277f", + "action": "add" + } + }, + "UserSearch": { + "1": { + "version": 1, + "hash": "69d1e10b81c8a4143cf70e4f911d8562732af2458ebbc455ca64542f11373dd1", + "action": "add" + } + }, + "UserView": { + "1": { + "version": 1, + "hash": "63289383fe7e7584652f242a4362ce6e2f0ade52f6416ab6149b326a506b0675", + "action": "add" + } + }, + "UserViewPage": { + "1": { + "version": 1, + "hash": "16dac6209b19a934d286ef1efa874379e0040c324e71023c57d1bc6d2d367171", + "action": "add" + } + }, + "UserPrivateKey": { + "1": { + "version": 1, + "hash": "7cb196587887f0f3bffb298dd9f3b88509e9b2748792bf8dc03bdd0d6b98714a", + "action": "add" + } + }, + "NodeSettingsUpdate": { + "1": { + "version": 1, + "hash": "b6ddc66ff270a3c2c4760e31e1a55d72ed04ccae2d0115ebe2fba6f2bf9bd119", + "action": "add" + } + }, + "NodeSettings": { + "1": { + "version": 1, + "hash": "b662047bb278f4f5db77c102f94b733c3a929839271b3d6b82ea174a60e2aaf0", + "action": "add" + }, + "2": { + "version": 2, + "hash": "29a82afcb006a044b6ae04c6ea8a067d145d28b4210bb038ea9fa86ebde108c8", + "action": "add" + } + }, + "HTTPConnection": { + "1": { + "version": 1, + "hash": "5ee19eaf55ecbe7945ea45924c036ec0f500114a2f64176620961a8c2ec94cdb", + "action": "add" + } + }, + "PythonConnection": { + "1": { + "version": 1, + "hash": "011946fc9af0a6987f5c7bc9b0208b2fae9d65217531430bced7ba542788da1a", + "action": "add" + } + }, + "DateTime": { + "1": { + "version": 1, + "hash": "7e9d89309a10d2110a7ae4f97d8f25a7914853269e8fa0c531630790c1253f17", + "action": "add" + } + }, + "ActionDataEmpty": { + "1": { + "version": 1, + "hash": "89b5912fe5416f922051b8068be6071a03c87a4ab264959de524f1b86e95f028", + "action": "add" + } + }, + "ActionFileData": { + "1": { + "version": 1, + "hash": "1f32d94b75b0a6b4e86cec93d94aa905738219e3e7e75f51dd335ee832a6ed3e", + "action": "add" + } + }, + "Action": { + "1": { + "version": 1, + "hash": "5cf71ee35097f17fbb1dd05096f875211d71cf07161205d7f6a9c11fd49d5272", + "action": "add" + }, + "2": { + "version": 2, + "hash": "a13b50c4d23bd6deb7896e394f2a20e6cef4c33c5e6f4ee30f19eaffab708f21", + "action": "add" + } + }, + "ActionObject": { + "1": { + "version": 1, + "hash": "632446f1415102490c93fafb56dd9eb29d79623bcc5e9f2e6e37c4f63c2c51c3", + "action": "add" + }, + "2": { + "version": 2, + "hash": "577aa1f010b90194958a18ec38ee21db3718bd96d9e036501c6ddeefabedf432", + "action": "add" + } + }, + "AnyActionObject": { + "1": { + "version": 1, + "hash": "bcb31f847907edc9c95d2d120dc5427854604f40940e3f41cd0474a1820ac65e", + "action": "add" + }, + "2": { + "version": 2, + "hash": "002d8be821140befebbc0503e6bc1ef8779094e24e46305e5da5af6eecb56b13", + "action": "add" + } + }, + "BlobFile": { + "1": { + "version": 1, + "hash": "47ed55183d619c6c624e35412360a41de42833e2c24223c1de1ad12a84fdafc2", + "action": "add" + }, + "2": { + "version": 2, + "hash": "f2b29d28fe81a04bf5e946c819010283a9f98a97d50519358bead773865a2e09", + "action": "add" + } + }, + "BlobFileOBject": { + "1": { + "version": 1, + "hash": "8da2c80ced4f0414c671313c4b63d05846df1e397c763d99d803be86c29755bb", + "action": "add" + } + }, + "SecureFilePathLocation": { + "1": { + "version": 1, + "hash": "7febc066e2ee5a3a4a891720afede3f5c155cacc0557662ac4d04bf67b964c6d", + "action": "add" + } + }, + "SeaweedSecureFilePathLocation": { + "1": { + "version": 1, + "hash": "5724a38b1a92b8a55da3d9cc34a720365a6d0c32683acda630fc44067173e201", + "action": "add" + } + }, + "BlobStorageEntry": { + "1": { + "version": 1, + "hash": "9f1b027cce390ee6f71c7a81e7420bb71a477b29c6c62ba74e781a97bc5434e6", + "action": "add" + }, + "2": { + "version": 2, + "hash": "5472bdd5bdce6d0b561543a6bac70d47bf0c05c141a21450751460cc538d6b55", + "action": "add" + } + }, + "BlobStorageMetadata": { + "1": { + "version": 1, + "hash": "6888943be3f97186190dd26d7eefbdf29b15c6f2fa459e13608065ebcdb799e2", + "action": "add" + }, + "2": { + "version": 2, + "hash": "674f4c52a8444289d5ef389b919008860e2b0e7acbaafa774d58e492d5b6741a", + "action": "add" + } + }, + "CreateBlobStorageEntry": { + "1": { + "version": 1, + "hash": "61a373336e83645f1b6d78a320323d9ea4ee91b3d87b730cb0608fbfa0072262", + "action": "add" + } + }, + "BlobRetrieval": { + "1": { + "version": 1, + "hash": "a8d7e1d6483e7a9b5a130e837fa398862aa6cbb316cc5f4470450d835755fdd9", + "action": "add" + }, + "2": { + "version": 2, + "hash": "4c4fbdb6df5bb9fcbe914a9890bd1c1b6a1b3f382a04cbc8752a5a1b03130111", + "action": "add" + } + }, + "SyftObjectRetrieval": { + "1": { + "version": 1, + "hash": "7ccc62d5b434d2d438b3df661b4d753b0c7c8d593d451d8b86d364da83998c89", + "action": "add" + }, + "2": { + "version": 2, + "hash": "d9d7a7e1b8843145c9687fd013c9223700285886073547734267e91ac53e0996", + "action": "add" + } + }, + "BlobRetrievalByURL": { + "2": { + "version": 2, + "hash": "8059ee03016c4d74e408dad9529e877f91829672e0cc42d8cfff9c8e14058adc", + "action": "add" + } + }, + "BlobDeposit": { + "1": { + "version": 1, + "hash": "c98e6da658a3be01ead4ea6ee6a4c10046879f0ce0f5fc5f946346671579b229", + "action": "add" + } + }, + "WorkerSettings": { + "1": { + "version": 1, + "hash": "0dcd95422ec8a7c74e45ee68a125084c08f898dc94a13d25fe5a5fd0e4fc5027", + "action": "add" + }, + "2": { + "version": 2, + "hash": "d623a8a0d6c83b26ba49686bd8be10eccb126f54626fef334a85396c3b8a8ed6", + "action": "add" + } + }, + "QueueItem": { + "1": { + "version": 1, + "hash": "5aa94681d9d0715d5b605f9625a54e114927271378cf2ea7245f85c488035e0b", + "action": "add" + }, + "2": { + "version": 2, + "hash": "9503b878de4b5b7a1793580301353523b7d6219ebd27d38abe598061979b7570", + "action": "add" + } + }, + "ActionQueueItem": { + "1": { + "version": 1, + "hash": "11a43caf9164eb2a5a21f4bcb0ca361d0a5d134bf3c60173f2c502d0d80219de", + "action": "add" + } + }, + "ZMQClientConfig": { + "1": { + "version": 1, + "hash": "e6054969b495791569caaf33239039beae3d116e1fe74e9575467c48b9007c45", + "action": "add" + }, + "2": { + "version": 2, + "hash": "0f9bc88d56cd6eed6fc75459d1f914aed840c66e1195b9e41cc501b488fef2ed", + "action": "add" + } + }, + "HTTPNodeRoute": { + "1": { + "version": 1, + "hash": "1901b9f53f9970ce2bd8307ba9f7cafc0e7eba1d2ec82e4014c6120e605e3741", + "action": "add" + } + }, + "PythonNodeRoute": { + "1": { + "version": 1, + "hash": "15711e6e7a1ef726c8e8b5c35a6cb2d30b56ba5213cba489524bf63489e136cf", + "action": "add" + } + }, + "EnclaveMetadata": { + "1": { + "version": 1, + "hash": "39f85e475015e6f860ddcc5fea819423eba2db8f4b7d8e004c05a44d6f8444c6", + "action": "add" + } + }, + "DataSubject": { + "1": { + "version": 1, + "hash": "0b8b049d4627727b444c419f5d6a97b7cb97a433088ebf744c854b6a470dadf1", + "action": "add" + } + }, + "DataSubjectCreate": { + "1": { + "version": 1, + "hash": "5a94f9fcba75c50d78d71222f0235c5fd4d8003ae0db4d74bdbc4d56a99de3aa", + "action": "add" + } + }, + "DataSubjectMemberRelationship": { + "1": { + "version": 1, + "hash": "0a820edc9f1a87387acc3c611fe852752fcb3dab7608058f2bc48211be7bfbd2", + "action": "add" + } + }, + "Contributor": { + "1": { + "version": 1, + "hash": "d1d4f25bb87e59c0414501d3335097de66815c164c9ed5a7850ff8bec69fbcdc", + "action": "add" + } + }, + "MarkdownDescription": { + "1": { + "version": 1, + "hash": "519328a3952049f57004013e4fb00840695b24b8575cad983056412c9c9d9ba6", + "action": "add" + } + }, + "Asset": { + "1": { + "version": 1, + "hash": "24350b8d9597df49999918ad42e0eece1328ea30389311f1e0a420be8f39b8a1", + "action": "add" + } + }, + "CreateAsset": { + "1": { + "version": 1, + "hash": "1b4c71569b8da64258672483bd36dc4aa99a32d4cb519659241d15bc898041a6", + "action": "add" + } + }, + "Dataset": { + "1": { + "version": 1, + "hash": "99ca2fa3e46fd9810222d269fac6accb546f632e94d5d57529016ba5e55af5a8", + "action": "add" + } + }, + "DatasetPageView": { + "1": { + "version": 1, + "hash": "b1de14bb9b6a259648dfc59b6a48fa526116afe50a689c24b8bb36fd0e6a97f8", + "action": "add" + } + }, + "CreateDataset": { + "1": { + "version": 1, + "hash": "3b020d9b8928cbd7e91f41c749ab4c932e19520696a183f2c7cd1312ebb640d1", + "action": "add" + } + }, + "JobItem": { + "1": { + "version": 1, + "hash": "7b8723861837b0b7e948b2cf9244159d232185f3407dd6bef108346f941ddf6e", + "action": "add" + }, + "2": { + "version": 2, + "hash": "e99cf5a78c6dd3a0adc37af3472c7c21570a9e747985dff540a2b06d24de6446", + "action": "add" + } + }, + "TwinObject": { + "1": { + "version": 1, + "hash": "c42455586b43724a7421becd99122b787a129798daf6081e96954ecaea228099", + "action": "add" + } + }, + "ExactMatch": { + "1": { + "version": 1, + "hash": "e497e2e2380db72766c5e219e8afd13136d8953933d6f1eaf83b14001e887cde", + "action": "add" + } + }, + "OutputHistory": { + "1": { + "version": 1, + "hash": "4ec6e6efd86a972b474251885151bdfe4ef262562174605e8ab6a8abba1aa867", + "action": "add" + } + }, + "OutputPolicyExecuteCount": { + "1": { + "version": 1, + "hash": "6bb24b3b35e19564c43b838ca3f46ccdeadb6596511917f2d220681a378e439d", + "action": "add" + } + }, + "OutputPolicyExecuteOnce": { + "1": { + "version": 1, + "hash": "32a40fc9966b277528eebc61c01041f3a5447417731954abdaffbb14dabc76bb", + "action": "add" + } + }, + "UserPolicy": { + "1": { + "version": 1, + "hash": "c69b17b1d96cace8b45da6d9639165f2da4aa7ff156b6fd922ac217bf7856d8a", + "action": "add" + } + }, + "SubmitUserPolicy": { + "1": { + "version": 1, + "hash": "96f7f39279fadc70c569b8d48ed4d6420a8132db51e37466d272fda19953554b", + "action": "add" + } + }, + "UserCode": { + "1": { + "version": 1, + "hash": "e14c22686cdc7d1fb2b0d01c0aebdea37e62a61b051677c1d30234214f05cd42", + "action": "add" + }, + "2": { + "version": 2, + "hash": "660e1abc15034f525e91ffdd820c2a2179bfddf83b7b9e3ce7823b2efc515c69", + "action": "add" + } + }, + "SubmitUserCode": { + "2": { + "version": 2, + "hash": "9b29e060973a3de8d3564a2b7d2bb5c53745aa445bf257576994b613505d7194", + "action": "add" + } + }, + "UserCodeExecutionResult": { + "1": { + "version": 1, + "hash": "49c32e85e78b7b189a7f13b7e26115ef94fcb0b60b578adcbe2b95e289f63a6e", + "action": "add" + } + }, + "CodeHistory": { + "1": { + "version": 1, + "hash": "a7baae93862ae0aa67675f1617574e31aafb15a9ebff633eb817278a3a867161", + "action": "add" + } + }, + "CodeHistoryView": { + "1": { + "version": 1, + "hash": "0ed1a2a04a962ecbcfa38b0b8a03c1e51e8946a4b80f6bf2557148ce658671ce", + "action": "add" + } + }, + "CodeHistoriesDict": { + "1": { + "version": 1, + "hash": "95288411cd5843834f3273a2fd66a7df2e603e980f4ab1d329f9ab17d5d2f643", + "action": "add" + } + }, + "UsersCodeHistoriesDict": { + "1": { + "version": 1, + "hash": "5e1f389c4565ee8558386dd5c934d81e0c68ab1434f86bb9065976b587ef44d1", + "action": "add" + } + }, + "NodePeer": { + "1": { + "version": 1, + "hash": "7b88de7e38490e2d69f31295137673e7ddabc16ab0e2272ff491f6cea1835d63", + "action": "add" + } + }, + "OnDiskBlobDeposit": { + "1": { + "version": 1, + "hash": "5efc230c1ee65c4626d334aa69ed458c796c45265e546a333844c6c2bcd0e6b0", + "action": "add" + } + }, + "SeaweedFSBlobDeposit": { + "1": { + "version": 1, + "hash": "382a9ac178deed2a9591e1ebbb39f265cbe67027fb93a420d473a4c26b7fda11", + "action": "add" + } + }, + "DictStoreConfig": { + "1": { + "version": 1, + "hash": "256e9c623ce0becd555ddd2a55a0c15514e162786b1549388cef98a92a9b18c9", + "action": "add" + } + }, + "NumpyArrayObject": { + "1": { + "version": 1, + "hash": "dcc7b44fa5ad22ae0bc576948f856c172dac1e9de2bc8e2a302e428f3309a278", + "action": "add" + }, + "2": { + "version": 2, + "hash": "2c631121d9211006edab5620b214dea83e2398bee92244d822227ee316647e22", + "action": "add" + } + }, + "NumpyScalarObject": { + "1": { + "version": 1, + "hash": "5c1b6b6e8ba88bc79e76646d621489b889fe8f9b9fd59f117d594be18a409633", + "action": "add" + }, + "2": { + "version": 2, + "hash": "0d5d81b9d45c140f6e07b43ed68d31e0ef060d6b4d0431c9b4795997bb35c69d", + "action": "add" + } + }, + "NumpyBoolObject": { + "1": { + "version": 1, + "hash": "a5c822a6a3ca9eefd6a2b68f7fd0bc614fba7995f6bcc30bdc9dc882296b9b16", + "action": "add" + }, + "2": { + "version": 2, + "hash": "24839ba1c88ed833a134124750d5f299abcdf318670315028ed87b254f4578b3", + "action": "add" + } + }, + "PandasDataframeObject": { + "1": { + "version": 1, + "hash": "35058924b3de2e0a604a92f91f4dd2e3cc0dac80c219d34f360e7cedd52f5f4c", + "action": "add" + }, + "2": { + "version": 2, + "hash": "66729d4ba7a92210d45c5a5c24fbdb4c8e58138a515a7bdb71ac8f6e8b868544", + "action": "add" + } + }, + "PandasSeriesObject": { + "1": { + "version": 1, + "hash": "2a0d8a55f1c27bd8fccd276cbe01bf272c40cab10417d7027273983fed423caa", + "action": "add" + }, + "2": { + "version": 2, + "hash": "cb05a714f75b1140a943f56a3622fcc0477b3a1f504cd545a98510959ffe1528", + "action": "add" + } + }, + "ReplyNotification": { + "1": { + "version": 1, + "hash": "34b2ad522f7406c2486573467d9c7acef5c1063a0d9f2177c3bda2d8c4f87572", + "action": "add" + } + }, + "Notification": { + "1": { + "version": 1, + "hash": "d13981f721fe2b3e2717640ee07dc716c596e4ecd442461665c3fdab0b85bf0e", + "action": "add" + } + }, + "CreateNotification": { + "1": { + "version": 1, + "hash": "b1f459de374fe674f873a4a5f3fb8a8aabe0d83faad84a933f0a77dd1141159a", + "action": "add" + } + }, + "Change": { + "1": { + "version": 1, + "hash": "aefebd1601cf5bfd4817b0db75300a78299cc4949ead735a90873cbd22c8d4bc", + "action": "add" + } + }, + "ChangeStatus": { + "1": { + "version": 1, + "hash": "627f6f8e42cc285336aa6fd4916285d796140f4ff901487b7cb3907ef0f116a6", + "action": "add" + } + }, + "ActionStoreChange": { + "1": { + "version": 1, + "hash": "17b865e75eb3fb2693924fb00ba87a25260be45d55a4eb2184c4ead22d787cbe", + "action": "add" + } + }, + "Request": { + "1": { + "version": 1, + "hash": "e054307eeb7f13683cde9ce7613d5ca2925a13fff7c345b1c9f729a12c955f90", + "action": "add" + } + }, + "RequestInfo": { + "1": { + "version": 1, + "hash": "b76075c138afc0563ce9ac7f6b1131f048951f7486cd516c02736dc1a2a23639", + "action": "add" + } + }, + "RequestInfoFilter": { + "1": { + "version": 1, + "hash": "7103abdc464ae71bb746410f5730f55dd8ed82268aa32bbb0a69e0070488a669", + "action": "add" + } + }, + "SubmitRequest": { + "1": { + "version": 1, + "hash": "96b4ec12beafd9d8a7c97399cb8a23dade4db16d8f521be3fe7b8fec99db5161", + "action": "add" + } + }, + "ObjectMutation": { + "1": { + "version": 1, + "hash": "0ee3dd38d6df0fe9a19d848e8f3aaaf13a6ba86afe3406c239caed6da185651a", + "action": "add" + } + }, + "EnumMutation": { + "1": { + "version": 1, + "hash": "4c02f956ec9b973064972cc57fc8dd9c525e683f93f804642b4e1bfee1b62e57", + "action": "add" + } + }, + "UserCodeStatusChange": { + "1": { + "version": 1, + "hash": "4f5b405cc2b3976ed8f7018df82e873435d9187dff15fa5a23bc85a738969f3f", + "action": "add" + }, + "2": { + "version": 2, + "hash": "d83e0905ae882c824ba8fbbf455cd3881906bf8b2ebbfff07bcf471ef869cedc", + "action": "add" + } + }, + "SyftLog": { + "1": { + "version": 1, + "hash": "bd3f62b8fe4b2718a6380c8f05a93c5c40169fc4ab174db291929298e588429e", + "action": "add" + }, + "2": { + "version": 2, + "hash": "d3ce45794da2e6c4b0cef63b98a553525af50c5d9db42d3d64caef3e7d22b4a9", + "action": "add" + } + }, + "SyftObjectMigrationState": { + "1": { + "version": 1, + "hash": "d3c8126bc15dae4dd243bb035530e3f56cd9e433d403dd6b5f3b45face6d281f", + "action": "add" + } + }, + "ProjectThreadMessage": { + "1": { + "version": 1, + "hash": "1118e935792e8e54103dbf91fa33edbf192a7767d2b1d4526dfa7d4a643cde2e", + "action": "add" + } + }, + "ProjectMessage": { + "1": { + "version": 1, + "hash": "55a3a5171b6949372b4125cc461bf39bc998565e07703804fca6c7ef99695ae4", + "action": "add" + } + }, + "ProjectRequestResponse": { + "1": { + "version": 1, + "hash": "d4c360e845697a0b24695143d0781626cd344cfde43162c90ae90fe67e00ae21", + "action": "add" + } + }, + "ProjectRequest": { + "1": { + "version": 1, + "hash": "514d189df335c68869eea36befcdcafec74bdc682eaf18871fe879e26da4dbb6", + "action": "add" + } + }, + "AnswerProjectPoll": { + "1": { + "version": 1, + "hash": "ff2e1ac7bb764c99d646b96eb3ebfbf9311599b7e3be07aa4a4eb4810bb6dd12", + "action": "add" + } + }, + "ProjectPoll": { + "1": { + "version": 1, + "hash": "b0ac8f1d9c06997374ddbc33fdf1d0af0da15fdb6899f52d91a8574106558964", + "action": "add" + } + }, + "Project": { + "1": { + "version": 1, + "hash": "ec5b7ac1c92808e266f06b175c6ebcd50be81777ad120c02ce8c6074d0004788", + "action": "add" + } + }, + "ProjectSubmit": { + "1": { + "version": 1, + "hash": "0374b37779497d7e0b2ffeabc38d35bfbae2ee762a7674a5a8af75e7c5545e61", + "action": "add" + } + }, + "ContainerImage": { + "1": { + "version": 1, + "hash": "776fc7cf7498b93e656a00fff03b86160d1b63e508e2143ac7932e7e38021b0c", + "action": "add" + } + }, + "SQLiteStoreConfig": { + "1": { + "version": 1, + "hash": "b656b26c14cf4e97aba702dd62a0927aec7f860c12eed512c2c688e1b7109aa5", + "action": "add" + } + }, + "Plan": { + "1": { + "version": 1, + "hash": "a0bba2b7792c9e08c453e9e256f0ac6e6185610726566bcd50b057ae83b42d9a", + "action": "add" + } + } +} + + This means the existing data will be automatically be migrated to + their latest class versions during the upgrade. diff --git a/packages/grid/helm/syft/templates/seaweedfs-service.yaml b/packages/grid/helm/syft/templates/seaweedfs-service.yaml index a425eccda00..78bccff1967 100644 --- a/packages/grid/helm/syft/templates/seaweedfs-service.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs-service.yaml @@ -19,6 +19,10 @@ spec: port: 8333 protocol: TCP targetPort: 8333 + - name: port-2 + port: 4001 + protocol: TCP + targetPort: 4001 selector: app.kubernetes.io/component: seaweedfs app.kubernetes.io/name: devspace-app diff --git a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml index ce32340ac8b..6afc4af8800 100644 --- a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml @@ -30,9 +30,7 @@ spec: affinity: null containers: - args: null - command: - - sh - - /etc/seaweedfs/start.sh + command: null env: - name: S3_VOLUME_SIZE_MB value: '1024' @@ -42,8 +40,10 @@ spec: value: admin - name: S3_PORT value: '8333' + - name: SEAWEED_MOUNT_PORT + value: '4001' envFrom: null - image: chrislusf/seaweedfs:3.59 + image: docker.io/openmined/grid-seaweedfs:0.8.3-beta.3 lifecycle: null livenessProbe: null name: container-0 diff --git a/packages/grid/seaweedfs/mount_command.sh b/packages/grid/seaweedfs/mount_command.sh index 9b4683cc3e5..f9c813ef8d5 100644 --- a/packages/grid/seaweedfs/mount_command.sh +++ b/packages/grid/seaweedfs/mount_command.sh @@ -1,9 +1,4 @@ -echo "remote.configure -name=$1 -type=azure -azure.account_name=$2 \ - -azure.account_key=$5" \ - | weed shell - -echo "s3.bucket.create -name $3" | weed shell - -echo "remote.mount -dir=/buckets/$3 -remote=$1/$4" | weed shell - +echo "remote.configure -name=$1 -type=azure -azure.account_name=$2 -azure.account_key=$5" | weed shell && \ +echo "s3.bucket.create -name=$3" | weed shell && \ +echo "remote.mount -dir=/buckets/$3 -remote=$1/$4" | weed shell && \ weed filer.remote.sync diff --git a/packages/grid/seaweedfs/seaweedfs.dockerfile b/packages/grid/seaweedfs/seaweedfs.dockerfile index 224041522ca..3982e621c3b 100644 --- a/packages/grid/seaweedfs/seaweedfs.dockerfile +++ b/packages/grid/seaweedfs/seaweedfs.dockerfile @@ -1,18 +1,15 @@ -FROM chrislusf/seaweedfs:3.57 +ARG SEAWEEDFS_VERSION + +FROM chrislusf/seaweedfs:${SEAWEEDFS_VERSION} WORKDIR / -RUN apk update && apk upgrade --available -RUN apk add --no-cache python3 py3-pip ca-certificates bash +RUN apk update && \ + apk add --no-cache python3 py3-pip ca-certificates bash -COPY ./requirements.txt /requirements.txt +COPY requirements.txt app.py / RUN pip install --no-cache-dir -r requirements.txt -COPY ./start.sh /start.sh -COPY ./mount_command.sh /mount_command.sh -COPY ./app.py /app.py - -RUN chmod +x /start.sh -RUN chmod +x /mount_command.sh +COPY --chmod=755 start.sh mount_command.sh / -ENTRYPOINT ["bash", "./start.sh"] +ENTRYPOINT ["/start.sh"] diff --git a/packages/grid/seaweedfs/start.sh b/packages/grid/seaweedfs/start.sh index f084ec521d7..7972664b44e 100644 --- a/packages/grid/seaweedfs/start.sh +++ b/packages/grid/seaweedfs/start.sh @@ -1,10 +1,7 @@ #!/usr/bin/env bash -echo "got api key" -echo ${STACK_API_KEY} -export STACK_API_KEY=${STACK_API_KEY} - +weed server -s3 -s3.port="$S3_PORT" -volume.max=500 -master.volumeSizeLimitMB="$S3_VOLUME_SIZE_MB" & echo "s3.configure -access_key $S3_ROOT_USER -secret_key $S3_ROOT_PWD \ --user iam -actions Read,Write,List,Tagging,Admin -apply" | weed shell > /dev/null 2>&1 & -weed server -s3 -s3.port=$S3_PORT -volume.max=500 -master.volumeSizeLimitMB=$S3_VOLUME_SIZE_MB & -flask run -p $SEAWEED_MOUNT_PORT --host=0.0.0.0 +-user iam -actions Read,Write,List,Tagging,Admin -apply" | weed shell > /dev/null 2>&1 + +flask run -p "$SEAWEED_MOUNT_PORT" --host=0.0.0.0 diff --git a/packages/grid/traefik/docker/dynamic.yml b/packages/grid/traefik/docker/dynamic.yml index 9b8923e386f..cc6a7bb7ee4 100644 --- a/packages/grid/traefik/docker/dynamic.yml +++ b/packages/grid/traefik/docker/dynamic.yml @@ -48,14 +48,6 @@ http: middlewares: - "blob-storage-url" - "blob-storage-host" - blob-storage-mount: - rule: "PathPrefix(`/mount`)" - entryPoints: - - web - - vpn - service: "seaweedfsmount" - middlewares: - - "blob-storage-mount-url" vpn: rule: "PathPrefix(`/vpn`)" entryPoints: @@ -81,10 +73,6 @@ http: stripprefix: prefixes: /blob forceslash: true - blob-storage-mount-url: - stripprefix: - prefixes: /mount - forceslash: true vpn-url: stripprefix: prefixes: /vpn