Skip to content

Commit f887a04

Browse files
rasswanth-srasswanth-s
authored
Merge branch 'dev' into dependabot/github_actions/dorny/paths-filter-3
2 parents 75e7c49 + df90d16 commit f887a04

26 files changed

+1223
-460
lines changed

‎notebooks/api/0.8/01-submit-code.ipynb

+5-5
Original file line numberDiff line numberDiff line change
@@ -424,7 +424,7 @@
424424
"outputs": [],
425425
"source": [
426426
"# Add a request to submit & execute the code\n",
427-
"new_project.create_code_request(sum_trade_value_mil, jane_client)"
427+
"result = new_project.create_code_request(sum_trade_value_mil, jane_client)"
428428
]
429429
},
430430
{
@@ -435,7 +435,7 @@
435435
},
436436
"outputs": [],
437437
"source": [
438-
"assert len(jane_client.code.get_all()) == 1"
438+
"assert len(jane_client.code.get_all()) == 1, str(result)"
439439
]
440440
},
441441
{
@@ -445,7 +445,7 @@
445445
"outputs": [],
446446
"source": [
447447
"# create the same code request with the exact same function should return an error\n",
448-
"new_project.create_code_request(sum_trade_value_mil, jane_client)"
448+
"result = new_project.create_code_request(sum_trade_value_mil, jane_client)"
449449
]
450450
},
451451
{
@@ -454,7 +454,7 @@
454454
"metadata": {},
455455
"outputs": [],
456456
"source": [
457-
"assert len(jane_client.code.get_all()) == 1"
457+
"assert len(jane_client.code.get_all()) == 1, str(result)"
458458
]
459459
},
460460
{
@@ -583,7 +583,7 @@
583583
"name": "python",
584584
"nbconvert_exporter": "python",
585585
"pygments_lexer": "ipython3",
586-
"version": "3.11.4"
586+
"version": "3.11.2"
587587
},
588588
"toc": {
589589
"base_numbering": 1,

‎notebooks/api/0.8/10-container-images.ipynb

+36-18
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,10 @@
3030
"import syft as sy\n",
3131
"\n",
3232
"sy.requires(SYFT_VERSION)\n",
33+
"\n",
3334
"# syft absolute\n",
3435
"from syft.service.worker.image_registry import SyftImageRegistry\n",
35-
"from syft.service.worker.worker_image import SyftWorkerImage\n",
36-
"\n",
37-
"sy.requires(SYFT_VERSION)"
36+
"from syft.service.worker.worker_image import SyftWorkerImage"
3837
]
3938
},
4039
{
@@ -486,17 +485,19 @@
486485
" # third party\n",
487486
" import requests\n",
488487
"\n",
489-
" repos = requests.get(\"http://localhost:5678/v2/_catalog\").json()[\"repositories\"]\n",
490-
" tags = requests.get(\n",
491-
" \"http://localhost:5678/v2/openmined/custom-worker/tags/list\"\n",
492-
" ).json()[\"tags\"]\n",
488+
" base_url = f\"http://{workerimage.image_identifier.registry_host}\"\n",
489+
" expected_tag = workerimage.image_identifier.tag\n",
490+
"\n",
491+
" repos = requests.get(f\"{base_url}/v2/_catalog\").json()[\"repositories\"]\n",
492+
" tags = requests.get(f\"{base_url}/v2/openmined/custom-worker/tags/list\").json()\n",
493+
" tags = tags[\"tags\"]\n",
493494
"\n",
494495
" assert (\n",
495496
" \"openmined/custom-worker\" in repos\n",
496497
" ), f\"'openmined/custom-worker' not uploaded to local registry | {repos}\"\n",
497498
" assert (\n",
498-
" \"0.7.8\" in tags\n",
499-
" ), f\"'openmined/custom-worker' with tag '0.7.8' not available | {tags}\""
499+
" expected_tag in tags\n",
500+
" ), f\"'openmined/custom-worker' with tag {expected_tag} not available | {tags}\""
500501
]
501502
},
502503
{
@@ -739,7 +740,7 @@
739740
"metadata": {},
740741
"outputs": [],
741742
"source": [
742-
"assert isinstance(worker_delete_res, sy.SyftSuccess)"
743+
"assert isinstance(worker_delete_res, sy.SyftSuccess), str(worker_delete_res)"
743744
]
744745
},
745746
{
@@ -981,7 +982,7 @@
981982
" res = domain_client.api.services.worker.delete(\n",
982983
" uid=worker.id,\n",
983984
" )\n",
984-
" assert isinstance(res, sy.SyftSuccess)\n",
985+
" assert isinstance(res, sy.SyftSuccess), str(res)\n",
985986
"\n",
986987
"# Adding some sleep to allow containers to be fully removed,\n",
987988
"# before removing the image\n",
@@ -1016,7 +1017,7 @@
10161017
"outputs": [],
10171018
"source": [
10181019
"# Since the containers are delete, we should be able to delete the image\n",
1019-
"assert isinstance(delete_res, sy.SyftSuccess)\n",
1020+
"assert isinstance(delete_res, sy.SyftSuccess), str(delete_res)\n",
10201021
"delete_res"
10211022
]
10221023
},
@@ -1158,7 +1159,7 @@
11581159
"metadata": {},
11591160
"outputs": [],
11601161
"source": [
1161-
"assert isinstance(req_result, sy.SyftSuccess)"
1162+
"assert isinstance(req_result, sy.SyftSuccess), str(req_result)"
11621163
]
11631164
},
11641165
{
@@ -1215,8 +1216,25 @@
12151216
"metadata": {},
12161217
"outputs": [],
12171218
"source": [
1218-
"test_build_res = docker_config_3.test_image_build(tag=docker_tag_3)\n",
1219-
"assert isinstance(test_build_res, sy.SyftSuccess)"
1219+
"# syft absolute\n",
1220+
"from syft.custom_worker.builder import CustomWorkerBuilder\n",
1221+
"from syft.service.response import SyftError\n",
1222+
"from syft.service.response import SyftSuccess\n",
1223+
"\n",
1224+
"\n",
1225+
"def test_image_build(config: str, tag: str, **kwargs):\n",
1226+
" builder = CustomWorkerBuilder()\n",
1227+
" try:\n",
1228+
" result = builder.build_image(\n",
1229+
" config=config, tag=tag, pull=True, rm=True, forcerm=True, **kwargs\n",
1230+
" )\n",
1231+
" return SyftSuccess(message=result.logs)\n",
1232+
" except Exception as e:\n",
1233+
" return SyftError(message=f\"Failed to build image !! Error: {str(e)}.\")\n",
1234+
"\n",
1235+
"\n",
1236+
"test_build_res = test_image_build(config=docker_config_3, tag=docker_tag_3)\n",
1237+
"assert isinstance(test_build_res, sy.SyftSuccess), str(test_build_res)"
12201238
]
12211239
},
12221240
{
@@ -1270,7 +1288,7 @@
12701288
"metadata": {},
12711289
"outputs": [],
12721290
"source": [
1273-
"assert isinstance(req_result, sy.SyftSuccess)"
1291+
"assert isinstance(req_result, sy.SyftSuccess), str(req_result)"
12741292
]
12751293
},
12761294
{
@@ -1343,7 +1361,7 @@
13431361
" for worker in worker_pool.workers:\n",
13441362
" res = domain_client.api.services.worker.delete(uid=worker.id, force=True)\n",
13451363
" print(res)\n",
1346-
" assert isinstance(res, sy.SyftSuccess)\n",
1364+
" assert isinstance(res, sy.SyftSuccess), str(res)\n",
13471365
"\n",
13481366
"# Adding some sleep to allow containers to be fully removed,\n",
13491367
"# before removing the image\n",
@@ -1388,7 +1406,7 @@
13881406
"name": "python",
13891407
"nbconvert_exporter": "python",
13901408
"pygments_lexer": "ipython3",
1391-
"version": "3.9.7"
1409+
"version": "3.11.2"
13921410
}
13931411
},
13941412
"nbformat": 4,

‎packages/grid/backend/backend.dockerfile

+4-4
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ RUN --mount=type=cache,target=/var/cache/apk,sharing=locked \
2222
apk update && \
2323
apk add build-base gcc tzdata python-$PYTHON_VERSION-dev py$PYTHON_VERSION-pip && \
2424
ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
25-
# uncomment for creating rootless user
26-
# && adduser -D -u $UID $USER
25+
# uncomment for creating rootless user
26+
# && adduser -D -u $UID $USER
2727

2828
# ==================== [BUILD STEP] Install Syft Dependency ==================== #
2929

@@ -73,12 +73,12 @@ ARG USER_GRP
7373
# Setup Python
7474
RUN --mount=type=cache,target=/var/cache/apk,sharing=locked \
7575
apk update && \
76-
apk add tzdata bash python-$PYTHON_VERSION py$PYTHON_VERSION-pip && \
76+
apk add tzdata git bash python-$PYTHON_VERSION py$PYTHON_VERSION-pip && \
7777
ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \
7878
# Uncomment for rootless user
7979
# adduser -D -u 1000 $USER && \
8080
mkdir -p /var/log/pygrid $HOME/data/creds $HOME/data/db $HOME/.cache $HOME/.local
81-
# chown -R $USER_GRP /var/log/pygrid $HOME/
81+
# chown -R $USER_GRP /var/log/pygrid $HOME/
8282

8383
USER $USER
8484
WORKDIR $APPDIR

‎packages/grid/devspace.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -66,10 +66,13 @@ deployments:
6666
syft:
6767
registry: ${CONTAINER_REGISTRY}
6868
version: dev-${DEVSPACE_TIMESTAMP}
69+
workerBuilds:
70+
mountInBackend: true
6971
node:
7072
settings:
7173
nodeName: ${NODE_NAME}
7274
nodeType: "domain"
75+
inMemoryWorkers: true
7376
configuration:
7477
devmode: True
7578

‎packages/grid/helm/syft/templates/backend-service-account.yaml

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: backend-service-account
5+
namespace: {{ .Release.Namespace }}
6+
labels:
7+
app.kubernetes.io/name: {{ .Chart.Name }}
8+
app.kubernetes.io/version: {{ .Chart.AppVersion }}
9+
app.kubernetes.io/managed-by: Helm
10+
11+
---
12+
13+
apiVersion: v1
14+
kind: Secret
15+
metadata:
16+
name: backend-service-secret
17+
namespace: {{ .Release.Namespace }}
18+
annotations:
19+
kubernetes.io/service-account.name: "backend-service-account"
20+
labels:
21+
app.kubernetes.io/name: {{ .Chart.Name }}
22+
app.kubernetes.io/version: {{ .Chart.AppVersion }}
23+
app.kubernetes.io/managed-by: Helm
24+
type: kubernetes.io/service-account-token
25+
26+
---
27+
28+
apiVersion: rbac.authorization.k8s.io/v1
29+
kind: Role
30+
metadata:
31+
name: backend-service-role
32+
namespace: {{ .Release.Namespace }}
33+
labels:
34+
app.kubernetes.io/name: {{ .Chart.Name }}
35+
app.kubernetes.io/version: {{ .Chart.AppVersion }}
36+
app.kubernetes.io/managed-by: Helm
37+
rules:
38+
- apiGroups: [""]
39+
resources: ["pods", "configmaps"]
40+
verbs: ["list", "create", "update", "patch", "delete", "get"]
41+
- apiGroups: [""]
42+
resources: ["pods/log"]
43+
verbs: ["get", "list", "watch"]
44+
- apiGroups: ["batch"]
45+
resources: ["jobs"]
46+
verbs: ["create", "delete", "get", "list", "watch", "update"]
47+
48+
---
49+
50+
apiVersion: rbac.authorization.k8s.io/v1
51+
kind: RoleBinding
52+
metadata:
53+
name: backend-service-role-binding
54+
namespace: {{ .Release.Namespace }}
55+
labels:
56+
app.kubernetes.io/name: {{ .Chart.Name }}
57+
app.kubernetes.io/version: {{ .Chart.AppVersion }}
58+
app.kubernetes.io/managed-by: Helm
59+
subjects:
60+
- kind: ServiceAccount
61+
name: backend-service-account
62+
namespace: {{ .Release.Namespace }}
63+
roleRef:
64+
kind: Role
65+
name: backend-service-role
66+
apiGroup: rbac.authorization.k8s.io

‎packages/grid/helm/syft/templates/backend-service.yaml

+6-3
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,22 @@
11
apiVersion: v1
22
kind: Service
33
metadata:
4+
name: backend
45
labels:
56
app.kubernetes.io/name: {{ .Chart.Name }}
67
app.kubernetes.io/version: {{ .Chart.AppVersion }}
78
app.kubernetes.io/managed-by: Helm
8-
name: backend
99
spec:
1010
externalIPs: null
1111
ports:
12-
- name: port-0
12+
- name: web
1313
port: 80
1414
protocol: TCP
1515
targetPort: 80
16+
- name: queue
17+
port: {{ .Values.queue.port }}
18+
protocol: TCP
19+
targetPort: {{ .Values.queue.port }}
1620
selector:
17-
app.kubernetes.io/name: {{ .Chart.Name }}
1821
app.kubernetes.io/component: backend
1922
type: ClusterIP

0 commit comments

Comments
 (0)