Merge branch 'dev' into filter_dup

.github/workflows/cd-syft-dev.yml

@@ -1,9 +1,8 @@
 name: CD - Syft - Deploy `dev` to K8s
 
 on:
-  # TODO: Re-enable once merged & tested
-  # schedule:
-  #   - cron: "0 */3 * * *"
+  schedule:
+    - cron: "0 */3 * * *"
 
   workflow_dispatch:
     inputs:
@@ -15,6 +14,19 @@ jobs:
   deploy-syft-dev:
     runs-on: om-ci-16vcpu-ubuntu2204
     steps:
+      - name: Check for new changes
+        id: cache
+        if: github.event_name == 'schedule'
+        uses: actions/cache@v3
+        with:
+          path: scripts/commit_hash # we don't care about the file, just the key
+          key: dev-commit-${{ github.sha }}
+          lookup-only: true
+
+      - name: Quit if no new changes
+        if: github.event_name == 'schedule' && steps.cache.outputs.cache-hit == 'true'
+        run: exit 0
+
       - name: Permission to home directory
         run: |
           sudo chown -R $USER:$USER $HOME
@@ -150,3 +162,24 @@ jobs:
           add: "."
           push: "origin main"
           cwd: "./infrastructure/"
+
+      - name: Cleanup Azure Container Registry
+        uses: azure/CLI@v1
+        with:
+          # SKIP_LINES = latest version dev & dev-<sha> (2 lines) + keep "n" previous version (n lines) + 1
+          inlineScript: |
+            ACR_REGISTRY_NAME=${{ secrets.ACR_REGISTRY_NAME }}
+            REPO_LIST=$(az acr repository list -n $ACR_REGISTRY_NAME -o tsv)
+
+            KEEP_PREV_VERSIONS=1
+            TAIL_FROM_LINE=$((2 + $KEEP_PREV_VERSIONS + 1))
+
+            for repo in $REPO_LIST
+            do
+                echo "Cleaning up '$repo'"
+                az acr repository show-tags --name $ACR_REGISTRY_NAME --repository $repo --orderby time_desc --output tsv | tail -n +$TAIL_FROM_LINE | xargs -r -I% az acr repository delete --name $ACR_REGISTRY_NAME --image $repo:% --yes
+            done
+
+      - name: Save Commit SHA
+        # only for cache to hit
+        run: echo "${{ github.sha }}" > scripts/commit_hash

.github/workflows/pr-tests-stack.yml

@@ -538,7 +538,7 @@ jobs:
           chmod 700 get_helm.sh
           ./get_helm.sh
 
-      - name: Run integration tests
+      - name: Run K8s & Helm integration tests
         if: steps.changes.outputs.stack == 'true'
         timeout-minutes: 60
         env:
@@ -556,7 +556,7 @@ jobs:
           curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
           chmod +x devspace
           devspace version
-          # tox -e stack.test.integration.k8s
+          tox -e stack.test.integration.k8s
           tox -e syft.build.helm
           tox -e syft.package.helm
           # tox -e syft.test.helm

packages/grid/helm/helm.py

@@ -64,11 +64,9 @@ def replace_variables(d: Any) -> None:
     if "kubernetes.io/ingress.class" in d:
         d["kubernetes.io/ingress.class"] = "{{ .Values.ingress.ingressClass }}"
 
-    if "host" in d:
-        d["host"] = "{{ .Values.node.settings.hostname }}"
-
-    if "hosts" in d:
-        d["hosts"] = ["{{ .Values.node.settings.hostname }}"]
+    if "kind" in d and d["kind"] == "Ingress" and "spec" in d:
+        d["spec"]["tls"] = [{"hosts": ["{{ .Values.node.settings.hostname }}"]}]
+        d["spec"]["rules"][0]["host"] = "{{ .Values.node.settings.hostname }}"
 
 
 # parse whole tree

packages/grid/helm/syft/values.yaml

@@ -22,7 +22,7 @@ db:
 
 node:
   settings:
-    hostname: "localhost"
+    hostname: ""
     nodeName: "mynode"
     nodeType: "domain"
     versionHash: "abc"

packages/grid/k8s/manifests/ingress.yaml

@@ -10,13 +10,8 @@ spec:
       name: proxy
       port:
         number: 80
-  tls:
-    - hosts:
-        - ""
-      # secretName: Add custom TLS secret here or default certificate is used
   rules:
-    - host: ""
-      http:
+    - http:
         paths:
           - backend:
               service:

packages/grid/vagrant/almalinux_9/arm_64/Vagrantfile

@@ -0,0 +1,38 @@
+Vagrant.configure("2") do |config|
+    config.vm.box = "almalinux/9.aarch64"
+    config.vm.hostname = "almalinux-9"
+
+    config.vm.provider "virtualbox" do |vb|
+        vb.memory = "4096"
+        vb.cpus = "2"
+    end
+
+    config.vm.provider "vmware_desktop" do |vd|
+        vd.memory = "4096"
+        vd.cpus = "2"
+    end
+
+    config.vm.box_check_update = false
+
+    config.vm.provision "shell", inline: <<-SHELL
+    #!/bin/bash
+    echo "Hello from the inline Bash script!"
+    # Flush existing rules
+    iptables-save > rules.v4.old
+    iptables -F
+
+    # Set the default policies to DROP
+    iptables -P INPUT DROP
+    iptables -P FORWARD DROP
+    iptables -P OUTPUT ACCEPT
+
+    # Allow incoming SSH traffic
+    iptables -A INPUT -p tcp --dport 22 -j ACCEPT
+
+    # Save the rules so they persist across reboots
+    iptables-save > rules.v4.new
+    pwd
+
+    SHELL
+
+  end

packages/grid/vagrant/almalinux_9/x86_64/Vagrantfile

@@ -0,0 +1,37 @@
+Vagrant.configure("2") do |config|
+    config.vm.box = "almalinux/9"
+    config.vm.hostname = "almalinux-9"
+
+    config.vm.provider "virtualbox" do |vb|
+        vb.memory = "4096"
+        vb.cpus = "2"
+    end
+
+    config.vm.provider "vmware_desktop" do |vd|
+        vd.memory = "4096"
+        vd.cpus = "2"
+    end
+    config.vm.box_check_update = false
+
+    config.vm.provision "shell", inline: <<-SHELL
+    #!/bin/bash
+    echo "Hello from the inline Bash script!"
+    # Flush existing rules
+    iptables-save > rules.v4.old
+    iptables -F
+
+    # Set the default policies to DROP
+    iptables -P INPUT DROP
+    iptables -P FORWARD DROP
+    iptables -P OUTPUT ACCEPT
+
+    # Allow incoming SSH traffic
+    iptables -A INPUT -p tcp --dport 22 -j ACCEPT
+
+    # Save the rules so they persist across reboots
+    iptables-save > rules.v4.new
+    pwd
+
+    SHELL
+
+  end

packages/grid/podman/vagrant/arm_64/Vagrantfile → packages/grid/vagrant/fedora_38/arm_64/Vagrantfile

@@ -11,7 +11,7 @@ Vagrant.configure("2") do |config|
         vd.memory = "4096"
         vd.cpus = "2"
     end
-    config.vm.synced_folder "../../podman-kube", "/home/vagrant/podman-kube" ,type: "rsync"
+    config.vm.synced_folder "../../../podman/podman-kube", "/home/vagrant/podman-kube" ,type: "rsync"
     config.vm.box_check_update = false
     config.vm.network "forwarded_port", guest: 8080, host: 8080, host_ip: "127.0.0.1"
 

packages/grid/podman/vagrant/x86_64/Vagrantfile → packages/grid/vagrant/fedora_38/x86_64/Vagrantfile

@@ -1,6 +1,6 @@
 Vagrant.configure("2") do |config|
-    config.vm.box = "bento/fedora-38-x86_64"
-    config.vm.hostname = "fedora-38"
+    config.vm.box = "bento/fedora-38"
+    config.vm.hostname = "fedora-38-x86_64"
 
     config.vm.provider "virtualbox" do |vb|
         vb.memory = "4096"
@@ -11,7 +11,7 @@ Vagrant.configure("2") do |config|
         vd.memory = "4096"
         vd.cpus = "2"
     end
-    config.vm.synced_folder "../../podman-kube", "/home/vagrant/podman-kube" ,type: "rsync"
+    config.vm.synced_folder "../../../podman/podman-kube", "/home/vagrant/podman-kube" ,type: "rsync"
     config.vm.box_check_update = false
     config.vm.network "forwarded_port", guest: 8080, host: 8080, host_ip: "127.0.0.1"
 

packages/syft/src/syft/exceptions/exception.py

@@ -0,0 +1,27 @@
+# stdlib
+from typing import List
+from typing import Optional
+
+# relative
+from ..service.context import NodeServiceContext
+from ..service.response import SyftError
+from ..service.user.user_roles import ServiceRole
+
+
+class PySyftException(Exception):
+    """Base class for all PySyft exceptions."""
+
+    def __init__(self, message: str, roles: Optional[List[ServiceRole]] = None):
+        super().__init__(message)
+        self.message = message
+        self.roles = roles if roles else [ServiceRole.ADMIN]
+
+    def raise_with_context(self, context: NodeServiceContext):
+        self.context = context
+        return self
+
+    def handle(self) -> SyftError:
+        # if self.context and self.context.role in self.roles:
+        return SyftError(message=self.message)
+        # else:
+        #     return SyftError(message="Access denied to exception message.")

packages/syft/src/syft/exceptions/user.py

@@ -0,0 +1,9 @@
+# stdlib
+
+# relative
+from ..service.user.user_roles import ServiceRole
+from .exception import PySyftException
+
+UserAlreadyExistsException = PySyftException(
+    message="User already exists", roles=[ServiceRole.ADMIN]
+)

packages/syft/src/syft/node/node.py

@@ -41,6 +41,7 @@
 from ..client.api import SyftAPICall
 from ..client.api import SyftAPIData
 from ..client.api import debox_signed_syftapicall_response
+from ..exceptions.exception import PySyftException
 from ..external import OBLV
 from ..serde.deserialize import _deserialize
 from ..serde.serialize import _serialize
@@ -806,6 +807,8 @@ def handle_api_call_with_unsigned_result(
             method = self.get_service_method(_private_api_path)
             try:
                 result = method(context, *api_call.args, **api_call.kwargs)
+            except PySyftException as e:
+                return e.handle()
             except Exception:
                 result = SyftError(
                     message=f"Exception calling {api_call.path}. {traceback.format_exc()}"

packages/syft/src/syft/service/dataset/dataset.py

@@ -169,6 +169,18 @@ def _repr_html_(self) -> Any:
             data_table_line = itables.to_html_datatable(df=self.data, css=itables_css)
         else:
             data_table_line = self.data
+
+        if isinstance(self.mock, ActionObject):
+            mock_table_line = itables.to_html_datatable(
+                df=self.mock.syft_action_data, css=itables_css
+            )
+        elif isinstance(self.data, pd.DataFrame):
+            mock_table_line = itables.to_html_datatable(df=self.mock, css=itables_css)
+        else:
+            mock_table_line = self.mock
+            if isinstance(mock_table_line, SyftError):
+                mock_table_line = mock_table_line.message
+
         return f"""
             <style>
             {fonts_css}
@@ -189,7 +201,7 @@ def _repr_html_(self) -> Any:
             <p><strong>Data:</strong></p>
             {data_table_line}
             <p><strong>Mock Data:</strong></p>
-            {itables.to_html_datatable(df=self.mock, css=itables_css)}
+            {mock_table_line}
             </div>"""
 
     def _repr_markdown_(self) -> str:

packages/syft/src/syft/service/user/user_service.py

@@ -6,6 +6,7 @@
 
 # relative
 from ...abstract_node import NodeType
+from ...exceptions.user import UserAlreadyExistsException
 from ...node.credentials import SyftSigningKey
 from ...node.credentials import SyftVerifyKey
 from ...node.credentials import UserLoginCredentials
@@ -235,9 +236,7 @@ def update(
                 email=user_update.email
             )
             if user_with_email_exists:
-                return SyftError(
-                    message=f"A user with the email {user_update.email} already exists."
-                )
+                raise UserAlreadyExistsException.raise_with_context(context=context)
 
         if result.is_err():
             error_msg = (

packages/syft/tests/syft/users/user_test.py

@@ -247,9 +247,7 @@ def test_guest_user_update_to_root_email_failed(
             uid=client.me.id, user_update=user_update_to_root_email
         )
         assert isinstance(res, SyftError)
-        assert (
-            res.message == f"A user with the email {default_root_email} already exists."
-        )
+        assert res.message == "User already exists"
 
 
 def test_user_view_set_password(worker: Worker, root_client: DomainClient) -> None:
@@ -299,36 +297,27 @@ def test_user_view_set_default_admin_email_failed(
     default_root_email = get_default_root_email()
     result = ds_client.me.set_email(default_root_email)
     assert isinstance(result, SyftError)
-    assert (
-        result.message == f"A user with the email {default_root_email} already exists."
-    )
+    assert result.message == "User already exists"
+
     result_2 = guest_client.me.set_email(default_root_email)
     assert isinstance(result_2, SyftError)
-    assert (
-        result.message == f"A user with the email {default_root_email} already exists."
-    )
+    assert result_2.message == "User already exists"
 
 
 def test_user_view_set_duplicated_email(
     root_client: DomainClient, ds_client: DomainClient, guest_client: DomainClient
 ) -> None:
     result = ds_client.me.set_email(root_client.me.email)
     result2 = guest_client.me.set_email(root_client.me.email)
+
     assert isinstance(result, SyftError)
-    assert (
-        result.message
-        == f"A user with the email {root_client.me.email} already exists."
-    )
+    assert result.message == "User already exists"
     assert isinstance(result2, SyftError)
-    assert (
-        result2.message
-        == f"A user with the email {root_client.me.email} already exists."
-    )
+    assert result2.message == "User already exists"
+
     result3 = guest_client.me.set_email(ds_client.me.email)
     assert isinstance(result3, SyftError)
-    assert (
-        result3.message == f"A user with the email {ds_client.me.email} already exists."
-    )
+    assert result3.message == "User already exists"
 
 
 def test_user_view_update_name_institution_website(

tox.ini

@@ -584,6 +584,7 @@ disable_error_code = attr-defined, valid-type, no-untyped-call, arg-type
 
 [testenv:stack.test.integration.k8s]
 description = Integration Tests for Core Stack
+basepython = python3
 deps =
     {[testenv:syft]deps}
     {[testenv:hagrid]deps}