From 9e887b5eaf78d15eebf8c80aedcba169f001fa71 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Wed, 19 Jun 2024 14:08:13 +0200 Subject: [PATCH 01/30] feat: add nh3 --- packages/syft/setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 59bfee973ea..855086084b8 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -65,6 +65,7 @@ syft = rich==13.7.1 jinja2==3.1.4 tenacity==8.3.0 + nh3==0.2.17 install_requires = %(syft)s From 11b71d4fc564f0ca4cb8851251a2409860922786 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Wed, 19 Jun 2024 17:31:04 +0200 Subject: [PATCH 02/30] fix: remove deprecated warning --- packages/syft/src/syft/__init__.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/packages/syft/src/syft/__init__.py b/packages/syft/src/syft/__init__.py index d7183898935..e442a5302d9 100644 --- a/packages/syft/src/syft/__init__.py +++ b/packages/syft/src/syft/__init__.py @@ -94,21 +94,6 @@ logger.start() -try: - # third party - from IPython import get_ipython - - get_ipython() # noqa: F821 - # TODO: add back later or auto detect - # display( - # Markdown( - # "\nWarning: syft is imported in light mode by default. \ - # \nTo switch to dark mode, please run `sy.options.color_theme = 'dark'`" - # ) - # ) -except: # noqa: E722 - pass # nosec - def _patch_ipython_autocompletion() -> None: try: From 8338ba256cd3dff0115d5cde64bfaa99a9deab2b Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Wed, 19 Jun 2024 22:07:56 +0200 Subject: [PATCH 03/30] feat: sanitize html --- .../syft/util/notebook_ui/components/tabulator_template.py | 3 ++- packages/syft/src/syft/util/notebook_ui/styles.py | 2 +- packages/syft/src/syft/util/table.py | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py index ee0576cc206..676dbe3151e 100644 --- a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py +++ b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py @@ -8,6 +8,7 @@ from IPython.display import display import jinja2 from loguru import logger +import nh3 # relative from ...assets import load_css @@ -130,7 +131,7 @@ def build_tabulator_table( uid=uid, columns=json.dumps(column_data), row_header=json.dumps(row_header), - data=json.dumps(table_data), + data=nh3.clean(json.dumps(table_data)), css=css, js=js, index_field_name=TABLE_INDEX_KEY, diff --git a/packages/syft/src/syft/util/notebook_ui/styles.py b/packages/syft/src/syft/util/notebook_ui/styles.py index a250c20a7dc..2e780394687 100644 --- a/packages/syft/src/syft/util/notebook_ui/styles.py +++ b/packages/syft/src/syft/util/notebook_ui/styles.py @@ -28,6 +28,6 @@ CSS_CODE = f""" """ diff --git a/packages/syft/src/syft/util/table.py b/packages/syft/src/syft/util/table.py index 998e022bdbd..cf05e5f7e45 100644 --- a/packages/syft/src/syft/util/table.py +++ b/packages/syft/src/syft/util/table.py @@ -8,6 +8,7 @@ # third party from loguru import logger +import nh3 # relative from .notebook_ui.components.table_template import TABLE_INDEX_KEY @@ -89,7 +90,7 @@ def _create_table_rows( if "id" in ret_val: del ret_val["id"] for key in ret_val.keys(): - cols[key].append(ret_val[key]) + cols[key].append(nh3.clean(ret_val[key])) else: for field in extra_fields: value = item @@ -134,7 +135,7 @@ def _create_table_rows( except Exception as e: print(e) value = None - cols[field].append(str(value)) + cols[field].append(nh3.clean(str(value))) col_lengths = {len(cols[col]) for col in cols.keys()} if len(col_lengths) != 1: From e485e54375293efb88e389ebcbc78b1997665898 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Wed, 19 Jun 2024 22:08:10 +0200 Subject: [PATCH 04/30] feat: add init sanitizer --- packages/syft/src/syft/__init__.py | 67 +++++++++++++++++++ .../components/tabulator_template.py | 2 +- packages/syft/src/syft/util/table.py | 5 +- 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/__init__.py b/packages/syft/src/syft/__init__.py index e442a5302d9..e3ecc2e472c 100644 --- a/packages/syft/src/syft/__init__.py +++ b/packages/syft/src/syft/__init__.py @@ -4,10 +4,13 @@ from collections.abc import Callable import pathlib from pathlib import Path +import re import sys from types import MethodType from typing import Any +from syft.types.dicttuple import DictTuple + # relative from .abstract_node import NodeSideType # noqa: F401 from .abstract_node import NodeType # noqa: F401 @@ -95,9 +98,73 @@ logger.start() +def _patch_ipython_sanitization() -> None: + from IPython import get_ipython + from IPython.display import display_html, display_markdown + + ip = get_ipython() + if ip is None: + return + + from importlib import resources + import nh3 + from .util.notebook_ui.styles import FONT_CSS, ITABLES_CSS, JS_DOWNLOAD_FONTS, CSS_CODE + from .util.assets import load_js, load_css + + tabulator_js = load_js('tabulator.min.js') + tabulator_js = tabulator_js.replace( + "define(t)", "define('tabulator-tables', [], t)" + ) + + SKIP_SANITIZE = [ + FONT_CSS, + ITABLES_CSS, + CSS_CODE, + JS_DOWNLOAD_FONTS, + tabulator_js, + load_css("tabulator_pysyft.min.css"), + load_js("table.js"), + ] + + css_reinsert = f""" + + +{JS_DOWNLOAD_FONTS} +{CSS_CODE} +""" + + escaped_js_css = re.compile("|".join(re.escape(substr) for substr in SKIP_SANITIZE), re.IGNORECASE | re.MULTILINE) + + table_template = resources.files('syft.assets.jinja').joinpath('table.jinja2').read_text() + table_template = table_template.strip() + table_template = re.sub(r'\\{\\{.*?\\}\\}', '.*?', re.escape(table_template)) + escaped_template = re.compile(table_template, re.DOTALL | re.VERBOSE) + + def display_sanitized_html(obj) -> None: + if hasattr(obj, "_repr_html_"): + _str = obj._repr_html_() + matching_template = escaped_template.findall(_str) + print("matching_template") + _str = escaped_template.sub('', _str) + _str = escaped_js_css.sub('', _str) + _str = nh3.clean(_str) + return f"{css_reinsert} {_str} {"\n".join(matching_template)}" + + def display_sanitized_md(obj) -> None: + if hasattr(obj, "_repr_markdown_"): + return nh3.clean(obj._repr_markdown_()) + + ip.display_formatter.formatters['text/html'].for_type(SyftObject, display_sanitized_html) + ip.display_formatter.formatters['text/html'].for_type(DictTuple, display_sanitized_html) + ip.display_formatter.formatters['text/markdown'].for_type(SyftObject, display_sanitized_md) + +_patch_ipython_sanitization() + + def _patch_ipython_autocompletion() -> None: try: # third party + from IPython import get_ipython from IPython.core.guarded_eval import EVALUATION_POLICIES except ImportError: return diff --git a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py index 676dbe3151e..eabaf8c2a70 100644 --- a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py +++ b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py @@ -131,7 +131,7 @@ def build_tabulator_table( uid=uid, columns=json.dumps(column_data), row_header=json.dumps(row_header), - data=nh3.clean(json.dumps(table_data)), + data=(json.dumps(table_data)), css=css, js=js, index_field_name=TABLE_INDEX_KEY, diff --git a/packages/syft/src/syft/util/table.py b/packages/syft/src/syft/util/table.py index cf05e5f7e45..8deecf0190b 100644 --- a/packages/syft/src/syft/util/table.py +++ b/packages/syft/src/syft/util/table.py @@ -90,7 +90,10 @@ def _create_table_rows( if "id" in ret_val: del ret_val["id"] for key in ret_val.keys(): - cols[key].append(nh3.clean(ret_val[key])) + # if isinstance(ret_val[key], str): + # cols[key].append(nh3.clean(ret_val[key])) + # else: + cols[key].append(ret_val[key]) else: for field in extra_fields: value = item From 3437e2f6f4218aebc036a309dd88cd5163b69e42 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 20 Jun 2024 21:02:35 +0200 Subject: [PATCH 05/30] fix: APISubModulesView now uses the new table --- packages/syft/src/syft/client/api.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/client/api.py b/packages/syft/src/syft/client/api.py index c4a3a1b40a9..67d5fb1f21e 100644 --- a/packages/syft/src/syft/client/api.py +++ b/packages/syft/src/syft/client/api.py @@ -23,6 +23,7 @@ from pydantic import TypeAdapter from result import OkErr from result import Result +from syft.util.notebook_ui.components.tabulator_template import build_tabulator_table, show_table from typeguard import check_type # relative @@ -730,9 +731,9 @@ def recursively_get_submodules( APISubModulesView(submodule=submodule_name, endpoints=child_paths) ) - return list_dict_repr_html(views) - # return NotImplementedError - + return build_tabulator_table(views) + + # should never happen? results = self.get_all() return results._repr_html_() From 95cc27e438ea3297ba053cbdbbfb72e0f29243cc Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 20 Jun 2024 23:35:13 +0200 Subject: [PATCH 06/30] fix: sanitize but keep id and type intact --- .../syft/util/notebook_ui/components/tabulator_template.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py index eabaf8c2a70..b7b07277181 100644 --- a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py +++ b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py @@ -69,7 +69,7 @@ def format_dict(data: Any) -> str: return data if set(data.keys()) != {"type", "value"}: - return str(data) + return nh3.clean(str(data)) if "badge" in data["type"]: return Badge(value=data["value"], badge_class=data["type"]).to_html() @@ -87,7 +87,7 @@ def format_table_data(table_data: list[dict[str, Any]]) -> list[dict[str, str]]: row_formatted: dict[str, str] = {} for k, v in row.items(): if isinstance(v, str): - row_formatted[k] = v.replace("\n", "
") + row_formatted[k] = nh3.clean(v.replace("\n", "
")) continue v_formatted = format_dict(v) row_formatted[k] = v_formatted @@ -131,7 +131,7 @@ def build_tabulator_table( uid=uid, columns=json.dumps(column_data), row_header=json.dumps(row_header), - data=(json.dumps(table_data)), + data=json.dumps(table_data), css=css, js=js, index_field_name=TABLE_INDEX_KEY, From bf1d42c4af2f6f9a32c2799490c0b827727fef21 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 20 Jun 2024 23:39:34 +0200 Subject: [PATCH 07/30] fix: move ipython patches to its own file --- packages/syft/src/syft/__init__.py | 147 +---------- .../syft/src/syft/util/ipython_patches.py | 231 ++++++++++++++++++ 2 files changed, 233 insertions(+), 145 deletions(-) create mode 100644 packages/syft/src/syft/util/ipython_patches.py diff --git a/packages/syft/src/syft/__init__.py b/packages/syft/src/syft/__init__.py index e3ecc2e472c..f6f0db9a929 100644 --- a/packages/syft/src/syft/__init__.py +++ b/packages/syft/src/syft/__init__.py @@ -10,6 +10,7 @@ from typing import Any from syft.types.dicttuple import DictTuple +from syft.util.ipython_patches import patch_ipython # relative from .abstract_node import NodeSideType # noqa: F401 @@ -97,151 +98,7 @@ logger.start() - -def _patch_ipython_sanitization() -> None: - from IPython import get_ipython - from IPython.display import display_html, display_markdown - - ip = get_ipython() - if ip is None: - return - - from importlib import resources - import nh3 - from .util.notebook_ui.styles import FONT_CSS, ITABLES_CSS, JS_DOWNLOAD_FONTS, CSS_CODE - from .util.assets import load_js, load_css - - tabulator_js = load_js('tabulator.min.js') - tabulator_js = tabulator_js.replace( - "define(t)", "define('tabulator-tables', [], t)" - ) - - SKIP_SANITIZE = [ - FONT_CSS, - ITABLES_CSS, - CSS_CODE, - JS_DOWNLOAD_FONTS, - tabulator_js, - load_css("tabulator_pysyft.min.css"), - load_js("table.js"), - ] - - css_reinsert = f""" - - -{JS_DOWNLOAD_FONTS} -{CSS_CODE} -""" - - escaped_js_css = re.compile("|".join(re.escape(substr) for substr in SKIP_SANITIZE), re.IGNORECASE | re.MULTILINE) - - table_template = resources.files('syft.assets.jinja').joinpath('table.jinja2').read_text() - table_template = table_template.strip() - table_template = re.sub(r'\\{\\{.*?\\}\\}', '.*?', re.escape(table_template)) - escaped_template = re.compile(table_template, re.DOTALL | re.VERBOSE) - - def display_sanitized_html(obj) -> None: - if hasattr(obj, "_repr_html_"): - _str = obj._repr_html_() - matching_template = escaped_template.findall(_str) - print("matching_template") - _str = escaped_template.sub('', _str) - _str = escaped_js_css.sub('', _str) - _str = nh3.clean(_str) - return f"{css_reinsert} {_str} {"\n".join(matching_template)}" - - def display_sanitized_md(obj) -> None: - if hasattr(obj, "_repr_markdown_"): - return nh3.clean(obj._repr_markdown_()) - - ip.display_formatter.formatters['text/html'].for_type(SyftObject, display_sanitized_html) - ip.display_formatter.formatters['text/html'].for_type(DictTuple, display_sanitized_html) - ip.display_formatter.formatters['text/markdown'].for_type(SyftObject, display_sanitized_md) - -_patch_ipython_sanitization() - - -def _patch_ipython_autocompletion() -> None: - try: - # third party - from IPython import get_ipython - from IPython.core.guarded_eval import EVALUATION_POLICIES - except ImportError: - return - - ipython = get_ipython() - if ipython is None: - return - - try: - # this allows property getters to be used in nested autocomplete - ipython.Completer.evaluation = "limited" - ipython.Completer.use_jedi = False - policy = EVALUATION_POLICIES["limited"] - - policy.allowed_getattr_external.update( - [ - ("syft.client.api", "APIModule"), - ("syft.client.api", "SyftAPI"), - ] - ) - original_can_get_attr = policy.can_get_attr - - def patched_can_get_attr(value: Any, attr: str) -> bool: - attr_name = "__syft_allow_autocomplete__" - # first check if exist to prevent side effects - if hasattr(value, attr_name) and attr in getattr(value, attr_name, []): - if attr in dir(value): - return True - else: - return False - else: - return original_can_get_attr(value, attr) - - policy.can_get_attr = patched_can_get_attr - except Exception: - print("Failed to patch ipython autocompletion for syft property getters") - - try: - # this constraints the completions for autocomplete. - # if __syft_dir__ is defined we only autocomplete those properties - # stdlib - import re - - original_attr_matches = ipython.Completer.attr_matches - - def patched_attr_matches(self, text: str) -> list[str]: # type: ignore - res = original_attr_matches(text) - m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer) - if not m2: - return res - expr, _ = m2.group(1, 2) - obj = self._evaluate_expr(expr) - if isinstance(obj, SyftObject) and hasattr(obj, "__syft_dir__"): - # here we filter all autocomplete results to only contain those - # defined in __syft_dir__, however the original autocomplete prefixes - # have the full path, while __syft_dir__ only defines the attr - attrs = set(obj.__syft_dir__()) - new_res = [] - for r in res: - splitted = r.split(".") - if len(splitted) > 1: - attr_name = splitted[-1] - if attr_name in attrs: - new_res.append(r) - return new_res - else: - return res - - ipython.Completer.attr_matches = MethodType( - patched_attr_matches, ipython.Completer - ) - except Exception: - print("Failed to patch syft autocompletion for __syft_dir__") - - -_patch_ipython_autocompletion() - +patch_ipython() def module_property(func: Any) -> Callable: """Decorator to turn module functions into properties. diff --git a/packages/syft/src/syft/util/ipython_patches.py b/packages/syft/src/syft/util/ipython_patches.py new file mode 100644 index 00000000000..e4ce3876aee --- /dev/null +++ b/packages/syft/src/syft/util/ipython_patches.py @@ -0,0 +1,231 @@ +# stdlib +from collections.abc import Callable +import pathlib +from pathlib import Path +import re +import sys +from types import MethodType +from typing import Any + +from syft.types.dicttuple import DictTuple + +# relative +from .abstract_node import NodeSideType # noqa: F401 +from .abstract_node import NodeType # noqa: F401 +from .client.client import connect # noqa: F401 +from .client.client import login # noqa: F401 +from .client.client import login_as_guest # noqa: F401 +from .client.client import register # noqa: F401 +from .client.domain_client import DomainClient # noqa: F401 +from .client.gateway_client import GatewayClient # noqa: F401 +from .client.registry import DomainRegistry # noqa: F401 +from .client.registry import EnclaveRegistry # noqa: F401 +from .client.registry import NetworkRegistry # noqa: F401 +from .client.search import Search # noqa: F401 +from .client.search import SearchResults # noqa: F401 +from .client.user_settings import UserSettings # noqa: F401 +from .client.user_settings import settings # noqa: F401 +from .custom_worker.config import DockerWorkerConfig # noqa: F401 +from .custom_worker.config import PrebuiltWorkerConfig # noqa: F401 +from .node.credentials import SyftSigningKey # noqa: F401 +from .node.domain import Domain # noqa: F401 +from .node.enclave import Enclave # noqa: F401 +from .node.gateway import Gateway # noqa: F401 +from .node.server import serve_node # noqa: F401 +from .node.server import serve_node as bind_worker # noqa: F401 +from .node.worker import Worker # noqa: F401 +from .orchestra import Orchestra as orchestra # noqa: F401 +from .protocol.data_protocol import bump_protocol_version # noqa: F401 +from .protocol.data_protocol import check_or_stage_protocol # noqa: F401 +from .protocol.data_protocol import get_data_protocol # noqa: F401 +from .protocol.data_protocol import stage_protocol_changes # noqa: F401 +from .serde import NOTHING # noqa: F401 +from .serde.deserialize import _deserialize as deserialize # noqa: F401 +from .serde.serializable import serializable # noqa: F401 +from .serde.serialize import _serialize as serialize # noqa: F401 +from .service.action.action_data_empty import ActionDataEmpty # noqa: F401 +from .service.action.action_object import ActionObject # noqa: F401 +from .service.action.plan import Plan # noqa: F401 +from .service.action.plan import planify # noqa: F401 +from .service.api.api import api_endpoint # noqa: F401 +from .service.api.api import api_endpoint_method # noqa: F401 +from .service.api.api import create_new_api_endpoint as TwinAPIEndpoint # noqa: F401 +from .service.code.user_code import UserCodeStatus # noqa: F401; noqa: F401 +from .service.code.user_code import syft_function # noqa: F401; noqa: F401 +from .service.code.user_code import syft_function_single_use # noqa: F401; noqa: F401 +from .service.data_subject import DataSubjectCreate as DataSubject # noqa: F401 +from .service.dataset.dataset import Contributor # noqa: F401 +from .service.dataset.dataset import CreateAsset as Asset # noqa: F401 +from .service.dataset.dataset import CreateDataset as Dataset # noqa: F401 +from .service.notification.notifications import NotificationStatus # noqa: F401 +from .service.policy.policy import CustomInputPolicy # noqa: F401 +from .service.policy.policy import CustomOutputPolicy # noqa: F401 +from .service.policy.policy import ExactMatch # noqa: F401 +from .service.policy.policy import SingleExecutionExactOutput # noqa: F401 +from .service.policy.policy import UserInputPolicy # noqa: F401 +from .service.policy.policy import UserOutputPolicy # noqa: F401 +from .service.project.project import ProjectSubmit as Project # noqa: F401 +from .service.request.request import SubmitRequest as Request # noqa: F401 +from .service.response import SyftError # noqa: F401 +from .service.response import SyftNotReady # noqa: F401 +from .service.response import SyftSuccess # noqa: F401 +from .service.user.roles import Roles as roles # noqa: F401 +from .service.user.user_service import UserService # noqa: F401 +from .stable_version import LATEST_STABLE_SYFT +from .types.syft_object import SyftObject +from .types.twin_object import TwinObject # noqa: F401 +from .types.uid import UID # noqa: F401 +from .util import filterwarnings # noqa: F401 +from .util import logger # noqa: F401 +from .util import options # noqa: F401 +from .util.autoreload import disable_autoreload # noqa: F401 +from .util.autoreload import enable_autoreload # noqa: F401 +from .util.telemetry import instrument # noqa: F401 +from .util.util import autocache # noqa: F401 +from .util.util import get_root_data_path # noqa: F401 +from .util.version_compare import make_requires + +def _patch_ipython_sanitization() -> None: + try: + from IPython import get_ipython + from IPython.display import display_html, display_markdown + except ImportError: + return + + ip = get_ipython() + if ip is None: + return + + from importlib import resources + import nh3 + from .util.notebook_ui.styles import FONT_CSS, ITABLES_CSS, JS_DOWNLOAD_FONTS, CSS_CODE + from .util.assets import load_js, load_css + + tabulator_js = load_js('tabulator.min.js') + tabulator_js = tabulator_js.replace( + "define(t)", "define('tabulator-tables', [], t)" + ) + + SKIP_SANITIZE = [ + FONT_CSS, + ITABLES_CSS, + CSS_CODE, + JS_DOWNLOAD_FONTS, + tabulator_js, + load_css("tabulator_pysyft.min.css"), + load_js("table.js"), + ] + + css_reinsert = f""" + + +{JS_DOWNLOAD_FONTS} +{CSS_CODE} +""" + + escaped_js_css = re.compile("|".join(re.escape(substr) for substr in SKIP_SANITIZE), re.IGNORECASE | re.MULTILINE) + + table_template = resources.files('syft.assets.jinja').joinpath('table.jinja2').read_text() + table_template = table_template.strip() + table_template = re.sub(r'\\{\\{.*?\\}\\}', '.*?', re.escape(table_template)) + escaped_template = re.compile(table_template, re.DOTALL | re.VERBOSE) + + def display_sanitized_html(obj) -> str | None: + if hasattr(obj, "_repr_html_") and callable(obj._repr_html_): + _str = obj._repr_html_() + matching_template = escaped_template.findall(_str) + _str = escaped_template.sub('', _str) + _str = escaped_js_css.sub('', _str) + _str = nh3.clean(_str) + return f"{css_reinsert} {_str} {"\n".join(matching_template)}" + + def display_sanitized_md(obj) -> None: + if hasattr(obj, "_repr_markdown_"): + return nh3.clean(obj._repr_markdown_()) + + ip.display_formatter.formatters['text/html'].for_type(SyftObject, display_sanitized_html) + ip.display_formatter.formatters['text/html'].for_type(DictTuple, display_sanitized_html) + ip.display_formatter.formatters['text/markdown'].for_type(SyftObject, display_sanitized_md) + + +def _patch_ipython_autocompletion() -> None: + try: + # third party + from IPython import get_ipython + from IPython.core.guarded_eval import EVALUATION_POLICIES + except ImportError: + return + + ipython = get_ipython() + if ipython is None: + return + + try: + # this allows property getters to be used in nested autocomplete + ipython.Completer.evaluation = "limited" + ipython.Completer.use_jedi = False + policy = EVALUATION_POLICIES["limited"] + + policy.allowed_getattr_external.update( + [ + ("syft.client.api", "APIModule"), + ("syft.client.api", "SyftAPI"), + ] + ) + original_can_get_attr = policy.can_get_attr + + def patched_can_get_attr(value: Any, attr: str) -> bool: + attr_name = "__syft_allow_autocomplete__" + # first check if exist to prevent side effects + if hasattr(value, attr_name) and attr in getattr(value, attr_name, []): + if attr in dir(value): + return True + else: + return False + else: + return original_can_get_attr(value, attr) + + policy.can_get_attr = patched_can_get_attr + except Exception: + print("Failed to patch ipython autocompletion for syft property getters") + + try: + # this constraints the completions for autocomplete. + # if __syft_dir__ is defined we only autocomplete those properties + original_attr_matches = ipython.Completer.attr_matches + + def patched_attr_matches(self, text: str) -> list[str]: # type: ignore + res = original_attr_matches(text) + m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer) + if not m2: + return res + expr, _ = m2.group(1, 2) + obj = self._evaluate_expr(expr) + if isinstance(obj, SyftObject) and hasattr(obj, "__syft_dir__"): + # here we filter all autocomplete results to only contain those + # defined in __syft_dir__, however the original autocomplete prefixes + # have the full path, while __syft_dir__ only defines the attr + attrs = set(obj.__syft_dir__()) + new_res = [] + for r in res: + splitted = r.split(".") + if len(splitted) > 1: + attr_name = splitted[-1] + if attr_name in attrs: + new_res.append(r) + return new_res + else: + return res + + ipython.Completer.attr_matches = MethodType( + patched_attr_matches, ipython.Completer + ) + except Exception: + print("Failed to patch syft autocompletion for __syft_dir__") + + +def patch_ipython(): + _patch_ipython_sanitization() + _patch_ipython_autocompletion() + + From cd2be2d37e9be6198e3a3653fb53abc313e0a761 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 20 Jun 2024 23:44:47 +0200 Subject: [PATCH 08/30] fix: more sanitization --- packages/syft/src/syft/__init__.py | 8 +-- packages/syft/src/syft/client/api.py | 5 +- .../syft/src/syft/util/ipython_patches.py | 67 +++++++++++-------- packages/syft/src/syft/util/table.py | 3 - 4 files changed, 44 insertions(+), 39 deletions(-) diff --git a/packages/syft/src/syft/__init__.py b/packages/syft/src/syft/__init__.py index f6f0db9a929..f4b28443e40 100644 --- a/packages/syft/src/syft/__init__.py +++ b/packages/syft/src/syft/__init__.py @@ -4,14 +4,9 @@ from collections.abc import Callable import pathlib from pathlib import Path -import re import sys -from types import MethodType from typing import Any -from syft.types.dicttuple import DictTuple -from syft.util.ipython_patches import patch_ipython - # relative from .abstract_node import NodeSideType # noqa: F401 from .abstract_node import NodeType # noqa: F401 @@ -75,7 +70,6 @@ from .service.user.roles import Roles as roles # noqa: F401 from .service.user.user_service import UserService # noqa: F401 from .stable_version import LATEST_STABLE_SYFT -from .types.syft_object import SyftObject from .types.twin_object import TwinObject # noqa: F401 from .types.uid import UID # noqa: F401 from .util import filterwarnings # noqa: F401 @@ -83,6 +77,7 @@ from .util import options # noqa: F401 from .util.autoreload import disable_autoreload # noqa: F401 from .util.autoreload import enable_autoreload # noqa: F401 +from .util.ipython_patches import patch_ipython from .util.telemetry import instrument # noqa: F401 from .util.util import autocache # noqa: F401 from .util.util import get_root_data_path # noqa: F401 @@ -100,6 +95,7 @@ patch_ipython() + def module_property(func: Any) -> Callable: """Decorator to turn module functions into properties. Function names must be prefixed with an underscore.""" diff --git a/packages/syft/src/syft/client/api.py b/packages/syft/src/syft/client/api.py index 67d5fb1f21e..6622ed3e81d 100644 --- a/packages/syft/src/syft/client/api.py +++ b/packages/syft/src/syft/client/api.py @@ -23,7 +23,6 @@ from pydantic import TypeAdapter from result import OkErr from result import Result -from syft.util.notebook_ui.components.tabulator_template import build_tabulator_table, show_table from typeguard import check_type # relative @@ -63,7 +62,7 @@ from ..types.uid import UID from ..util.autoreload import autoreload_enabled from ..util.markdown import as_markdown_python_code -from ..util.table import list_dict_repr_html +from ..util.notebook_ui.components.tabulator_template import build_tabulator_table from ..util.telemetry import instrument from ..util.util import prompt_warning_message from .connection import NodeConnection @@ -732,7 +731,7 @@ def recursively_get_submodules( ) return build_tabulator_table(views) - + # should never happen? results = self.get_all() return results._repr_html_() diff --git a/packages/syft/src/syft/util/ipython_patches.py b/packages/syft/src/syft/util/ipython_patches.py index e4ce3876aee..ab8e0a354d9 100644 --- a/packages/syft/src/syft/util/ipython_patches.py +++ b/packages/syft/src/syft/util/ipython_patches.py @@ -1,15 +1,10 @@ # stdlib -from collections.abc import Callable -import pathlib -from pathlib import Path import re -import sys from types import MethodType from typing import Any -from syft.types.dicttuple import DictTuple - # relative +from ..types.dicttuple import DictTuple from .abstract_node import NodeSideType # noqa: F401 from .abstract_node import NodeType # noqa: F401 from .client.client import connect # noqa: F401 @@ -71,7 +66,6 @@ from .service.response import SyftSuccess # noqa: F401 from .service.user.roles import Roles as roles # noqa: F401 from .service.user.user_service import UserService # noqa: F401 -from .stable_version import LATEST_STABLE_SYFT from .types.syft_object import SyftObject from .types.twin_object import TwinObject # noqa: F401 from .types.uid import UID # noqa: F401 @@ -83,12 +77,12 @@ from .util.telemetry import instrument # noqa: F401 from .util.util import autocache # noqa: F401 from .util.util import get_root_data_path # noqa: F401 -from .util.version_compare import make_requires + def _patch_ipython_sanitization() -> None: try: + # third party from IPython import get_ipython - from IPython.display import display_html, display_markdown except ImportError: return @@ -96,12 +90,21 @@ def _patch_ipython_sanitization() -> None: if ip is None: return + # stdlib from importlib import resources + + # third party import nh3 - from .util.notebook_ui.styles import FONT_CSS, ITABLES_CSS, JS_DOWNLOAD_FONTS, CSS_CODE - from .util.assets import load_js, load_css - tabulator_js = load_js('tabulator.min.js') + # relative + from .util.assets import load_css + from .util.assets import load_js + from .util.notebook_ui.styles import CSS_CODE + from .util.notebook_ui.styles import FONT_CSS + from .util.notebook_ui.styles import ITABLES_CSS + from .util.notebook_ui.styles import JS_DOWNLOAD_FONTS + + tabulator_js = load_js("tabulator.min.js") tabulator_js = tabulator_js.replace( "define(t)", "define('tabulator-tables', [], t)" ) @@ -123,29 +126,41 @@ def _patch_ipython_sanitization() -> None: {CSS_CODE} """ - escaped_js_css = re.compile("|".join(re.escape(substr) for substr in SKIP_SANITIZE), re.IGNORECASE | re.MULTILINE) + escaped_js_css = re.compile( + "|".join(re.escape(substr) for substr in SKIP_SANITIZE), + re.IGNORECASE | re.MULTILINE, + ) - table_template = resources.files('syft.assets.jinja').joinpath('table.jinja2').read_text() + table_template = ( + resources.files("syft.assets.jinja").joinpath("table.jinja2").read_text() + ) table_template = table_template.strip() - table_template = re.sub(r'\\{\\{.*?\\}\\}', '.*?', re.escape(table_template)) + table_template = re.sub(r"\\{\\{.*?\\}\\}", ".*?", re.escape(table_template)) escaped_template = re.compile(table_template, re.DOTALL | re.VERBOSE) - def display_sanitized_html(obj) -> str | None: - if hasattr(obj, "_repr_html_") and callable(obj._repr_html_): - _str = obj._repr_html_() + def display_sanitized_html(obj: SyftObject | DictTuple) -> str | None: + if hasattr(obj, "_repr_html_") and callable(obj._repr_html_): # type: ignore + _str = obj._repr_html_() # type: ignore matching_template = escaped_template.findall(_str) - _str = escaped_template.sub('', _str) - _str = escaped_js_css.sub('', _str) + _str = escaped_template.sub("", _str) + _str = escaped_js_css.sub("", _str) _str = nh3.clean(_str) return f"{css_reinsert} {_str} {"\n".join(matching_template)}" + return None - def display_sanitized_md(obj) -> None: + def display_sanitized_md(obj: SyftObject) -> None: if hasattr(obj, "_repr_markdown_"): return nh3.clean(obj._repr_markdown_()) - ip.display_formatter.formatters['text/html'].for_type(SyftObject, display_sanitized_html) - ip.display_formatter.formatters['text/html'].for_type(DictTuple, display_sanitized_html) - ip.display_formatter.formatters['text/markdown'].for_type(SyftObject, display_sanitized_md) + ip.display_formatter.formatters["text/html"].for_type( + SyftObject, display_sanitized_html + ) + ip.display_formatter.formatters["text/html"].for_type( + DictTuple, display_sanitized_html + ) + ip.display_formatter.formatters["text/markdown"].for_type( + SyftObject, display_sanitized_md + ) def _patch_ipython_autocompletion() -> None: @@ -224,8 +239,6 @@ def patched_attr_matches(self, text: str) -> list[str]: # type: ignore print("Failed to patch syft autocompletion for __syft_dir__") -def patch_ipython(): +def patch_ipython() -> None: _patch_ipython_sanitization() _patch_ipython_autocompletion() - - diff --git a/packages/syft/src/syft/util/table.py b/packages/syft/src/syft/util/table.py index 8deecf0190b..f4965cb1ef0 100644 --- a/packages/syft/src/syft/util/table.py +++ b/packages/syft/src/syft/util/table.py @@ -90,9 +90,6 @@ def _create_table_rows( if "id" in ret_val: del ret_val["id"] for key in ret_val.keys(): - # if isinstance(ret_val[key], str): - # cols[key].append(nh3.clean(ret_val[key])) - # else: cols[key].append(ret_val[key]) else: for field in extra_fields: From 1ac431f8db36f41cec6994a79f7f07ba3b3bf2a1 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 20 Jun 2024 23:55:37 +0200 Subject: [PATCH 09/30] fix: imports in ipython_patches.py --- .../syft/src/syft/util/ipython_patches.py | 71 ------------------- 1 file changed, 71 deletions(-) diff --git a/packages/syft/src/syft/util/ipython_patches.py b/packages/syft/src/syft/util/ipython_patches.py index ab8e0a354d9..04bea086c5e 100644 --- a/packages/syft/src/syft/util/ipython_patches.py +++ b/packages/syft/src/syft/util/ipython_patches.py @@ -5,78 +5,7 @@ # relative from ..types.dicttuple import DictTuple -from .abstract_node import NodeSideType # noqa: F401 -from .abstract_node import NodeType # noqa: F401 -from .client.client import connect # noqa: F401 -from .client.client import login # noqa: F401 -from .client.client import login_as_guest # noqa: F401 -from .client.client import register # noqa: F401 -from .client.domain_client import DomainClient # noqa: F401 -from .client.gateway_client import GatewayClient # noqa: F401 -from .client.registry import DomainRegistry # noqa: F401 -from .client.registry import EnclaveRegistry # noqa: F401 -from .client.registry import NetworkRegistry # noqa: F401 -from .client.search import Search # noqa: F401 -from .client.search import SearchResults # noqa: F401 -from .client.user_settings import UserSettings # noqa: F401 -from .client.user_settings import settings # noqa: F401 -from .custom_worker.config import DockerWorkerConfig # noqa: F401 -from .custom_worker.config import PrebuiltWorkerConfig # noqa: F401 -from .node.credentials import SyftSigningKey # noqa: F401 -from .node.domain import Domain # noqa: F401 -from .node.enclave import Enclave # noqa: F401 -from .node.gateway import Gateway # noqa: F401 -from .node.server import serve_node # noqa: F401 -from .node.server import serve_node as bind_worker # noqa: F401 -from .node.worker import Worker # noqa: F401 -from .orchestra import Orchestra as orchestra # noqa: F401 -from .protocol.data_protocol import bump_protocol_version # noqa: F401 -from .protocol.data_protocol import check_or_stage_protocol # noqa: F401 -from .protocol.data_protocol import get_data_protocol # noqa: F401 -from .protocol.data_protocol import stage_protocol_changes # noqa: F401 -from .serde import NOTHING # noqa: F401 -from .serde.deserialize import _deserialize as deserialize # noqa: F401 -from .serde.serializable import serializable # noqa: F401 -from .serde.serialize import _serialize as serialize # noqa: F401 -from .service.action.action_data_empty import ActionDataEmpty # noqa: F401 -from .service.action.action_object import ActionObject # noqa: F401 -from .service.action.plan import Plan # noqa: F401 -from .service.action.plan import planify # noqa: F401 -from .service.api.api import api_endpoint # noqa: F401 -from .service.api.api import api_endpoint_method # noqa: F401 -from .service.api.api import create_new_api_endpoint as TwinAPIEndpoint # noqa: F401 -from .service.code.user_code import UserCodeStatus # noqa: F401; noqa: F401 -from .service.code.user_code import syft_function # noqa: F401; noqa: F401 -from .service.code.user_code import syft_function_single_use # noqa: F401; noqa: F401 -from .service.data_subject import DataSubjectCreate as DataSubject # noqa: F401 -from .service.dataset.dataset import Contributor # noqa: F401 -from .service.dataset.dataset import CreateAsset as Asset # noqa: F401 -from .service.dataset.dataset import CreateDataset as Dataset # noqa: F401 -from .service.notification.notifications import NotificationStatus # noqa: F401 -from .service.policy.policy import CustomInputPolicy # noqa: F401 -from .service.policy.policy import CustomOutputPolicy # noqa: F401 -from .service.policy.policy import ExactMatch # noqa: F401 -from .service.policy.policy import SingleExecutionExactOutput # noqa: F401 -from .service.policy.policy import UserInputPolicy # noqa: F401 -from .service.policy.policy import UserOutputPolicy # noqa: F401 -from .service.project.project import ProjectSubmit as Project # noqa: F401 -from .service.request.request import SubmitRequest as Request # noqa: F401 -from .service.response import SyftError # noqa: F401 -from .service.response import SyftNotReady # noqa: F401 -from .service.response import SyftSuccess # noqa: F401 -from .service.user.roles import Roles as roles # noqa: F401 -from .service.user.user_service import UserService # noqa: F401 from .types.syft_object import SyftObject -from .types.twin_object import TwinObject # noqa: F401 -from .types.uid import UID # noqa: F401 -from .util import filterwarnings # noqa: F401 -from .util import logger # noqa: F401 -from .util import options # noqa: F401 -from .util.autoreload import disable_autoreload # noqa: F401 -from .util.autoreload import enable_autoreload # noqa: F401 -from .util.telemetry import instrument # noqa: F401 -from .util.util import autocache # noqa: F401 -from .util.util import get_root_data_path # noqa: F401 def _patch_ipython_sanitization() -> None: From d48082933e9e35856789917f7a96b1194dfff968 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Thu, 20 Jun 2024 23:57:49 +0200 Subject: [PATCH 10/30] fix: ipython_patches.py -> patch_ipython.py --- packages/syft/src/syft/__init__.py | 2 +- .../syft/src/syft/util/{ipython_patches.py => patch_ipython.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename packages/syft/src/syft/util/{ipython_patches.py => patch_ipython.py} (100%) diff --git a/packages/syft/src/syft/__init__.py b/packages/syft/src/syft/__init__.py index f4b28443e40..d1df56cf81a 100644 --- a/packages/syft/src/syft/__init__.py +++ b/packages/syft/src/syft/__init__.py @@ -77,7 +77,7 @@ from .util import options # noqa: F401 from .util.autoreload import disable_autoreload # noqa: F401 from .util.autoreload import enable_autoreload # noqa: F401 -from .util.ipython_patches import patch_ipython +from .util.patch_ipython import patch_ipython from .util.telemetry import instrument # noqa: F401 from .util.util import autocache # noqa: F401 from .util.util import get_root_data_path # noqa: F401 diff --git a/packages/syft/src/syft/util/ipython_patches.py b/packages/syft/src/syft/util/patch_ipython.py similarity index 100% rename from packages/syft/src/syft/util/ipython_patches.py rename to packages/syft/src/syft/util/patch_ipython.py From d0ad3b0b2b1aa3259334b5457e7ff44d8eb025ec Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 00:09:36 +0200 Subject: [PATCH 11/30] fix: import typo --- packages/syft/src/syft/util/patch_ipython.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/util/patch_ipython.py b/packages/syft/src/syft/util/patch_ipython.py index 04bea086c5e..175c251febb 100644 --- a/packages/syft/src/syft/util/patch_ipython.py +++ b/packages/syft/src/syft/util/patch_ipython.py @@ -5,7 +5,7 @@ # relative from ..types.dicttuple import DictTuple -from .types.syft_object import SyftObject +from ..types.syft_object import SyftObject def _patch_ipython_sanitization() -> None: From ffe480be4dcf89ec18689994c9d57e2360973ed7 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 00:15:51 +0200 Subject: [PATCH 12/30] fix: support SyftResponseMessage --- packages/syft/src/syft/util/patch_ipython.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/packages/syft/src/syft/util/patch_ipython.py b/packages/syft/src/syft/util/patch_ipython.py index 175c251febb..93962b6c5f8 100644 --- a/packages/syft/src/syft/util/patch_ipython.py +++ b/packages/syft/src/syft/util/patch_ipython.py @@ -4,6 +4,7 @@ from typing import Any # relative +from ..service.response import SyftResponseMessage from ..types.dicttuple import DictTuple from ..types.syft_object import SyftObject @@ -26,12 +27,12 @@ def _patch_ipython_sanitization() -> None: import nh3 # relative - from .util.assets import load_css - from .util.assets import load_js - from .util.notebook_ui.styles import CSS_CODE - from .util.notebook_ui.styles import FONT_CSS - from .util.notebook_ui.styles import ITABLES_CSS - from .util.notebook_ui.styles import JS_DOWNLOAD_FONTS + from .assets import load_css + from .assets import load_js + from .notebook_ui.styles import CSS_CODE + from .notebook_ui.styles import FONT_CSS + from .notebook_ui.styles import ITABLES_CSS + from .notebook_ui.styles import JS_DOWNLOAD_FONTS tabulator_js = load_js("tabulator.min.js") tabulator_js = tabulator_js.replace( @@ -77,10 +78,14 @@ def display_sanitized_html(obj: SyftObject | DictTuple) -> str | None: return f"{css_reinsert} {_str} {"\n".join(matching_template)}" return None - def display_sanitized_md(obj: SyftObject) -> None: + def display_sanitized_md(obj: SyftObject) -> str | None: if hasattr(obj, "_repr_markdown_"): return nh3.clean(obj._repr_markdown_()) + return None + ip.display_formatter.formatters["text/html"].for_type( + SyftResponseMessage, display_sanitized_html + ) ip.display_formatter.formatters["text/html"].for_type( SyftObject, display_sanitized_html ) From 8a62934598831098ea199413bed6a18f7ba2d162 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 00:24:21 +0200 Subject: [PATCH 13/30] fix: support response and exception --- packages/syft/src/syft/service/response.py | 5 +++-- packages/syft/src/syft/util/patch_ipython.py | 4 ---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/syft/src/syft/service/response.py b/packages/syft/src/syft/service/response.py index 723970cdfff..f924dfc48d7 100644 --- a/packages/syft/src/syft/service/response.py +++ b/packages/syft/src/syft/service/response.py @@ -4,6 +4,7 @@ from typing import Any # third party +import nh3 from result import Err # relative @@ -44,7 +45,7 @@ def _repr_html_(self) -> str: f'
' f"{type(self).__name__}: " f'
'
-            f"{self.message}

" + f"{nh3.clean(self.message)}
" ) @@ -107,7 +108,7 @@ def _repr_html_class_(self) -> str: def _repr_html_(self) -> str: return ( f'
' - + f"{type(self).__name__}: {self.args}

" + + f"{type(self).__name__}: {nh3.clean(self.args)}
" ) @staticmethod diff --git a/packages/syft/src/syft/util/patch_ipython.py b/packages/syft/src/syft/util/patch_ipython.py index 93962b6c5f8..822e7df2e45 100644 --- a/packages/syft/src/syft/util/patch_ipython.py +++ b/packages/syft/src/syft/util/patch_ipython.py @@ -4,7 +4,6 @@ from typing import Any # relative -from ..service.response import SyftResponseMessage from ..types.dicttuple import DictTuple from ..types.syft_object import SyftObject @@ -83,9 +82,6 @@ def display_sanitized_md(obj: SyftObject) -> str | None: return nh3.clean(obj._repr_markdown_()) return None - ip.display_formatter.formatters["text/html"].for_type( - SyftResponseMessage, display_sanitized_html - ) ip.display_formatter.formatters["text/html"].for_type( SyftObject, display_sanitized_html ) From 7ca9121d16aa4a1829c93bc57d9ab8d71b243589 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 01:01:26 +0200 Subject: [PATCH 14/30] fix: more sanitization --- .../src/syft/util/notebook_ui/components/tabulator_template.py | 2 +- packages/syft/src/syft/util/patch_ipython.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py index b7b07277181..a4a136deb39 100644 --- a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py +++ b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py @@ -78,7 +78,7 @@ def format_dict(data: Any) -> str: if "clipboard" in data["type"]: return CopyButton(copy_text=data["value"]).to_html() - return str(data) + return nh3.clean(str(data)) def format_table_data(table_data: list[dict[str, Any]]) -> list[dict[str, str]]: diff --git a/packages/syft/src/syft/util/patch_ipython.py b/packages/syft/src/syft/util/patch_ipython.py index 822e7df2e45..79ebdeabf71 100644 --- a/packages/syft/src/syft/util/patch_ipython.py +++ b/packages/syft/src/syft/util/patch_ipython.py @@ -78,7 +78,7 @@ def display_sanitized_html(obj: SyftObject | DictTuple) -> str | None: return None def display_sanitized_md(obj: SyftObject) -> str | None: - if hasattr(obj, "_repr_markdown_"): + if hasattr(obj, "_repr_markdown_") and callable(obj._repr_markdown_): return nh3.clean(obj._repr_markdown_()) return None From 0ff5389da44a88073d4e067ba14704f21db443c9 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 03:22:58 +0200 Subject: [PATCH 15/30] refactor: extract common logic for tabulator table rendering - Created `_render_tabulator_table` to handle shared rendering logic - Updated `build_tabulator_table` to use the new helper function - Created `build_tabulator_table_with_data` for cases where the data and metadata are passed and there's no need to create an extra SyftObject for table rows. --- .../components/tabulator_template.py | 107 +++++++++++------- 1 file changed, 68 insertions(+), 39 deletions(-) diff --git a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py index a4a136deb39..69c172181b7 100644 --- a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py +++ b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py @@ -95,6 +95,71 @@ def format_table_data(table_data: list[dict[str, Any]]) -> list[dict[str, str]]: return formatted +def _render_tabulator_table( + uid: str, + table_data: list[dict], + table_metadata: dict, + max_height: int | None, + pagination: bool, + header_sort: bool, +) -> str: + table_template = env.get_template("table.jinja2") + tabulator_js = load_js("tabulator.min.js") + tabulator_css = load_css("tabulator_pysyft.min.css") + js = load_js("table.js") + css = load_css("style.css") + + # Add tabulator as a named module for VSCode compatibility + tabulator_js = tabulator_js.replace( + "define(t)", "define('tabulator-tables', [], t)" + ) + + icon = table_metadata.get("icon", None) + if icon is None: + icon = Icon.TABLE.svg + + column_data, row_header = create_tabulator_columns( + table_metadata["columns"], header_sort=header_sort + ) + table_data = format_table_data(table_data) + table_html = table_template.render( + uid=uid, + columns=json.dumps(column_data), + row_header=json.dumps(row_header), + data=json.dumps(table_data), + css=css, + js=js, + index_field_name=TABLE_INDEX_KEY, + icon=icon, + name=table_metadata["name"], + tabulator_js=tabulator_js, + tabulator_css=tabulator_css, + max_height=json.dumps(max_height), + pagination=json.dumps(pagination), + header_sort=json.dumps(header_sort), + ) + + return table_html + + +def build_tabulator_table_with_data( + table_data: list[dict], + table_metadata: dict, + uid: str | None = None, + max_height: int | None = None, + pagination: bool = True, + header_sort: bool = True, +) -> str | None: + try: + uid = uid if uid is not None else secrets.token_hex(4) + return _render_tabulator_table( + uid, table_data, table_metadata, max_height, pagination, header_sort + ) + except Exception as e: + logger.debug("error building table", e) + return None + + def build_tabulator_table( obj: Any, uid: str | None = None, @@ -106,49 +171,13 @@ def build_tabulator_table( table_data, table_metadata = prepare_table_data(obj) if len(table_data) == 0: return obj.__repr__() - - table_template = env.get_template("table.jinja2") - tabulator_js = load_js("tabulator.min.js") - tabulator_css = load_css("tabulator_pysyft.min.css") - js = load_js("table.js") - css = load_css("style.css") - - # Add tabulator as a named module for VSCode compatibility - tabulator_js = tabulator_js.replace( - "define(t)", "define('tabulator-tables', [], t)" - ) - - icon = table_metadata.get("icon", None) - if icon is None: - icon = Icon.TABLE.svg - uid = uid if uid is not None else secrets.token_hex(4) - column_data, row_header = create_tabulator_columns( - table_metadata["columns"], header_sort=header_sort - ) - table_data = format_table_data(table_data) - table_html = table_template.render( - uid=uid, - columns=json.dumps(column_data), - row_header=json.dumps(row_header), - data=json.dumps(table_data), - css=css, - js=js, - index_field_name=TABLE_INDEX_KEY, - icon=icon, - name=table_metadata["name"], - tabulator_js=tabulator_js, - tabulator_css=tabulator_css, - max_height=json.dumps(max_height), - pagination=json.dumps(pagination), - header_sort=json.dumps(header_sort), + return _render_tabulator_table( + uid, table_data, table_metadata, max_height, pagination, header_sort ) - - return table_html except Exception as e: logger.debug("error building table", e) - - return None + return None def show_table(obj: Any) -> None: From 219aa915c622874dbc6d93d5f6e0c41247615022 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 03:38:22 +0200 Subject: [PATCH 16/30] feat: update code_history with new table style --- .../syft/service/code_history/code_history.py | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/packages/syft/src/syft/service/code_history/code_history.py b/packages/syft/src/syft/service/code_history/code_history.py index b5e893c87bf..092505a2278 100644 --- a/packages/syft/src/syft/service/code_history/code_history.py +++ b/packages/syft/src/syft/service/code_history/code_history.py @@ -11,7 +11,9 @@ from ...types.syft_object import SyftObject from ...types.syft_object import SyftVerifyKey from ...types.uid import UID -from ...util.notebook_ui.components.table_template import create_table_template +from ...util.notebook_ui.components.tabulator_template import ( + build_tabulator_table_with_data, +) from ...util.table import prepare_table_data from ..code.user_code import UserCode from ..response import SyftError @@ -55,8 +57,8 @@ def _coll_repr_(self) -> dict[str, int]: return {"Number of versions": len(self.user_code_history)} def _repr_html_(self) -> str: - # TODO techdebt: move this to _coll_repr_ - rows, _ = prepare_table_data(self.user_code_history) + rows, metadata = prepare_table_data(self.user_code_history) + for i, r in enumerate(rows): r["Version"] = f"v{i}" raw_code = self.user_code_history[i].raw_code @@ -64,8 +66,11 @@ def _repr_html_(self) -> str: if n_code_lines > 5: raw_code = "\n".join(raw_code.split("\n", 5)) r["Code"] = raw_code - # rows = sorted(rows, key=lambda x: x["Version"]) - return create_table_template(rows, "CodeHistory", icon=None) + + metadata["name"] = "Code History" + metadata["columns"] += ["Version", "Code"] + + return build_tabulator_table_with_data(rows, metadata) def __getitem__(self, index: int | str) -> UserCode | SyftError: if isinstance(index, str): @@ -138,6 +143,12 @@ def __getitem__(self, key: str | int) -> CodeHistoriesDict | SyftError: def _repr_html_(self) -> str: rows = [ - {"user": user, "UserCodes": funcs} for user, funcs in self.user_dict.items() + {"User": user, "UserCodes": ", ".join(funcs)} + for user, funcs in self.user_dict.items() ] - return create_table_template(rows, "UserCodeHistory", icon=None) + metadata = { + "name": "UserCode Histories", + "columns": ["User", "UserCodes"], + "icon": None, + } + return build_tabulator_table_with_data(rows, metadata) From dcc859e00a43684b73885541b274d686e3580145 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 03:38:50 +0200 Subject: [PATCH 17/30] refactor: display formatters in patch_ipython.py --- packages/syft/src/syft/util/patch_ipython.py | 21 +++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/packages/syft/src/syft/util/patch_ipython.py b/packages/syft/src/syft/util/patch_ipython.py index 79ebdeabf71..904db6bf561 100644 --- a/packages/syft/src/syft/util/patch_ipython.py +++ b/packages/syft/src/syft/util/patch_ipython.py @@ -68,18 +68,21 @@ def _patch_ipython_sanitization() -> None: escaped_template = re.compile(table_template, re.DOTALL | re.VERBOSE) def display_sanitized_html(obj: SyftObject | DictTuple) -> str | None: - if hasattr(obj, "_repr_html_") and callable(obj._repr_html_): # type: ignore - _str = obj._repr_html_() # type: ignore - matching_template = escaped_template.findall(_str) - _str = escaped_template.sub("", _str) - _str = escaped_js_css.sub("", _str) - _str = nh3.clean(_str) - return f"{css_reinsert} {_str} {"\n".join(matching_template)}" + if callable(getattr(obj, "_repr_html_", None)): + html_str = obj._repr_html_() + if html_str is not None: + matching_template = escaped_template.findall(html_str) + sanitized_str = escaped_template.sub("", html_str) + sanitized_str = escaped_js_css.sub("", sanitized_str) + sanitized_str = nh3.clean(sanitized_str) + return f"{css_reinsert} {sanitized_str} {'\n'.join(matching_template)}" return None def display_sanitized_md(obj: SyftObject) -> str | None: - if hasattr(obj, "_repr_markdown_") and callable(obj._repr_markdown_): - return nh3.clean(obj._repr_markdown_()) + if callable(getattr(obj, "_repr_markdown_", None)): + md = obj._repr_markdown_() + if md is not None: + return nh3.clean(md) return None ip.display_formatter.formatters["text/html"].for_type( From 700fbe93f5513876dd80ac257e1fd2a9535e459a Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 03:39:17 +0200 Subject: [PATCH 18/30] feat: remove old table code and old js --- .../notebook_ui/components/table_template.py | 330 ------------------ packages/syft/src/syft/util/table.py | 22 +- packages/syft/tests/syft/notebook_ui_test.py | 10 - 3 files changed, 1 insertion(+), 361 deletions(-) delete mode 100644 packages/syft/src/syft/util/notebook_ui/components/table_template.py diff --git a/packages/syft/src/syft/util/notebook_ui/components/table_template.py b/packages/syft/src/syft/util/notebook_ui/components/table_template.py deleted file mode 100644 index 7de891af6a9..00000000000 --- a/packages/syft/src/syft/util/notebook_ui/components/table_template.py +++ /dev/null @@ -1,330 +0,0 @@ -# stdlib -from collections.abc import Sequence -import json -from string import Template - -# relative -from ....types.uid import UID -from ..icons import Icon -from ..styles import CSS_CODE - -TABLE_INDEX_KEY = "_table_repr_index" - -custom_code = """ - - -
-
-
${icon}
-

${list_name}

-
- -
-
-
-
-
- -
- -
- -
- -

0

-
-
- -
-
- -
-
-""" - - -def create_table_template( - table_data: Sequence, - name: str, - rows: int = 5, - icon: str | None = None, - grid_template_columns: str | None = None, - grid_template_cell_columns: str | None = None, - **kwargs: dict, -) -> str: - if icon is None: - icon = Icon.TABLE.svg - if grid_template_columns is None: - grid_template_columns = "1fr repeat({cols}, 1fr)" - if grid_template_cell_columns is None: - grid_template_cell_columns = "span 4" - - items_dict = json.dumps(table_data) - code = CSS_CODE + custom_code - template = Template(code) - rows = min(len(table_data), rows) - if len(table_data) == 0: - cols = 0 - else: - col_names = [k for k in table_data[0].keys() if k != TABLE_INDEX_KEY] - cols = (len(col_names)) * 4 - if "{cols}" in grid_template_columns: - grid_template_columns = grid_template_columns.format(cols=cols) - final_html = template.substitute( - uid=str(UID()), - element=items_dict, - list_name=name, - cols=cols, - rows=rows, - icon=icon, - searchIcon=Icon.SEARCH.svg, - clipboardIconEscaped=Icon.CLIPBOARD.js_escaped_svg, - grid_template_columns=grid_template_columns, - grid_template_cell_columns=grid_template_cell_columns, - ) - return final_html diff --git a/packages/syft/src/syft/util/table.py b/packages/syft/src/syft/util/table.py index f4965cb1ef0..5d380a65648 100644 --- a/packages/syft/src/syft/util/table.py +++ b/packages/syft/src/syft/util/table.py @@ -7,14 +7,12 @@ from typing import Any # third party -from loguru import logger import nh3 # relative -from .notebook_ui.components.table_template import TABLE_INDEX_KEY -from .notebook_ui.components.table_template import create_table_template from .util import full_name_with_qualname +TABLE_INDEX_KEY = "_table_repr_index" def _syft_in_mro(self: Any, item: Any) -> bool: if hasattr(type(item), "mro") and type(item) != type: @@ -239,21 +237,3 @@ def prepare_table_data( } return table_data, table_metadata - - -def list_dict_repr_html(self: Mapping | Set | Iterable) -> str | None: - try: - table_data, table_metadata = prepare_table_data(self) - if len(table_data) == 0: - # TODO cleanup tech debt: _repr_html_ is used in syft without `None` fallback. - return self.__repr__() - return create_table_template( - table_data=table_data, - **table_metadata, - ) - - except Exception as e: - logger.debug(f"Could not create table: {e}") - - # _repr_html_ returns None -> fallback to default repr - return None diff --git a/packages/syft/tests/syft/notebook_ui_test.py b/packages/syft/tests/syft/notebook_ui_test.py index eebc249dc82..32f591fedad 100644 --- a/packages/syft/tests/syft/notebook_ui_test.py +++ b/packages/syft/tests/syft/notebook_ui_test.py @@ -39,16 +39,6 @@ def table_test_cases() -> list[tuple[list, str | None]]: ] -@pytest.mark.parametrize("test_case", table_test_cases()) -def test_list_dict_repr_html(test_case): - obj, expected = test_case - - assert (obj._repr_html_() is not None) == expected - assert (dict(enumerate(obj))._repr_html_() is not None) == expected - assert (set(obj)._repr_html_() is not None) == expected - assert (tuple(obj)._repr_html_() is not None) == expected - - def test_sort_table_rows(): emails = [ "x@y.z", From 1331f732e84c07ca86901d245291f15d78ae67f6 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 03:45:30 +0200 Subject: [PATCH 19/30] lint --- packages/syft/src/syft/service/code_history/code_history.py | 4 ++-- packages/syft/src/syft/util/table.py | 1 + packages/syft/tests/syft/notebook_ui_test.py | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/code_history/code_history.py b/packages/syft/src/syft/service/code_history/code_history.py index 092505a2278..c3b1151fe2e 100644 --- a/packages/syft/src/syft/service/code_history/code_history.py +++ b/packages/syft/src/syft/service/code_history/code_history.py @@ -56,7 +56,7 @@ class CodeHistoryView(SyftObject): def _coll_repr_(self) -> dict[str, int]: return {"Number of versions": len(self.user_code_history)} - def _repr_html_(self) -> str: + def _repr_html_(self) -> str | None: rows, metadata = prepare_table_data(self.user_code_history) for i, r in enumerate(rows): @@ -141,7 +141,7 @@ def __getitem__(self, key: str | int) -> CodeHistoriesDict | SyftError: ) return api.services.code_history.get_history_for_user(key) - def _repr_html_(self) -> str: + def _repr_html_(self) -> str | None: rows = [ {"User": user, "UserCodes": ", ".join(funcs)} for user, funcs in self.user_dict.items() diff --git a/packages/syft/src/syft/util/table.py b/packages/syft/src/syft/util/table.py index 5d380a65648..34439fa95df 100644 --- a/packages/syft/src/syft/util/table.py +++ b/packages/syft/src/syft/util/table.py @@ -14,6 +14,7 @@ TABLE_INDEX_KEY = "_table_repr_index" + def _syft_in_mro(self: Any, item: Any) -> bool: if hasattr(type(item), "mro") and type(item) != type: mro = type(item).mro() diff --git a/packages/syft/tests/syft/notebook_ui_test.py b/packages/syft/tests/syft/notebook_ui_test.py index 32f591fedad..ddff0824467 100644 --- a/packages/syft/tests/syft/notebook_ui_test.py +++ b/packages/syft/tests/syft/notebook_ui_test.py @@ -1,6 +1,5 @@ # third party import numpy as np -import pytest # syft absolute from syft.service.action.action_object import ActionObject From f9550f9a2eac64fe8e32e1889de0a71614fac49f Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 03:57:09 +0200 Subject: [PATCH 20/30] fix: f-string backslash --- packages/syft/src/syft/util/patch_ipython.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/util/patch_ipython.py b/packages/syft/src/syft/util/patch_ipython.py index 904db6bf561..c1a91f4cb73 100644 --- a/packages/syft/src/syft/util/patch_ipython.py +++ b/packages/syft/src/syft/util/patch_ipython.py @@ -72,10 +72,11 @@ def display_sanitized_html(obj: SyftObject | DictTuple) -> str | None: html_str = obj._repr_html_() if html_str is not None: matching_template = escaped_template.findall(html_str) + matching_template = '\n'.join(matching_template) sanitized_str = escaped_template.sub("", html_str) sanitized_str = escaped_js_css.sub("", sanitized_str) sanitized_str = nh3.clean(sanitized_str) - return f"{css_reinsert} {sanitized_str} {'\n'.join(matching_template)}" + return f"{css_reinsert} {sanitized_str} {matching_template}" return None def display_sanitized_md(obj: SyftObject) -> str | None: From b57b056823dd26f84494e6a01611e14b3fda7134 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 03:59:57 +0200 Subject: [PATCH 21/30] tests: reintroduce list_dict_repr --- packages/syft/tests/syft/notebook_ui_test.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/syft/tests/syft/notebook_ui_test.py b/packages/syft/tests/syft/notebook_ui_test.py index ddff0824467..eebc249dc82 100644 --- a/packages/syft/tests/syft/notebook_ui_test.py +++ b/packages/syft/tests/syft/notebook_ui_test.py @@ -1,5 +1,6 @@ # third party import numpy as np +import pytest # syft absolute from syft.service.action.action_object import ActionObject @@ -38,6 +39,16 @@ def table_test_cases() -> list[tuple[list, str | None]]: ] +@pytest.mark.parametrize("test_case", table_test_cases()) +def test_list_dict_repr_html(test_case): + obj, expected = test_case + + assert (obj._repr_html_() is not None) == expected + assert (dict(enumerate(obj))._repr_html_() is not None) == expected + assert (set(obj)._repr_html_() is not None) == expected + assert (tuple(obj)._repr_html_() is not None) == expected + + def test_sort_table_rows(): emails = [ "x@y.z", From 5a1863f3b428ba50d47bbd58cf9503835e5258fe Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 04:08:14 +0200 Subject: [PATCH 22/30] lint --- packages/syft/src/syft/util/patch_ipython.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/util/patch_ipython.py b/packages/syft/src/syft/util/patch_ipython.py index c1a91f4cb73..642b75a590d 100644 --- a/packages/syft/src/syft/util/patch_ipython.py +++ b/packages/syft/src/syft/util/patch_ipython.py @@ -72,11 +72,11 @@ def display_sanitized_html(obj: SyftObject | DictTuple) -> str | None: html_str = obj._repr_html_() if html_str is not None: matching_template = escaped_template.findall(html_str) - matching_template = '\n'.join(matching_template) + template = "\n".join(matching_template) sanitized_str = escaped_template.sub("", html_str) sanitized_str = escaped_js_css.sub("", sanitized_str) sanitized_str = nh3.clean(sanitized_str) - return f"{css_reinsert} {sanitized_str} {matching_template}" + return f"{css_reinsert} {sanitized_str} {template}" return None def display_sanitized_md(obj: SyftObject) -> str | None: From 696e73b3933561ec5e23c4284ae9acd7b65d3390 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 12:06:23 +0200 Subject: [PATCH 23/30] bump --- packages/syft/src/syft/assets/css/style.css | 18 ++++++++++++++++++ packages/syft/src/syft/service/response.py | 4 ++-- .../syft/util/notebook_ui/components/sync.py | 1 + .../components/tabulator_template.py | 6 +++--- packages/syft/src/syft/util/patch_ipython.py | 8 ++++++-- packages/syft/src/syft/util/table.py | 2 +- 6 files changed, 31 insertions(+), 8 deletions(-) diff --git a/packages/syft/src/syft/assets/css/style.css b/packages/syft/src/syft/assets/css/style.css index 528046d6668..113a9908055 100644 --- a/packages/syft/src/syft/assets/css/style.css +++ b/packages/syft/src/syft/assets/css/style.css @@ -5,6 +5,7 @@ body.vscode-dark { --tertiary-color: #cfcdd6; --button-color: #111111; --colors-black: #ffffff; + --surface-color: #fff; } body { @@ -13,6 +14,7 @@ body { --tertiary-color: #000000de; --button-color: #d1d5db; --colors-black: #17161d; + --surface-color: #464158; } .header-1 { @@ -564,3 +566,19 @@ body { .syft-widget li a:hover { background-color: #c2def0; } + +.syft-user_code, .syft-project, .syft-project-create, .syft-dataset, .syft-syncstate { + color: var(--surface-color); +} + +.syft-dataset h3, +.syft-dataset p, +.syft-syncstate h3, +.syft-syncstate p { + font-family: 'Open Sans'; +} + +.diff-container { + border: 0.5px solid #B4B0BF; +} + diff --git a/packages/syft/src/syft/service/response.py b/packages/syft/src/syft/service/response.py index f924dfc48d7..12070a42b54 100644 --- a/packages/syft/src/syft/service/response.py +++ b/packages/syft/src/syft/service/response.py @@ -45,7 +45,7 @@ def _repr_html_(self) -> str: f'
' f"{type(self).__name__}: " f'
'
-            f"{nh3.clean(self.message)}

" + f"{nh3.clean(self.message, clean_content_tags={"script", "style"}, attributes={"*": {"style", "class"}})}
" ) @@ -108,7 +108,7 @@ def _repr_html_class_(self) -> str: def _repr_html_(self) -> str: return ( f'
' - + f"{type(self).__name__}: {nh3.clean(self.args)}

" + + f"{type(self).__name__}: {nh3.clean(self.args, clean_content_tags={"script", "style"}, attributes={"*": {"style", "class"}})}
" ) @staticmethod diff --git a/packages/syft/src/syft/util/notebook_ui/components/sync.py b/packages/syft/src/syft/util/notebook_ui/components/sync.py index 4fdd0adf1b3..9e8877ed70c 100644 --- a/packages/syft/src/syft/util/notebook_ui/components/sync.py +++ b/packages/syft/src/syft/util/notebook_ui/components/sync.py @@ -3,6 +3,7 @@ from typing import Any # third party +import nh3 from pydantic import model_validator # relative diff --git a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py index 69c172181b7..9199aa0acbc 100644 --- a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py +++ b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py @@ -69,7 +69,7 @@ def format_dict(data: Any) -> str: return data if set(data.keys()) != {"type", "value"}: - return nh3.clean(str(data)) + return nh3.clean(str(data), clean_content_tags={"script", "style"}, attributes={"*": {"style", "class"}}) if "badge" in data["type"]: return Badge(value=data["value"], badge_class=data["type"]).to_html() @@ -78,7 +78,7 @@ def format_dict(data: Any) -> str: if "clipboard" in data["type"]: return CopyButton(copy_text=data["value"]).to_html() - return nh3.clean(str(data)) + return nh3.clean(str(data), clean_content_tags={"script", "style"}, attributes={"*": {"style", "class"}}) def format_table_data(table_data: list[dict[str, Any]]) -> list[dict[str, str]]: @@ -87,7 +87,7 @@ def format_table_data(table_data: list[dict[str, Any]]) -> list[dict[str, str]]: row_formatted: dict[str, str] = {} for k, v in row.items(): if isinstance(v, str): - row_formatted[k] = nh3.clean(v.replace("\n", "
")) + row_formatted[k] = nh3.clean(v.replace("\n", "
"), clean_content_tags={"script", "style"}, attributes={"*": {"style", "class"}}) continue v_formatted = format_dict(v) row_formatted[k] = v_formatted diff --git a/packages/syft/src/syft/util/patch_ipython.py b/packages/syft/src/syft/util/patch_ipython.py index 642b75a590d..79d050ba955 100644 --- a/packages/syft/src/syft/util/patch_ipython.py +++ b/packages/syft/src/syft/util/patch_ipython.py @@ -32,6 +32,8 @@ def _patch_ipython_sanitization() -> None: from .notebook_ui.styles import FONT_CSS from .notebook_ui.styles import ITABLES_CSS from .notebook_ui.styles import JS_DOWNLOAD_FONTS + from .notebook_ui.components.sync import ALERT_CSS + from .notebook_ui.components.sync import COPY_CSS tabulator_js = load_js("tabulator.min.js") tabulator_js = tabulator_js.replace( @@ -53,6 +55,8 @@ def _patch_ipython_sanitization() -> None: {JS_DOWNLOAD_FONTS} {CSS_CODE} + + """ escaped_js_css = re.compile( @@ -75,7 +79,7 @@ def display_sanitized_html(obj: SyftObject | DictTuple) -> str | None: template = "\n".join(matching_template) sanitized_str = escaped_template.sub("", html_str) sanitized_str = escaped_js_css.sub("", sanitized_str) - sanitized_str = nh3.clean(sanitized_str) + sanitized_str = nh3.clean(sanitized_str, clean_content_tags={"script", "style"}, attributes={"*": {"style", "class"}}) return f"{css_reinsert} {sanitized_str} {template}" return None @@ -83,7 +87,7 @@ def display_sanitized_md(obj: SyftObject) -> str | None: if callable(getattr(obj, "_repr_markdown_", None)): md = obj._repr_markdown_() if md is not None: - return nh3.clean(md) + return nh3.clean(md, clean_content_tags={"script", "style"}, attributes={"*": {"style", "class"}}) return None ip.display_formatter.formatters["text/html"].for_type( diff --git a/packages/syft/src/syft/util/table.py b/packages/syft/src/syft/util/table.py index 34439fa95df..60510161a5c 100644 --- a/packages/syft/src/syft/util/table.py +++ b/packages/syft/src/syft/util/table.py @@ -134,7 +134,7 @@ def _create_table_rows( except Exception as e: print(e) value = None - cols[field].append(nh3.clean(str(value))) + cols[field].append(nh3.clean(str(value), clean_content_tags={"script", "style"}, attributes={"*": {"style", "class"}})) col_lengths = {len(cols[col]) for col in cols.keys()} if len(col_lengths) != 1: From a95efc3d4be4bcebdd01f551fbb2e49dc9673094 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 13:39:09 +0200 Subject: [PATCH 24/30] feat: move sanitization fn to utils --- packages/syft/src/syft/service/response.py | 6 ++--- .../syft/util/notebook_ui/components/sync.py | 1 - .../components/tabulator_template.py | 9 +++---- packages/syft/src/syft/util/patch_ipython.py | 12 ++++----- packages/syft/src/syft/util/table.py | 6 ++--- packages/syft/src/syft/util/util.py | 27 +++++++++++++++++++ 6 files changed, 41 insertions(+), 20 deletions(-) diff --git a/packages/syft/src/syft/service/response.py b/packages/syft/src/syft/service/response.py index 12070a42b54..adce4b8af73 100644 --- a/packages/syft/src/syft/service/response.py +++ b/packages/syft/src/syft/service/response.py @@ -4,12 +4,12 @@ from typing import Any # third party -import nh3 from result import Err # relative from ..serde.serializable import serializable from ..types.base import SyftBaseModel +from ..util.util import sanitize_html class SyftResponseMessage(SyftBaseModel): @@ -45,7 +45,7 @@ def _repr_html_(self) -> str: f'
' f"{type(self).__name__}: " f'
'
-            f"{nh3.clean(self.message, clean_content_tags={"script", "style"}, attributes={"*": {"style", "class"}})}

" + f"{sanitize_html(self.message)}
" ) @@ -108,7 +108,7 @@ def _repr_html_class_(self) -> str: def _repr_html_(self) -> str: return ( f'
' - + f"{type(self).__name__}: {nh3.clean(self.args, clean_content_tags={"script", "style"}, attributes={"*": {"style", "class"}})}

" + + f"{type(self).__name__}: {sanitize_html(self.args)}
" ) @staticmethod diff --git a/packages/syft/src/syft/util/notebook_ui/components/sync.py b/packages/syft/src/syft/util/notebook_ui/components/sync.py index 9e8877ed70c..4fdd0adf1b3 100644 --- a/packages/syft/src/syft/util/notebook_ui/components/sync.py +++ b/packages/syft/src/syft/util/notebook_ui/components/sync.py @@ -3,7 +3,6 @@ from typing import Any # third party -import nh3 from pydantic import model_validator # relative diff --git a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py index 9199aa0acbc..69e0fdbff0f 100644 --- a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py +++ b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py @@ -8,13 +8,13 @@ from IPython.display import display import jinja2 from loguru import logger -import nh3 # relative from ...assets import load_css from ...assets import load_js from ...table import TABLE_INDEX_KEY from ...table import prepare_table_data +from ...util import sanitize_html from ..icons import Icon DEFAULT_ID_WIDTH = 110 @@ -69,8 +69,7 @@ def format_dict(data: Any) -> str: return data if set(data.keys()) != {"type", "value"}: - return nh3.clean(str(data), clean_content_tags={"script", "style"}, attributes={"*": {"style", "class"}}) - + return sanitize_html(str(data)) if "badge" in data["type"]: return Badge(value=data["value"], badge_class=data["type"]).to_html() elif "label" in data["type"]: @@ -78,7 +77,7 @@ def format_dict(data: Any) -> str: if "clipboard" in data["type"]: return CopyButton(copy_text=data["value"]).to_html() - return nh3.clean(str(data), clean_content_tags={"script", "style"}, attributes={"*": {"style", "class"}}) + return sanitize_html(str(data)) def format_table_data(table_data: list[dict[str, Any]]) -> list[dict[str, str]]: @@ -87,7 +86,7 @@ def format_table_data(table_data: list[dict[str, Any]]) -> list[dict[str, str]]: row_formatted: dict[str, str] = {} for k, v in row.items(): if isinstance(v, str): - row_formatted[k] = nh3.clean(v.replace("\n", "
"), clean_content_tags={"script", "style"}, attributes={"*": {"style", "class"}}) + row_formatted[k] = sanitize_html(v.replace("\n", "
")) continue v_formatted = format_dict(v) row_formatted[k] = v_formatted diff --git a/packages/syft/src/syft/util/patch_ipython.py b/packages/syft/src/syft/util/patch_ipython.py index 79d050ba955..5e47ac5ef98 100644 --- a/packages/syft/src/syft/util/patch_ipython.py +++ b/packages/syft/src/syft/util/patch_ipython.py @@ -6,6 +6,7 @@ # relative from ..types.dicttuple import DictTuple from ..types.syft_object import SyftObject +from .util import sanitize_html def _patch_ipython_sanitization() -> None: @@ -22,18 +23,15 @@ def _patch_ipython_sanitization() -> None: # stdlib from importlib import resources - # third party - import nh3 - # relative from .assets import load_css from .assets import load_js + from .notebook_ui.components.sync import ALERT_CSS + from .notebook_ui.components.sync import COPY_CSS from .notebook_ui.styles import CSS_CODE from .notebook_ui.styles import FONT_CSS from .notebook_ui.styles import ITABLES_CSS from .notebook_ui.styles import JS_DOWNLOAD_FONTS - from .notebook_ui.components.sync import ALERT_CSS - from .notebook_ui.components.sync import COPY_CSS tabulator_js = load_js("tabulator.min.js") tabulator_js = tabulator_js.replace( @@ -79,7 +77,7 @@ def display_sanitized_html(obj: SyftObject | DictTuple) -> str | None: template = "\n".join(matching_template) sanitized_str = escaped_template.sub("", html_str) sanitized_str = escaped_js_css.sub("", sanitized_str) - sanitized_str = nh3.clean(sanitized_str, clean_content_tags={"script", "style"}, attributes={"*": {"style", "class"}}) + sanitized_str = sanitize_html(sanitized_str) return f"{css_reinsert} {sanitized_str} {template}" return None @@ -87,7 +85,7 @@ def display_sanitized_md(obj: SyftObject) -> str | None: if callable(getattr(obj, "_repr_markdown_", None)): md = obj._repr_markdown_() if md is not None: - return nh3.clean(md, clean_content_tags={"script", "style"}, attributes={"*": {"style", "class"}}) + return sanitize_html(md) return None ip.display_formatter.formatters["text/html"].for_type( diff --git a/packages/syft/src/syft/util/table.py b/packages/syft/src/syft/util/table.py index 60510161a5c..296596f33a9 100644 --- a/packages/syft/src/syft/util/table.py +++ b/packages/syft/src/syft/util/table.py @@ -6,11 +6,9 @@ import re from typing import Any -# third party -import nh3 - # relative from .util import full_name_with_qualname +from .util import sanitize_html TABLE_INDEX_KEY = "_table_repr_index" @@ -134,7 +132,7 @@ def _create_table_rows( except Exception as e: print(e) value = None - cols[field].append(nh3.clean(str(value), clean_content_tags={"script", "style"}, attributes={"*": {"style", "class"}})) + cols[field].append(sanitize_html(str(value))) col_lengths = {len(cols[col]) for col in cols.keys()} if len(col_lengths) != 1: diff --git a/packages/syft/src/syft/util/util.py b/packages/syft/src/syft/util/util.py index b0affa2b1a0..59f7974ae9c 100644 --- a/packages/syft/src/syft/util/util.py +++ b/packages/syft/src/syft/util/util.py @@ -29,12 +29,14 @@ import types from types import ModuleType from typing import Any +from copy import deepcopy # third party from IPython.display import display from forbiddenfruit import curse from nacl.signing import SigningKey from nacl.signing import VerifyKey +import nh3 import requests # relative @@ -919,3 +921,28 @@ def get_queue_address(port: int) -> str: def get_dev_mode() -> bool: return str_to_bool(os.getenv("DEV_MODE", "False")) + + +def sanitize_html(html: str) -> str: + policy = { + 'tags': ['svg', 'strong', 'rect', 'path', 'circle'], + 'attributes': { + '*': {'class', 'style'}, + 'svg': {'class', 'style', 'xmlns', 'width', 'height', 'viewBox', 'fill', 'stroke', 'stroke-width'}, + 'path': {'d', 'fill', 'stroke', 'stroke-width'}, + 'rect': {'x', 'y', 'width', 'height', 'fill', 'stroke', 'stroke-width'}, + 'circle': {'cx', 'cy', 'r', 'fill', 'stroke', 'stroke-width'}, + }, + 'remove': {'script', 'style'} + } + + tags = nh3.ALLOWED_TAGS + for tag in policy['tags']: + tags.add(tag) + + attributes = deepcopy(nh3.ALLOWED_ATTRIBUTES) + attributes = {**attributes, **policy['attributes']} + + return nh3.clean( + html, tags=tags, clean_content_tags=policy['remove'], attributes=attributes, + ) From c4f167124c4d25b0bcb7713ac5c440aa4b4c132f Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 13:57:38 +0200 Subject: [PATCH 25/30] fix: broken svg, lint styles --- packages/syft/src/syft/assets/css/style.css | 11 +++++++---- packages/syft/src/syft/assets/svg/copy.svg | 6 ++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/syft/src/syft/assets/css/style.css b/packages/syft/src/syft/assets/css/style.css index 113a9908055..b4e624170b0 100644 --- a/packages/syft/src/syft/assets/css/style.css +++ b/packages/syft/src/syft/assets/css/style.css @@ -567,7 +567,11 @@ body { background-color: #c2def0; } -.syft-user_code, .syft-project, .syft-project-create, .syft-dataset, .syft-syncstate { +.syft-user_code, +.syft-project, +.syft-project-create, +.syft-dataset, +.syft-syncstate { color: var(--surface-color); } @@ -575,10 +579,9 @@ body { .syft-dataset p, .syft-syncstate h3, .syft-syncstate p { - font-family: 'Open Sans'; + font-family: "Open Sans"; } .diff-container { - border: 0.5px solid #B4B0BF; + border: 0.5px solid #b4b0bf; } - diff --git a/packages/syft/src/syft/assets/svg/copy.svg b/packages/syft/src/syft/assets/svg/copy.svg index aadd5116ebb..9e43a5b27f2 100644 --- a/packages/syft/src/syft/assets/svg/copy.svg +++ b/packages/syft/src/syft/assets/svg/copy.svg @@ -1,5 +1,3 @@ - - \ No newline at end of file + + From a1fb642e2ceff33c80dd63f095930c32938b19b0 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 14:00:13 +0200 Subject: [PATCH 26/30] lint --- packages/syft/src/syft/util/util.py | 37 +++++++++++++++++++---------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/packages/syft/src/syft/util/util.py b/packages/syft/src/syft/util/util.py index 59f7974ae9c..979c6673021 100644 --- a/packages/syft/src/syft/util/util.py +++ b/packages/syft/src/syft/util/util.py @@ -7,6 +7,7 @@ from concurrent.futures import ProcessPoolExecutor from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager +from copy import deepcopy import functools import hashlib from itertools import repeat @@ -29,7 +30,6 @@ import types from types import ModuleType from typing import Any -from copy import deepcopy # third party from IPython.display import display @@ -925,24 +925,37 @@ def get_dev_mode() -> bool: def sanitize_html(html: str) -> str: policy = { - 'tags': ['svg', 'strong', 'rect', 'path', 'circle'], - 'attributes': { - '*': {'class', 'style'}, - 'svg': {'class', 'style', 'xmlns', 'width', 'height', 'viewBox', 'fill', 'stroke', 'stroke-width'}, - 'path': {'d', 'fill', 'stroke', 'stroke-width'}, - 'rect': {'x', 'y', 'width', 'height', 'fill', 'stroke', 'stroke-width'}, - 'circle': {'cx', 'cy', 'r', 'fill', 'stroke', 'stroke-width'}, + "tags": ["svg", "strong", "rect", "path", "circle"], + "attributes": { + "*": {"class", "style"}, + "svg": { + "class", + "style", + "xmlns", + "width", + "height", + "viewBox", + "fill", + "stroke", + "stroke-width", + }, + "path": {"d", "fill", "stroke", "stroke-width"}, + "rect": {"x", "y", "width", "height", "fill", "stroke", "stroke-width"}, + "circle": {"cx", "cy", "r", "fill", "stroke", "stroke-width"}, }, - 'remove': {'script', 'style'} + "remove": {"script", "style"}, } tags = nh3.ALLOWED_TAGS - for tag in policy['tags']: + for tag in policy["tags"]: tags.add(tag) attributes = deepcopy(nh3.ALLOWED_ATTRIBUTES) - attributes = {**attributes, **policy['attributes']} + attributes = {**attributes, **policy["attributes"]} return nh3.clean( - html, tags=tags, clean_content_tags=policy['remove'], attributes=attributes, + html, + tags=tags, + clean_content_tags=policy["remove"], + attributes=attributes, ) From 6cc4507f7fd925a6a2985b2f13b7278619e3a0be Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 14:31:50 +0200 Subject: [PATCH 27/30] feat: add more styles to styles.css --- packages/syft/src/syft/assets/css/style.css | 33 ++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/assets/css/style.css b/packages/syft/src/syft/assets/css/style.css index b4e624170b0..d5ebf822b62 100644 --- a/packages/syft/src/syft/assets/css/style.css +++ b/packages/syft/src/syft/assets/css/style.css @@ -570,13 +570,20 @@ body { .syft-user_code, .syft-project, .syft-project-create, +.syft-settings, .syft-dataset, -.syft-syncstate { +.syft-asset, +.syft-contributor, +.syft-request, +.syft-syncstate, +.job-info { color: var(--surface-color); } .syft-dataset h3, .syft-dataset p, +.syft-asset h3, +.syft-asset p, .syft-syncstate h3, .syft-syncstate p { font-family: "Open Sans"; @@ -585,3 +592,27 @@ body { .diff-container { border: 0.5px solid #b4b0bf; } + +.syft-container { + padding: 5px; + font-family: 'Open Sans'; +} + +.syft-alert-info { + color: #1F567A; + background-color: #C2DEF0; + border-radius: 4px; + padding: 5px; + padding: 13px 10px +} + +.syft-code-block { + background-color: #f7f7f7; + border: 1px solid #cfcfcf; + padding: 0px 2px; +} + +.syft-space { + margin-top: 1em; +} + From d0cd6fd71eb259f90b9cf6354f9e0d0f0aab5b97 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 14:42:26 +0200 Subject: [PATCH 28/30] lint --- packages/syft/src/syft/assets/css/style.css | 9 ++++----- packages/syft/src/syft/util/util.py | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/syft/src/syft/assets/css/style.css b/packages/syft/src/syft/assets/css/style.css index d5ebf822b62..beece1fa2c0 100644 --- a/packages/syft/src/syft/assets/css/style.css +++ b/packages/syft/src/syft/assets/css/style.css @@ -595,15 +595,15 @@ body { .syft-container { padding: 5px; - font-family: 'Open Sans'; + font-family: "Open Sans"; } .syft-alert-info { - color: #1F567A; - background-color: #C2DEF0; + color: #1f567a; + background-color: #c2def0; border-radius: 4px; padding: 5px; - padding: 13px 10px + padding: 13px 10px; } .syft-code-block { @@ -615,4 +615,3 @@ body { .syft-space { margin-top: 1em; } - diff --git a/packages/syft/src/syft/util/util.py b/packages/syft/src/syft/util/util.py index 979c6673021..0860eb9e5e4 100644 --- a/packages/syft/src/syft/util/util.py +++ b/packages/syft/src/syft/util/util.py @@ -950,8 +950,8 @@ def sanitize_html(html: str) -> str: for tag in policy["tags"]: tags.add(tag) - attributes = deepcopy(nh3.ALLOWED_ATTRIBUTES) - attributes = {**attributes, **policy["attributes"]} + _attributes = deepcopy(nh3.ALLOWED_ATTRIBUTES) + attributes = {**_attributes, **policy["attributes"]} # type: ignore return nh3.clean( html, From eb3c196629be87835db3534756e516f18fcba2dd Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 15:34:50 +0200 Subject: [PATCH 29/30] fix: sanitize support for jobs --- packages/syft/src/syft/util/patch_ipython.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/util/patch_ipython.py b/packages/syft/src/syft/util/patch_ipython.py index 5e47ac5ef98..98666d62641 100644 --- a/packages/syft/src/syft/util/patch_ipython.py +++ b/packages/syft/src/syft/util/patch_ipython.py @@ -69,14 +69,19 @@ def _patch_ipython_sanitization() -> None: table_template = re.sub(r"\\{\\{.*?\\}\\}", ".*?", re.escape(table_template)) escaped_template = re.compile(table_template, re.DOTALL | re.VERBOSE) + jobs_repr_template = r'(.*?)' + jobs_pattern = re.compile(jobs_repr_template, re.DOTALL) + def display_sanitized_html(obj: SyftObject | DictTuple) -> str | None: if callable(getattr(obj, "_repr_html_", None)): html_str = obj._repr_html_() if html_str is not None: - matching_template = escaped_template.findall(html_str) - template = "\n".join(matching_template) + matching_table = escaped_template.findall(html_str) + matching_jobs = jobs_pattern.findall(html_str) + template = "\n".join(matching_table + matching_jobs) sanitized_str = escaped_template.sub("", html_str) sanitized_str = escaped_js_css.sub("", sanitized_str) + sanitized_str = jobs_pattern.sub("", html_str) sanitized_str = sanitize_html(sanitized_str) return f"{css_reinsert} {sanitized_str} {template}" return None From d0c378e5c179885c9e988ec0cbd771462058d4a3 Mon Sep 17 00:00:00 2001 From: Thiago Costa Porto Date: Fri, 21 Jun 2024 15:35:24 +0200 Subject: [PATCH 30/30] lint --- packages/syft/src/syft/util/patch_ipython.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/util/patch_ipython.py b/packages/syft/src/syft/util/patch_ipython.py index 98666d62641..4910eb77d52 100644 --- a/packages/syft/src/syft/util/patch_ipython.py +++ b/packages/syft/src/syft/util/patch_ipython.py @@ -69,7 +69,9 @@ def _patch_ipython_sanitization() -> None: table_template = re.sub(r"\\{\\{.*?\\}\\}", ".*?", re.escape(table_template)) escaped_template = re.compile(table_template, re.DOTALL | re.VERBOSE) - jobs_repr_template = r'(.*?)' + jobs_repr_template = ( + r"(.*?)" + ) jobs_pattern = re.compile(jobs_repr_template, re.DOTALL) def display_sanitized_html(obj: SyftObject | DictTuple) -> str | None: