Merge pull request #8968 from OpenMined/yash/tracing2

OpenTelemetry #2

packages/grid/backend/backend.dockerfile

@@ -41,7 +41,7 @@ COPY syft/src/syft/VERSION ./syft/src/syft/
 RUN --mount=type=cache,target=/root/.cache,sharing=locked \
     # remove torch because we already have the cpu version pre-installed
     sed --in-place /torch==/d ./syft/setup.cfg && \
-    uv pip install -e ./syft[data_science]
+    uv pip install -e ./syft[data_science,telemetry]
 
 # ==================== [Final] Setup Syft Server ==================== #
 
@@ -79,6 +79,7 @@ ENV \
     RELEASE="production" \
     DEV_MODE="False" \
     DEBUGGER_ENABLED="False" \
+    TRACING="False" \
     CONTAINER_HOST="docker" \
     DEFAULT_ROOT_EMAIL="[email protected]" \
     DEFAULT_ROOT_PASSWORD="changethis" \

packages/grid/backend/grid/core/config.py

@@ -159,6 +159,8 @@ def get_emails_enabled(self) -> Self:
     REVERSE_TUNNEL_ENABLED: bool = str_to_bool(
         os.getenv("REVERSE_TUNNEL_ENABLED", "false")
     )
+    TRACING_ENABLED: bool = str_to_bool(os.getenv("TRACING", "False"))
+
     model_config = SettingsConfigDict(case_sensitive=True)
 
 

packages/grid/backend/grid/core/server.py

@@ -10,8 +10,8 @@
 from syft.server.server import get_server_side_type
 from syft.server.server import get_server_type
 from syft.server.server import get_server_uid_env
-from syft.service.queue.zmq_queue import ZMQClientConfig
-from syft.service.queue.zmq_queue import ZMQQueueConfig
+from syft.service.queue.zmq_client import ZMQClientConfig
+from syft.service.queue.zmq_client import ZMQQueueConfig
 from syft.store.blob_storage.seaweedfs import SeaweedFSClientConfig
 from syft.store.blob_storage.seaweedfs import SeaweedFSConfig
 from syft.store.mongo_client import MongoStoreClientConfig

packages/grid/backend/grid/main.py

@@ -77,3 +77,58 @@ def healthcheck() -> dict[str, str]:
     probe on the pods backing the Service.
     """
     return {"status": "ok"}
+
+
+if settings.TRACING_ENABLED:
+    try:
+        # stdlib
+        import os
+
+        endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", None)
+        # third party
+        from opentelemetry._logs import set_logger_provider
+        from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
+        from opentelemetry.sdk._logs import LoggerProvider
+        from opentelemetry.sdk._logs import LoggingHandler
+        from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+        from opentelemetry.sdk.resources import Resource
+
+        logger_provider = LoggerProvider(
+            resource=Resource.create(
+                {
+                    "service.name": "backend-container",
+                }
+            ),
+        )
+        set_logger_provider(logger_provider)
+
+        exporter = OTLPLogExporter(insecure=True, endpoint=endpoint)
+
+        logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
+        handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider)
+
+        # Attach OTLP handler to root logger
+        logging.getLogger().addHandler(handler)
+        logger = logging.getLogger(__name__)
+        message = "> Added OTEL BatchLogRecordProcessor"
+        print(message)
+        logger.info(message)
+
+    except Exception as e:
+        print(f"Failed to load OTLPLogExporter. {e}")
+
+    # third party
+    try:
+        # third party
+        from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+
+        FastAPIInstrumentor.instrument_app(app)
+        message = "> Added OTEL FastAPIInstrumentor"
+        print(message)
+        logger = logging.getLogger(__name__)
+        logger.info(message)
+    except Exception as e:
+        error = f"Failed to load FastAPIInstrumentor. {e}"
+        print(error)
+        logger = logging.getLogger(__name__)
+        logger.error(message)

packages/grid/backend/grid/start.sh

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 set -e
 
-echo "Running Syft with RELEASE=${RELEASE} and $(id)"
+echo "Running Syft with RELEASE=${RELEASE}"
 
 APP_MODULE=grid.main:app
 LOG_LEVEL=${LOG_LEVEL:-info}
@@ -10,19 +10,40 @@ PORT=${PORT:-80}
 SERVER_TYPE=${SERVER_TYPE:-datasite}
 APPDIR=${APPDIR:-$HOME/app}
 RELOAD=""
-DEBUG_CMD=""
+ROOT_PROC=""
+
+echo "Starting with TRACING=${TRACING}"
 
 if [[ ${DEV_MODE} == "True" ]];
 then
-    echo "DEV_MODE Enabled"
+    echo "Hot-reload Enabled"
     RELOAD="--reload"
 fi
 
 # only set by kubernetes to avoid conflict with docker tests
 if [[ ${DEBUGGER_ENABLED} == "True" ]];
 then
+    echo "Debugger Enabled"
     uv pip install debugpy
-    DEBUG_CMD="python -m debugpy --listen 0.0.0.0:5678 -m"
+    ROOT_PROC="python -m debugpy --listen 0.0.0.0:5678 -m"
+fi
+
+if [[ ${TRACING} == "true" ]];
+then
+    echo "OpenTelemetry Enabled"
+
+    # TODOs:
+    # ! Handle case when OTEL_EXPORTER_OTLP_ENDPOINT is not set.
+    # ! syft-signoz-otel-collector.platform:4317 should be plumbed through helm charts
+    # ? Kubernetes OTel operator is recommended by signoz
+    export OTEL_PYTHON_LOG_CORRELATION=${OTEL_PYTHON_LOG_CORRELATION:-true}
+    export OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-"http://syft-signoz-otel-collector.platform:4317"}
+    export OTEL_EXPORTER_OTLP_PROTOCOL=${OTEL_EXPORTER_OTLP_PROTOCOL:-grpc}
+
+    # TODO: uvicorn postfork is not stable with OpenTelemetry
+    # ROOT_PROC="opentelemetry-instrument"
+else
+    echo "OpenTelemetry Disabled"
 fi
 
 export CREDENTIALS_PATH=${CREDENTIALS_PATH:-$HOME/data/creds/credentials.json}
@@ -33,4 +54,4 @@ export SERVER_TYPE=$SERVER_TYPE
 echo "SERVER_UID=$SERVER_UID"
 echo "SERVER_TYPE=$SERVER_TYPE"
 
-exec $DEBUG_CMD uvicorn $RELOAD --host $HOST --port $PORT --log-config=$APPDIR/grid/logging.yaml --log-level $LOG_LEVEL "$APP_MODULE"
+exec $ROOT_PROC uvicorn $RELOAD --host $HOST --port $PORT --log-config=$APPDIR/grid/logging.yaml "$APP_MODULE"

packages/grid/default.env

@@ -95,11 +95,6 @@ REDIS_HOST=redis
 CONTAINER_HOST=docker
 RELATIVE_PATH=""
 
-# Jaeger
-TRACE=False
-JAEGER_HOST=localhost
-JAEGER_PORT=14268
-
 # Syft
 SYFT_TUTORIAL_MODE=False
 ENABLE_WARNINGS=True

packages/grid/devspace.yaml

@@ -124,6 +124,14 @@ profiles:
         value:
           side: low
 
+  - name: tracing
+    description: "Enable Tracing"
+    patches:
+      - op: add
+        path: deployments.syft.helm.values.server
+        value:
+          tracing: true
+
   - name: bigquery-scenario-tests
     description: "Deploy a datasite for bigquery scenario testing"
     patches:

packages/grid/helm/examples/dev/base.yaml

@@ -7,6 +7,7 @@ global:
 server:
   rootEmail: [email protected]
   associationRequestAutoApproval: true
+  tracing: false
 
   resourcesPreset: null
   resources: null

packages/grid/helm/syft/templates/backend/backend-statefulset.yaml

@@ -87,7 +87,7 @@ spec:
             {{- end }}
             {{- if .Values.server.debuggerEnabled }}
             - name: DEBUGGER_ENABLED
-              value: "true"
+              value: "True"
             {{- end }}
             {{- if eq .Values.server.type "gateway" }}
             - name: ASSOCIATION_REQUEST_AUTO_APPROVAL
@@ -142,14 +142,8 @@ spec:
             - name: MIN_SIZE_BLOB_STORAGE_MB
               value: {{ .Values.seaweedfs.minSizeBlobStorageMB | quote }}
             # Tracing
-            - name: TRACE
-              value: "false"
-            - name: SERVICE_NAME
-              value: "backend"
-            - name: JAEGER_HOST
-              value: "localhost"
-            - name: JAEGER_PORT
-              value: "14268"
+            - name: TRACING
+              value: {{ .Values.server.tracing | default "False" | quote }}
             # Enclave attestation
             {{- if .Values.attestation.enabled }}
             - name: ENCLAVE_ATTESTATION_ENABLED

packages/grid/helm/syft/values.yaml

@@ -10,7 +10,6 @@ global:
   kaniko:
     version: "v1.23.2"
 
-
 # =================================================================================
 
 mongo:
@@ -174,6 +173,7 @@ server:
   debuggerEnabled: false
   associationRequestAutoApproval: false
   useInternalRegistry: true
+  tracing: false
 
   # Default Worker pool settings
   defaultWorkerPool:
@@ -185,7 +185,7 @@ server:
   smtp:
     # Existing secret for SMTP with key 'smtpPassword'
     existingSecret: null
-    host: smtp.sendgrid.net
+    host: hostname
     port: 587
     from: [email protected]
     username: apikey

packages/syft/setup.cfg

@@ -104,12 +104,20 @@ dev =
     aiosmtpd==1.4.6
 
 telemetry =
-    opentelemetry-api==1.14.0
-    opentelemetry-sdk==1.14.0
-    opentelemetry-exporter-jaeger==1.14.0
-    opentelemetry-instrumentation==0.35b0
-    opentelemetry-instrumentation-requests==0.35b0
-    ; opentelemetry-instrumentation-digma==0.9.1-beta.7
+    opentelemetry-api==1.27.0
+    opentelemetry-sdk==1.27.0
+    opentelemetry-exporter-otlp==1.27.0
+    opentelemetry-instrumentation==0.48b0
+    opentelemetry-instrumentation-requests==0.48b0
+    opentelemetry-instrumentation-fastapi==0.48b0
+    opentelemetry-instrumentation-pymongo==0.48b0
+    opentelemetry-instrumentation-botocore==0.48b0
+    opentelemetry-instrumentation-logging==0.48b0
+    ; opentelemetry-instrumentation-asyncio==0.48b0
+    ; opentelemetry-instrumentation-sqlite3==0.48b0
+    ; opentelemetry-instrumentation-threading==0.48b0
+    ; opentelemetry-instrumentation-jinja2==0.48b0
+    ; opentelemetry-instrumentation-system-metrics==0.48b0
 
 # pytest>=8.0 broke pytest-lazy-fixture which doesn't seem to be actively maintained
 # temporarily pin to pytest<8

packages/syft/src/syft/client/api.py

@@ -59,7 +59,6 @@
 from ..util.autoreload import autoreload_enabled
 from ..util.markdown import as_markdown_python_code
 from ..util.notebook_ui.components.tabulator_template import build_tabulator_table
-from ..util.telemetry import instrument
 from ..util.util import index_syft_by_module_name
 from ..util.util import prompt_warning_message
 from .connection import ServerConnection
@@ -237,7 +236,6 @@ def is_valid(self) -> bool:
         return True
 
 
-@instrument
 @serializable()
 class SyftAPICall(SyftObject):
     # version
@@ -264,7 +262,6 @@ def __repr__(self) -> str:
         return f"SyftAPICall(path={self.path}, args={self.args}, kwargs={self.kwargs}, blocking={self.blocking})"
 
 
-@instrument
 @serializable()
 class SyftAPIData(SyftBaseObject):
     # version
@@ -874,7 +871,6 @@ def result_needs_api_update(api_call_result: Any) -> bool:
     return False
 
 
-@instrument
 @serializable(
     attrs=[
         "endpoints",

packages/syft/src/syft/client/client.py

@@ -55,7 +55,6 @@
 from ..types.server_url import ServerURL
 from ..types.syft_object import SYFT_OBJECT_VERSION_1
 from ..types.uid import UID
-from ..util.telemetry import instrument
 from ..util.util import prompt_warning_message
 from ..util.util import thread_ident
 from ..util.util import verify_tls
@@ -648,7 +647,6 @@ def get_client_type(self) -> type[SyftClient]:
             raise SyftException(message=f"Unknown server type {metadata.server_type}")
 
 
-@instrument
 @serializable(canonical_name="SyftClient", version=1)
 class SyftClient:
     connection: ServerConnection
@@ -1117,7 +1115,6 @@ def refresh_callback() -> SyftAPI:
         return _api
 
 
-@instrument
 def connect(
     url: str | ServerURL = DEFAULT_SYFT_UI_ADDRESS,
     server: AbstractServer | None = None,
@@ -1135,7 +1132,6 @@ def connect(
     return client_type(connection=connection)
 
 
-@instrument
 def register(
     url: str | ServerURL,
     port: int,
@@ -1155,7 +1151,6 @@ def register(
     )
 
 
-@instrument
 def login_as_guest(
     # HTTPConnection
     url: str | ServerURL = DEFAULT_SYFT_UI_ADDRESS,
@@ -1179,7 +1174,6 @@ def login_as_guest(
     return _client.guest()
 
 
-@instrument
 def login(
     email: str,
     # HTTPConnection