feat: added the OpenTelemetry trace support for pd-store

apache · Mar 15, 2024 · d9387ea · d9387ea
1 parent d56ad10
commit d9387ea
Show file tree

Hide file tree

Showing 2 changed files with 100 additions and 8 deletions.
diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh
@@ -23,12 +23,17 @@ fi
 if [ -z "$USER_OPTION" ];then
   USER_OPTION=""
 fi
+if [ -z "$OPEN_TELEMETRY" ];then
+  OPEN_TELEMETRY="false"
+fi
 
-while getopts "g:j:v" arg; do
+while getopts "g:j:y:" arg; do
     case ${arg} in
         g) GC_OPTION="$OPTARG" ;;
         j) USER_OPTION="$OPTARG" ;;
-        ?) echo "USAGE: $0 [-g g1] [-j xxx] [-v]" && exit 1 ;;
+        # Telemetry is used to collect metrics, traces and logs
+        y) OPEN_TELEMETRY="$OPTARG" ;;
+        ?) echo "USAGE: $0 [-g g1] [-j xxx] [-y true|false]" && exit 1 ;;
     esac
 done
 
@@ -46,13 +51,16 @@ BIN=$(abs_path)
 TOP="$(cd "$BIN"/../ && pwd)"
 CONF="$TOP/conf"
 LIB="$TOP/lib"
+PLUGINS="$TOP/plugins"
 LOGS="$TOP/logs"
 OUTPUT=${LOGS}/hugegraph-pd-stdout.log
+GITHUB="https://github.com"
 PID_FILE="$BIN/pid"
 
 . "$BIN"/util.sh
 
-mkdir -p ${LOGS}
+ensure_path_writable "$LOGS"
+ensure_path_writable "$PLUGINS"
 
 # The maximum and minium heap memory that service can use
 MAX_MEM=$((32 * 1024))
@@ -104,6 +112,44 @@ case "$GC_OPTION" in
         exit 1
 esac
 
+if [ "${OPEN_TELEMETRY}" == "true" ]; then
+    OT_JAR="opentelemetry-javaagent.jar"
+    OT_JAR_PATH="${PLUGINS}/${OT_JAR}"
+
+    if [[ ! -e "${OT_JAR_PATH}" ]]; then
+        echo "## Downloading ${OT_JAR}..."
+        download "${PLUGINS}" \
+            "${GITHUB}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.1.0/${OT_JAR}"
+
+        if [[ ! -e "${OT_JAR_PATH}" ]]; then
+            echo "## Error: Failed to download ${OT_JAR}." >>${OUTPUT}
+            exit 1
+        fi
+    fi
+
+    # Note: remember update it if we change the jar
+    expected_md5="e3bcbbe8ed9b6d840fa4c333b36f369f"
+    actual_md5=$(md5sum "${OT_JAR_PATH}" | awk '{print $1}')
+
+    if [[ "${expected_md5}" != "${actual_md5}" ]]; then
+        echo "## Error: MD5 checksum verification failed for ${OT_JAR_PATH}." >>${OUTPUT}
+        echo "## Tips: Remove the file and try again." >>${OUTPUT}
+        exit 1
+    fi
+
+    # Note: check carefully if multi "javeagent" params are set
+    export JAVA_TOOL_OPTIONS="-javaagent:${PLUGINS}/${OT_JAR}"
+    export OTEL_TRACES_EXPORTER=otlp
+    export OTEL_METRICS_EXPORTER=none
+    export OTEL_LOGS_EXPORTER=none
+    export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc
+    # 127.0.0.1:4317 is the port of otel-collector running in Docker located in
+    # 'hugegraph-server/hugegraph-dist/docker/example/docker-compose-trace.yaml'.
+    # Make sure the otel-collector is running before starting HugeGraphPD.
+    export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://127.0.0.1:4317
+    export OTEL_RESOURCE_ATTRIBUTES=service.name=pd
+fi
+
 #if [ "${JMX_EXPORT_PORT}" != "" ] && [ ${JMX_EXPORT_PORT} -ne 0 ] ; then
 #  JAVA_OPTIONS="${JAVA_OPTIONS} -javaagent:${LIB}/jmx_prometheus_javaagent-0.16.1.jar=${JMX_EXPORT_PORT}:${CONF}/jmx_exporter.yml"
 #fi

diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh
@@ -31,12 +31,14 @@ BIN=$(abs_path)
 TOP="$(cd "$BIN"/../ && pwd)"
 CONF="$TOP/conf"
 LIB="$TOP/lib"
+PLUGINS="$TOP/plugins"
 LOGS="$TOP/logs"
 OUTPUT=${LOGS}/hugegraph-store-server.log
+GITHUB="https://github.com"
 PID_FILE="$BIN/pid"
 arch=$(arch)
 
-# TODO: repalce it with uname -a?
+# TODO: replace it with uname -a?
 echo "Current arch: ", ${arch}
 #if [[ $arch =~ "aarch64" ]];then
 #	  export LD_PRELOAD="$TOP/bin/libjemalloc_aarch64.so"
@@ -77,13 +79,17 @@ fi
 if [ -z "$USER_OPTION" ];then
   USER_OPTION=""
 fi
+if [ -z "$OPEN_TELEMETRY" ];then
+  OPEN_TELEMETRY="false"
+fi
 
-while getopts "g:j:v" arg; do
+while getopts "g:j:y:" arg; do
     case ${arg} in
         g) GC_OPTION="$OPTARG" ;;
         j) USER_OPTION="$OPTARG" ;;
-        v) VERBOSE="verbose" ;;
-        ?) echo "USAGE: $0 [-g g1] [-j xxx] [-v]" && exit 1 ;;
+        # Telemetry is used to collect metrics, traces and logs
+        y) OPEN_TELEMETRY="$OPTARG" ;;
+        ?) echo "USAGE: $0 [-g g1] [-j xxx] [-y true|false]" && exit 1 ;;
     esac
 done
 
@@ -92,7 +98,8 @@ done
 
 . "$BIN"/util.sh
 
-mkdir -p "${LOGS}"
+ensure_path_writable "$LOGS"
+ensure_path_writable "$PLUGINS"
 
 # The maximum and minimum heap memory that service can use (for production env set it 36GB)
 MAX_MEM=$((2 * 1024))
@@ -145,6 +152,45 @@ case "$GC_OPTION" in
 esac
 
 JVM_OPTIONS="-Dlog4j.configurationFile=${CONF}/log4j2.xml -Dfastjson.parser.safeMode=true"
+
+if [ "${OPEN_TELEMETRY}" == "true" ]; then
+    OT_JAR="opentelemetry-javaagent.jar"
+    OT_JAR_PATH="${PLUGINS}/${OT_JAR}"
+
+    if [[ ! -e "${OT_JAR_PATH}" ]]; then
+        echo "## Downloading ${OT_JAR}..."
+        download "${PLUGINS}" \
+            "${GITHUB}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.1.0/${OT_JAR}"
+
+        if [[ ! -e "${OT_JAR_PATH}" ]]; then
+            echo "## Error: Failed to download ${OT_JAR}." >>${OUTPUT}
+            exit 1
+        fi
+    fi
+
+    # Note: remember update it if we change the jar
+    expected_md5="e3bcbbe8ed9b6d840fa4c333b36f369f"
+    actual_md5=$(md5sum "${OT_JAR_PATH}" | awk '{print $1}')
+
+    if [[ "${expected_md5}" != "${actual_md5}" ]]; then
+        echo "## Error: MD5 checksum verification failed for ${OT_JAR_PATH}." >>${OUTPUT}
+        echo "## Tips: Remove the file and try again." >>${OUTPUT}
+        exit 1
+    fi
+
+    # Note: check carefully if multi "javeagent" params are set
+    export JAVA_TOOL_OPTIONS="-javaagent:${PLUGINS}/${OT_JAR}"
+    export OTEL_TRACES_EXPORTER=otlp
+    export OTEL_METRICS_EXPORTER=none
+    export OTEL_LOGS_EXPORTER=none
+    export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc
+    # 127.0.0.1:4317 is the port of otel-collector running in Docker located in
+    # 'hugegraph-server/hugegraph-dist/docker/example/docker-compose-trace.yaml'.
+    # Make sure the otel-collector is running before starting HugeGraphStore.
+    export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://127.0.0.1:4317
+    export OTEL_RESOURCE_ATTRIBUTES=service.name=store
+fi
+
 #if [ "${JMX_EXPORT_PORT}" != "" ] && [ ${JMX_EXPORT_PORT} -ne 0 ] ; then
 #  JAVA_OPTIONS="${JAVA_OPTIONS} -javaagent:${LIB}/jmx_prometheus_javaagent-0.16.1.jar=${JMX_EXPORT_PORT}:${CONF}/jmx_exporter.yml"
 #fi