Add experimental instrumentation for OpenAI client

CHANGELOG.next-release.md

@@ -1,3 +1,4 @@
 * add dynamically disabled instrumentation capability - #422
 * add disable all instrumentations option - #471
 * add stop-sending option - #474
+* Add OpenAI client instrumentation - #498

agentextension/build.gradle.kts

@@ -2,7 +2,7 @@ plugins {
   id("elastic-otel.java-conventions")
   id("elastic-otel.sign-and-publish-conventions")
   id("elastic-otel.license-report-conventions")
-  id("com.github.johnrengelman.shadow")
+  id("com.gradleup.shadow")
 }
 
 description = "Bundles all elastic extensions in a fat-jar to be used" +

buildSrc/build.gradle.kts

@@ -15,8 +15,15 @@ repositories {
 
 dependencies {
     implementation(catalog.spotlessPlugin)
-    implementation(catalog.shadowPlugin)
     implementation(catalog.licenseReportPlugin)
+    // muzzle pulls in ancient versions of http components which conflict with other plugins, such as jib
+    implementation(catalog.muzzleGenerationPlugin) {
+      exclude(group = "org.apache.httpcomponents" )
+    }
+    implementation(catalog.muzzleCheckPlugin) {
+      exclude(group = "org.apache.httpcomponents" )
+    }
+    implementation(catalog.shadowPlugin)
     // The ant dependency is required to add custom transformers for the shadow plugin
     // but it is unfortunately not exposed as transitive dependency
     implementation(catalog.ant)

buildSrc/src/main/kotlin/elastic-otel.agent-packaging-conventions.gradle.kts

@@ -8,7 +8,7 @@ import org.objectweb.asm.Opcodes
 
 plugins {
   id("elastic-otel.java-conventions")
-  id("com.github.johnrengelman.shadow")
+  id("com.gradleup.shadow")
 }
 
 

buildSrc/src/main/kotlin/elastic-otel.instrumentation-conventions.gradle.kts

@@ -0,0 +1,93 @@
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+
+plugins {
+  `java-library`
+  id("elastic-otel.java-conventions")
+  id("io.opentelemetry.instrumentation.muzzle-generation")
+  id("io.opentelemetry.instrumentation.muzzle-check")
+}
+
+// Other instrumentations to include for testing
+val testInstrumentation: Configuration by configurations.creating {
+  isCanBeConsumed = false
+}
+val agentForTesting: Configuration by configurations.creating {
+  isCanBeConsumed = false
+}
+
+//https://github.com/gradle/gradle/issues/15383
+val catalog = extensions.getByType<VersionCatalogsExtension>().named("catalog")
+dependencies {
+  agentForTesting(platform(catalog.findLibrary("opentelemetryInstrumentationAlphaBom").get()))
+  agentForTesting("io.opentelemetry.javaagent:opentelemetry-agent-for-testing")
+
+  compileOnly("io.opentelemetry:opentelemetry-sdk")
+  compileOnly("io.opentelemetry.instrumentation:opentelemetry-instrumentation-api")
+  compileOnly("io.opentelemetry.javaagent:opentelemetry-javaagent-extension-api")
+
+  testImplementation("io.opentelemetry.javaagent:opentelemetry-testing-common")
+  testImplementation("io.opentelemetry:opentelemetry-sdk-testing")
+
+  val agentVersion = catalog.findVersion("opentelemetryJavaagentAlpha").get()
+  add("codegen", "io.opentelemetry.javaagent:opentelemetry-javaagent-tooling:${agentVersion}")
+  add("muzzleBootstrap", "io.opentelemetry.instrumentation:opentelemetry-instrumentation-annotations-support:${agentVersion}")
+  add("muzzleTooling", "io.opentelemetry.javaagent:opentelemetry-javaagent-extension-api:${agentVersion}")
+  add("muzzleTooling", "io.opentelemetry.javaagent:opentelemetry-javaagent-tooling:${agentVersion}")
+}
+
+fun relocatePackages( shadowJar : ShadowJar) {
+  // rewrite dependencies calling Logger.getLogger
+  shadowJar.relocate("java.util.logging.Logger", "io.opentelemetry.javaagent.bootstrap.PatchLogger")
+
+  // prevents conflict with library instrumentation, since these classes live in the bootstrap class loader
+  shadowJar.relocate("io.opentelemetry.instrumentation", "io.opentelemetry.javaagent.shaded.instrumentation") {
+    // Exclude resource providers since they live in the agent class loader
+    exclude("io.opentelemetry.instrumentation.resources.*")
+    exclude("io.opentelemetry.instrumentation.spring.resources.*")
+  }
+
+  // relocate(OpenTelemetry API) since these classes live in the bootstrap class loader
+  shadowJar.relocate("io.opentelemetry.api", "io.opentelemetry.javaagent.shaded.io.opentelemetry.api")
+  shadowJar.relocate("io.opentelemetry.semconv", "io.opentelemetry.javaagent.shaded.io.opentelemetry.semconv")
+  shadowJar.relocate("io.opentelemetry.context", "io.opentelemetry.javaagent.shaded.io.opentelemetry.context")
+  shadowJar.relocate("io.opentelemetry.extension.incubator", "io.opentelemetry.javaagent.shaded.io.opentelemetry.extension.incubator")
+
+  // relocate the OpenTelemetry extensions that are used by instrumentation modules
+  // these extensions live in the AgentClassLoader, and are injected into the user's class loader
+  // by the instrumentation modules that use them
+  shadowJar.relocate("io.opentelemetry.extension.aws", "io.opentelemetry.javaagent.shaded.io.opentelemetry.extension.aws")
+  shadowJar.relocate("io.opentelemetry.extension.kotlin", "io.opentelemetry.javaagent.shaded.io.opentelemetry.extension.kotlin")
+}
+
+tasks {
+  shadowJar {
+    configurations = listOf(project.configurations.runtimeClasspath.get(), testInstrumentation)
+    mergeServiceFiles()
+
+    archiveFileName.set("agent-testing.jar")
+    relocatePackages(this)
+  }
+}
+
+tasks.withType<Test>().configureEach {
+  dependsOn(tasks.shadowJar, agentForTesting)
+
+  jvmArgs(
+    "-Dotel.javaagent.debug=true",
+    "-javaagent:${agentForTesting.files.first().absolutePath}",
+    // loads the given just jar, but in contrast to external extensions doesn't perform runtime shading
+    // instead the instrumentations are expected to be correctly shaded already in the jar
+    // Also the classes end up in the agent classloader instead of the extension loader
+    "-Dotel.javaagent.experimental.initializer.jar=${tasks.shadowJar.get().archiveFile.get().asFile.absolutePath}",
+    "-Dotel.javaagent.testing.additional-library-ignores.enabled=false",
+    "-Dotel.javaagent.testing.fail-on-context-leak=true",
+    "-Dotel.javaagent.testing.transform-safe-logging.enabled=true",
+    "-Dotel.metrics.exporter=otlp"
+  )
+
+  // The sources are packaged into the testing jar so we need to make sure to exclude from the test
+  // classpath, which automatically inherits them, to ensure our shaded versions are used.
+  classpath = classpath.filter {
+    return@filter !(it == file("${layout.buildDirectory.get()}/resources/main") || it == file("${layout.buildDirectory.get()}/classes/java/main"))
+  }
+}

custom/build.gradle.kts

@@ -2,11 +2,19 @@ plugins {
   id("elastic-otel.library-packaging-conventions")
 }
 
+val instrumentations = listOf<String>(
+  ":instrumentation:openai-client-instrumentation"
+)
+
 dependencies {
   implementation(project(":common"))
   implementation(project(":inferred-spans"))
   implementation(project(":universal-profiling-integration"))
   implementation(project(":resources"))
+  instrumentations.forEach {
+    implementation(project(it))
+  }
+
   compileOnly("io.opentelemetry:opentelemetry-sdk")
   compileOnly("io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi")
   compileOnly("io.opentelemetry.javaagent:opentelemetry-javaagent-extension-api")
@@ -37,6 +45,21 @@ dependencies {
   testImplementation("io.opentelemetry:opentelemetry-sdk-testing")
   testImplementation(libs.freemarker)
 }
+
+tasks {
+  instrumentations.forEach {
+    // TODO: instrumentation dependencies must be declared here explicitly atm, otherwise gradle complains
+    // about it being missing we need to figure out a way of including it in the "compileJava" task
+    // to not have to do this
+    javadoc {
+      dependsOn("$it:byteBuddyJava")
+    }
+    compileTestJava {
+      dependsOn("$it:byteBuddyJava")
+    }
+  }
+}
+
 tasks.withType<Test> {
   val overrideConfig = project.properties["elastic.otel.overwrite.config.docs"]
   if (overrideConfig != null) {

docs/configure.md

@@ -111,6 +111,26 @@ _Currently there are no additional `OTEL_` options waiting to be contributed ups
 | `ELASTIC_OTEL_UNIVERSAL_PROFILING_INTEGRATION_*` | [Universal profiling integration](https://github.com/elastic/elastic-otel-java/tree/main/universal-profiling-integration) | Correlates traces with profiling data from the Elastic universal profiler. |
 | `ELASTIC_OTEL_JAVA_SPAN_STACKTRACE_MIN_DURATION` | [Span stacktrace capture](https://github.com/open-telemetry/opentelemetry-java-contrib/tree/main/span-stacktrace) | Define the minimum duration for attaching stack traces to spans. Defaults to 5ms. |
 
+### Instrumentations that are _only_ available in EDOT Java
+
+Some instrumentation are only available in EDOT Java and might or might not be added upstream in future versions.
+
+#### OpenAI Java Client
+
+Instrumentation for the [official OpenAI Java Client](https://github.com/openai/openai-java).
+It supports:
+ * Tracing for requests, including GenAI-specific attributes such as token usage
+ * Opt-In logging of OpenAI request and response content payloads
+
+This instrumentation is currently **experimental**. It can by disabled by setting either the `OTEL_INSTRUMENTATION_OPENAI_CLIENT_ENABLED` environment variable or the `otel.instrumentation.openai-client.enabled` JVM property to `false`.
+
+In addition, this instrumentation provides the following configuration options:
+
+| Option(s)                                             | Description                                                                                                                                                                                                                                                                                                                                                                                                   |
+|-------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `ELASTIC_OTEL_JAVA_INSTRUMENTATION_GENAI_EMIT_EVENTS` | If set to `true`, the agent will generate log events for OpenAI requests and responses. Potentially sensitive content will only be included if `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` is true. Defaults to the value of `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`, so that just enabling `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` ensures that log events are generated. |
+| `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`  | If set to `true`, enables the capturing of OpenAI request and response content in the log events outputted by the agent. Defaults to `false`                                                                                                                                                                                                                                                                  |
+
 <!-- ✅ List auth methods -->
 ## Authentication methods
 

gradle/libs.versions.toml

@@ -1,5 +1,5 @@
 [versions]
-shadow = "8.1.1"
+shadow = "8.3.5"
 jib = "3.4.4"
 spotless = "7.0.1"
 junit = "5.11.4"
@@ -22,6 +22,9 @@ opentelemetryContribAlpha = "1.42.0-alpha"
 # reference the "incubating" version explicitly
 opentelemetrySemconvAlpha = "1.29.0-alpha"
 
+# instrumented libraries
+openaiClient = "0.11.0"
+
 [libraries]
 
 # transitively provides 'opentelemetry-instrumentation-bom' (non-alpha)
@@ -44,6 +47,7 @@ autoservice-annotations = { group = "com.google.auto.service", name = "auto-serv
 assertj-core = "org.assertj:assertj-core:3.27.2"
 awaitility = "org.awaitility:awaitility:4.2.2"
 findbugs-jsr305 = "com.google.code.findbugs:jsr305:3.0.2"
+wiremock = "com.github.tomakehurst:wiremock-jre8:2.35.2"
 testcontainers = "org.testcontainers:testcontainers:1.20.4"
 logback = "ch.qos.logback:logback-classic:1.5.16"
 jackson = "com.fasterxml.jackson.core:jackson-databind:2.18.2"
@@ -66,13 +70,18 @@ asyncprofiler = "tools.profiler:async-profiler:3.0"
 freemarker = "org.freemarker:freemarker:2.3.34"
 
 spotlessPlugin = { group = "com.diffplug.spotless", name = "spotless-plugin-gradle", version.ref = "spotless" }
-shadowPlugin = { group = "com.github.johnrengelman", name = "shadow", version.ref = "shadow" }
+shadowPlugin = { group = "com.gradleup.shadow", name = "shadow-gradle-plugin", version.ref = "shadow" }
 licenseReportPlugin = "com.github.jk1.dependency-license-report:com.github.jk1.dependency-license-report.gradle.plugin:2.9"
+muzzleCheckPlugin = { group = "io.opentelemetry.instrumentation.muzzle-check", name = "io.opentelemetry.instrumentation.muzzle-check.gradle.plugin", version.ref = "opentelemetryJavaagentAlpha" }
+muzzleGenerationPlugin = { group = "io.opentelemetry.instrumentation.muzzle-generation", name = "io.opentelemetry.instrumentation.muzzle-generation.gradle.plugin", version.ref = "opentelemetryJavaagentAlpha" }
 # Ant should be kept in sync with the version used in the shadow plugin
 ant = "org.apache.ant:ant:1.10.15"
 # ASM is currently only used during compile-time, so it is okay to diverge from the version used in ByteBuddy
 asm = "org.ow2.asm:asm:9.7"
 
+# Instrumented libraries
+openaiClient = {group = "com.openai", name = "openai-java", version.ref ="openaiClient"}
+
 [bundles]
 
 semconv = ["opentelemetrySemconv", "opentelemetrySemconvIncubating"]
@@ -84,4 +93,3 @@ taskinfo = { id = "org.barfuin.gradle.taskinfo", version = '2.2.0' }
 jmh = {id = "me.champeau.jmh", version = "0.7.2"}
 nexusPublish = { id = "io.github.gradle-nexus.publish-plugin", version = '2.0.0' }
 dockerJavaApplication = { id = "com.bmuschko.docker-java-application", version = "9.4.0" }
-shadow = { id = "com.github.johnrengelman.shadow", version.ref = "shadow" }

instrumentation/openai-client-instrumentation/build.gradle.kts

@@ -0,0 +1,19 @@
+plugins {
+  id("elastic-otel.instrumentation-conventions")
+}
+
+dependencies {
+  compileOnly(catalog.openaiClient)
+  testImplementation(catalog.openaiClient)
+
+  testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.18.2")
+  testImplementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.18.2")
+  testImplementation("org.slf4j:slf4j-simple:2.0.16")
+  testImplementation(catalog.wiremock)
+}
+
+muzzle {
+  // TODO: setup muzzle to check older versions of openAI client
+  // See the docs on how to do it:
+  // https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/main/docs/contributing/muzzle.md
+}