Mask or hide http.url in Jaeger UI because of sensitive query strings.

[yayaythegreat]

Hello,

As mentioned above, I am trying to mask my http.url due to some request includes query strings that contains sensitive data.

I have setup jaeger with istio-envoy

meshConfig:
    accessLogFile: /dev/stdout
    accessLogEncoding: JSON
    accessLogFormat: |
      {
        "protocol": "%PROTOCOL%",
        "method": "%REQ(:METHOD)%",
        "path": "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%",
        "responseCode": "%RESPONSE_CODE%",
        "clientDuration": "%DURATION%",
        "responseCodeDetails": "%RESPONSE_CODE_DETAILS%",
        "connectionTerminationDetails": "%CONNECTION_TERMINATION_DETAILS%",
        "targetDuration": "%RESPONSE_DURATION%",
        "upstreamName": "%UPSTREAM_CLUSTER%",
        "traceId": "%REQ(X-B3-Traceid)%",
        "responseFlags": "%RESPONSE_FLAGS%",
        "routeName": "%ROUTE_NAME%",
        "downstreamRemoteAddress": "%DOWNSTREAM_REMOTE_ADDRESS%",
        "upstreamHost": "%UPSTREAM_HOST%",
        "downstreamLocalURISan": "%DOWNSTREAM_LOCAL_URI_SAN%",
        "requestHeaders": "%DYNAMIC_METADATA(envoy.lua:request_headers)%",
        "requestBody": "%DYNAMIC_METADATA(envoy.lua:request_body)%",
        "responseHeaders": "%DYNAMIC_METADATA(envoy.lua:response_headers)%",
        "responseBody": "%DYNAMIC_METADATA(envoy.lua:response_body)%"
      }
    defaultConfig:
      proxyMetadata: {}
      tracing:
        zipkin:
          address: jaeger-collector.jaegertracing.svc.cluster.local:9411

I have tried using EnvoyFilter to intecept traffic going to jaeger-collector and replace the http.url to a different value.

The code for EnvoyFilter is as follows

apiVersion: networking.istio.io/v1alpha3
kind: EnvoyFilter
metadata:
  name: request-response-filter
  namespace: istio-system
spec:
  configPatches:
  - applyTo: HTTP_FILTER
    match:
      context: ANY
      listener:
        filterChain:
          filter:
            name: "envoy.filters.network.http_connection_manager"
            subFilter:
              name: "envoy.filters.http.router"
    patch:
      operation: INSERT_BEFORE
      value:
        name: envoy.lua
        typed_config:
            "@type": "type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua"
            inlineCode: |
              function envoy_on_request(request_handle)
                local headers = request_handle:headers()
                local headersMap = {}
                for key, value in pairs(headers) do
                  headersMap[key] = value
                end              
                request_handle:streamInfo():dynamicMetadata():set("envoy.lua","request_headers",headersMap)                    
                local requestBody = ""
                for chunk in request_handle:bodyChunks() do
                  if (chunk:length() > 0) then
                    requestBody = requestBody .. chunk:getBytes(0, chunk:length())
                  end
                end
                local modifiedRequestBody = string.gsub(requestBody, "(\"http%.url\":\")(.-)(\")", "%1http://*******/*****%3")

                request_handle:streamInfo():dynamicMetadata():set("envoy.lua","request_body",modifiedRequestBody)                    
              end

Upon checking the logs of istio-proxy inside jaeger-collector pod the Request body was modified into something like http://*******/***** but it doesn't reflect to Jaeger-UI.

Here's a sample payload for jaeger-collector:

{
  "method": "POST",
  "path": "/api/v2/spans",
  "upstreamHost": "172.19.4.89:9411",
  "responseHeaders": null,
  "routeName": "default",
  "responseFlags": "-",
  "downstreamLocalURISan": null,
  "protocol": "HTTP/1.1",
  "requestBody": "[{\"tags\":{\**"http.url\":\"http://*******/*****\**",\"node_id\":\"sidecar~172.19.4.31~productpage-v1-6b8b984bd4-d2rmv.default~default.svc.cluster.local\",\"upstream_cluster\":\"outbound|9080||details.default.svc.cluster.local\",\"upstream_cluster.name\":\"outbound|9080||details.default.svc.cluster.local\",\"guid:x-request-id\":\"e480180c-7a05-9610-8434-37c0360ebac7\",\"response_size\":\"178\",\"user_agent\":\"curl/7.79.1\",\"component\":\"proxy\",\"downstream_cluster\":\"-\",\"istio.canonical_service\":\"productpage\",\"istio.mesh_id\":\"cluster.local\",\"istio.canonical_revision\":\"v1\",\"http.method\":\"GET\",\"request_size\":\"0\",\"peer.address\":\"172.19.4.31\",\"http.protocol\":\"HTTP/1.1\",\"response_flags\":\"-\",\"istio.namespace\":\"default\",\"http.status_code\":\"200\"},\"localEndpoint\":{\"ipv4\":\"172.19.4.31\",\"serviceName\":\"productpage.default\",\"port\":0},\"annotations\":[{\"timestamp\":1687342787002381,\"value\":\"cr\"}],\"parentId\":\"2ae586b42e6884e6\",\"id\":\"fe2db7ec814af12d\",\"duration\":5858,\"name\":\"details.default.svc.cluster.local:9080/*\",\"timestamp\":1687342786996409,\"traceId\":\"808868d8f245f34fcb88cdac1214be63\",\"kind\":\"CLIENT\"},{\"localEndpoint\":{\"serviceName\":\"productpage.default\",\"port\":0,\"ipv4\":\"172.19.4.31\"},\"kind\":\"CLIENT\",\"id\":\"2bab9313a42507df\",\"tags\":{\"downstream_cluster\":\"-\",\"response_flags\":\"-\",\"user_agent\":\"curl/7.79.1\",\"istio.canonical_service\":\"productpage\",\"upstream_cluster.name\":\"outbound|9080||reviews.default.svc.cluster.local\",\"http.url\":\"**http://*******/*****\**",\"http.status_code\":\"200\",\"upstream_cluster\":\"outbound|9080||reviews.default.svc.cluster.local\",\"response_size\":\"358\",\"component\":\"proxy\",\"peer.address\":\"172.19.4.31\",\"request_size\":\"0\",\"istio.canonical_revision\":\"v1\",\"node_id\":\"sidecar~172.19.4.31~productpage-v1-6b8b984bd4-d2rmv.default~default.svc.cluster.local\",\"istio.mesh_id\":\"cluster.local\",\"http.method\":\"GET\",\"guid:x-request-id\":\"e480180c-7a05-9610-8434-37c0360ebac7\",\"http.protocol\":\"HTTP/1.1\",\"istio.namespace\":\"default\"},\"duration\":5627,\"name\":\"reviews.default.svc.cluster.local:9080/*\",\"annotations\":[{\"value\":\"cr\",\"timestamp\":1687342787023761}],\"timestamp\":1687342787018026,\"parentId\":\"2ae586b42e6884e6\",\"traceId\":\"808868d8f245f34fcb88cdac1214be63\"}]",
  "downstreamRemoteAddress": "172.19.4.31:0",
  "upstreamName": "inbound|9411||",
  "traceId": "3ccf4222672042042118ba08fcc1b434",
  "responseBody": null,
  "targetDuration": 1,
  "clientDuration": 1,
  "responseCode": 202,
  "requestHeaders": {
    "x-forwarded-for": "172.19.4.31",
    ":authority": "zipkin",
    "x-envoy-expected-rq-timeout-ms": "5000",
    ":scheme": "http",
    ":path": "/api/v2/spans",
    "x-envoy-internal": "true",
    "x-forwarded-proto": "http",
    "content-type": "application/json",
    "x-request-id": "f4f027db-0900-4604-bef9-51646eb19127",
    ":method": "POST"
  },
  "responseCodeDetails": "via_upstream",
  "connectionTerminationDetails": null
}

And this is screenshot of Jaeger UI with the corresponding TraceID

Not sure if I made a mistake somewhere or missing something.

Thank you!

[yurishkuro]

This is a complex topic. For example, why do you want to store sensitive data in Jaeger but not have it displayed? Should it be sanitized during ingestion instead? You can probably do this today already with some processor in OTEL Collector.

[yayaythegreat]

Hi @yurishkuro. I'm using istio-sidecar-proxy to ingest data and send it directly to Jaeger. So, every tags sent is already pre-defined by istio including http.url that has query-parameters.

Few alternative I can think of for my end is:

1. Hide tags that have sensitive data. This is not yet supported I believe
2. Make http.url request without parameters. Not really sure how to do this

[yurishkuro]

You can pass data through OTEL Collector first and use various processors to sanitize the data, e.g. https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/attributesprocessor/README.md