Merge branch 'refs/heads/main' into support/httpserver-response-NR-27…

…7771

.github/workflows/publish-release-to-maven.yml

@@ -12,8 +12,8 @@ jobs:
     with:
       apm-repo: 'k2io/newrelic-java-agent'
       apm-source-ref: 'csec-dev'
-      csec-run-unittest: 'true'
-      csec-run-instrumentation-verify: 'true'
+      csec-run-unittest: 'false'
+      csec-run-instrumentation-verify: 'false'
       is-release: 'true'
       version-suffix: ''
       slack-notify: 'true'

Changelog.md

@@ -4,6 +4,122 @@ Noteworthy changes to the agent are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.5.0] - 2024-9-25
+### New features
+- Json Version bump to 1.2.9.
+- [PR-327](https://github.com/newrelic/csec-java-agent/pull/327) Application endpoint detection for gRPC Server [NR-303616](https://new-relic.atlassian.net/browse/NR-303616)
+- [PR-326](https://github.com/newrelic/csec-java-agent/pull/326) Add IAST Scan start time and Traffic Start Time in Health Check [NR-308822](https://new-relic.atlassian.net/browse/NR-308822)
+- [PR-320](https://github.com/newrelic/csec-java-agent/pull/320) Add feature to allow IAST Scan Scheduling. [NR-301534](https://new-relic.atlassian.net/browse/NR-301534)
+  Configuration via yaml:
+  ```yaml
+  security:
+      scan_schedule:
+        # The delay field specifies the delay in minutes before the IAST scan starts. This allows to schedule the scan to start at a later time.
+        delay: 0        #In minutes, default is 0 min
+
+        # The duration field specifies the duration of the IAST scan in minutes. This determines how long the scan will run.
+        duration: 0      #In minutes, default is forever
+
+        # The schedule field specifies a cron expression that defines when the IAST scan should start.
+        #schedule: ""   #By default, schedule is inactive
+
+        # Allow continuously sample collection of IAST events
+        always_sample_traces: false # Default is false
+  ```
+- [PR-320](https://github.com/newrelic/csec-java-agent/pull/320) Add feature to ignore IAST Scan of certain APIs, categories, or parameters. [NR-301856](https://new-relic.atlassian.net/browse/NR-301856)
+  Configuration via yaml:
+  ```yaml
+  security:
+     # The exclude_from_iast_scan configuration allows to specify APIs, parameters, and categories that should not be scanned by Security Agents.
+    exclude_from_iast_scan:
+      # The api field specifies list of APIs using regular expression (regex) patterns that follow the syntax of Perl 5. The regex pattern should provide a complete match for the URL without the endpoint.
+      # Example:
+      #   api:
+      #    - .*account.*
+      #    - .*/\api\/v1\/.*?\/login
+      api: []
+
+      # The parameters configuration allows users to specify headers, query parameters, and body keys that should be excluded from IAST scans.
+      # Example:
+      #   http_request_parameters:
+      #    header:
+      #      - X-Forwarded-For
+      #    query:
+      #      - username
+      #      - password
+      #    body:
+      #      - account.email
+      #      - account.contact
+      http_request_parameters:
+        # A list of HTTP header keys. If a request includes any headers with these keys, the corresponding IAST scan will be skipped.
+        header: []
+        # A list of query parameter keys. The presence of these parameters in the request's query string will lead to skipping the IAST scan.
+        query: []
+        # A list of keys within the request body. If these keys are found in the body content, the IAST scan will be omitted.
+        body: []
+
+      # The iast_detection_category configuration allows to specify which categories of vulnerabilities should not be detected by Security Agents.
+      # If any of these categories are set to true, Security Agents will not generate events or flag vulnerabilities for that category.
+      iast_detection_category:
+        insecure_settings: false
+        invalid_file_access: false
+        sql_injection: false
+        nosql_injection: false
+        ldap_injection: false
+        javascript_injection: false
+        command_injection: false
+        xpath_injection: false
+        ssrf: false
+        rxss: false
+  ```
+- [PR-321](https://github.com/newrelic/csec-java-agent/pull/321) Add feature to rate limit the IAST replay requests. [NR-304574](https://new-relic.atlassian.net/browse/NR-304574)
+  ```yaml
+  security:
+    scan_controllers:
+      # The scan_request_rate_limit configuration allows to specify maximum number of replay request played per minute.
+      iast_scan_request_rate_limit: 3600 # Number of IAST replay request played per minute, Default is 3600
+  ```
+- [PR-315](https://github.com/newrelic/csec-java-agent/pull/315) GraphQL Support : The security agent now also supports GraphQL Version 16.0.0 and above, default is disabled. [NR-299885](https://new-relic.atlassian.net/browse/NR-299885)
+
+### Fixes
+- [PR-322](https://github.com/newrelic/csec-java-agent/pull/322) Report Application endpoints immediately upon detecting new endpoints. [NR-287324](https://new-relic.atlassian.net/browse/NR-287324)
+- [PR-323](https://github.com/newrelic/csec-java-agent/pull/323) Extract Server Configuration to resolve IAST localhost connection with application for WebSphere Liberty server [NR-303483](https://new-relic.atlassian.net/browse/NR-303483)
+- [PR-327](https://github.com/newrelic/csec-java-agent/pull/327) Fix for User Class Detection in gRPC Server [NR-303616](https://new-relic.atlassian.net/browse/NR-303616)
+- [PR-328](https://github.com/newrelic/csec-java-agent/pull/328) Fix for multiple Reflected Events observed in Jersey Framework [NR-307644](https://new-relic.atlassian.net/browse/NR-307644)
+- [PR-325](https://github.com/newrelic/csec-java-agent/pull/325) Fix for incorrect Application endpoints detected for Servlet Framework [NR-303615](https://new-relic.atlassian.net/browse/NR-303615)
+- [PR-320](https://github.com/newrelic/csec-java-agent/pull/320) Report only uncaught exceptions in IAST Error inbox. [NR-313412](https://new-relic.atlassian.net/browse/NR-313412)
+
+### Deprecations
+- Status File Used for Debugging: This feature has been deprecated. All debugging capabilities have been moved to either Init Logging or [Error Inbox](https://docs.newrelic.com/docs/errors-inbox/errors-inbox/) and will be removed in a future agent release. [NR-293966](https://new-relic.atlassian.net/browse/NR-293966)
+
+
+## [1.4.1] - 2024-8-14
+### Adds
+- [PR-296](https://github.com/newrelic/csec-java-agent/pull/296) Apache Solr Support: The security agent now also supports Apache Solr Version 4.0.0 and above. [NR-288599](https://new-relic.atlassian.net/browse/NR-288599)
+- [PR-275](https://github.com/newrelic/csec-java-agent/pull/275) The maximum permissible size for a request body for scan will be set at 500KB. [NR-174195](https://new-relic.atlassian.net/browse/NR-174195)
+- [PR-306](https://github.com/newrelic/csec-java-agent/pull/306) Add csec prefix to all instrumentation Jar, this resolves CVE flagged by third party scanners on our instrumentation JARs. [NR-289249](https://new-relic.atlassian.net/browse/NR-289249)
+- [PR-303](https://github.com/newrelic/csec-java-agent/pull/303) Honour OFF Flag, Handle Boolean values for config log_level. [NR-293102](https://new-relic.atlassian.net/browse/NR-293102)
+- [PR-299](https://github.com/newrelic/csec-java-agent/pull/299) Support Authentication capabilities for Proxy Settings. [NR-283945](https://new-relic.atlassian.net/browse/NR-283945)
+- [PR-313](https://github.com/newrelic/csec-java-agent/pull/313) Processing of the security agent will persist even if the creation of the security home directory encounters an issue. [NR-297206](https://new-relic.atlassian.net/browse/NR-297206)
+- [PR-277](https://github.com/newrelic/csec-java-agent/pull/277) Improve Management of Log file size and its count. [NR-272900](https://new-relic.atlassian.net/browse/NR-272900)
+- [PR-314](https://github.com/newrelic/csec-java-agent/pull/314) Report error to Error Inbox upon connection failure to Security Engine. [NR-299700](https://new-relic.atlassian.net/browse/NR-299700)
+- [PR-316](https://github.com/newrelic/csec-java-agent/pull/316) Detailed IAST Scan metric reporting via HealthCheck. [NR-267166](https://new-relic.atlassian.net/browse/NR-267166)
+- [PR-302](https://github.com/newrelic/csec-java-agent/pull/302) Detect API Endpoint of the Application for Vertx Framework. [NR-287771](https://new-relic.atlassian.net/browse/NR-287771)
+- [PR-293](https://github.com/newrelic/csec-java-agent/pull/293), [PR-284](https://github.com/newrelic/csec-java-agent/pull/284), [PR-302](https://github.com/newrelic/csec-java-agent/pull/302) Detect route of an incoming request for mule server, play framework and Vertx Framework. [NR-283915](https://new-relic.atlassian.net/browse/NR-283915), [NR-265915](https://new-relic.atlassian.net/browse/NR-265915), [NR-287771](https://new-relic.atlassian.net/browse/NR-287771)
+
+### Changes
+- [PR-265](https://github.com/newrelic/csec-java-agent/pull/265) Improve Secure Cookie event reporting to provide detailed vulnerability. [NR-273609](https://new-relic.atlassian.net/browse/NR-273609)
+- [PR-283](https://github.com/newrelic/csec-java-agent/pull/283) Update IAST Header Parsing Minimum Expected Length Set to 8. [NR-282647](https://new-relic.atlassian.net/browse/NR-282647)
+- [PR-308](https://github.com/newrelic/csec-java-agent/pull/308) Remove jackson-dataformat-properties to address [CVE-2023-3894](https://www.cve.org/CVERecord?id=CVE-2023-3894) and exclude transitive dependency junit to address [CVE-2020-15250](https://www.cve.org/CVERecord?id=CVE-2020-15250) [NR-295033](https://new-relic.atlassian.net/browse/NR-295033)
+
+### Fixes
+- [PR-292](https://github.com/newrelic/csec-java-agent/pull/292) Fix for ClassNotFoundException observed in glassfish server [NR-262453](https://new-relic.atlassian.net/browse/NR-262453)
+- [PR-286](https://github.com/newrelic/csec-java-agent/pull/286) Detect correct user class in Netty Reactor Server [NR-253551](https://new-relic.atlassian.net/browse/NR-253551)
+- [PR-317](https://github.com/newrelic/csec-java-agent/pull/317) Add a workaround for an issue where New Relic Security Agent breaks the gRPC endpoints [#130](https://github.com/newrelic/csec-java-agent/issues/310). [NR-299709](https://new-relic.atlassian.net/browse/NR-299709)
+
+### Deprecations
+- Status File Used for Debugging: This feature has been deprecated. All debugging capabilities have been moved to either Init Logging or [Error Inbox](https://docs.newrelic.com/docs/errors-inbox/errors-inbox/) and will be removed in a future agent release. [NR-293966](https://new-relic.atlassian.net/browse/NR-293966)
+
 ## [1.4.0] - 2024-6-24
 ### Changes
 - Json Version bump to 1.2.3 due to [NR-254157](https://new-relic.atlassian.net/browse/NR-254157) implementation.
@@ -16,8 +132,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [PR-256](https://github.com/newrelic/csec-java-agent/pull/256), [PR-259](https://github.com/newrelic/csec-java-agent/pull/259), [PR-258](https://github.com/newrelic/csec-java-agent/pull/258) Feature to detect route of an incoming request for Jax-RS and Spring Framework. [NR-265913](https://new-relic.atlassian.net/browse/NR-265913), [NR-261653](https://new-relic.atlassian.net/browse/NR-261653), [NR-273605](https://new-relic.atlassian.net/browse/NR-273605)
 - [PR-126](https://github.com/newrelic/csec-java-agent/pull/126), [PR-127](https://github.com/newrelic/csec-java-agent/pull/127), [PR-128](https://github.com/newrelic/csec-java-agent/pull/128), [PR-129](https://github.com/newrelic/csec-java-agent/pull/129) Jedis Support : The security agent now also supports Jedis Version 1.4.0 and above. [NR-174176](https://new-relic.atlassian.net/browse/NR-174176)
 - [PR-287](https://github.com/newrelic/csec-java-agent/pull/287) Support for Proxy Settings for Connecting to the Security Engine, with known limitation of missing Authentication capabilities.
+
 ### Fixes
-- [PR-255](https://github.com/newrelic/csec-java-agent/pull/255) Handle InvalidPathException thrown by Paths.get method [NR-262452](https://new-relic.atlassian.net/browse/)
+- [PR-255](https://github.com/newrelic/csec-java-agent/pull/255) Handle InvalidPathException thrown by Paths.get method [NR-262452](https://new-relic.atlassian.net/browse/NR-262452)
 - [PR-216](https://github.com/newrelic/csec-java-agent/pull/216) Extract Server Configuration to resolve IAST localhost connection with application for Glassfish Server. [NR-223808](https://new-relic.atlassian.net/browse/NR-223808)
 - [PR-214](https://github.com/newrelic/csec-java-agent/pull/214) Extract Server Configuration to resolve IAST localhost connection with application for Weblogic Server. [NR-223809](https://new-relic.atlassian.net/browse/NR-223809)
 - [PR-242](https://github.com/newrelic/csec-java-agent/pull/242) Fix for User Class detection in Play Framework [NR-264101](https://new-relic.atlassian.net/browse/NR-264101)

README.md

@@ -33,7 +33,7 @@ The agent automatically instruments the following frameworks.
 - Resin 3.1.9 to 4.0.x
 - Jetty 9.3.0.M1 to latest
 - Mule ESB 3.6 to 3.9.x
-- gRPC 1.4.0 to latest**
+- gRPC 1.4.0 to latest [**](#grpc-instrumentation)
 - Jersey 2.0 to latest
 - Akka Server 10.0 to latest (with scala 2.11 and above)
 - Spray Can 1.3.1 to latest (with scala 2.11 and above)
@@ -42,8 +42,18 @@ The agent automatically instruments the following frameworks.
 - Netty Server 4.0.0.Final to latest
 - Netty Reactor Server 0.7.0.RELEASE to latest
 - Vertx web 3.2.0 to latest
-
-** IAST for **gRPC** requires the dependency [protobuf-java-util](https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java-util) for IAST request replay.
+- GraphQL 16.0.0 to latest [**](#graphql-instrumentation)
+
+#### gRPC Instrumentation
+IAST for **gRPC** requires the dependency [protobuf-java-util](https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java-util) for IAST request replay.
+
+#### GraphQL Instrumentation
+By default, GraphQL instrumentation is disabled in IAST as it is an experimental feature. To take advantage of this feature enable GraphQL instrumentation, update your configuration by adding the following settings under the class_transformer section:
+```yaml
+class_transformer:
+  com.newrelic.instrumentation.security.graphql-java-16.2:
+    enabled: true
+```
 
 ### Java Native Operations
 

buildSrc/src/main/java/com/nr/builder/publish/PublishConfig.java

@@ -75,7 +75,7 @@ private static void configureRepo(MavenArtifactRepository repo, String projectVe
     private static void configurePom(MavenPom pom) {
         pom.getUrl().set("https://github.com/newrelic/csec-java-agent");
         pom.licenses(spec -> spec.license(license -> {
-            license.getName().set("New Relic Pre-Release Software License");
+            license.getName().set("New Relic Software License, Version 1.0");
             license.getUrl().set("https://github.com/newrelic/csec-java-agent/blob/main/LICENSE.md");
             license.getDistribution().set("repo");
         }));

gradle.properties

@@ -1,6 +1,6 @@
 # The agent version.
-agentVersion=1.4.0
-jsonVersion=1.2.3
+agentVersion=1.5.0
+jsonVersion=1.2.9
 # Updated exposed NR APM API version.
 nrAPIVersion=8.12.0
 

gradle/script/java.gradle

@@ -264,7 +264,7 @@ test {
 }
 
 dependencies {
-    testImplementation("junit:junit:4.12")
+    testImplementation("junit:junit:4.13.2")
     testImplementation("org.mockito:mockito-core:3.9.0")
     testImplementation("org.hamcrest:hamcrest-library:1.3")
     testImplementation(project(":test-annotations"))

instrumentation-security-test/build.gradle

@@ -14,13 +14,15 @@ dependencies {
     implementation("com.newrelic.agent.java:newrelic-weaver-api:${nrAPIVersion}")
     implementation("com.newrelic.agent.java:newrelic-api:${nrAPIVersion}")
     implementation("com.newrelic.agent.java:newrelic-agent:${nrAgentVersion}")
-    implementation 'org.apache.commons:commons-text:1.7'
+    implementation ('org.apache.commons:commons-text:1.10.0')
     implementation("com.newrelic.agent.java:agent-bridge:${nrAPIVersion}")
     implementation("com.newrelic.agent.java:agent-bridge-datastore:${nrAPIVersion}")
     implementation("commons-net:commons-net:3.9.0")
     implementation("org.mockftpserver:MockFtpServer:3.1.0")
 
-    api("org.apache.httpcomponents:httpclient:4.5.13")
+    api("org.apache.httpcomponents:httpclient:4.5.13"){
+        exclude(module: 'commons-codec', group: 'commons-codec')
+    }
     api("org.nanohttpd:nanohttpd:2.3.1")
 }
 

instrumentation-security/akka-http-2.11_10.0.0/src/main/scala/akka/http/scaladsl/server/AkkaCoreUtils.java

@@ -60,6 +60,9 @@ public static boolean acquireServletLockIfPossible() {
     }
 
     public static void postProcessHttpRequest(Boolean isServletLockAcquired, StringBuilder responseBody, String contentType, String className, String methodName, Token token) {
+        if(NewRelicSecurity.getAgent().getIastDetectionCategory().getRxssEnabled()){
+            return;
+        }
         try {
             token.linkAndExpire();
             ServletHelper.executeBeforeExitingTransaction();

instrumentation-security/akka-http-2.11_10.0.0/src/main/scala/akka/http/scaladsl/server/CsecAkkaHttpContextFunction.scala

@@ -8,6 +8,7 @@
 package akka.http.scaladsl.server
 
 import akka.Done
+import akka.http.scaladsl.model.HttpEntity
 import akka.stream.javadsl.Source
 import akka.stream.scaladsl.Sink
 import akka.util.ByteString
@@ -55,15 +56,17 @@ class CsecContextWrapper(original: Function1[RequestContext, Future[RouteResult]
   override def apply(ctx: RequestContext): Future[RouteResult] = {
     try {
 
-      var httpRequest = ctx.request;
+      val httpRequest = ctx.request;
       val body: lang.StringBuilder = new lang.StringBuilder();
       val dataBytes: Source[ByteString, AnyRef] = httpRequest.entity.getDataBytes()
       val isLockAquired = AkkaCoreUtils.acquireServletLockIfPossible();
-      val sink: Sink[ByteString, Future[Done]] = Sink.foreach[ByteString] { byteString =>
-        val chunk = byteString.utf8String
-        body.append(chunk)
+      if (!httpRequest.entity.isInstanceOf[HttpEntity.Chunked]) {
+        val sink: Sink[ByteString, Future[Done]] = Sink.foreach[ByteString] { byteString =>
+          val chunk = byteString.utf8String
+          body.append(chunk)
+        }
+        val processingResult: Future[Done] = dataBytes.runWith(sink, ctx.materializer)
       }
-      val processingResult: Future[Done] = dataBytes.runWith(sink, ctx.materializer)
       AkkaCoreUtils.preProcessHttpRequest(isLockAquired, httpRequest, body, NewRelic.getAgent.getTransaction.getToken);
       original.apply(ctx)
     } catch {

instrumentation-security/akka-http-core-10.0/src/main/scala/akka/http/scaladsl/AkkaAsyncRequestHandler.scala

@@ -8,7 +8,7 @@
 package akka.http.scaladsl
 
 import akka.Done
-import akka.http.scaladsl.model.{HttpRequest, HttpResponse}
+import akka.http.scaladsl.model.{HttpEntity, HttpRequest, HttpResponse}
 import akka.stream.Materializer
 import akka.stream.javadsl.Source
 import akka.stream.scaladsl.Sink
@@ -26,11 +26,14 @@ class AkkaAsyncRequestHandler(handler: HttpRequest ⇒ Future[HttpResponse])(imp
     val body: lang.StringBuilder = new lang.StringBuilder();
     val dataBytes: Source[ByteString, AnyRef] = param.entity.getDataBytes()
     val isLockAquired = AkkaCoreUtils.acquireServletLockIfPossible();
-    val sink: Sink[ByteString, Future[Done]] = Sink.foreach[ByteString] { byteString =>
-      val chunk = byteString.utf8String
-      body.append(chunk)
+    if (!param.entity.isInstanceOf[HttpEntity.Chunked]) {
+      val sink: Sink[ByteString, Future[Done]] = Sink.foreach[ByteString] { byteString =>
+        val chunk = byteString.utf8String
+        body.append(chunk)
+      }
+      val processingResult: Future[Done] = dataBytes.runWith(sink, materializer)
     }
-    val processingResult: Future[Done] = dataBytes.runWith(sink, materializer)
+
     AkkaCoreUtils.preProcessHttpRequest(isLockAquired, param, body, NewRelic.getAgent.getTransaction.getToken);
     val futureResponse: Future[HttpResponse] = handler.apply(param)
     futureResponse.flatMap(ResponseFutureHelper.wrapResponseAsync(NewRelic.getAgent.getTransaction.getToken, materializer))