Template healthcheck + integration test improvements and fixes #3616

gradle/build-java.gradle

@@ -76,10 +76,6 @@ subprojects{
     }
 
     test {
-        filter {
-             excludeTestsMatching "*ManualTest"
-             excludeTestsMatching "*IntTest"
-        }
 
 	    /* Per default GRADLE stops the build if one single test fails. We want to have all tests executed. */
         ignoreFailures = true

sechub-api-java/fullRegenerateOpenAPIClassFiles.sh

@@ -11,7 +11,6 @@
 # again with necessary system properties defined!
 set -e
 
-
 echo "-----------------------------------"
 echo "Regenerate OpenAPI class files"
 echo "-----------------------------------"
@@ -29,5 +28,8 @@ echo "- generate open api file"
 echo "- generate sechub java api"
 
 cd ..
-./gradlew spotlessApply generateOpenapi 
+echo ">> Generate OpenAPI (sechub-openapi-java-client)"
+./gradlew spotlessApply generateOpenapi
+
+echo ">> Generate OpenAPI (sechub-api-java) [DEPRECATED]"
 ./gradlew :sechub-api-java:build  -Dsechub.build.stage=all 

sechub-developertools/src/main/java/com/mercedesbenz/sechub/developertools/admin/ui/action/integrationtestserver/IntegrationTestAction.java

@@ -5,7 +5,7 @@
 
 import com.mercedesbenz.sechub.developertools.admin.ui.UIContext;
 import com.mercedesbenz.sechub.developertools.admin.ui.action.AbstractUIAction;
-import com.mercedesbenz.sechub.integrationtest.api.IntegrationTestSetup;
+import com.mercedesbenz.sechub.integrationtest.api.IntegrationTestSupport;
 import com.mercedesbenz.sechub.integrationtest.internal.IntegrationTestContext;
 
 /**
@@ -46,7 +46,7 @@ private boolean checkIntegrationTestServerRunning() {
         integrationTestContext.setPort(getContext().getPort());
 
         String isAliveURL = integrationTestContext.getUrlBuilder().buildIntegrationTestIsAliveUrl();
-        if (!Boolean.TRUE.equals(IntegrationTestSetup.fetchTestServerStatus(isAliveURL))) {
+        if (!Boolean.TRUE.equals(IntegrationTestSupport.fetchTestServerStatus(isAliveURL))) {
             warn("You are not running an integration test server, so cannot exeucte action!");
             return false;
         }

sechub-doc/openapi-workaround.gradle

@@ -31,6 +31,9 @@ stringType.put("type", "string")
    
    Remark: if you still find any `oneOf` elements inside the `openApi3.json` file you have
    to fix another one this inside this gradle file!
+   
+   Remark 2025-01-30, de-jcup: This ugly workaround gradle file will be removed completely
+                               when we switch to use sechub-openapi-java-client only!!!
 */
 void postProcessOpenApiGeneration(){
 

sechub-doc/src/docs/asciidoc/documents/techdoc/01_development.adoc

@@ -121,13 +121,19 @@ with
 -Dsechub.integrationtest.running=true
 ----
 
-Some integration tests are long running. These tests are only run when you set
-----
--Dsechub.integrationtest.longrunning=true
-----
-
 So you can easily switch or enable all by enabling dedicated system property.
 
+[TIP]
+====
+To make things easier there is the possibility to define this property also
+inside file:
+`~/.sechub/sechub-developer.properties`
+
+This file is used inside integration tests automatically.
+If you set there a line like
+`sechub.integrationtest.running=true`
+====
+
 ====== Console
 Next line starts integration test server, execute tests and automatically stops
 server after done or failed.

sechub-doc/src/docs/asciidoc/documents/techdoc/03_coding_conventions.adoc

@@ -394,15 +394,54 @@ TIP: For an example look into `JobRepositoryDBTest`
 ===== Integration tests
 In project `sechub-integrationtest` full integration tests are settled.
 The project needs a running server in profile `integrationtest`.
-For more details about those tests read the `README.md` file inside
+
+====== Example for an integration test
+[source, java]
+----
+import static com.mercedesbenz.sechub.integrationtest.api.TestAPI.*;
+
+import org.junit.jupiter.api.Test;
+import com.mercedesbenz.sechub.integrationtest.api.IntegrationTestExtension;
+import com.mercedesbenz.sechub.integrationtest.api.WithTestScenario;
+
+
+@ExtendWith(IntegrationTestExtension.class)//<1>
+@WithTestScenario(Scenario6.class)//<2>
+public class ExampleIntTest {
+
+	@Test
+	void example_doing_some_test_stuff(){
+	   // now use the TestAPI methods to test your wanted stuff
+	}
+}
+----
+<1> Annotation used to mark this test as a {sechub} Integration Test
+<2> Annotation used to select the scenario to use inside the integration test.
+    (Scenarios are predefined and use an automated setup with test data which
+    will be always cleaned up and provided automatically)
+
+[TIP]
+====
+For more details about those tests inspect existing tests. You can also read the `README.md` file inside
 projects root folder.
+====
+
+[IMPORTANT]
+====
+There are old existing integration tests which are not using the Junit5 extension `IntegrationTestExtension`
+but the old Junit4 rule `IntegrationTestSetup`. Those old junit4 tests shall be replaced step by step (in future with
+the new Junit5 way.
+
+**New integration tests shall always use the new Junit5 annotations.**
+New assert methods created for TestAPI shall use Junit5 only as well.
+
+====
+
 
-====== Integration tests using RestAPI
 These tests will be called `${name}Scenario${n}IntTest.java`
 
 ====== Integration tests using SecHubClient
 Some integration tests do need a build SecHub client and execute the client.
-These tests will be called `${name}Scenario${n}SecHubClientIntTest.java`
 
 TIP: If these tests are failing, please check you have called `gradlew buildGo` before,
      otherwise no {sechub} client is available for testing...

sechub-doc/src/docs/asciidoc/sechub-developer-quickstart-guide.adoc

@@ -559,8 +559,8 @@ Other SecHub server integration tests can be run in the same way.
 +
 image::eclipse-run-server-integrationtest.png[]
 
-. Your first run will fail, as the run needs to be configured with the variables `-Dsechub.integrationtest.running=true` or `-Dsechub.integrationtest.longrunning=true`.
-Therefor open the run configurations and select the `UserRegistrationScenario1IntTest`.
+. Your first run will fail, as the run needs to be configured with the variables `-Dsechub.integrationtest.running=true`.
+Therefore open the run configurations and select the `UserRegistrationScenario1IntTest`.
 Click on the `Arguments` tab and paste the flag `-Dsechub.integrationtest.running=true` in the VM arguments field, click apply and run the test again.
 
 image::eclipse-add-VM-argument-to-UnitTest.png[]
@@ -690,8 +690,8 @@ Other SecHub server integration tests can be run in the same way.
 
 . Right-click on `UserRegistrationScenario1IntTest.java` to open the context menu and select `test`.
 
-. Your first run will fail, as the run needs to be configured with the variables `-Dsechub.integrationtest.running=true` or `-Dsechub.integrationtest.longrunning=true`.
-Therefor open the run configurations and select the `UserRegistrationScenario1IntTest`.
+. Your first run will fail, as the run needs to be configured with the variables `-Dsechub.integrationtest.running=true`.
+Therefore open the run configurations and select the `UserRegistrationScenario1IntTest`.
 Click on the `Add VM options` tab and paste the flag `-Dsechub.integrationtest.running=true` in the VM arguments field, click apply and run the test again.
 
 . Check the JUnit tab, the result should be green indicating a successful run.

sechub-doc/src/test/java/com/mercedesbenz/sechub/restdoc/TemplateRestControllerRestDocTest.java

@@ -14,6 +14,7 @@
 import java.lang.annotation.Annotation;
 import java.util.List;
 
+import org.assertj.core.util.Arrays;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -34,16 +35,21 @@
 import com.mercedesbenz.sechub.commons.model.template.TemplateDefinition.TemplateVariableValidation;
 import com.mercedesbenz.sechub.commons.model.template.TemplateType;
 import com.mercedesbenz.sechub.docgen.util.RestDocFactory;
+import com.mercedesbenz.sechub.domain.scan.template.TemplateHealthCheckProblemType;
 import com.mercedesbenz.sechub.domain.scan.template.TemplateRepository;
 import com.mercedesbenz.sechub.domain.scan.template.TemplateRestController;
 import com.mercedesbenz.sechub.domain.scan.template.TemplateService;
+import com.mercedesbenz.sechub.domain.scan.template.TemplatesHealthCheckResult;
+import com.mercedesbenz.sechub.domain.scan.template.TemplatesHealthCheckService;
+import com.mercedesbenz.sechub.domain.scan.template.TemplatesHealthCheckStatus;
 import com.mercedesbenz.sechub.sharedkernel.Profiles;
 import com.mercedesbenz.sechub.sharedkernel.logging.AuditLogService;
 import com.mercedesbenz.sechub.sharedkernel.logging.LogSanitizer;
 import com.mercedesbenz.sechub.sharedkernel.security.RoleConstants;
 import com.mercedesbenz.sechub.sharedkernel.usecases.UseCaseRestDoc;
 import com.mercedesbenz.sechub.sharedkernel.usecases.admin.config.UseCaseAdminCreatesOrUpdatesTemplate;
 import com.mercedesbenz.sechub.sharedkernel.usecases.admin.config.UseCaseAdminDeletesTemplate;
+import com.mercedesbenz.sechub.sharedkernel.usecases.admin.config.UseCaseAdminExecutesTemplatesHealthcheck;
 import com.mercedesbenz.sechub.sharedkernel.usecases.admin.config.UseCaseAdminFetchesAllTemplateIds;
 import com.mercedesbenz.sechub.sharedkernel.usecases.admin.config.UseCaseAdminFetchesTemplate;
 import com.mercedesbenz.sechub.test.ExampleConstants;
@@ -67,6 +73,9 @@ public class TemplateRestControllerRestDocTest implements TestIsNecessaryForDocu
     @MockBean
     private TemplateRepository templateRepository;
 
+    @MockBean
+    private TemplatesHealthCheckService templateHealthCheckService;
+
     @MockBean
     TemplateService templateService;
 
@@ -257,4 +266,80 @@ public void restdoc_admin_fetches_templatelist() throws Exception {
 
         /* @formatter:on */
     }
+
+    @Test
+    @UseCaseRestDoc(useCase = UseCaseAdminExecutesTemplatesHealthcheck.class)
+    public void restdoc_admin_executes_templates_healthcheck() throws Exception {
+
+        /* prepare */
+
+        String json = """
+                {
+                  "status" : "WARNING",
+                  "entries" : [ {
+                    "type" : "ERROR",
+                    "description" : "The file 'asset-1/pds-product1-id.zip' does not exist!",
+                    "templateId" : "template-1",
+                    "projects" : [ "project-1" ],
+                    "executorConfigUUID" : "349ea899-e780-4553-bd50-06c12fe96c9e",
+                    "profiles" : [ "profile-1" ],
+                    "hints" : [ "At least one combination of executor and profile is enabled.", "At least one executor config is enabled.", "At least one profile is enabled." ],
+                    "solution" : "Upload a file 'pds-product1-id.zip' to asset folder 'asset-1'",
+                    "assetId" : "asset-1",
+                    "fileName" : "pds-product1-id.zip"
+                  }, {
+                    "type" : "WARNING",
+                    "description" : "The file 'asset-1/pds-product2-id.zip' does not exist!",
+                    "templateId" : "template-1",
+                    "projects" : [ "project-2" ],
+                    "executorConfigUUID" : "2b25b007-f3d2-4591-ba42-409e19d9a5e8",
+                    "profiles" : [ "profile-2" ],
+                    "hints" : [ "At least one executor config is not enabled.", "At least one profile is enabled." ],
+                    "solution" : "Upload a file 'pds-product2-id.zip' to asset folder 'asset-1'",
+                    "assetId" : "asset-1",
+                    "fileName" : "pds-product2-id.zip"
+                  } ]
+                }
+                """;
+        TemplatesHealthCheckResult healthCheckResult = TemplatesHealthCheckResult.fromJson(json);
+
+        when(templateHealthCheckService.executeHealthCheck()).thenReturn(healthCheckResult);
+
+        String apiEndpoint = https(PORT_USED).buildAdminExecutesTemplatesCheck();
+        Class<? extends Annotation> useCase = UseCaseAdminExecutesTemplatesHealthcheck.class;
+
+        /* execute + test @formatter:off */
+        this.mockMvc.perform(
+                get(apiEndpoint).
+                contentType(MediaType.APPLICATION_JSON_VALUE).
+                header(TestAuthenticationHelper.HEADER_NAME, TestAuthenticationHelper.getHeaderValue())
+                ).
+        andExpect(status().isOk()).
+        andDo(defineRestService().
+                with().
+                useCaseData(useCase).
+                tag(RestDocFactory.extractTag(apiEndpoint)).
+                requestSchema(TestOpenApiSchema.TEMPLATES.getSchema()).
+                and().
+                document(
+                    responseFields(
+                        fieldWithPath("status").description("Represents overall healthcheck status. Can be one of : "+ Arrays.asList(TemplatesHealthCheckStatus.values())),
+                        fieldWithPath("entries").description("A list of healthcheck status entries. Each entry represents a problem or an information."),
+                        fieldWithPath("entries[].type").description("Type of this entry. Can be one of : "+ Arrays.asList(TemplateHealthCheckProblemType.values())),
+                        fieldWithPath("entries[].description").description("A description about the the entry"),
+                        fieldWithPath("entries[].templateId").description("The template id for the template where the problem/information is related to"),
+                        fieldWithPath("entries[].projects").description("A list of projects which have the template assigned"),
+                        fieldWithPath("entries[].executorConfigUUID").description("The uuid of the product executor config where the problem/info is related to (in combination with template)"),
+                        fieldWithPath("entries[].profiles").description("A list of the invovled profiles, means using executor config and being assigned to projects"),
+                        fieldWithPath("entries[].hints").description("A list of hints which gives additinal information. E.g. A disabled executor configuration which leads to problems will lead to a WARNING, but not to an ERROR. In this case a hint 'At least one executor config is not enabled' would be added."),
+                        fieldWithPath("entries[].solution").description("A solution how to fix/resolve the problem"),
+                        fieldWithPath("entries[].assetId").description("The asset identifier which is used to locate the file"),
+                        fieldWithPath("entries[].fileName").description("The name of the file inside the asset")
+                        )
+                    )
+               );
+
+        /* @formatter:on */
+    }
+
 }

sechub-integrationtest/build.gradle

@@ -28,6 +28,10 @@ dependencies {
     api library.junit4
 
     api library.apache_commons_lang3
+
+    implementation spring_boot_dependency.junit_jupiter
+    implementation spring_boot_dependency.mockito_core
+    implementation spring_boot_dependency.assertj_core
 }
 import java.nio.file.Files;
 import java.nio.file.Paths;
@@ -157,11 +161,18 @@ task stopIntegrationTestInstances(dependsOn: [stopIntegrationTestServer, stopInt
 
 }
 
+task cleanIntegrationTest(type: Delete) {
+  delete "${project.buildDir}/test-results", "${project.buildDir}/reports/tests", "${project.buildDir}/sechub-testreports"
+}
 
 /**
  * Own task for integration tests.
+
+ * We use cleanIntegrationTest task as dependency here - reason: When tests are executed before in CI/CD, the integration
+ * tests are skipped - which is okay. Unfortunately gradle will assume that there are no changes in tests
+ * (source has not changed, test was executed, so means still valid...). To avoid this we do the cleanTest
  */
-task integrationtest(type: Test, dependsOn: startIntegrationTestInstances) {
+task integrationtest(type: Test, dependsOn: [cleanIntegrationTest, startIntegrationTestInstances]) {
    group 'sechub'
    description 'Starts integration test server, execute tests and automatically stops server after done or failed'
    // integration tests seems to be very often  "up-to-date"
@@ -179,13 +190,21 @@ task integrationtest(type: Test, dependsOn: startIntegrationTestInstances) {
     * integration test properties here
     */
    systemProperty "sechub.integrationtest.running", "true"
-   systemProperty "sechub.integrationtest.longrunning", "true"
 
    finalizedBy "stopIntegrationTestInstances"
-
-   filter {
-         includeTestsMatching "*IntTest"
-   }
+
+   /*
+    * we do NOT something like 
+    * ``` 
+    *  filter {
+    *      includeTestsMatching "*IntTest"
+    *   }
+    * ```
+    * 
+    * Why? Because we need the cleanIntegrationTest task run (see documentation on top of task for reasons) and this
+    * will also cleanup the "normal" unit test results inside the gradle sub module "sechub-integration".
+    * Means if we would filter here, the tests would no longer exist.
+    */   
 
 }
 

sechub-integrationtest/src/main/java/com/mercedesbenz/sechub/integrationtest/api/AsUser.java

@@ -44,6 +44,7 @@
 import com.mercedesbenz.sechub.domain.administration.project.ProjectDetailInformation;
 import com.mercedesbenz.sechub.domain.scan.asset.AssetDetailData;
 import com.mercedesbenz.sechub.domain.scan.project.FalsePositiveProjectData;
+import com.mercedesbenz.sechub.domain.scan.template.TemplatesHealthCheckResult;
 import com.mercedesbenz.sechub.integrationtest.JSONTestSupport;
 import com.mercedesbenz.sechub.integrationtest.internal.IntegrationTestContext;
 import com.mercedesbenz.sechub.integrationtest.internal.IntegrationTestExampleConstants;
@@ -1480,6 +1481,12 @@ public List<String> fetchTemplateList() {
         return JSONConverter.get().fromJSONtoListOf(String.class, json);
     }
 
+    public TemplatesHealthCheckResult executeTemplatesHealthcheck() {
+        String url = getUrlBuilder().buildAdminExecutesTemplatesCheck();
+        String json = getRestHelper().getJSON(url);
+        return TemplatesHealthCheckResult.fromJson(json);
+    }
+
     public AsUser uploadAssetFile(String assetId, File file) {
         String url = getUrlBuilder().buildAdminUploadsAssetFile(assetId);
         String checkSum = TestAPI.createSHA256Of(file);