From dc2e9b74c203f41f8c3e0011e8388fc44f773417 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 15:13:47 +0100 Subject: [PATCH 1/3] [frontend] Update dependency globals to v15.13.0 (#2046) --- openbas-front/package.json | 2 +- openbas-front/yarn.lock | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/openbas-front/package.json b/openbas-front/package.json index f649a3efc6..b71cf564ac 100644 --- a/openbas-front/package.json +++ b/openbas-front/package.json @@ -113,7 +113,7 @@ "eslint-plugin-simple-import-sort": "12.1.1", "express": "4.21.1", "fs-extra": "11.2.0", - "globals": "15.12.0", + "globals": "15.13.0", "jsdom": "25.0.1", "monocart-coverage-reports": "2.11.3", "monocart-reporter": "2.9.11", diff --git a/openbas-front/yarn.lock b/openbas-front/yarn.lock index 4c84f9a938..c93ab129e6 100644 --- a/openbas-front/yarn.lock +++ b/openbas-front/yarn.lock @@ -6220,10 +6220,10 @@ __metadata: languageName: node linkType: hard -"globals@npm:15.12.0": - version: 15.12.0 - resolution: "globals@npm:15.12.0" - checksum: 10c0/f34e0a1845b694f45188331742af9f488b07ba7440a06e9d2039fce0386fbbfc24afdbb9846ebdccd4092d03644e43081c49eb27b30f4b88e43af156e1c1dc34 +"globals@npm:15.13.0": + version: 15.13.0 + resolution: "globals@npm:15.13.0" + checksum: 10c0/640365115ca5f81d91e6a7667f4935021705e61a1a5a76a6ec5c3a5cdf6e53f165af7f9db59b7deb65cf2e1f83d03ac8d6660d0b14c569c831a9b6483eeef585 languageName: node linkType: hard @@ -8971,7 +8971,7 @@ __metadata: final-form: "npm:4.20.10" final-form-arrays: "npm:3.1.0" fs-extra: "npm:11.2.0" - globals: "npm:15.12.0" + globals: "npm:15.13.0" history: "npm:5.3.0" html-react-parser: "npm:5.2.0" html-to-image: "npm:1.11.11" From fa353426ef64ed4217a0ca5cbdb0483e9aa153cc Mon Sep 17 00:00:00 2001 From: Hedi Date: Tue, 17 Dec 2024 11:36:47 +0100 Subject: [PATCH 2/3] [backend] Fix for the bug breaking openbas when running multiple agent (#2023) --- .../service/CalderaExecutorService.java | 116 +++++++------- .../service/CalderaExecutorServiceTest.java | 150 ++++++++++++++++++ 2 files changed, 211 insertions(+), 55 deletions(-) create mode 100644 openbas-api/src/test/java/io/openbas/executors/caldera/service/CalderaExecutorServiceTest.java diff --git a/openbas-api/src/main/java/io/openbas/executors/caldera/service/CalderaExecutorService.java b/openbas-api/src/main/java/io/openbas/executors/caldera/service/CalderaExecutorService.java index 24dcd57374..19b55a9b31 100644 --- a/openbas-api/src/main/java/io/openbas/executors/caldera/service/CalderaExecutorService.java +++ b/openbas-api/src/main/java/io/openbas/executors/caldera/service/CalderaExecutorService.java @@ -3,6 +3,7 @@ import static java.time.Instant.now; import static java.time.ZoneOffset.UTC; +import com.cronutils.utils.VisibleForTesting; import io.openbas.asset.EndpointService; import io.openbas.database.model.*; import io.openbas.executors.caldera.client.CalderaExecutorClient; @@ -112,30 +113,21 @@ public void run() { this.client.agents().stream() .filter(agent -> !agent.getExe_name().contains("implant")) .toList(); - List endpoints = toEndpoint(agents).stream().filter(Asset::getActive).toList(); - log.info("Caldera executor provisioning based on " + endpoints.size() + " assets"); - endpoints.forEach( - endpoint -> { - List existingEndpoints = - this.endpointService - .findAssetsForInjectionByHostname(endpoint.getHostname()) - .stream() - .filter( - endpoint1 -> - Arrays.stream(endpoint1.getIps()) - .anyMatch( - s -> Arrays.stream(endpoint.getIps()).toList().contains(s))) - .toList(); - if (existingEndpoints.isEmpty()) { + log.info("Caldera executor provisioning based on " + agents.size() + " assets"); + agents.forEach( + agent -> { + Optional existingEndpoint = findExistingEndpointForAnAgent(agent); + + if (existingEndpoint.isEmpty()) { Optional endpointByExternalReference = - endpointService.findByExternalReference(endpoint.getExternalReference()); + endpointService.findByExternalReference(agent.getPaw()); if (endpointByExternalReference.isPresent()) { - this.updateEndpoint(endpoint, List.of(endpointByExternalReference.get())); + this.updateEndpoint(agent, endpointByExternalReference.get()); } else { - this.endpointService.createEndpoint(endpoint); + this.endpointService.createEndpoint(toEndpoint(agent)); } } else { - this.updateEndpoint(endpoint, existingEndpoints); + this.updateEndpoint(agent, existingEndpoint.get()); } }); List inactiveEndpoints = @@ -162,63 +154,77 @@ public void run() { // -- PRIVATE -- + @VisibleForTesting + protected Optional findExistingEndpointForAnAgent(@NotNull final Agent agent) { + return this.endpointService.findAssetsForInjectionByHostname(agent.getHost()).stream() + .filter( + endpoint -> + Arrays.stream(endpoint.getIps()) + .anyMatch(Arrays.asList(agent.getHost_ip_addrs())::contains) + && endpoint.getExecutor() != null + && CALDERA_EXECUTOR_TYPE.equals(endpoint.getExecutor().getType())) + .findFirst(); + } + + private Endpoint toEndpoint(@NotNull final Agent agent) { + Endpoint endpoint = new Endpoint(); + endpoint.setExecutor(this.executor); + endpoint.setExternalReference(agent.getPaw()); + endpoint.setName(agent.getHost()); + endpoint.setDescription("Asset collected by Caldera executor context."); + endpoint.setIps(agent.getHost_ip_addrs()); + endpoint.setHostname(agent.getHost()); + endpoint.setPlatform(toPlatform(agent.getPlatform())); + endpoint.setArch(toArch(agent.getArchitecture())); + endpoint.setProcessName(agent.getExe_name()); + endpoint.setLastSeen(toInstant(agent.getLast_seen())); + return endpoint; + } + private List toEndpoint(@NotNull final List agents) { - return agents.stream() - .map( - (agent) -> { - Endpoint endpoint = new Endpoint(); - endpoint.setExecutor(this.executor); - endpoint.setExternalReference(agent.getPaw()); - endpoint.setName(agent.getHost()); - endpoint.setDescription("Asset collected by Caldera executor context."); - endpoint.setIps(agent.getHost_ip_addrs()); - endpoint.setHostname(agent.getHost()); - endpoint.setPlatform(toPlatform(agent.getPlatform())); - endpoint.setArch(toArch(agent.getArchitecture())); - endpoint.setProcessName(agent.getExe_name()); - endpoint.setLastSeen(toInstant(agent.getLast_seen())); - return endpoint; - }) - .toList(); + return agents.stream().map(this::toEndpoint).toList(); } private void updateEndpoint( - @NotNull final Endpoint external, @NotNull final List existingList) { - Endpoint matchingExistingEndpoint = existingList.getFirst(); - matchingExistingEndpoint.setLastSeen(external.getLastSeen()); - matchingExistingEndpoint.setExternalReference(external.getExternalReference()); - matchingExistingEndpoint.setName(external.getName()); - matchingExistingEndpoint.setIps(external.getIps()); - matchingExistingEndpoint.setHostname(external.getHostname()); - matchingExistingEndpoint.setProcessName(external.getProcessName()); - matchingExistingEndpoint.setPlatform(external.getPlatform()); - matchingExistingEndpoint.setArch(external.getArch()); - matchingExistingEndpoint.setExecutor(this.executor); - if ((now().toEpochMilli() - matchingExistingEndpoint.getClearedAt().toEpochMilli()) - > CLEAR_TTL) { + @NotNull final Agent agent, @NotNull final Endpoint existingEndpoint) { + existingEndpoint.setLastSeen(toInstant(agent.getLast_seen())); + existingEndpoint.setExternalReference(agent.getPaw()); + existingEndpoint.setName(agent.getHost()); + existingEndpoint.setIps(agent.getHost_ip_addrs()); + existingEndpoint.setHostname(agent.getHost()); + existingEndpoint.setProcessName(agent.getExe_name()); + existingEndpoint.setPlatform(toPlatform(agent.getPlatform())); + existingEndpoint.setArch(toArch(agent.getArchitecture())); + existingEndpoint.setExecutor(this.executor); + if ((now().toEpochMilli() - existingEndpoint.getClearedAt().toEpochMilli()) > CLEAR_TTL) { try { - log.info("Clearing endpoint " + matchingExistingEndpoint.getHostname()); + log.info("Clearing endpoint " + existingEndpoint.getHostname()); Iterable injectors = injectorService.injectors(); injectors.forEach( injector -> { if (injector.getExecutorClearCommands() != null) { - this.calderaExecutorContextService.launchExecutorClear( - injector, matchingExistingEndpoint); + this.calderaExecutorContextService.launchExecutorClear(injector, existingEndpoint); } }); - matchingExistingEndpoint.setClearedAt(now()); + existingEndpoint.setClearedAt(now()); } catch (RuntimeException e) { log.info("Failed clear agents"); } } - this.endpointService.updateEndpoint(matchingExistingEndpoint); + this.endpointService.updateEndpoint(existingEndpoint); } - private Instant toInstant(@NotNull final String lastSeen) { + @VisibleForTesting + protected Instant toInstant(@NotNull final String lastSeen) { String pattern = "yyyy-MM-dd'T'HH:mm:ss'Z'"; DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(pattern, Locale.getDefault()); LocalDateTime localDateTime = LocalDateTime.parse(lastSeen, dateTimeFormatter); ZonedDateTime zonedDateTime = localDateTime.atZone(UTC); return zonedDateTime.toInstant(); } + + @VisibleForTesting + protected void setExecutor(Executor executor) { + this.executor = executor; + } } diff --git a/openbas-api/src/test/java/io/openbas/executors/caldera/service/CalderaExecutorServiceTest.java b/openbas-api/src/test/java/io/openbas/executors/caldera/service/CalderaExecutorServiceTest.java new file mode 100644 index 0000000000..fb7a1a7a71 --- /dev/null +++ b/openbas-api/src/test/java/io/openbas/executors/caldera/service/CalderaExecutorServiceTest.java @@ -0,0 +1,150 @@ +package io.openbas.executors.caldera.service; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.openbas.asset.EndpointService; +import io.openbas.database.model.Endpoint; +import io.openbas.database.model.Executor; +import io.openbas.executors.caldera.client.CalderaExecutorClient; +import io.openbas.executors.caldera.config.CalderaExecutorConfig; +import io.openbas.executors.caldera.model.Agent; +import io.openbas.integrations.ExecutorService; +import io.openbas.integrations.InjectorService; +import io.openbas.service.PlatformSettingsService; +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class CalderaExecutorServiceTest { + private static final String CALDERA_AGENT_HOSTNAME = "calderaHostname"; + private static final String CALDERA_AGENT_EXTERNAL_REF = "calderaExt"; + private static final String CALDERA_AGENT_IP = "10.10.10.10"; + + private static final String CALDERA_EXECUTOR_TYPE = "openbas_caldera"; + private static final String CALDERA_EXECUTOR_NAME = "Caldera"; + + private static final String DATE = "2024-12-11T12:45:30Z"; + + @Mock private ExecutorService executorService; + + @Mock private CalderaExecutorClient client; + + @Mock private CalderaExecutorConfig config; + + @Mock private CalderaExecutorContextService calderaExecutorContextService; + + @Mock private EndpointService endpointService; + + @Mock private InjectorService injectorService; + + @Mock private PlatformSettingsService platformSettingsService; + + @Mock private Executor executor; + + @InjectMocks private CalderaExecutorService calderaExecutorService; + + private Endpoint calderaEndpoint; + private Endpoint randomEndpoint; + private Agent calderaAgent; + private Agent randomAgent; + private Executor calderaExecutor; + private Executor randomExecutor; + + @BeforeEach + void setUp() { + calderaAgent = new Agent(); + calderaAgent.setArchitecture("Arch"); + calderaAgent.setPaw(CALDERA_AGENT_EXTERNAL_REF); + calderaExecutor = new Executor(); + calderaExecutor.setName(CALDERA_EXECUTOR_NAME); + calderaExecutor.setType(CALDERA_EXECUTOR_TYPE); + randomExecutor = new Executor(); + randomExecutor.setName("NAME"); + randomExecutor.setType("TYPE"); + calderaExecutorService.setExecutor(calderaExecutor); + + calderaAgent = + createAgent(CALDERA_AGENT_HOSTNAME, CALDERA_AGENT_IP, CALDERA_AGENT_EXTERNAL_REF); + randomAgent = createAgent("hostname", "1.1.1.1", "ref"); + calderaEndpoint = createEndpoint(calderaAgent, calderaExecutor); + randomEndpoint = createEndpoint(randomAgent, randomExecutor); + + when(endpointService.findAssetsForInjectionByHostname(CALDERA_AGENT_HOSTNAME)) + .thenReturn(List.of(calderaEndpoint, randomEndpoint)); + } + + private Endpoint createEndpoint(Agent agent, Executor executor) { + Endpoint endpoint = new Endpoint(); + endpoint.setExecutor(executor); + endpoint.setExternalReference(agent.getPaw()); + endpoint.setName(agent.getHost()); + endpoint.setDescription("Asset collected by Caldera executor context."); + endpoint.setIps(agent.getHost_ip_addrs()); + endpoint.setHostname(agent.getHost()); + endpoint.setPlatform(CalderaExecutorService.toPlatform("windows")); + endpoint.setArch(CalderaExecutorService.toArch("amd64")); + endpoint.setProcessName(agent.getExe_name()); + endpoint.setLastSeen(calderaExecutorService.toInstant(DATE)); + return endpoint; + } + + private Agent createAgent(String hostname, String ip, String externalRef) { + Agent agent = new Agent(); + agent.setArchitecture("amd64"); + agent.setPaw(externalRef); + agent.setPlatform("windows"); + agent.setExe_name("exe"); + agent.setLast_seen(DATE); + agent.setHost_ip_addrs(new String[] {ip}); + agent.setHost(hostname); + return agent; + } + + @Test + void test_run_WITH_one_endpoint() throws Exception { + when(client.agents()).thenReturn(List.of(calderaAgent)); + calderaExecutorService.run(); + verify(endpointService).updateEndpoint(calderaEndpoint); + } + + @Test + void test_run_WITH_2_existing_endpoint_same_machine() throws Exception { + when(client.agents()).thenReturn(List.of(calderaAgent)); + randomEndpoint.setHostname(CALDERA_AGENT_HOSTNAME); + randomEndpoint.setIps(new String[] {CALDERA_AGENT_IP}); + calderaExecutorService.run(); + verify(endpointService).updateEndpoint(calderaEndpoint); + } + + @Test + void test_findExistingEndpointForAnAgent_WITH_2_existing_endpoint_same_host() throws Exception { + Optional result = calderaExecutorService.findExistingEndpointForAnAgent(calderaAgent); + assertEquals(calderaEndpoint, result.get()); + } + + @Test + void test_findExistingEndpointForAnAgent_WITH_no_existing_endpoint() throws Exception { + when(endpointService.findAssetsForInjectionByHostname(CALDERA_AGENT_HOSTNAME)) + .thenReturn(List.of()); + Optional result = calderaExecutorService.findExistingEndpointForAnAgent(calderaAgent); + assertTrue(result.isEmpty()); + } + + @Test + void test_findExistingEndpointForAnAgent_WITH_1_enpdoint_with_null_executor() throws Exception { + randomEndpoint.setExecutor(null); + randomEndpoint.setHostname(CALDERA_AGENT_HOSTNAME); + randomEndpoint.setIps(new String[] {CALDERA_AGENT_IP}); + Optional result = calderaExecutorService.findExistingEndpointForAnAgent(calderaAgent); + assertEquals(calderaEndpoint, result.get()); + } +} From fece7c255240c91a15a6854d11f837e721508eb9 Mon Sep 17 00:00:00 2001 From: Guillaume Date: Tue, 17 Dec 2024 20:17:08 +0100 Subject: [PATCH 3/3] [backend] fix finished exercises query by excluding disabled injects (#2091) --- .../java/io/openbas/database/repository/ExerciseRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbas-model/src/main/java/io/openbas/database/repository/ExerciseRepository.java b/openbas-model/src/main/java/io/openbas/database/repository/ExerciseRepository.java index b2be2d5836..2014ab9288 100644 --- a/openbas-model/src/main/java/io/openbas/database/repository/ExerciseRepository.java +++ b/openbas-model/src/main/java/io/openbas/database/repository/ExerciseRepository.java @@ -82,7 +82,7 @@ List userCountGroupByWeek( @Query( value = "select e.*, se.scenario_id from exercises e " - + "left join injects as inject on e.exercise_id = inject.inject_exercise " + + "left join injects as inject on e.exercise_id = inject.inject_exercise and inject.inject_enabled = 'true' " + "left join injects_statuses as status on inject.inject_id = status.status_inject and status.status_name != 'PENDING'" + "left join scenarios_exercises as se on e.exercise_id = se.exercise_id " + "where e.exercise_status = 'RUNNING' group by e.exercise_id, se.scenario_id having count(status) = count(inject);",