Skip to content

Commit

Permalink
Merge branch 'master' into bugfix/template-tech-impr
Browse files Browse the repository at this point in the history
  • Loading branch information
savacano28 authored Dec 18, 2024
2 parents 2ddf044 + e31e29d commit 66d8047
Show file tree
Hide file tree
Showing 32 changed files with 1,697 additions and 1,551 deletions.
59 changes: 42 additions & 17 deletions .circleci/config.yml

Large diffs are not rendered by default.

10 changes: 5 additions & 5 deletions openbas-api/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -6,23 +6,23 @@
<parent>
<groupId>io.openbas</groupId>
<artifactId>openbas-platform</artifactId>
<version>1.9.1</version>
<version>1.10.0</version>
</parent>

<artifactId>openbas-api</artifactId>
<name>OpenBAS API</name>
<description>OpenBAS api</description>

<properties>
<bcpg-jdk18on.version>1.78.1</bcpg-jdk18on.version>
<bcpg-jdk18on.version>1.79</bcpg-jdk18on.version>
<commons-collections4.version>4.4</commons-collections4.version>
<commons-email.version>1.5</commons-email.version>
<opensaml.version>4.3.2</opensaml.version>
<springdoc.version>2.6.0</springdoc.version>
<springdoc-plugin.version>1.4</springdoc-plugin.version>
<cron-utils.version>9.2.1</cron-utils.version>
<apache-poi.version>5.3.0</apache-poi.version>
<opentelemetry.version>1.44.1</opentelemetry.version>
<opentelemetry.version>1.45.0</opentelemetry.version>
<opentelemetry-semconv.version>1.27.0-alpha</opentelemetry-semconv.version>
</properties>

Expand All @@ -48,12 +48,12 @@
<dependency>
<groupId>io.openbas</groupId>
<artifactId>openbas-framework</artifactId>
<version>1.9.1</version>
<version>1.10.0</version>
</dependency>
<dependency>
<groupId>com.rabbitmq</groupId>
<artifactId>amqp-client</artifactId>
<version>5.22.0</version>
<version>5.24.0</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import static java.time.Instant.now;
import static java.time.ZoneOffset.UTC;

import com.cronutils.utils.VisibleForTesting;
import io.openbas.asset.EndpointService;
import io.openbas.database.model.*;
import io.openbas.executors.caldera.client.CalderaExecutorClient;
Expand Down Expand Up @@ -112,30 +113,21 @@ public void run() {
this.client.agents().stream()
.filter(agent -> !agent.getExe_name().contains("implant"))
.toList();
List<Endpoint> endpoints = toEndpoint(agents).stream().filter(Asset::getActive).toList();
log.info("Caldera executor provisioning based on " + endpoints.size() + " assets");
endpoints.forEach(
endpoint -> {
List<Endpoint> existingEndpoints =
this.endpointService
.findAssetsForInjectionByHostname(endpoint.getHostname())
.stream()
.filter(
endpoint1 ->
Arrays.stream(endpoint1.getIps())
.anyMatch(
s -> Arrays.stream(endpoint.getIps()).toList().contains(s)))
.toList();
if (existingEndpoints.isEmpty()) {
log.info("Caldera executor provisioning based on " + agents.size() + " assets");
agents.forEach(
agent -> {
Optional<Endpoint> existingEndpoint = findExistingEndpointForAnAgent(agent);

if (existingEndpoint.isEmpty()) {
Optional<Endpoint> endpointByExternalReference =
endpointService.findByExternalReference(endpoint.getExternalReference());
endpointService.findByExternalReference(agent.getPaw());
if (endpointByExternalReference.isPresent()) {
this.updateEndpoint(endpoint, List.of(endpointByExternalReference.get()));
this.updateEndpoint(agent, endpointByExternalReference.get());
} else {
this.endpointService.createEndpoint(endpoint);
this.endpointService.createEndpoint(toEndpoint(agent));
}
} else {
this.updateEndpoint(endpoint, existingEndpoints);
this.updateEndpoint(agent, existingEndpoint.get());
}
});
List<Endpoint> inactiveEndpoints =
Expand All @@ -162,63 +154,77 @@ public void run() {

// -- PRIVATE --

@VisibleForTesting
protected Optional<Endpoint> findExistingEndpointForAnAgent(@NotNull final Agent agent) {
return this.endpointService.findAssetsForInjectionByHostname(agent.getHost()).stream()
.filter(
endpoint ->
Arrays.stream(endpoint.getIps())
.anyMatch(Arrays.asList(agent.getHost_ip_addrs())::contains)
&& endpoint.getExecutor() != null
&& CALDERA_EXECUTOR_TYPE.equals(endpoint.getExecutor().getType()))
.findFirst();
}

private Endpoint toEndpoint(@NotNull final Agent agent) {
Endpoint endpoint = new Endpoint();
endpoint.setExecutor(this.executor);
endpoint.setExternalReference(agent.getPaw());
endpoint.setName(agent.getHost());
endpoint.setDescription("Asset collected by Caldera executor context.");
endpoint.setIps(agent.getHost_ip_addrs());
endpoint.setHostname(agent.getHost());
endpoint.setPlatform(toPlatform(agent.getPlatform()));
endpoint.setArch(toArch(agent.getArchitecture()));
endpoint.setProcessName(agent.getExe_name());
endpoint.setLastSeen(toInstant(agent.getLast_seen()));
return endpoint;
}

private List<Endpoint> toEndpoint(@NotNull final List<Agent> agents) {
return agents.stream()
.map(
(agent) -> {
Endpoint endpoint = new Endpoint();
endpoint.setExecutor(this.executor);
endpoint.setExternalReference(agent.getPaw());
endpoint.setName(agent.getHost());
endpoint.setDescription("Asset collected by Caldera executor context.");
endpoint.setIps(agent.getHost_ip_addrs());
endpoint.setHostname(agent.getHost());
endpoint.setPlatform(toPlatform(agent.getPlatform()));
endpoint.setArch(toArch(agent.getArchitecture()));
endpoint.setProcessName(agent.getExe_name());
endpoint.setLastSeen(toInstant(agent.getLast_seen()));
return endpoint;
})
.toList();
return agents.stream().map(this::toEndpoint).toList();
}

private void updateEndpoint(
@NotNull final Endpoint external, @NotNull final List<Endpoint> existingList) {
Endpoint matchingExistingEndpoint = existingList.getFirst();
matchingExistingEndpoint.setLastSeen(external.getLastSeen());
matchingExistingEndpoint.setExternalReference(external.getExternalReference());
matchingExistingEndpoint.setName(external.getName());
matchingExistingEndpoint.setIps(external.getIps());
matchingExistingEndpoint.setHostname(external.getHostname());
matchingExistingEndpoint.setProcessName(external.getProcessName());
matchingExistingEndpoint.setPlatform(external.getPlatform());
matchingExistingEndpoint.setArch(external.getArch());
matchingExistingEndpoint.setExecutor(this.executor);
if ((now().toEpochMilli() - matchingExistingEndpoint.getClearedAt().toEpochMilli())
> CLEAR_TTL) {
@NotNull final Agent agent, @NotNull final Endpoint existingEndpoint) {
existingEndpoint.setLastSeen(toInstant(agent.getLast_seen()));
existingEndpoint.setExternalReference(agent.getPaw());
existingEndpoint.setName(agent.getHost());
existingEndpoint.setIps(agent.getHost_ip_addrs());
existingEndpoint.setHostname(agent.getHost());
existingEndpoint.setProcessName(agent.getExe_name());
existingEndpoint.setPlatform(toPlatform(agent.getPlatform()));
existingEndpoint.setArch(toArch(agent.getArchitecture()));
existingEndpoint.setExecutor(this.executor);
if ((now().toEpochMilli() - existingEndpoint.getClearedAt().toEpochMilli()) > CLEAR_TTL) {
try {
log.info("Clearing endpoint " + matchingExistingEndpoint.getHostname());
log.info("Clearing endpoint " + existingEndpoint.getHostname());
Iterable<Injector> injectors = injectorService.injectors();
injectors.forEach(
injector -> {
if (injector.getExecutorClearCommands() != null) {
this.calderaExecutorContextService.launchExecutorClear(
injector, matchingExistingEndpoint);
this.calderaExecutorContextService.launchExecutorClear(injector, existingEndpoint);
}
});
matchingExistingEndpoint.setClearedAt(now());
existingEndpoint.setClearedAt(now());
} catch (RuntimeException e) {
log.info("Failed clear agents");
}
}
this.endpointService.updateEndpoint(matchingExistingEndpoint);
this.endpointService.updateEndpoint(existingEndpoint);
}

private Instant toInstant(@NotNull final String lastSeen) {
@VisibleForTesting
protected Instant toInstant(@NotNull final String lastSeen) {
String pattern = "yyyy-MM-dd'T'HH:mm:ss'Z'";
DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(pattern, Locale.getDefault());
LocalDateTime localDateTime = LocalDateTime.parse(lastSeen, dateTimeFormatter);
ZonedDateTime zonedDateTime = localDateTime.atZone(UTC);
return zonedDateTime.toInstant();
}

@VisibleForTesting
protected void setExecutor(Executor executor) {
this.executor = executor;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ public ResponseEntity<Flux<AiResult>> queryAi(String body) {
.POST(HttpRequest.BodyPublishers.ofString(body))
.header("Authorization", "Bearer " + aiConfig.getToken())
.header("Accept", "text/event-stream")
.header("Content-Type", "application/json")
.build();
Flux<AiResult> dataFlux =
Flux.create(
Expand Down
Empty file.
4 changes: 2 additions & 2 deletions openbas-api/src/main/resources/application.properties
Original file line number Diff line number Diff line change
Expand Up @@ -161,12 +161,12 @@ executor.tanium.unix-package-id=

# valid values: local | repository
# default: local
executor.openbas.binaries.origin=
executor.openbas.binaries.origin=local

# if executor.openbas.agent.binaries.origin is set to "local",
# this config is ignored
# default: the OpenBAS instance's version
executor.openbas.binaries.version=
executor.openbas.binaries.version=@project.version@

#############
# INJECTORS #
Expand Down
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
package io.openbas.executors.caldera.service;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import io.openbas.asset.EndpointService;
import io.openbas.database.model.Endpoint;
import io.openbas.database.model.Executor;
import io.openbas.executors.caldera.client.CalderaExecutorClient;
import io.openbas.executors.caldera.config.CalderaExecutorConfig;
import io.openbas.executors.caldera.model.Agent;
import io.openbas.integrations.ExecutorService;
import io.openbas.integrations.InjectorService;
import io.openbas.service.PlatformSettingsService;
import java.util.List;
import java.util.Optional;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;

@ExtendWith(MockitoExtension.class)
public class CalderaExecutorServiceTest {
private static final String CALDERA_AGENT_HOSTNAME = "calderaHostname";
private static final String CALDERA_AGENT_EXTERNAL_REF = "calderaExt";
private static final String CALDERA_AGENT_IP = "10.10.10.10";

private static final String CALDERA_EXECUTOR_TYPE = "openbas_caldera";
private static final String CALDERA_EXECUTOR_NAME = "Caldera";

private static final String DATE = "2024-12-11T12:45:30Z";

@Mock private ExecutorService executorService;

@Mock private CalderaExecutorClient client;

@Mock private CalderaExecutorConfig config;

@Mock private CalderaExecutorContextService calderaExecutorContextService;

@Mock private EndpointService endpointService;

@Mock private InjectorService injectorService;

@Mock private PlatformSettingsService platformSettingsService;

@Mock private Executor executor;

@InjectMocks private CalderaExecutorService calderaExecutorService;

private Endpoint calderaEndpoint;
private Endpoint randomEndpoint;
private Agent calderaAgent;
private Agent randomAgent;
private Executor calderaExecutor;
private Executor randomExecutor;

@BeforeEach
void setUp() {
calderaAgent = new Agent();
calderaAgent.setArchitecture("Arch");
calderaAgent.setPaw(CALDERA_AGENT_EXTERNAL_REF);
calderaExecutor = new Executor();
calderaExecutor.setName(CALDERA_EXECUTOR_NAME);
calderaExecutor.setType(CALDERA_EXECUTOR_TYPE);
randomExecutor = new Executor();
randomExecutor.setName("NAME");
randomExecutor.setType("TYPE");
calderaExecutorService.setExecutor(calderaExecutor);

calderaAgent =
createAgent(CALDERA_AGENT_HOSTNAME, CALDERA_AGENT_IP, CALDERA_AGENT_EXTERNAL_REF);
randomAgent = createAgent("hostname", "1.1.1.1", "ref");
calderaEndpoint = createEndpoint(calderaAgent, calderaExecutor);
randomEndpoint = createEndpoint(randomAgent, randomExecutor);

when(endpointService.findAssetsForInjectionByHostname(CALDERA_AGENT_HOSTNAME))
.thenReturn(List.of(calderaEndpoint, randomEndpoint));
}

private Endpoint createEndpoint(Agent agent, Executor executor) {
Endpoint endpoint = new Endpoint();
endpoint.setExecutor(executor);
endpoint.setExternalReference(agent.getPaw());
endpoint.setName(agent.getHost());
endpoint.setDescription("Asset collected by Caldera executor context.");
endpoint.setIps(agent.getHost_ip_addrs());
endpoint.setHostname(agent.getHost());
endpoint.setPlatform(CalderaExecutorService.toPlatform("windows"));
endpoint.setArch(CalderaExecutorService.toArch("amd64"));
endpoint.setProcessName(agent.getExe_name());
endpoint.setLastSeen(calderaExecutorService.toInstant(DATE));
return endpoint;
}

private Agent createAgent(String hostname, String ip, String externalRef) {
Agent agent = new Agent();
agent.setArchitecture("amd64");
agent.setPaw(externalRef);
agent.setPlatform("windows");
agent.setExe_name("exe");
agent.setLast_seen(DATE);
agent.setHost_ip_addrs(new String[] {ip});
agent.setHost(hostname);
return agent;
}

@Test
void test_run_WITH_one_endpoint() throws Exception {
when(client.agents()).thenReturn(List.of(calderaAgent));
calderaExecutorService.run();
verify(endpointService).updateEndpoint(calderaEndpoint);
}

@Test
void test_run_WITH_2_existing_endpoint_same_machine() throws Exception {
when(client.agents()).thenReturn(List.of(calderaAgent));
randomEndpoint.setHostname(CALDERA_AGENT_HOSTNAME);
randomEndpoint.setIps(new String[] {CALDERA_AGENT_IP});
calderaExecutorService.run();
verify(endpointService).updateEndpoint(calderaEndpoint);
}

@Test
void test_findExistingEndpointForAnAgent_WITH_2_existing_endpoint_same_host() throws Exception {
Optional<Endpoint> result = calderaExecutorService.findExistingEndpointForAnAgent(calderaAgent);
assertEquals(calderaEndpoint, result.get());
}

@Test
void test_findExistingEndpointForAnAgent_WITH_no_existing_endpoint() throws Exception {
when(endpointService.findAssetsForInjectionByHostname(CALDERA_AGENT_HOSTNAME))
.thenReturn(List.of());
Optional<Endpoint> result = calderaExecutorService.findExistingEndpointForAnAgent(calderaAgent);
assertTrue(result.isEmpty());
}

@Test
void test_findExistingEndpointForAnAgent_WITH_1_enpdoint_with_null_executor() throws Exception {
randomEndpoint.setExecutor(null);
randomEndpoint.setHostname(CALDERA_AGENT_HOSTNAME);
randomEndpoint.setIps(new String[] {CALDERA_AGENT_IP});
Optional<Endpoint> result = calderaExecutorService.findExistingEndpointForAnAgent(calderaAgent);
assertEquals(calderaEndpoint, result.get());
}
}
Loading

0 comments on commit 66d8047

Please sign in to comment.