Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add RecentLogs support #1237

Draft
wants to merge 6 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -48,4 +48,8 @@ Mono<MetaResponse> meta(MetaRequest request) {
Mono<ReadResponse> read(ReadRequest request) {
return get(request, ReadResponse.class, "read", request.getSourceId()).checkpoint();
}

Mono<ReadResponse> recentLogs(ReadRequest request) {
return get(request, ReadResponse.class, "read", request.getSourceId()).checkpoint();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,11 @@ public Mono<ReadResponse> read(ReadRequest request) {
return getReactorLogCacheEndpoints().read(request);
}

@Override
public Mono<ReadResponse> recentLogs(ReadRequest request) {
return getReactorLogCacheEndpoints().recentLogs(request);
}

/**
* The connection context
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
import org.cloudfoundry.reactor.TestRequest;
import org.cloudfoundry.reactor.TestResponse;
import org.cloudfoundry.reactor.client.AbstractClientApiTest;
import org.cloudfoundry.reactor.client.v3.serviceinstances.ReactorServiceInstancesV3;
import org.junit.jupiter.api.Test;
import reactor.test.StepVerifier;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,15 @@ public interface DopplerClient {
*/
Flux<Envelope> firehose(FirehoseRequest request);

//TODO Adapt the message
/**
* Makes the <a href="https://github.com/cloudfoundry/loggregator/tree/develop/src/trafficcontroller#endpoints">Recent Logs</a> request
*
* @deprecated Do not use this type directly, it exists only for the <em>Jackson</em>-binding infrastructure
* @param request the Recent Logs request
* @return the events from the recent logs
*/
@Deprecated
Flux<Envelope> recentLogs(RecentLogsRequest request);

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,4 +46,12 @@ public interface LogCacheClient {
* @return the read response
*/
Mono<ReadResponse> read(ReadRequest request);

/**
* Makes the Log Cache RecentLogs /api/v1/read request
*
* @param request the Recent Logs request
* @return the events from the recent logs
*/
Mono<ReadResponse> recentLogs(ReadRequest request);
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.cloudfoundry.client.v3.spaces.ListSpacesRequest;
import org.cloudfoundry.client.v3.spaces.SpaceResource;
import org.cloudfoundry.doppler.DopplerClient;
import org.cloudfoundry.logcache.v1.LogCacheClient;
import org.cloudfoundry.networking.NetworkingClient;
import org.cloudfoundry.operations.advanced.Advanced;
import org.cloudfoundry.operations.advanced.DefaultAdvanced;
Expand Down Expand Up @@ -79,7 +80,7 @@ public Advanced advanced() {
@Override
@Value.Derived
public Applications applications() {
return new DefaultApplications(getCloudFoundryClientPublisher(), getDopplerClientPublisher(), getSpaceId());
return new DefaultApplications(getCloudFoundryClientPublisher(), getDopplerClientPublisher(), getLogCacheClientPublisher(), getSpaceId());
}

@Override
Expand Down Expand Up @@ -185,6 +186,19 @@ Mono<DopplerClient> getDopplerClientPublisher() {
.orElse(Mono.error(new IllegalStateException("DopplerClient must be set")));
}

/**
* The {@link LogCacheClient} to use for operations functionality
*/
@Nullable
abstract LogCacheClient getLogCacheClient();

@Value.Derived
Mono<LogCacheClient> getLogCacheClientPublisher() {
return Optional.ofNullable(getLogCacheClient())
.map(Mono::just)
.orElse(Mono.error(new IllegalStateException("LogCacheClient must be set")));
}

/**
* The {@link NetworkingClient} to use for operations functionality
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
package org.cloudfoundry.operations.applications;

import org.cloudfoundry.doppler.LogMessage;
import org.cloudfoundry.logcache.v1.Log;
import org.cloudfoundry.logcache.v1.ReadRequest;
import org.cloudfoundry.logcache.v1.ReadResponse;

import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;

Expand Down Expand Up @@ -120,7 +124,7 @@ public interface Applications {
* @param request the application logs request
* @return the applications logs
*/
Flux<LogMessage> logs(LogsRequest request);
Flux<Log> logs(LogsRequest request);

/**
* Push a specific application
Expand Down

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
import org.cloudfoundry.client.v3.spaces.SpacesV3;
import org.cloudfoundry.client.v3.tasks.Tasks;
import org.cloudfoundry.doppler.DopplerClient;
import org.cloudfoundry.logcache.v1.LogCacheClient;
import org.cloudfoundry.routing.RoutingClient;
import org.cloudfoundry.routing.v1.routergroups.RouterGroups;
import org.cloudfoundry.uaa.UaaClient;
Expand Down Expand Up @@ -101,6 +102,8 @@ public abstract class AbstractOperationsTest {

protected final DopplerClient dopplerClient = mock(DopplerClient.class, RETURNS_SMART_NULLS);

protected final LogCacheClient logCacheClient = mock(LogCacheClient.class, RETURNS_SMART_NULLS);

protected final Events events = mock(Events.class, RETURNS_SMART_NULLS);

protected final FeatureFlags featureFlags = mock(FeatureFlags.class, RETURNS_SMART_NULLS);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import static org.cloudfoundry.client.v3.LifecycleType.BUILDPACK;
import static org.cloudfoundry.client.v3.LifecycleType.DOCKER;
import static org.cloudfoundry.operations.TestObjects.fill;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.RETURNS_SMART_NULLS;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
Expand Down Expand Up @@ -144,10 +145,18 @@
import org.cloudfoundry.doppler.LogMessage;
import org.cloudfoundry.doppler.RecentLogsRequest;
import org.cloudfoundry.doppler.StreamRequest;
import org.cloudfoundry.logcache.v1.EnvelopeBatch;
import org.cloudfoundry.logcache.v1.EnvelopeType;
import org.cloudfoundry.logcache.v1.Log;
import org.cloudfoundry.logcache.v1.LogCacheClient;
import org.cloudfoundry.logcache.v1.LogType;
import org.cloudfoundry.logcache.v1.ReadRequest;
import org.cloudfoundry.logcache.v1.ReadResponse;
import org.cloudfoundry.operations.AbstractOperationsTest;
import org.cloudfoundry.util.DateUtils;
import org.cloudfoundry.util.FluentMap;
import org.cloudfoundry.util.ResourceMatchingUtils;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.core.io.ClassPathResource;
import reactor.core.publisher.Flux;
Expand All @@ -163,6 +172,7 @@ final class DefaultApplicationsTest extends AbstractOperationsTest {
new DefaultApplications(
Mono.just(this.cloudFoundryClient),
Mono.just(this.dopplerClient),
Mono.just(this.logCacheClient),
this.randomWords,
Mono.just(TEST_SPACE_ID));

Expand Down Expand Up @@ -1313,12 +1323,12 @@ void logs() {
"test-application-name",
TEST_SPACE_ID,
"test-metadata-id");
requestLogsStream(this.dopplerClient, "test-metadata-id");
requestLogsRecentLogCache(this.logCacheClient, "test-application-name");

this.applications
.logs(LogsRequest.builder().name("test-application-name").recent(false).build())
.logs(LogsRequest.builder().name("test-application-name").recent(true).build())
.as(StepVerifier::create)
.expectNext(fill(LogMessage.builder(), "log-message-").build())
.expectNextMatches(log -> log.getPayload().equals("test-payload"))
.expectComplete()
.verify(Duration.ofSeconds(5));
}
Expand All @@ -1339,39 +1349,40 @@ void logsNoApp() {
.verify(Duration.ofSeconds(5));
}

@Test
void logsRecent() {
requestApplications(
this.cloudFoundryClient,
"test-application-name",
TEST_SPACE_ID,
"test-metadata-id");
requestLogsRecent(this.dopplerClient, "test-metadata-id");

this.applications
.logs(LogsRequest.builder().name("test-application-name").recent(true).build())
.as(StepVerifier::create)
.expectNext(fill(LogMessage.builder(), "log-message-").build())
.expectComplete()
.verify(Duration.ofSeconds(5));
}

@Test
void logsRecentNotSet() {
requestApplications(
this.cloudFoundryClient,
"test-application-name",
TEST_SPACE_ID,
"test-metadata-id");
requestLogsStream(this.dopplerClient, "test-metadata-id");

this.applications
.logs(LogsRequest.builder().name("test-application-name").build())
.as(StepVerifier::create)
.expectNext(fill(LogMessage.builder(), "log-message-").build())
.expectComplete()
.verify(Duration.ofSeconds(5));
}
// TODO: it's not passing since recentLogs is not properly implemented yet with logcacheclient
@Test
void logsRecent() {
requestApplications(
this.cloudFoundryClient,
"test-application-name",
TEST_SPACE_ID,
"test-metadata-id");
requestLogsRecentLogCache(this.logCacheClient, "test-metadata-id");

this.applications
.logs(LogsRequest.builder().name("test-application-name").build())
.as(StepVerifier::create)
.expectNext(fill(Log.builder(), "log-message-").build())
.expectComplete()
.verify(Duration.ofSeconds(5));
}
// TODO: it's not passing since recentLogs is not properly implemented yet with logcacheclient
@Test
void logsRecentNotSet() {
requestApplications(
this.cloudFoundryClient,
"test-application-name",
TEST_SPACE_ID,
"test-metadata-id");
requestLogsStream(this.dopplerClient, "test-metadata-id");

this.applications
.logs(LogsRequest.builder().name("test-application-name").build())
.as(StepVerifier::create)
.expectNext(fill(Log.builder(), "log-message-").build())
.expectComplete()
.verify(Duration.ofSeconds(5));
}

@Test
void pushDocker() {
Expand Down Expand Up @@ -5248,17 +5259,21 @@ private static void requestListTasksEmpty(
.build()));
}

private static void requestLogsRecent(DopplerClient dopplerClient, String applicationId) {
when(dopplerClient.recentLogs(
RecentLogsRequest.builder().applicationId(applicationId).build()))
.thenReturn(
Flux.just(
Envelope.builder()
.eventType(EventType.LOG_MESSAGE)
.logMessage(
fill(LogMessage.builder(), "log-message-").build())
.origin("rsp")
.build()));
private static void requestLogsRecentLogCache(LogCacheClient logCacheClient, String applicationId) {
when(logCacheClient.recentLogs(
any()))
.thenReturn(
Mono.just(fill(ReadResponse.builder())
.envelopes(fill(EnvelopeBatch.builder())
.batch(fill(org.cloudfoundry.logcache.v1.Envelope
.builder())
.log(fill(Log.builder())
.payload("test-payload")
.type(LogType.OUT)
.build())
.build())
.build())
.build()));
}

private static void requestLogsStream(DopplerClient dopplerClient, String applicationId) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import org.cloudfoundry.client.v2.stacks.ListStacksRequest;
import org.cloudfoundry.client.v2.userprovidedserviceinstances.CreateUserProvidedServiceInstanceRequest;
import org.cloudfoundry.doppler.DopplerClient;
import org.cloudfoundry.logcache.v1.LogCacheClient;
import org.cloudfoundry.logcache.v1.TestLogCacheEndpoints;
import org.cloudfoundry.networking.NetworkingClient;
import org.cloudfoundry.operations.DefaultCloudFoundryOperations;
Expand Down Expand Up @@ -254,6 +255,7 @@ ReactorCloudFoundryClient cloudFoundryClient(
DefaultCloudFoundryOperations cloudFoundryOperations(
CloudFoundryClient cloudFoundryClient,
DopplerClient dopplerClient,
LogCacheClient logCacheClient,
NetworkingClient networkingClient,
RoutingClient routingClient,
UaaClient uaaClient,
Expand All @@ -263,6 +265,7 @@ DefaultCloudFoundryOperations cloudFoundryOperations(
.cloudFoundryClient(cloudFoundryClient)
.dopplerClient(dopplerClient)
.networkingClient(networkingClient)
.logCacheClient(logCacheClient)
.routingClient(routingClient)
.uaaClient(uaaClient)
.organization(organizationName)
Expand Down Expand Up @@ -547,7 +550,7 @@ Mono<String> stackId(CloudFoundryClient cloudFoundryClient, String stackName) {

@Bean
String stackName() {
return "cflinuxfs3";
return "cflinuxfs4";
}

@Bean(initMethod = "block")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,11 @@
import org.cloudfoundry.AbstractIntegrationTest;
import org.cloudfoundry.CloudFoundryVersion;
import org.cloudfoundry.IfCloudFoundryVersion;
import org.cloudfoundry.doppler.Envelope;
import org.cloudfoundry.doppler.LogMessage;
import org.cloudfoundry.doppler.MessageType;
import org.cloudfoundry.logcache.v1.LogType;
import org.cloudfoundry.logcache.v1.ReadRequest;
import org.cloudfoundry.operations.applications.ApplicationDetail;
import org.cloudfoundry.operations.applications.ApplicationEnvironments;
import org.cloudfoundry.operations.applications.ApplicationEvent;
Expand Down Expand Up @@ -494,19 +497,15 @@ public void logs() throws IOException {
this.cloudFoundryOperations,
new ClassPathResource("test-application.zip").getFile().toPath(),
applicationName,
false)
.thenMany(
this.cloudFoundryOperations
false)
.thenMany(this.cloudFoundryOperations
.applications()
.logs(
LogsRequest.builder()
.name(applicationName)
.recent(true)
.logs(ReadRequest.builder()
.sourceId(applicationName)
.build()))
.map(LogMessage::getMessageType)
.next()
.map(org.cloudfoundry.logcache.v1.Log::getType)
.as(StepVerifier::create)
.expectNext(MessageType.OUT)
.expectNext(org.cloudfoundry.logcache.v1.LogType.OUT)
.expectComplete()
.verify(Duration.ofMinutes(5));
}
Expand Down
Loading