From 859e0137b0c1971c5a3811769c54e98effe1810f Mon Sep 17 00:00:00 2001 From: bgehrels Date: Mon, 9 Jul 2018 14:55:04 +0200 Subject: [PATCH 01/18] Adding a spring boot 2 test. At the moment, it only tests successfull startup. Let's see what comes next. --- .../pom.xml | 86 +++++++++++++++++++ .../nakadiproducer/tests/Application.java | 32 +++++++ .../src/main/resources/application.properties | 2 + .../resources/db/migration/V1.1__noop.sql | 0 .../main/resources/tokens/nakadi-token-secret | 1 + .../main/resources/tokens/nakadi-token-type | 1 + .../nakadiproducer/tests/ApplicationIT.java | 34 ++++++++ pom.xml | 1 + 8 files changed, 157 insertions(+) create mode 100644 nakadi-producer-starter-spring-boot-2-test/pom.xml create mode 100644 nakadi-producer-starter-spring-boot-2-test/src/main/java/org/zalando/nakadiproducer/tests/Application.java create mode 100644 nakadi-producer-starter-spring-boot-2-test/src/main/resources/application.properties create mode 100644 nakadi-producer-starter-spring-boot-2-test/src/main/resources/db/migration/V1.1__noop.sql create mode 100644 nakadi-producer-starter-spring-boot-2-test/src/main/resources/tokens/nakadi-token-secret create mode 100644 nakadi-producer-starter-spring-boot-2-test/src/main/resources/tokens/nakadi-token-type create mode 100644 nakadi-producer-starter-spring-boot-2-test/src/test/java/org/zalando/nakadiproducer/tests/ApplicationIT.java diff --git a/nakadi-producer-starter-spring-boot-2-test/pom.xml b/nakadi-producer-starter-spring-boot-2-test/pom.xml new file mode 100644 index 00000000..1ad73e53 --- /dev/null +++ b/nakadi-producer-starter-spring-boot-2-test/pom.xml @@ -0,0 +1,86 @@ + + + 4.0.0 + + nakadi-producer-starter-spring-boot-2-test + org.zalando + + + + org.springframework.boot + spring-boot-starter-parent + 2.0.3.RELEASE + + + + + org.zalando + nakadi-producer-spring-boot-starter + 4.1.3 + + + org.springframework.boot + spring-boot-starter-web + + + org.zalando.stups + tokens + 0.12.0-beta-2 + + + org.springframework.boot + spring-boot-starter-jdbc + 2.0.3.RELEASE + + + com.opentable.components + otj-pg-embedded + 0.12.0 + + + postgresql + postgresql + + + + + org.springframework.boot + spring-boot-starter-test + test + + + com.github.stefanbirkner + system-rules + 1.18.0 + test + + + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.22.0 + + + + integration-test + verify + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + + \ No newline at end of file diff --git a/nakadi-producer-starter-spring-boot-2-test/src/main/java/org/zalando/nakadiproducer/tests/Application.java b/nakadi-producer-starter-spring-boot-2-test/src/main/java/org/zalando/nakadiproducer/tests/Application.java new file mode 100644 index 00000000..c7b2cd1f --- /dev/null +++ b/nakadi-producer-starter-spring-boot-2-test/src/main/java/org/zalando/nakadiproducer/tests/Application.java @@ -0,0 +1,32 @@ +package org.zalando.nakadiproducer.tests; + +import com.opentable.db.postgres.embedded.EmbeddedPostgres; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Primary; +import org.zalando.nakadiproducer.EnableNakadiProducer; + +import javax.sql.DataSource; + +import java.io.IOException; + +@EnableAutoConfiguration +@EnableNakadiProducer +public class Application { + + public static void main(String[] args) throws Exception { + SpringApplication.run(Application.class, args); + } + + @Bean + @Primary + public DataSource dataSource() throws IOException { + return embeddedPostgres().getPostgresDatabase(); + } + + @Bean + public EmbeddedPostgres embeddedPostgres() throws IOException { + return EmbeddedPostgres.start(); + } +} diff --git a/nakadi-producer-starter-spring-boot-2-test/src/main/resources/application.properties b/nakadi-producer-starter-spring-boot-2-test/src/main/resources/application.properties new file mode 100644 index 00000000..83fa688c --- /dev/null +++ b/nakadi-producer-starter-spring-boot-2-test/src/main/resources/application.properties @@ -0,0 +1,2 @@ +nakadi-producer.access-token-uri: http://localhost:1234 +nakadi-producer.nakadi-base-uri: https://nakadi.example.org:5432 \ No newline at end of file diff --git a/nakadi-producer-starter-spring-boot-2-test/src/main/resources/db/migration/V1.1__noop.sql b/nakadi-producer-starter-spring-boot-2-test/src/main/resources/db/migration/V1.1__noop.sql new file mode 100644 index 00000000..e69de29b diff --git a/nakadi-producer-starter-spring-boot-2-test/src/main/resources/tokens/nakadi-token-secret b/nakadi-producer-starter-spring-boot-2-test/src/main/resources/tokens/nakadi-token-secret new file mode 100644 index 00000000..98592b28 --- /dev/null +++ b/nakadi-producer-starter-spring-boot-2-test/src/main/resources/tokens/nakadi-token-secret @@ -0,0 +1 @@ +my fake token \ No newline at end of file diff --git a/nakadi-producer-starter-spring-boot-2-test/src/main/resources/tokens/nakadi-token-type b/nakadi-producer-starter-spring-boot-2-test/src/main/resources/tokens/nakadi-token-type new file mode 100644 index 00000000..83f31cba --- /dev/null +++ b/nakadi-producer-starter-spring-boot-2-test/src/main/resources/tokens/nakadi-token-type @@ -0,0 +1 @@ +Bearer \ No newline at end of file diff --git a/nakadi-producer-starter-spring-boot-2-test/src/test/java/org/zalando/nakadiproducer/tests/ApplicationIT.java b/nakadi-producer-starter-spring-boot-2-test/src/test/java/org/zalando/nakadiproducer/tests/ApplicationIT.java new file mode 100644 index 00000000..e652413d --- /dev/null +++ b/nakadi-producer-starter-spring-boot-2-test/src/test/java/org/zalando/nakadiproducer/tests/ApplicationIT.java @@ -0,0 +1,34 @@ +package org.zalando.nakadiproducer.tests; + +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.contrib.java.lang.system.EnvironmentVariables; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +import java.io.File; + +@RunWith(SpringRunner.class) +@SpringBootTest( + classes = Application.class, + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT +) +public class ApplicationIT { + + @ClassRule + public static final EnvironmentVariables environmentVariables + = new EnvironmentVariables(); + + @BeforeClass + public static void fakeCredentialsDir() { + environmentVariables.set("CREDENTIALS_DIR", new File("src/main/resources/tokens").getAbsolutePath()); + } + + @Test + public void foo() { + } + + +} diff --git a/pom.xml b/pom.xml index 02280087..ad226d28 100644 --- a/pom.xml +++ b/pom.xml @@ -22,6 +22,7 @@ nakadi-producer nakadi-producer-spring-boot-starter + nakadi-producer-starter-spring-boot-2-test From 49d61b190f9081f0d65e8672a7f55506aa5470e9 Mon Sep 17 00:00:00 2001 From: bgehrels Date: Mon, 9 Jul 2018 20:31:03 +0200 Subject: [PATCH 02/18] Upgrading to Spring Boot 2, which will probably be a new major release - Documented that 4.2.0 should be used, if one is still using Spring Boot 1 - Spring Boot acutator got a big API rework in Spring Boot 2 so a lot of adaptions were neccessary here: -- Endpoints are not exposed via HTTP by default any more, has to be configured explicitly => documented -- Endpoint code is now technology independent and works out of the box with HTTP and JMX => removed the MvcEndpoint -- Because the endpoint id is automagically used in http paths and config property names and some parts of springs property handly code doesn't support underscores, we had to switch to dashes - The new spring boot version automatically collects *all* FlywayCallback Beans and adds them to the main Flyway instance. Since we want to isolate our flyway config, we had to make sure that our callbacks have a different class name. - The Spring Boot Flyway AutoConfiguration can be disabled. In this case no FlywayProperties will exist leading to an NPE in FlywayMigrator => Fixed - There seems to be no need any more to wrap the Actuator config in a @ManagementContextConfiguration - Added a seperate test module that uses the k8s way to configure access tokens. It can later be copy'n'pasted to test multiple permutation of optional dependencies - Deleted the deprecated Snapshot event provider, so that we don't carry deprecated interfaces into a new major release --- README.md | 24 +-- nakadi-producer-spring-boot-starter/pom.xml | 10 +- .../nakadiproducer/FlywayMigrator.java | 123 +++++++++++++- .../NakadiProducerAutoConfiguration.java | 76 ++------- .../NakadiProducerFlywayCallback.java | 152 ++++++++++++++++-- .../impl/SnapshotEventCreationEndpoint.java | 21 +-- .../SnapshotEventCreationMvcEndpoint.java | 36 ----- .../main/resources/META-INF/spring.factories | 2 - .../NakadiProducerFlywayCallbackIT.java | 22 ++- .../NonNakadiProducerFlywayCallbackIT.java | 23 ++- .../SnapshotEventGenerationWebEndpointIT.java | 23 +-- ...shotEventGeneratorAutoconfigurationIT.java | 30 ---- .../pom.xml | 17 +- .../nakadiproducer/tests/Application.java | 25 +++ .../src/main/resources/application.properties | 2 - .../src/main/resources/application.yml | 4 + .../nakadiproducer/tests/ApplicationIT.java | 8 + nakadi-producer/pom.xml | 2 +- .../snapshots/SnapshotEventProvider.java | 54 ------- pom.xml | 4 +- 20 files changed, 410 insertions(+), 248 deletions(-) delete mode 100644 nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/snapshots/impl/SnapshotEventCreationMvcEndpoint.java delete mode 100644 nakadi-producer-spring-boot-starter/src/main/resources/META-INF/spring.factories delete mode 100644 nakadi-producer-starter-spring-boot-2-test/src/main/resources/application.properties create mode 100644 nakadi-producer-starter-spring-boot-2-test/src/main/resources/application.yml delete mode 100644 nakadi-producer/src/main/java/org/zalando/nakadiproducer/snapshots/SnapshotEventProvider.java diff --git a/README.md b/README.md index 4e380d11..c2397ae8 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,8 @@ You may of course always setup a fresh system with the newest version. ## Prerequisites -This library was tested with Spring Boot 1.5.3.RELEASE and relies on an existing configured PostgreSQL DataSource. +This library was tested with Spring Boot 2.0.3.RELEASE and relies on an existing configured PostgreSQL DataSource. If +you are still using Spring Boot 1.x, please use version 4.2.0 ([Release Notes](https://github.com/zalando-nakadi/nakadi-producer-spring-boot-starter/releases/tag/4.2.0), [Documentation](https://github.com/zalando-nakadi/nakadi-producer-spring-boot-starter/blob/4.2.0/README.md)) versions of this library. This library also uses: @@ -50,7 +51,7 @@ This library also uses: * jackson >= 2.7.0 * (optional) Zalando's [tracer-spring-boot-starter](https://github.com/zalando/tracer) * (optional) Zalando's [tokens library](https://github.com/zalando/tokens) >= 0.10.0 - * Please note that [tokens-spring-boot-starter](https://github.com/zalando-stups/spring-boot-zalando-stups-tokens) 0.10.0 comes with tokens 0.9.9, which is not enough. You can manually add tokens 0.10.0 with that starter, though. + * Please note that [tokens-spring-boot-starter](https://github.com/zalando-stups/spring-boot-zalando-stups-tokens) 0.10.0 comes with tokens 0.9.9, which is not enough. You can manually add tokens 0.10.0 with that starter, though. To be used in zalandos k8s environment, you must at least use 0.11.0. ## Usage @@ -207,21 +208,26 @@ creation is a irregular, manually triggered maintenance task. This library provides a Spring Boot Actuator endpoint named `snapshot_event_creation` that can be used to trigger a Snapshot for a given event type. Assuming your management port is set to `7979`, - GET localhost:7979/snapshot_event_creation + GET localhost:7979/actuator/snapshot-event-creation will return a list of all event types available for snapshot creation and POST localhost:7979/snapshot_event_creation/my.event-type -will trigger a snapshot for the event type `my.event-type`. The (optional) request body is a "filter specifier". +will trigger a snapshot for the event type `my.event-type`. There is an optional request parameter called "filter" that +will be passed to your application to implement some application specific event filtering logic. -This will only work if your application has configured spring-boot-actuator +This will only work if your application has configured spring-boot-actuator, ```xml org.springframework.boot spring-boot-starter-actuator ``` +your `application.properties` includes +``` +management.endpoints.web.exposure.include=snapshot-event-creation,your-other-endpoints,...` +``` and if one or more Spring Beans implement the `org.zalando.nakadiproducer.snapshots.SnapshotEventGenerator` interface. Otherwise (or if the generator is not for the event type you requested), the library will respond with an error message when you request a snapshot creation. The request body (the "filter specifier") of the trigger request will be passed as a string parameter to the SnapshotEventGenerator's `generateSnapshots` method. @@ -259,12 +265,12 @@ By default, the library will pick up your flyway data source (or the primary dat configured), create its own schema and start setting up its tables in there. You can customize this process in two ways: If you want to use a different data source for schema maintainence (for example to use a different username) and -configuring the Spring flyway datasource is not enough, your can define a spring bean of type `DataSource` and annotate +configuring the Spring Flyway datasource is not enough, your can define a spring bean of type `DataSource` and annotate it with `@NakadiProducerDataSource`. -You may also define a spring bean of type `FlywayCallback` and annotate it with `@NakadiProducerFlywayCallback`. The -interface provide several hook into the schema management lifecycle that may, for example, be used to - `SET ROLE migrator` before and `RESET ROLE` after each migration. +You may also define a spring bean of type `NakadiProducerFlywayCallback`. The interface provides several hooks into the +schema management lifecycle that may, for example, be used to `SET ROLE migrator` before and `RESET ROLE` after each +migration. ### Test support diff --git a/nakadi-producer-spring-boot-starter/pom.xml b/nakadi-producer-spring-boot-starter/pom.xml index 023a3fa3..57679bc2 100644 --- a/nakadi-producer-spring-boot-starter/pom.xml +++ b/nakadi-producer-spring-boot-starter/pom.xml @@ -10,7 +10,7 @@ org.zalando nakadi-producer-reactor - 4.1.3 + 5.0.0-SNAPSHOT nakadi-producer-spring-boot-starter @@ -102,6 +102,14 @@ spring-boot-starter-web test + + org.springframework.boot + spring-boot-autoconfigure + + + org.springframework.boot + spring-boot-actuator-autoconfigure + diff --git a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/FlywayMigrator.java b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/FlywayMigrator.java index fa09a32d..b220ef4c 100644 --- a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/FlywayMigrator.java +++ b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/FlywayMigrator.java @@ -4,11 +4,18 @@ import javax.sql.DataSource; import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.MigrationInfo; +import org.flywaydb.core.api.callback.BaseFlywayCallback; import org.flywaydb.core.api.callback.FlywayCallback; +import org.flywaydb.core.api.configuration.ConfigurationAware; +import org.flywaydb.core.api.configuration.FlywayConfiguration; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.flyway.FlywayDataSource; import org.springframework.boot.autoconfigure.flyway.FlywayProperties; +import java.sql.Connection; +import java.util.List; + public class FlywayMigrator { @Autowired(required = false) @NakadiProducerFlywayDataSource @@ -22,10 +29,9 @@ public class FlywayMigrator { private DataSource dataSource; @Autowired(required = false) - @NakadiProducerFlywayCallback - private FlywayCallback callback; + private List callback; - @Autowired + @Autowired(required = false) private FlywayProperties flywayProperties; @PostConstruct @@ -34,7 +40,7 @@ public void migrateFlyway() { if (this.nakadiProducerFlywayDataSource != null) { flyway.setDataSource(nakadiProducerFlywayDataSource); - } else if (this.flywayProperties.isCreateDataSource()) { + } else if (this.flywayProperties != null && this.flywayProperties.isCreateDataSource()) { flyway.setDataSource(this.flywayProperties.getUrl(), this.flywayProperties.getUser(), this.flywayProperties.getPassword(), this.flywayProperties.getInitSqls().toArray(new String[0])); @@ -47,10 +53,117 @@ public void migrateFlyway() { flyway.setLocations("classpath:db_nakadiproducer/migrations"); flyway.setSchemas("nakadi_events"); if (callback != null) { - flyway.setCallbacks(callback); + flyway.setCallbacks(callback.stream().map(FlywayCallbackAdaptor::new).toArray(FlywayCallback[]::new)); } + flyway.setBaselineOnMigrate(true); flyway.setBaselineVersionAsString("2133546886.1.0"); flyway.migrate(); } + + private static class FlywayCallbackAdaptor extends BaseFlywayCallback { + + private NakadiProducerFlywayCallback callback; + + private FlywayCallbackAdaptor(NakadiProducerFlywayCallback callback) { + this.callback = callback; + } + + @Override + public void setFlywayConfiguration(FlywayConfiguration flywayConfiguration) { + if (callback instanceof ConfigurationAware) { + ((ConfigurationAware) callback).setFlywayConfiguration(flywayConfiguration); + } + } + + @Override + public void beforeClean(Connection connection) { + callback.beforeClean(connection); + } + + @Override + public void afterClean(Connection connection) { + callback.afterClean(connection); + } + + @Override + public void beforeMigrate(Connection connection) { + callback.beforeMigrate(connection); + } + + @Override + public void afterMigrate(Connection connection) { + callback.afterMigrate(connection); + } + + @Override + public void beforeEachMigrate(Connection connection, MigrationInfo info) { + callback.beforeEachMigrate(connection, info); + } + + @Override + public void afterEachMigrate(Connection connection, MigrationInfo info) { + callback.afterEachMigrate(connection, info); + } + + @Override + public void beforeUndo(Connection connection) { + callback.beforeUndo(connection); + } + + @Override + public void beforeEachUndo(Connection connection, MigrationInfo info) { + callback.beforeEachUndo(connection, info); + } + + @Override + public void afterEachUndo(Connection connection, MigrationInfo info) { + callback.afterEachUndo(connection, info); + } + + @Override + public void afterUndo(Connection connection) { + callback.afterUndo(connection); + } + + @Override + public void beforeValidate(Connection connection) { + callback.beforeValidate(connection); + } + + @Override + public void afterValidate(Connection connection) { + callback.afterValidate(connection); + } + + @Override + public void beforeBaseline(Connection connection) { + callback.beforeBaseline(connection); + } + + @Override + public void afterBaseline(Connection connection) { + callback.afterBaseline(connection); + } + + @Override + public void beforeRepair(Connection connection) { + callback.beforeRepair(connection); + } + + @Override + public void afterRepair(Connection connection) { + callback.afterRepair(connection); + } + + @Override + public void beforeInfo(Connection connection) { + callback.beforeInfo(connection); + } + + @Override + public void afterInfo(Connection connection) { + callback.afterInfo(connection); + } + } } diff --git a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerAutoConfiguration.java b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerAutoConfiguration.java index 29816e52..cd001930 100644 --- a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerAutoConfiguration.java +++ b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerAutoConfiguration.java @@ -1,18 +1,15 @@ package org.zalando.nakadiproducer; -import static java.util.stream.Collectors.toList; - import java.net.URI; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.stream.Stream; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.actuate.autoconfigure.ManagementContextConfiguration; -import org.springframework.boot.actuate.condition.ConditionalOnEnabledEndpoint; import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.AutoConfigureBefore; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -30,13 +27,9 @@ import org.zalando.nakadiproducer.flowid.FlowIdComponent; import org.zalando.nakadiproducer.flowid.NoopFlowIdComponent; import org.zalando.nakadiproducer.flowid.TracerFlowIdComponent; -import org.zalando.nakadiproducer.snapshots.SimpleSnapshotEventGenerator; -import org.zalando.nakadiproducer.snapshots.Snapshot; import org.zalando.nakadiproducer.snapshots.SnapshotEventGenerator; -import org.zalando.nakadiproducer.snapshots.SnapshotEventProvider; import org.zalando.nakadiproducer.snapshots.impl.SnapshotCreationService; import org.zalando.nakadiproducer.snapshots.impl.SnapshotEventCreationEndpoint; -import org.zalando.nakadiproducer.snapshots.impl.SnapshotEventCreationMvcEndpoint; import org.zalando.nakadiproducer.transmission.NakadiPublishingClient; import org.zalando.nakadiproducer.transmission.impl.EventTransmissionService; import org.zalando.nakadiproducer.transmission.impl.EventTransmitter; @@ -111,70 +104,23 @@ public FlowIdComponent flowIdComponent() { } } - @ManagementContextConfiguration - static class ManagementEndpointConfiguration { - @Bean - @ConditionalOnMissingBean - public SnapshotEventCreationEndpoint snapshotEventCreationEndpoint( - SnapshotCreationService snapshotCreationService) { - return new SnapshotEventCreationEndpoint(snapshotCreationService); - } - - @Bean - @ConditionalOnBean(SnapshotEventCreationEndpoint.class) - @ConditionalOnEnabledEndpoint("snapshot_event_creation") - public SnapshotEventCreationMvcEndpoint snapshotEventCreationMvcEndpoint( - SnapshotEventCreationEndpoint snapshotEventCreationEndpoint) { - return new SnapshotEventCreationMvcEndpoint(snapshotEventCreationEndpoint); - } + @Bean + @ConditionalOnMissingBean + public SnapshotEventCreationEndpoint snapshotEventCreationEndpoint( + SnapshotCreationService snapshotCreationService) { + return new SnapshotEventCreationEndpoint(snapshotCreationService); } @Bean public SnapshotCreationService snapshotCreationService( Optional> snapshotEventGenerators, - Optional snapshotEventProvider, EventLogWriter eventLogWriter) { - final Stream legacyGenerators = - snapshotEventProvider.map(this::wrapInSnapshotEventGenerators) - .orElseGet(Stream::empty); - final Stream nonLegacyGenerators = - snapshotEventGenerators.map(List::stream) - .orElseGet(Stream::empty); - final List allGenerators = - Stream.concat(legacyGenerators, nonLegacyGenerators) - .collect(toList()); - return new SnapshotCreationService(allGenerators, eventLogWriter); - } - - /** - * This method (and the following three) support the legacy {@link SnapshotEventProvider} interface, - * mapping it to the new logic (several {@link SnapshotEventGenerator}s). - * - * It will be removed when we don't support that interface anymore. - */ - private Stream wrapInSnapshotEventGenerators(SnapshotEventProvider p) { - return p.getSupportedEventTypes() - .stream() - .map(t -> wrapInSnapshotEventGenerator(p, t)); - } - - private SnapshotEventGenerator wrapInSnapshotEventGenerator(SnapshotEventProvider provider, String eventType) { - return new SimpleSnapshotEventGenerator( - eventType, - (cursor) -> createNonLegacySnapshots(provider, eventType, cursor) + EventLogWriter eventLogWriter) { + return new SnapshotCreationService( + snapshotEventGenerators.orElse(Collections.emptyList()), + eventLogWriter ); } - private List createNonLegacySnapshots(SnapshotEventProvider provider, String eventType, Object cursor) { - return provider.getSnapshot(eventType, cursor) - .stream() - .map(this::mapLegacyToNewSnapshot) - .collect(toList()); - } - - private Snapshot mapLegacyToNewSnapshot(SnapshotEventProvider.Snapshot snapshot) { - return new Snapshot(snapshot.getId(), snapshot.getDataType(), snapshot.getData()); - } - @Bean public EventLogWriter eventLogWriter(EventLogRepository eventLogRepository, ObjectMapper objectMapper, FlowIdComponent flowIdComponent) { diff --git a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerFlywayCallback.java b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerFlywayCallback.java index e5552e12..9db6b6e4 100644 --- a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerFlywayCallback.java +++ b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerFlywayCallback.java @@ -1,18 +1,148 @@ package org.zalando.nakadiproducer; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; +import org.flywaydb.core.api.MigrationInfo; +import org.flywaydb.core.api.configuration.ConfigurationAware; -import org.springframework.beans.factory.annotation.Qualifier; +import java.sql.Connection; /** - * Qualifier annotation for a FlywayCallback to be injected in to nakadi-producer's Flyway instance. + * This is the main callback interface that should be implemented to get access to flyway lifecycle notifications. + * Simply add code to the callback method you are interested in having. + * + *

If a callback also implements the {@link ConfigurationAware} interface, + * a {@link org.flywaydb.core.api.configuration.FlywayConfiguration} object will automatically be injected before + * calling any methods, giving the callback access to the core flyway configuration.

+ * + *

Each callback method will run within its own transaction.

*/ -@Target({ ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.TYPE, - ElementType.ANNOTATION_TYPE }) -@Retention(RetentionPolicy.RUNTIME) -@Qualifier -public @interface NakadiProducerFlywayCallback { +public interface NakadiProducerFlywayCallback { + /** + * Runs before the clean task executes. + * + * @param connection A valid connection to the database. + */ + void beforeClean(Connection connection); + + /** + * Runs after the clean task executes. + * + * @param connection A valid connection to the database. + */ + void afterClean(Connection connection); + + /** + * Runs before the migrate task executes. + * + * @param connection A valid connection to the database. + */ + void beforeMigrate(Connection connection); + + /** + * Runs after the migrate task executes. + * + * @param connection A valid connection to the database. + */ + void afterMigrate(Connection connection); + + /** + * Runs before the undo task executes. + * + * @param connection A valid connection to the database. + */ + void beforeUndo(Connection connection); + + /** + * Runs before each migration script is undone. + * + * @param connection A valid connection to the database. + * @param info The current MigrationInfo for the migration to be undone. + */ + void beforeEachUndo(Connection connection, MigrationInfo info); + + /** + * Runs after each migration script is undone. + * + * @param connection A valid connection to the database. + * @param info The current MigrationInfo for the migration just undone. + */ + void afterEachUndo(Connection connection, MigrationInfo info); + + /** + * Runs after the undo task executes. + * + * @param connection A valid connection to the database. + */ + void afterUndo(Connection connection); + + /** + * Runs before each migration script is executed. + * + * @param connection A valid connection to the database. + * @param info The current MigrationInfo for this migration. + */ + void beforeEachMigrate(Connection connection, MigrationInfo info); + + /** + * Runs after each migration script is executed. + * + * @param connection A valid connection to the database. + * @param info The current MigrationInfo for this migration. + */ + void afterEachMigrate(Connection connection, MigrationInfo info); + + /** + * Runs before the validate task executes. + * + * @param connection A valid connection to the database. + */ + void beforeValidate(Connection connection); + + /** + * Runs after the validate task executes. + * + * @param connection A valid connection to the database. + */ + void afterValidate(Connection connection); + + /** + * Runs before the baseline task executes. + * + * @param connection A valid connection to the database. + */ + void beforeBaseline(Connection connection); + + /** + * Runs after the baseline task executes. + * + * @param connection A valid connection to the database. + */ + void afterBaseline(Connection connection); + + /** + * Runs before the repair task executes. + * + * @param connection A valid connection to the database. + */ + void beforeRepair(Connection connection); + + /** + * Runs after the repair task executes. + * + * @param connection A valid connection to the database. + */ + void afterRepair(Connection connection); + + /** + * Runs before the info task executes. + * + * @param connection A valid connection to the database. + */ + void beforeInfo(Connection connection); + + /** + * Runs after the info task executes. + * + * @param connection A valid connection to the database. + */ + void afterInfo(Connection connection); } diff --git a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/snapshots/impl/SnapshotEventCreationEndpoint.java b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/snapshots/impl/SnapshotEventCreationEndpoint.java index 3545fbf9..404bafb0 100644 --- a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/snapshots/impl/SnapshotEventCreationEndpoint.java +++ b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/snapshots/impl/SnapshotEventCreationEndpoint.java @@ -2,28 +2,31 @@ import java.util.Set; -import org.springframework.boot.actuate.endpoint.AbstractEndpoint; -import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.actuate.endpoint.annotation.Endpoint; +import org.springframework.boot.actuate.endpoint.annotation.ReadOperation; +import org.springframework.boot.actuate.endpoint.annotation.Selector; +import org.springframework.boot.actuate.endpoint.annotation.WriteOperation; import lombok.AllArgsConstructor; import lombok.Getter; +import org.springframework.lang.Nullable; -@ConfigurationProperties("endpoints.snapshot-event-creation") -public class SnapshotEventCreationEndpoint extends AbstractEndpoint { +@Endpoint(id = "snapshot-event-creation") +public class SnapshotEventCreationEndpoint { private final SnapshotCreationService snapshotCreationService; public SnapshotEventCreationEndpoint(SnapshotCreationService snapshotCreationService) { - super("snapshot_event_creation", true, true); this.snapshotCreationService = snapshotCreationService; } - @Override - public SnapshotReport invoke() { + @ReadOperation + public SnapshotReport getSupportedEventTypes() { return new SnapshotReport(snapshotCreationService.getSupportedEventTypes()); } - public void invoke(String eventType, String filter) { - snapshotCreationService.createSnapshotEvents(eventType, filter); + @WriteOperation + public void createFilteredSnapshotEvents(@Selector String arg0, @Nullable String filter) { + snapshotCreationService.createSnapshotEvents(arg0, filter); } diff --git a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/snapshots/impl/SnapshotEventCreationMvcEndpoint.java b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/snapshots/impl/SnapshotEventCreationMvcEndpoint.java deleted file mode 100644 index fffc577f..00000000 --- a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/snapshots/impl/SnapshotEventCreationMvcEndpoint.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.zalando.nakadiproducer.snapshots.impl; - -import org.springframework.boot.actuate.endpoint.mvc.EndpointMvcAdapter; -import org.springframework.boot.actuate.endpoint.mvc.HypermediaDisabled; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.ResponseBody; - -public class SnapshotEventCreationMvcEndpoint extends EndpointMvcAdapter { - private final SnapshotEventCreationEndpoint delegate; - - public SnapshotEventCreationMvcEndpoint(SnapshotEventCreationEndpoint delegate) { - super(delegate); - this.delegate = delegate; - } - - @RequestMapping(value = "/{eventType:.*}", method = RequestMethod.POST) - @ResponseBody - @HypermediaDisabled - public ResponseEntity createSnapshot(@PathVariable String eventType, - @RequestBody(required = false) String filter) { - if (!this.delegate.isEnabled()) { - // Shouldn't happen - MVC endpoint shouldn't be registered when delegate's - // disabled - return getDisabledResponse(); - } - - delegate.invoke(eventType, filter); - return ResponseEntity.ok().build(); - } - - -} diff --git a/nakadi-producer-spring-boot-starter/src/main/resources/META-INF/spring.factories b/nakadi-producer-spring-boot-starter/src/main/resources/META-INF/spring.factories deleted file mode 100644 index 7fac8de6..00000000 --- a/nakadi-producer-spring-boot-starter/src/main/resources/META-INF/spring.factories +++ /dev/null @@ -1,2 +0,0 @@ -# org.springframework.boot.autoconfigure.EnableAutoConfiguration=org.zalando.nakadiproducer.NakadiProducerAutoConfiguration -org.springframework.boot.actuate.autoconfigure.ManagementContextConfiguration=org.zalando.nakadiproducer.NakadiProducerAutoConfiguration.ManagementEndpointConfiguration \ No newline at end of file diff --git a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NakadiProducerFlywayCallbackIT.java b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NakadiProducerFlywayCallbackIT.java index 638a0c82..5e5caa8d 100644 --- a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NakadiProducerFlywayCallbackIT.java +++ b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NakadiProducerFlywayCallbackIT.java @@ -6,18 +6,32 @@ import java.sql.Connection; -import org.flywaydb.core.api.callback.FlywayCallback; +import org.flywaydb.core.api.configuration.ConfigurationAware; +import org.flywaydb.core.api.configuration.FlywayConfiguration; import org.junit.Test; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.annotation.DirtiesContext; public class NakadiProducerFlywayCallbackIT extends BaseMockedExternalCommunicationIT { @MockBean - @NakadiProducerFlywayCallback - private FlywayCallback flywayCallback; + private NakadiProducerFlywayCallback flywayCallback; + + @MockBean + private ConfigurationAwareNakadiProducerFlywayCallback configurationAwareNakadiProducerFlywayCallback; @Test + @DirtiesContext // Needed to make sure that flyway gets executed for each of the tests and Callbacks are called again public void flywayCallbackIsCalledIfAnnotatedWithQualifierAnnotation() { - verify(flywayCallback, times(1)).beforeValidate(any(Connection.class)); + verify(flywayCallback, times(1)).beforeMigrate(any(Connection.class)); + } + + @Test + @DirtiesContext // Needed to make sure that flyway gets executed for each of the tests and Callbacks are called again + public void flywayConfigurationIsSetIfCallbackIsConfigurationAware() { + verify(configurationAwareNakadiProducerFlywayCallback).setFlywayConfiguration(any(FlywayConfiguration.class)); + } + + public interface ConfigurationAwareNakadiProducerFlywayCallback extends NakadiProducerFlywayCallback, ConfigurationAware { } } diff --git a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NonNakadiProducerFlywayCallbackIT.java b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NonNakadiProducerFlywayCallbackIT.java index 59330475..ce023b00 100644 --- a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NonNakadiProducerFlywayCallbackIT.java +++ b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NonNakadiProducerFlywayCallbackIT.java @@ -8,15 +8,34 @@ import org.flywaydb.core.api.callback.FlywayCallback; import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.zalando.nakadiproducer.config.EmbeddedDataSourceConfig; -public class NonNakadiProducerFlywayCallbackIT extends BaseMockedExternalCommunicationIT { +@ActiveProfiles("test") +@RunWith(SpringRunner.class) +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.MOCK, + properties = { "zalando.team.id:alpha-local-testing", "nakadi-producer.scheduled-transmission-enabled:false", "spring.flyway.enabled:false"}, + classes = { TestApplication.class, EmbeddedDataSourceConfig.class } +) +public class NonNakadiProducerFlywayCallbackIT { @MockBean private FlywayCallback flywayCallback; @Test - public void flywayCallbackIsNotUsedIfNotAnnotatedWithQualifierAnnotation() { + public void flywayCallbacksFromOurHostApplicationAreNotUsedByUs() { verify(flywayCallback, never()).beforeValidate(any(Connection.class)); } + + @Test + public void ourOwnFlywayConfigurationStillWorksFineWhenSpringsFlywayAutoconfigIsDisabled() { + // Yes, this is redundant to the other test in here. + // We consider it important to document the requirement, so it is here nonetheless. + // This test does just enough to test it. + } } diff --git a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGenerationWebEndpointIT.java b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGenerationWebEndpointIT.java index 5569eb21..2846c2bd 100644 --- a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGenerationWebEndpointIT.java +++ b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGenerationWebEndpointIT.java @@ -8,7 +8,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.actuate.autoconfigure.LocalManagementPort; +import org.springframework.boot.actuate.autoconfigure.web.server.LocalManagementPort; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -21,13 +21,13 @@ @RunWith(SpringRunner.class) @SpringBootTest( webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - properties = { "management.security.enabled=false", "zalando.team.id:alpha-local-testing", "nakadi-producer.scheduled-transmission-enabled:false" }, + properties = { "management.security.enabled=false", "zalando.team.id:alpha-local-testing", "nakadi-producer.scheduled-transmission-enabled:false", "management.endpoints.web.exposure.include:snapshot-event-creation" }, classes = { TestApplication.class, EmbeddedDataSourceConfig.class, SnapshotEventGenerationWebEndpointIT.Config.class } ) public class SnapshotEventGenerationWebEndpointIT { private static final String MY_EVENT_TYPE = "my.event-type"; - private static final String MY_REQUEST_BODY = "my request body"; + private static final String filter = "myRequestBody"; @LocalManagementPort private int managementPort; @@ -36,20 +36,21 @@ public class SnapshotEventGenerationWebEndpointIT { private SnapshotEventGenerator snapshotEventGenerator; @Test - public void passesRequestBodyIfPresent() { + public void passesFilterIfPresent() { given().baseUri("http://localhost:" + managementPort) - .body(MY_REQUEST_BODY) - .when().post("/snapshot_event_creation/" + MY_EVENT_TYPE) - .then().statusCode(200); + .contentType("application/json") + .when().post("/actuator/snapshot-event-creation/" + MY_EVENT_TYPE + "?filter=" + filter) + .then().statusCode(204); - verify(snapshotEventGenerator).generateSnapshots(null, MY_REQUEST_BODY); + verify(snapshotEventGenerator).generateSnapshots(null, filter); } @Test - public void passesNullIfNoRequestBodyPresent() { + public void passesNullIfNoFilterIsPresent() { given().baseUri("http://localhost:" + managementPort) - .when().post("/snapshot_event_creation/" + MY_EVENT_TYPE) - .then().statusCode(200); + .contentType("application/json") + .when().post("/actuator/snapshot-event-creation/" + MY_EVENT_TYPE) + .then().statusCode(204); verify(snapshotEventGenerator).generateSnapshots(null, null); } diff --git a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGeneratorAutoconfigurationIT.java b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGeneratorAutoconfigurationIT.java index 9b52eb24..37c00718 100644 --- a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGeneratorAutoconfigurationIT.java +++ b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGeneratorAutoconfigurationIT.java @@ -3,9 +3,6 @@ import static org.junit.Assert.fail; import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -29,12 +26,6 @@ public void picksUpDefinedSnapshotEventProviders() { // expect no exceptions snapshotCreationService.createSnapshotEvents("B", ""); - // expect no exceptions - snapshotCreationService.createSnapshotEvents("C", ""); - - // expect no exceptions - snapshotCreationService.createSnapshotEvents("D", ""); - try { snapshotCreationService.createSnapshotEvents("not defined", ""); } catch (final UnknownEventTypeException e) { @@ -56,26 +47,5 @@ public SnapshotEventGenerator snapshotEventProviderA() { public SnapshotEventGenerator snapshotEventProviderB() { return new SimpleSnapshotEventGenerator("B", (x) -> Collections.emptyList()); } - - @Bean - public SnapshotEventProvider snapshotEventProviderCandD() { - return new SnapshotEventProvider() { - @Override - public List getSnapshot(String eventType, Object withIdGreaterThan) { - if (!getSupportedEventTypes().contains(eventType)) { - throw new UnknownEventTypeException(eventType); - } - return Collections.emptyList(); - } - - @Override - public Set getSupportedEventTypes() { - final HashSet types = new HashSet<>(); - types.add("C"); - types.add("D"); - return types; - } - }; - } } } diff --git a/nakadi-producer-starter-spring-boot-2-test/pom.xml b/nakadi-producer-starter-spring-boot-2-test/pom.xml index 1ad73e53..4e867ab2 100644 --- a/nakadi-producer-starter-spring-boot-2-test/pom.xml +++ b/nakadi-producer-starter-spring-boot-2-test/pom.xml @@ -8,16 +8,16 @@ - org.springframework.boot - spring-boot-starter-parent - 2.0.3.RELEASE + org.zalando + nakadi-producer-reactor + 5.0.0-SNAPSHOT org.zalando nakadi-producer-spring-boot-starter - 4.1.3 + ${project.version} org.springframework.boot @@ -44,6 +44,10 @@ + + org.springframework.boot + spring-boot-starter-actuator + org.springframework.boot spring-boot-starter-test @@ -55,6 +59,11 @@ 1.18.0 test + + io.rest-assured + rest-assured + 3.1.0 + diff --git a/nakadi-producer-starter-spring-boot-2-test/src/main/java/org/zalando/nakadiproducer/tests/Application.java b/nakadi-producer-starter-spring-boot-2-test/src/main/java/org/zalando/nakadiproducer/tests/Application.java index c7b2cd1f..9a874c57 100644 --- a/nakadi-producer-starter-spring-boot-2-test/src/main/java/org/zalando/nakadiproducer/tests/Application.java +++ b/nakadi-producer-starter-spring-boot-2-test/src/main/java/org/zalando/nakadiproducer/tests/Application.java @@ -6,10 +6,17 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Primary; import org.zalando.nakadiproducer.EnableNakadiProducer; +import org.zalando.nakadiproducer.snapshots.SimpleSnapshotEventGenerator; +import org.zalando.nakadiproducer.snapshots.Snapshot; +import org.zalando.nakadiproducer.snapshots.SnapshotEventGenerator; import javax.sql.DataSource; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.function.BiFunction; @EnableAutoConfiguration @EnableNakadiProducer @@ -29,4 +36,22 @@ public DataSource dataSource() throws IOException { public EmbeddedPostgres embeddedPostgres() throws IOException { return EmbeddedPostgres.start(); } + + @Bean + public SnapshotEventGenerator snapshotEventGenerator() { + return new SimpleSnapshotEventGenerator("eventtype", new BiFunction>() { + @Override + public List apply(Object withIdGreaterThan, String filter) { + if (withIdGreaterThan == null) { + return Collections.singletonList(new Snapshot("1", "foo", (Object) filter)); + } else if (withIdGreaterThan.equals("1")) { + return Collections.singletonList(new Snapshot("2", "foo", (Object) filter)); + } else { + return new ArrayList<>(); + } + } + }); + + // Todo: Test that some events arrive at a local nakadi mock + } } diff --git a/nakadi-producer-starter-spring-boot-2-test/src/main/resources/application.properties b/nakadi-producer-starter-spring-boot-2-test/src/main/resources/application.properties deleted file mode 100644 index 83fa688c..00000000 --- a/nakadi-producer-starter-spring-boot-2-test/src/main/resources/application.properties +++ /dev/null @@ -1,2 +0,0 @@ -nakadi-producer.access-token-uri: http://localhost:1234 -nakadi-producer.nakadi-base-uri: https://nakadi.example.org:5432 \ No newline at end of file diff --git a/nakadi-producer-starter-spring-boot-2-test/src/main/resources/application.yml b/nakadi-producer-starter-spring-boot-2-test/src/main/resources/application.yml new file mode 100644 index 00000000..47ed5c12 --- /dev/null +++ b/nakadi-producer-starter-spring-boot-2-test/src/main/resources/application.yml @@ -0,0 +1,4 @@ +nakadi-producer: + access-token-uri: http://localhost:1234 + nakadi-base-uri: https://nakadi.example.org:5432 +management.endpoints.web.exposure.include: snapshot-event-creation \ No newline at end of file diff --git a/nakadi-producer-starter-spring-boot-2-test/src/test/java/org/zalando/nakadiproducer/tests/ApplicationIT.java b/nakadi-producer-starter-spring-boot-2-test/src/test/java/org/zalando/nakadiproducer/tests/ApplicationIT.java index e652413d..0336b65a 100644 --- a/nakadi-producer-starter-spring-boot-2-test/src/test/java/org/zalando/nakadiproducer/tests/ApplicationIT.java +++ b/nakadi-producer-starter-spring-boot-2-test/src/test/java/org/zalando/nakadiproducer/tests/ApplicationIT.java @@ -5,17 +5,22 @@ import org.junit.Test; import org.junit.contrib.java.lang.system.EnvironmentVariables; import org.junit.runner.RunWith; +import org.springframework.boot.actuate.autoconfigure.web.server.LocalManagementPort; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.junit4.SpringRunner; import java.io.File; +import static io.restassured.RestAssured.given; + @RunWith(SpringRunner.class) @SpringBootTest( classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT ) public class ApplicationIT { + @LocalManagementPort + private int localManagementPort; @ClassRule public static final EnvironmentVariables environmentVariables @@ -28,6 +33,9 @@ public static void fakeCredentialsDir() { @Test public void foo() { + given().baseUri("http://localhost:" + localManagementPort).contentType("application/json") + .when().post("/actuator/snapshot-event-creation/eventtype") + .then().statusCode(204); } diff --git a/nakadi-producer/pom.xml b/nakadi-producer/pom.xml index cfc905f4..e91aa8a6 100644 --- a/nakadi-producer/pom.xml +++ b/nakadi-producer/pom.xml @@ -10,7 +10,7 @@ org.zalando nakadi-producer-reactor - 4.1.3 + 5.0.0-SNAPSHOT nakadi-producer diff --git a/nakadi-producer/src/main/java/org/zalando/nakadiproducer/snapshots/SnapshotEventProvider.java b/nakadi-producer/src/main/java/org/zalando/nakadiproducer/snapshots/SnapshotEventProvider.java deleted file mode 100644 index 47ed4dd2..00000000 --- a/nakadi-producer/src/main/java/org/zalando/nakadiproducer/snapshots/SnapshotEventProvider.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.zalando.nakadiproducer.snapshots; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -import java.util.List; -import java.util.Set; - -/** - * The {@code SnapshotEventProvider} interface should be implemented by any - * Event Producer that wants to support snapshot events feature. The - * class must define a method {@code getSnapshot}. - * - * @deprecated Please use one or more {@link SnapshotEventGenerator} instances instead - */ -@Deprecated -public interface SnapshotEventProvider { - - /** - * Returns a batch of snapshots of given type (event type is an event channel topic name). The implementation may - * return an arbitrary amount of results, but it must return at least one element if there are entities - * matching the parameters. - * - * The library will call your implementation like this: - * Request: getSnapshot(eventType, null) Response: 1,2,3 - * Request: getSnapshot(eventType, 3) Response: 4,5 - * Request: getSnapshot(eventType, 5) Response: emptyList - * It is your responsibility to make sure that the returned events are ordered by their id asc and that, given you - * return a list of events for entities with ids {id1, ..., idN}, there exists no entity with an id between id1 and idN, that - * is not part of the result. - * - * - * @param eventType event type to make a snapshot of - * @param withIdGreaterThan if not null, only events for entities with an id greater than the given one must be returned - * @return list of elements ordered by their id - * @throws UnknownEventTypeException if {@code eventType} is unknown - */ - @Deprecated - List getSnapshot(String eventType, Object withIdGreaterThan); - - @Deprecated - Set getSupportedEventTypes(); - - @AllArgsConstructor - @Getter - @Deprecated - class Snapshot { - private Object id; - private String eventType; - private String dataType; - private Object data; - } - -} diff --git a/pom.xml b/pom.xml index ad226d28..a2fe8c86 100644 --- a/pom.xml +++ b/pom.xml @@ -10,12 +10,12 @@ org.springframework.boot spring-boot-starter-parent - 1.5.3.RELEASE + 2.0.3.RELEASE nakadi-producer-reactor org.zalando - 4.1.3 + 5.0.0-SNAPSHOT pom Nakadi Event Producer Reactor From 66040d4ea7226fdf46576b01cab4d41a04b1a927 Mon Sep 17 00:00:00 2001 From: bgehrels Date: Wed, 11 Jul 2018 18:38:39 +0200 Subject: [PATCH 03/18] Switching to Version 20.0.0 since it matches the 2.0 of Spring boot so nicely and gives us 16 Major Releases time before the Spring Boot 1.x release train overtakes the Spring Boot 2 one --- README.md | 4 ++-- nakadi-producer-spring-boot-starter/pom.xml | 2 +- nakadi-producer-starter-spring-boot-2-test/pom.xml | 2 +- nakadi-producer/pom.xml | 2 +- pom.xml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 45d9eb03..b48ec9d5 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,8 @@ You may of course always setup a fresh system with the newest version. ## Prerequisites -This library was tested with Spring Boot 2.0.3.RELEASE and relies on an existing configured PostgreSQL DataSource. If -you are still using Spring Boot 1.x, please use version 4.2.0 ([Release Notes](https://github.com/zalando-nakadi/nakadi-producer-spring-boot-starter/releases/tag/4.2.0), [Documentation](https://github.com/zalando-nakadi/nakadi-producer-spring-boot-starter/blob/4.2.0/README.md)) versions of this library. +This library was tested with Spring Boot 2.0.3.RELEASE and relies on an existing configured PostgreSQL DataSource. +**If you are still using Spring Boot 1.x, please use versions < 20.0.0, they are still actively maintained ([Documentation](https://github.com/zalando-nakadi/nakadi-producer-spring-boot-starter/tree/spring-boot-1)).** This library also uses: diff --git a/nakadi-producer-spring-boot-starter/pom.xml b/nakadi-producer-spring-boot-starter/pom.xml index 57679bc2..7236d2a0 100644 --- a/nakadi-producer-spring-boot-starter/pom.xml +++ b/nakadi-producer-spring-boot-starter/pom.xml @@ -10,7 +10,7 @@ org.zalando nakadi-producer-reactor - 5.0.0-SNAPSHOT + 20.0.0-SNAPSHOT nakadi-producer-spring-boot-starter diff --git a/nakadi-producer-starter-spring-boot-2-test/pom.xml b/nakadi-producer-starter-spring-boot-2-test/pom.xml index 4e867ab2..d2f6c912 100644 --- a/nakadi-producer-starter-spring-boot-2-test/pom.xml +++ b/nakadi-producer-starter-spring-boot-2-test/pom.xml @@ -10,7 +10,7 @@ org.zalando nakadi-producer-reactor - 5.0.0-SNAPSHOT + 20.0.0-SNAPSHOT diff --git a/nakadi-producer/pom.xml b/nakadi-producer/pom.xml index e91aa8a6..cd1ab088 100644 --- a/nakadi-producer/pom.xml +++ b/nakadi-producer/pom.xml @@ -10,7 +10,7 @@ org.zalando nakadi-producer-reactor - 5.0.0-SNAPSHOT + 20.0.0-SNAPSHOT nakadi-producer diff --git a/pom.xml b/pom.xml index a2fe8c86..83d16fc0 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ nakadi-producer-reactor org.zalando - 5.0.0-SNAPSHOT + 20.0.0-SNAPSHOT pom Nakadi Event Producer Reactor From e74994e984919c96c14e201b3951de97dfe6c15c Mon Sep 17 00:00:00 2001 From: bgehrels Date: Thu, 12 Jul 2018 16:55:45 +0200 Subject: [PATCH 04/18] better naming of a test, removed a dead import --- .../zalando/nakadiproducer/NakadiProducerAutoConfiguration.java | 1 - .../java/org/zalando/nakadiproducer/tests/ApplicationIT.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerAutoConfiguration.java b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerAutoConfiguration.java index cd001930..0ad70345 100644 --- a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerAutoConfiguration.java +++ b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerAutoConfiguration.java @@ -9,7 +9,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.AutoConfigureAfter; -import org.springframework.boot.autoconfigure.AutoConfigureBefore; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; diff --git a/nakadi-producer-starter-spring-boot-2-test/src/test/java/org/zalando/nakadiproducer/tests/ApplicationIT.java b/nakadi-producer-starter-spring-boot-2-test/src/test/java/org/zalando/nakadiproducer/tests/ApplicationIT.java index 0336b65a..b0344459 100644 --- a/nakadi-producer-starter-spring-boot-2-test/src/test/java/org/zalando/nakadiproducer/tests/ApplicationIT.java +++ b/nakadi-producer-starter-spring-boot-2-test/src/test/java/org/zalando/nakadiproducer/tests/ApplicationIT.java @@ -32,7 +32,7 @@ public static void fakeCredentialsDir() { } @Test - public void foo() { + public void shouldSuccessfullyStartAndSnapshotCanBeTriggered() { given().baseUri("http://localhost:" + localManagementPort).contentType("application/json") .when().post("/actuator/snapshot-event-creation/eventtype") .then().statusCode(204); From 405ca3a74cd4ffa1bfafe8508e83bbd9c538efe7 Mon Sep 17 00:00:00 2001 From: bgehrels Date: Tue, 17 Jul 2018 16:40:17 +0200 Subject: [PATCH 05/18] LBJ-982 Fixed actuator path in the readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b48ec9d5..dad6273c 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,7 @@ This library provides a Spring Boot Actuator endpoint named `snapshot_event_crea will return a list of all event types available for snapshot creation and - POST localhost:7979/snapshot_event_creation/my.event-type + POST localhost:7979/actuator/snapshot-event-creation/my.event-type will trigger a snapshot for the event type `my.event-type`. There is an optional request parameter called "filter" that will be passed to your application to implement some application specific event filtering logic. From 2159c004ffbcb906ac24500f17d86e3a97a19048 Mon Sep 17 00:00:00 2001 From: bgehrels Date: Wed, 18 Jul 2018 10:01:31 +0200 Subject: [PATCH 06/18] LBJ-982 make the wording of the docs more consistent regarding snapshot creation --- README.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index dad6273c..663f9941 100644 --- a/README.md +++ b/README.md @@ -212,7 +212,8 @@ process step the event is reporting. ### Event snapshots (optional) A Snapshot event is a special type of data change event (data operation) defined by Nakadi. -It does not represent a change of the state of a resource, but a current snapshot of the state of the resource. +It does not represent a change of the state of a resource, but a current snapshot of its state. It can be usefull to +bootstrap a new consumer or to recover from inconsistencies between sender and consumer after an incident. You can create snapshot events programmatically (using EventLogWriter.fireSnapshotEvent), but usually snapshot event creation is a irregular, manually triggered maintenance task. @@ -225,10 +226,16 @@ will return a list of all event types available for snapshot creation and POST localhost:7979/actuator/snapshot-event-creation/my.event-type -will trigger a snapshot for the event type `my.event-type`. There is an optional request parameter called "filter" that -will be passed to your application to implement some application specific event filtering logic. +will trigger a snapshot for the event type `my.event-type`. You can change the port, the authentication scheme and the +path prefix as part of your Spring Boot Actuator configuration. -This will only work if your application has configured spring-boot-actuator, +You can provide an optional filter specifier that will be passed to your application to implement any application +specific event/entity filtering logic. It can be provided either as a request parameter called `filter`, or as a +request body + + {"filter":"myFilter"} + +This endpoint will only work if your application includes spring-boot-actuator, ```xml org.springframework.boot @@ -239,8 +246,9 @@ your `application.properties` includes ``` management.endpoints.web.exposure.include=snapshot-event-creation,your-other-endpoints,...` ``` -and if one or more Spring Beans implement the `org.zalando.nakadiproducer.snapshots.SnapshotEventGenerator` interface. Otherwise (or if the generator is not for the event type you requested), the library will respond with an error message when you request a snapshot creation. -The request body (the "filter specifier") of the trigger request will be passed as a string parameter to the SnapshotEventGenerator's `generateSnapshots` method. +and if one or more Spring Beans implement the `org.zalando.nakadiproducer.snapshots.SnapshotEventGenerator` interface. +The optional filter specifier of the trigger request will be passed as a string parameter to the +SnapshotEventGenerator's `generateSnapshots` method and may be null, if none is given. We provide a `SimpleSnapshotEventGenerator` to ease bean creation using a more functional style: ```java From dc6733d9e5edb5506b9b12d6d33f805c8c95fb9e Mon Sep 17 00:00:00 2001 From: bgehrels Date: Wed, 18 Jul 2018 10:07:31 +0200 Subject: [PATCH 07/18] LBJ-982 minor code beauty in the Flyway Migrator --- .../org/zalando/nakadiproducer/FlywayMigrator.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/FlywayMigrator.java b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/FlywayMigrator.java index b220ef4c..bf3570f4 100644 --- a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/FlywayMigrator.java +++ b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/FlywayMigrator.java @@ -29,7 +29,7 @@ public class FlywayMigrator { private DataSource dataSource; @Autowired(required = false) - private List callback; + private List callbacks; @Autowired(required = false) private FlywayProperties flywayProperties; @@ -52,8 +52,8 @@ public void migrateFlyway() { flyway.setLocations("classpath:db_nakadiproducer/migrations"); flyway.setSchemas("nakadi_events"); - if (callback != null) { - flyway.setCallbacks(callback.stream().map(FlywayCallbackAdaptor::new).toArray(FlywayCallback[]::new)); + if (callbacks != null) { + flyway.setCallbacks(callbacks.stream().map(FlywayCallbackAdapter::new).toArray(FlywayCallback[]::new)); } flyway.setBaselineOnMigrate(true); @@ -61,11 +61,11 @@ public void migrateFlyway() { flyway.migrate(); } - private static class FlywayCallbackAdaptor extends BaseFlywayCallback { + private static class FlywayCallbackAdapter extends BaseFlywayCallback { private NakadiProducerFlywayCallback callback; - private FlywayCallbackAdaptor(NakadiProducerFlywayCallback callback) { + private FlywayCallbackAdapter(NakadiProducerFlywayCallback callback) { this.callback = callback; } From 9a85b62089c1c749a2615079835baaa50c98a766 Mon Sep 17 00:00:00 2001 From: bgehrels Date: Wed, 18 Jul 2018 10:12:04 +0200 Subject: [PATCH 08/18] LBJ-982 added default implementations to all NakadiProducerFlywayCallback methods --- .../NakadiProducerFlywayCallback.java | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerFlywayCallback.java b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerFlywayCallback.java index 9db6b6e4..bf7e4ff9 100644 --- a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerFlywayCallback.java +++ b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/NakadiProducerFlywayCallback.java @@ -21,35 +21,35 @@ public interface NakadiProducerFlywayCallback { * * @param connection A valid connection to the database. */ - void beforeClean(Connection connection); + default void beforeClean(Connection connection) {} /** * Runs after the clean task executes. * * @param connection A valid connection to the database. */ - void afterClean(Connection connection); + default void afterClean(Connection connection) {}; /** * Runs before the migrate task executes. * * @param connection A valid connection to the database. */ - void beforeMigrate(Connection connection); + default void beforeMigrate(Connection connection) {}; /** * Runs after the migrate task executes. * * @param connection A valid connection to the database. */ - void afterMigrate(Connection connection); + default void afterMigrate(Connection connection) {} /** * Runs before the undo task executes. * * @param connection A valid connection to the database. */ - void beforeUndo(Connection connection); + default void beforeUndo(Connection connection) {} /** * Runs before each migration script is undone. @@ -57,7 +57,7 @@ public interface NakadiProducerFlywayCallback { * @param connection A valid connection to the database. * @param info The current MigrationInfo for the migration to be undone. */ - void beforeEachUndo(Connection connection, MigrationInfo info); + default void beforeEachUndo(Connection connection, MigrationInfo info) {} /** * Runs after each migration script is undone. @@ -65,14 +65,14 @@ public interface NakadiProducerFlywayCallback { * @param connection A valid connection to the database. * @param info The current MigrationInfo for the migration just undone. */ - void afterEachUndo(Connection connection, MigrationInfo info); + default void afterEachUndo(Connection connection, MigrationInfo info) {} /** * Runs after the undo task executes. * * @param connection A valid connection to the database. */ - void afterUndo(Connection connection); + default void afterUndo(Connection connection) {} /** * Runs before each migration script is executed. @@ -80,7 +80,7 @@ public interface NakadiProducerFlywayCallback { * @param connection A valid connection to the database. * @param info The current MigrationInfo for this migration. */ - void beforeEachMigrate(Connection connection, MigrationInfo info); + default void beforeEachMigrate(Connection connection, MigrationInfo info) {} /** * Runs after each migration script is executed. @@ -88,61 +88,61 @@ public interface NakadiProducerFlywayCallback { * @param connection A valid connection to the database. * @param info The current MigrationInfo for this migration. */ - void afterEachMigrate(Connection connection, MigrationInfo info); + default void afterEachMigrate(Connection connection, MigrationInfo info) {} /** * Runs before the validate task executes. * * @param connection A valid connection to the database. */ - void beforeValidate(Connection connection); + default void beforeValidate(Connection connection) {} /** * Runs after the validate task executes. * * @param connection A valid connection to the database. */ - void afterValidate(Connection connection); + default void afterValidate(Connection connection) {} /** * Runs before the baseline task executes. * * @param connection A valid connection to the database. */ - void beforeBaseline(Connection connection); + default void beforeBaseline(Connection connection) {} /** * Runs after the baseline task executes. * * @param connection A valid connection to the database. */ - void afterBaseline(Connection connection); + default void afterBaseline(Connection connection) {} /** * Runs before the repair task executes. * * @param connection A valid connection to the database. */ - void beforeRepair(Connection connection); + default void beforeRepair(Connection connection) {} /** * Runs after the repair task executes. * * @param connection A valid connection to the database. */ - void afterRepair(Connection connection); + default void afterRepair(Connection connection) {} /** * Runs before the info task executes. * * @param connection A valid connection to the database. */ - void beforeInfo(Connection connection); + default void beforeInfo(Connection connection) {} /** * Runs after the info task executes. * * @param connection A valid connection to the database. */ - void afterInfo(Connection connection); + default void afterInfo(Connection connection) {} } From 3a2cdd38c17e197cff8e270620e41b6b02f995e9 Mon Sep 17 00:00:00 2001 From: bgehrels Date: Wed, 18 Jul 2018 10:17:48 +0200 Subject: [PATCH 09/18] LBJ-982 verify that flyway config is set *before* the callbacks are called --- .../nakadiproducer/NakadiProducerFlywayCallbackIT.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NakadiProducerFlywayCallbackIT.java b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NakadiProducerFlywayCallbackIT.java index 5e5caa8d..5d2e6275 100644 --- a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NakadiProducerFlywayCallbackIT.java +++ b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NakadiProducerFlywayCallbackIT.java @@ -1,6 +1,7 @@ package org.zalando.nakadiproducer; import static org.mockito.Matchers.any; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -9,6 +10,7 @@ import org.flywaydb.core.api.configuration.ConfigurationAware; import org.flywaydb.core.api.configuration.FlywayConfiguration; import org.junit.Test; +import org.mockito.InOrder; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.test.annotation.DirtiesContext; @@ -29,7 +31,10 @@ public void flywayCallbackIsCalledIfAnnotatedWithQualifierAnnotation() { @Test @DirtiesContext // Needed to make sure that flyway gets executed for each of the tests and Callbacks are called again public void flywayConfigurationIsSetIfCallbackIsConfigurationAware() { - verify(configurationAwareNakadiProducerFlywayCallback).setFlywayConfiguration(any(FlywayConfiguration.class)); + InOrder inOrder = inOrder(configurationAwareNakadiProducerFlywayCallback); + inOrder.verify(configurationAwareNakadiProducerFlywayCallback).setFlywayConfiguration(any(FlywayConfiguration.class)); + inOrder.verify(configurationAwareNakadiProducerFlywayCallback, times(1)).beforeMigrate(any(Connection.class)); + } public interface ConfigurationAwareNakadiProducerFlywayCallback extends NakadiProducerFlywayCallback, ConfigurationAware { From 09c486a94ee04de1c4c44921e326d7092ef9f904 Mon Sep 17 00:00:00 2001 From: bgehrels Date: Wed, 18 Jul 2018 10:18:40 +0200 Subject: [PATCH 10/18] LBJ-982 verify that flyway config is set *before* the callbacks are called --- .../nakadiproducer/NonNakadiProducerFlywayCallbackIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NonNakadiProducerFlywayCallbackIT.java b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NonNakadiProducerFlywayCallbackIT.java index ce023b00..e6f19af7 100644 --- a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NonNakadiProducerFlywayCallbackIT.java +++ b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/NonNakadiProducerFlywayCallbackIT.java @@ -36,6 +36,6 @@ public void flywayCallbacksFromOurHostApplicationAreNotUsedByUs() { public void ourOwnFlywayConfigurationStillWorksFineWhenSpringsFlywayAutoconfigIsDisabled() { // Yes, this is redundant to the other test in here. // We consider it important to document the requirement, so it is here nonetheless. - // This test does just enough to test it. + // The test setup done by the class annotations does just enough to test it } } From 7da346b74b7385475a7ab81d1fa0f2e8ecd23b16 Mon Sep 17 00:00:00 2001 From: bgehrels Date: Wed, 18 Jul 2018 10:52:12 +0200 Subject: [PATCH 11/18] LBJ-982 add a test for filter specification in the request body --- .../SnapshotEventGenerationWebEndpointIT.java | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGenerationWebEndpointIT.java b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGenerationWebEndpointIT.java index 2846c2bd..02f38546 100644 --- a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGenerationWebEndpointIT.java +++ b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGenerationWebEndpointIT.java @@ -21,13 +21,18 @@ @RunWith(SpringRunner.class) @SpringBootTest( webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - properties = { "management.security.enabled=false", "zalando.team.id:alpha-local-testing", "nakadi-producer.scheduled-transmission-enabled:false", "management.endpoints.web.exposure.include:snapshot-event-creation" }, + properties = { + "management.security.enabled=false", + "zalando.team.id:alpha-local-testing", + "nakadi-producer.scheduled-transmission-enabled:false", + "management.endpoints.web.exposure.include:snapshot-event-creation" + }, classes = { TestApplication.class, EmbeddedDataSourceConfig.class, SnapshotEventGenerationWebEndpointIT.Config.class } ) public class SnapshotEventGenerationWebEndpointIT { private static final String MY_EVENT_TYPE = "my.event-type"; - private static final String filter = "myRequestBody"; + private static final String FILTER = "myRequestBody"; @LocalManagementPort private int managementPort; @@ -36,13 +41,24 @@ public class SnapshotEventGenerationWebEndpointIT { private SnapshotEventGenerator snapshotEventGenerator; @Test - public void passesFilterIfPresent() { + public void passesFilterIfPresentInUrl() { given().baseUri("http://localhost:" + managementPort) .contentType("application/json") - .when().post("/actuator/snapshot-event-creation/" + MY_EVENT_TYPE + "?filter=" + filter) + .when().post("/actuator/snapshot-event-creation/" + MY_EVENT_TYPE + "?filter=" + FILTER) .then().statusCode(204); - verify(snapshotEventGenerator).generateSnapshots(null, filter); + verify(snapshotEventGenerator).generateSnapshots(null, FILTER); + } + + @Test + public void passesFilterIfPresentInBody() { + given().baseUri("http://localhost:" + managementPort) + .contentType("application/json") + .body("{\"filter\":\"" + FILTER + "\"}") + .when().post("/actuator/snapshot-event-creation/" + MY_EVENT_TYPE) + .then().statusCode(204); + + verify(snapshotEventGenerator).generateSnapshots(null, FILTER); } @Test From 501bc6a35321f5a11fb8381a9e7db2783c18ed48 Mon Sep 17 00:00:00 2001 From: bgehrels Date: Wed, 18 Jul 2018 10:53:49 +0200 Subject: [PATCH 12/18] LBJ-982 more precise docs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 663f9941..bd8831ac 100644 --- a/README.md +++ b/README.md @@ -230,7 +230,7 @@ will trigger a snapshot for the event type `my.event-type`. You can change the p path prefix as part of your Spring Boot Actuator configuration. You can provide an optional filter specifier that will be passed to your application to implement any application -specific event/entity filtering logic. It can be provided either as a request parameter called `filter`, or as a +specific event/entity filtering logic. It can be provided either as a query parameter called `filter`, or as a request body {"filter":"myFilter"} From e92b70587de0d580c23e667afb1835e9c02580f5 Mon Sep 17 00:00:00 2001 From: bgehrels Date: Wed, 18 Jul 2018 10:55:30 +0200 Subject: [PATCH 13/18] LBJ-982 use lambda instead of inner class --- .../nakadiproducer/tests/Application.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/nakadi-producer-starter-spring-boot-2-test/src/main/java/org/zalando/nakadiproducer/tests/Application.java b/nakadi-producer-starter-spring-boot-2-test/src/main/java/org/zalando/nakadiproducer/tests/Application.java index 9a874c57..e80b3cca 100644 --- a/nakadi-producer-starter-spring-boot-2-test/src/main/java/org/zalando/nakadiproducer/tests/Application.java +++ b/nakadi-producer-starter-spring-boot-2-test/src/main/java/org/zalando/nakadiproducer/tests/Application.java @@ -39,16 +39,13 @@ public EmbeddedPostgres embeddedPostgres() throws IOException { @Bean public SnapshotEventGenerator snapshotEventGenerator() { - return new SimpleSnapshotEventGenerator("eventtype", new BiFunction>() { - @Override - public List apply(Object withIdGreaterThan, String filter) { - if (withIdGreaterThan == null) { - return Collections.singletonList(new Snapshot("1", "foo", (Object) filter)); - } else if (withIdGreaterThan.equals("1")) { - return Collections.singletonList(new Snapshot("2", "foo", (Object) filter)); - } else { - return new ArrayList<>(); - } + return new SimpleSnapshotEventGenerator("eventtype", (withIdGreaterThan, filter) -> { + if (withIdGreaterThan == null) { + return Collections.singletonList(new Snapshot("1", "foo", (Object) filter)); + } else if (withIdGreaterThan.equals("1")) { + return Collections.singletonList(new Snapshot("2", "foo", (Object) filter)); + } else { + return new ArrayList<>(); } }); From 9b2bb3749edd3bf41db8a4e89b8c2fe154ff511a Mon Sep 17 00:00:00 2001 From: bgehrels Date: Wed, 18 Jul 2018 11:02:29 +0200 Subject: [PATCH 14/18] moved mok k8s token config to src/test --- .../java/org/zalando/nakadiproducer/tests/ApplicationIT.java | 2 +- .../src/{main => test}/resources/tokens/nakadi-token-secret | 0 .../src/{main => test}/resources/tokens/nakadi-token-type | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename nakadi-producer-starter-spring-boot-2-test/src/{main => test}/resources/tokens/nakadi-token-secret (100%) rename nakadi-producer-starter-spring-boot-2-test/src/{main => test}/resources/tokens/nakadi-token-type (100%) diff --git a/nakadi-producer-starter-spring-boot-2-test/src/test/java/org/zalando/nakadiproducer/tests/ApplicationIT.java b/nakadi-producer-starter-spring-boot-2-test/src/test/java/org/zalando/nakadiproducer/tests/ApplicationIT.java index b0344459..cce10fd9 100644 --- a/nakadi-producer-starter-spring-boot-2-test/src/test/java/org/zalando/nakadiproducer/tests/ApplicationIT.java +++ b/nakadi-producer-starter-spring-boot-2-test/src/test/java/org/zalando/nakadiproducer/tests/ApplicationIT.java @@ -28,7 +28,7 @@ public class ApplicationIT { @BeforeClass public static void fakeCredentialsDir() { - environmentVariables.set("CREDENTIALS_DIR", new File("src/main/resources/tokens").getAbsolutePath()); + environmentVariables.set("CREDENTIALS_DIR", new File("src/main/test/tokens").getAbsolutePath()); } @Test diff --git a/nakadi-producer-starter-spring-boot-2-test/src/main/resources/tokens/nakadi-token-secret b/nakadi-producer-starter-spring-boot-2-test/src/test/resources/tokens/nakadi-token-secret similarity index 100% rename from nakadi-producer-starter-spring-boot-2-test/src/main/resources/tokens/nakadi-token-secret rename to nakadi-producer-starter-spring-boot-2-test/src/test/resources/tokens/nakadi-token-secret diff --git a/nakadi-producer-starter-spring-boot-2-test/src/main/resources/tokens/nakadi-token-type b/nakadi-producer-starter-spring-boot-2-test/src/test/resources/tokens/nakadi-token-type similarity index 100% rename from nakadi-producer-starter-spring-boot-2-test/src/main/resources/tokens/nakadi-token-type rename to nakadi-producer-starter-spring-boot-2-test/src/test/resources/tokens/nakadi-token-type From e6ac4511fc63de27dc5438fc730e3821410121de Mon Sep 17 00:00:00 2001 From: bgehrels Date: Wed, 18 Jul 2018 11:07:14 +0200 Subject: [PATCH 15/18] fixed broken IT --- .../snapshots/SnapshotEventGenerationWebEndpointIT.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGenerationWebEndpointIT.java b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGenerationWebEndpointIT.java index 02f38546..c61f4c80 100644 --- a/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGenerationWebEndpointIT.java +++ b/nakadi-producer-spring-boot-starter/src/test/java/org/zalando/nakadiproducer/snapshots/SnapshotEventGenerationWebEndpointIT.java @@ -2,9 +2,11 @@ import static com.jayway.restassured.RestAssured.given; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; @@ -40,6 +42,11 @@ public class SnapshotEventGenerationWebEndpointIT { @Autowired private SnapshotEventGenerator snapshotEventGenerator; + @Before + public void resetMocks() { + reset(snapshotEventGenerator); + } + @Test public void passesFilterIfPresentInUrl() { given().baseUri("http://localhost:" + managementPort) From 9f0993f5d2043c2d58d141076342387ad1c0d10d Mon Sep 17 00:00:00 2001 From: bgehrels Date: Wed, 18 Jul 2018 11:33:39 +0200 Subject: [PATCH 16/18] documented why i choose some weird variable name --- .../snapshots/impl/SnapshotEventCreationEndpoint.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/snapshots/impl/SnapshotEventCreationEndpoint.java b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/snapshots/impl/SnapshotEventCreationEndpoint.java index 404bafb0..ef423054 100644 --- a/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/snapshots/impl/SnapshotEventCreationEndpoint.java +++ b/nakadi-producer-spring-boot-starter/src/main/java/org/zalando/nakadiproducer/snapshots/impl/SnapshotEventCreationEndpoint.java @@ -25,7 +25,12 @@ public SnapshotReport getSupportedEventTypes() { } @WriteOperation - public void createFilteredSnapshotEvents(@Selector String arg0, @Nullable String filter) { + public void createFilteredSnapshotEvents( + // this is the event type. Could have a better name, but since Spring Boot relies on the -parameters + // compiler flag being set to resolve path parameter names, it would then get trickier to reliably run this + // Test in the IDE. So let's stick with arg0 for now. + @Selector String arg0, + @Nullable String filter) { snapshotCreationService.createSnapshotEvents(arg0, filter); } From 21a6fd6c265df8e624e06a0ebbec7937f66c5eaa Mon Sep 17 00:00:00 2001 From: bgehrels Date: Wed, 18 Jul 2018 12:01:19 +0200 Subject: [PATCH 17/18] removed unused imports --- .../main/java/org/zalando/nakadiproducer/tests/Application.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/nakadi-producer-starter-spring-boot-2-test/src/main/java/org/zalando/nakadiproducer/tests/Application.java b/nakadi-producer-starter-spring-boot-2-test/src/main/java/org/zalando/nakadiproducer/tests/Application.java index e80b3cca..30929521 100644 --- a/nakadi-producer-starter-spring-boot-2-test/src/main/java/org/zalando/nakadiproducer/tests/Application.java +++ b/nakadi-producer-starter-spring-boot-2-test/src/main/java/org/zalando/nakadiproducer/tests/Application.java @@ -15,8 +15,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.List; -import java.util.function.BiFunction; @EnableAutoConfiguration @EnableNakadiProducer From d5f3c582c07c7598e7b430c50ac285f54b5bd372 Mon Sep 17 00:00:00 2001 From: Paul Ebermann Date: Wed, 18 Jul 2018 13:32:23 +0200 Subject: [PATCH 18/18] Some typo fixes in the README --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index bd8831ac..f624cc61 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ The goal of this Spring Boot starter is to simplify the reliable integration bet There are already [multiple clients for the Nakadi REST API](https://zalando.github.io/nakadi/manual.html#using_clients), but none of them solves the mentioned issues. -We solved them by persisting new events in a log table as part of the producing JDBC transaction. They will then be sent asynchonously to Nakadi after the transaction completed. If the transaction is rolled back, the events will vanish too. As a result, events will always be sent if and only if the transaction succeeded. +We solved them by persisting new events in a log table as part of the producing JDBC transaction. They will then be sent asynchronously to Nakadi after the transaction completed. If the transaction is rolled back, the events will vanish too. As a result, events will always be sent if and only if the transaction succeeded. The Transmitter generates a strictly monotonically increasing event id that can be used for ordering the events during retrieval. It is not guaranteed, that events will be sent to Nakadi in the order they have been produced. If an event could not be sent to Nakadi, the library will periodically retry the transmission. @@ -51,14 +51,14 @@ This library also uses: * jackson >= 2.7.0 * (optional) Zalando's [tracer-spring-boot-starter](https://github.com/zalando/tracer) * (optional) Zalando's [tokens library](https://github.com/zalando/tokens) >= 0.10.0 - * Please note that [tokens-spring-boot-starter](https://github.com/zalando-stups/spring-boot-zalando-stups-tokens) 0.10.0 comes with tokens 0.9.9, which is not enough. You can manually add tokens 0.10.0 with that starter, though. To be used in zalandos k8s environment, you must at least use 0.11.0. + * Please note that [tokens-spring-boot-starter](https://github.com/zalando-stups/spring-boot-zalando-stups-tokens) 0.10.0 comes with tokens 0.9.9, which is not enough. You can manually add tokens 0.10.0 with that starter, though. To be used in zalando's k8s environment, you must at least use 0.11.0. ## Usage ### Setup -If you are using maven, include the library in your `pom.xml`: +If you are using Maven, include the library in your `pom.xml`: ```xml @@ -82,7 +82,7 @@ public class Application { } ``` -The library uses flyway migrations to set up its own database schema `nakadi_events`. +The library uses Flyway migrations to set up its own database schema `nakadi_events`. ### Nakadi communication configuration @@ -212,7 +212,7 @@ process step the event is reporting. ### Event snapshots (optional) A Snapshot event is a special type of data change event (data operation) defined by Nakadi. -It does not represent a change of the state of a resource, but a current snapshot of its state. It can be usefull to +It does not represent a change of the state of a resource, but a current snapshot of its state. It can be useful to bootstrap a new consumer or to recover from inconsistencies between sender and consumer after an incident. You can create snapshot events programmatically (using EventLogWriter.fireSnapshotEvent), but usually snapshot event @@ -283,7 +283,7 @@ tracer: By default, the library will pick up your flyway data source (or the primary data source if no flyway data source is configured), create its own schema and start setting up its tables in there. You can customize this process in two ways: -If you want to use a different data source for schema maintainence (for example to use a different username) and +If you want to use a different data source for schema maintenance (for example to use a different username) and configuring the Spring Flyway datasource is not enough, your can define a spring bean of type `DataSource` and annotate it with `@NakadiProducerDataSource`.