diff --git a/README.adoc b/README.adoc
index d8b05de..9f05689 100644
--- a/README.adoc
+++ b/README.adoc
@@ -9,7 +9,7 @@ This repo contains the source code of the https://www.maoudia.com[maoudia.com] w
built using https://gohugo.io/[Hugo] + https://github.com/Lednerb/bilberry-hugo-theme[Billeberry] theme and hosted on https://maoudia.pages.dev/[CloudFlare Pages] + https://pages.github.com/[GitHub Pages].
== Requirements
-1. https://gohugo.io/getting-started/installing/[Hugo] (0.98.x)
+1. https://gohugo.io/getting-started/installing/[Hugo] (0.100.x)
2. https://www.ruby-lang.org/en/documentation/installation/[Ruby] (2.7.x)
3. https://nodejs.org/[NodeJS] (16.15.x)
4. https://asciidoctor.org/[Asciidoctor] with required https://asciidoctor.org/docs/extensions/[extensions]:
diff --git a/archetypes/article.adoc b/archetypes/article.adoc
index 3e5e868..371a669 100644
--- a/archetypes/article.adoc
+++ b/archetypes/article.adoc
@@ -6,10 +6,12 @@ lastmod = 2021-05-19T08:30:00+01:00
showPublishDate = true
showLastModificationDate = true
showReadingTime = true
+readingtime = 10
+showAuthor = true
images = ["featuredImage.jpg"]
categories = ["Tutorial", "Tip"]
tags = ["Java", "Kotlin", "Android", "Hugo", "SpringBoot", "GraphQL", "Programming", "Quarkus"]
-project_url = "https://github.com/gohugoio/hugo"
+project_url = "https=//github.com/gohugoio/hugo"
author = "Moncef AOUDIA"
series = []
audio = []
@@ -19,8 +21,7 @@ resizeImages = false
excludeFromIndex = false
draft = true
slug = "/"
-exclude = false
+exclude : false
+++
-
diff --git a/assets/css/4-aoudiamoncef.css b/assets/css/4-aoudiamoncef.css
index 052eba2..2b1a776 100644
--- a/assets/css/4-aoudiamoncef.css
+++ b/assets/css/4-aoudiamoncef.css
@@ -21,10 +21,18 @@ h1, h2, h3, h4, h5, h6 {
}
.bilberry-hugo-theme article .content {
- padding: 0em;
+ padding: 0;
margin: 1.5em;
}
+.bilberry-hugo-theme article .content h2 {
+ font-size: 1.8em;
+}
+
+.bilberry-hugo-theme article .content h3 {
+ font-size: 1.4em;
+}
+
.bilberry-hugo-theme header {
background-color: #000a12;
}
@@ -172,19 +180,19 @@ h1, h2, h3, h4, h5, h6 {
}
.bilberry-hugo-theme footer .container ul a {
- color: #2b5c00;
+ color: #377300;
}
.bilberry-hugo-theme footer .container .right .languages a {
- color: #2b5c00;
+ color: #377300;
}
.bilberry-hugo-theme .credits .container .author {
- color: #2b5c00;
+ color: #377300;
}
.bilberry-hugo-theme .credits .container a {
- color: #2b5c00;
+ color: #377300;
}
.bilberry-hugo-theme footer .container a:hover {
@@ -207,13 +215,13 @@ h1, h2, h3, h4, h5, h6 {
}
.bilberry-hugo-theme footer .container .right .languages span {
- color: #2b5c00;
+ color: #377300;
}
.bilberry-hugo-theme header .logo img {
max-width: 100%;
border-radius: 50%;
- height: auto;
+ height: inherit;
}
.bilberry-hugo-theme pre code {
@@ -235,6 +243,10 @@ h1, h2, h3, h4, h5, h6 {
height: auto;
}
+.bilberry-hugo-theme article .content .meta {
+ color: #737373;
+}
+
.bilberry-hugo-theme article .content img {
width: 100%;
height: auto;
@@ -242,6 +254,27 @@ h1, h2, h3, h4, h5, h6 {
margin: 0 auto;
}
+.bilberry-hugo-theme article .content.schema img {
+ width: 60%;
+}
+
+@media (max-width: 1200px) {
+ .bilberry-hugo-theme article .content.schema img {
+ width: 70%;
+ }
+}
+
+@media (max-width: 1024px) {
+ .bilberry-hugo-theme article .content.schema img {
+ width: 80%;
+ }
+}
+@media (max-width: 768px) {
+ .bilberry-hugo-theme article .content.schema img {
+ width: 100%;
+ }
+}
+
.badge .image {
display: inline-table;
}
@@ -292,6 +325,32 @@ details.badge summary {
font-size: 0.95em;
}
-.content .listingblock .content {
+.bilberry-hugo-theme .content .listingblock .content {
margin: 0;
}
+
+.bilberry-hugo-theme .container {
+ width: 1200px;
+ max-width: 100%;
+}
+
+.bilberry-hugo-theme ul {
+ padding-left: revert;
+}
+
+.bilberry-hugo-theme ol li {
+ list-style: decimal;
+}
+
+.bilberry-hugo-theme ol {
+ padding-left: revert;
+ margin-top: 0;
+}
+
+.bilberry-hugo-theme .ulist ul {
+ padding-left: inherit;
+}
+
+.bilberry-hugo-theme .toc ul li ul {
+ padding-left: inherit;
+}
diff --git a/config/_default/config.toml b/config/_default/config.toml
index a2db448..e7f1b7a 100644
--- a/config/_default/config.toml
+++ b/config/_default/config.toml
@@ -31,17 +31,17 @@ disableAliases = false
[markup]
[markup.tableOfContents]
- endLevel = 5
- ordered = false
+ endLevel = 4
+ ordered = true
startLevel = 2
[markup.asciidocExt]
backend = "html5"
extensions = ["asciidoctor-html5s"]
failureLevel = "fatal"
noHeaderOrFooter = true
- preserveTOC = false
+ preserveTOC = true
safeMode = "unsafe"
- sectionNumbers = false
+ sectionNumbers = true
trace = false
verbose = false
workingFolderCurrent = false
diff --git a/config/_default/params.toml b/config/_default/params.toml
index f976cd9..1c86b6a 100644
--- a/config/_default/params.toml
+++ b/config/_default/params.toml
@@ -34,7 +34,7 @@ resizeImages = false
gravatarEMail = "mf.aoudia@gmail.com"
# avatarEmail = "mf.aoudia@gmail.com"
# set an path to the image file you want to use | overwrites gravatar
-customImage = "/images/favicons/splash.webp"
+customImage = "/images/logo.png"
# define the icon you want to use for the overlay for the customImage or gravatar.
overlayIcon = "fa-home"
# always display the top navigation (with pages and search) on non-mobile screens
@@ -65,6 +65,8 @@ showReadingTime = true
showPublishDate = true
showLastModificationDate = true
+
+showAuthor = true
# Minimum word count to display the Table of Contents
tocMinWordCount = 400
# Footer configuration
diff --git a/content/blog/bulk-update-with-spring-data-mongodb-reactive/featuredImage.fr.png b/content/blog/bulk-update-with-spring-data-mongodb-reactive/featuredImage.fr.png
new file mode 100644
index 0000000..6fd5604
Binary files /dev/null and b/content/blog/bulk-update-with-spring-data-mongodb-reactive/featuredImage.fr.png differ
diff --git a/content/blog/bulk-update-with-spring-data-mongodb-reactive/featuredImage.png b/content/blog/bulk-update-with-spring-data-mongodb-reactive/featuredImage.png
new file mode 100644
index 0000000..ec122dc
Binary files /dev/null and b/content/blog/bulk-update-with-spring-data-mongodb-reactive/featuredImage.png differ
diff --git a/content/blog/bulk-update-with-spring-data-mongodb-reactive/index.adoc b/content/blog/bulk-update-with-spring-data-mongodb-reactive/index.adoc
new file mode 100644
index 0000000..760cae4
--- /dev/null
+++ b/content/blog/bulk-update-with-spring-data-mongodb-reactive/index.adoc
@@ -0,0 +1,649 @@
++++
+title = "Bulk Update With Spring Data MongoDB Reactive"
+date = 2022-06-20T00:00:00+02:00
+description = "We will implement a solution to enrich and update efficiently a large amount of data using Spring Data MongoDB Reactive."
+author = "Moncef AOUDIA"
+showAuthor = false
+showReadingTime = true
+readingtime = 10
+tags = ["EIP", "Java", "Reactor", "MongoDB", "Spring Boot", "Spring Data", "Spring WebFlux", "Docker Compose", "TestContainers"]
+series = ["MongoDB Reactive CLI"]
+categories = ["Tutorial", "Reactive Programming"]
+slug = "bulk-update-with-spring-data-mongodb-reactive"
+type = "article"
+[twitter]
+ card = "summary_large_image"
+ site = "@AoudiaMoncef"
+ creator = "@AoudiaMoncef"
+ title = "Bulk Update With Spring Data MongoDB Reactive"
+ description = "We will implement a solution to enrich and update efficiently a large amount of data using Spring Data MongoDB Reactive."
+ image = "https://www.maoudia.com/blog/bulk-update-with-spring-data-mongodb-reactive/featuredImage.png"
++++
+
+:toc: macro
+:toc-title: Table of contents
+:toclevels: 4
+:imagesdir: /images/blog/bulk-update-with-spring-data-mongodb-reactive
+ifdef::env-github[]
+:imagesdir: ../../static/images/bulk-update-with-spring-data-mongodb-reactive
+endif::[]
+
+In order to update documents in a MongoDB collection, we often use update requests, if the volume of data is too large,
+it could lead to performance issues and overconsumption of hardware resources.
+
+We will implement a solution to enrich and update efficiently a large amount of data
+using Spring Data MongoDB Reactive.
+
+
+
+toc::[]
+
+Before continuing the reading, if you are not familiar with Spring reactive stack and MongoDB,
+I suggest you to check the *resources* section.
+
+== EIP content enricher
+
+++++
+
+
+
+
+
+++++
+
+Enterprise Integration Pattern _Content Enricher_ appends information to an existing message from an external source.
+It uses information inside the incoming message to perform the enrichment operation.
+
+We will implement a simplified version of the EIP :
+
+. Input message : represented by a MongoDB document.
+. Enricher : our application.
+. Resource : call to a RESTful API.
+. Output message : we will keep only the enriched document.
+
+=== Integration flow
+
+++++
+
+
+
+
+
+++++
+
+The application will read the address documents, add the product and save the enriched documents to the MongoDB database.
+
+== Project setup
+
+=== Requirements
+
+* Java 1.8+
+* Maven 3+
+* Docker Compose
+* MongoDB Database Tools
+
+=== Generation
+
+We generate the project skeleton from https://start.spring.io/#!type=maven-project&language=java&platformVersion=2.7.0&packaging=jar&jvmVersion=1.8&groupId=com.maoudia&artifactId=bulk-update-with-spring-data-mongodb&name=Bulk%20Update%20with%20Spring%20Data%20MongoDB%20Reactive&description=Bulk%20Update%20with%20Spring%20data%20MongoDB%20reactive&packageName=com.maoudia.tutorial&dependencies=data-mongodb-reactive,webflux,testcontainers[`Spring Initializr`, window=\"_blank\"].
+
+=== Structure
+
+[source,shell,indent=0,linenums=true]
+----
+.
+│ .gitignore
+│ docker-compose.yml
+│ pom.xml
+│ README.adoc
+├───data
+│ ├───mongodb
+│ │ address.ndjson
+│ └───product
+│ db.json
+└───src
+ ├───main
+ │ ├───java
+ │ │ └───com
+ │ │ └───maoudia
+ │ │ └───tutorial
+ │ │ Application.java
+ │ │ AppProperties.java
+ │ │ CollectionService.java
+ │ │ NetworkConfig.java
+ │ └───resources
+ │ application.yml
+ └───test
+ └───java
+ └───com
+ └───maoudia
+ └───tutorial
+ CollectionServiceTest.java
+----
+
+=== Containers
+
+Download https://github.com/maoudia/code.maoudia.com/tree/main/bulk-update-with-spring-data-mongodb-reactive/data[`data`] directory to the root of the project.
+
+We use `docker-compose` to create the needed containers for this tutorial.
+
+[source,yml,indent=0,linenums=true]
+.docker-compose.yml
+----
+services:
+ mongodb: // <1>
+ container_name: maoudia-mongodb
+ image: mongo:5.0.8
+ environment:
+ - MONGO_INITDB_DATABASE=test
+ - MONGO_INITDB_ROOT_USERNAME=admin
+ - MONGO_INITDB_ROOT_PASSWORD=password
+ networks:
+ - mongodb-network
+ ports:
+ - 15015:27017
+ volumes:
+ - ./data/mongodb:/data/mongodb
+
+ mongo-express: // <2>
+ container_name: maoudia-mongo-express
+ image: mongo-express:0.54.0
+ depends_on:
+ - mongodb
+ networks:
+ - mongodb-network
+ environment:
+ - ME_CONFIG_MONGODB_SERVER=maoudia-mongodb
+ - ME_CONFIG_MONGODB_ADMINUSERNAME=admin
+ - ME_CONFIG_MONGODB_ADMINPASSWORD=password
+ ports:
+ - 1515:8081
+ volumes:
+ - ./data/mongodb:/data/mongodb
+
+ product-api: // <3>
+ container_name: maoudia-product-api
+ image: clue/json-server:latest
+ ports:
+ - 1519:80
+ volumes:
+ - ./data/product/db.json:/data/db.json
+
+networks:
+ mongodb-network:
+ driver: bridge
+----
+
+<1> MongoDB initialized with the `test` database.
+<2> MongoExpress is a MongoDB administration interface.
+<3> Product API which is configured from `db.json` file.
+
+
+We start up the services :
+
+[source,shell,indent=0,linenums=true]
+----
+docker-compose up -d
+----
+
+=== Data initialization
+
+We use a JSON document from the French address database.
+
+.Address
+[source,json,indent=0,linenums=true]
+----
+{
+ "id": "59350",
+ "type": "municipality",
+ "name": "Lille",
+ "postcode": [
+ "59000",
+ "59800",
+ "59260",
+ "59777",
+ "59160"
+ ],
+ "citycode": "59350",
+ "x": 703219.96,
+ "y": 7059335.72,
+ "lon": 3.045433,
+ "lat": 50.630992,
+ "population": 234475,
+ "city": "Lille",
+ "context": "59, Nord, Hauts-de-France",
+ "importance": 0.56333
+}
+----
+
+Import address collection :
+
+[source,shell,indent=0,linenums=true]
+----
+mongoimport --uri "mongodb://admin:password@localhost:15015" --authenticationDatabase=admin --db test --collection address ./data/mongodb/address.ndjson
+----
+
+Ou :
+
+We use *MongoExpress* which is available at http://localhost:1515[`http://localhost:1515`].
+
+Product represents a satellite internet offer.
+
+.Product
+[source,json,indent=0,linenums=true]
+----
+{
+ "id": 1,
+ "available": true,
+ "company": "SPACEX",
+ "provider": "STARLINK",
+ "type": "SATELLITE"
+}
+----
+
+Product API is available at http://localhost:1519[`http://localhost:1519`].
+
+== Application
+
+=== Configuration
+
+We change file extension from `application.properties` to `application.yml`.
+
+[source,yml,indent=0,linenums=true]
+.application.yml
+----
+app:
+ buffer-max-size: 500
+ bulk-size: 100
+ collection-name: address
+ enriching-key: product
+ enriching-uri: http://localhost:1519/products/1
+spring:
+ main:
+ web-application-type: none
+ data:
+ mongodb:
+ database: test
+ uri: mongodb://admin:password@localhost:15015
+---
+spring.config.activate.on-profile: dev
+logging:
+ level:
+ org.mongodb.driver: debug
+---
+spring.config.activate.on-profile: test
+app:
+ bulk-size: 2
+----
+
+We declare a class which contains application configuration properties.
+
+[source,java,indent=0,linenums=true]
+.AppProperties.java
+----
+@ConfigurationProperties(prefix = "app")
+public class AppProperties {
+ private int bulkSize;
+ private int bufferMaxSize;
+ private String collectionName;
+ private String enrichingKey;
+ private String enrichingUri;
+ // Getter and Setter are omitted
+}
+----
+
+We create a `@Bean` of Spring non-blocking HTTP client.
+
+[source,java,indent=0,linenums=true]
+.NetworkConfig.java
+----
+@Configuration
+public class NetworkConfig {
+
+ @Bean
+ public WebClient client() {
+ return WebClient.create();
+ }
+
+}
+----
+
+=== Implementation
+
+We create a `@Service` which contains application business logic.
+
+[source,java,indent=0,linenums=true]
+.CollectionService.java
+----
+@Service
+public class CollectionService {
+ private final AppProperties properties;
+ private final ReactiveMongoTemplate template;
+ private final WebClient client;
+
+ public CollectionService(AppProperties properties,
+ ReactiveMongoTemplate template,
+ WebClient client) {
+ this.properties = properties;
+ this.template = template;
+ this.client = client;
+ }
+
+ public Flux enrichAll(String collectionName, String enrichingKey, String enrichingUri) {
+ return template.findAll(Document.class, collectionName) // <1>
+ .onBackpressureBuffer(properties.getBufferMaxSize()) // <2>
+ .flatMap(document -> enrich(document, enrichingKey, enrichingUri)) // <3>
+ .map(CollectionService::toReplaceOneModel) // <4>
+ .window(properties.getBulkSize()) // <5>
+ .flatMap(replaceOneModelFlux -> bulkWrite(replaceOneModelFlux, collectionName)); // <6>
+ }
+}
+----
+
+<1> Creates a stream of documents from the collection.
+<2> Limits the maximum number of loaded documents in the _RAM_ in case of consumption process is slower than production.
+If the maximum buffer size is exceeded, an `IllegalStateException` is thrown.
+<3> Enriches document asynchronously with the external one.
+<4> Creates a `ReplaceOneModel` from document.
+<5> Group documents into streams of fixed size. The last stream can be smaller.
+<6> Calls bulk write function.
+
+[NOTE]
+====
+Configuration property `app.bulk-size` can be adjusted according to the project needs and available hardware resources.
+The larger the value of the maximum size, the higher the memory consumption and the size of the requests.
+====
+
+We create document enrichment functions.
+
+[source,java,indent=0,linenums=true]
+.CollectionService.java
+----
+private Publisher enrich(Document document, String enrichingKey, String enrichingUri) { // <1>
+ return getEnrichingDocument(enrichingUri)
+ .map(enrichingDocument -> {
+ document.put(enrichingKey, enrichingDocument);
+ document.put("updatedAt", new Date());
+ return document;
+ });
+}
+
+private Mono getEnrichingDocument(String enrichingUri) { // <2>
+ return client.get()
+ .uri(URI.create(enrichingUri))
+ .retrieve()
+ .bodyToMono(Document.class);
+}
+----
+
+<1> Adds the retrieved document from HTTP call to root of document to be enriched with the key passed in parameter.
+<2> Retrieves a document from an URI.
+
+[NOTE]
+====
+MongoDB converts and stores dates in UTC by default.
+====
+
+
+[source,java,indent=0,linenums=true]
+.CollectionService.java
+----
+private static final ReplaceOptions REPLACE_OPTIONS = new ReplaceOptions(); // <1>
+private static ReplaceOneModel toReplaceOneModel (Document document) {
+ return new ReplaceOneModel<>(
+ Filters.eq("_id", document.get("_id")), // <2>
+ document, // <3>
+ REPLACE_OPTIONS
+ );
+}
+----
+
+<1> Instantiates default replacement configuration.
+<2> Filter that allows matching by document identifier.
+<3> Content to be replaced, represents the complete enriched document.
+
+[source,java,indent=0,linenums=true]
+.CollectionService.java
+----
+private static final BulkWriteOptions BULK_WRITE_OPTIONS = new BulkWriteOptions().ordered(false); // <1>
+private Flux bulkWrite(Flux> updateOneModelFlux, String collectionName) {
+ return updateOneModelFlux.collectList() // <2>
+ .flatMapMany(unused -> template.getCollection(collectionName) // <3>
+ .flatMapMany(collection -> collection.bulkWrite(updateOneModels, BULK_WRITE_OPTIONS))); // <4>
+}
+----
+
+<1> Instantiates writing options with disabling operations order.
+<2> Collects the stream into a list.
+<3> Retrieves the collection passed as a parameter.
+<4> Bulk writes documents into MongoDB collection.
+
+[NOTE]
+====
+Transactions are supported on Replicaset since MongoDB 4.2.
+If transactions are enabled, we can use `@Transactional` or `TransactionalOperator` to make a method transactional.
+====
+
+We implement the following interfaces:
+
+* `CommandLineRunner` : runs enrichment command at application startup.
+* `ExitCodeGenerator` : manages application system exit code.
+
+[source,java,indent=0,linenums=true]
+.Application.java
+----
+@SpringBootApplication(exclude = MongoReactiveRepositoriesAutoConfiguration.class) // <1>
+@ConfigurationPropertiesScan("com.maoudia.tutorial") // <2>
+public class Application implements CommandLineRunner, ExitCodeGenerator {
+ private static final Logger LOGGER = LoggerFactory.getLogger(Application.class);
+ private final AppProperties properties;
+ private final CollectionService service;
+ private int exitCode = 255;
+
+ public static void main(String[] args) {
+ System.exit(SpringApplication.exit(SpringApplication.run(Application.class, args)));
+ }
+
+ public Application(AppProperties properties, CollectionService service) {
+ this.properties = properties;
+ this.service = service;
+ }
+
+ @Override
+ public void run(final String... args) {
+ service.enrichAll(properties.getCollectionName(), properties.getEnrichingKey(), properties.getEnrichingUri())
+ .doOnSubscribe(unused -> LOGGER.info("------------------< Staring Collection Enriching Command >-------------------")) // <3>
+ .doOnNext(bulkWriteResult -> LOGGER.info("Bulk write result with {} modified document(s)", bulkWriteResult.getModifiedCount()))
+ .doOnError(throwable -> {
+ exitCode = 1;
+ LOGGER.error("Collection enriching failed due to : {}", throwable.getMessage(), throwable);
+ })
+ .doOnComplete(() -> exitCode = 0)
+ .doOnTerminate(() -> LOGGER.info("------------------< Collection Enriching Command Finished >------------------"))
+ .blockLast(); // <4>
+ }
+
+ @Override
+ public int getExitCode() {
+ return exitCode;
+ }
+
+}
+----
+
+<1> Disables auto-configuration of repositories, as we use `MongoReactiveTemplate` only.
+<2> Allows scanning and detecting beans that carry the `@ConfigProperties` annotation.
+<3> Subscribing to stream triggers the processing.
+<4> Without a running web server, we have to subscribe indefinitely to the `Publisher` in order to trigger
+and wait until the end of the execution.
+
+=== Demo
+
+We launch the application :
+
+[source,shell,indent=0,linenums=true]
+----
+mvn spring-boot:run
+----
+
+Output :
+
+[source,console,indent=0,linenums=true]
+----
+...
+2022-06-10 00:36:45.152 INFO 7036 --- [ main] com.maoudia.tutorial.Application : Started Application in 2.755 seconds (JVM running for 3.251)
+2022-06-10 00:36:45.227 INFO 7036 --- [ main] com.maoudia.tutorial.Application : ------------------< Staring Collection Enriching Command >-------------------
+2022-06-10 00:36:45.297 INFO 7036 --- [ main] org.mongodb.driver.cluster : No server chosen by com.mongodb.reactivestreams.client.internal.ClientSessionHelper$$Lambda$543/543409470@4647881c from cluster description ClusterDescription{type=UNKNOWN, connectionMode=SINGLE, serverDescriptions=[ServerDescription{address=localhost:15015, type=UNKNOWN, state=CONNECTING}]}. Waiting for 30000 ms before timing out
+2022-06-10 00:36:46.527 INFO 7036 --- [localhost:15015] org.mongodb.driver.connection : Opened connection [connectionId{localValue:1, serverValue:39}] to localhost:15015
+2022-06-10 00:36:46.527 INFO 7036 --- [localhost:15015] org.mongodb.driver.connection : Opened connection [connectionId{localValue:2, serverValue:40}] to localhost:15015
+2022-06-10 00:36:46.527 INFO 7036 --- [localhost:15015] org.mongodb.driver.cluster : Monitor thread successfully connected to server with description ServerDescription{address=localhost:15015, type=STANDALONE, state=CONNECTED, ok=true, minWireVersion=0, maxWireVersion=13, maxDocumentSize=16777216, logicalSessionTimeoutMinutes=30, roundTripTimeNanos=61576400}
+2022-06-10 00:36:46.692 INFO 7036 --- [ntLoopGroup-2-3] org.mongodb.driver.connection : Opened connection [connectionId{localValue:3, serverValue:41}] to localhost:15015
+2022-06-10 00:36:48.355 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:48.482 INFO 7036 --- [ntLoopGroup-2-4] org.mongodb.driver.connection : Opened connection [connectionId{localValue:4, serverValue:42}] to localhost:15015
+2022-06-10 00:36:48.562 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:48.742 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:48.982 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:49.222 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:49.488 INFO 7036 --- [ntLoopGroup-2-4] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:49.701 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:49.852 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:50.031 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:50.105 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:50.106 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : ------------------< Collection Enriching Command Finished >------------------
+[INFO] ------------------------------------------------------------------------
+[INFO] BUILD SUCCESS
+[INFO] ------------------------------------------------------------------------
+[INFO] Total time: 17.315 s
+[INFO] Finished at: 2022-06-10T00:36:54+02:00
+[INFO] ------------------------------------------------------------------------
+
+Process finished with exit code 0
+----
+
+=== VisuelVM report
+
+*VisualVM* is a lightweight profiling tool. It is used to have an overview of the threads which are launched by the application.
+
+++++
+
+
+
+
+
+++++
+
+There are two groups of threads that execute operations in parallel, each group forms an _event loop_.
+
+* MongoDB requests are executed by `nioEventLoopGroup`.
+* HTTP requests are executed by `reactor-http-nio`.
+
+== Integration tests
+
+We use *JUnit 5* and the *Testcontainers MongoDB* module for the integration tests.
+It allows to have a feedback close to the real behaviour of the application which essentially do read/write operations.
+
+To keep this tutorial short, we will only write one test.
+
+[source,java,indent=0,linenums=true]
+.CollectionServiceTest.java
+----
+@Profile("test")
+@SpringBootTest
+@Testcontainers // <1>
+class CollectionServiceTest {
+
+ @Container
+ private static final MongoDBContainer mongoDBContainer = new MongoDBContainer("mongo:5.0.8") // <2>
+ .withReuse(true);
+
+ @DynamicPropertySource
+ private static void setProperties(DynamicPropertyRegistry registry) {
+ registry.add("spring.data.mongodb.uri", mongoDBContainer::getReplicaSetUrl); // <3>
+ }
+
+ @Autowired
+ private AppProperties properties;
+ @Autowired
+ private CollectionService command;
+ @Autowired
+ private ReactiveMongoTemplate template;
+
+ @Test
+ void multipleBulkWriteResultsAreReturned() {
+ Document givenDocument1 = new Document();
+ givenDocument1.put("_id", "628ea3edb5110304e5e814f6");
+ givenDocument1.put("type", "municipality");
+ Document givenDocument2 = new Document();
+ givenDocument2.put("_id", "628ea3edb5110304e5e814f7");
+ givenDocument2.put("type", "street");
+ Document givenDocument3 = new Document();
+ givenDocument3.put("_id", "628ea3edb5110304e5e814f8");
+ givenDocument3.put("type", "housenumber");
+
+ template.insert(Arrays.asList(givenDocument1, givenDocument2, givenDocument3), properties.getCollectionName()).blockLast();
+
+ BulkWriteResult expectedBulkWriteResult1 = BulkWriteResult.acknowledged(WriteRequest.Type.REPLACE, 2, 2, Collections.emptyList(),
+ Collections.emptyList());
+ BulkWriteResult expectedBulkWriteResult2 = BulkWriteResult.acknowledged(WriteRequest.Type.REPLACE, 1, 1, Collections.emptyList(),
+ Collections.emptyList());
+
+ command.enrichAll( properties.getCollectionName(), properties.getEnrichingKey() , properties.getEnrichingUri())
+ .as(StepVerifier::create) // <4>
+ .expectNext(expectedBulkWriteResult1)
+ .expectNext(expectedBulkWriteResult2)
+ .verifyComplete();
+ }
+}
+----
+
+<1> Adds TestContainers Junit 5 extension.
+<2> Starts a MongoDB container.
+<3> Sets up application with container's URI.
+<4> Uses `StepVerifier` from *Reactor Test* to assert output stream.
+
+We launch the integration tests :
+
+[source,shell,indent=0,linenums=true]
+----
+mvn test -Dspring.profiles.active=test
+----
+
+Test results :
+
+[source,console,indent=0,linenums=true]
+----
+...
+[INFO] Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 20.563 s - in com.maoudia.tutorial.CollectionServiceTest
+[INFO]
+[INFO] Results:
+[INFO]
+[INFO] Tests run: 1, Failures: 0, Errors: 0, Skipped: 0
+[INFO]
+[INFO] ------------------------------------------------------------------------
+[INFO] BUILD SUCCESS
+[INFO] ------------------------------------------------------------------------
+[INFO] Total time: 32.100 s
+[INFO] Finished at: 2022-06-10T01:02:17+02:00
+[INFO] ------------------------------------------------------------------------
+----
+
+== Conclusion
+
+In this tutorial, we managed to implement a complete solution to enrich and update efficiently a MongoDB collection.
+Moreover, we have seen how to write integration tests with JUnit 5 and Testcontainers.
+
+The complete source code is available on https://github.com/maoudia/code.maoudia.com/tree/main/bulk-update-with-spring-data-mongodb-reactive[Github].
+
+In the next chapter of *MongoDB Reactive CLI* series, we will add new features and use https://picocli.info/[Picocli] to facilitate interactions
+with the application.
+
+== Resources
+
+* https://www.enterpriseintegrationpatterns.com/DataEnricher.html[EIP Data enricher]
+* https://www.mongodb.com/try/download/database-tools[MongoDB Database Tools]
+* https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/[French Adresses Data]
+* https://mongodb.github.io/mongo-java-driver/4.6/driver-reactive/tutorials/bulk-writes/[MongoDB Java Driver Bulk operations]
+* https://projectreactor.io/docs/core/release/reference/[Reactor 3 Reference Guide]
+* https://docs.spring.io/spring-data/mongodb/docs/current/reference/html/[Spring Data MongoDB Reference]
+* https://docs.spring.io/spring-framework/docs/current/reference/html/web-reactive.html[Web on Reactive Stack]
+* https://visualvm.github.io/[VisualVM]
+* https://www.testcontainers.org/modules/databases/mongodb/[Testcontainers MongoDB]
diff --git a/content/blog/bulk-update-with-spring-data-mongodb-reactive/index.fr.adoc b/content/blog/bulk-update-with-spring-data-mongodb-reactive/index.fr.adoc
new file mode 100644
index 0000000..12ba170
--- /dev/null
+++ b/content/blog/bulk-update-with-spring-data-mongodb-reactive/index.fr.adoc
@@ -0,0 +1,649 @@
++++
+title = "Mise À Jour En Masse Avec Spring Data MongoDB Reactive"
+date = 2022-06-20T00:00:00+02:00
+description = "On va implémenter une solution pour enrichir et mettre à jour efficacement un grand volume de données en utilisant Spring Data MongoDB Reactive."
+author = "Moncef AOUDIA"
+showAuthor = false
+showReadingTime = true
+readingtime = 10
+tags = ["EIP", "Java", "Reactor", "MongoDB", "Spring Boot", "Spring Data", "Spring WebFlux", "Docker Compose", "TestContainers"]
+series = ["MongoDB Reactive CLI"]
+categories = ["Tutorial", "Reactive Programming"]
+slug = "mise-a-jour-en-masse-avec-spring-data-mongodb-reactive"
+type = "article"
+[twitter]
+ card = "summary_large_image"
+ site = "@AoudiaMoncef"
+ creator = "@AoudiaMoncef"
+ title = "Mise À Jour En Masse Avec Spring Data MongoDB Reactive"
+ description = "On va implémenter une solution pour enrichir et mettre à jour efficacement un grand volume de données en utilisant Spring Data MongoDB Reactive."
+ image = "https://www.maoudia.com/fr/blog/mise-a-jour-en-masse-avec-spring-data-mongodb-reactive/featuredImage.png"
++++
+
+:toc: macro
+:toc-title: Sommaire
+:toclevels: 4
+:imagesdir: /images/blog/bulk-update-with-spring-data-mongodb-reactive
+ifdef::env-github[]
+:imagesdir: ../../static/images/bulk-update-with-spring-data-mongodb-reactive
+endif::[]
+
+Afin de mettre à jour les documents d'une collection MongoDB, on passe souvent par des requêtes de mise à jour, si le volume des données est conséquent,
+cela peut conduire à des problèmes de performance et une surconsommation des ressources matérielles.
+
+On va implémenter une solution pour enrichir et mettre à jour efficacement un grand volume de données
+en utilisant Spring Data MongoDB Reactive.
+
+
+
+toc::[]
+
+Avant de continuer la lecture, si vous n'êtes pas familier avec la pile réactive de Spring et MongoDB,
+je suggère de consulter la section *ressources*.
+
+== EIP enrichisseur de contenu
+
+++++
+
+
+
+
+
+++++
+
+Le modèle d'intégration d'entreprise _enrichisseur de contenu_ permet d'ajouter des informations à un message existant depuis une source externe.
+Il utilise les informations contenues dans le message entrant pour effectuer l'opération d'enrichissement.
+
+On va implémenter une version simplifiée de l'_EIP_ :
+
+. Message en entrée : représenté par un document MongoDB.
+. Enrichisseur : notre application.
+. Ressource : appel à une API RESTful.
+. Message en sortie : nous ne conserverons que le document enrichi.
+
+=== Flux d'intégration
+
+++++
+
+
+
+
+
+++++
+
+L'application va lire les documents adresse, ajouter le produit et sauvegarder les documents enrichis dans la base MongoDB.
+
+== Configuration du projet
+
+=== Prérequis
+
+* Java 1.8+
+* Maven 3+
+* Docker Compose
+* MongoDB Database Tools
+
+=== Génération
+
+On génère le squelette du projet depuis https://start.spring.io/#!type=maven-project&language=java&platformVersion=2.7.0&packaging=jar&jvmVersion=1.8&groupId=com.maoudia&artifactId=bulk-update-with-spring-data-mongodb&name=Bulk%20Update%20with%20Spring%20Data%20MongoDB%20Reactive&description=Bulk%20Update%20with%20Spring%20data%20MongoDB%20reactive&packageName=com.maoudia.tutorial&dependencies=data-mongodb-reactive,webflux,testcontainers[`Spring Initializr`, window=\"_blank\"].
+
+=== Structure
+
+[source,shell,indent=0,linenums=true]
+----
+.
+│ .gitignore
+│ docker-compose.yml
+│ pom.xml
+│ README.adoc
+├───data
+│ ├───mongodb
+│ │ address.ndjson
+│ └───product
+│ db.json
+└───src
+ ├───main
+ │ ├───java
+ │ │ └───com
+ │ │ └───maoudia
+ │ │ └───tutorial
+ │ │ Application.java
+ │ │ AppProperties.java
+ │ │ CollectionService.java
+ │ │ NetworkConfig.java
+ │ └───resources
+ │ application.yml
+ └───test
+ └───java
+ └───com
+ └───maoudia
+ └───tutorial
+ CollectionServiceTest.java
+----
+
+=== Conteneurs
+
+On télécharge le dossier https://github.com/maoudia/code.maoudia.com/tree/main/bulk-update-with-spring-data-mongodb-reactive/data[`data`] vers la racine du projet.
+
+On utilise `docker-compose` pour créer les conteneurs nécessaires pour ce tutoriel.
+
+[source,yml,indent=0,linenums=true]
+.docker-compose.yml
+----
+services:
+ mongodb: // <1>
+ container_name: maoudia-mongodb
+ image: mongo:5.0.8
+ environment:
+ - MONGO_INITDB_DATABASE=test
+ - MONGO_INITDB_ROOT_USERNAME=admin
+ - MONGO_INITDB_ROOT_PASSWORD=password
+ networks:
+ - mongodb-network
+ ports:
+ - 15015:27017
+ volumes:
+ - ./data/mongodb:/data/mongodb
+
+ mongo-express: // <2>
+ container_name: maoudia-mongo-express
+ image: mongo-express:0.54.0
+ depends_on:
+ - mongodb
+ networks:
+ - mongodb-network
+ environment:
+ - ME_CONFIG_MONGODB_SERVER=maoudia-mongodb
+ - ME_CONFIG_MONGODB_ADMINUSERNAME=admin
+ - ME_CONFIG_MONGODB_ADMINPASSWORD=password
+ ports:
+ - 1515:8081
+ volumes:
+ - ./data/mongodb:/data/mongodb
+
+ product-api: // <3>
+ container_name: maoudia-product-api
+ image: clue/json-server:latest
+ ports:
+ - 1519:80
+ volumes:
+ - ./data/product/db.json:/data/db.json
+
+networks:
+ mongodb-network:
+ driver: bridge
+----
+
+<1> MongoDB initialisé avec la base de données `test`.
+<2> MongoExpress est une interface d'administration MongoDB.
+<3> L'API produit est configurée depuis le fichier `db.json`.
+
+
+On démarre les services :
+
+[source,shell,indent=0,linenums=true]
+----
+docker-compose up -d
+----
+
+=== Initialisation des données
+
+On utilise un document JSON issu de la base d'adresses française.
+
+.Adresse
+[source,json,indent=0,linenums=true]
+----
+{
+ "id": "59350",
+ "type": "municipality",
+ "name": "Lille",
+ "postcode": [
+ "59000",
+ "59800",
+ "59260",
+ "59777",
+ "59160"
+ ],
+ "citycode": "59350",
+ "x": 703219.96,
+ "y": 7059335.72,
+ "lon": 3.045433,
+ "lat": 50.630992,
+ "population": 234475,
+ "city": "Lille",
+ "context": "59, Nord, Hauts-de-France",
+ "importance": 0.56333
+}
+----
+
+On importe la collection d'adresses :
+
+[source,shell,indent=0,linenums=true]
+----
+mongoimport --uri "mongodb://admin:password@localhost:15015" --authenticationDatabase=admin --db test --collection address ./data/mongodb/address.ndjson
+----
+
+Ou :
+
+On utilise MongoExpress qui est accessible sur http://localhost:1515[`http://localhost:1515`].
+
+Le produit représente une offre d'internet par satellite.
+
+.Produit
+[source,json,indent=0,linenums=true]
+----
+{
+ "id": 1,
+ "available": true,
+ "company": "SPACEX",
+ "provider": "STARLINK",
+ "type": "SATELLITE"
+}
+----
+
+L'API produit est accessible sur http://localhost:1519[`http://localhost:1519`].
+
+== Application
+
+=== Configuration
+
+On change l'extension du fichier de `application.properties` vers `application.yml`.
+
+[source,yml,indent=0,linenums=true]
+.application.yml
+----
+app:
+ buffer-max-size: 500
+ bulk-size: 100
+ collection-name: address
+ enriching-key: product
+ enriching-uri: http://localhost:1519/products/1
+spring:
+ main:
+ web-application-type: none
+ data:
+ mongodb:
+ database: test
+ uri: mongodb://admin:password@localhost:15015
+---
+spring.config.activate.on-profile: dev
+logging:
+ level:
+ org.mongodb.driver: debug
+---
+spring.config.activate.on-profile: test
+app:
+ bulk-size: 2
+----
+
+On déclare une classe qui va contenir les propriétés de configuration de l'application.
+
+[source,java,indent=0,linenums=true]
+.AppProperties.java
+----
+@ConfigurationProperties(prefix = "app")
+public class AppProperties {
+ private int bulkSize;
+ private int bufferMaxSize;
+ private String collectionName;
+ private String enrichingKey;
+ private String enrichingUri;
+ // Les Getter et Setter sont omis
+}
+----
+
+On crée un `@Bean` du client HTTP non bloquant de Spring.
+
+[source,java,indent=0,linenums=true]
+.NetworkConfig.java
+----
+@Configuration
+public class NetworkConfig {
+
+ @Bean
+ public WebClient client() {
+ return WebClient.create();
+ }
+
+}
+----
+
+=== Implémentation
+
+On crée le `@Service` qui va contenir la logique métier de l'application.
+
+[source,java,indent=0,linenums=true]
+.CollectionService.java
+----
+@Service
+public class CollectionService {
+ private final AppProperties properties;
+ private final ReactiveMongoTemplate template;
+ private final WebClient client;
+
+ public CollectionService(AppProperties properties,
+ ReactiveMongoTemplate template,
+ WebClient client) {
+ this.properties = properties;
+ this.template = template;
+ this.client = client;
+ }
+
+ public Flux enrichAll(String collectionName, String enrichingKey, String enrichingUri) {
+ return template.findAll(Document.class, collectionName) // <1>
+ .onBackpressureBuffer(properties.getBufferMaxSize()) // <2>
+ .flatMap(document -> enrich(document, enrichingKey, enrichingUri)) // <3>
+ .map(CollectionService::toReplaceOneModel) // <4>
+ .window(properties.getBulkSize()) // <5>
+ .flatMap(replaceOneModelFlux -> bulkWrite(replaceOneModelFlux, collectionName)); // <6>
+ }
+}
+----
+
+<1> Crée un flux de documents à partir de la collection.
+<2> Limite le nombre maximum de documents chargés dans la _RAM_ en cas de consommation plus lente que la production.
+Si la taille maximale du tampon est dépassée, une `IllegalStateException` est levée.
+<3> Enrichie le document avec le document externe d'une façon asynchrone.
+<4> Crée un `ReplaceOneModel` à partir du document.
+<5> Regroupe les documents en flux de taille fixe. Le dernier flux peut être de taille inférieure.
+<6> Appel la fonction d'écriture en masse.
+
+[NOTE]
+====
+La propriété de configuration `app.bulk-size` peut être ajustée en fonction des besoins et ressources matérielles disponibles.
+Plus la taille du _bulk_ est grande, plus la consommation de mémoire et la taille des requêtes seront élevées.
+====
+
+On crée les fonctions d'enrichissement de document.
+
+[source,java,indent=0,linenums=true]
+.CollectionService.java
+----
+private Publisher enrich(Document document, String enrichingKey, String enrichingUri) { // <1>
+ return getEnrichingDocument(enrichingUri)
+ .map(enrichingDocument -> {
+ document.put(enrichingKey, enrichingDocument);
+ document.put("updatedAt", new Date());
+ return document;
+ });
+}
+
+private Mono getEnrichingDocument(String enrichingUri) { // <2>
+ return client.get()
+ .uri(URI.create(enrichingUri))
+ .retrieve()
+ .bodyToMono(Document.class);
+}
+----
+
+<1> Ajoute le document récupéré depuis l'appel _HTTP_ à la racine du document à enrichir avec la clef passée en paramètre.
+<2> Récupère le document depuis l'_URI_.
+
+[NOTE]
+====
+MongoDB convertie et stocke les dates en UTC par défaut.
+====
+
+[source,java,indent=0,linenums=true]
+.CollectionService.java
+----
+private static final ReplaceOptions REPLACE_OPTIONS = new ReplaceOptions(); // <1>
+private static ReplaceOneModel toReplaceOneModel (Document document) {
+ return new ReplaceOneModel<>(
+ Filters.eq("_id", document.get("_id")), // <2>
+ document, // <3>
+ REPLACE_OPTIONS
+ );
+}
+----
+
+<1> Instancie la configuration de remplacement par défaut.
+<2> Le filtre permet la correspondance par identifiant document.
+<3> Le contenu à remplacer, représente l'intégralité du document enrichi.
+
+
+[source,java,indent=0,linenums=true]
+.CollectionService.java
+----
+private static final BulkWriteOptions BULK_WRITE_OPTIONS = new BulkWriteOptions().ordered(false); // <1>
+private Flux bulkWrite(Flux> updateOneModelFlux, String collectionName) {
+ return updateOneModelFlux.collectList() // <2>
+ .flatMapMany(unused -> template.getCollection(collectionName) // <3>
+ .flatMapMany(collection -> collection.bulkWrite(updateOneModels, BULK_WRITE_OPTIONS))); // <4>
+}
+----
+
+<1> Instancie les options d'écritures en désactivant l'ordre des opérations.
+<2> Collecte le flux dans une liste.
+<3> Récupère la collection passée en paramètre.
+<4> Écrit en masse les documents dans la collection MongoDB.
+
+[NOTE]
+====
+Les transactions sont supportées sur les _Replicaset_ depuis MongoDB 4.2.
+Si les transactions sont activées, on peut utiliser `@Transactional` ou `TransactionalOperator` pour rendre une méthode transactionnelle.
+====
+
+On implémente les interfaces suivantes :
+
+* `CommandLineRunner` : exécute la commande d'enrichissement au démarrage de l'application.
+* `ExitCodeGenerator` : gère le code de sortie système.
+
+[source,java,indent=0,linenums=true]
+.Application.java
+----
+@SpringBootApplication(exclude = MongoReactiveRepositoriesAutoConfiguration.class) // <1>
+@ConfigurationPropertiesScan("com.maoudia.tutorial") // <2>
+public class Application implements CommandLineRunner, ExitCodeGenerator {
+ private static final Logger LOGGER = LoggerFactory.getLogger(Application.class);
+ private final AppProperties properties;
+ private final CollectionService service;
+ private int exitCode = 255;
+
+ public static void main(String[] args) {
+ System.exit(SpringApplication.exit(SpringApplication.run(Application.class, args)));
+ }
+
+ public Application(AppProperties properties, CollectionService service) {
+ this.properties = properties;
+ this.service = service;
+ }
+
+ @Override
+ public void run(final String... args) {
+ service.enrichAll(properties.getCollectionName(), properties.getEnrichingKey(), properties.getEnrichingUri())
+ .doOnSubscribe(unused -> LOGGER.info("------------------< Staring Collection Enriching Command >-------------------")) // <3>
+ .doOnNext(bulkWriteResult -> LOGGER.info("Bulk write result with {} modified document(s)", bulkWriteResult.getModifiedCount()))
+ .doOnError(throwable -> {
+ exitCode = 1;
+ LOGGER.error("Collection enriching failed due to : {}", throwable.getMessage(), throwable);
+ })
+ .doOnComplete(() -> exitCode = 0)
+ .doOnTerminate(() -> LOGGER.info("------------------< Collection Enriching Command Finished >------------------"))
+ .blockLast(); // <4>
+ }
+
+ @Override
+ public int getExitCode() {
+ return exitCode;
+ }
+
+}
+----
+
+<1> Désactive l'auto-configuration des repositories, car on utilise `MongoReactiveTemplate` seulement.
+<2> Permet de scanner et détecter les _beans_ qui portent l'annotation `@ConfigProperties`.
+<3> L'inscription au flux déclenche le traitement.
+<4> Sans un serveur web en fonctionnement, nous devons nous abonner indéfiniment au `Publisher` afin de déclencher
+et attendre la fin de l'exécution.
+
+=== Démo
+
+On lance l'application :
+
+[source,shell,indent=0,linenums=true]
+----
+mvn spring-boot:run
+----
+
+Sortie :
+
+[source,console,indent=0,linenums=true]
+----
+...
+2022-06-10 00:36:45.152 INFO 7036 --- [ main] com.maoudia.tutorial.Application : Started Application in 2.755 seconds (JVM running for 3.251)
+2022-06-10 00:36:45.227 INFO 7036 --- [ main] com.maoudia.tutorial.Application : ------------------< Staring Collection Enriching Command >-------------------
+2022-06-10 00:36:45.297 INFO 7036 --- [ main] org.mongodb.driver.cluster : No server chosen by com.mongodb.reactivestreams.client.internal.ClientSessionHelper$$Lambda$543/543409470@4647881c from cluster description ClusterDescription{type=UNKNOWN, connectionMode=SINGLE, serverDescriptions=[ServerDescription{address=localhost:15015, type=UNKNOWN, state=CONNECTING}]}. Waiting for 30000 ms before timing out
+2022-06-10 00:36:46.527 INFO 7036 --- [localhost:15015] org.mongodb.driver.connection : Opened connection [connectionId{localValue:1, serverValue:39}] to localhost:15015
+2022-06-10 00:36:46.527 INFO 7036 --- [localhost:15015] org.mongodb.driver.connection : Opened connection [connectionId{localValue:2, serverValue:40}] to localhost:15015
+2022-06-10 00:36:46.527 INFO 7036 --- [localhost:15015] org.mongodb.driver.cluster : Monitor thread successfully connected to server with description ServerDescription{address=localhost:15015, type=STANDALONE, state=CONNECTED, ok=true, minWireVersion=0, maxWireVersion=13, maxDocumentSize=16777216, logicalSessionTimeoutMinutes=30, roundTripTimeNanos=61576400}
+2022-06-10 00:36:46.692 INFO 7036 --- [ntLoopGroup-2-3] org.mongodb.driver.connection : Opened connection [connectionId{localValue:3, serverValue:41}] to localhost:15015
+2022-06-10 00:36:48.355 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:48.482 INFO 7036 --- [ntLoopGroup-2-4] org.mongodb.driver.connection : Opened connection [connectionId{localValue:4, serverValue:42}] to localhost:15015
+2022-06-10 00:36:48.562 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:48.742 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:48.982 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:49.222 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:49.488 INFO 7036 --- [ntLoopGroup-2-4] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:49.701 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:49.852 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:50.031 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:50.105 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : Bulk write result with 100 modified document(s)
+2022-06-10 00:36:50.106 INFO 7036 --- [ntLoopGroup-2-3] com.maoudia.tutorial.Application : ------------------< Collection Enriching Command Finished >------------------
+[INFO] ------------------------------------------------------------------------
+[INFO] BUILD SUCCESS
+[INFO] ------------------------------------------------------------------------
+[INFO] Total time: 17.315 s
+[INFO] Finished at: 2022-06-10T00:36:54+02:00
+[INFO] ------------------------------------------------------------------------
+
+Process finished with exit code 0
+----
+
+=== Rapport VisuelVM
+
+*VisualVM* est un outil de profilage léger. On l'utilise pour avoir une vue d'ensemble sur les threads qui sont lancés par l'application.
+
+++++
+
+
+
+
+
+++++
+
+On observe deux groupes de threads qui exécutent les opérations en parallèle, chaque groupe forme une l'_event loop_.
+
+* Les requêtes MongoDB sont exécutées par `nioEventLoopGroup`.
+* Les requêtes HTTP sont exécutées par `reactor-http-nio`.
+
+== Tests d'intégration
+
+On utilise *JUnit 5* et le module *Testcontainers MongoDB* pour les tests d'intégration.
+Cela permet d'avoir un retour proche du comportement réel de l'application qui fait essentiellement des opérations de lecture/écriture.
+
+Pour que ce tutoriel reste court, on va se contenter d'écrire qu'un seul test.
+
+[source,java,indent=0,linenums=true]
+.CollectionServiceTest.java
+----
+@Profile("test")
+@SpringBootTest
+@Testcontainers // <1>
+class CollectionServiceTest {
+
+ @Container
+ private static final MongoDBContainer mongoDBContainer = new MongoDBContainer("mongo:5.0.8") // <2>
+ .withReuse(true);
+
+ @DynamicPropertySource
+ private static void setProperties(DynamicPropertyRegistry registry) {
+ registry.add("spring.data.mongodb.uri", mongoDBContainer::getReplicaSetUrl); // <3>
+ }
+
+ @Autowired
+ private AppProperties properties;
+ @Autowired
+ private CollectionService command;
+ @Autowired
+ private ReactiveMongoTemplate template;
+
+ @Test
+ void multipleBulkWriteResultsAreReturned() {
+ Document givenDocument1 = new Document();
+ givenDocument1.put("_id", "628ea3edb5110304e5e814f6");
+ givenDocument1.put("type", "municipality");
+ Document givenDocument2 = new Document();
+ givenDocument2.put("_id", "628ea3edb5110304e5e814f7");
+ givenDocument2.put("type", "street");
+ Document givenDocument3 = new Document();
+ givenDocument3.put("_id", "628ea3edb5110304e5e814f8");
+ givenDocument3.put("type", "housenumber");
+
+ template.insert(Arrays.asList(givenDocument1, givenDocument2, givenDocument3), properties.getCollectionName()).blockLast();
+
+ BulkWriteResult expectedBulkWriteResult1 = BulkWriteResult.acknowledged(WriteRequest.Type.REPLACE, 2, 2, Collections.emptyList(),
+ Collections.emptyList());
+ BulkWriteResult expectedBulkWriteResult2 = BulkWriteResult.acknowledged(WriteRequest.Type.REPLACE, 1, 1, Collections.emptyList(),
+ Collections.emptyList());
+
+ command.enrichAll( properties.getCollectionName(), properties.getEnrichingKey() , properties.getEnrichingUri())
+ .as(StepVerifier::create) // <4>
+ .expectNext(expectedBulkWriteResult1)
+ .expectNext(expectedBulkWriteResult2)
+ .verifyComplete();
+ }
+}
+----
+
+<1> Ajoute l'extension Junit 5 de TestContainers.
+<2> Démarre un conteneur MongoDB.
+<3> Configure l'application avec l'URI du conteneur.
+<4> Utilise `StepVerifier` de *Reactor Test* pour faire des assertions sur le flux en sortie.
+
+On lance les tests d'intégration :
+
+[source,shell,indent=0,linenums=true]
+----
+mvn test -Dspring.profiles.active=test
+----
+
+Résultats des tests :
+
+[source,console,indent=0,linenums=true]
+----
+...
+[INFO] Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 20.563 s - in com.maoudia.tutorial.CollectionServiceTest
+[INFO]
+[INFO] Results:
+[INFO]
+[INFO] Tests run: 1, Failures: 0, Errors: 0, Skipped: 0
+[INFO]
+[INFO] ------------------------------------------------------------------------
+[INFO] BUILD SUCCESS
+[INFO] ------------------------------------------------------------------------
+[INFO] Total time: 32.100 s
+[INFO] Finished at: 2022-06-10T01:02:17+02:00
+[INFO] ------------------------------------------------------------------------
+----
+
+== Conclusion
+
+Dans ce tutoriel, on a réussi à implémenter une solution complète pour enrichir et mettre à jour efficacement une collection MongoDB.
+De plus, on a vu comment écrire des tests d'intégration avec JUnit 5 et Testcontainers.
+
+Le code source complet est disponible sur https://github.com/maoudia/code.maoudia.com/tree/main/bulk-update-with-spring-data-mongodb-reactive[Github].
+
+Dans le prochain chapitre de la série *MongoDB Reactive CLI*, on ajoutera de nouvelles fonctionnalités et utilisera https://picocli.info/[Picocli] afin de faciliter les interactions
+avec l'application.
+
+== Ressources
+
+* https://www.enterpriseintegrationpatterns.com/DataEnricher.html[EIP Data enricher]
+* https://www.mongodb.com/try/download/database-tools[MongoDB Database Tools]
+* https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/[French Adresses Data]
+* https://mongodb.github.io/mongo-java-driver/4.6/driver-reactive/tutorials/bulk-writes/[MongoDB Java Driver Bulk operations]
+* https://projectreactor.io/docs/core/release/reference/[Reactor 3 Reference Guide]
+* https://docs.spring.io/spring-data/mongodb/docs/current/reference/html/[Spring Data MongoDB Reference]
+* https://docs.spring.io/spring-framework/docs/current/reference/html/web-reactive.html[Web on Reactive Stack]
+* https://visualvm.github.io/[VisualVM]
+* https://www.testcontainers.org/modules/databases/mongodb/[Testcontainers MongoDB]
diff --git a/content/page/about/index.adoc b/content/page/about/index.adoc
index 552fc49..5acb1a8 100644
--- a/content/page/about/index.adoc
+++ b/content/page/about/index.adoc
@@ -7,6 +7,13 @@ showPublishDate = false
showLastModificationDate = false
showReadingTime = false
slug = "about"
+[twitter]
+ card = "summary_large_image"
+ site = "@AoudiaMoncef"
+ creator = "@AoudiaMoncef"
+ title = "Moncef AOUDIA"
+ description = "1x engineer, open-source enthusiast/maintainer, simracing/flightsim lover."
+ image = "https://www.maoudia.com/images/banners/banner-1200x600.png"
+++
:badges: /images/badges
@@ -180,7 +187,7 @@ slug = "about"
:zulip: image:{badges}/zulip.svg[zulip]
-I am a software developer, open-source enthusiast/maintainer.
+1x engineer, open-source enthusiast/maintainer, simracing/flightsim lover.
diff --git a/content/page/about/index.fr.adoc b/content/page/about/index.fr.adoc
index 6370a64..841c477 100644
--- a/content/page/about/index.fr.adoc
+++ b/content/page/about/index.fr.adoc
@@ -7,6 +7,13 @@ showPublishDate = false
showLastModificationDate = false
showReadingTime = false
slug = "about"
+[twitter]
+ card = "summary_large_image"
+ site = "@AoudiaMoncef"
+ creator = "@AoudiaMoncef"
+ title = "Moncef AOUDIA"
+ description = "Ingénieur 1x, passionné/mainteneur d'open-source, amoureux de simulations automobiles et vols."
+ image = "https://www.maoudia.com/images/banners/banner-1200x600.png"
+++
:badges: /images/badges
@@ -179,7 +186,7 @@ slug = "about"
:yammer: image:{badges}/yammer.svg[yammer]
:zulip: image:{badges}/zulip.svg[zulip]
-Je suis développeur logiciels, passionné/mainteneur d'open-source.
+Ingénieur 1x, passionné/mainteneur d'open-source, amoureux de simulations automobiles et vols.
diff --git a/content/page/uses/index.adoc b/content/page/uses/index.adoc
index 9762757..bf16cf1 100644
--- a/content/page/uses/index.adoc
+++ b/content/page/uses/index.adoc
@@ -7,6 +7,13 @@ showPublishDate = false
showLastModificationDate = false
showReadingTime = false
slug = "uses"
+[twitter]
+ card = "summary_large_image"
+ site = "@AoudiaMoncef"
+ creator = "@AoudiaMoncef"
+ title = "Moncef AOUDIA"
+ description = "Here are the software and hardware I use."
+ image = "https://www.maoudia.com/images/banners/banner-1200x600.png"
+++
:oss: image:/images/badges/oss.svg[open source badge, 50, 20]
diff --git a/content/page/uses/index.fr.adoc b/content/page/uses/index.fr.adoc
index 8210064..3ac0b55 100644
--- a/content/page/uses/index.fr.adoc
+++ b/content/page/uses/index.fr.adoc
@@ -7,6 +7,13 @@ showPublishDate = false
showLastModificationDate = false
showReadingTime = false
slug = "uses"
+[twitter]
+ card = "summary_large_image"
+ site = "@AoudiaMoncef"
+ creator = "@AoudiaMoncef"
+ title = "Moncef AOUDIA"
+ description = "Voici les logiciels et le matériel que j'utilise."
+ image = "https://www.maoudia.com/images/banners/banner-1200x600.png"
+++
:oss: image:/images/badges/oss.svg[open source badge, 50, 20]
diff --git a/content/status/website/index.adoc b/content/status/website/index.adoc
deleted file mode 100644
index 291a08d..0000000
--- a/content/status/website/index.adoc
+++ /dev/null
@@ -1,22 +0,0 @@
-+++
-date = "2021-05-19T07:30:00+01:00"
-publishDate = "2021-05-19T07:30:00+01:00"
-showPublishDate = false
-noSummary = true
-resizeImages = false
-type = "status"
-exclude = true
-+++
-
-.Available pages 🔗
-[NOTE]
-====
-* link:https://www.maoudia.com/about/[About]
-
-* link:https://www.maoudia.com/uses/[Uses]
-====
-
-[WARNING]
-====
-Work in progress... 👨💻
-====
diff --git a/content/status/website/index.fr.adoc b/content/status/website/index.fr.adoc
deleted file mode 100644
index f44b7fb..0000000
--- a/content/status/website/index.fr.adoc
+++ /dev/null
@@ -1,22 +0,0 @@
-+++
-date = "2021-05-19T07:30:00+01:00"
-publishDate = "2021-05-19T07:30:00+01:00"
-showPublishDate = false
-noSummary = true
-resizeImages = false
-type = "status"
-exclude = true
-+++
-
-.Pages disponibles 🔗
-[NOTE]
-====
-* link:https://www.maoudia.com/fr/about/[À propos]
-
-* link:https://www.maoudia.com/fr/uses/[Usages]
-====
-
-[WARNING]
-====
-Travail en cours… 👨💻
-====
\ No newline at end of file
diff --git a/i18n/en.toml b/i18n/en.toml
index a3ead95..cd8658e 100644
--- a/i18n/en.toml
+++ b/i18n/en.toml
@@ -33,7 +33,7 @@ other = "Other languages"
[readingTime]
one = "1 min read"
-other = "{{.Count}} min read"
+other = "{{.Count}} mins read"
[tableOfContents]
other = "Table of Contents"
@@ -78,4 +78,13 @@ other = "Rerouting..."
other = "Offline mode ❌"
[offlineMessage]
-other = "Oops, please wait until your internet connection is back"
\ No newline at end of file
+other = "Oops, please wait until your internet connection is back"
+
+[recentContent]
+other = "Recent content"
+
+[in]
+other = "in"
+
+[on]
+other = "on"
\ No newline at end of file
diff --git a/i18n/fr.toml b/i18n/fr.toml
index 05061dd..c64c1ec 100644
--- a/i18n/fr.toml
+++ b/i18n/fr.toml
@@ -20,20 +20,20 @@ other = "Chercher ..."
other = "Rien n'a été trouvé."
[olderPosts]
-other = "Articles plus anciens"
+other = "Postes plus anciens"
[newerPosts]
-other = "messages plus récents"
+other = "Postes plus récents"
[continueReading]
-other = "continuer la lecture"
+other = "Continuer la lecture"
[otherLanguages]
other = "Autres langues"
[readingTime]
-one = "1 Min. lecture"
-other = "{{.Count}} Min. lecture"
+one = "1 min de lecture"
+other = "{{.Count}} mins lecture"
[series]
other = "Séries"
@@ -75,4 +75,13 @@ other = "Redirection..."
other = "Mode hors ligne ❌"
[offlineMessage]
-other = "Oops, veuillez attendre le retour de votre connexion internet"
\ No newline at end of file
+other = "Oops, veuillez attendre le retour de votre connexion internet"
+
+[recentContent]
+other = "Contenu récent"
+
+[in]
+other = "sur"
+
+[on]
+other = "sur"
\ No newline at end of file
diff --git a/layouts/_default/baseof.html b/layouts/_default/baseof.html
index c7b01d1..f763613 100644
--- a/layouts/_default/baseof.html
+++ b/layouts/_default/baseof.html
@@ -9,8 +9,8 @@
{{ end }}
{{ .Scratch.Set "image" $imageUrl }}
- {{ .Scratch.Set "imageWidth" 700 }}
- {{ .Scratch.Set "imageHeight" 350 }}
+ {{ .Scratch.Set "imageWidth" 1200 }}
+ {{ .Scratch.Set "imageHeight" 600 }}
{{ else if .Resources.GetMatch "featuredImage.*" }}
{{ $imageUrl := "" }}
@@ -22,17 +22,17 @@
{{ end }}
{{ .Scratch.Set "image" $imageUrl }}
- {{ .Scratch.Set "imageWidth" 700 }}
- {{ .Scratch.Set "imageHeight" 350 }}
+ {{ .Scratch.Set "imageWidth" 1200 }}
+ {{ .Scratch.Set "imageHeight" 600 }}
{{ else if .Params.featuredImage }}
{{ .Scratch.Set "image" (.Params.featuredImage | absURL) }}
{{ else if .Params.mp4videoImage }}
{{ .Scratch.Set "image" (.Params.mp4videoImage | absURL) }}
{{ else }}
- {{ .Scratch.Set "image" (printf "https://www.gravatar.com/avatar/%s?size=200" (md5 .Site.Params.gravatarEMail)) }}
- {{ .Scratch.Set "imageWidth" 200 }}
- {{ .Scratch.Set "imageHeight" 200 }}
+ {{ .Scratch.Set "image" ("/images/banners/banner-1200x600.png" | absURL) }}
+ {{ .Scratch.Set "imageWidth" 1200 }}
+ {{ .Scratch.Set "imageHeight" 600 }}
{{ end }}
{{- if ne .Description "" -}}
@@ -92,6 +92,10 @@
+ {{ if .Site.Params.enableTwitterCard | default true }}
+ {{ partial "twitter-card.html" . }}
+ {{ end }}
+
{{ if .Site.Params.enableOpenGraph | default true }}
{{ partial "opengraph.html" . }}
{{ end }}
@@ -119,10 +123,6 @@
{{ end }}
-
- {{ if .Site.Params.enableTwitterCard | default true }}
- {{ partial "twitter-card.html" . }}
- {{ end }}
diff --git a/layouts/blog/rss.xml b/layouts/blog/rss.xml
index 37b8c23..300b03b 100644
--- a/layouts/blog/rss.xml
+++ b/layouts/blog/rss.xml
@@ -9,16 +9,16 @@ xmlns:webfeeds="http://webfeeds.org/rss/1.0">
{{ if eq .Title .Site.Title }}{{ .Site.Title }}{{ else }}{{ with .Title }}{{.}} on {{ end }}{{ .Site.Title }}{{ end }}
{{ .Permalink }}
- {{ .Site.BaseURL }}images/favicons/splash.webp
+ {{ .Site.BaseURL }}images/logo.png512512{{ if eq .Title .Site.Title }}{{ .Site.Title }}{{ else }}{{ with .Title }}{{.}} on {{ end }}{{ .Site.Title }}{{ end }}
{{ .Permalink }}
- {{ .Site.BaseURL }}images/banners/banner-700x350.webp
- 700
- 150
+ {{ .Site.BaseURL }}images/banners/banner-1200x600.png
+ 1200
+ 600Hugo -- gohugo.io{{ with .Site.LanguageCode }}
{{.}}{{end}}{{ with .Site.Author.email }}
@@ -30,9 +30,9 @@ xmlns:webfeeds="http://webfeeds.org/rss/1.0">
{{ printf "" .Permalink .MediaType | safeHTML }}
{{ end }}
000A12
-
+ {{ .Site.BaseURL }}images/favicons/favicon.ico
- {{ .Site.BaseURL }}images/favicons/splash.webp
+ {{ .Site.BaseURL }}images/logo.png
{{ if .Site.GoogleAnalytics }}
diff --git a/layouts/index.rss.xml b/layouts/index.rss.xml
index 1c03ca8..7683a0a 100644
--- a/layouts/index.rss.xml
+++ b/layouts/index.rss.xml
@@ -3,22 +3,22 @@ xmlns:atom="http://www.w3.org/2005/Atom"
xmlns:media="http://search.yahoo.com/mrss/"
xmlns:webfeeds="http://webfeeds.org/rss/1.0">
- {{ if eq .Title .Site.Title }}{{ .Site.Title }}{{ else }}{{ with .Title }}{{.}} on {{ end }}{{ .Site.Title }}{{ end }}
+ {{ if eq .Title .Site.Title }}{{ .Site.Title }}{{ else }}{{ with .Title }}{{.}} {{ i18n "on" }} {{ end }}{{ .Site.Title }}{{ end }}
{{ .Permalink }}
- Recent content {{ if ne .Title .Site.Title }}{{ with .Title }}in {{.}} {{ end }}{{ end }}on {{ .Site.Title }}
+ {{ i18n "recentContent" }} {{ if ne .Title .Site.Title }}{{ with .Title }}{{ i18n "in" }} {{.}} {{ end }}{{ end }}{{ i18n "on" }} {{ .Site.Title }}
- {{ if eq .Title .Site.Title }}{{ .Site.Title }}{{ else }}{{ with .Title }}{{.}} on {{ end }}{{ .Site.Title }}{{ end }}
+ {{ if eq .Title .Site.Title }}{{ .Site.Title }}{{ else }}{{ with .Title }}{{.}} {{ i18n "on" }} {{ end }}{{ .Site.Title }}{{ end }}
{{ .Permalink }}
- {{ "images/favicons/splash.webp" | absURL }}
+ {{ "images/logo.png" | absURL }}512512
- {{ if eq .Title .Site.Title }}{{ .Site.Title }}{{ else }}{{ with .Title }}{{.}} on {{ end }}{{ .Site.Title }}{{ end }}
+ {{ if eq .Title .Site.Title }}{{ .Site.Title }}{{ else }}{{ with .Title }}{{.}} {{ i18n "on" }} {{ end }}{{ .Site.Title }}{{ end }}
{{ .Permalink }}
- {{ "images/banners/banner-700x350.webp" | absURL }}
- 700
- 150
+ {{ "images/banners/banner-1200x600.png" | absURL }}
+ 1200
+ 600Hugo -- gohugo.io{{ with .Site.LanguageCode }}
{{.}}{{end}}{{ with .Site.Author.email }}
@@ -30,9 +30,9 @@ xmlns:webfeeds="http://webfeeds.org/rss/1.0">
{{ printf "" .Permalink .MediaType | safeHTML }}
{{ end }}
000A12
-
+ {{ "images/favicons/favicon.ico" | absURL }}
- {{ "images/favicons/splash.webp" | absURL }}
+ {{ "images/logo.png" | absURL }}
{{ if .Site.GoogleAnalytics }}
diff --git a/layouts/partials/default-content.html b/layouts/partials/default-content.html
index cc72678..67c1666 100644
--- a/layouts/partials/default-content.html
+++ b/layouts/partials/default-content.html
@@ -7,8 +7,8 @@
+ 
+ 
+ 
+