From 74714578d3a2c3380a5e364e48a43fdd810dafe1 Mon Sep 17 00:00:00 2001 From: Oliver Howell Date: Thu, 5 Dec 2024 12:11:01 +0000 Subject: [PATCH 1/3] DOC-303: Data Connection nav and tweaks --- docs/modules/ROOT/nav.adoc | 6 +++- ... => build-map-loader-data-connection.adoc} | 16 +++++----- ...ild-pipeline-service-data-connection.adoc} | 12 +++---- .../pages/data-connection-service.adoc | 31 +++++++++---------- .../data-connections/partials/nav.adoc | 3 +- 5 files changed, 35 insertions(+), 33 deletions(-) rename docs/modules/data-connections/pages/{how-to-map-loader-data-connection.adoc => build-map-loader-data-connection.adoc} (93%) rename docs/modules/data-connections/pages/{how-to-pipeline-service-data-connection.adoc => build-pipeline-service-data-connection.adoc} (93%) diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index ad42e9642..df70ba87e 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -157,7 +157,11 @@ include::cp-subsystem:partial$nav.adoc[] * xref:storage:high-density-memory.adoc[] include::tiered-storage:partial$nav.adoc[] * xref:cluster-performance:thread-per-core-tpc.adoc[] -include::data-connections:partial$nav.adoc[] +* Data Connections +** xref:data-connections:data-connections-configuration.adoc[Configure Data Connections] +** xref:data-connections:data-connection-service.adoc[] +** xref:data-connections:build-map-loader-data-connection.adoc[Build MapLoader] +** xref:data-connections:build-pipeline-service-data-connection.adoc[Build pipeline service] include::wan:partial$nav.adoc[] * xref:extending-hazelcast:extending-hazelcast.adoc[] ** xref:extending-hazelcast:operationparker.adoc[] diff --git a/docs/modules/data-connections/pages/how-to-map-loader-data-connection.adoc b/docs/modules/data-connections/pages/build-map-loader-data-connection.adoc similarity index 93% rename from docs/modules/data-connections/pages/how-to-map-loader-data-connection.adoc rename to docs/modules/data-connections/pages/build-map-loader-data-connection.adoc index d42dc5930..ddd524f84 100644 --- a/docs/modules/data-connections/pages/how-to-map-loader-data-connection.adoc +++ b/docs/modules/data-connections/pages/build-map-loader-data-connection.adoc @@ -1,10 +1,10 @@ -= Map Loader using Data Connection += Build MapLoader using Data Connection -:description: In this tutorial you build a custom map loader that uses a configured data connection to load data not present in an IMap. +:description: This tutorial shows how to build a custom map loader that uses a configured data connection to load data not present in an IMap. {description} -NOTE: This tutorial builds a custom implementation of MapLoader. For the most common use cases we also provide an out-of-the-box implementation xref:mapstore:configuring-a-generic-maploader.adoc[GenericMapLoader]. +NOTE: This tutorial builds a custom implementation of MapLoader. For the most common use cases an out-of-the-box implementation is also provided via xref:mapstore:configuring-a-generic-maploader.adoc[GenericMapLoader]. == Before you begin @@ -23,7 +23,7 @@ To complete this tutorial, you need the following: |=== -=== Step 1. Create and Populate the Database +=== Step 1 - create and populate the database This tutorial uses Docker to run the Postgres database. @@ -59,7 +59,7 @@ INSERT INTO my_table VALUES (8, 'eight'); INSERT INTO my_table VALUES (9, 'nine'); ---- -== Step 2. Create a New Java Project +== Step 2 - create a new java project Create a blank Java project named pipeline-service-data-connection-example and copy the Gradle or Maven file into it: @@ -106,7 +106,7 @@ Create a blank Java project named pipeline-service-data-connection-example and c ---- -== Step 3. MapLoader +== Step 3 - MapLoader The following map loader implements the `com.hazelcast.map.MapLoader` and `com.hazelcast.map.MapLoaderLifecycleSupport` interfaces. @@ -121,7 +121,7 @@ public class SimpleMapLoader implements MapLoader, MapLoaderLif } ---- -To implement the `MapLoaderLifecycleSupport` interface we need the following methods: +To implement the `MapLoaderLifecycleSupport` interface you need the following methods: [source,java] ---- @@ -207,7 +207,7 @@ To implement the `MapLoader` interface we need the following methods: } ---- -== Step 4. MapLoader Example App +== Step 4 - example MapLoader app Configure the data connection: diff --git a/docs/modules/data-connections/pages/how-to-pipeline-service-data-connection.adoc b/docs/modules/data-connections/pages/build-pipeline-service-data-connection.adoc similarity index 93% rename from docs/modules/data-connections/pages/how-to-pipeline-service-data-connection.adoc rename to docs/modules/data-connections/pages/build-pipeline-service-data-connection.adoc index d14e105ce..5bb5e8c4e 100644 --- a/docs/modules/data-connections/pages/how-to-pipeline-service-data-connection.adoc +++ b/docs/modules/data-connections/pages/build-pipeline-service-data-connection.adoc @@ -1,6 +1,6 @@ -= Pipeline Service using Data Connection += Build pipeline service using Data Connection -:description: This tutorial builds a service that transforms a stream of items. The service uses a data connection to retrieve a connection to a relational database, uses a table in the database to enrich a stream of numbers with a textual representation of the last digit. +:description: This tutorial builds a service that transforms a stream of items. The service uses a data connection to retrieve a connection to a relational database, and uses a table in the database to enrich a stream of numbers with a textual representation of the last digit. {description} @@ -21,7 +21,7 @@ To complete this tutorial, you need the following: |=== -== Step 1. Create and Populate the Database +== Step 1 - create and populate the database This tutorial uses Docker to run the Postgres database. @@ -57,9 +57,9 @@ INSERT INTO my_table VALUES (8, 'eight'); INSERT INTO my_table VALUES (9, 'nine'); ---- -== Step 2. Create a New Java Project +== Step 2 - create a new java project -Create a blank Java project named pipeline-service-data-connection-example and copy the Gradle or Maven file into it: +Create a blank Java project named `pipeline-service-data-connection-example`` and copy the Gradle or Maven file into it: [source,xml] ---- @@ -104,7 +104,7 @@ Create a blank Java project named pipeline-service-data-connection-example and c ---- -== Step 3. Create the Pipeline and the Job +== Step 3 - create pipeline and job Create a class `EnrichUsingDataConnection` with `main` method that will be the main job class. The `main` method will do the following steps: diff --git a/docs/modules/data-connections/pages/data-connection-service.adoc b/docs/modules/data-connections/pages/data-connection-service.adoc index b8ed3c8bc..708158071 100644 --- a/docs/modules/data-connections/pages/data-connection-service.adoc +++ b/docs/modules/data-connections/pages/data-connection-service.adoc @@ -1,19 +1,19 @@ -= Using Data Connections in custom components += Use Data Connection service :description: Using the Data Connection Service gives access to the configured xref:data-connections-configuration.adoc[data connections] in custom components. {description} -== Typical Usage +== Typical usage The typical steps to use a data connection are as follows: 1. Obtain the data connection from the data connection service. -2. Retrieve the underlying resource from the `DataConnection` instance. This step varies based on the specific implementation of `DataConnection` (e.g., `JdbcDataConnection` provides `getConnection()` which returns a `java.sql.Connection`; `HazelcastDataConnection` provides `getClient()` which returns a `HazelcastInstance`). +2. Retrieve the underlying resource from the `DataConnection` instance. This step varies based on the specific implementation of `DataConnection` (e.g. `JdbcDataConnection` provides `getConnection()` which returns a `java.sql.Connection`; `HazelcastDataConnection` provides `getClient()` which returns a `HazelcastInstance`). 3. Use the resource to perform the required operations. 4. Dispose of the resource (e.g., by calling `Connection#close` or `HazelcastInstance#destroy`). 5. Release the `DataConnection` instance (by calling `DataConnection#release()`). -Steps 2, 3, and 4 should be completed as quickly as possible to maximize the efficiency of connection pooling. +You should complete steps 2, 3, and 4 as quickly as possible to maximize the efficiency of connection pooling. [source,java] ---- @@ -31,7 +31,7 @@ try (Connection connection = jdbcDataConnection.getConnection()) { <2> jdbcDataConnection.release(); <4> ---- -== Retrieve Data Connection Service +== Retrieve Data Connection service Before working with data connections you need to retrieve an instance of the `DataConnectionService`. Use https://docs.hazelcast.org/docs/{full-version}/javadoc/com/hazelcast/core/HazelcastInstance.html#getDataConnectionService()[`HazelcastInstance#getDataConnectionService()`] @@ -43,32 +43,31 @@ to the `HazelcastInstance`. In the pipeline API you can use https://docs.hazelcast.org/docs/{full-version}/javadoc/com/hazelcast/jet/core/ProcessorMetaSupplier.Context.html#dataConnectionService()[ProcessorMetaSupplier.Context#dataConnectionService()]. -NOTE: The Data Connection Service is only available on the member side. Calling `getDataConnectionService()` on client will result in `UnsupportedOperationException`. +NOTE: The Data Connection Service is only available on the member side. Calling `getDataConnectionService()` on a client results in `UnsupportedOperationException`. -== Retrieve Configured DataConnection +== Retrieve configured DataConnection Use the `DataConnectionService` to get an instance of previously configured data connection https://docs.hazelcast.org/docs/{full-version}/javadoc/com/hazelcast/dataconnection/DataConnectionService.html#getAndRetainDataConnection(java.lang.String,java.lang.Class)[DataConnectionService#getAndRetainDataConnection(String, Class)]. For details how to configure a data connection, please refer to the xref:data-connections-configuration.adoc[Configuring Data Connections] page. -== Data Connection Scope +== Data Connection scope The data connection configuration is per-member. For example, when a data connection is created -with maximum pool size of 10 and the cluster has 3 members, there will be up to 30 connections +with maximum pool size of 10 and the cluster has 3 members, up to 30 connections will be created. -== Data Connection Sharing +== Data Connection sharing -Data connection is shared by default. It means that when the data connection is requested in multiple places, the same -underlying resource (e.g. Jdbc pool, remote client) is used. +Data connection is shared by default. This means that when the data connection is requested in multiple places, the same +underlying resource (e.g. JDBC pool, remote client) is used. If you want to share the data connection configuration, but use a different instance of the underlying resource, set the `DataConnectionConfig#setShared` to false. -== Configuration Considerations +== Configuration considerations If the data connection is defined in the Hazelcast configuration, it remains immutable for the entire lifespan of the Hazelcast member. In this case, whether you retrieve the DataConnection instance once or each time before accessing the underlying resource, the result will be the same. -However, if the data connection is created dynamically via SQL, it can be replaced using `CREATE OR REPLACE DATA CONNECTION` -(see xref:sql.adoc). +However, if the data connection is created dynamically via SQL, it can be replaced using `CREATE OR REPLACE DATA CONNECTION`. For more information, see xref:sql:get-started-sql.adoc[]. In such cases, the DataConnection instance will stay valid until you release it, allowing you to retrieve the underlying resource as needed. This approach can be useful for adapting to changes in data connection configuration. -For example, if you are running a batch job and want to use the same data connection throughout, request the connection at the start of the job. For a streaming job that may need updated configurations, retrieve both the data connection and the underlying resource just before use (e.g., when processing each item in the pipeline). +For example, if you are running a batch job and want to use the same data connection throughout, request the connection at the start of the job. For a streaming job that may need updated configurations, retrieve both the data connection and the underlying resource just before use (e.g. when processing each item in the pipeline). diff --git a/docs/modules/data-connections/partials/nav.adoc b/docs/modules/data-connections/partials/nav.adoc index 9c78b9652..8b1378917 100644 --- a/docs/modules/data-connections/partials/nav.adoc +++ b/docs/modules/data-connections/partials/nav.adoc @@ -1,2 +1 @@ -* Data Connections -** xref:data-connections:data-connections-configuration.adoc[] + From bfd350802617c2f2ac572844fc9658d0c2bb7797 Mon Sep 17 00:00:00 2001 From: Oliver Howell Date: Thu, 5 Dec 2024 12:15:49 +0000 Subject: [PATCH 2/3] DOC-303: tweak titles --- .../pages/build-map-loader-data-connection.adoc | 8 ++++---- .../pages/build-pipeline-service-data-connection.adoc | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/modules/data-connections/pages/build-map-loader-data-connection.adoc b/docs/modules/data-connections/pages/build-map-loader-data-connection.adoc index ddd524f84..84eec5f35 100644 --- a/docs/modules/data-connections/pages/build-map-loader-data-connection.adoc +++ b/docs/modules/data-connections/pages/build-map-loader-data-connection.adoc @@ -23,7 +23,7 @@ To complete this tutorial, you need the following: |=== -=== Step 1 - create and populate the database +=== Step 1. Create and populate database This tutorial uses Docker to run the Postgres database. @@ -59,7 +59,7 @@ INSERT INTO my_table VALUES (8, 'eight'); INSERT INTO my_table VALUES (9, 'nine'); ---- -== Step 2 - create a new java project +== Step 2. Create new java project Create a blank Java project named pipeline-service-data-connection-example and copy the Gradle or Maven file into it: @@ -106,7 +106,7 @@ Create a blank Java project named pipeline-service-data-connection-example and c ---- -== Step 3 - MapLoader +== Step 3. Implement MapLoader The following map loader implements the `com.hazelcast.map.MapLoader` and `com.hazelcast.map.MapLoaderLifecycleSupport` interfaces. @@ -207,7 +207,7 @@ To implement the `MapLoader` interface we need the following methods: } ---- -== Step 4 - example MapLoader app +== Step 4. Create example MapLoader app Configure the data connection: diff --git a/docs/modules/data-connections/pages/build-pipeline-service-data-connection.adoc b/docs/modules/data-connections/pages/build-pipeline-service-data-connection.adoc index 5bb5e8c4e..0d04e2a0d 100644 --- a/docs/modules/data-connections/pages/build-pipeline-service-data-connection.adoc +++ b/docs/modules/data-connections/pages/build-pipeline-service-data-connection.adoc @@ -21,7 +21,7 @@ To complete this tutorial, you need the following: |=== -== Step 1 - create and populate the database +== Step 1. Create and populate database This tutorial uses Docker to run the Postgres database. @@ -57,7 +57,7 @@ INSERT INTO my_table VALUES (8, 'eight'); INSERT INTO my_table VALUES (9, 'nine'); ---- -== Step 2 - create a new java project +== Step 2. Create new java project Create a blank Java project named `pipeline-service-data-connection-example`` and copy the Gradle or Maven file into it: @@ -104,7 +104,7 @@ Create a blank Java project named `pipeline-service-data-connection-example`` an ---- -== Step 3 - create pipeline and job +== Step 3. Create pipeline and job Create a class `EnrichUsingDataConnection` with `main` method that will be the main job class. The `main` method will do the following steps: From 2b93398b998124c4b90493a95e42cbe43f40c9c2 Mon Sep 17 00:00:00 2001 From: Oliver Howell Date: Tue, 17 Dec 2024 17:09:10 +0000 Subject: [PATCH 3/3] DOC-303: remove old partial nav and move data connections up --- docs/modules/ROOT/nav.adoc | 10 +++++----- docs/modules/data-connections/partials/nav.adoc | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) delete mode 100644 docs/modules/data-connections/partials/nav.adoc diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index df70ba87e..abf63b260 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -80,6 +80,11 @@ include::clusters:partial$nav.adoc[] ** xref:query:overview.adoc[] include::mapstore:partial$nav.adoc[] include::pipelines:partial$nav.adoc[] +* Data Connections +** xref:data-connections:data-connections-configuration.adoc[Configure Data Connections] +** xref:data-connections:data-connection-service.adoc[] +** xref:data-connections:build-map-loader-data-connection.adoc[Build MapLoader] +** xref:data-connections:build-pipeline-service-data-connection.adoc[Build pipeline service] * SQL ** xref:sql:sql-overview.adoc[Overview] ** SQL Over Maps @@ -157,11 +162,6 @@ include::cp-subsystem:partial$nav.adoc[] * xref:storage:high-density-memory.adoc[] include::tiered-storage:partial$nav.adoc[] * xref:cluster-performance:thread-per-core-tpc.adoc[] -* Data Connections -** xref:data-connections:data-connections-configuration.adoc[Configure Data Connections] -** xref:data-connections:data-connection-service.adoc[] -** xref:data-connections:build-map-loader-data-connection.adoc[Build MapLoader] -** xref:data-connections:build-pipeline-service-data-connection.adoc[Build pipeline service] include::wan:partial$nav.adoc[] * xref:extending-hazelcast:extending-hazelcast.adoc[] ** xref:extending-hazelcast:operationparker.adoc[] diff --git a/docs/modules/data-connections/partials/nav.adoc b/docs/modules/data-connections/partials/nav.adoc deleted file mode 100644 index 8b1378917..000000000 --- a/docs/modules/data-connections/partials/nav.adoc +++ /dev/null @@ -1 +0,0 @@ -