diff --git a/antora-playbook-local.yml b/antora-playbook-local.yml index c720cb829..c598cebda 100644 --- a/antora-playbook-local.yml +++ b/antora-playbook-local.yml @@ -1,7 +1,7 @@ site: title: Documentation url: http:localhost:5000 - start_page: hazelcast:getting-started:install-hazelcast.adoc + start_page: hazelcast:getting-started:get-started-cli.adoc robots: disallow keys: docsearch_id: 'QK2EAH8GB0' @@ -12,6 +12,9 @@ content: - url: . branches: HEAD start_path: docs + - url: https://github.com/hazelcast/cloud-docs + branches: [main] + start_path: docs ui: bundle: url: https://github.com/hazelcast/hazelcast-docs-ui/releases/latest/download/ui-bundle.zip #../hazelcast-docs-ui/build/ui-bundle.zip diff --git a/antora-playbook.yml b/antora-playbook.yml index d1c2cbc84..265173168 100644 --- a/antora-playbook.yml +++ b/antora-playbook.yml @@ -1,7 +1,7 @@ site: title: Documentation url: https://hardcore-allen-f5257d.netlify.app/ - start_page: hazelcast:getting-started:install-hazelcast.adoc + start_page: hazelcast:getting-started:get-started-cli.adoc robots: disallow keys: docsearch_id: 'QK2EAH8GB0' diff --git a/docs/modules/ROOT/images/client-server-pc-mobile.svg b/docs/modules/ROOT/images/client-server-pc-mobile.svg new file mode 100644 index 000000000..f06811774 --- /dev/null +++ b/docs/modules/ROOT/images/client-server-pc-mobile.svg @@ -0,0 +1,181 @@ + + + + + + + + + + + + + + + + + + + + + + + + + +Member + + + + + + + + + + + + + + + + + + + + +Member + + + + + + + + + + + + + + + + + + + + +Member +Cluster + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Client App + diff --git a/docs/modules/ROOT/images/multi-embedded.svg b/docs/modules/ROOT/images/multi-embedded.svg new file mode 100644 index 000000000..45f8bb7b3 --- /dev/null +++ b/docs/modules/ROOT/images/multi-embedded.svg @@ -0,0 +1,156 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Member + +Java Application + + + + + + + + + + + + + + + + + + + + + + + + + +Member + +Java Application + + + + + + + + + + + + + + + + + + + + + + + + + +Member + +Cluster + +Java Application + diff --git a/docs/modules/ROOT/images/single-embedded.svg b/docs/modules/ROOT/images/single-embedded.svg new file mode 100644 index 000000000..01d04b8d3 --- /dev/null +++ b/docs/modules/ROOT/images/single-embedded.svg @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Member + +Java Application + diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index a3de50bf9..4fb310271 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -1,8 +1,3 @@ - -.Quickstart -* xref:getting-started:quickstart.adoc[Start a Local Cluster in 5 minutes] -* xref:cloud:ROOT:getting-started.adoc[Try Hazelcast Cloud for Free] - .Get Started * xref:index.adoc[] include::getting-started:partial$nav.adoc[] @@ -42,21 +37,36 @@ include::cp-subsystem:partial$nav.adoc[] include::test:partial$nav.adoc[] include::troubleshoot:partial$nav.adoc[] -.Deploy and Manage Clusters +.Deploy Clusters * xref:production-checklist.adoc[] * xref:capacity-planning.adoc[] -* xref:topologies.adoc[] -* Enterprise Licenses -** xref:deploy:enterprise-licenses.adoc[] -** xref:deploy:updating-license-rest.adoc[] -include::deploy:partial$nav.adoc[] +* Installation +** xref:deploy:choosing-a-deployment-option.adoc[] +** xref:getting-started:editions.adoc[Editions and Distributions] +** xref:getting-started:install-hazelcast.adoc[Open Source] +** xref:getting-started:install-enterprise.adoc[Enterprise] +* Public Cloud +** xref:deploy:deploying-in-cloud.adoc[Overview] +** xref:deploy:deploying-on-aws.adoc[] +** xref:deploy:deploying-on-azure.adoc[] +** xref:deploy:deploying-on-gcp.adoc[] +* xref:deploy:deploying-with-docker.adoc[] +* Kubernetes +** xref:deploy:deploying-in-kubernetes.adoc[Overview] +** xref:deploy:configuring-kubernetes.adoc[] +** xref:deploy:scaling-and-upgrading-kubernetes.adoc[] +** xref:operator:ROOT:get-started.adoc[Hazelcast Platform Operator] +* xref:deploy:deploying-in-vmware-tanzu.adoc[] + +.Configure and Manage Clusters +include::configuration:partial$nav.adoc[] +* xref:deploy:enterprise-licenses.adoc[] * xref:maintain-cluster:logging.adoc[] include::maintain-cluster:partial$nav.adoc[] -* Configuring Clusters +* Partition Groups and Networking ** xref:clusters:creating-clusters.adoc[Grouping Clusters] ** xref:clusters:partition-group-configuration.adoc[Grouping Partitions] ** xref:clusters:network-configuration.adoc[Networking] -** xref:configuration:jet-configuration.adoc[Configuring the Jet Engine] include::cluster-performance:partial$nav.adoc[] include::secure-cluster:partial$nav.adoc[] include::fault-tolerance:partial$nav.adoc[] @@ -77,9 +87,8 @@ include::wan:partial$nav.adoc[] include::migrate:partial$nav.adoc[] .Reference -include::architecture:partial$nav.adoc[] -include::configuration:partial$nav.adoc[] +include::architecture:partial$nav.adoc[] * xref:deploy:supported-jvms.adoc[] diff --git a/docs/modules/ROOT/pages/topologies.adoc b/docs/modules/ROOT/pages/topologies.adoc deleted file mode 100644 index 3d9022500..000000000 --- a/docs/modules/ROOT/pages/topologies.adoc +++ /dev/null @@ -1,94 +0,0 @@ -= Hazelcast Topologies -:description: Hazelcast allows you to use it in either embedded mode or client/server mode. The mode you choose has an effect on your applications performance, your ability to scale your cluster, and how you communicate with your cluster. -[[hazelcast-topology]] - -{description} - -== Embedded Mode - -Hazelcast is a Java library, which means that you can embed it into your -own distributed application. - -NOTE: To use this mode, you must write your application in Java. - -In this mode, each instance of your application -starts a Hazelcast member, and all members automatically form their own cluster. To interact with the cluster, you use a local member in your application. - -If you are a Java developer working in an IDE, this is a very convenient -way to run a cluster on your development machine. There is nothing -stopping you from starting your program several times to get a -cluster on the same physical machine. - -The main advantage of embedded mode is that all your Java -classes are visible to Hazelcast. This way, you don't need to add classes to your members for tasks such as entry processing or submitting jobs. - -image:ROOT:Embedded.png[Embedded Mode] - -TIP: xref:getting-started:install-hazelcast.adoc#use-java[Add Hazelcast to your Java project] to get started with embedded mode. - -== Client/Server Mode - -In client/server mode, the Hazelcast cluster is independant of your application, which means that they can be independently created and scaled. -To interact with the cluster, you use one of the available clients. - -image:ROOT:ClientServer.png[Client/Server Mode] - -Client/server mode has advantages including more predictable and reliable Hazelcast -performance, easier identification of problem causes and, most importantly, better scalability. -When you need to scale in this mode, just add more Hazelcast members. You -can address client and server scalability concerns separately. - -Client/server mode also provides higher availability. If there are any issues in running either -your application or Hazelcast members, the issue and any potential failure is isolated from the rest -of the application and members. - -TIP: Download and xref:getting-started:install-hazelcast.adoc[install the binary] to get started with client/server mode. - -== Comparing Topologies - -Use this table to compare embedded mode and client/server mode in terms of latency, scalability, and available client languages. - -[cols="h,a,a"] -.Comparison of Hazelcast Modes -|=== -| |Embedded|Client/Server - -|Low-latency -|Yes -|If used with xref:clients:java.adoc#configuring-client-near-cache[Near Cache] to store frequently used data in the client's local memory. - -|Scalability -|The application and the cluster must be scaled together -|You can scale the cluster independently of your application - -|Supported clients -| Java -| -[.tablelist] -include::ROOT:partial$clients.adoc[] - -|=== - -== Development Clusters - -You can create a cluster for development and testing purposes. -This cluster may have one or multiple members running on a single machine. -It does not access the network and offers quickly starting and stopping a member, -adding more members, basic operations on various data structures, a minified cluster -simulating the production one, etc. See xref:getting-started:get-started-binary.adoc[here] on how to create a local cluster. - -== Production Clusters - -A production cluster is a network of multiple members that run Hazelcast. - -These clusters are accessed and managed as a single group. - -Besides having a single production cluster, you can also create multiple clusters, -for example to have data centers in different cities, over WAN. In this case, Hazelcast -offers its WAN replication feature to synchronize these clusters to the same state. See the xref:wan:wan.adoc[Synchronizing Data Across a WAN section] for more information. - -You also have the option to use the well-known “Blue-Green Deployment”. In this deployment model -you have two identical and alive clusters one of which is active (blue) and the other one is -idle (green). This way Hazelcast provides a disaster recovery for the client-cluster connections; -a client is automatically diverted to another cluster on demand or when the intended cluster becomes -unavailable. See the xref:clients:java.adoc#blue-green-deployment-and-disaster-recovery[Java client docs] for more information. \ No newline at end of file diff --git a/docs/modules/architecture/pages/018-kinesis-connector.adoc b/docs/modules/architecture/pages/018-kinesis-connector.adoc deleted file mode 100644 index 0f25e0f71..000000000 --- a/docs/modules/architecture/pages/018-kinesis-connector.adoc +++ /dev/null @@ -1,436 +0,0 @@ -= 018 - Kinesis Connector - -== Summary - -link:https://aws.amazon.com/kinesis/data-streams/[Amazon Kinesis Data Streams] (KDS) is a -massively scalable and durable real-time data streaming service. As part -of the Amazon Web Services offering, KDS manages the infrastructure, -storage, networking, and configuration needed to stream your data at the -level of your data throughput. You do not have to worry about -provisioning, deployment, ongoing-maintenance of hardware, software, or -other services for your data streams. Also, Amazon Kinesis Data Streams -synchronously replicates data across three availability zones, providing -high availability and data durability. - -The purpose of this document is to describe the implementation of -distributed Jet sources and sinks, which make it possible to read data -from and write data into Kinesis via Jet. - -== Key Concepts - -A **shard** is the base throughput unit of KDS. Shards help break the -stream's data flow into independent substreams, which can be processed -in parallel. Shards preserve the order of the data items they ingest -while ordering among different shards' items is undefined. One shard -provides a capacity of 1MiB/sec data input and 2MiB/sec data output. One -shard can support up to 1000 record publications per second. You will -specify the number of shards needed when you create a data stream. For -example, you can create a data stream with two shards. This data stream -has a throughput of 2MiB/sec data input and 4MiB/sec data output and -allows up to 2000 record publications per second. You can monitor -shard-level metrics in Kinesis and add or remove shards from your data -stream dynamically as your data throughput changes by resharding the -data stream. - -A **record** is the unit of data stored in Kinesis. A record is composed -of a sequence number, partition key, and data blob. Data blob is the -data of interest your data producer adds to a data stream. The maximum -size of a data blob (the data payload before Base64-encoding) is 1 MiB. - -A **partition key** is used to assign records to different shards of a -data stream. Items with the same partition key always belong to the same -shard. Since shards preserve the order of the items they ingest, the -ordering of records with the same partition key is also preserved. The -partition key is specified by your data producer while adding data to -KDS. - -A **sequence number** is a unique identifier for each record within its -shard. Sequence numbers are assigned by KDS when a data producer -publishes data into it. They can be used as offsets of the ordered -series of records of a shard. - -== APIs - -Amazon offers various choices of libraries that can be used to interact -with KDS: - -* **Kinesis Client Library (KCL)** and **Kinesis Producer Library -(KPL)** are high-level libraries that are easy to use because they -abstract away many concerns. They manage their threading policy, hide -away the REST-based nature of Kinesis behind asynchronous constructs, -balance load, handle failures, and react to resharding. However, all -this convenience makes them unsuitable for building Jet connectors, -where we need the most control possible to make choices that are -suitable to Jet's architecture. -* **Amazon Kinesis Data Streams API** via **AWS SDK for Java** is the -lower-level library that allows sufficient control when interacting -with Kinesis. It consists of a simple set of link:https://docs.aws.amazon.com/kinesis/latest/APIReference/API_Operations.html[REST-based operations]. -Every other concern mentioned above, when discussing the high-level -libraries, has to be handled explicitly. This is the library used in -the Jet source and sink implementations. - -== Quotas - -Amazon Kinesis Data Streams enforces quite a few quotas and limits, -which our sources and sinks need to comply with: - -* A single shard can ingest up to 1 MiB of data per second (including partition keys) or 1,000 records per second for writes. -* The maximum size of the data payload of a record is 1 MiB. -* The link:https://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetRecords.html[GetRecords] operation can retrieve up to 10 MiB of data per call from a single shard and up to 10,000 records per call. -* Each shard can support up to 5 GetRecords operations per second. -* Each shard can support up to a maximum total data read rate of 2 MiB per second. -* The link:https://docs.aws.amazon.com/kinesis/latest/APIReference/API_ListShards.html[ListShards] operation has a limit of 100 transactions per second, per data stream. Each such transaction is able to return at most 100 shards, and if the stream has more, then multiple transactions need to be used for a full listing. (For details, see the <> section.) -* The link:https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html[PutRecords] operation can write at most 500 records into the stream. Each record in the request can be as large as 1MiB, up to a limit of 5MiB for the entire request, including partition keys. Each shard can support writes up to 1,000 records per second, up to a maximum data write total of 1 MiB per second. -* The link:https://docs.aws.amazon.com/kinesis/latest/APIReference/API_DescribeStreamSummary.html[DescribeStreamSummary] operation has a limit of 20 transactions per second per account. (For details, see the <> section.) - -== Source - -The Kinesis source is a _streaming_, _distributed_, and _fault-tolerant_ -data source for Jet. It supports both the _at-least-once_ and -_exactly-once_ xref:fault-tolerance:fault-tolerance.adoc#processing-guarantee-is-a-shared-concern[processing guarantees]. - -=== Distribution - -Being a distributed source, it has multiple instances running in each -Hazelcast cluster member. Each instance is responsible for reading from zero, -one or more KDS <>. Each shard will be read by exactly one -source instance, the assignment is deterministic. - -Record keys, or partition keys as Kinesis calls them, are Unicode -strings, with a maximum length limit of 256 characters. The stream uses -the MD5 hash function to map these strings to 128-bit integer values. -The range of these values is thus [0 .. 2^128). Each Kinesis shard has a -continuous chunk of this range assigned to it, called the shard's hash -range. The stream assigns a record to a shard if the record's partition -key hashes into the shard's range. - -In the Jet Kinesis source, we use similar logic for assigning shards to -source instances. Each of our sources gets a part of the hash range -assigned to it. We say that a source owns a specific shard if and only -if the shard's hash range's starting point is inside the source's hash -range. Any similar range matching logic would work, as long as it's -non-ambiguous. - -=== Discovery - -Reading records from shards assigned to them is only a part of the -responsibility of sources. Sources also need a way to discover currently -active shards in the stream to take responsibility for them. Moreover, -this discovery process can't just happen once, on start-up, because -shards are dynamic, shards can be closed, and new shards can pop up at -any time. For details, see the <> section. - -Continuously monitoring the set of active shards in the stream is the -responsibility of **one** of the local source instances in each Jet -cluster member. This is an optimization. If all sources would run the -discovery, they would still obtain the same data, just with a multiplied -effort and cost. Monitoring means continuously polling the stream for -the list of all shards in it. - -Monitoring needs to take care not to cross the rate limit imposed by -Kinesis on this operation. For details, see the <> -section. - -=== Resharding - -Kinesis supports resharding, which lets you adjust the number of shards -in your stream to adapt to changes in data flow rate through the stream. -(Amazon charges on a per-shard basis, that's why it's desirable to have -the smallest amount of shards possible.) - -There are two types of resharding operations: shard **split** and shard -**merge**. In a shard split, you divide a single shard into two adjacent -shards. In a shard merge, you combine two adjacent shards into a single -shard. By "adjacent", we mean that one's hash range starts where the -other one's ends. - -Splitting increases the number of shards in your stream and therefore -increases the data capacity (and cost) of the stream. Similarly, merging -reduces the number of shards in your stream and therefore decreases the -data capacity (and cost). - -Resharding is always pairwise in the sense that you cannot split into -more than two shards in a single operation, and you cannot merge more -than two shards in a single operation. The shard or pair of shards that -the resharding operation acts on are called parent shards. The shard or -pair of shards that result from the resharding operation are called -child shards. - -When child shards, resulting from a split or merge, activate, their -parents get deactivated and will no longer get data inserted into them. -From that point onward, data goes into the children. - -=== Read Order - -Resharding does not suspend the stream's dataflow, while it's going on. -Data continues to be ingested into the stream, and at some point, it -just stops being put into the parent shards and starts being put into -the child shards. - -The Kinesis Jet source would need to make sure that it finishes reading -from parents before reading from their children. However, this is not -possible since the children might end up being owned by an entirely -different instance of the source than their parents (for example, in a -split), possibly located in an entirely different Hazelcast cluster member. - -Moreover, it's not enough to finish reading from the parent before -reading from the children. Even if that was achieved, data from parents -might overtake data from children further down the pipeline, simply -because it's a parallel flow. A Kinesis source would need to make sure -that it has read all data from the parents and that data has fully -passed through the pipeline before starting to read from the -children. Only then it could provide the same ordering as KDS while -resharding. - -This is currently not possible in Jet. Hopefully, future versions will -address the problem. Users of the Kinesis source need to be aware that -some data reordering might occur on resharding and try to time their -resharding activities, if possible, to utilize lulls in the data flow. - -=== Fault Tolerance - -The Kinesis Jet source supports pipelines with both at-least-once and -exactly-once processing guarantees. It achieves this by saving KDS -offsets into its snapshots and starting the reading from saved offsets -when restarted. - -The offsets are saved on a per-shard basis, and on restart, each source -instance receives all saved offsets for all shards, so it can function -properly regardless of how shards are assigned to sources after the -restart. - -=== Watermarks - -The Kinesis source can provide native timestamps because the link:https://docs.aws.amazon.com/kinesis/latest/APIReference/API_Record.html[record data structure] has a field that can be turned towards this purpose -(`ApproximateArrivalTimestamp`). However, it should be pointed out that -these watermarks are "native" only from Jet's point of view. They are -KDS ingestion times, i.e., whenever a KDS producer managed to push said -record into the data stream. We have no way of knowing what's the real -event time of a record. - -Watermarks are also saved to and recovered from snapshots. - -=== Metrics - -When receiving record batches, the link:https://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetRecords.html#API_GetRecords_ResponseSyntax[data structure] -contains a field called `MillisBehindLatest` defined as following: - -> The number of milliseconds the GetRecords response is from the -> stream's tip, indicating how far behind the current time the consumer -> is. A value of zero indicates that record processing caught up, and -> there are no new records to process at this moment. - -This value can be useful for monitoring, so the sources publish it as a -per-processor metric. - -=== Code Example - -A typical example of setting up a Kinesis source in Jet would look like -this: - -```java -KinesisSources.kinesis("myStream") - .withRegion("us-east-1") - .withEndpoint("http://localhost:12345") - .withCredentials("accesskey", "secretkey") - .withRetryStrategy(RetryStrategies.indefinitely(250)) - .build(); -``` - -The only mandatory property is the Kinesis `stream name`. The others are -optional and can be specified via a fluent builder. - -If `region` is not specified, then _us-east-1_ will be used by default. - -If `endpoint` is not specified, then the region's default endpoint will -be used. - -If `credentials` aren't specified, then the link:https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default[Default Credential Provider Chain] -will be followed. - -If `retry strategy` is not specified, then a default will be used -(defined by us - retry indefinitely, with exponential backoff limited to -a maximum of 3 seconds). A source's retry strategy applies to failures -of reading records from or listing shards of a stream. - -The actual source created will be of type -`StreamSource>`, so basically a stream of -partition key - record data blob pairs. - -== Sink - -The Kinesis sink is a _distributed_, _fault-tolerant_ data sink for Jet. -It supports both _streaming_ and _batching_ pipelines. The -fault-tolerance guarantee it can offer is only _at-least-once_ since -Kinesis does not offer transaction support. - -=== Distribution - -Being a distributed sink, it has multiple instances running in each Jet -cluster member. When used in a pipeline, this sink forces its incoming -edges to be _distributed_ and _partitioned_. The partition keys used by -the edges are the same as the Kinesis <>. -This ensures that all data with the same partition key will end up in -the same global sink instance and the same shard. - -=== Flow Control - -Writing data into a Kinesis Data Stream is governed by multiple -limitations: - -* no more than 500 records can be written in one batch -* each record must contain no more than 1M of data -* each batch must contain no more than 5M of data -* each shard can ingest no more than 1,000 records per second - -While most of these limitations are simple to enforce, the shard -ingestion rate is not. Different partition keys get assigned to a shard -based on a hashing function, so partition keys going into the same shard -can be written by different sink instances. Currently, Jet has no -capability for computing and coordinating such a per-shard rate among -all its distributed sink instances. - -The sink takes a different approach to comply with this limitation. It -allows for the rate to be tripped (i.e., it doesn't attempt to prevent -it from happening), but once it gets tripped, sinks try to slow down the -amount of data they write to keep the rate violation as an occasional, -rare event and not a continuous storm. - -The source achieves this flow control in two ways: - -* by decreasing the send batch size; the default is the maximum of 500, which it will reduce, if necessary, to as low as 10 records/batch -* by adding a delay between two subsequent send actions (which can be as little as 100ms, a reasonable value in case of Kinesis and as much as 10 seconds, which is a lot, but would occur only in an unreasonably -sized stream, as far as shard count is concerned - ultimately the -owner of the stream is responsible for setting up enough shards to be -able to handle his data rates) - -The flow control process is _adaptive_ in the sense that: - -* it kicks in only when batches start failing due to shard ingestion rates being tripped -* as long as failures repeat, it keeps quickly increasing the sleep delays to stop them from happening -* once failures stop, it slowly decreases the sleep delays until they are eliminated (i.e., the data volume spike was only temporary) or until failures start happening again - -Under normal circumstances, if there are enough shards in the stream and -their data ingestion rate covers the data flow, this whole flow control -process stays shut off. The sink publishes data with the lowest possible -latency. - -=== Discovery - -As we've seen in the <> section, one element -used to control the throughput is batch size. Under normal conditions, -the sink uses the default/maximum batch size of 500. When flow control -kicks in, a new batch size is picked as a function of the number of open -shards in the stream. - -For this to happen, the sinks need to have a relatively up-to-date -information about the number of open shards. The sink achieves this by -using a mechanism very similar to the <>. The only real difference is that the sinks use the -link:https://docs.aws.amazon.com/kinesis/latest/APIReference/API_DescribeStreamSummary.html[DescribeStreamSummary] -operation instead of the -link:https://docs.aws.amazon.com/kinesis/latest/APIReference/API_ListShards.html[ListShards] -one. - -=== Write Order - -Under normal circumstances, the Kinesis sink preserves the order of -items belonging to the same partition key. However, when the <> mechanism kicks in, the ordering might be lost -on occasion. - -This fact originates in the way how KDS handles shard ingestion rate -violations. When KDS receives a batch to be ingested, it processes each -item in it one by one, and if some fail, it doesn't stop processing the -batch. The result is that some items from a batch get rejected, some get -ingested, but in a random manner. The sink does resend the non-ingested -item, they won't get lost, but there is nothing it can do to preserve -the initial ordering. - -The advice we can give to Kinesis sink users, if they care about -ordering at all, is to try to have enough shards to accommodate even -occasional spikes in their data rate and to make sure that their -partition keys are spread out adequately over all shards. - -=== Fault Tolerance - -Since there is no transaction support in Kinesis, the sink can't support -exactly-once delivery. It can, however, support at-least-once -processing. It does that by ensuring it flushes all data it has taken -ownership of (taken from the `Inbox` is the more accurate "dev-speak") -out to Kinesis, before saving its snapshots. - -A further reason why exactly-once support is not possible is the API -used to implement the sink, the AWS SDK itself. It has internal retry -mechanisms, which can lead to duplicate publishing of records. For -details, see the link:https://docs.aws.amazon.com/streams/latest/dev/kinesis-record-processor-duplicates.html#kinesis-record-processor-duplicates-producer[relevant parts of its documentation]. - -=== Metrics - -Two metrics that should be useful to populate on a per-sink basis are -parameters related to <>: - -* batch size -* sleep between two consecutive send attempts - -=== Code Example - -A typical example of setting up a Kinesis sink in Jet would look like -this: - -```java -KinesisSinks.kinesis("myStream") - .withRegion("us-east-1") - .withEndpoint("http://localhost:12345") - .withCredentials("accesskey", "secretkey") - .withRetryStrategy(RetryStrategies.indefinitely(250)) - .build(); -``` - -The properties here work exactly like the ones for the -<>. What's worth noting, though, is that this -version is a simplified form. It is able to accept only input items of -the form of `Map.Entry` (so partition key - data blob -pairs). - -A more generic form, which can accept any item stream, is of the form: - -```java -KinesisSinks.kinesis( - @Nonnull String stream, - @Nonnull FunctionEx keyFn, - @Nonnull FunctionEx valueFn -) -``` - -It has two more mandatory parameters: - -* a `key function` that specifies how to compute the partition key from an input item -* a `value function` that specifies how to compute the data blob from an input item - -== Testing - -Both the Kinesis source and sink can be covered by integration tests in -which the AWS backend is mocked with the help of -link:https://github.com/localstack/localstack[LocalStack] and -link:https://www.testcontainers.org/[Testcontainers]. - -This mock is pretty reliable, with only small disadvantages. One of them -is that it doesn't enforce the intake rate of shards, so we can't write -tests to verify the sink's flow control behavior when trying to publish -more data than the stream can ingest. Another disadvantage is that it -ignores credentials (accepts anything), so we can't test behavior when -credentials are incorrect. These scenarios can, however, be tested -manually on the real AWS backend. - -== Future Improvements - -One extra Kinesis connector we could add to Jet in the future would be a -version of the source which supports _enhanced fan-out_. Such a Kinesis -consumer is different in two ways: it has dedicated throughput, and it -gets data pushed to it, doesn't have to poll. Implementing such a source -in future versions, though, needs to be motivated with concrete needs. - -Another future improvement would be adding a generic mechanism to Jet, -which would enable us to solve the <>. This would be some kind of signaling mechanism -we could use in a Kinesis source to check that certain previously -dispatched items have cleared the entire pipeline. It's not clear how -exactly this would work and if it will be implemented at all. diff --git a/docs/modules/architecture/pages/architecture.adoc b/docs/modules/architecture/pages/architecture.adoc new file mode 100644 index 000000000..930049de6 --- /dev/null +++ b/docs/modules/architecture/pages/architecture.adoc @@ -0,0 +1,114 @@ += Hazelcast Architecture +:description: Hazelcast is a distributed computation and storage platform for consistent low-latency querying, aggregation, and stateful computation against event streams and traditional data sources. This guide outlines how Hazelcast works under the hood to help you understand some important concepts. +:page-aliases: overview:architecture.adoc + +{description} + +In Hazelcast, data is load-balanced in-memory across a cluster. +This cluster is a network of members each of which runs Hazelcast. A cluster of Hazelcast members share both the data storage and computational +load which can dynamically scale up and down. When you add new members to the cluster, both the data and computations are automatically rebalanced across the cluster. + +image:ROOT:HighLevelArch.png[Hazelcast High-Level Architecture] + +== Discovery and Clustering + +Members xref:clusters:discovery-mechanisms.adoc[discover each other automatically] and form a cluster. after the cluster is formed, members +communicate with each other via TCP. + +Hazelcast supports automatic discovery in cloud environments such as Amazon EC2, Google Cloud Platform and Azure. You can also configure Hazelcast to discover members by TCP/IP or multicast. In addition, you can make use of the automatic member discovery in the Kubernetes environment. + +== Fault Tolerance + +Hazelcast distributes your storage data, computational data, and backups, among all cluster members. This way, if a member is lost, +Hazelcast can restore the data from these backups, providing continuous availability. + +Backups are distributed and stored in memory (RAM). +The distribution happens on the partition level, where the primary data and its backups are stored in +the xref:overview:data-partitioning.adoc[partitions]. + +When a member in your cluster is lost, Hazelcast redistributes the backups on the +remaining members so that every partition has a backup. This makes Hazelcast resilient +to data loss. The number of backups is configurable. Based on the configuration, data +can be kept in multiple replicas of a partition. + +== High Availability + +In Hazelcast, cluster members monitor the health of each other. When a cluster +member becomes inaccessible due to an event such as a network failure, other members +cooperatively diagnose the state and immediately take over the responsibility +of the failed member. To determine if a member is unreachable or crashed, Hazelcast +provides built-in xref:clusters:failure-detector-configuration.adoc[failure detectors]. + +Hazelcast also provides replication over WAN for high availability. You can have deployments across multiple data centers using the xref:wan:wan.adoc[WAN replication] mechanism which offers protection against a data center or wider network failures. + +== AP/CP + +In the context of the https://en.wikipedia.org/wiki/CAP_theorem[CAP theorem^], +Hazelcast offers AP and CP data structures: + +* *Availability:* All working members in a distributed system return a valid +response to any request, without exceptions. +* *Consistency:* All the clients connected to the system see the same data at the same time, +no matter which member they connect to; whenever data is written to a member, it is instantly +replicated to all the other members in the system. +* *Partition Tolerance:* _Partition_ refers to a lost or temporarily delayed connection between +the members; partition tolerance refers to continued work of the cluster despite any number of +communication breakdowns between the members of the cluster. + +Hazelcast as an AP system delivers availability and partition tolerance at the expense of +consistency. When a communication breakdown occurs, all members remain available, +however some members affected by the partition might return an older version of data than others. +When the partition is resolved, Hazelcast typically synchronizes the members to repair all inconsistencies in the system. + +As a CP system, Hazelcast delivers consistency and partition tolerance at the expense of availability. +When a partition occurs between any members, Hazelcast makes the non-consistent member unavailable +until the partition is resolved. + +Data structures exposed under `HazelcastInstance` API are all AP data structures and the ones +accessed via `HazelcastInstance.getCPSubsytem()` provides CP structures and APIs, which are built +on the Raft consensus algorithm. See the xref:consistency-and-replication:consistency.adoc[Consistency and Replication Model section] +and xref:cp-subsystem:cp-subsystem.adoc[CP Subsystem section]. + +== Storage Engine + +The storage engine provides caching and data processing on top of Hazelcast’s key/value store called IMap. +You can xref:data-structures:working-with-external-data.adoc[load and store] data +from/to external data sources, control the xref:data-structures:managing-map-memory.adoc[eviction] +of data entries and xref:computing:entry-processor.adoc[execute] your codes on them, use Hazelcast's +specification-compliant xref:jcache:jcache.adoc[JCache] implementation and integrate your cache with Spring. + +== Jet (Streaming) Engine + +The Jet engine is responsible for running streaming applications. For this, it uses many different connectors (sources and sinks) +to ingest data into Hazelcast and output data that data, and stateless/stateful transforms with operators including join, aggregate and sort +to process data. + +A data pipeline is a series of processing steps that consist of a source, one or more processing steps, +and a sink. It may enable the flow of data from an application to a data warehouse, from +a data lake to an analytics database, or into a payment processing system. Hazelcast allows you to create +data pipelines, using either SQL or the Hazelcast Java API. See xref:pipelines:overview.adoc[]. + +== Distributed System Tools + +Hazelcast offers distributed implementations of standard collections, concurrency utilities, and +publish/subscribe messaging model. + +Standard collection implementations include Maps, Queues, Lists, Sets and Ringbuffers. +Concurrency utilities include AtomicLongs, AtomicReferences, Semaphores and CountDownLatches. +It also provides a broadcast messaging system based on publish/subscribe model; it lets applications +communicate in real time at high speed. Your applications can publish to specific channels +called topics, and then one or more subscribers can read the messages from those topics. See the +xref:data-structures:distributed-data-structures.adoc[Distributed Data Structures]. + +Hazelcast also provides a CP subsystem for distributed coordination use cases such as leader +election, distributed locking, synchronization and metadata management. See the +xref:cp-subsystem:cp-subsystem.adoc[CP Subsystem section]. + +== SQL Service + +Hazelcast’s xref:sql:sql-overview.adoc[SQL service] lets you query static and streaming data. You can load static data +from sources (files and maps) to transform and analyze it. Hazelcast can also load and query +real-time streaming data as it is being generated, which is ideal for use cases that require +complex queries with low latency, e.g., fraud detection. Besides performing transformations +and queries, you can store your transform/query results in one or more systems; it is useful +for sending results to other systems or caching results in Hazelcast to avoid running redundant queries. diff --git a/docs/modules/architecture/pages/data-partitioning.adoc b/docs/modules/architecture/pages/data-partitioning.adoc new file mode 100644 index 000000000..796e26d51 --- /dev/null +++ b/docs/modules/architecture/pages/data-partitioning.adoc @@ -0,0 +1,233 @@ += Data Partitioning and Replication +:description: Partitioning and replication are two common techniques used together in distributed systems to achieve scalable, available, and transparent data distribution. +:page-aliases: overview:data-partitioning.adoc, consistency-and-replication:replication-algorithm.adoc, consistency-and-replication:invocation-lifecycle.adoc, consistency-and-replication:indeterminateoperationstateexception.adoc, consistency-and-replication:exactly-once-execution.adoc, consistency-and-replication:consistency.adoc + +{description} + +By default, Hazelcast creates a single replica of each partition. You can configure Hazelcast so that +each partition can have multiple replicas. One of these replicas is called _primary_ +and the others are called _backups_. + +The cluster member that owns the _primary_ replica of a partition +is called the _partition owner_. When you read or write a particular data entry, you transparently +talk to the partition owner that contains the data entry. + +By default, Hazelcast offers 271 partitions. When you start a cluster with a single member, +it owns all 271 partitions. + +The amount of data entries that each partition can hold is limited by the physical capacity of your system. + +image:ROOT:NodePartition.jpg[Single Member with Partitions] + +When you start a second member in the same cluster, +the partition replicas are distributed between both members. + +image:ROOT:BackupPartitions.jpg[Cluster with Two Members - Backups are Created] + +NOTE: Partitions are not usually distributed in any order, but are distributed randomly. Partition distributions are displayed in order for the sake of simplicity and for descriptive purposes. + +The partition replicas with black text are primaries and the partition replicas +with blue text are backups. The first member has primary replicas of 135 partitions and +each of these partitions are backed up by the second member. At the same time, the first member also has the backup replicas of +the second member's primary partition replicas. + +As you add more members, Hazelcast moves some of the primary and backup partition replicas to +the new members one by one, making all members equal and redundant. Thanks to the consistent +hashing algorithm, only the minimum amount of partitions are moved to scale out Hazelcast. + +image:ROOT:4NodeCluster.jpg[Cluster with Four Members] + +Hazelcast distributes the partitions' primary and backup replicas equally among cluster members. Backup replicas of the partitions are maintained for redundancy. + +NOTE: Your data can have multiple copies on partition primaries and backups, depending on your +backup count. See xref:data-structures:backing-up-maps.adoc[]. + +== Lite Members + +Hazelcast also offers lite members. These members do not own any partition. Lite members are +intended for use in computationally-heavy task executions and listener registrations. Although +they do not own any partitions, +they can access partitions that are owned by other members in the cluster. + +NOTE: See the xref:management:cluster-utilities.adoc#enabling-lite-members[Enabling Lite Members section]. + +[[how-the-data-is-partitioned]] +== How Data is Partitioned + +Hazelcast distributes data entries into the partitions using a hashing algorithm. Given an object +key such as for a map or an object name such as for a topic: + +* the key or name is serialized (converted into a byte array) +* this byte array is hashed +* the result of the hash is mod by the number of partitions. + +The result of this modulo - *MOD(hash result, partition count)* - is the partition in which the +data will be stored, that is the _partition ID_. For all members you have in your cluster, the +partition ID for a given key is always the same. + +[[partition-table]] +== Partition Table + +The partition table stores the partition IDs and the addresses of cluster members to +which they belong. The purpose of this table is to make all members, including lite members, in the cluster aware of this information so that each member knows where the data is. + +When you start your first member, a partition table is created within it. As you start +additional members, that first member becomes the oldest member, also known as the _master member_ and updates the partition +table accordingly. This member periodically sends the partition +table to all other members. This way, each member in the cluster is informed about any changes to +partition ownership. The ownerships may be changed when, for example, a new member joins the cluster, or when a member leaves the cluster. + +NOTE: If the master member goes down, the next oldest member sends the partition table information to the other ones. + +You can configure how often the member sends the partition table information +by using the `hazelcast.partition.table.send.interval` system property. By default, the frequency is 15 seconds. + +[[repartitioning]] +== Repartitioning + +Repartitioning is the process of redistribution of partition ownerships. Hazelcast performs the +repartitioning when a member joins or leaves the cluster. + +In these cases, the partition table in the master member is updated with the new partition +ownerships. If a lite member joins or leaves a cluster, repartitioning is not triggered +since lite members do not own any partitions. + +== Replication Algorithm for AP Data Structures + +For AP data structures, Hazelcast employs a combination of primary-copy and +configurable lazy replication techniques. Each data entry is mapped to +a single Hazelcast partition and put into replicas of that partition. One of +the replicas is elected as the primary replica, which is responsible for +performing operations on that partition. When you read or +write a map entry, you transparently talk to the Hazelcast member to which +the primary replica of the corresponding partition is assigned. +This way, each request hits the most up-to-date version of +a particular data entry in a stable cluster. Backup replicas stay +in standby mode until the primary replica fails. +Upon failure of the primary replica, one of the backup replicas is promoted to the primary role. + +With _lazy replication_, when the primary replica receives +an update operation for a key, it executes the update locally and +propagates it to backup replicas. It marks each update with +a logical timestamp so that backups apply them in the correct order and +converge to the same state with the primary. Backup replicas can be used to +scale reads with +no strong consistency but monotonic reads guarantee. See xref:data-structures:backing-up-maps.adoc[]. + +Two types of backup replication are available: _sync_ and _async_. +Despite what their names imply, both types are still implementations of +the lazy (async) replication model. The only difference between +_sync_ and _async_ is that, the former makes the caller block until +backup updates are applied by backup replicas and acknowledgments are sent back to +the caller, but the latter is just fire & forget. Number of sync and +async backups are defined in the data structure configurations, and you can use +a combination of sync and async backups. + +When backup updates are propagated, the response of the execution, including +the number of sync backup updates, is sent to the caller and after receiving +the response, the caller waits to receive the specified number of +sync backup acknowledgements for a predefined timeout. +This timeout is 5 seconds by default and defined by +the system property xref:ROOT:system-properties.adoc[`hazelcast.operation.backup.timeout.millis`]. + +A backup update can be missed because of a few reasons, such as +a stale partition table information about a backup replica member, +network interruption, or a member crash. That's why sync backup acks require +a timeout to give up. Regardless of being a sync or async backup, if a backup update is missed, +the periodically running anti-entropy mechanism detects the inconsistency and +synchronizes backup replicas with the primary. Also the graceful shutdown procedure ensures +that all backup replicas for partitions whose primary replicas are assigned to +the shutting down member will be consistent. + +In some cases, although the target member of an invocation is assumed to be +alive by the failure detector, the target may not execute the operation or +send the response back in time. Network splits, long pauses caused by +high load, GC or I/O (disk, network) can be listed as a few possible reasons. +When an invocation doesn't receive any response from the member that owns +the primary replica, then invocation fails with an `OperationTimeoutException`. +This timeout is 2 minutes by default and defined by +the system property xref:ROOT:system-properties.adoc[`hazelcast.operation.call.timeout.millis`]. +When the timeout is passed, the result of the invocation will be indeterminate. + +=== Execution Guarantees + +Hazelcast, as an AP product, does not provide the exactly-once guarantee. +In general, Hazelcast tends to be an at-least-once solution. + +In the following failure case, the exactly-once guarantee can be broken: The target member of a pending invocation leaves the cluster while +the invocation is waiting for a response, that invocation is re-submitted to +its new target due to the new partition table. It can be that, it has +already been executed on the leaving member and backup updates are propagated to +the backup replicas, but the response is not received by the caller. +If that happens, the operation will be executed twice. + +In the following failure case, invocation state becomes indeterminate: +When an invocation does not receive a response in time, +invocation fails with an `OperationTimeoutException`. This exception does not +say anything about the outcome of the operation, meaning the operation may not be +executed at all, or it may be executed once or twice. + +=== Throwing an IndeterminateOperationStateException + +If the xref:ROOT:system-properties.adoc[`hazelcast.operation.fail.on.indeterminate.state`] system property is +enabled, a *mutating* operation throws an `IndeterminateOperationStateException` when +it encounters the following cases: + +- The operation fails on partition primary replica member with `MemberLeftException`. +In this case, the caller may not determine the status of the operation. +It could happen that the primary replica executes the operation, but fails before +replicating it to all the required backup replicas. Even if the caller receives +backup acks from some backup replicas, it cannot decide if it has received +all required ack responses, since it does not know how many acks it should wait for. + +- There is at least one missing ack from the backup replicas for the given timeout duration. +In this case, the caller knows that the operation is executed on the primary replica, +but some backup may have missed it. It could be also a false-positive, +if the backup timeout duration is configured with a very small value. +However, Hazelcast's active anti-entropy mechanism eventually kicks in and +resolves durability of the write on all available backup replicas as long as +the primary replica member is alive. + +When an invocation fails with `IndeterminateOperationStateException`, +the system does not try to rollback the changes which are executed on healthy replicas. +Effect of a failed invocation may be even observed by another caller, +if the invocation has succeeded on the primary replica. +Hence, this new behavior does not guarantee linearizability. +However, if an invocation completes without `IndeterminateOperationStateException` when +the configuration is enabled, it is guaranteed that the operation has been +executed exactly-once on the primary replica and specified number of backup replicas of the partition. + +Please note that `IndeterminateOperationStateException` does not apply to +read-only operations, such as `map.get()`. If a partition primary replica member crashes before +replying to a read-only operation, the operation is retried on the new owner of the primary replica. + +=== Best-Effort Consistency + +The replication algorithm for AP data structures enables Hazelcast clusters to offer high throughput. +However, due to temporary situations in the system, such as +network interruption, backup replicas can miss some updates and +diverge from the primary. Backup replicas can also hit +VM or long GC pauses, and fall behind the primary, which is a situation called as +_replication lag_. If a Hazelcast partition primary replica member crashes while +there is a replication lag between itself and the backups, strong consistency of the data can be lost. + +Please note that CP systems can have similar problems as well. +However, in a CP system, once a replica performs +an update locally (i.e., _commits_ the update), the underlying consensus algorithm guarantees +durability of the update for the rest of the execution. + +On the other hand, in AP systems like Hazelcast, a replica can perform +an update locally, even if the update is not to be performed on other replicas. +This is a fair trade-off to reduce amount of coordination among replicas and +maintain high throughput & high availability of the system. +These systems employ additional measurements to maintain consistency in a +best-effort manner. In this regard, Hazelcast tries to minimize the effect of +such scenarios using an active anti-entropy solution as follows: + +* Each Hazelcast member runs a periodic task in the background. +* For each primary replica it is assigned, it creates a summary information and +sends it to the backups. +* Then, each backup member compares the summary information with its own data to +see if it is up-to-date with the primary. +* If a backup member detects a missing update, it triggers +the synchronization process with the primary. \ No newline at end of file diff --git a/docs/modules/architecture/partials/nav.adoc b/docs/modules/architecture/partials/nav.adoc index 44cd7c3f1..50f71fd1c 100644 --- a/docs/modules/architecture/partials/nav.adoc +++ b/docs/modules/architecture/partials/nav.adoc @@ -1,12 +1,6 @@ * Architecture -** xref:overview:architecture.adoc[Overview] -** xref:overview:data-partitioning.adoc[] -** Consistency and Replication -*** xref:consistency-and-replication:consistency.adoc[] -*** xref:consistency-and-replication:replication-algorithm.adoc[] -*** xref:consistency-and-replication:invocation-lifecycle.adoc[] -*** xref:consistency-and-replication:exactly-once-execution.adoc[] -*** xref:consistency-and-replication:indeterminateoperationstateexception.adoc[] +** xref:architecture:architecture.adoc[Overview] +** xref:architecture:data-partitioning.adoc[] ** xref:architecture:distributed-computing.adoc[] ** xref:architecture:execution-engine.adoc[] ** xref:architecture:event-time-processing.adoc[] diff --git a/docs/modules/cache/pages/overview.adoc b/docs/modules/cache/pages/overview.adoc index 16a62f83d..d55834796 100644 --- a/docs/modules/cache/pages/overview.adoc +++ b/docs/modules/cache/pages/overview.adoc @@ -60,7 +60,7 @@ Allows you to write polyglot applications that can all connect to the same cache |=== -For more information about topologies, see xref:ROOT:topologies.adoc[]. +For more information about topologies, see xref:deploy:choosing-a-deployment-option.adoc[]. == Caching Data on the Client-Side diff --git a/docs/modules/clients/pages/hazelcast-clients.adoc b/docs/modules/clients/pages/hazelcast-clients.adoc index 8ef31d9b0..bb292e002 100644 --- a/docs/modules/clients/pages/hazelcast-clients.adoc +++ b/docs/modules/clients/pages/hazelcast-clients.adoc @@ -31,7 +31,7 @@ handle all the requests. For example, if a member has 4 cores, it will have 80 t == Serialization in Client/Server Mode -In xref:overview:topology.adoc[client/server topologies], you can serialize your data +In xref:deploy:choosing-a-deployment-option.adoc[client/server topologies], you can serialize your data on the client side before sending it to a member. For example, you can serialize data in JSON and add it to a map. This option is useful if you plan on using Hazelcast to store your data in binary. Clients can handle serialization without the members needing to know how to do so. diff --git a/docs/modules/configuration/pages/configuring-declaratively.adoc b/docs/modules/configuration/pages/configuring-declaratively.adoc index 431f2a146..a9b3f5f0d 100644 --- a/docs/modules/configuration/pages/configuring-declaratively.adoc +++ b/docs/modules/configuration/pages/configuring-declaratively.adoc @@ -1,5 +1,5 @@ -= Configuration Files -:description: You can declare settings for Hazelcast members and clients, using a YAML or XML configuration file. Although Hazelcast members and clients read from a single configuration file, you can store settings in multiple files and import them into a single root file to enable reuse. += Member Configuration Files +:description: You can declare settings for Hazelcast members, using a YAML or XML configuration file. Although Hazelcast members read from a single configuration file, you can store settings in multiple files and import them into a single root file to enable reuse. :page-aliases: using-variables.adoc, overriding-configuration-settings.adoc, variable-replacers.adoc {description} @@ -83,12 +83,9 @@ xref:clients:java.adoc#blue-green-deployment-and-disaster-recovery[Java client d == Setting the Path to a Configuration File -Before looking for configuration files either in your working directory or in the classpath, Hazelcast checks the following system properties: +Before looking for configuration files either in your working directory or in the classpath, Hazelcast checks the `hazelcast.config` system property. -- `hazelcast.config` for member configuration -- `hazelcast.client.config` for client configuration - -You may want to set these properties if you have configuration files for different environments and you want to start members with different configurations. For example, you may have a test configuration file and a production configuration file. +You may want to set this property if you have configuration files for different environments and you want to start members with different configurations. For example, you may have a test configuration file and a production configuration file. ``` -Dhazelcast.config=`*` diff --git a/docs/modules/configuration/pages/configuring-programmatically.adoc b/docs/modules/configuration/pages/configuring-programmatically.adoc index 7f07c1b58..d99814dfd 100644 --- a/docs/modules/configuration/pages/configuring-programmatically.adoc +++ b/docs/modules/configuration/pages/configuring-programmatically.adoc @@ -1,9 +1,9 @@ -= Configuration Object -:description: You can use a configuration object (config object) to configure members and clients with any Hazelcast client or the Java member API. To create new instances of members or clients, pass the config object to those instances. += Configuring Hazelcast in Embedded Mode +:description: You can use a configuration object (config object) to configure members with the Java member API. To create new instances of members, pass the config object to those instances. {description} -For example, in Java, you can create a `Config` object, set/change its properties and attributes, +You can create a `Config` object, set/change its properties and attributes, and use it to create new instances of members and clients. The following example configures some network and map properties for a member. [source,java] @@ -55,7 +55,7 @@ Hazelcast configuration also by using its default lookup mechanism. |link:https://docs.hazelcast.org/docs/{full-version}/javadoc/com/hazelcast/config/Config.html#loadFromString-java.lang.String-java.util.Properties-[`Config.loadFromString()`] |Creates `Config` from the provided XML or YAML string content. -|link:https://docs.hazelcast.org/docs/5.0.2/javadoc/com/hazelcast/config/Config.html#loadFromStream-java.io.InputStream-java.util.Properties-[`Config.loadFromStream()`] +|link:https://docs.hazelcast.org/docs/{full-version}/javadoc/com/hazelcast/config/Config.html#loadFromStream-java.io.InputStream-java.util.Properties-[`Config.loadFromStream()`] |Creates `Config` from the provided XML or YAML stream content. |link:https://docs.hazelcast.org/docs/{full-version}/javadoc/com/hazelcast/config/XmlConfigBuilder.html[] @@ -80,7 +80,7 @@ XML:: - link:https://docs.hazelcast.org/docs/{full-version}/javadoc/com/hazelcast/config/FileSystemXmlConfig.html[`FileSystemXmlConfig`] -- link:https://docs.hazelcast.org/docs/5.0.2/javadoc/com/hazelcast/config/UrlXmlConfig.html[`UrlXmlConfig`] +- link:https://docs.hazelcast.org/docs/{full-version}/javadoc/com/hazelcast/config/UrlXmlConfig.html[`UrlXmlConfig`] - link:https://docs.hazelcast.org/docs/{full-version}/javadoc/com/hazelcast/config/InMemoryXmlConfig.html[`InMemoryXmlConfig`] -- @@ -92,7 +92,7 @@ YAML:: - link:https://docs.hazelcast.org/docs/{full-version}/javadoc/com/hazelcast/config/FileSystemYamlConfig.html[`FileSystemYamlConfig`] -- link:https://docs.hazelcast.org/docs/5.0.2/javadoc/com/hazelcast/config/UrlYamlConfig.html[`UrlYamlConfig`] +- link:https://docs.hazelcast.org/docs/{full-version}/javadoc/com/hazelcast/config/UrlYamlConfig.html[`UrlYamlConfig`] - link:https://docs.hazelcast.org/docs/{full-version}/javadoc/com/hazelcast/config/InMemoryYamlConfig.html[`InMemoryYamlConfig`] -- @@ -100,7 +100,7 @@ YAML:: By default, Hazelcast searches for `hazelcast.yaml` or `hazelcast.xml` files in the classpath and -working directory. You can control the name of the files that Hazelcast search for, using the +working directory. You can control the name of the files that Hazelcast searches for, using the relevant system properties such as `hazelcast.config`. This sample code creates a new Jet engine configuration using diff --git a/docs/modules/configuration/pages/dynamic-config.adoc b/docs/modules/configuration/pages/dynamic-config.adoc index 1cf90252a..7e4eebb9f 100644 --- a/docs/modules/configuration/pages/dynamic-config.adoc +++ b/docs/modules/configuration/pages/dynamic-config.adoc @@ -41,7 +41,7 @@ NOTE: To use the REST endpoints you need an Enterprise license. | Update a Hazelcast Enterprise license | No -| xref:deploy:updating-license-rest.adoc[Yes] +| xref:deploy:enterprise-licenses.adoc#update[Yes] | xref:dynamic-config-update-and-reload.adoc[Yes] | No | xref:{page-latest-supported-mc}@management-center:clusters:update-config.adoc[Yes] diff --git a/docs/modules/consistency-and-replication/pages/consistency.adoc b/docs/modules/consistency-and-replication/pages/consistency.adoc deleted file mode 100644 index 07fb4e5a0..000000000 --- a/docs/modules/consistency-and-replication/pages/consistency.adoc +++ /dev/null @@ -1,44 +0,0 @@ -= Consistency and Replication Model - -== A Brief Overview of Consistency and Replication in Distributed Systems - -Partitioning and replication are the two common techniques used together in -distributed databases to achieve scalable, available and transparent data distribution. -The data space is divided into partitions, each of which contains -a distinct portion of the overall data set. For these partitions, multiple copies called replicas -are created. Partition replicas are distributed among the cluster members. -Each member is assigned to at most a single replica for a partition. -In this setting, different replication techniques can be used to access the data and -keep the replicas in sync on updates. The technique being used directly affects -the guarantees and properties a distributed data store provides, due to -the CAP (**C**onsistency, **A**vailability and **P**artition Tolerance) principle. - -One aspect of replication techniques is about where a replicated data set is -accessed and updated. For instance, primary-copy systems first elect -a replica, which can be called as primary, master, etc., and use that -replica to access the data. Changes in the data on the primary replica are -propagated to other replicas. This approach has different namings, such as -_primary-copy_, _single-master_, _passive replication_. The primary-copy technique is -a powerful model as it prevents conflicts, deadlocks among the replicas. -However, primary replicas can become bottlenecks. -On the other hand, we can have a different technique by eliminating -the primary-copy and treating each replica as equal. These systems can -achieve a higher level of availability as a data entry can be accessed and -updated using any replica. However, it can become more difficult to keep -the replicas in sync with each other. - -Replication techniques also differ in how updates are propagated among replicas. -One option is to update each replica as part of a single atomic transaction, called as -_eager replication_ or _synchronous replication_. Consensus algorithms apply -this approach to achieve strong consistency on a replicated data set. -The main drawback is the amount of coordination and communication required while -running the replication algorithm. CP systems implement consensus algorithms under the hood. -Another option is the _lazy replication_ technique, which is also called as -_asynchronous replication_. Lazy replication algorithms execute updates on replicas with -separate transactions. They generally work with best-effort. -By this way, the amount of coordination among the replicas are degraded and -data can be accessed in a more performant manner. Yet, it can happen that -a particular update is executed on some replicas but not on others, which causes -replicas to diverge. Such problems can be resolved with different approaches, such as -_read-repair_, _write-repair_, _anti-entropy_. Lazy replication techniques are -popular among AP systems. \ No newline at end of file diff --git a/docs/modules/consistency-and-replication/pages/exactly-once-execution.adoc b/docs/modules/consistency-and-replication/pages/exactly-once-execution.adoc deleted file mode 100644 index b16e4c8e6..000000000 --- a/docs/modules/consistency-and-replication/pages/exactly-once-execution.adoc +++ /dev/null @@ -1,18 +0,0 @@ -= Exactly-once, At-least-once or At-most-once Execution - -Hazelcast, as an AP product, does not provide the exactly-once guarantee. -In general, Hazelcast tends to be an at-least-once solution. - -In the following failure case, exactly-once guarantee can be broken: -When the target member of a pending invocation leaves the cluster while -the invocation is waiting for a response, that invocation is re-submitted to -its new target due to the new partition table. It can be that, it has -already been executed on the leaving member and backup updates are propagated to -the backup replicas, but the response is not received by the caller. -If that happens, the operation will be executed twice. - -In the following failure case, invocation state becomes indeterminate: -As explained above, when an invocation does not receive a response in time, -invocation fails with an `OperationTimeoutException`. This exception does not -say anything about outcome of the operation, that means operation may not be -executed at all, it may be executed once or twice (due to member left case explained above). \ No newline at end of file diff --git a/docs/modules/consistency-and-replication/pages/indeterminateoperationstateexception.adoc b/docs/modules/consistency-and-replication/pages/indeterminateoperationstateexception.adoc deleted file mode 100644 index 9a7b2bfc2..000000000 --- a/docs/modules/consistency-and-replication/pages/indeterminateoperationstateexception.adoc +++ /dev/null @@ -1,39 +0,0 @@ -= IndeterminateOperationStateException - -As described in xref:invocation-lifecycle.adoc[Invocation Lifecycle] section, -for partition-based *mutating* invocations, such as `map.put()`, -a caller waits with a timeout for the operation that is executed on -corresponding partition's primary replica and backup replicas, based on -the sync backup configuration of the distributed data structure. -Hazelcast 3.9 introduces a new mechanism to detect indeterminate situations while -making such invocations. If `hazelcast.operation.fail.on.indeterminate.state` system property is -enabled, a *mutating* invocation throws `IndeterminateOperationStateException` when -it encounters the following cases: - -- The operation fails on partition primary replica member with `MemberLeftException`. -In this case, the caller may not determine the status of the operation. -It could happen that the primary replica executes the operation, but fails before -replicating it to all the required backup replicas. Even if the caller receives -backup acks from some backup replicas, it cannot decide if it has received -all required ack responses, since it does not know how many acks it should wait for. - -- There is at least one missing ack from the backup replicas for the given timeout duration. -In this case, the caller knows that the operation is executed on the primary replica, -but some backup may have missed it. It could be also a false-positive, -if the backup timeout duration is configured with a very small value. -However, Hazelcast's active anti-entropy mechanism eventually kicks in and -resolves durability of the write on all available backup replicas as long as -the primary replica member is alive. - -When an invocation fails with `IndeterminateOperationStateException`, -the system does not try to rollback the changes which are executed on healthy replicas. -Effect of a failed invocation may be even observed by another caller, -if the invocation has succeeded on the primary replica. -Hence, this new behavior does not guarantee linearizability. -However, if an invocation completes without `IndeterminateOperationStateException` when -the configuration is enabled, it is guaranteed that the operation has been -executed exactly-once on the primary replica and specified number of backup replicas of the partition. - -Please note that `IndeterminateOperationStateException` does not apply to -read-only operations, such as `map.get()`. If a partition primary replica member crashes before -replying to a read-only operation, the operation is retried on the new owner of the primary replica. \ No newline at end of file diff --git a/docs/modules/consistency-and-replication/pages/invocation-lifecycle.adoc b/docs/modules/consistency-and-replication/pages/invocation-lifecycle.adoc deleted file mode 100644 index 66ee679a0..000000000 --- a/docs/modules/consistency-and-replication/pages/invocation-lifecycle.adoc +++ /dev/null @@ -1,53 +0,0 @@ -= Invocation Lifecycle - -When a write is requested with the methods, such as -`map.put()` or `queue.offer()`, a write operation is submitted to -the Hazelcast member that owns the primary replica of the specific partition. -Partition of an operation is determined based on a parameter (key of an entry or -name of the data structure, etc.) related to that operation depending on -the data structure. Target Hazelcast member is figured out by looking up -a local partition assignment/ownership table, which is updated on -each partition migration and broadcasted to all cluster eventually. - -When a Hazelcast member receives a partition specific operation, -it executes the operation and propagates it to backup replica(s) with -a logical timestamp. Number of backups for each operation depends on -the data structure and its configuration. See -xref:performance:threading-model.adoc#operation-threading[Threading Model - Operation Threading] for threading details. - -Two types of backup replication are available: _sync_ and _async_. -Despite what their names imply, both types are still implementations of -the lazy (async) replication model. The only difference between -_sync_ and _async_ is that, the former makes the caller block until -backup updates are applied by backup replicas and acknowledgments are sent back to -the caller, but the latter is just fire & forget. Number of sync and -async backups are defined in the data structure configurations, and you can use -a combination of sync and async backups. - -When backup updates are propagated, response of the execution including -number of sync backup updates is sent to the caller and after receiving -the response, caller waits to receive the specified number of -sync backup acknowledgements for a predefined timeout. -This timeout is 5 seconds by default and defined by -the system property `hazelcast.operation.backup.timeout.millis` -(see xref:ROOT:system-properties.adoc[System Properties appendix]). - -A backup update can be missed because of a few reasons, such as -a stale partition table information about a backup replica member, -network interruption, or a member crash. That's why sync backup acks require -a timeout to give up. Regardless of being a sync or async backup, if a backup update is missed, -the periodically running anti-entropy mechanism detects the inconsistency and -synchronizes backup replicas with the primary. Also the graceful shutdown procedure ensures -that all backup replicas for partitions whose primary replicas are assigned to -the shutting down member will be consistent. - -In some cases, although the target member of an invocation is assumed to be -alive by the failure detector, the target may not execute the operation or -send the response back in time. Network splits, long pauses caused by -high load, GC or I/O (disk, network) can be listed as a few possible reasons. -When an invocation doesn't receive any response from the member that owns -primary replica, then invocation fails with an `OperationTimeoutException`. -This timeout is 2 minutes by default and defined by -the system property `hazelcast.operation.call.timeout.millis` -(xref:ROOT:system-properties.adoc[System Properties appendix]). -When timeout is passed, result of the invocation will be indeterminate. \ No newline at end of file diff --git a/docs/modules/consistency-and-replication/pages/replication-algorithm.adoc b/docs/modules/consistency-and-replication/pages/replication-algorithm.adoc deleted file mode 100644 index fee80f8d1..000000000 --- a/docs/modules/consistency-and-replication/pages/replication-algorithm.adoc +++ /dev/null @@ -1,74 +0,0 @@ -= Hazelcast's Replication Algorithm - -The discussion here generally applies to any system that maintains multiple copies of -a data set. It applies to Hazelcast as well. In the context of CAP principle, **Hazelcast offers -AP and CP functionality with different data structure implementations**. -Data structures exposed under `HazelcastInstance` API are all *AP* data structures. -Hazelcast also contains a *CP* subsystem, built on the Raft consensus algorithm and -accessed via `HazelcastInstance.getCPSubsytem()` which provides *CP* data structures and APIs. - -NOTE: The replication algorithm and consistency model explained below apply -to AP data structures only. For CP subsystem and CP data structures, -see the xref:cp-subsystem:cp-subsystem.adoc[CP Subsystem] section. - -For AP data structures, Hazelcast employs the combination of primary-copy and -configurable lazy replication techniques. As briefly described in -the xref:overview:data-partitioning.adoc[Data Partitioning section], each data entry is mapped to -a single Hazelcast partition and put into replicas of that partition. One of -the replicas is elected as the primary replica, which is responsible for -performing operations on that partition. When you read or -write a map entry, you transparently talk to the Hazelcast member to which -primary replica of the corresponding partition is assigned. -By this way, each request hits the most up-to-date version of -a particular data entry in a stable cluster. Backup replicas stay -in standby mode until the primary replica fails. -Upon failure of the primary replica, one of the backup replicas is promoted to the primary role. - -With _lazy replication_, when the primary replica receives -an update operation for a key, it executes the update locally and -propagates it to backup replicas. It marks each update with -a logical timestamp so that backups apply them in the correct order and -converge to the same state with the primary. Backup replicas can be used to -scale reads with -no strong consistency but monotonic reads guarantee. See xref:data-structures:backing-up-maps.adoc[]. - -Hazelcast offers features such as _SplitBrainProtection_, _ILock_ and _AtomicLong_. -In the journey of being a highly elastic, dynamic and easy to -use product, Hazelcast tries to provide best-effort consistency guarantees without -being a complete CP solution. Therefore, we recommend these features to be used for -efficiency purposes in general, instead of correctness. For instance, they can be used to -prevent to run a resource-extensive computation multiple times, which would not -create any correctness problem if runs more than once. See the -<> and xref:network-partitioning:network-partitioning.adoc[Network Partitioning] sections -for more information. - -== Best-Effort Consistency - -Hazelcast's replication technique enables Hazelcast clusters to offer high throughput. -However, due to temporary situations in the system, such as -network interruption, backup replicas can miss some updates and -diverge from the primary. Backup replicas can also hit -VM or long GC pauses, and fall behind the primary, which is a situation called as -_replication lag_. If a Hazelcast partition primary replica member crashes while -there is a replication lag between itself and the backups, strong consistency of the data can be lost. - -Please note that CP systems can have similar problems as well. -However, in a CP system, once a replica performs -an update locally (i.e., _commits_ the update), the underlying consensus algorithm guarantees -durability of the update for the rest of the execution. - -On the other hand, in AP systems like Hazelcast, a replica can perform -an update locally, even if the update is not to be performed on other replicas. -This is a fair trade-off to reduce amount of coordination among replicas and -maintain high throughput & high availability of the system. -These systems employ additional measurements to maintain consistency in a -best-effort manner. In this regard, Hazelcast tries to minimize the effect of -such scenarios using an active anti-entropy solution as follows: - -* Each Hazelcast member runs a periodic task in the background. -* For each primary replica it is assigned, it creates a summary information and -sends it to the backups. -* Then, each backup member compares the summary information with its own data to -see if it is up-to-date with the primary. -* If a backup member detects a missing update, it triggers -the synchronization process with the primary. \ No newline at end of file diff --git a/docs/modules/consistency-and-replication/partials/nav.adoc b/docs/modules/consistency-and-replication/partials/nav.adoc deleted file mode 100644 index 479b353fa..000000000 --- a/docs/modules/consistency-and-replication/partials/nav.adoc +++ /dev/null @@ -1,5 +0,0 @@ -* xref:consistency-and-replication:consistency.adoc[] -** xref:consistency-and-replication:replication-algorithm.adoc[] -** xref:consistency-and-replication:invocation-lifecycle.adoc[] -** xref:consistency-and-replication:exactly-once-execution.adoc[] -** xref:consistency-and-replication:indeterminateoperationstateexception.adoc[] \ No newline at end of file diff --git a/docs/modules/data-structures/pages/reading-map-metrics.adoc b/docs/modules/data-structures/pages/reading-map-metrics.adoc index 8bda80e8d..080dff110 100644 --- a/docs/modules/data-structures/pages/reading-map-metrics.adoc +++ b/docs/modules/data-structures/pages/reading-map-metrics.adoc @@ -7,7 +7,7 @@ == Getting Statistics about All Map Entries -To get statistics for all map entries that are owned by a member, use the `getLocalMapStats()` method of the Java member API (xref:overview:topology.adoc[embedded mode]). This method returns data only if the `statistics-enabled` element is set to `true`, which is the default. +To get statistics for all map entries that are owned by a member, use the `getLocalMapStats()` method of the Java member API (xref:deploy:choosing-a-deployment-option.adoc[embedded mode]). This method returns data only if the `statistics-enabled` element is set to `true`, which is the default. NOTE: If you disable map statistics, statistics will not be available via the API, or in xref:{page-latest-supported-mc}@management-center:ROOT:index.adoc[Hazelcast Management Center]. diff --git a/docs/modules/deploy/pages/choosing-a-deployment-option.adoc b/docs/modules/deploy/pages/choosing-a-deployment-option.adoc new file mode 100644 index 000000000..d25f04d0d --- /dev/null +++ b/docs/modules/deploy/pages/choosing-a-deployment-option.adoc @@ -0,0 +1,80 @@ += Choosing an Application Topology +:description: Hazelcast offers two topologies to use Hazelcast: embedded in a Java application or in a client/server topology. To help you choose a topology that meets your requirements, this topic describes each option. +:page-aliases: ROOT:topologies.adoc + +You can deploy Hazelcast in two topologies, depending on where you want the Hazelcast members to run: + +- *Embedded mode:* Hazelcast members run in the same Java process as your application. +- *Client/server mode:* Hazelcast members run remotely outside of your application, allowing you to scale them independently and connect to them through any of the supported clients. + +TIP: If you don't want to deploy your own self-managed cluster, try xref:cloud:ROOT:index.adoc[Hazelcast Cloud] for free. + +== Comparing Topologies + +This table compares embedded mode and client/server mode in terms of latency, scalability, and available programming languages. + +[cols="h,a,a"] +.Comparison of Hazelcast Topologies +|=== +| |Embedded|Client/Server + +|Low-latency +|Yes +|If used with xref:clients:java.adoc#configuring-client-near-cache[Near Cache] to store frequently used data in the client's local memory. + +|Scalability +|The application and the cluster must be scaled together +|You can scale the cluster independently of your application + +|Supported clients +| Java +| +[.tablelist] +include::ROOT:partial$clients.adoc[] + +|=== + +== Embedded Mode + +Hazelcast is written in Java, which means that Java developers can integrate, or embed, it as a library in their applications. This topology is called _embedded_ mode. + +All you need to do is add the Hazelcast JAR file to the application’s classpath and start a member with the Java API. Then all the benefits of Hazelcast are available right inside the application. + +image::ROOT:single-embedded.svg[A single Hazelcast member is embedded in a Java application,200] + +In this mode, each instance of your application +starts a Hazelcast member, and all members automatically form their own cluster. To interact with the cluster, you use a local member in your application. + +image::ROOT:multi-embedded.svg[A cluster of Hazelcast members are embedded in Java applications,600] + +If you are a Java developer working in an IDE, this is a convenient +way to run a cluster on your development machine. There is nothing +stopping you from starting your program several times to get a +cluster on the same physical machine. + +The main advantage of embedded mode is that all your Java +classes are visible to Hazelcast. This way, you don't need to add classes to your members for tasks such as entry processing or submitting jobs. + +xref:getting-started:get-started-java.adoc[Get started with Hazelcast embedded]. + +== Client/Server Mode + +In client/server mode, the Hazelcast cluster is independant of your application, which means that they can be independently scaled. +To interact with the cluster, you use one of the available clients. + +image::ROOT:client-server-pc-mobile.svg[Client/Server Mode,600] + +Client/server mode has advantages, including more predictable and reliable +performance, easier identification of problem causes and, most importantly, better scalability. +When you need to scale in this mode, just add more Hazelcast members. You +can address client and server scalability concerns separately. + +Client/server mode also provides higher availability. If there are any issues in running either +your application or Hazelcast members, the issue and any potential failure is isolated from the rest +of the application and members. + +Get started with client/server: + +- xref:getting-started:get-started-docker.adoc[Docker] +- xref:getting-started:get-started-cli.adoc[CLI] +- xref:getting-started:get-started-binary.adoc[Binary] \ No newline at end of file diff --git a/docs/modules/deploy/pages/installing-using-docker.adoc b/docs/modules/deploy/pages/deploying-with-docker.adoc similarity index 93% rename from docs/modules/deploy/pages/installing-using-docker.adoc rename to docs/modules/deploy/pages/deploying-with-docker.adoc index 3a849a679..fcdb68820 100644 --- a/docs/modules/deploy/pages/installing-using-docker.adoc +++ b/docs/modules/deploy/pages/deploying-with-docker.adoc @@ -1,38 +1,9 @@ = Deploying on Docker :description: In this deployment guide, you will learn how to form a cluster from Hazelcast member containers running on more than one Docker host in the same local area network (LAN). - -[[installing-using-docker]] +:page-aliases: installing-using-docker.adoc {description} -== Pulling the Docker Image - -Hazelcast host a Docker image for the open source edition as well as the Enterprise edition on Docker Hub. - -[tabs] -==== -Open Source:: -+ --- -// tag::pull-docker-oss[] -[source,bash,subs="attributes+"] ----- -docker pull hazelcast/hazelcast:{full-version} ----- -// end::pull-docker-oss[] --- -Enterprise:: -+ --- -// tag::pull-docker-ee[] -[source,bash,subs="attributes+"] ----- -docker pull hazelcast/hazelcast-enterprise:{full-version} ----- -// end::pull-docker-ee[] --- -==== - == Starting a Cluster The default network mode in Docker is bridge. In this mode, all containers run in a single network that is not accessible outside of the Docker host. If you run member Docker containers on different hosts, you must allow them to discover and connect to each other using one of the following methods: @@ -269,7 +240,7 @@ docker run -v /path/to/log4j2.properties:/opt/hazelcast/log4j2.properties hazelc == Building a Custom Image from the Slim Image -Hazelcast offers a xref:installing-upgrading.adoc#full-slim[slim] Docker image that +Hazelcast offers a xref:getting-started:editions.adoc#full-slim[slim] Docker image that contains only the core Hazelcast engine. When image size is a concern, you can use it as the starting point to build your custom image with just the extensions you need. diff --git a/docs/modules/deploy/pages/enterprise-licenses.adoc b/docs/modules/deploy/pages/enterprise-licenses.adoc index 566a5cb37..822b053c6 100644 --- a/docs/modules/deploy/pages/enterprise-licenses.adoc +++ b/docs/modules/deploy/pages/enterprise-licenses.adoc @@ -1,13 +1,13 @@ = Managing Enterprise License Keys :description: To use Hazelcast Enterprise, you need an Enterprise license key. This topic explains how to get a license key, install it on a member, and manage it. -:page-aliases: installing-license-keys.adoc, monitoring-license-keys.adoc, renewing-license-keys.adoc +:page-aliases: installing-license-keys.adoc, monitoring-license-keys.adoc, renewing-license-keys.adoc, deploy:updating-license-rest.adoc :page-enterprise: true {description} -// tag::get-license[] == Getting an Enterprise License Key +// tag::get-license[] Hazelcast Enterprise requires a license key. You can get a 30-day trial license from the link:https://hazelcast.com/get-started[Hazelcast website]. @@ -21,9 +21,7 @@ Hazelcast Enterprise requires a license key. You can get a - ``: Maximum number of members that may use the license at the same time. - ``: Machine-readable license key. -The first two strings up to the second hash (#) character are optional. These strings just provide a way for you to know important information about the license key. - -For example, both of the following formats are valid: +The first two strings up to the second hash (#) character are optional. These strings provide information about the license key. For example, both of the following formats are valid: .Full-form license ``` @@ -39,10 +37,10 @@ NOTE: These licenses are examples and will not work if you install them on membe // end::get-license[] -// tag::install-license[] -== Installing a License Key on Members +== Configuring a License Key on Members -To use Hazelcast Enterprise, you need to install a license +// tag::install-license[] +To use Hazelcast Enterprise, you need to configure a license key on all members in a cluster. NOTE: Hazelcast Enterprise license keys are required only for members. @@ -223,6 +221,43 @@ Reach out to Hazelcast Support to resolve any issues with an expired license. // tag::update-license[] +[[update]] == Updating a License Key at Runtime -You can use xref:configuration:dynamic-config.adoc[dynamic configuration] to update a license key at runtime. \ No newline at end of file +You can use any xref:configuration:dynamic-config.adoc[dynamic configuration] option to update a license key at runtime. + +To use the dedicated `/license` endpoint, do the following: + +. xref:clients:rest.adoc#using-the-rest-endpoint-groups[Enable the REST API]. ++ +WARNING: By default, if a member shuts down, its license key will not be persisted. To persist the license key, xref:configuration:dynamic-config.adoc#persistence[enable persistence for dynamic configuration]. + +. Send a `POST` request to the `/license` endpoint. ++ +```bash +curl --data "{cluster-name}&{password}&{license}" http://localhost:5001/hazelcast/rest/license +``` ++ +NOTE: The request parameters must be URL-encoded as described in the xref:clients:rest.adoc[REST Client section]. + +This command updates the license on all running Hazelcast members of the cluster. +If successful, you should see something like the following: + +```json +{ + "status": "success", + "licenseInfo": { + "expiryDate": 1560380399161, + "maxNodeCount": 10, + "type": -1, + "companyName": "ExampleCompany", + "ownerEmail": "info@example.com", + "keyHash": "ml/u6waTNQ+T4EWxnDRykJpwBmaV9uj+skZzv0SzDhs=" + }, + "message": "License updated at run time - please make sure to update the license in the persistent configuration to avoid losing the changes on restart." +} +``` + +If the update fails on some members, the whole operation fails, leaving the cluster in +a potentially inconsistent state where some members have been switched to the new license +while some have not. You must resolve this situation manually. \ No newline at end of file diff --git a/docs/modules/deploy/pages/installing-upgrading.adoc b/docs/modules/deploy/pages/installing-upgrading.adoc deleted file mode 100644 index c85591177..000000000 --- a/docs/modules/deploy/pages/installing-upgrading.adoc +++ /dev/null @@ -1,447 +0,0 @@ -= Installing Hazelcast On-Premise -:page-aliases: installing-using-download-archives.adoc - -You can install and deploy Hazelcast using the options described -in this section. - -[[installing-using-cli]] -== Package Manager - -This is the fastest way of getting Hazelcast running. This option is for development purposes. For production, use the other deployment options explained on this page. - -// tag::cli[] -. To install Hazelcast, use one of the following methods, depending on your operating system: -+ -[tabs] -==== -Mac:: -+ --- - -To install on macOS, use the Homebrew package manager. - -[source,bash,subs="attributes+"] ----- -brew tap hazelcast/hz -brew install hazelcast@{full-version} ----- --- - -Linux:: -+ --- - -To install on Linux, you can use either of these package managers, depending on your Linux distribution: -ifdef::snapshot[] -.Debian -[source,shell] ----- -wget -qO - https://repository.hazelcast.com/api/gpg/key/public | gpg --dearmor | sudo tee /usr/share/keyrings/hazelcast-archive-keyring.gpg > /dev/null -echo "deb [signed-by=/usr/share/keyrings/hazelcast-archive-keyring.gpg] https://repository.hazelcast.com/debian snapshot main" | sudo tee -a /etc/apt/sources.list -sudo apt update && sudo apt install hazelcast ----- - -.RPM -[source,shell] ----- -wget https://repository.hazelcast.com/rpm/snapshot/hazelcast-rpm.repo -O hazelcast-snapshot-rpm.repo -sudo mv hazelcast-snapshot-rpm.repo /etc/yum.repos.d/ -sudo yum install hazelcast ----- -endif::[] -ifndef::snapshot[] -.Debian -[source,shell,subs="attributes+"] ----- -wget -qO - https://repository.hazelcast.com/api/gpg/key/public | gpg --dearmor | sudo tee /usr/share/keyrings/hazelcast-archive-keyring.gpg > /dev/null -echo "deb [signed-by=/usr/share/keyrings/hazelcast-archive-keyring.gpg] https://repository.hazelcast.com/debian stable main" | sudo tee -a /etc/apt/sources.list -sudo apt update && sudo apt install hazelcast={full-version} ----- - -.RPM -[source,shell,subs="attributes+"] ----- -wget https://repository.hazelcast.com/rpm/stable/hazelcast-rpm-stable.repo -O hazelcast-rpm-stable.repo -sudo mv hazelcast-rpm-stable.repo /etc/yum.repos.d/ -sudo yum install hazelcast-{full-version} ----- -endif::[] --- -Windows:: -+ --- -At the moment, Hazelcast does not support any Windows package managers. - -To get started with Hazelcast on Windows, you can use <>, the <>, or <>. --- -==== - -. Check that Hazelcast is installed. -+ -[source,shell] ----- -hz -V ----- -+ -You should see the version of both Hazelcast and the command-line client (CLI). - -For more information about the package manager, see the link:https://github.com/hazelcast/hazelcast-packaging[GitHub repository]. - -// end::cli[] - -[[installing-using-maven]] -== Maven - -// tag::maven-install[] -Hazelcast runs on Java, which means you can add it as a dependency in your Java project. - -The Java package includes both a member API and a Java client API. The member API is for xref:ROOT:topologies.adoc[embedded topologies] where you want to deploy and manage a cluster in the same Java Virtual Machine (JVM) as your applications. The Java client is for connecting to an existing member in a client/server topology. - -. Download and install a xref:deploy:supported-jvms.adoc[supported JDK]. -+ -TIP: We recommend installing the link:https://adoptopenjdk.net[AdoptOpenJDK^]. - -. Add the following to your `pom.xml` file. -// end::maven-install[] -+ -[tabs] -==== -Open Source:: -+ --- -// tag::maven-full-oss[] -ifdef::snapshot[] -[source,xml,subs="attributes+"] ----- - - - snapshot-repository - Maven2 Snapshot Repository - https://oss.sonatype.org/content/repositories/snapshots - - false - - - - - - - com.hazelcast - hazelcast - {full-version} - - ----- -endif::[] -ifndef::snapshot[] -[source,xml,subs="attributes+"] ----- - - - com.hazelcast - hazelcast - {full-version} - - ----- -endif::[] -// end::maven-full-oss[] --- -Enterprise:: -+ --- -// tag::maven-full-ee[] -ifdef::snapshot[] -[source,xml,subs="attributes+"] ----- - - - private-repository - Hazelcast Private Repository - https://repository.hazelcast.com/release/ - - true - - - true - - - - - - - com.hazelcast - hazelcast-enterprise - {full-version} - - ----- -endif::[] -ifndef::snapshot[] -[source,xml,subs="attributes+"] ----- - - - private-repository - Hazelcast Private Repository - https://repository.hazelcast.com/release/ - - true - - - false - - - - - - - com.hazelcast - hazelcast-enterprise - {full-version} - - ----- -endif::[] -// end::maven-full-ee[] --- -==== - -[[installing-using-download-archives]] -== ZIP/TAR Packages - -The ZIP/TAR packages provide the most flexibility and -all the tooling, but takes a little longer to install. - -To download and install a package, do the following: - -=== Open Source - -// tag::download-package-oss[] -[tabs] -==== -Mac:: -+ --- -ifdef::snapshot[] -Go to the link:https://oss.sonatype.org/content/repositories/snapshots/com/hazelcast/hazelcast-distribution/{full-version}/[snapshot repository] and click the download link for the TAR file that has the most up-to-date timestamp in the *Last Modified* columns. -endif::[] -ifndef::snapshot[] -. Download a package from https://hazelcast.com/get-started/download/[hazelcast.com^]. -. Extract the downloaded package. -endif::[] --- -Linux:: -+ --- -ifdef::snapshot[] -Go to the link:https://oss.sonatype.org/content/repositories/snapshots/com/hazelcast/hazelcast-distribution/{full-version}/[snapshot repository] and click the download link for the TAR file that has the most up-to-date timestamp in the *Last Modified* columns. -endif::[] -ifndef::snapshot[] -. Download a package from https://hazelcast.com/get-started/download/[hazelcast.com^]. -. Extract the downloaded package. -endif::[] --- -Windows:: -+ --- -ifdef::snapshot[] -Go to the link:https://oss.sonatype.org/content/repositories/snapshots/com/hazelcast/hazelcast-distribution/{full-version}/[snapshot repository] and click the download link for the ZIP file that has the most up-to-date timestamp in the *Last Modified* columns. -endif::[] -ifndef::snapshot[] -. Download a package from https://hazelcast.com/get-started/download/[hazelcast.com^]. -. Extract the downloaded package. -endif::[] --- -==== -// end::download-package-oss[] - -=== Enterprise - -// tag::download-package-ee[] -[tabs] -==== -Mac:: -+ --- -[source,bash,subs="attributes+"] ----- -curl -L 'https://repository.hazelcast.com/download/hazelcast-enterprise/hazelcast-enterprise-{full-version}.tar.gz' | tar xvzf - ----- --- -Linux:: -+ --- -[source,bash,subs="attributes+"] ----- -wget -O - 'https://repository.hazelcast.com/download/hazelcast-enterprise/hazelcast-enterprise-{full-version}.tar.gz' | tar xvzf - ----- --- -Windows:: -+ --- -Download and extract the link:https://repository.hazelcast.com/download/hazelcast-enterprise/hazelcast-enterprise-{full-version}.zip[Hazelcast ZIP file]. --- -==== -// end::download-package-ee[] - - -[[full-slim]] -== Full and Slim Distributions - -Some installation options offer a full and slim distribution. - -The full distribution contains all available Hazelcast connectors, libraries, and Management Center. - -.What's included in the full distribution -[%collapsible] -==== -[source,plain,subs="attributes+"] ----- -├── LICENSE -├── NOTICE -├── bin -│ ├── common.sh -│ ├── hz-cli -│ ├── hz-cli.bat -│ ├── hz-cluster-admin -│ ├── hz-cluster-cp-admin -│ ├── hz-healthcheck -│ ├── hz-start -│ ├── hz-start.bat -│ ├── hz-stop -│ └── hz-stop.bat -├── config -│ ├── examples -│ │ ├── hazelcast-client-full-example.xml -│ │ ├── hazelcast-client-full-example.yaml -│ │ ├── hazelcast-client.yaml -│ │ ├── hazelcast-full-example.xml -│ │ ├── hazelcast-full-example.yaml -│ │ ├── hazelcast-security-hardened.yaml -│ │ └── hazelcast.yaml -│ ├── hazelcast-client.xml -│ ├── hazelcast.xml -│ ├── jmx_agent_config.yaml -│ ├── jvm-client.options -│ ├── jvm.options -│ └── log4j2.properties -├── custom-lib -│ ├── hazelcast-3-connector-impl-{full-version}.jar -│ ├── hazelcast-3.12.12.jar -│ └── hazelcast-client-3.12.12.jar -├── lib -│ ├── cache-api-1.1.1.jar -│ ├── hazelcast-3-connector-common-{full-version}.jar -│ ├── hazelcast-3-connector-interface-{full-version}.jar -│ ├── hazelcast-{full-version}.jar -│ ├── hazelcast-download.properties -│ ├── hazelcast-hibernate53-2.1.1.jar -│ ├── hazelcast-jet-avro-{full-version}.jar -│ ├── hazelcast-jet-cdc-debezium-{full-version}.jar -│ ├── hazelcast-jet-cdc-mysql-{full-version}.jar -│ ├── hazelcast-jet-cdc-postgres-{full-version}.jar -│ ├── hazelcast-jet-csv-{full-version}.jar -│ ├── hazelcast-jet-elasticsearch-7-{full-version}.jar -│ ├── hazelcast-jet-files-azure-{full-version}.jar -│ ├── hazelcast-jet-files-gcs-{full-version}.jar -│ ├── hazelcast-jet-files-s3-{full-version}.jar -│ ├── hazelcast-jet-grpc-{full-version}.jar -│ ├── hazelcast-jet-hadoop-all-{full-version}.jar -│ ├── hazelcast-jet-kafka-{full-version}.jar -│ ├── hazelcast-jet-kinesis-{full-version}.jar -│ ├── hazelcast-jet-protobuf-{full-version}.jar -│ ├── hazelcast-jet-python-{full-version}.jar -│ ├── hazelcast-jet-s3-{full-version}.jar -│ ├── hazelcast-sql-{full-version}.jar -│ ├── hazelcast-wm-4.0.jar -│ ├── jansi-2.1.0.jar -│ ├── jline-reader-3.19.0.jar -│ ├── jline-terminal-3.19.0.jar -│ ├── jline-terminal-jansi-3.19.0.jar -│ ├── jmx_prometheus_javaagent-0.14.0.jar -│ ├── log4j-api-2.14.0.jar -│ ├── log4j-core-2.14.0.jar -│ ├── log4j-slf4j-impl-2.14.0.jar -│ ├── picocli-3.9.0.jar -│ └── slf4j-api-1.7.30.jar -└── licenses - ├── THIRD-PARTY.txt - ├── apache-v2-license. - ├── attribution.txt - └── hazelcast-community-license.txt -├── management-center -│   ├── ThirdPartyNotices.txt -│   ├── bin -│   │   ├── mc-conf.bat -│   │   ├── mc-conf.sh -│   │   ├── mc-start.cmd -│   │   ├── mc-start.sh -│   │   ├── start.bat -│   │   ├── start.sh -│   │   └── user-lib -│   ├── hazelcast-management-center-{full-version}.jar -│   └── license.txt -└── release_notes.txt ----- -==== - -The slim distribution excludes Management Center and connectors. You can use the slim version -to save memory and later include the additional components, as needed. - -.What's included in the slim distribution -[%collapsible] -==== -[source,plain,subs="attributes+"] ----- -├── LICENSE -├── NOTICE -├── bin -│ ├── common.sh -│ ├── hz-cli -│ ├── hz-cli.bat -│ ├── hz-cluster-admin -│ ├── hz-cluster-cp-admin -│ ├── hz-healthcheck -│ ├── hz-start -│ ├── hz-start.bat -│ ├── hz-stop -│ └── hz-stop.bat -├── config -│ ├── examples -│ │ ├── hazelcast-client-full-example.xml -│ │ ├── hazelcast-client-full-example.yaml -│ │ ├── hazelcast-client.yaml -│ │ ├── hazelcast-full-example.xml -│ │ ├── hazelcast-full-example.yaml -│ │ ├── hazelcast-security-hardened.yaml -│ │ └── hazelcast.yaml -│ ├── hazelcast-client.xml -│ ├── hazelcast.xml -│ ├── jmx_agent_config.yaml -│ ├── jvm-client.options -│ ├── jvm.options -│ └── log4j2.properties -├── lib -│ ├── cache-api-1.1.1.jar -│ ├── hazelcast-{full-version}.jar -│ ├── hazelcast-download.properties -│ ├── hazelcast-hibernate53-2.1.1.jar -│ ├── hazelcast-sql-{full-version}.jar -│ ├── hazelcast-wm-4.0.jar -│ ├── jansi-2.1.0.jar -│ ├── jline-reader-3.19.0.jar -│ ├── jline-terminal-3.19.0.jar -│ ├── jline-terminal-jansi-3.19.0.jar -│ ├── jmx_prometheus_javaagent-0.14.0.jar -│ ├── log4j-api-2.14.0.jar -│ ├── log4j-core-2.14.0.jar -│ ├── log4j-slf4j-impl-2.14.0.jar -│ ├── picocli-3.9.0.jar -│ └── slf4j-api-1.7.30.jar -└── licenses - ├── THIRD-PARTY.txt - ├── apache-v2-license.txt - └── hazelcast-community-license.txt ----- -==== diff --git a/docs/modules/deploy/pages/running-in-modular-java.adoc b/docs/modules/deploy/pages/running-in-modular-java.adoc deleted file mode 100644 index dc044bf9c..000000000 --- a/docs/modules/deploy/pages/running-in-modular-java.adoc +++ /dev/null @@ -1,60 +0,0 @@ -= Running in Modular Java -[[running-in-modular-java]] - -If you're using JDK 11 or later, you can use Hazelcast as a module in the http://openjdk.java.net/projects/jigsaw/[Java Platform Module System] (JPMS). - -To run your application with Hazelcast -libraries on the modulepath, use the following module name: - -* `com.hazelcast.core` for `hazelcast-{full-version}.jar` and -`hazelcast-enterprise-{full-version}.jar` - -The JPMS comes with stricter visibility rules. It affects -Hazelcast which uses the internal Java API to reach the best performance results. - -Hazelcast needs the `java.se` module and access to the following Java -packages: - -* `java.base/jdk.internal.ref` -* `java.base/java.nio` _(reflective access)_ -* `java.base/sun.nio.ch` _(reflective access)_ -* `java.base/java.lang` _(reflective access)_ -* `jdk.management/com.ibm.lang.management.internal` _(reflective access)_ -* `jdk.management/com.sun.management.internal` _(reflective access)_ -* `java.management/sun.management` _(reflective access)_ - -You can provide access to these packages by using -`--add-exports` and `--add-opens` (for reflective access) Java arguments. - -== Running a Member on the Classpath - -[source,bash,subs="attributes+"] ----- -java --add-modules java.se \ - --add-exports java.base/jdk.internal.ref=ALL-UNNAMED \ - --add-opens java.base/java.lang=ALL-UNNAMED \ - --add-opens java.base/java.nio=ALL-UNNAMED \ - --add-opens java.base/sun.nio.ch=ALL-UNNAMED \ - --add-opens java.management/sun.management=ALL-UNNAMED \ - --add-opens jdk.management/com.ibm.lang.management.internal=ALL-UNNAMED \ - --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED \ - -jar hazelcast-{full-version}.jar ----- - -== Running a Member on the Modulepath - -[source,bash] ----- -java --add-modules java.se \ - --add-exports java.base/jdk.internal.ref=com.hazelcast.core \ - --add-opens java.base/java.lang=com.hazelcast.core \ - --add-opens java.base/java.nio=com.hazelcast.core \ - --add-opens java.base/sun.nio.ch=com.hazelcast.core \ - --add-opens java.management/sun.management=com.hazelcast.core \ - --add-opens jdk.management/com.ibm.lang.management.internal=com.hazelcast.core \ - --add-opens jdk.management/com.sun.management.internal=com.hazelcast.core \ - --module-path lib \ <1> - --module com.hazelcast.core/com.hazelcast.core.server.HazelcastMemberStarter ----- - -<1> This example expects the `hazelcast-{full-version}.jar` file in the `lib` directory. \ No newline at end of file diff --git a/docs/modules/deploy/pages/updating-license-rest.adoc b/docs/modules/deploy/pages/updating-license-rest.adoc deleted file mode 100644 index 49196fe42..000000000 --- a/docs/modules/deploy/pages/updating-license-rest.adoc +++ /dev/null @@ -1,41 +0,0 @@ -= Updating a License Key at Runtime with the REST API -:description: You can use the /license endpoint of the REST API to update the license on a running member. - -You can use the `/license` endpoint of the REST API to update the license on a running member. - -. xref:clients:rest.adoc#using-the-rest-endpoint-groups[Enable the REST API]. - -WARNING: By default, if a member shuts down, its license key will not be persisted. To persist the license key, xref:configuration:dynamic-config.adoc#persistence[enable persistence for dynamic configuration]. - -== Update a License Key - -To update the license of a running cluster, send a `POST` -request to the `/license` endpoint. - -```bash -curl --data "{cluster-name}&{password}&{license}" http://localhost:5001/hazelcast/rest/license -``` - -NOTE: The request parameters must be URL-encoded as described in the xref:clients:rest.adoc[REST Client section]. - -This command updates the license on all running Hazelcast members of the cluster. -If successful, you should see something like the following: - -```json -{ - "status": "success", - "licenseInfo": { - "expiryDate": 1560380399161, - "maxNodeCount": 10, - "type": -1, - "companyName": "ExampleCompany", - "ownerEmail": "info@example.com", - "keyHash": "ml/u6waTNQ+T4EWxnDRykJpwBmaV9uj+skZzv0SzDhs=" - }, - "message": "License updated at run time - please make sure to update the license in the persistent configuration to avoid losing the changes on restart." -} -``` - -If the update fails on some members, the whole operation fails, leaving the cluster in -a potentially inconsistent state where some members have been switched to the new license -while some have not. You must resolve this situation manually. \ No newline at end of file diff --git a/docs/modules/deploy/partials/nav.adoc b/docs/modules/deploy/partials/nav.adoc index 2e13f4ce8..23f7f7eb1 100644 --- a/docs/modules/deploy/partials/nav.adoc +++ b/docs/modules/deploy/partials/nav.adoc @@ -3,19 +3,5 @@ ** xref:clusters:discovering-by-tcp.adoc[] ** xref:clusters:discovering-by-multicast.adoc[] ** xref:clusters:discovering-native-clients.adoc[] -* Deploying a Cluster -** xref:deploy:installing-upgrading.adoc[] -** xref:deploy:deploying-hazelcast-cloud.adoc[] -** Public Cloud Environments -*** xref:deploy:deploying-in-cloud.adoc[Overview] -*** xref:deploy:deploying-on-aws.adoc[] -*** xref:deploy:deploying-on-azure.adoc[] -*** xref:deploy:deploying-on-gcp.adoc[] -** xref:deploy:installing-using-docker.adoc[] -** Kubernetes -*** xref:deploy:deploying-in-kubernetes.adoc[Overview] -*** xref:deploy:configuring-kubernetes.adoc[] -*** xref:deploy:scaling-and-upgrading-kubernetes.adoc[] -*** xref:operator:ROOT:get-started.adoc[Hazelcast Platform Operator] -** xref:deploy:deploying-in-vmware-tanzu.adoc[] -** xref:deploy:running-in-modular-java.adoc[] + + diff --git a/docs/modules/getting-started/pages/authenticate-clients.adoc b/docs/modules/getting-started/pages/authenticate-clients.adoc index a82e3cc68..7d2c9a356 100644 --- a/docs/modules/getting-started/pages/authenticate-clients.adoc +++ b/docs/modules/getting-started/pages/authenticate-clients.adoc @@ -1,5 +1,6 @@ = Authenticate Client Connections :description: This tutorial introduces you to member security by setting up authentication for client connections. At the end of this tutorial, you'll know how to configure member and client security settings to allow only authorized clients to connect to a cluster. +:page-enterprise: true {description} @@ -19,11 +20,8 @@ To complete this tutorial, you need the following: |=== |Prerequisites|Useful resources -|Trial license -|https://trialrequest.hazelcast.com/[Hazelcast website] - -|Docker image for the full Hazelcast distribution -|xref:get-started-enterprise.adoc[Install Hazelcast Enterprise] +|Docker image for Hazelcast Enterprise and an Enterprise license +|xref:getting-started:enterprise-overview.adoc[] |=== diff --git a/docs/modules/getting-started/pages/before-you-begin.adoc b/docs/modules/getting-started/pages/before-you-begin.adoc deleted file mode 100644 index 4eda96297..000000000 --- a/docs/modules/getting-started/pages/before-you-begin.adoc +++ /dev/null @@ -1,28 +0,0 @@ -= Before you Install Hazelcast -:description: Hazelcast comes in different formats and distributions to accomodate a variety of use cases and application topologies. Before you install Hazelcast, you should learn about the different options and choose one. - -{description} - -== Choosing a Distribution - -Hazelcast is available in two distributions: - -- *Full:* Includes Management Center and all connectors. -- *Slim:* Excludes Management Center and connectors. - -The getting started tutorials use the full distribution to walk you through important features. - -If you later decide that you do not need these features, you can use the slim version to save memory. For more information about these distributions, see xref:deploy:installing-upgrading.adoc#full-slim[Full and Slim Packages]. - -== Choosing an Application Topology - -You can use Hazelcast in two topologies, depending on where you want the Hazelcast servers (members) to run: - -- Client/server mode: Hazelcast members run remotely outside of your application, allowing you to scale them independently and connect to them through any of the supported clients. -- Embedded mode: Hazelcast members run in the same Java process as your application. - -For details about these topologies, see xref:ROOT:topologies.adoc[]. - -To get started with client/server mode, install Hazelcast with <>, <>, or <>. - -To get started with embedded mode, <>. \ No newline at end of file diff --git a/docs/modules/getting-started/pages/blue-green.adoc b/docs/modules/getting-started/pages/blue-green.adoc index efa5ceaca..e5018d0fa 100644 --- a/docs/modules/getting-started/pages/blue-green.adoc +++ b/docs/modules/getting-started/pages/blue-green.adoc @@ -1,5 +1,6 @@ = Deploy Blue-Green Clusters :description: This tutorial introduces you to blue-green cluster deployments. At the end of this tutorial, you'll know how to set up a client to operate continuously even when its connected cluster fails. +:page-enterprise: true {description} @@ -25,11 +26,8 @@ To complete this tutorial, you need the following: |=== |Prerequisites|Useful resources -|Trial license -|https://trialrequest.hazelcast.com/[Hazelcast website] - -|Docker image for the full Hazelcast distribution -|xref:get-started-enterprise.adoc[Install Hazelcast Enterprise] +|Docker image for Hazelcast Enterprise and an Enterprise license +|xref:getting-started:enterprise-overview.adoc[] |Docker network with the name `hazelcast-network` |Use the `docker network create hazelcast-network` command @@ -378,4 +376,4 @@ move to the other tutorials. To shut down the cluster, close the terminals in wh If you're interested in learning more about the topics introduced in this tutorial, see xref:clients:java#blue-green-deployment-and-disaster-recovery[Blue-Green Deployment]. -This is the last tutorial in this Hazelcast Enterprise series. To learn more, see these xref:getting-started:get-started-enterprise.adoc#next-steps[links]. +This is the last tutorial in this Hazelcast Enterprise series. To learn more, see these xref:getting-started:install-hazelcast.adoc#next-steps[links]. diff --git a/docs/modules/getting-started/pages/editions.adoc b/docs/modules/getting-started/pages/editions.adoc index 09b4c0023..f52e462fd 100644 --- a/docs/modules/getting-started/pages/editions.adoc +++ b/docs/modules/getting-started/pages/editions.adoc @@ -1,51 +1,204 @@ -= Hazelcast Editions and Licenses -:description: Hazelcast has Open Source and Enterprise editions. Hazelcast Enterprise is the licensed commercial offering built on top of the Open Source product. += Hazelcast Editions and Distributions +:description: Hazelcast offers Open Source and Enterprise editions of its member software as well as full and slim distributions. +:page-aliases: before-you-begin.adoc {description} -== Features in Hazelcast Open Source +== Editions -The Open Source edition of Hazelcast offers the following features: +Hazelcast Open Source edition is free and covered by Apache License, Version 2.0 +and https://hazelcast.com/hazelcast-community-license/[Hazelcast Community License]. Permissions are granted +to use, reproduce, and distribute it along with any kind of open +source and closed source applications. -* Distributed computation, data structures, and events -* Streaming data processing -* Connectors to read from/write to systems like Apache Kafka, JMS, JDBC and HDMS -* Querying with SQL and predicates -* CP subsystem for distributed coordination use cases -* JCache implementation -* Replication of web sessions (filter, Tomcat, Jetty based) -* Administration and monitoring utilities including Management Center, JMX, metrics and diagnostics +Hazelcast Enterprise is a commercial product that offers additional features to the Open Source edition, including the security suite and blue/green deployments for clients. -== Features in Hazelcast Enterprise +=== Features in Hazelcast Enterprise -The Enterprise edition of Hazelcast offers the following features in addition to the -ones listed above: +The Enterprise edition of Hazelcast offers the following Enterprise features. -* High-Density Memory Store (off-heap memory) -* Persistence -* Lossless cluster restarts -* Rolling upgrades -* Job upgrades -* Unlimited members on Hazelcast Management Center (it is restricted to 3 members on the freely available version of Management Center) -* CP Subsystem Persistence -* Security Suite (RBAC, TLS, Mutual Auth, Client Certificates) -* WAN Replication (syncing data between two geo-replicated clusters) -* Blue/Green Deployments +If a feature requires an Enterprise license, it is labelled [.enterprise]*Enterprise* in the documentation. -NOTE: All preexisting Hazelcast licenses (former Pro/Enterprise/Enterprise HD licenses) -work with Hazelcast Platform. +* xref:security:overview.adoc[Security suite] +* xref:wan:wan.adoc[] +* xref:cp-subsystem:persistence.adoc[] +* xref:deploy:deploying-in-vmware-tanzu.adoc[Deploying in VMware Tanzu] +* xref:deploy:deploying-in-kubernetes.adoc[Deploying in Openshift container platform] +* xref:maintain-cluster:monitoring.adoc#clustered-jmx-and-rest-via-management-center[Clustered REST] +* xref:maintain-cluster:monitoring.adoc#clustered-jmx-and-rest-via-management-center[Clustered JMX] +* xref:maintain-cluster:rolling-upgrades.adoc[Rolling upgrades] +* xref:storage:high-density-memory.adoc[] +* xref:storage:persistence.adoc[] +* xref:clients:java.adoc#blue-green-deployment-and-disaster-recovery[Blue/Green client support] +* xref:pipelines:job-update.adoc[Job updates] -== Open Source License +NOTE: All pre-existing Hazelcast licenses such as the Pro/Enterprise/Enterprise HD licenses, +are valid for Hazelcast Platform. -Hazelcast Open Source edition is free and covered by Apache License, Version 2.0 -and https://hazelcast.com/hazelcast-community-license/[Hazelcast Community License]. Permissions are granted -to use, reproduce and distribute it along with any kind of open -source and closed source applications. +== Full and Slim Distributions +[[full-slim]] + +The following installation options offer a full and slim distribution: + +- Docker +- ZIP/TAR Binaries +- Java + +Other installation options offer only the full distribution. + +The full distribution contains all available Hazelcast connectors, libraries, and Management Center. + +.What's included in the full distribution +[%collapsible] +==== +[source,plain,subs="attributes+"] +---- +├── LICENSE +├── NOTICE +├── bin +│ ├── common.sh +│ ├── hz-cli +│ ├── hz-cli.bat +│ ├── hz-cluster-admin +│ ├── hz-cluster-cp-admin +│ ├── hz-healthcheck +│ ├── hz-start +│ ├── hz-start.bat +│ ├── hz-stop +│ └── hz-stop.bat +├── config +│ ├── examples +│ │ ├── hazelcast-client-full-example.xml +│ │ ├── hazelcast-client-full-example.yaml +│ │ ├── hazelcast-client.yaml +│ │ ├── hazelcast-full-example.xml +│ │ ├── hazelcast-full-example.yaml +│ │ ├── hazelcast-security-hardened.yaml +│ │ └── hazelcast.yaml +│ ├── hazelcast-client.xml +│ ├── hazelcast.xml +│ ├── jmx_agent_config.yaml +│ ├── jvm-client.options +│ ├── jvm.options +│ └── log4j2.properties +├── custom-lib +│ ├── hazelcast-3-connector-impl-{full-version}.jar +│ ├── hazelcast-3.12.12.jar +│ └── hazelcast-client-3.12.12.jar +├── lib +│ ├── cache-api-1.1.1.jar +│ ├── hazelcast-3-connector-common-{full-version}.jar +│ ├── hazelcast-3-connector-interface-{full-version}.jar +│ ├── hazelcast-{full-version}.jar +│ ├── hazelcast-download.properties +│ ├── hazelcast-hibernate53-2.1.1.jar +│ ├── hazelcast-jet-avro-{full-version}.jar +│ ├── hazelcast-jet-cdc-debezium-{full-version}.jar +│ ├── hazelcast-jet-cdc-mysql-{full-version}.jar +│ ├── hazelcast-jet-cdc-postgres-{full-version}.jar +│ ├── hazelcast-jet-csv-{full-version}.jar +│ ├── hazelcast-jet-elasticsearch-7-{full-version}.jar +│ ├── hazelcast-jet-files-azure-{full-version}.jar +│ ├── hazelcast-jet-files-gcs-{full-version}.jar +│ ├── hazelcast-jet-files-s3-{full-version}.jar +│ ├── hazelcast-jet-grpc-{full-version}.jar +│ ├── hazelcast-jet-hadoop-all-{full-version}.jar +│ ├── hazelcast-jet-kafka-{full-version}.jar +│ ├── hazelcast-jet-kinesis-{full-version}.jar +│ ├── hazelcast-jet-protobuf-{full-version}.jar +│ ├── hazelcast-jet-python-{full-version}.jar +│ ├── hazelcast-jet-s3-{full-version}.jar +│ ├── hazelcast-sql-{full-version}.jar +│ ├── hazelcast-wm-4.0.jar +│ ├── jansi-2.1.0.jar +│ ├── jline-reader-3.19.0.jar +│ ├── jline-terminal-3.19.0.jar +│ ├── jline-terminal-jansi-3.19.0.jar +│ ├── jmx_prometheus_javaagent-0.14.0.jar +│ ├── log4j-api-2.14.0.jar +│ ├── log4j-core-2.14.0.jar +│ ├── log4j-slf4j-impl-2.14.0.jar +│ ├── picocli-3.9.0.jar +│ └── slf4j-api-1.7.30.jar +└── licenses + ├── THIRD-PARTY.txt + ├── apache-v2-license. + ├── attribution.txt + └── hazelcast-community-license.txt +├── management-center +│   ├── ThirdPartyNotices.txt +│   ├── bin +│   │   ├── mc-conf.bat +│   │   ├── mc-conf.sh +│   │   ├── mc-start.cmd +│   │   ├── mc-start.sh +│   │   ├── start.bat +│   │   ├── start.sh +│   │   └── user-lib +│   ├── hazelcast-management-center-{full-version}.jar +│   └── license.txt +└── release_notes.txt +---- +==== + +The slim distribution excludes Management Center and connectors. You can use the slim version +to save memory and later include the additional components, as needed. -== Enterprise License +To get a slim distribution, use one of the available xref:install-hazelcast.adoc[installation options], but add `-slim` to `{full-version}`: `{full-version}-slim`. -Hazelcast Enterprise edition is a commercial product of Hazelcast, Inc. and distributed under -a commercial license that must be acquired -before using it in any type of released software. Feel free to contact -http://hazelcast.com/contact/[Hazelcast sales department^] -for more information about commercial offers. \ No newline at end of file +.What's included in the slim distribution +[%collapsible] +==== +[source,plain,subs="attributes+"] +---- +├── LICENSE +├── NOTICE +├── bin +│ ├── common.sh +│ ├── hz-cli +│ ├── hz-cli.bat +│ ├── hz-cluster-admin +│ ├── hz-cluster-cp-admin +│ ├── hz-healthcheck +│ ├── hz-start +│ ├── hz-start.bat +│ ├── hz-stop +│ └── hz-stop.bat +├── config +│ ├── examples +│ │ ├── hazelcast-client-full-example.xml +│ │ ├── hazelcast-client-full-example.yaml +│ │ ├── hazelcast-client.yaml +│ │ ├── hazelcast-full-example.xml +│ │ ├── hazelcast-full-example.yaml +│ │ ├── hazelcast-security-hardened.yaml +│ │ └── hazelcast.yaml +│ ├── hazelcast-client.xml +│ ├── hazelcast.xml +│ ├── jmx_agent_config.yaml +│ ├── jvm-client.options +│ ├── jvm.options +│ └── log4j2.properties +├── lib +│ ├── cache-api-1.1.1.jar +│ ├── hazelcast-{full-version}.jar +│ ├── hazelcast-download.properties +│ ├── hazelcast-hibernate53-2.1.1.jar +│ ├── hazelcast-sql-{full-version}.jar +│ ├── hazelcast-wm-4.0.jar +│ ├── jansi-2.1.0.jar +│ ├── jline-reader-3.19.0.jar +│ ├── jline-terminal-3.19.0.jar +│ ├── jline-terminal-jansi-3.19.0.jar +│ ├── jmx_prometheus_javaagent-0.14.0.jar +│ ├── log4j-api-2.14.0.jar +│ ├── log4j-core-2.14.0.jar +│ ├── log4j-slf4j-impl-2.14.0.jar +│ ├── picocli-3.9.0.jar +│ └── slf4j-api-1.7.30.jar +└── licenses + ├── THIRD-PARTY.txt + ├── apache-v2-license.txt + └── hazelcast-community-license.txt +---- +==== \ No newline at end of file diff --git a/docs/modules/getting-started/pages/enterprise-overview.adoc b/docs/modules/getting-started/pages/enterprise-overview.adoc index bfaf9d795..b581ccee4 100644 --- a/docs/modules/getting-started/pages/enterprise-overview.adoc +++ b/docs/modules/getting-started/pages/enterprise-overview.adoc @@ -1,11 +1,20 @@ -= Overview += Get Started with Hazelcast Enterprise :description: This series of tutorials showcases some of the most useful features of Hazelcast Enterprise. +:page-enterprise: true {description} -NOTE: To complete these tutorials, you'll need Docker. See xref:getting-started:get-started-enterprise.adoc#using-the-enterprise-docker-image[Using the Enterprise Docker Image]. +== Step 1. Pull the Hazelcast Enterprise Docker Image -In this series, you will learn how to do the following: +include::install-enterprise.adoc[tag=docker] + +== Step 2. Get your Enterprise License + +include::deploy:enterprise-licenses.adoc[tag=get-license] + +== Next Steps + +Continue your journey by following another tutorial in this series: * xref:getting-started:persistence.adoc[Persist Maps on Disk] * xref:getting-started:authenticate-clients.adoc[Authenticate Client Connections] diff --git a/docs/modules/getting-started/pages/get-started-binary.adoc b/docs/modules/getting-started/pages/get-started-binary.adoc index b87a21ec8..bb77c6668 100644 --- a/docs/modules/getting-started/pages/get-started-binary.adoc +++ b/docs/modules/getting-started/pages/get-started-binary.adoc @@ -20,9 +20,9 @@ xref:deploy:running-in-modular-java.adoc[Using JDK 11+ with Modular Java] link:https://adoptopenjdk.net[AdoptOpenJDK^] |A full Hazelcast distribution -|xref:install-hazelcast.adoc#using-the-binary[Install Hazelcast] +|xref:getting-started:install-hazelcast.adoc#using-the-binary[Install Hazelcast] -xref:get-started-enterprise.adoc[Install Hazelcast Enterprise] +xref:getting-started:install-hazelcast.adoc[Install Hazelcast Enterprise] |Your internal IP address |link:https://lifehacker.com/how-to-find-your-local-and-external-ip-address-5833108[How to Find Your Internal IP Address^] @@ -194,7 +194,7 @@ Java:: + -- -. xref:install-hazelcast.adoc#use-java[Install the Java client library] +. xref:getting-started:install-hazelcast.adoc#use-java[Install the Java client library] . Add the following to your file: + diff --git a/docs/modules/getting-started/pages/quickstart.adoc b/docs/modules/getting-started/pages/get-started-cli.adoc similarity index 92% rename from docs/modules/getting-started/pages/quickstart.adoc rename to docs/modules/getting-started/pages/get-started-cli.adoc index d5c1ddaaa..9ae00470f 100644 --- a/docs/modules/getting-started/pages/quickstart.adoc +++ b/docs/modules/getting-started/pages/get-started-cli.adoc @@ -1,6 +1,6 @@ -= Hazelcast Quickstart -:description: Follow this five-minute tutorial to set up a Hazelcast cluster for testing or evaluation purposes. In five minutes, you will learn how to install Hazelcast, start a local cluster, write data to memory and visualize that data. -:!page-pagination: += Start a Local Cluster with the CLI +:description: Follow this five-minute tutorial to set up a Hazelcast cluster for testing or evaluation purposes. In five minutes, you will learn how to install Hazelcast with a package manager, start a local cluster with the CLI, write data to memory and visualize that data. +:page-aliases: quickstart.adoc ++++