diff --git a/.gitignore b/.gitignore index b9599e62e..88579df42 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ .DS_Store +# DynamoDb local simulator data directory +docker-dynamodblocal-data/ + # ephemeral EventStoreDB certs created via `docker compose up` certs/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 60cb8d856..d983eed68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ The `Unreleased` section name is replaced by the expected version of next releas ### Added - `Equinox`: `Decider.Transact(interpret : 'state -> Async<'event list>)` [#314](https://github.com/jet/equinox/pull/314) +- `CosmosStore.Prometheus`: Add `rut` tag to enable filtering/grouping by Read vs Write activity as per `DynamoDB` [#321](https://github.com/jet/equinox/pull/321) +- `DynamoDb`/`DynamoDb.Prometheus`: Implements the majority of the `CosmosStore` functionality via `FSharp.AWS.DynamoDB` [#321](https://github.com/jet/equinox/pull/321) - `EventStoreDb`: As per `EventStore` module, but using the modern `EventStore.Client.Grpc.Streams` client [#196](https://github.com/jet/equinox/pull/196) ### Changed diff --git a/Equinox.sln b/Equinox.sln index 2a91415bf..f3bda112f 100644 --- a/Equinox.sln +++ b/Equinox.sln @@ -95,6 +95,12 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Equinox.EventStoreDb", "src EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Equinox.EventStoreDb.Integration", "tests\Equinox.EventStoreDb.Integration\Equinox.EventStoreDb.Integration.fsproj", "{BA63048B-3CA3-448D-A4CD-0C772D57B6F8}" EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Equinox.DynamoStore", "src\Equinox.DynamoStore\Equinox.DynamoStore.fsproj", "{E04E86B4-4E35-4AC9-8D8F-B01297484FC1}" +EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Equinox.DynamoStore.Integration", "tests\Equinox.DynamoStore.Integration\Equinox.DynamoStore.Integration.fsproj", "{2C8FCD63-4A3C-4EA6-88E0-E0F287B0F47A}" +EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Equinox.DynamoStore.Prometheus", "src\Equinox.DynamoStore.Prometheus\Equinox.DynamoStore.Prometheus.fsproj", "{A9AF41B3-AB28-4296-B4A4-B90DA7821476}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -209,6 +215,18 @@ Global {BA63048B-3CA3-448D-A4CD-0C772D57B6F8}.Debug|Any CPU.Build.0 = Debug|Any CPU {BA63048B-3CA3-448D-A4CD-0C772D57B6F8}.Release|Any CPU.ActiveCfg = Release|Any CPU {BA63048B-3CA3-448D-A4CD-0C772D57B6F8}.Release|Any CPU.Build.0 = Release|Any CPU + {E04E86B4-4E35-4AC9-8D8F-B01297484FC1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E04E86B4-4E35-4AC9-8D8F-B01297484FC1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E04E86B4-4E35-4AC9-8D8F-B01297484FC1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E04E86B4-4E35-4AC9-8D8F-B01297484FC1}.Release|Any CPU.Build.0 = Release|Any CPU + {2C8FCD63-4A3C-4EA6-88E0-E0F287B0F47A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2C8FCD63-4A3C-4EA6-88E0-E0F287B0F47A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2C8FCD63-4A3C-4EA6-88E0-E0F287B0F47A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2C8FCD63-4A3C-4EA6-88E0-E0F287B0F47A}.Release|Any CPU.Build.0 = Release|Any CPU + {A9AF41B3-AB28-4296-B4A4-B90DA7821476}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A9AF41B3-AB28-4296-B4A4-B90DA7821476}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A9AF41B3-AB28-4296-B4A4-B90DA7821476}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A9AF41B3-AB28-4296-B4A4-B90DA7821476}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/README.md b/README.md index 221db657a..f907979b1 100644 --- a/README.md +++ b/README.md @@ -101,10 +101,28 @@ _If you're looking to learn more about and/or discuss Event Sourcing and it's my - It should be noted that from a querying perspective, the `Tip` shares the same structure as `Batch` documents (a potential future extension would be to carry some events in the `Tip` as [some interim versions of the implementation once did](https://github.com/jet/equinox/pull/58), see also [#109](https://github.com/jet/equinox/pull/109). - **`Equinox.CosmosStore` `RollingState` and `Custom` 'non-event-sourced' modes**: - Uses 'Tip with Unfolds' encoding to avoid having to write event documents at all. This option benefits from the caching and consistency management mechanisms because the cost of writing and storing infinitely increasing events are removed. Search for `transmute` or `RollingState` in the `samples` and/or see [the `Checkpoint` Aggregate in Propulsion](https://github.com/jet/propulsion/blob/master/src/Propulsion.EventStore/Checkpoint.fs). One chief use of this mechanism is for tracking Summary Event feeds in [the `dotnet-templates` `summaryConsumer` template](https://github.com/jet/dotnet-templates/tree/master/propulsion-summary-consumer). +- **`Equinox.DynamoStore`**: + - Most features and behaviors are as per `Equinox.CosmosStore`, with the following key differences: + - Instead of using a Stored Procedure as `CosmosStore` does, the implementation involves: + - conditional `PutItem` and [`UpdateItem`](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html) requests to accumulate events in the Tip (where there is space available). + - At the point where the Tip exceeds any of the configured and/or implicit limits, a [`TransactWriteItems`](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_TransactWriteItems.html) request is used (see [implementation in `FSharp.AWS.DynamoDB`](https://github.com/fsprojects/FSharp.AWS.DynamoDB/pull/48)): + - maximum event count (not limited by default) + - maximum accumulated event size (default 32KiB) + - DynamoDB Item Size Limit (hard limit of 400KiB) + - DynamoDB does not support an etag-checked Read API, which means a cache hit is not as efficient as it is on CosmosDB (and the data hence travels and is deserialized unnecessarily) + - Concurrency conflicts necessitate an additional roundtrip to resync [as the DynamoDB Service does not yield the item in the event of a `ConditionalCheckFailedException`](https://stackoverflow.com/questions/71622525) + - `Equinox.Cosmos.Core.Events.appendAtEnd`/`NonIdempotentAppend` has not been ported (there's no obvious clean and efficient way to do a conditional insert/update/split as the CosmosDB stored proc can, and this is a low usage feature) + - The implementation uses [the excellent `FSharp.AWS.DynamoDB` library](https://github.com/fsprojects/FSharp.AWS.DynamoDB)) (which wraps the standard AWS `AWSSDK.DynamoDBv2` SDK Package), and leans on [significant preparatory research](https://github.com/pierregoudjo/dynamodb_conditional_writes) :pray: [@pierregoudjo](https://github.com/pierregoudjo) + - `CosmosStore` dictates (as of V4) that event bodies be supplied as `System.Text.Json.JsonElement`s (in order that events can be included in the Document/ Items as JSON directly. This is also to underscore the fact that the only reasonable format to use is valid JSON; binary data would need to be base64 encoded. `DynamoStore` accepts and yields event bodies as arbitrary `ReadOnlyMemory` BLOBs (the AWS SDK round-trips such blobs as a `MemoryStream` and does not impose any restrictions on the blobs in terms of required format). + - `CosmosStore` defaults to compressing (with `System.IO.Compression.DeflateStream`) the event bodies for Unfolds; `DynamoStore` provides for (and defaults to) compressing event bodies for both Events and Unfolds. While both compression behaviors can be disabled (particularly if your Event Encoding already compresses, or the nature of your data or format is such that it will never compress), it should be noted that the key reason why this facility is provided is that minimizing Request Charges is imperative when request size directly maps to financial charges, 429s, reduced throughput and a lowered scaling ceiling. + - Azure CosmosDB's ChangeFeed API intrinsically supports replays of all the events in a Store, whereas the DynamoDB Streams facility only retains 24h of actions. As a result, there are ancillary components that provide equivalent functionality composed of: + - `Propulsion.DynamoStore.Lambda`: an AWS Lambda that is configured via a DynamoDB Streams Trigger to Index the Events (represented as Equinox Streams, typically in a separated `-index` Table) as they are appended + - `Propulsion.DynamoStore.DynamoStoreSource`: consumes the Index Streams akin to how `Propulsion.CosmosStore.CosmosStoreSource` consumes the CosmosDB Change Feed # Currently Supported Data Stores - [Azure Cosmos DB](https://docs.microsoft.com/en-us/azure/cosmos-db): contains some fragments of code dating back to 2016, however [the storage model](DOCUMENTATION.md#Cosmos-Storage-Model) was arrived at based on intensive benchmarking (squash-merged in [#42](https://github.com/jet/equinox/pull/42)). The V2 and V3 release lines are being used in production systems. (The V3 release provides support for significantly more efficient packing of events ([storing events in the 'Tip'](https://github.com/jet/equinox/pull/251))). +- [Amazon Dynamo DB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Introduction.html): Shares most features with `Equinox.CosmosStore` ([from which it was ported in #321](https://github.com/jet/equinox/pull/321)). See above for detailed comparison. - [EventStoreDB](https://eventstore.org/): this codebase itself has been in production since 2017 (see commit history), with key elements dating back to approx 2016. Current versions require EventStoreDB Server editions `21.10` or later, and communicate over the modern gRPC interface. - [SqlStreamStore](https://github.com/SQLStreamStore/SQLStreamStore): bindings for the powerful and widely used SQL-backed Event Storage system, derived from the EventStoreDB adapter. [See SqlStreamStore docs](https://sqlstreamstore.readthedocs.io/en/latest/#introduction). :pray: [@rajivhost](https://github.com/rajivhost) - `MemoryStore`: In-memory store (volatile, for unit or integration test purposes). Fulfils the full contract Equinox imposes on a store, but without I/O costs [(it's ~100 LOC wrapping a `ConcurrentDictionary`)](https://github.com/jet/equinox/blob/master/src/Equinox.MemoryStore/MemoryStore.fs). Also enables [take serialization/deserialization out of the picture](https://github.com/jet/FsCodec#boxcodec) in tests. @@ -136,6 +154,8 @@ The components within this repository are delivered as multi-targeted Nuget pack - `Equinox.MemoryStore` [![MemoryStore NuGet](https://img.shields.io/nuget/v/Equinox.MemoryStore.svg)](https://www.nuget.org/packages/Equinox.MemoryStore/): In-memory store for integration testing/performance base-lining/providing out-of-the-box zero dependency storage for examples. ([depends](https://www.fuget.org/packages/Equinox.MemoryStore) on `Equinox.Core`, `FsCodec`) - `Equinox.CosmosStore` [![CosmosStore NuGet](https://img.shields.io/nuget/v/Equinox.CosmosStore.svg)](https://www.nuget.org/packages/Equinox.CosmosStore/): Azure CosmosDB Adapter with integrated 'unfolds' feature, facilitating optimal read performance in terms of latency and RU costs, instrumented to meet Jet's production monitoring requirements. ([depends](https://www.fuget.org/packages/Equinox.CosmosStore) on `Equinox.Core`, `Microsoft.Azure.Cosmos >= 3.25`, `FsCodec`, `System.Text.Json`, `FSharp.Control.AsyncSeq >= 2.0.23`) - `Equinox.CosmosStore.Prometheus` [![CosmosStore.Prometheus NuGet](https://img.shields.io/nuget/v/Equinox.CosmosStore.Prometheus.svg)](https://www.nuget.org/packages/Equinox.CosmosStore.Prometheus/): Integration package providing a `Serilog.Core.ILogEventSink` that extracts detailed metrics information attached to the `LogEvent`s and feeds them to the `prometheus-net`'s `Prometheus.Metrics` static instance. ([depends](https://www.fuget.org/packages/Equinox.CosmosStore.Prometheus) on `Equinox.CosmosStore`, `prometheus-net >= 3.6.0`) +- `Equinox.DynamoStore` [![DynamoStore NuGet](https://img.shields.io/nuget/v/Equinox.DynamoStore.svg)](https://www.nuget.org/packages/Equinox.DynamoStore/): Amazon DynamoDB Adapter with integrated 'unfolds' feature, facilitating optimal read performance in terms of latency and RC costs, patterned after `Equinox.CosmosStore`. ([depends](https://www.fuget.org/packages/Equinox.CosmosStore) on `Equinox.Core`, `FSharp.AWS.DynamoDB >= 0.11.0-beta`, `FsCodec`, `FSharp.Control.AsyncSeq >= 2.0.23`) +- `Equinox.DynamoStore.Prometheus` [![DynamoStore.Prometheus NuGet](https://img.shields.io/nuget/v/Equinox.DynamoStore.Prometheus.svg)](https://www.nuget.org/packages/Equinox.DynamoStore.Prometheus/): Integration package providing a `Serilog.Core.ILogEventSink` that extracts detailed metrics information attached to the `LogEvent`s and feeds them to the `prometheus-net`'s `Prometheus.Metrics` static instance. ([depends](https://www.fuget.org/packages/Equinox.CosmosStore.Prometheus) on `Equinox.DynamoStore`, `prometheus-net >= 3.6.0`) - `Equinox.EventStore` [![EventStore NuGet](https://img.shields.io/nuget/v/Equinox.EventStore.svg)](https://www.nuget.org/packages/Equinox.EventStore/): [EventStoreDB](https://eventstore.org/) Adapter designed to meet Jet's production monitoring requirements. ([depends](https://www.fuget.org/packages/Equinox.EventStore) on `Equinox.Core`, `EventStore.Client >= 22.0.0-preview`, `FSharp.Control.AsyncSeq >= 2.0.23`), EventStore Server version `21.10` or later) - `Equinox.EventStoreDb` [![EventStoreDb NuGet](https://img.shields.io/nuget/v/Equinox.EventStoreDb.svg)](https://www.nuget.org/packages/Equinox.EventStoreDb/): Production-strength [EventStoreDB](https://eventstore.org/) Adapter. ([depends](https://www.fuget.org/packages/Equinox.EventStoreDb) on `Equinox.Core`, `EventStore.Client.Grpc.Streams` >= `22.0.0, `FSharp.Control.AsyncSeq` v `2.0.23`, EventStore Server version `21.10` or later) - `Equinox.SqlStreamStore` [![SqlStreamStore NuGet](https://img.shields.io/nuget/v/Equinox.SqlStreamStore.svg)](https://www.nuget.org/packages/Equinox.SqlStreamStore/): [SqlStreamStore](https://github.com/SQLStreamStore/SQLStreamStore) Adapter derived from `Equinox.EventStore` - provides core facilities (but does not connect to a specific database; see sibling `SqlStreamStore`.* packages). ([depends](https://www.fuget.org/packages/Equinox.SqlStreamStore) on `Equinox.Core`, `FsCodec`, `SqlStreamStore >= 1.2.0-beta.8`, `FSharp.Control.AsyncSeq`) @@ -151,6 +171,7 @@ Equinox does not focus on projection logic - each store brings its own strengths - `Propulsion` [![Propulsion NuGet](https://img.shields.io/nuget/v/Propulsion.svg)](https://www.nuget.org/packages/Propulsion/): A library that provides an easy way to implement projection logic. It defines `Propulsion.Streams.StreamEvent` used to interop with `Propulsion.*` in processing pipelines for the `proProjector` and `proSync` templates in the [templates repo](https://github.com/jet/dotnet-templates), together with the `Ingestion`, `Streams`, `Progress` and `Parallel` modules that get composed into those processing pipelines. ([depends](https://www.fuget.org/packages/Propulsion) on `Serilog`) - `Propulsion.Cosmos` [![Propulsion.Cosmos NuGet](https://img.shields.io/nuget/v/Propulsion.Cosmos.svg)](https://www.nuget.org/packages/Propulsion.Cosmos/): Wraps the [Microsoft .NET `ChangeFeedProcessor` library](https://github.com/Azure/azure-documentdb-changefeedprocessor-dotnet) providing a [processor loop](DOCUMENTATION.md#change-feed-processors) that maintains a continuous query loop per CosmosDB Physical Partition (Range) yielding new or updated documents (optionally unrolling events written by `Equinox.CosmosStore` for processing or forwarding). ([depends](https://www.fuget.org/packages/Propulsion.Cosmos) on `Equinox.Cosmos`, `Microsoft.Azure.DocumentDb.ChangeFeedProcessor >= 2.2.5`) - `Propulsion.CosmosStore` [![Propulsion.CosmosStore NuGet](https://img.shields.io/nuget/v/Propulsion.CosmosStore.svg)](https://www.nuget.org/packages/Propulsion.CosmosStore/): Wraps the CosmosDB V3 SDK's Change Feed API, providing a [processor loop](DOCUMENTATION.md#change-feed-processors) that maintains a continuous query loop per CosmosDB Physical Partition (Range) yielding new or updated documents (optionally unrolling events written by `Equinox.CosmosStore` for processing or forwarding). Used in the [`propulsion project stats cosmos`](dotnet-tool-provisioning--benchmarking-tool) tool command; see [`dotnet new proProjector` to generate a sample app](#quickstart) using it. ([depends](https://www.fuget.org/packages/Propulsion.CosmosStore) on `Equinox.CosmosStore`) +- `Propulsion.DynamoStore` [![Propulsion.DynamoStore NuGet](https://img.shields.io/nuget/v/Propulsion.DynamoStore.svg)](https://www.nuget.org/packages/Propulsion.DynamoStore/): Indexes events written by `Equinox.DynamoStore` via a DynamoDB Streams-triggered Lambda. Provides a `DynamoStoreSource` that provides equivalent functionality to `Propulsion.CosmosStore`; see [`dotnet new proProjector` to generate a sample app](#quickstart) using it. ([depends](https://www.fuget.org/packages/Propulsion.DynamoStore) on `Equinox.DynamoStore`) - `Propulsion.EventStore` [![Propulsion.EventStore NuGet](https://img.shields.io/nuget/v/Propulsion.EventStore.svg)](https://www.nuget.org/packages/Propulsion.EventStore/) Used in the [`propulsion project es`](dotnet-tool-provisioning--benchmarking-tool) tool command; see [`dotnet new proSync` to generate a sample app](#quickstart) using it. ([depends](https://www.fuget.org/packages/Propulsion.EventStore) on `Equinox.EventStore`) - `Propulsion.EventStoreDb` [![Propulsion.EventStoreDb NuGet](https://img.shields.io/nuget/v/Propulsion.EventStoreDb.svg)](https://www.nuget.org/packages/Propulsion.EventStoreDb/) Consumes from `EventStoreDB` v `21.10` or later using the gRPC interface. ([depends](https://www.fuget.org/packages/Propulsion.EventStoreDb) on `Equinox.EventStoreDb`) - `Propulsion.Kafka` [![Propulsion.Kafka NuGet](https://img.shields.io/nuget/v/Propulsion.Kafka.svg)](https://www.nuget.org/packages/Propulsion.Kafka/): Provides a canonical `RenderedSpan` that can be used as a default format when projecting events via e.g. the Producer/Consumer pair in `dotnet new proProjector -k; dotnet new proConsumer`. ([depends](https://www.fuget.org/packages/Propulsion.Kafka) on `Newtonsoft.Json >= 11.0.2`, `Propulsion`, `FsKafka`) @@ -162,6 +183,7 @@ Equinox does not focus on projection logic - each store brings its own strengths - can render events from any of the stores via `eqx dump`. - incorporates a benchmark scenario runner, running load tests composed of transactions in `samples/Store` and `samples/TodoBackend` against any supported store; this allows perf tuning and measurement in terms of both latency and transaction charge aspects. (Install via: `dotnet tool install Equinox.Tool -g`) - can configure indices in Azure CosmosDB for an `Equinox.CosmosStore` Container via `eqx init`. See [here](https://github.com/jet/equinox#store-data-in-azure-cosmosdb). + - can create tables in Amazon DynamoDB for `Equinox.DynamoStore` via `eqx initAws`. - can initialize databases for `SqlStreamStore` via `eqx config` ## Starter Project Templates and Sample Applications @@ -471,6 +493,47 @@ eqx run -t saveforlater -f 50 -d 5 -C -U pg -c "connectionstring" -p "u=un;p=pas eqx dump -s "SavedForLater-ab25cc9f24464d39939000aeb37ea11a" pg -c "connectionstring" -p "u=un;p=password" -s "schema" # show stored JSON (Guid shown in eqx run output) ``` + +### Use [Amazon DynamoDB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Introduction.html) + +DynamoDB is supported in the samples and the `eqx` tool equivalent to the CosmosDB support as described: +- being able to supply `dynamo` source to `eqx run` wherever `cosmos` works, e.g. `eqx run -t cart -f 50 -d 5 -CU dynamo -s http://localhost:8000 -t TableName` +- being able to supply `dynamo` flag to `eqx dump`, e.g. `eqx dump -CU -s "Favorites-ab25cc9f24464d39939000aeb37ea11a" dynamo` +- being able to supply `dynamo` flag to Web sample, e.g. `dotnet run --project samples/Web/ -- dynamo -s http://localhost:8000` +- being able to supply `dynamo` flag to `eqx initAws` command e.g. `eqx initAws -rru 10 -wru 10 dynamo -t TableName` + +1. The tooling and samples in this repo default to using the following environment variables (see [AWS CLI UserGuide for more detailed guidance as to specific configuration](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html)) + + ```zsh + $env:EQUINOX_DYNAMO_SERVICE_URL="https://dynamodb.us-west-2.amazonaws.com" # Simulator: "http://localhost:8000" + $env:EQUINOX_DYNAMO_ACCESS_KEY_ID="AKIAIOSFODNN7EXAMPLE" + $env:EQUINOX_DYNAMO_SECRET_ACCESS_KEY="AwJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + $env:EQUINOX_DYNAMO_TABLE="equinox-test" + $env:EQUINOX_DYNAMO_TABLE_ARCHIVE="equinox-test-archive" + ``` + +2. Tour of the tools/samples: + + ```zsh + cd ~/code/equinox + + # start the simulator at http://localhost:8000 and an admin console at http://localhost:8001/ + docker compose up dynamodb-local dynamodb-admin -d + + # Establish the table + dotnet run --project tools/Equinox.Tool -- initAws -rru 10 -wru 10 dynamo -t TableName + + # run a benchmark + dotnet run -c Release --project tools/Equinox.Tool -- run -t saveforlater -f 50 -d 5 -CU dynamo + + # run the webserver + dotnet run --project samples/Web/ -- dynamo -t TableName + + # run a benchmark connecting to the webserver + eqx run -t saveforlater -f 50 -d 5 -CU web + eqx dump -s "SavedForLater-ab25cc9f24464d39939000aeb37ea11a" dynamo # show stored JSON (Guid shown in eqx run output) + ``` + ### BENCHMARKS A key facility of this repo is being able to run load tests, either in process against a nominated store, or via HTTP to a nominated instance of `samples/Web` ASP.NET Core host app. The following test suites are implemented at present: @@ -873,7 +936,7 @@ I'd present the fact that Equinox: - was initially generalised and extracted from working code using ESDB in (by most measures) a successful startup written in a mix of F# and C# by the usual mix of devs, experience levels, awareness of event sourcing patterns and language/domain backgrounds - for a long time only had its MemoryStore as the thing to force it to be store agnostic - did not fundamentally change to add idiomatic support for a Document database (CosmosDB) -- will not fundamentally change to add idiomatic support for DynamoDB +- did not change to add idiomatic support for DynamoDB - can and has been used at every stage in an e-commerce pipeline - is presently aligning pretty neatly with diverse domains without any changes/extensions, both for me and others @@ -960,19 +1023,20 @@ Next, I'd like to call out some things that Equinox is focused on delivering, re - to a single consistency control unit (stream) at a time (underlying stores in general rarely provide support for more than that, but more importantly, a huge number of use cases in a huge number of systems have natural mappings to this without anyone having to do evil things or write thousands of lines of code) - no major focus on blind-writes, even if there is low level support and/or things work if you go direct and do it out of band) - provide a good story for managing the writing of the first event in a stream in an efficient manner - - have a story for providing a changefeed - - typically via a matching Propulsion library (fully ordered for SSS and ESDB, ordered at stream level for CosmosDB, similar for DynamoDB if/when that happens) - - have a story for caching and efficient usage of the store + - have a story for providing a Change Feed + - typically via a matching Propulsion library (fully ordered for SSS and ESDB, ordered at stream level for CosmosDB and DynamoDB) + - have a story for caching and efficient usage of each store to the best degree possible - `Equinox.SqlStreamStore` - caching is supported and recommended to minimise reads - in-stream snapshots sometimes help but there are tradeoffs - `Equinox.EventStore` - caching is supported, but less important/relevant than it is with SSS as ESDB has good caching support and options - Equinox in-stream snapshots sometimes help but there are tradeoffs) + - `Equinox.CosmosStore` and `Equinox.DynamoStore` + - multiple events are packed into each document (critical to avoid per-Item space and indexing overhead - limits are configurable) + - etag-checked RollingState access mode enables allow you to achieve optimal perf and RU cost via the same API without writing an event every time - `Equinox.CosmosStore` - etag-checked read caching (use without that is not recommended in general, though you absolutely will and should turn it off for some streams) - - multiple events are packed into each document (critical to avoid per document overhead - this is configurable) - - etag-checked RollingState access mode enables allow you to achieve optimal perf and RU cost via the same API without writing an event every time **The provision of the changefeed needs to be considered as a primary factor in the overall design if you're trying to build a general store - the nature of what you are seeking to provide (max latency and ordering guarantees etc) will be a major factor in designing the schema for how you manage the encoding and updating of the items in the store** diff --git a/build.proj b/build.proj index b08c27936..c50d64fab 100644 --- a/build.proj +++ b/build.proj @@ -18,6 +18,8 @@ + + diff --git a/docker-compose.yml b/docker-compose.yml index dab921cb1..c7b4f65c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -145,6 +145,29 @@ services: clusternetwork: ipv4_address: 172.30.240.13 + dynamodb-local: + image: amazon/dynamodb-local + container_name: dynamodb-local + hostname: dynamodb-local + restart: always + volumes: + - ./docker-dynamodblocal-data:/home/dynamodblocal/data + ports: + - 8000:8000 + command: "-jar DynamoDBLocal.jar -sharedDb -dbPath /home/dynamodblocal/data/" + + dynamodb-admin: + image: aaronshaf/dynamodb-admin + ports: + - "8001:8001" + environment: + DYNAMO_ENDPOINT: "http://dynamodb-local:8000" + AWS_REGION: "us-west-2" + AWS_ACCESS_KEY_ID: local + AWS_SECRET_ACCESS_KEY: local + depends_on: + - dynamodb-local + networks: clusternetwork: name: eventstoredb.local diff --git a/samples/Infrastructure/Infrastructure.fsproj b/samples/Infrastructure/Infrastructure.fsproj index 85f8101d7..96806eaf1 100644 --- a/samples/Infrastructure/Infrastructure.fsproj +++ b/samples/Infrastructure/Infrastructure.fsproj @@ -16,6 +16,7 @@ + @@ -25,7 +26,7 @@ - + diff --git a/samples/Infrastructure/Services.fs b/samples/Infrastructure/Services.fs index b0b40ae28..0944ee89e 100644 --- a/samples/Infrastructure/Services.fs +++ b/samples/Infrastructure/Services.fs @@ -17,6 +17,9 @@ type StreamResolver(storage) = | Storage.StorageConfig.Cosmos (store, caching, unfolds) -> let accessStrategy = if unfolds then Equinox.CosmosStore.AccessStrategy.Snapshot snapshot else Equinox.CosmosStore.AccessStrategy.Unoptimized Equinox.CosmosStore.CosmosStoreCategory<'event,'state,_>(store, codec.ToJsonElementCodec(), fold, initial, caching, accessStrategy).Resolve + | Storage.StorageConfig.Dynamo (store, caching, unfolds) -> + let accessStrategy = if unfolds then Equinox.DynamoStore.AccessStrategy.Snapshot snapshot else Equinox.DynamoStore.AccessStrategy.Unoptimized + Equinox.DynamoStore.DynamoStoreCategory<'event,'state,_>(store, codec, fold, initial, caching, accessStrategy).Resolve | Storage.StorageConfig.Es (context, caching, unfolds) -> let accessStrategy = if unfolds then Equinox.EventStoreDb.AccessStrategy.RollingSnapshots snapshot |> Some else None Equinox.EventStoreDb.EventStoreCategory<'event,'state,_>(context, codec, fold, initial, ?caching = caching, ?access = accessStrategy).Resolve diff --git a/samples/Infrastructure/Storage.fs b/samples/Infrastructure/Storage.fs index 7d88135c1..c72c6c94e 100644 --- a/samples/Infrastructure/Storage.fs +++ b/samples/Infrastructure/Storage.fs @@ -9,6 +9,7 @@ type StorageConfig = // For MemoryStore, we keep the events as UTF8 arrays - we could use FsCodec.Codec.Box to remove the JSON encoding, which would improve perf but can conceal problems | Memory of Equinox.MemoryStore.VolatileStore> | Cosmos of Equinox.CosmosStore.CosmosStoreContext * Equinox.CosmosStore.CachingStrategy * unfolds: bool + | Dynamo of Equinox.DynamoStore.DynamoStoreContext * Equinox.DynamoStore.CachingStrategy * unfolds: bool | Es of Equinox.EventStoreDb.EventStoreContext * Equinox.EventStoreDb.CachingStrategy option * unfolds: bool | Sql of Equinox.SqlStreamStore.SqlStreamStoreContext * Equinox.SqlStreamStore.CachingStrategy option * unfolds: bool @@ -125,6 +126,69 @@ module Cosmos = let cacheStrategy = match cache with Some c -> CachingStrategy.SlidingWindow (c, TimeSpan.FromMinutes 20.) | None -> CachingStrategy.NoCaching StorageConfig.Cosmos (context, cacheStrategy, unfolds) +module Dynamo = + + open Equinox.DynamoStore + let [] SERVICE_URL = "EQUINOX_DYNAMO_SERVICE_URL" + let [] ACCESS_KEY = "EQUINOX_DYNAMO_ACCESS_KEY_ID" + let [] SECRET_KEY = "EQUINOX_DYNAMO_SECRET_ACCESS_KEY" + let [] TABLE = "EQUINOX_DYNAMO_TABLE" + type [] Arguments = + | [] VerboseStore + | [] ServiceUrl of string + | [] AccessKey of string + | [] SecretKey of string + | [] Table of string + | [] ArchiveTable of string + | [] Retries of int + | [] RetriesTimeoutS of float + | [] TipMaxBytes of int + | [] TipMaxEvents of int + | [] QueryMaxItems of int + interface IArgParserTemplate with + member a.Usage = a |> function + | VerboseStore -> "Include low level Store logging." + | ServiceUrl _ -> "specify a server endpoint for a Dynamo account. (optional if environment variable " + SERVICE_URL + " specified)" + | AccessKey _ -> "specify an access key id for a Dynamo account. (optional if environment variable " + ACCESS_KEY + " specified)" + | SecretKey _ -> "specify a secret access key for a Dynamo account. (optional if environment variable " + SECRET_KEY + " specified)" + | Table _ -> "specify a table name for the primary store. (optional if environment variable " + TABLE + " specified)" + | ArchiveTable _ -> "specify a table name for the Archive. Default: Do not attempt to look in an Archive store as a Fallback to locate pruned events." + | Retries _ -> "specify operation retries (default: 1)." + | RetriesTimeoutS _ -> "specify max wait-time including retries in seconds (default: 5)" + | TipMaxBytes _ -> "specify maximum number of bytes to hold in Tip before calving off to a frozen Batch. Default: 32K" + | TipMaxEvents _ -> "specify maximum number of events to hold in Tip before calving off to a frozen Batch. Default: limited by Max Bytes" + | QueryMaxItems _ -> "specify maximum number of batches of events to retrieve in per query response. Default: 10" + type Info(args : ParseResults) = + let serviceUrl = args.TryGetResult ServiceUrl |> defaultWithEnvVar SERVICE_URL "ServiceUrl" + let accessKey = args.TryGetResult AccessKey |> defaultWithEnvVar ACCESS_KEY "AccessKey" + let secretKey = args.TryGetResult SecretKey |> defaultWithEnvVar SECRET_KEY "SecretKey" + let retries = args.GetResult(Retries, 1) + let timeout = args.GetResult(RetriesTimeoutS, 5.) |> TimeSpan.FromSeconds + member val Connector = DynamoStoreConnector(serviceUrl, accessKey, secretKey, retries, timeout) + + member val Table = args.TryGetResult Table |> defaultWithEnvVar TABLE "Table" + member val ArchiveTable = args.TryGetResult ArchiveTable + + member x.TipMaxEvents = args.TryGetResult TipMaxEvents + member x.TipMaxBytes = args.GetResult(TipMaxBytes, 32 * 1024) + member x.QueryMaxItems = args.GetResult(QueryMaxItems, 10) + + let logTable (log: ILogger) endpoint role table = + log.Information("DynamoDB {name:l} {endpoint} Table {table}", role, endpoint, table) + let createStoreClient (log : ILogger) (a : Info) = + let client = a.Connector.CreateClient() + let storeClient = DynamoStoreClient(client, a.Table, ?archiveTableName = a.ArchiveTable) + logTable log a.Connector.Endpoint "Primary" a.Table + match a.ArchiveTable with None -> () | Some at -> logTable log a.Connector.Endpoint "Archive" at + storeClient + let config (log : ILogger) (cache, unfolds) (a : Info) = + let storeClient = createStoreClient log a + log.Information("DynamoStore Max Events in Tip: {maxTipBytes}b {maxTipEvents}e Query Limit: {queryMaxItems} items", + a.TipMaxBytes, a.TipMaxEvents, a.QueryMaxItems) + let context = DynamoStoreContext(storeClient, maxBytes = a.TipMaxBytes, queryMaxItems = a.QueryMaxItems, ?tipMaxEvents = a.TipMaxEvents) + let cacheStrategy = match cache with Some c -> CachingStrategy.SlidingWindow (c, TimeSpan.FromMinutes 20.) | None -> CachingStrategy.NoCaching + StorageConfig.Dynamo (context, cacheStrategy, unfolds) + /// To establish a local node to run the tests against, follow https://developers.eventstore.com/server/v21.10/installation.html#use-docker-compose /// and/or do `docker compose up` in github.com/jet/equinox module EventStore = diff --git a/samples/Tutorial/AsAt.fsx b/samples/Tutorial/AsAt.fsx index 3aa9c33c2..cbc5020a1 100644 --- a/samples/Tutorial/AsAt.fsx +++ b/samples/Tutorial/AsAt.fsx @@ -11,7 +11,7 @@ // - the same general point applies to over-using querying of streams for read purposes as we do here; // applying CQRS principles can often lead to a better model regardless of raw necessity -#if !LOCAL +#if LOCAL // Compile Tutorial.fsproj by either a) right-clicking or b) typing // dotnet build samples/Tutorial before attempting to send this to FSI with Alt-Enter #if VISUALSTUDIO diff --git a/samples/Tutorial/FulfilmentCenter.fsx b/samples/Tutorial/FulfilmentCenter.fsx index 556ec4c5a..ea9f6a566 100644 --- a/samples/Tutorial/FulfilmentCenter.fsx +++ b/samples/Tutorial/FulfilmentCenter.fsx @@ -1,4 +1,4 @@ -#if !LOCAL +#if LOCAL #I "bin/Debug/net6.0/" #r "Serilog.dll" #r "Serilog.Sinks.Console.dll" diff --git a/samples/Web/Startup.fs b/samples/Web/Startup.fs index fe6b4d36b..8d8274353 100644 --- a/samples/Web/Startup.fs +++ b/samples/Web/Startup.fs @@ -16,6 +16,7 @@ type Arguments = | [] Unfolds | [] Memory of ParseResults | [] Cosmos of ParseResults + | [] Dynamo of ParseResults | [] Es of ParseResults | [] MsSql of ParseResults | [] MySql of ParseResults @@ -27,6 +28,7 @@ type Arguments = | Cached -> "employ a 50MB cache." | Unfolds -> "employ a store-appropriate Rolling Snapshots and/or Unfolding strategy." | Cosmos _ -> "specify storage in CosmosDB (--help for options)." + | Dynamo _ -> "specify storage in DynamoDB (--help for options)." | Es _ -> "specify storage in EventStore (--help for options)." | Memory _ -> "specify In-Memory Volatile Store (Default store)." | MsSql _ -> "specify storage in Sql Server (--help for options)." @@ -68,6 +70,10 @@ type Startup() = let storeLog = createStoreLog <| sargs.Contains Storage.Cosmos.Arguments.VerboseStore log.Information("CosmosDB Storage options: {options:l}", options) Storage.Cosmos.config log (cache, unfolds) (Storage.Cosmos.Info sargs), storeLog + | Some (Dynamo sargs) -> + let storeLog = createStoreLog <| sargs.Contains Storage.Dynamo.Arguments.VerboseStore + log.Information("DynamoDB Storage options: {options:l}", options) + Storage.Dynamo.config log (cache, unfolds) (Storage.Dynamo.Info sargs), storeLog | Some (Es sargs) -> let storeLog = createStoreLog <| sargs.Contains Storage.EventStore.Arguments.VerboseStore log.Information("EventStoreDB Storage options: {options:l}", options) diff --git a/src/Equinox.CosmosStore.Prometheus/CosmosStorePrometheus.fs b/src/Equinox.CosmosStore.Prometheus/CosmosStorePrometheus.fs index 4984944f0..00a5802ea 100644 --- a/src/Equinox.CosmosStore.Prometheus/CosmosStorePrometheus.fs +++ b/src/Equinox.CosmosStore.Prometheus/CosmosStorePrometheus.fs @@ -7,12 +7,12 @@ module private Impl = module private Histograms = - let labelNames tagNames = Array.append tagNames [| "facet"; "op"; "db"; "con"; "cat" |] - let labelValues tagValues (facet, op, db, con, cat) = Array.append tagValues [| facet; op; db; con; cat |] + let labelNames tagNames = Array.append tagNames [| "rut"; "facet"; "op"; "db"; "con"; "cat" |] + let labelValues tagValues (rut, facet, op, db, con, cat) = Array.append tagValues [| rut; facet; op; db; con; cat |] let private mkHistogram (cfg : Prometheus.HistogramConfiguration) name desc = let h = Prometheus.Metrics.CreateHistogram(name, desc, cfg) - fun tagValues (facet : string, op : string) (db, con, cat : string) s -> - h.WithLabels(labelValues tagValues (facet, op, db, con, cat)).Observe(s) + fun tagValues (rut, facet : string, op : string) (db, con, cat : string) s -> + h.WithLabels(labelValues tagValues (rut, facet, op, db, con, cat)).Observe(s) // Given we also have summary metrics with equivalent labels, we focus the bucketing on LAN latencies let private sHistogram tagNames = let sBuckets = [| 0.0005; 0.001; 0.002; 0.004; 0.008; 0.016; 0.5; 1.; 2.; 4.; 8. |] @@ -26,9 +26,9 @@ module private Histograms = let baseName, baseDesc = Impl.baseName stat, Impl.baseDesc desc let observeS = sHistogram tagNames (baseName + "_seconds") (baseDesc + " latency") let observeRu = ruHistogram tagNames (baseName + "_ru") (baseDesc + " charge") - fun (facet, op) (db, con, cat, s : System.TimeSpan, ru) -> - observeS tagValues (facet, op) (db, con, cat) s.TotalSeconds - observeRu tagValues (facet, op) (db, con, cat) ru + fun (rut, facet, op) (db, con, cat, s : System.TimeSpan, ru) -> + observeS tagValues (rut, facet, op) (db, con, cat) s.TotalSeconds + observeRu tagValues (rut, facet, op) (db, con, cat) ru module private Summaries = @@ -84,40 +84,40 @@ type LogSink(customTags: seq) = let payloadCounters = Counters.eventsAndBytesPair tags "payload" "Payload, " let cacheCounter = Counters.total tags "cache" "Cache" - let observeLatencyAndCharge (facet, op) (db, con, cat, s, ru) = - opHistogram (facet, op) (db, con, cat, s, ru) + let observeLatencyAndCharge (rut, facet, op) (db, con, cat, s, ru) = + opHistogram (rut, facet, op) (db, con, cat, s, ru) opSummary facet (db, con, s, ru) - let observeLatencyAndChargeWithEventCounts (facet, op, outcome) (db, con, cat, s, ru, count, bytes) = - observeLatencyAndCharge (facet, op) (db, con, cat, s, ru) + let observeLatencyAndChargeWithEventCounts (rut, facet, op, outcome) (db, con, cat, s, ru, count, bytes) = + observeLatencyAndCharge (rut, facet, op) (db, con, cat, s, ru) payloadCounters (facet, op, outcome) (db, con, cat, float count, if bytes = -1 then None else Some (float bytes)) let (|CatSRu|) ({ interval = i; ru = ru } : Measurement as m) = let cat, _id = FsCodec.StreamName.splitCategoryAndId (FSharp.UMX.UMX.tag m.stream) m.database, m.container, cat, i.Elapsed, ru - let observeRes (facet, _op as stat) (CatSRu (db, con, cat, s, ru)) = + let observeRes (_rut, facet, _op as stat) (CatSRu (db, con, cat, s, ru)) = roundtripHistogram stat (db, con, cat, s, ru) roundtripSummary facet (db, con, s, ru) let observe_ stat (CatSRu (db, con, cat, s, ru)) = observeLatencyAndCharge stat (db, con, cat, s, ru) - let observe (facet, op, outcome) (CatSRu (db, con, cat, s, ru) as m) = - observeLatencyAndChargeWithEventCounts (facet, op, outcome) (db, con, cat, s, ru, m.count, m.bytes) - let observeTip (facet, op, outcome, cacheOutcome) (CatSRu (db, con, cat, s, ru) as m) = - observeLatencyAndChargeWithEventCounts (facet, op, outcome) (db, con, cat, s, ru, m.count, m.bytes) + let observe (rut, facet, op, outcome) (CatSRu (db, con, cat, s, ru) as m) = + observeLatencyAndChargeWithEventCounts (rut, facet, op, outcome) (db, con, cat, s, ru, m.count, m.bytes) + let observeTip (rut, facet, op, outcome, cacheOutcome) (CatSRu (db, con, cat, s, ru) as m) = + observeLatencyAndChargeWithEventCounts (rut, facet, op, outcome) (db, con, cat, s, ru, m.count, m.bytes) cacheCounter (facet, op, cacheOutcome) (db, con, cat) 1. interface Serilog.Core.ILogEventSink with member _.Emit logEvent = logEvent |> function | MetricEvent cm -> cm |> function - | Op (Operation.Tip, m) -> observeTip ("query", "tip", "ok", "200") m - | Op (Operation.Tip404, m) -> observeTip ("query", "tip", "ok", "404") m - | Op (Operation.Tip304, m) -> observeTip ("query", "tip", "ok", "304") m - | Op (Operation.Query, m) -> observe ("query", "query", "ok") m - | QueryRes (_direction, m) -> observeRes ("query", "queryPage") m - | Op (Operation.Write, m) -> observe ("transact", "sync", "ok") m - | Op (Operation.Conflict, m) -> observe ("transact", "conflict", "conflict") m - | Op (Operation.Resync, m) -> observe ("transact", "resync", "conflict") m - | Op (Operation.Prune, m) -> observe_ ("prune", "pruneQuery") m - | PruneRes ( m) -> observeRes ("prune", "pruneQueryPage") m - | Op (Operation.Delete, m) -> observe ("prune", "delete", "ok") m - | Op (Operation.Trim, m) -> observe ("prune", "trim", "ok") m + | Op (Operation.Tip, m) -> observeTip ("R", "query", "tip", "ok", "200") m + | Op (Operation.Tip404, m) -> observeTip ("R", "query", "tip", "ok", "404") m + | Op (Operation.Tip304, m) -> observeTip ("R", "query", "tip", "ok", "304") m + | Op (Operation.Query, m) -> observe ("R", "query", "query", "ok") m + | QueryRes (_direction, m) -> observeRes ("R", "query", "queryPage") m + | Op (Operation.Write, m) -> observe ("W", "transact", "sync", "ok") m + | Op (Operation.Conflict, m) -> observe ("W", "transact", "conflict", "conflict") m + | Op (Operation.Resync, m) -> observe ("W", "transact", "resync", "conflict") m + | Op (Operation.Prune, m) -> observe_ ("R", "prune", "pruneQuery") m + | PruneRes ( m) -> observeRes ("R", "prune", "pruneQueryPage") m + | Op (Operation.Delete, m) -> observe ("W", "prune", "delete", "ok") m + | Op (Operation.Trim, m) -> observe ("W", "prune", "trim", "ok") m | _ -> () diff --git a/src/Equinox.CosmosStore/CosmosStore.fs b/src/Equinox.CosmosStore/CosmosStore.fs index 55436dcdd..df61d41a8 100644 --- a/src/Equinox.CosmosStore/CosmosStore.fs +++ b/src/Equinox.CosmosStore/CosmosStore.fs @@ -323,24 +323,25 @@ module Log = "Prune", Stats.LogSink.Prune "Delete", Stats.LogSink.Delete "Trim", Stats.LogSink.Trim ] - let mutable rows, totalCount, totalRc, totalMs = 0, 0L, 0., 0L - let logActivity name count rc lat = - log.Information("{name}: {count:n0} requests costing {ru:n0} RU (average: {avg:n2}); Average latency: {lat:n0}ms", - name, count, rc, (if count = 0L then Double.NaN else rc/float count), (if count = 0L then Double.NaN else float lat/float count)) + let mutable rows, totalCount, totalRRu, totalWRu, totalMs = 0, 0L, 0., 0., 0L + let logActivity name count ru lat = + let aru, ams = (if count = 0L then Double.NaN else ru/float count), (if count = 0L then Double.NaN else float lat/float count) + let rut = match name with "TOTAL" -> "" | "Read" | "Prune" -> totalRRu <- totalRRu + ru; "R" | _ -> totalWRu <- totalWRu + ru; "W" + log.Information("{name}: {count:n0} requests costing {ru:n0}{rut:l}RU (average: {avgRu:n1}); Average latency: {lat:n0}ms", + name, count, ru, rut, aru, ams) for name, stat in stats do if stat.count <> 0L then let ru = float stat.rux100 / 100. totalCount <- totalCount + stat.count - totalRc <- totalRc + ru totalMs <- totalMs + stat.ms logActivity name stat.count ru stat.ms rows <- rows + 1 // Yes, there's a minor race here between the use of the values and the reset let duration = Stats.LogSink.Restart() - if rows > 1 then logActivity "TOTAL" totalCount totalRc totalMs + if rows > 1 then logActivity "TOTAL" totalCount (totalRRu + totalWRu) totalMs let measures : (string * (TimeSpan -> float)) list = [ "s", fun x -> x.TotalSeconds(*; "m", fun x -> x.TotalMinutes; "h", fun x -> x.TotalHours*) ] - let logPeriodicRate name count ru = log.Information("rp{name} {count:n0} = ~{ru:n0} RU", name, count, ru) - for uom, f in measures do let d = f duration in if d <> 0. then logPeriodicRate uom (float totalCount/d |> int64) (totalRc/d) + let logPeriodicRate name count rru wru = log.Information("rp{name} {count:n0} = ~{rru:n1}R/{wru:n1}W RU", name, count, rru, wru) + for uom, f in measures do let d = f duration in if d <> 0. then logPeriodicRate uom (float totalCount/d |> int64) (totalRRu/d) (totalWRu/d) [] module private MicrosoftAzureCosmosWrappers = @@ -604,9 +605,9 @@ module internal Tip = let log bytes count (f : Log.Measurement -> _) = log |> Log.event (f { database = container.Database.Id; container = container.Id; stream = stream; interval = t; bytes = bytes; count = count; ru = ru }) match res with | ReadResult.NotModified -> - (log 0 0 Log.Metric.TipNotModified).Information("EqxCosmos {action:l} {stream} {res} {ms}ms {ru}RU", "Tip", stream, 304, (let e = t.Elapsed in e.TotalMilliseconds), ru) + (log 0 0 Log.Metric.TipNotModified).Information("EqxCosmos {action:l} {stream} {res} {ms:f1}ms {ru}RU", "Tip", stream, 304, (let e = t.Elapsed in e.TotalMilliseconds), ru) | ReadResult.NotFound -> - (log 0 0 Log.Metric.TipNotFound).Information("EqxCosmos {action:l} {stream} {res} {ms}ms {ru}RU", "Tip", stream, 404, (let e = t.Elapsed in e.TotalMilliseconds), ru) + (log 0 0 Log.Metric.TipNotFound).Information("EqxCosmos {action:l} {stream} {res} {ms:f1}ms {ru}RU", "Tip", stream, 404, (let e = t.Elapsed in e.TotalMilliseconds), ru) | ReadResult.Found tip -> let log = let count, bytes = tip.u.Length, if verbose then Enum.Unfolds tip.u |> Log.batchLen else 0 @@ -614,7 +615,7 @@ module internal Tip = let log = if verbose then log |> Log.propDataUnfolds tip.u else log let log = match maybePos with Some p -> log |> Log.propStartPos p |> Log.propStartEtag p | None -> log let log = log |> Log.prop "_etag" tip._etag |> Log.prop "n" tip.n - log.Information("EqxCosmos {action:l} {stream} {res} {ms}ms {ru}RU", "Tip", stream, 200, (let e = t.Elapsed in e.TotalMilliseconds), ru) + log.Information("EqxCosmos {action:l} {stream} {res} {ms:f1}ms {ru}RU", "Tip", stream, 200, (let e = t.Elapsed in e.TotalMilliseconds), ru) return ru, res } type [] Result = NotModified | NotFound | Found of Position * i : int64 * ITimelineEvent[] /// `pos` being Some implies that the caller holds a cached value and hence is ready to deal with Result.NotModified @@ -681,7 +682,7 @@ module internal Query = (log|> Log.prop "bytes" bytes |> match minIndex with None -> id | Some i -> Log.prop "minIndex" i |> match maxIndex with None -> id | Some i -> Log.prop "maxIndex" i) - .Information("EqxCosmos {action:l} {count}/{batches} {direction} {ms}ms i={index} {ru}RU", + .Information("EqxCosmos {action:l} {count}/{batches} {direction} {ms:f1}ms i={index} {ru}RU", "Response", count, batches.Length, direction, (let e = t.Elapsed in e.TotalMilliseconds), index, ru) let maybePosition = batches |> Array.tryPick Position.tryFromBatch events, maybePosition, ru @@ -693,7 +694,7 @@ module internal Query = let evt = Log.Metric.Query (direction, responsesCount, reqMetric) let action = match direction with Direction.Forward -> "QueryF" | Direction.Backward -> "QueryB" (log |> Log.prop "bytes" bytes |> Log.event evt).Information( - "EqxCosmos {action:l} {stream} v{n} {count}/{responses} {ms}ms {ru}RU", + "EqxCosmos {action:l} {stream} v{n} {count}/{responses} {ms:f1}ms {ru}RU", action, streamName, n, count, responsesCount, (let e = interval.Elapsed in e.TotalMilliseconds), ru) let private calculateUsedVersusDroppedPayload stopIndex (xs : ITimelineEvent[]) : int * int = @@ -884,7 +885,7 @@ module Prune = let rc, ms = res.RequestCharge, (let e = t.Elapsed in e.TotalMilliseconds) let reqMetric : Log.Measurement = { database = container.Database.Id; container = container.Id; stream = stream; interval = t; bytes = -1; count = count; ru = rc } let log = let evt = Log.Metric.Delete reqMetric in log |> Log.event evt - log.Information("EqxCosmos {action:l} {id} {ms}ms {ru}RU", "Delete", id, ms, rc) + log.Information("EqxCosmos {action:l} {id} {ms:f1}ms {ru}RU", "Delete", id, ms, rc) return rc } let trimTip expectedI count = async { @@ -900,7 +901,7 @@ module Prune = let rc, ms = tipRu + updateRes.RequestCharge, (let e = t.Elapsed in e.TotalMilliseconds) let reqMetric : Log.Measurement = { database = container.Database.Id; container = container.Id; stream = stream; interval = t; bytes = -1; count = count; ru = rc } let log = let evt = Log.Metric.Trim reqMetric in log |> Log.event evt - log.Information("EqxCosmos {action:l} {count} {ms}ms {ru}RU", "Trim", count, ms, rc) + log.Information("EqxCosmos {action:l} {count} {ms:f1}ms {ru}RU", "Trim", count, ms, rc) return rc } let log = log |> Log.prop "index" indexInclusive @@ -913,7 +914,7 @@ module Prune = let next = Array.tryLast batches |> Option.map (fun x -> x.n) |> Option.toNullable let reqMetric : Log.Measurement = { database = container.Database.Id; container = container.Id; stream = stream; interval = t; bytes = -1; count = batches.Length; ru = rc } let log = let evt = Log.Metric.PruneResponse reqMetric in log |> Log.prop "batchIndex" i |> Log.event evt - log.Information("EqxCosmos {action:l} {batches} {ms}ms n={next} {ru}RU", "PruneResponse", batches.Length, ms, next, rc) + log.Information("EqxCosmos {action:l} {batches} {ms:f1}ms n={next} {ru}RU", "PruneResponse", batches.Length, ms, next, rc) batches, rc let! pt, outcomes = let isTip (x : BatchIndices) = x.id = Tip.WellKnownDocumentId @@ -964,7 +965,7 @@ module Prune = let reqMetric : Log.Measurement = { database = container.Database.Id; container = container.Id; stream = stream; interval = pt; bytes = eventsDeleted; count = batches; ru = queryCharges } let log = let evt = Log.Metric.Prune (responses, reqMetric) in log |> Log.event evt let lwm = lwm |> Option.defaultValue 0L // If we've seen no batches at all, then the write position is 0L - log.Information("EqxCosmos {action:l} {events}/{batches} lwm={lwm} {ms}ms queryRu={queryRu} deleteRu={deleteRu} trimRu={trimRu}", + log.Information("EqxCosmos {action:l} {events}/{batches} lwm={lwm} {ms:f1}ms queryRu={queryRu} deleteRu={deleteRu} trimRu={trimRu}", "Prune", eventsDeleted, batches, lwm, (let e = pt.Elapsed in e.TotalMilliseconds), queryCharges, delCharges, trimCharges) return eventsDeleted, eventsDeferred, lwm } diff --git a/src/Equinox.DynamoStore.Prometheus/DynamoStorePrometheus.fs b/src/Equinox.DynamoStore.Prometheus/DynamoStorePrometheus.fs new file mode 100644 index 000000000..cae34a96d --- /dev/null +++ b/src/Equinox.DynamoStore.Prometheus/DynamoStorePrometheus.fs @@ -0,0 +1,122 @@ +namespace Equinox.DynamoStore.Prometheus + +module private Impl = + + let baseName stat = "equinox_ddb_" + stat + let baseDesc desc = "Equinox DynamoDB " + desc + +module private Histograms = + + let labelNames tagNames = Array.append tagNames [| "rut"; "facet"; "op"; "table"; "cat" |] + let labelValues tagValues (rut, facet, op, table, cat) = Array.append tagValues [| rut; facet; op; table; cat |] + let private mkHistogram (cfg : Prometheus.HistogramConfiguration) name desc = + let h = Prometheus.Metrics.CreateHistogram(name, desc, cfg) + fun tagValues (rut, facet : string, op : string) (table, cat : string) s -> + h.WithLabels(labelValues tagValues (rut, facet, op, table, cat)).Observe(s) + // Given we also have summary metrics with equivalent labels, we focus the bucketing on LAN latencies + let private sHistogram tagNames = + let sBuckets = [| 0.0005; 0.001; 0.002; 0.004; 0.008; 0.016; 0.5; 1.; 2.; 4.; 8. |] + let sCfg = Prometheus.HistogramConfiguration(Buckets = sBuckets, LabelNames = labelNames tagNames) + mkHistogram sCfg + let private ruHistogram tagNames = + let ruBuckets = Prometheus.Histogram.ExponentialBuckets(1., 2., 11) // 1 .. 1024 + let ruCfg = Prometheus.HistogramConfiguration(Buckets = ruBuckets, LabelNames = labelNames tagNames) + mkHistogram ruCfg + let sAndRuPair (tagNames, tagValues) stat desc = + let baseName, baseDesc = Impl.baseName stat, Impl.baseDesc desc + let observeS = sHistogram tagNames (baseName + "_seconds") (baseDesc + " latency") + let observeRu = ruHistogram tagNames (baseName + "_ru") (baseDesc + " charge") + fun (rut, facet, op) (table, cat, s : System.TimeSpan, ru) -> + observeS tagValues (rut, facet, op) (table, cat) s.TotalSeconds + observeRu tagValues (rut, facet, op) (table, cat) ru + +module private Summaries = + + let labelNames tagNames = Array.append tagNames [| "facet"; "table" |] + let labelValues tagValues (facet, table) = Array.append tagValues [| facet; table |] + let private mkSummary (cfg : Prometheus.SummaryConfiguration) name desc = + let s = Prometheus.Metrics.CreateSummary(name, desc, cfg) + fun tagValues (facet : string) table o -> s.WithLabels(labelValues tagValues (facet, table)).Observe(o) + let config tagNames = + let inline qep q e = Prometheus.QuantileEpsilonPair(q, e) + let objectives = [| qep 0.50 0.05; qep 0.95 0.01; qep 0.99 0.01 |] + Prometheus.SummaryConfiguration(Objectives = objectives, LabelNames = labelNames tagNames, MaxAge = System.TimeSpan.FromMinutes 1.) + let sAndRuPair (tagNames, tagValues) stat desc = + let baseName, baseDesc = Impl.baseName stat, Impl.baseDesc desc + let observeS = mkSummary (config tagNames) (baseName + "_seconds") (baseDesc + " latency") tagValues + let observeRu = mkSummary (config tagNames) (baseName + "_ru") (baseDesc + " charge") tagValues + fun facet (table, s : System.TimeSpan, ru) -> + observeS facet table s.TotalSeconds + observeRu facet table ru + +module private Counters = + + let labelNames tagNames = Array.append tagNames [| "facet"; "op"; "outcome"; "table"; "cat" |] + let labelValues tagValues (facet, op, outcome, table, cat) = Array.append tagValues [| facet; op; outcome; table; cat |] + let private mkCounter (cfg : Prometheus.CounterConfiguration) name desc = + let h = Prometheus.Metrics.CreateCounter(name, desc, cfg) + fun tagValues (facet : string, op : string, outcome : string) (table, cat) c -> + h.WithLabels(labelValues tagValues (facet, op, outcome, table, cat)).Inc(c) + let config tagNames = Prometheus.CounterConfiguration(LabelNames = labelNames tagNames) + let total (tagNames, tagValues) stat desc = + let name = Impl.baseName (stat + "_total") + let desc = Impl.baseDesc desc + mkCounter (config tagNames) name desc tagValues + let eventsAndBytesPair tags stat desc = + let observeE = total tags (stat + "_events") (desc + "Events") + let observeB = total tags (stat + "_bytes") (desc + "Bytes") + fun ctx (table, cat, e, b) -> + observeE ctx (table, cat) e + match b with None -> () | Some b -> observeB ctx (table, cat) b + +open Equinox.DynamoStore.Core.Log + +/// An ILogEventSink that publishes to Prometheus +/// Custom tags to annotate the metric we're publishing where such tag manipulation cannot better be achieved via the Prometheus scraper config. +type LogSink(customTags: seq) = + + let tags = Array.ofSeq customTags |> Array.unzip + + let opHistogram = Histograms.sAndRuPair tags "op" "Operation" + let roundtripHistogram = Histograms.sAndRuPair tags "roundtrip" "Fragment" + let opSummary = Summaries.sAndRuPair tags "op_summary" "Operation Summary" + let roundtripSummary = Summaries.sAndRuPair tags "roundtrip_summary" "Fragment Summary" + let payloadCounters = Counters.eventsAndBytesPair tags "payload" "Payload, " + let cacheCounter = Counters.total tags "cache" "Cache" + + let observeLatencyAndCharge (rut, facet, op) (table, cat, s, ru) = + opHistogram (rut, facet, op) (table, cat, s, ru) + opSummary facet (table, s, ru) + let observeLatencyAndChargeWithEventCounts (rut, facet, op, outcome) (table, cat, s, ru, count, bytes) = + observeLatencyAndCharge (rut, facet, op) (table, cat, s, ru) + payloadCounters (facet, op, outcome) (table, cat, float count, if bytes = -1 then None else Some (float bytes)) + + let (|CatSRu|) ({ interval = i; ru = ru } : Measurement as m) = + let cat, _id = FsCodec.StreamName.splitCategoryAndId (FSharp.UMX.UMX.tag m.stream) + m.table, cat, i.Elapsed, ru + let observeRes (_rut, facet, _op as stat) (CatSRu (table, cat, s, ru)) = + roundtripHistogram stat (table, cat, s, ru) + roundtripSummary facet (table, s, ru) + let observe_ stat (CatSRu (table, cat, s, ru)) = + observeLatencyAndCharge stat (table, cat, s, ru) + let observe (rut, facet, op, outcome) (CatSRu (table, cat, s, ru) as m) = + observeLatencyAndChargeWithEventCounts (rut, facet, op, outcome) (table, cat, s, ru, m.count, m.bytes) + let observeTip (rut, facet, op, outcome, cacheOutcome) (CatSRu (table, cat, s, ru) as m) = + observeLatencyAndChargeWithEventCounts (rut, facet, op, outcome) (table, cat, s, ru, m.count, m.bytes) + cacheCounter (facet, op, cacheOutcome) (table, cat) 1. + + interface Serilog.Core.ILogEventSink with + member _.Emit logEvent = logEvent |> function + | MetricEvent cm -> cm |> function + | Op (Operation.Tip, m) -> observeTip ("R", "query", "tip", "ok", "200") m + | Op (Operation.Tip404, m) -> observeTip ("R", "query", "tip", "ok", "404") m + | Op (Operation.Tip304, m) -> observeTip ("R", "query", "tip", "ok", "304") m + | Op (Operation.Query, m) -> observe ("R", "query", "query", "ok") m + | QueryRes (_direction, m) -> observeRes ("R", "query", "queryPage") m + | Op (Operation.Write, m) -> observe ("W", "transact", "sync", "ok") m + | Op (Operation.Conflict, m) -> observe ("W", "transact", "conflict", "conflict") m + | Op (Operation.Prune, m) -> observe_ ("R", "prune", "pruneQuery") m + | PruneRes m -> observeRes ("R", "prune", "pruneQueryPage") m + | Op (Operation.Delete, m) -> observe ("W", "prune", "delete", "ok") m + | Op (Operation.Trim, m) -> observe ("W", "prune", "trim", "ok") m + | _ -> () diff --git a/src/Equinox.DynamoStore.Prometheus/Equinox.DynamoStore.Prometheus.fsproj b/src/Equinox.DynamoStore.Prometheus/Equinox.DynamoStore.Prometheus.fsproj new file mode 100644 index 000000000..8c2b2ad95 --- /dev/null +++ b/src/Equinox.DynamoStore.Prometheus/Equinox.DynamoStore.Prometheus.fsproj @@ -0,0 +1,26 @@ + + + + netstandard2.1 + true + true + + + + + + + + + + + + + + + + + + + + diff --git a/src/Equinox.DynamoStore/DynamoStore.fs b/src/Equinox.DynamoStore/DynamoStore.fs new file mode 100644 index 000000000..dd73bb878 --- /dev/null +++ b/src/Equinox.DynamoStore/DynamoStore.fs @@ -0,0 +1,1499 @@ +namespace Equinox.DynamoStore.Core + +open Equinox.Core +open FsCodec +open FSharp.AWS.DynamoDB +open FSharp.Control +open Serilog +open System +open System.IO + +[] +type EncodedBody = { isCompressed : bool; data : MemoryStream } +module private EncodedBody = + let ofRawAndCompressed (raw : MemoryStream option, compressed : MemoryStream option) : EncodedBody = + let compressed = Option.toObj compressed + if compressed <> null then { isCompressed = true; data = compressed } + else { isCompressed = false; data = Option.toObj raw } + let toRawAndCompressed (encoded : EncodedBody) = + if encoded.isCompressed then None, Option.ofObj encoded.data + else Option.ofObj encoded.data, None + let bytes (x : EncodedBody) = + if x.data = null then 0 + else int x.data.Length + +/// A single Domain Event from the array held in a Batch +[] +type Event = + { /// Index number within stream, not persisted (computed from Batch's `n` and the index within `e`) + i : int + + /// Creation Timestamp, as set by the application layer at the point of rendering the Event + t : DateTimeOffset + + /// The Event Type (Case) that defines the content of the Data (and Metadata) fields + c : string + + /// Main event body; required + d : EncodedBody + + /// Optional metadata, encoded as per 'd'; can be Empty + m : EncodedBody + + /// CorrelationId; stored as x (signifying transactionId), or null + correlationId : string option + + /// CausationId; stored as y (signifying why), or null + causationId : string option } + interface ITimelineEvent with + member x.Index = x.i + member x.Context = null + member x.IsUnfold = false + member x.EventType = x.c + member x.Data = x.d + member x.Meta = x.m + member _.EventId = Guid.Empty + member x.CorrelationId = Option.toObj x.correlationId + member x.CausationId = Option.toObj x.causationId + member x.Timestamp = x.t +module Event = + let private len = function Some (s : string) -> s.Length | None -> 0 + let bytes (x : Event) = x.c.Length + EncodedBody.bytes x.d + EncodedBody.bytes x.m + len x.correlationId + len x.causationId + 100 + let arrayBytes (xs : Event array) = Array.sumBy bytes xs + +/// Compaction/Snapshot/Projection Event based on the state at a given point in time `i` +[] +type Unfold = + { /// Base: Stream Position (Version) of State from which this Unfold was generated + i : int64 + + /// Generation datetime + t : DateTimeOffset + + /// The Case (Event Type) of this snapshot, used to drive deserialization + c : string // required + + /// Event body + d : EncodedBody // required + + /// Optional metadata, can be Empty + m : EncodedBody } + interface ITimelineEvent with + member x.Index = x.i + member x.Context = null + member x.IsUnfold = true + member x.EventType = x.c + member x.Data = x.d + member x.Meta = x.m + member _.EventId = Guid.Empty + member x.CorrelationId = null + member x.CausationId = null + member x.Timestamp = x.t +module Unfold = + let private bytes (x : Unfold) = x.c.Length + EncodedBody.bytes x.d + EncodedBody.bytes x.m + 50 + let arrayBytes (xs : Unfold array) = match xs with null -> 0 | u -> Array.sumBy bytes u + +/// The abstract storage format for a 'normal' (frozen, not Tip) Batch of Events (without any Unfolds) +/// NOTE See BatchSchema for what actually gets stored +/// NOTE names are intended to generally align with CosmosStore naming. Key Diffs: +/// - no mandatory `id` and/or requirement for it to be a `string` -> replaced with `i` as an int64 (also Tip magic value is tipMagicI: int.MaxValue, not "-1") +/// - etag is managed explicitly (on Cosmos DB, its managed by the service and named "_etag") +/// NOTE see the BatchSchema buddy type for what the store internally has +[] +type Batch = + { p : string // "{streamName}" + + /// base 'i' value for the Events held herein + i : int64 // tipMagicI for the Tip + + /// Marker on which compare-and-swap operations on Tip are predicated + etag : string + + /// `i` value for successor batch (to facilitate identifying which Batch a given startPos is within) + n : int64 + + /// The Domain Events (as opposed to Unfolded Events, see Tip) at this offset in the stream + e : Event array + + /// Compaction/Snapshot/Projection quasi-events + u : Unfold array } +module Batch = + /// NOTE QueryIAndNOrderByNAscending and others rely on this, when used as the [], sorting after the other items + let tipMagicI = int64 Int32.MaxValue + let tableKeyForStreamTip stream = TableKey.Combined(stream, tipMagicI) + let isTip i = i = tipMagicI + /// The concrete storage format + [] + type Schema = + { [] + p : string + [] + i : int64 // tipMagicI for the Tip + etag : string option + n : int64 + // Count of items written in the most recent insert/update - used by the DDB Streams Consumer to identify the fresh events + a : int + // NOTE the per-event e.c values are actually stored here, so they can be selected out without hydrating the bodies + c : string array + // NOTE as per Event, but without c and t fields; we instead unroll those as arrays at top level + e : EventSchema array + u : UnfoldSchema array } + and [] EventSchema = + { t : DateTimeOffset // NOTE there has to be a single non-`option` field per record, or a trailing insert will be stripped + d : MemoryStream option; D : MemoryStream option // D if compressed, d if raw // required + m : MemoryStream option; M : MemoryStream option // M if compressed, m if raw + x : string option + y : string option } + and [] UnfoldSchema = + { i : int64 + t : DateTimeOffset + c : string // required + d : MemoryStream option; D : MemoryStream option // D if compressed, d if raw // required + m : MemoryStream option; M : MemoryStream option } // M if compressed, m if raw + let private toEventSchema (x : Event) : EventSchema = + let (d, D), (m, M) = EncodedBody.toRawAndCompressed x.d, EncodedBody.toRawAndCompressed x.m + { t = x.t; d = d; D = D; m = m; M = M; x = x.correlationId; y = x.causationId } + let eventsToSchema (xs : Event array) : (*case*) string array * EventSchema array = + xs |> Array.map (fun x -> x.c), xs |> Array.map toEventSchema + let private toUnfoldSchema (x : Unfold) : UnfoldSchema = + let (d, D), (m, M) = EncodedBody.toRawAndCompressed x.d, EncodedBody.toRawAndCompressed x.m + { i = x.i; t = x.t; c = x.c; d = d; D = D; m = m; M = M } + let unfoldsToSchema = Array.map toUnfoldSchema + let private ofUnfoldSchema (x : UnfoldSchema) : Unfold = + { i = x.i; t = x.t; c = x.c; d = EncodedBody.ofRawAndCompressed (x.d, x.D); m = EncodedBody.ofRawAndCompressed (x.m, x.M) } + let ofSchema (x : Schema) : Batch = + let baseIndex = int x.n - x.e.Length + let events = + Seq.zip x.c x.e + |> Seq.mapi (fun i (c, e) -> + let d, m = EncodedBody.ofRawAndCompressed (e.d, e.D), EncodedBody.ofRawAndCompressed (e.m, e.M) + { i = baseIndex + i; t = e.t; d = d; m = m; correlationId = e.x; causationId = e.y; c = c }) + { p = x.p; i = x.i; etag = Option.toObj x.etag; n = x.n; e = Seq.toArray events; u = x.u |> Array.map ofUnfoldSchema } + let enumEvents (minIndex, maxIndex) (x : Batch) : Event seq = + let indexMin, indexMax = defaultArg minIndex 0L, defaultArg maxIndex Int64.MaxValue + // If we're loading from a nominated position, we need to discard items in the batch before/after the start on the start page + x.e |> Seq.filter (fun e -> let i = int64 e.i in i >= indexMin && int64 i < indexMax) + + /// Computes base Index for the Item (`i` can bear the the magic value TipI when the Item is the Tip) + let baseIndex (x : Batch) = x.n - x.e.LongLength + let bytesUnfolds (x : Batch) = Unfold.arrayBytes x.u + let bytesBase (x : Batch) = 80 + x.p.Length + String.length x.etag + Event.arrayBytes x.e + let bytesTotal (xs : Batch seq) = xs |> Seq.sumBy (fun x -> bytesBase x + bytesUnfolds x) + +type EventBody = ReadOnlyMemory +module EventBody = + + (* All EncodedBody can potentially hold compressed content, that we'll inflate on demand *) + + let private inflate (loaded : MemoryStream) : byte array = + let decompressor = new System.IO.Compression.DeflateStream(loaded, System.IO.Compression.CompressionMode.Decompress) + let output = new MemoryStream() + decompressor.CopyTo(output) + output.ToArray() + let decode (encoded : EncodedBody) : EventBody = + if encoded.isCompressed then inflate encoded.data |> ReadOnlyMemory + elif encoded.data = null then ReadOnlyMemory.Empty + else encoded.data.ToArray() |> ReadOnlyMemory + let decodeEvent = FsCodec.Core.TimelineEvent.Map decode + + (* Compression is conditional on the input meeting a minimum size, and the result meeting a required gain *) + + let private compress (eventBody : ReadOnlyMemory) : MemoryStream = + let output = new MemoryStream() // NB not `use` - Dispose would Close the stream, and AWS SDK requires it open + let compressor = new System.IO.Compression.DeflateStream(output, System.IO.Compression.CompressionLevel.Optimal) + compressor.Write(eventBody.Span) + compressor.Flush() // NB not `Close` as that would Close the `output`, and AWS SDK requires the Stream open + output + let private encodeUncompressed (raw : EventBody) = { isCompressed = false; data = new MemoryStream(raw.ToArray(), writable = false) } + let internal encode (minSize, minGain) (raw : EventBody) : EncodedBody = + if raw.IsEmpty then { isCompressed = false; data = null } + elif raw.Length < minSize then encodeUncompressed raw + else match compress raw with + | tmp when raw.Length > int tmp.Length + minGain -> { isCompressed = true; data = tmp } + | _ -> encodeUncompressed raw + +type EncodingOptions = + { eventData : int; eventMeta : int; unfoldData : int; unfoldMeta : int; minGain : int } + member x.EncodeEventData raw = EventBody.encode (x.eventData, x.minGain) raw + member x.EncodeEventMeta raw = EventBody.encode (x.eventMeta, x.minGain) raw + member x.EncodeUnfoldData raw = EventBody.encode (x.unfoldData, x.minGain) raw + member x.EncodeUnfoldMeta raw = EventBody.encode (x.unfoldMeta, x.minGain) raw + +// We only capture the total RUs, without attempting to split them into read/write as the service does not actually populate the associated fields +// See https://github.com/aws/aws-sdk-go/issues/2699#issuecomment-514378464 +[] +type RequestConsumption = { total : float } + +[] +type Direction = Forward | Backward override this.ToString() = match this with Forward -> "Forward" | Backward -> "Backward" + +module Log = + + /// Name of Property used for Metric in LogEvents. + let [] PropertyTag = "ddbEvt" + + [] + type Measurement = + { table : string; stream : string + interval : StopwatchInterval; bytes : int; count : int; ru : float } + let inline metric table stream t bytes count rc : Measurement = + { table = table; stream = stream; interval = t; bytes = bytes; count = count; ru = rc.total } + [] + type Metric = + /// Individual read request for the Tip + | Tip of Measurement + /// Individual read request for the Tip, not found + | TipNotFound of Measurement + /// Tip read but etag unchanged, signifying payload not inspected as it can be trusted not to have been altered + /// (NOTE the read is still fully charged on Dynamo, as opposed to Cosmos where it is 1 RU regardless of size) + | TipNotModified of Measurement + + /// Summarizes a set of Responses for a given Read request + | Query of Direction * responses : int * Measurement + /// Individual read request in a Batch + /// Charges are rolled up into Query Metric (so do not double count) + | QueryResponse of Direction * Measurement + + | SyncSuccess of Measurement + | SyncConflict of Measurement + + /// Summarizes outcome of request to trim batches from head of a stream and events in Tip + /// count in Measurement is number of batches (documents) deleted + /// bytes in Measurement is number of events deleted + | Prune of responsesHandled : int * Measurement + /// Handled response from listing of batches in a stream + /// Charges are rolled up into the Prune Metric (so do not double count) + | PruneResponse of Measurement + /// Deleted an individual Batch + | Delete of Measurement + /// Trimmed the Tip + | Trim of Measurement + let tms (t : StopwatchInterval) = let e = t.Elapsed in e.TotalMilliseconds + let internal prop name value (log : ILogger) = log.ForContext(name, value) + + /// Include a LogEvent property bearing metrics + // Sidestep Log.ForContext converting to a string; see https://github.com/serilog/serilog/issues/1124 + let internal event (value : Metric) (log : ILogger) = + let enrich (e : Serilog.Events.LogEvent) = + e.AddPropertyIfAbsent(Serilog.Events.LogEventProperty(PropertyTag, Serilog.Events.ScalarValue(value))) + log.ForContext({ new Serilog.Core.ILogEventEnricher with member _.Enrich(evt, _) = enrich evt }) + let internal (|SerilogScalar|_|) : Serilog.Events.LogEventPropertyValue -> obj option = function + | :? Serilog.Events.ScalarValue as x -> Some x.Value + | _ -> None + let (|MetricEvent|_|) (logEvent : Serilog.Events.LogEvent) : Metric option = + match logEvent.Properties.TryGetValue(PropertyTag) with + | true, SerilogScalar (:? Metric as e) -> Some e + | _ -> None + [] + type Operation = Tip | Tip404 | Tip304 | Query | Write | Conflict | Prune | Delete | Trim + let (|Op|QueryRes|PruneRes|) = function + | Metric.Tip s -> Op (Operation.Tip, s) + | Metric.TipNotFound s -> Op (Operation.Tip404, s) + | Metric.TipNotModified s -> Op (Operation.Tip304, s) + + | Metric.Query (_, _, s) -> Op (Operation.Query, s) + | Metric.QueryResponse (direction, s) -> QueryRes (direction, s) + + | Metric.SyncSuccess s -> Op (Operation.Write, s) + | Metric.SyncConflict s -> Op (Operation.Conflict, s) + + | Metric.Prune (_, s) -> Op (Operation.Prune, s) + | Metric.PruneResponse s -> PruneRes s + | Metric.Delete s -> Op (Operation.Delete, s) + | Metric.Trim s -> Op (Operation.Trim, s) + + module InternalMetrics = + + module Stats = + + type internal Counter = + { mutable rux100 : int64; mutable count : int64; mutable ms : int64 } + static member Create() = { rux100 = 0L; count = 0L; ms = 0L } + member x.Ingest(ru, ms) = + System.Threading.Interlocked.Increment(&x.count) |> ignore + System.Threading.Interlocked.Add(&x.rux100, int64 (ru * 100.)) |> ignore + System.Threading.Interlocked.Add(&x.ms, ms) |> ignore + let inline private (|RuMs|) ({ interval = i; ru = ru } : Measurement) = ru, int64 (tms i) + type LogSink() = + static let epoch = System.Diagnostics.Stopwatch.StartNew() + static member val internal Read = Counter.Create() with get, set + static member val internal Write = Counter.Create() with get, set + static member val internal Conflict = Counter.Create() with get, set + static member val internal Prune = Counter.Create() with get, set + static member val internal Delete = Counter.Create() with get, set + static member val internal Trim = Counter.Create() with get, set + static member Restart() = + LogSink.Read <- Counter.Create() + LogSink.Write <- Counter.Create() + LogSink.Conflict <- Counter.Create() + LogSink.Prune <- Counter.Create() + LogSink.Delete <- Counter.Create() + LogSink.Trim <- Counter.Create() + let span = epoch.Elapsed + epoch.Restart() + span + interface Serilog.Core.ILogEventSink with + member _.Emit logEvent = + match logEvent with + | MetricEvent cm -> + match cm with + | Op ((Operation.Tip | Operation.Tip404 | Operation.Tip304 | Operation.Query), RuMs m) -> + LogSink.Read.Ingest m + | QueryRes (_direction, _) -> () + | Op (Operation.Write, RuMs m) -> LogSink.Write.Ingest m + | Op (Operation.Conflict, RuMs m) -> LogSink.Conflict.Ingest m + | Op (Operation.Prune, RuMs m) -> LogSink.Prune.Ingest m + | PruneRes _ -> () + | Op (Operation.Delete, RuMs m) -> LogSink.Delete.Ingest m + | Op (Operation.Trim, RuMs m) -> LogSink.Trim.Ingest m + | _ -> () + + /// Relies on feeding of metrics from Log through to Stats.LogSink + /// Use Stats.LogSink.Restart() to reset the start point (and stats) where relevant + let dump (log : ILogger) = + let stats = + [ "Read", Stats.LogSink.Read + "Write", Stats.LogSink.Write + "Conflict", Stats.LogSink.Conflict + "Prune", Stats.LogSink.Prune + "Delete", Stats.LogSink.Delete + "Trim", Stats.LogSink.Trim ] + let mutable rows, totalCount, totalRRu, totalWRu, totalMs = 0, 0L, 0., 0., 0L + let logActivity name count ru lat = + let aru, ams = (if count = 0L then Double.NaN else ru/float count), (if count = 0L then Double.NaN else float lat/float count) + let rut = match name with "TOTAL" -> "" | "Read" | "Prune" -> totalRRu <- totalRRu + ru; "R" | _ -> totalWRu <- totalWRu + ru; "W" + log.Information("{name}: {count:n0} requests costing {ru:n0}{rut:l}RU (average: {avgRu:n1}); Average latency: {lat:n0}ms", + name, count, ru, rut, aru, ams) + for name, stat in stats do + if stat.count <> 0L then + let ru = float stat.rux100 / 100. + totalCount <- totalCount + stat.count + totalMs <- totalMs + stat.ms + logActivity name stat.count ru stat.ms + rows <- rows + 1 + // Yes, there's a minor race here between the use of the values and the reset + let duration = Stats.LogSink.Restart() + if rows > 1 then logActivity "TOTAL" totalCount (totalRRu + totalWRu) totalMs + let measures : (string * (TimeSpan -> float)) list = [ "s", fun x -> x.TotalSeconds(*; "m", fun x -> x.TotalMinutes; "h", fun x -> x.TotalHours*) ] + let logPeriodicRate name count rru wru = log.Information("rp{name} {count:n0} = ~{rru:n1}R/{wru:n1}W RU", name, count, rru, wru) + for uom, f in measures do let d = f duration in if d <> 0. then logPeriodicRate uom (float totalCount/d |> int64) (totalRRu/d) (totalWRu/d) + +module Initialization = + + open Amazon.DynamoDBv2 + let private prepare (client : IAmazonDynamoDB) tableName maybeThroughput : Async = + let context = TableContext(client, tableName) + match maybeThroughput with Some throughput -> context.VerifyOrCreateTableAsync(throughput) | None -> context.VerifyTableAsync() + + /// Verify the specified tableName is present and adheres to the correct schema. + let verify (client : IAmazonDynamoDB) tableName : Async = + let context = TableContext(client, tableName) + context.VerifyTableAsync() + + [] + type StreamingMode = Off | Default | NewAndOld + let toStreaming = function + | StreamingMode.Off -> Streaming.Disabled + | StreamingMode.Default -> Streaming.Enabled StreamViewType.NEW_IMAGE + | StreamingMode.NewAndOld -> Streaming.Enabled StreamViewType.NEW_AND_OLD_IMAGES + + /// Create the specified tableName if it does not exist. Will throw if it exists but the schema mismatches. + let createIfNotExists (client : IAmazonDynamoDB) tableName (throughput, streamingMode) : Async = + let context = TableContext(client, tableName) + context.VerifyOrCreateTableAsync(throughput, toStreaming streamingMode) + + /// Provision (or re-provision) the specified table with the specified Throughput. Will throw if schema mismatches. + let provision (client : IAmazonDynamoDB) tableName (throughput, streamingMode) = async { + let context = TableContext(client, tableName) + do! context.VerifyOrCreateTableAsync(throughput, toStreaming streamingMode) + do! context.UpdateTableIfRequiredAsync(throughput, toStreaming streamingMode) } + +type private Metrics() = + let mutable t = 0. + member _.Add(x : RequestMetrics) = + for x in x.ConsumedCapacity do + t <- t + x.CapacityUnits + member _.Consumed : RequestConsumption = { total = t } + +type internal BatchIndices = { isTip : bool; index : int64; n : int64 } +type Container(tableName, createContext : (RequestMetrics -> unit) -> TableContext) = + + member _.Context(collector) = createContext collector + member _.TableName = tableName + + /// As per Equinox.CosmosStore, we assume the table to be provisioned correctly (see DynamoStoreClient.Connect(ConnectMode) re validating on startup) + static member Create(client, tableName) = + let createContext collector = TableContext(client, tableName, metricsCollector = collector) + Container(tableName, createContext) + + member x.TryGetTip(stream : string) : Async = async { + let rm = Metrics() + let context = createContext rm.Add + let pk = Batch.tableKeyForStreamTip stream + let! item = context.TryGetItemAsync(pk) + return item |> Option.map Batch.ofSchema, rm.Consumed } + member x.TryUpdateTip(stream : string, updateExpr : Quotations.Expr Batch.Schema>, ?precondition) : Async = async { + let rm = Metrics() + let context = createContext rm.Add + let pk = Batch.tableKeyForStreamTip stream + let! item = context.UpdateItemAsync(pk, updateExpr, ?precondition = precondition) + return item |> Batch.ofSchema, rm.Consumed } + member _.QueryBatches(stream, minN, maxI, backwards, batchSize) : AsyncSeq = + let compile = (createContext ignore).Template.PrecomputeConditionalExpr + let kc = match maxI with + | Some maxI -> compile <@ fun (b : Batch.Schema) -> b.p = stream && b.i < maxI @> + | None -> compile <@ fun (b : Batch.Schema) -> b.p = stream @> + let fc = match minN with + | Some minN -> compile <@ fun (b : Batch.Schema) -> b.n > minN @> |> Some + | None -> None + let rec aux (i, le) = asyncSeq { + // TOCONSIDER could avoid projecting `p` + let rm = Metrics() + let context = createContext rm.Add + let! t, res = context.QueryPaginatedAsync(kc, ?filterCondition = fc, limit = batchSize, ?exclusiveStartKey = le, scanIndexForward = not backwards) + |> Stopwatch.Time + yield i, t, Array.map Batch.ofSchema res.Records, rm.Consumed + match res.LastEvaluatedKey with + | None -> () + | le -> yield! aux (i + 1, le) + } + aux (0, None) + member internal _.QueryIAndNOrderByNAscending(stream, maxItems) : AsyncSeq = + let rec aux (index, lastEvaluated) = asyncSeq { + let rm = Metrics() + let context = createContext rm.Add + let keyCond = <@ fun (b : Batch.Schema) -> b.p = stream @> + let proj = <@ fun (b : Batch.Schema) -> b.i, b.c, b.n @> // TOCONSIDER want len of c, but b.e.Length explodes in empty array case, so no choice but to return the full thing + let! t, res = context.QueryProjectedPaginatedAsync(keyCond, proj, ?exclusiveStartKey = lastEvaluated, scanIndexForward = true, limit = maxItems) + |> Stopwatch.Time + yield index, t, [| for i, c, n in res -> { isTip = Batch.isTip i; index = n - int64 c.Length; n = n } |], rm.Consumed + match res.LastEvaluatedKey with + | None -> () + | le -> yield! aux (index + 1, le) } + aux (0, None) + member x.DeleteItem(stream : string, i) : Async = async { + let rm = Metrics() + let context = createContext rm.Add + let pk = TableKey.Combined(stream, i) + let! _item = context.DeleteItemAsync(pk) + return rm.Consumed } + +/// Represents the State of the Stream for the purposes of deciding how to map a Sync request to DynamoDB operations +[] +type Position = + { index : int64; etag : string; baseBytes : int; unfoldsBytes : int; events : Event array } + override x.ToString() = sprintf "{ n=%d; etag=%s; e=%d; b=%d+%d }" x.index x.etag x.events.Length x.baseBytes x.unfoldsBytes +module internal Position = + + let fromTip (x : Batch) = { index = x.n; etag = x.etag; events = x.e; baseBytes = Batch.bytesBase x; unfoldsBytes = Batch.bytesUnfolds x } + let fromElements (p, n, e, u, etag) = fromTip { p = p; i = Unchecked.defaultof<_>; n = n; e = e; u = u; etag = etag } + let tryFromBatch (x : Batch) = if Batch.isTip x.i then fromTip x |> Some else None + let toIndex = function Some p -> p.index | None -> 0 + let toEtag = function Some p -> p.etag | None -> null + let null_ i = { index = i; etag = null; baseBytes = 0; unfoldsBytes = 0; events = Array.empty } + let flatten = function Some p -> p | None -> null_ 0 + +module internal Sync = + + [] + type internal Exp = + | Version of int64 + | Etag of string + + let private cce (ce : Quotations.Expr bool>) = template.PrecomputeConditionalExpr ce + let private cue (ue : Quotations.Expr Batch.Schema>) = template.PrecomputeUpdateExpr ue + let private batchDoesNotExistCondition = cce <@ fun t -> NOT_EXISTS t.i @> + let private putItemIfNotExists item = TransactWrite.Put (item, Some batchDoesNotExistCondition) + let private updateTip stream updater cond = TransactWrite.Update (Batch.tableKeyForStreamTip stream, Some (cce cond), cue updater) + + let private generateRequests (stream : string) (tipEmpty, exp, n', calve : Event array, append : Event array, u, eventCount) etag' + : TransactWrite list = + let u = Batch.unfoldsToSchema u + let tipEventCount = append.LongLength + let tipIndex = n' - tipEventCount + let appA = min eventCount append.Length + let insertCalf () = + let calfEventCount = calve.LongLength + let calfC, calfE = Batch.eventsToSchema calve + let calveA = eventCount - appA + putItemIfNotExists { p = stream; i = tipIndex - calfEventCount; n = tipIndex; a = calveA; c = calfC; e = calfE; etag = None; u = [||] } + let appC, appE = Batch.eventsToSchema append + let insertFreshTip () = + putItemIfNotExists { p = stream; i = Batch.tipMagicI; n = n'; a = appA; c = appC; e = appE; etag = Some etag'; u = u } + let updateTipIf condExpr = + let updExpr : Quotations.Expr Batch.Schema> = + // TOCONSIDER figure out whether there is a way to stop DDB choking on the Array.append below when its empty instead of this special casing + if calve.Length <> 0 || (tipEmpty && tipEventCount <> 0) then + <@ fun t -> { t with a = appA; c = appC; e = appE; n = n' + etag = Some etag'; u = u } @> + elif tipEventCount <> 0 then + <@ fun t -> { t with a = appA; c = Array.append t.c appC; e = Array.append t.e appE; n = t.n + tipEventCount + etag = Some etag'; u = u } @> + else <@ fun t -> { t with etag = Some etag'; u = u } @> + updateTip stream updExpr condExpr + [ if calve.Length > 0 then + insertCalf () + match exp with + | Exp.Version 0L | Exp.Etag null -> insertFreshTip () + | Exp.Etag etag -> updateTipIf <@ fun t -> t.etag = Some etag @> + | Exp.Version ver -> updateTipIf <@ fun t -> t.n = ver @> ] + + [] + type private TransactResult = + | Written of etag' : string + | ConflictUnknown + + let private (|DynamoDbConflict|_|) : exn -> _ = function + | Precondition.CheckFailed + | TransactWriteItemsRequest.TransactionCanceledConditionalCheckFailed -> Some () + | _ -> None + let private transact (container : Container, stream : string) (tipEmpty, exp, n', calve, append, unfolds, eventCount) + : Async = async { + let etag' = let g = Guid.NewGuid() in g.ToString "N" + let actions = generateRequests stream (tipEmpty, exp, n', calve, append, unfolds, eventCount) etag' + let rm = Metrics() + try do! let context = container.Context(rm.Add) + match actions with + | [ TransactWrite.Put (item, Some cond) ] -> context.PutItemAsync(item, cond) |> Async.Ignore + | [ TransactWrite.Update (key, Some cond, updateExpr) ] -> context.UpdateItemAsync(key, updateExpr, cond) |> Async.Ignore + | actions -> context.TransactWriteItems actions + return rm.Consumed, TransactResult.Written etag' + with DynamoDbConflict -> + return rm.Consumed, TransactResult.ConflictUnknown } + + let private transactLogged (container, stream) (tipBaseSize, tipEvents, tipEmpty, exp : Exp, n', calve, append, unfolds, eventCount) (log : ILogger) + : Async = async { + let! t, ({ total = ru } as rc, result) = transact (container, stream) (tipEmpty, exp, n', calve, append, unfolds, eventCount) |> Stopwatch.Time + let calveBytes, tipBytes = Event.arrayBytes calve, Event.arrayBytes append + Unfold.arrayBytes unfolds + let unfoldsCount = Array.length unfolds + let log = + let reqMetric = Log.metric container.TableName stream t (calveBytes + tipBytes) (eventCount + unfoldsCount) rc + log + |> match exp with + | Exp.Etag et -> Log.prop "expectedEtag" et + | Exp.Version ev -> Log.prop "expectedVersion" ev + |> match result with + | TransactResult.Written etag' -> + Log.prop "nextPos" n' >> Log.prop "nextEtag" etag' >> Log.event (Log.Metric.SyncSuccess reqMetric) + | TransactResult.ConflictUnknown -> + Log.prop "conflict" true + >> Log.prop "eventTypes" (Seq.truncate 5 (seq { for x in append -> x.c })) + >> Log.event (Log.Metric.SyncConflict reqMetric) + log.Information("EqxDynamo {action:l} {stream:l} {events} {ms:f1}ms {ru}RU Tip {tipBase}+{tipBytes}b {tipEvents}e {tipUnfolds}u Calf {calveBytes}b {calveEvents}e {exp:l}", + "Sync", stream, eventCount, Log.tms t, ru, tipBaseSize, tipBytes, tipEvents, unfoldsCount, calveBytes, calve.Length, exp) + return result } + + [] + type Result = + | Written of etag : string * events : Event array * unfolds : Unfold array + | ConflictUnknown + + let private maxDynamoDbItemSize = 400 * 1024 + let handle log (maxEvents, maxBytes, eo : EncodingOptions) (container, stream) + (pos, exp, n', events : IEventData array, unfolds : IEventData array) = async { + let baseIndex = int n' - events.Length + let events : Event array = events |> Array.mapi (fun i e -> + { i = baseIndex + i; t = e.Timestamp + c = e.EventType; d = eo.EncodeEventData e.Data; m = eo.EncodeEventMeta e.Meta + correlationId = Option.ofObj e.CorrelationId; causationId = Option.ofObj e.CausationId }) + let unfolds : Unfold array = unfolds |> Array.map (fun (x : IEventData<_>) -> + { i = n'; t = x.Timestamp + c = x.EventType; d = eo.EncodeUnfoldData x.Data; m = eo.EncodeUnfoldMeta x.Meta }) + if Array.isEmpty events && Array.isEmpty unfolds then invalidOp "Must write either events or unfolds." + let cur = Position.flatten pos + let calfEvents, tipOrAppendEvents, tipEvents' = + let eventOverflow = maxEvents |> Option.exists (fun limit -> events.Length + cur.events.Length > limit) + if eventOverflow || cur.baseBytes + Unfold.arrayBytes unfolds + Event.arrayBytes events > maxBytes then + let calfEvents, residualEvents = ResizeArray(cur.events.Length + events.Length), ResizeArray() + let mutable calfFull, calfSize = false, 1024 + for e in Seq.append cur.events events do + match calfFull, calfSize + Event.bytes e with + | false, calfSize' when calfSize' < maxDynamoDbItemSize -> calfSize <- calfSize'; calfEvents.Add e + | _ -> calfFull <- true; residualEvents.Add e + let tipEvents = residualEvents.ToArray() + calfEvents.ToArray(), tipEvents, tipEvents + else Array.empty, events, Array.append cur.events events + let tipEmpty, eventCount = Array.isEmpty cur.events, Array.length events + match! transactLogged (container, stream) (cur.baseBytes, tipEvents'.Length, tipEmpty, exp pos, n', calfEvents, tipOrAppendEvents, unfolds, eventCount) log with + | TransactResult.ConflictUnknown -> return Result.ConflictUnknown + | TransactResult.Written etag' -> return Result.Written (etag', tipEvents', unfolds) } + +module internal Tip = + + [] + type Res<'T> = + | Found of 'T + | NotFound + | NotModified + let private get (container : Container, stream : string) (maybePos : Position option) = async { + match! container.TryGetTip(stream) with + | Some { etag = fe }, rc when fe = Position.toEtag maybePos -> return rc, Res.NotModified + | Some t, rc -> return rc, Res.Found t + | None, rc -> return rc, Res.NotFound } + let private loggedGet (get : Container * string -> Position option -> Async<_>) (container, stream) (maybePos : Position option) (log : ILogger) = async { + let log = log |> Log.prop "stream" stream + let! t, ({ total = ru } as rc, res : Res<_>) = get (container, stream) maybePos |> Stopwatch.Time + let logMetric bytes count (f : Log.Measurement -> _) = log |> Log.event (f (Log.metric container.TableName stream t bytes count rc)) + match res with + | Res.NotModified -> + (logMetric 0 0 Log.Metric.TipNotModified).Information("EqxDynamo {action:l} {stream:l} {res} {ms:f1}ms {ru}RU", "Tip", stream, 304, Log.tms t, ru) + | Res.NotFound -> + (logMetric 0 0 Log.Metric.TipNotFound).Information("EqxDynamo {action:l} {stream:l} {res} {ms:f1}ms {ru}RU", "Tip", stream, 404, Log.tms t, ru) + | Res.Found tip -> + let eventsCount, unfoldsCount, bb, ub = tip.e.Length, tip.u.Length, Batch.bytesBase tip, Batch.bytesUnfolds tip + let log = logMetric (bb + ub) (eventsCount + unfoldsCount) Log.Metric.Tip + let log = match maybePos with Some p -> log |> Log.prop "startPos" p |> Log.prop "startEtag" p | None -> log + let log = log |> Log.prop "etag" tip.etag //|> Log.prop "n" tip.n + log.Information("EqxDynamo {action:l} {stream:l} {res} {ms:f1}ms {ru}RU v{n} {events}+{unfolds}e {baseBytes}+{unfoldsBytes}b", + "Tip", stream, 200, Log.tms t, ru, tip.n, eventsCount, unfoldsCount, bb, ub) + return ru, res } + let private enumEventsAndUnfolds (minIndex, maxIndex) (x : Batch) : ITimelineEvent array = + Seq.append> (Batch.enumEvents (minIndex, maxIndex) x |> Seq.cast) (x.u |> Seq.cast) + // where Index is equal, unfolds get delivered after the events so the fold semantics can be 'idempotent' + |> Seq.sortBy (fun x -> x.Index, x.IsUnfold) + |> Array.ofSeq + /// `pos` being Some implies that the caller holds a cached value and hence is ready to deal with Result.NotModified + let tryLoad (log : ILogger) containerStream (maybePos : Position option, maxIndex) : Async array>> = async { + let! _rc, res = loggedGet get containerStream maybePos log + match res with + | Res.NotModified -> return Res.NotModified + | Res.NotFound -> return Res.NotFound + | Res.Found tip -> + let minIndex = maybePos |> Option.map (fun x -> x.index) + return Res.Found (Position.fromTip tip, Batch.baseIndex tip, tip |> enumEventsAndUnfolds (minIndex, maxIndex)) } + +module internal Query = + + let private mkQuery (log : ILogger) (container : Container, stream : string) maxItems (direction : Direction, minIndex, maxIndex) = + let minN, maxI = minIndex, maxIndex + log.Debug("EqxDynamo Query {stream}; minIndex={minIndex} maxIndex={maxIndex}", stream, minIndex, maxIndex) + container.QueryBatches(stream, minN, maxI, (direction = Direction.Backward), maxItems) + + // Unrolls the Batches in a response + // NOTE when reading backwards, the events are emitted in reverse Index order to suit the takeWhile consumption + let private mapPage direction (container : Container, stream : string) (minIndex, maxIndex) (maxRequests : int option) + (log : ILogger) (i, t, batches : Batch array, rc) + : Event array * Position option * RequestConsumption = + let log = log |> Log.prop "batchIndex" i + match maxRequests with + | Some mr when i >= mr -> log.Information "batch Limit exceeded"; invalidOp "batch Limit exceeded" + | _ -> () + let unwrapBatch (x : Batch) = + Batch.enumEvents (minIndex, maxIndex) x + |> if direction = Direction.Backward then Seq.rev else id + let events = batches |> Seq.collect unwrapBatch |> Array.ofSeq + let count, bytes = events.Length, Batch.bytesTotal batches + let index = if count = 0 then Nullable () else Nullable (Seq.map Batch.baseIndex batches |> Seq.min) + (log|> Log.event (Log.Metric.QueryResponse (direction, Log.metric container.TableName stream t bytes count rc)) + |> Log.prop "bytes" bytes + |> match minIndex with None -> id | Some i -> Log.prop "minIndex" i + |> match maxIndex with None -> id | Some i -> Log.prop "maxIndex" i) + .Information("EqxDynamo {action:l} {count}/{batches} {direction} {ms:f1}ms i={index} {ru}RU", + "Response", count, batches.Length, direction, Log.tms t, index, rc.total) + let maybePosition = batches |> Array.tryPick Position.tryFromBatch + events, maybePosition, rc + + let private logQuery direction (container : Container, stream) interval (responsesCount, events : Event array) n (rc : RequestConsumption) (log : ILogger) = + let count, bytes = events.Length, Event.arrayBytes events + let reqMetric = Log.metric container.TableName stream interval bytes count rc + let evt = Log.Metric.Query (direction, responsesCount, reqMetric) + let action = match direction with Direction.Forward -> "QueryF" | Direction.Backward -> "QueryB" + (log |> Log.prop "bytes" bytes |> Log.event evt).Information( + "EqxDynamo {action:l} {stream} v{n} {count}/{responses} {ms:f1}ms {ru}RU", + action, stream, n, count, responsesCount, Log.tms interval, rc.total) + + let private calculateUsedVersusDroppedPayload stopIndex (xs : Event array) : int * int = + let mutable used, dropped = 0, 0 + let mutable found = false + for x in xs do + let bytes = Event.bytes x + if found then dropped <- dropped + bytes + else used <- used + bytes + if x.i = stopIndex then found <- true + used, dropped + + [] + type ScanResult<'event> = { found : bool; minIndex : int64; next : int64; maybeTipPos : Position option; events : 'event array } + + let scanTip (tryDecode : ITimelineEvent -> 'event option, isOrigin : 'event -> bool) (pos : Position, i : int64, xs : ITimelineEvent array) + : ScanResult<'event> = + let items = ResizeArray() + let isOrigin' e = + match tryDecode e with + | None -> false + | Some e -> + items.Insert(0, e) // WalkResult always renders events ordered correctly - here we're aiming to align with Enum.EventsAndUnfolds + isOrigin e + let f, e = xs |> Seq.map EventBody.decodeEvent |> Seq.tryFindBack isOrigin' |> Option.isSome, items.ToArray() + { found = f; maybeTipPos = Some pos; minIndex = i; next = pos.index + 1L; events = e } + + // Yields events in ascending Index order + let scan<'event> (log : ILogger) (container, stream) maxItems maxRequests direction + (tryDecode : ITimelineEvent -> 'event option, isOrigin : 'event -> bool) + (minIndex, maxIndex) + : Async option> = async { + let mutable found = false + let mutable responseCount = 0 + let mergeBatches (log : ILogger) (batchesBackward : AsyncSeq) = async { + let mutable lastResponse, maybeTipPos, ru = None, None, 0. + let! events = + batchesBackward + |> AsyncSeq.map (fun (events, maybePos, rc) -> + if Option.isNone maybeTipPos then maybeTipPos <- maybePos + lastResponse <- Some events; ru <- ru + rc.total + responseCount <- responseCount + 1 + seq { for x in events -> x, x |> EventBody.decodeEvent |> tryDecode }) + |> AsyncSeq.concatSeq + |> AsyncSeq.takeWhileInclusive (function + | x,Some e when isOrigin e -> + found <- true + match lastResponse with + | None -> log.Information("EqxDynamo Stop stream={stream} at={index} {case}", stream, x.i, x.c) + | Some batch -> + let used, residual = batch |> calculateUsedVersusDroppedPayload x.i + log.Information("EqxDynamo Stop stream={stream} at={index} {case} used={used} residual={residual}", + stream, x.i, x.c, used, residual) + false + | _ -> true) + |> AsyncSeq.toArrayAsync + return events, maybeTipPos, { total = ru } } + let log = log |> Log.prop "batchSize" maxItems |> Log.prop "stream" stream + let readLog = log |> Log.prop "direction" direction + let batches : AsyncSeq = + mkQuery readLog (container, stream) maxItems (direction, minIndex, maxIndex) + |> AsyncSeq.map (mapPage direction (container, stream) (minIndex, maxIndex) maxRequests readLog) + let! t, (events, maybeTipPos, ru) = mergeBatches log batches |> Stopwatch.Time + let raws = Array.map fst events + let decoded = if direction = Direction.Forward then Array.choose snd events else Seq.choose snd events |> Seq.rev |> Array.ofSeq + let minMax = (None, raws) ||> Array.fold (fun acc x -> let i = int64 x.i in Some (match acc with None -> i, i | Some (n, x) -> min n i, max x i)) + let version = + match maybeTipPos, minMax with + | Some { index = max }, _ + | _, Some (_, max) -> max + 1L + | None, None -> 0L + log |> logQuery direction (container, stream) t (responseCount, raws) version ru + match minMax, maybeTipPos with + | Some (i, m), _ -> return Some { found = found; minIndex = i; next = m + 1L; maybeTipPos = maybeTipPos; events = decoded } + | None, Some { index = tipI } -> return Some { found = found; minIndex = tipI; next = tipI; maybeTipPos = maybeTipPos; events = [||] } + | None, _ -> return None } + + let walkLazy<'event> (log : ILogger) (container, stream) maxItems maxRequests + (tryDecode : ITimelineEvent -> 'event option, isOrigin : 'event -> bool) + (direction, minIndex, maxIndex) + : AsyncSeq<'event array> = asyncSeq { + let query = mkQuery log (container, stream) maxItems (direction, minIndex, maxIndex) + + let readPage = mapPage direction (container, stream) (minIndex, maxIndex) maxRequests + let log = log |> Log.prop "batchSize" maxItems |> Log.prop "stream" stream + let readLog = log |> Log.prop "direction" direction + let query = query |> AsyncSeq.map (readPage readLog) + let startTicks = System.Diagnostics.Stopwatch.GetTimestamp() + let allEvents = ResizeArray() + let mutable i, ru = 0, 0. + try let mutable ok = true + let e = query.GetEnumerator() + while ok do + let batchLog = readLog |> Log.prop "batchIndex" i + match maxRequests with + | Some mr when i + 1 >= mr -> batchLog.Information "batch Limit exceeded"; invalidOp "batch Limit exceeded" + | _ -> () + + match! e.MoveNext() with + | None -> ok <- false // rest of block does not happen, while exits + | Some (events, _pos, rc) -> + + ru <- ru + rc.total + allEvents.AddRange(events) + + let acc = ResizeArray() + for x in events do + match x |> EventBody.decodeEvent |> tryDecode with + | Some e when isOrigin e -> + let used, residual = events |> calculateUsedVersusDroppedPayload x.i + log.Information("EqxDynamo Stop stream={stream} at={index} {case} used={used} residual={residual}", + stream, x.i, x.c, used, residual) + ok <- false + acc.Add e + | Some e -> acc.Add e + | None -> () + i <- i + 1 + yield acc.ToArray() + finally + let endTicks = System.Diagnostics.Stopwatch.GetTimestamp() + let t = StopwatchInterval(startTicks, endTicks) + log |> logQuery direction (container, stream) t (i, allEvents.ToArray()) -1L { total = ru } } + type [] LoadRes = Pos of Position | Empty | Next of int64 + let toPosition = function Pos p -> Some p | Empty -> None | Next _ -> failwith "unexpected" + /// Manages coalescing of spans of events obtained from various sources: + /// 1) Tip Data and/or Conflicting events + /// 2) Querying Primary for predecessors of what's obtained from 1 + /// 3) Querying Archive for predecessors of what's obtained from 2 + let load (log : ILogger) (minIndex, maxIndex) (tip : ScanResult<'event> option) + (primary : int64 option * int64 option -> Async option>) + // Choice1Of2 -> indicates whether it's acceptable to ignore missing events; Choice2Of2 -> Fallback store + (fallback : Choice Async option>>) + : Async = async { + let minI = defaultArg minIndex 0L + match tip with + | Some { found = true; maybeTipPos = Some p; events = e } -> return Some p, e + | Some { minIndex = i; maybeTipPos = Some p; events = e } when i <= minI -> return Some p, e + | _ -> + + let i, events, pos = + match tip with + | Some { minIndex = i; maybeTipPos = p; events = e } -> Some i, e, p + | None -> maxIndex, Array.empty, None + let! primary = primary (minIndex, i) + let events, pos = + match primary with + | None -> events, match pos with Some p -> Pos p | None -> Empty + | Some primary -> Array.append primary.events events, match pos |> Option.orElse primary.maybeTipPos with Some p -> Pos p | None -> Next primary.next + let inline logMissing (minIndex, maxIndex) message = + if log.IsEnabled Events.LogEventLevel.Debug then + (log|> fun log -> match minIndex with None -> log | Some mi -> log |> Log.prop "minIndex" mi + |> fun log -> match maxIndex with None -> log | Some mi -> log |> Log.prop "maxIndex" mi) + .Debug(message) + + match primary, fallback with + | Some { found = true }, _ -> return toPosition pos, events // origin found in primary, no need to look in fallback + | Some { minIndex = i }, _ when i <= minI -> return toPosition pos, events // primary had required earliest event Index, no need to look at fallback + | None, _ when Option.isNone tip -> return toPosition pos, events // initial load where no documents present in stream + | _, Choice1Of2 allowMissing -> + logMissing (minIndex, i) "Origin event not found; no Archive Table supplied" + if allowMissing then return toPosition pos, events + else return failwithf "Origin event not found; no Archive Table supplied" + | _, Choice2Of2 fallback -> + + let maxIndex = match primary with Some p -> Some p.minIndex | None -> maxIndex // if no batches in primary, high water mark from tip is max + let! fallback = fallback (minIndex, maxIndex) + let events = + match fallback with + | Some s -> Array.append s.events events + | None -> events + match fallback with + | Some { minIndex = i } when i <= minI -> () + | Some { found = true } -> () + | _ -> logMissing (minIndex, maxIndex) "Origin event not found in Archive Table" + return toPosition pos, events } + +// Manages deletion of (full) Batches, and trimming of events in Tip, maintaining ordering guarantees by never updating non-Tip batches +// Additionally, the nature of the fallback algorithm requires that deletions be carried out in sequential order so as not to leave gaps +// NOTE: module is public so BatchIndices can be deserialized into +module Prune = + + let until (log : ILogger) (container : Container, stream : string) maxItems indexInclusive : Async = async { + let log = log |> Log.prop "stream" stream + let deleteItem i count : Async = async { + let! t, rc = container.DeleteItem(stream, i) |> Stopwatch.Time + let reqMetric = Log.metric container.TableName stream t -1 count rc + let log = let evt = Log.Metric.Delete reqMetric in log |> Log.event evt + log.Information("EqxDynamo {action:l} {i} {ms:f1}ms {ru}RU", "Delete", i, Log.tms t, rc) + return rc + } + let trimTip expectedN count = async { + match! container.TryGetTip(stream) with + | None, _rc -> return failwith "unexpected NotFound" + | Some tip, _rc when tip.n <> expectedN -> return failwithf "Concurrent write detected; Expected n=%d actual=%d" expectedN tip.n + | Some tip, tipRc -> + + let tC, tE = Batch.eventsToSchema tip.e + let tC', tE' = Array.skip count tC, Array.skip count tE + let updEtag = let g = Guid.NewGuid() in g.ToString "N" + let condExpr : Quotations.Expr bool> = <@ fun t -> t.etag = Some tip.etag @> + let updateExpr : Quotations.Expr _> = <@ fun t -> { t with etag = Some updEtag; c = tC'; e = tE' } @> + let! t, (_updated, updRc) = container.TryUpdateTip(stream, updateExpr, condExpr) |> Stopwatch.Time + let rc = { total = tipRc.total + updRc.total } + let reqMetric = Log.metric container.TableName stream t -1 count rc + let log = let evt = Log.Metric.Trim reqMetric in log |> Log.event evt + log.Information("EqxDynamo {action:l} {count} {ms:f1}ms {ru}RU", "Trim", count, Log.tms t, rc) + return rc + } + let log = log |> Log.prop "index" indexInclusive + // need to sort by n to guarantee we don't ever leave an observable gap in the sequence + let query = container.QueryIAndNOrderByNAscending(stream, maxItems) + let mapPage (i, t : StopwatchInterval, batches : BatchIndices array, rc) = + let next = Array.tryLast batches |> Option.map (fun x -> x.n) |> Option.toNullable + let reqMetric = Log.metric container.TableName stream t -1 batches.Length rc + let log = let evt = Log.Metric.PruneResponse reqMetric in log |> Log.prop "batchIndex" i |> Log.event evt + log.Information("EqxDynamo {action:l} {batches} {ms:f1}ms n={next} {ru}RU", "PruneResponse", batches.Length, Log.tms t, next, rc) + batches, rc + let! pt, outcomes = + let isRelevant (x : BatchIndices) = x.index <= indexInclusive || x.isTip + let handle (batches : BatchIndices array, rc) = async { + let mutable delCharges, batchesDeleted, trimCharges, batchesTrimmed, eventsDeleted, eventsDeferred = 0., 0, 0., 0, 0, 0 + let mutable lwm = None + for x in batches |> Seq.takeWhile (fun x -> isRelevant x || lwm = None) do + let batchSize = x.n - x.index |> int + let eligibleEvents = max 0 (min batchSize (int (indexInclusive + 1L - x.index))) + if x.isTip then // Even if we remove the last event from the Tip, we need to retain a) unfolds b) position (n) + if eligibleEvents <> 0 then + let! charge = trimTip x.n eligibleEvents + trimCharges <- trimCharges + charge.total + batchesTrimmed <- batchesTrimmed + 1 + eventsDeleted <- eventsDeleted + eligibleEvents + if lwm = None then + lwm <- Some (x.index + int64 eligibleEvents) + elif x.n <= indexInclusive + 1L then + let! charge = deleteItem x.index batchSize + delCharges <- delCharges + charge.total + batchesDeleted <- batchesDeleted + 1 + eventsDeleted <- eventsDeleted + batchSize + else // can't update a non-Tip batch, or it'll be ordered wrong from a CFP perspective + eventsDeferred <- eventsDeferred + eligibleEvents + if lwm = None then + lwm <- Some x.index + return (rc, delCharges, trimCharges), lwm, (batchesDeleted + batchesTrimmed, eventsDeleted, eventsDeferred) + } + let hasRelevantItems (batches, _rc) = batches |> Array.exists isRelevant + query + |> AsyncSeq.map mapPage + |> AsyncSeq.takeWhile hasRelevantItems + |> AsyncSeq.mapAsync handle + |> AsyncSeq.toArrayAsync + |> Stopwatch.Time + let mutable queryCharges, delCharges, trimCharges, responses, batches, eventsDeleted, eventsDeferred = 0., 0., 0., 0, 0, 0, 0 + let mutable lwm = None + for (qc, dc, tc), bLwm, (bCount, eDel, eDef) in outcomes do + lwm <- max lwm bLwm + queryCharges <- queryCharges + qc.total + delCharges <- delCharges + dc + trimCharges <- trimCharges + tc + responses <- responses + 1 + batches <- batches + bCount + eventsDeleted <- eventsDeleted + eDel + eventsDeferred <- eventsDeferred + eDef + let reqMetric = Log.metric container.TableName stream pt eventsDeleted batches { total = queryCharges } + let log = let evt = Log.Metric.Prune (responses, reqMetric) in log |> Log.event evt + let lwm = lwm |> Option.defaultValue 0L // If we've seen no batches at all, then the write position is 0L + log.Information("EqxDynamo {action:l} {events}/{batches} lwm={lwm} {ms:f1}ms queryRu={queryRu} deleteRu={deleteRu} trimRu={trimRu}", + "Prune", eventsDeleted, batches, lwm, Log.tms pt, queryCharges, delCharges, trimCharges) + return eventsDeleted, eventsDeferred, lwm + } + +type [] Token = { pos : Position option } +module Token = + + let create_ pos : StreamToken = { value = box { pos = pos }; version = Position.toIndex pos } + let create : Position -> StreamToken = Some >> create_ + let empty = create_ None + let (|Unpack|) (token : StreamToken) : Position option = let t = unbox token.value in t.pos + let supersedes (Unpack currentPos) (Unpack xPos) = + match currentPos, xPos with + | Some currentPos, Some xPos -> + let currentVersion, newVersion = currentPos.index, xPos.index + let currentETag, newETag = currentPos.etag, xPos.etag + newVersion > currentVersion || currentETag <> newETag + | None, Some _ -> true + | Some _, None + | None, None -> false + +[] +module Internal = + + [] + type InternalSyncResult = Written of StreamToken | ConflictUnknown + + [] + type LoadFromTokenResult<'event> = Unchanged | Found of StreamToken * 'event array + + // Item writes are charged in 1K blocks; reads in 4K blocks. + // Selecting an appropriate limit is a trade-off between + // - Read costs, Table Item counts and database usage (fewer, larger Items will imply lower querying and storage costs) + // - Tip Write costs - appending an event also incurs a cost to rewrite the existing content of the Tip + // - Uncached roundtrip latency - using the normal access strategies, the Tip is loaded as a point read before querying is triggered. + // Ideally that gets all the data - smaller tip sizes reduce the likelihood of that + // - Calving costs - Calving a batch from the Tip every time is cost-prohibitive for at least the following reasons + // - TransactWriteItems is more than twice the cost in Write RU vs a normal UpdateItem, with associated latency impacts + // - various other considerations, e.g. we need to re-read the Tip next time around see https://stackoverflow.com/a/71706015/11635 + let defaultTipMaxBytes = 32 * 1024 + // In general we want to minimize round-trips, but we'll get better diagnostic feedback under load if we constrain our + // queries to shorter pages. The effect of this is of course highly dependent on the max Item size, which is + // dictated by the TipOptions - in the default configuration that's controlled by defaultTipMaxBytes + let defaultMaxItems = 32 + // TL;DR in general it's worth compressing everything to minimize RU consumption both on insert and update + // Every time we need to calve from the tip, the RU impact of using TransactWriteItems is significant, + // so preventing or delaying that is of critical significance + // Empirically not much JSON below 48 bytes actually compresses - while we don't assume that, it is what is guiding the derivation of the default + let defaultEncodingOptions : EncodingOptions = { eventData = 48; eventMeta = 48; unfoldData = 48; unfoldMeta = 48; minGain = 4 } + +/// Defines the policies in force regarding how to split up calls when loading Event Batches via queries +type QueryOptions + ( // Max number of Batches to return per paged query response. Default: 32. + [] ?maxItems : int, + // Dynamic version of `maxItems`, allowing one to react to dynamic configuration changes. Default: use `maxItems` value. + [] ?getMaxItems : unit -> int, + // Maximum number of trips to permit when slicing the work into multiple responses based on `MaxItems`. Default: unlimited. + [] ?maxRequests, + // Inhibit throwing when events are missing, but no fallback Table has been supplied. Default: false. + [] ?ignoreMissingEvents) = + let getMaxItems = defaultArg getMaxItems (fun () -> defaultArg maxItems defaultMaxItems) + /// Limit for Maximum number of `Batch` records in a single query batch response + member _.MaxItems = getMaxItems () + /// Maximum number of trips to permit when slicing the work into multiple responses based on `MaxItems` + member _.MaxRequests = maxRequests + /// Whether to inhibit throwing when events are missing, but no Archive Table has been supplied as a fallback + member val IgnoreMissingEvents = defaultArg ignoreMissingEvents false + +/// Defines the policies in force regarding accumulation/retention of Events in Tip +type TipOptions + ( // Maximum serialized size to permit to accumulate in Tip before events get moved out to a standalone Batch. Default: 32K. + [] ?maxBytes, + // Optional maximum number of events permitted in Tip. When this is exceeded, events are moved out to a standalone Batch. Default: limited by MaxBytes + [] ?maxEvents) = + /// Maximum number of events permitted in Tip. When this is exceeded, events are moved out to a standalone Batch. Default: limited by MaxBytes + member val MaxEvents : int option = maxEvents + /// Maximum serialized size to permit to accumulate in Tip before events get moved out to a standalone Batch. Default: 32K. + member val MaxBytes = defaultArg maxBytes defaultTipMaxBytes + +type internal StoreClient(container : Container, fallback : Container option, query : QueryOptions, tip : TipOptions, enc : EncodingOptions) = + + let loadTip log stream pos = Tip.tryLoad log (container, stream) (pos, None) + + // Always yields events forward, regardless of direction + member _.Read(log, stream, direction, (tryDecode, isOrigin), ?minIndex, ?maxIndex, ?tip) : Async = async { + let tip = tip |> Option.map (Query.scanTip (tryDecode, isOrigin)) + let maxIndex = match maxIndex with + | Some _ as mi -> mi + | None when Option.isSome tip -> Some Batch.tipMagicI + | None -> None + let walk log container = Query.scan log (container, stream) query.MaxItems query.MaxRequests direction (tryDecode, isOrigin) + let walkFallback = + match fallback with + | None -> Choice1Of2 query.IgnoreMissingEvents + | Some f -> Choice2Of2 (walk (log |> Log.prop "fallback" true) f) + + let log = log |> Log.prop "stream" stream + let! pos, events = Query.load log (minIndex, maxIndex) tip (walk log container) walkFallback + return Token.create_ pos, events } + member _.ReadLazy(log, batching : QueryOptions, stream, direction, (tryDecode, isOrigin), ?minIndex, ?maxIndex) : AsyncSeq<'event array> = + Query.walkLazy log (container, stream) batching.MaxItems batching.MaxRequests (tryDecode, isOrigin) (direction, minIndex, maxIndex) + + member store.Load(log, (stream, maybePos), (tryDecode, isOrigin), checkUnfolds : bool) : Async = + if not checkUnfolds then store.Read(log, stream, Direction.Backward, (tryDecode, isOrigin)) + else async { + match! loadTip log stream maybePos with + | Tip.Res.NotFound -> return Token.empty, Array.empty + | Tip.Res.NotModified -> return invalidOp "Not applicable" + | Tip.Res.Found (pos, i, xs) -> return! store.Read(log, stream, Direction.Backward, (tryDecode, isOrigin), tip = (pos, i, xs)) } + member _.GetPosition(log, stream, ?pos) : Async = async { + match! loadTip log stream pos with + | Tip.Res.NotFound -> return Token.empty + | Tip.Res.NotModified -> return Token.create pos.Value + | Tip.Res.Found (pos, _i, _unfoldsAndEvents) -> return Token.create pos } + member store.Reload(log, (stream, maybePos : Position option), (tryDecode, isOrigin), ?preview): Async> = + let read tipContent = async { + let! res = store.Read(log, stream, Direction.Backward, (tryDecode, isOrigin), minIndex = Position.toIndex maybePos, tip = tipContent) + return LoadFromTokenResult.Found res } + match preview with + | Some (pos, i, xs) -> read (pos, i, xs) + | None -> async { + match! loadTip log stream maybePos with + | Tip.Res.NotFound -> return LoadFromTokenResult.Found (Token.empty, Array.empty) + | Tip.Res.NotModified -> return LoadFromTokenResult.Unchanged + | Tip.Res.Found (pos, i, xs) -> return! read (pos, i, xs) } + + member _.Sync(log, stream, pos, exp, n' : int64, eventsEncoded, unfoldsEncoded) : Async = async { + match! Sync.handle log (tip.MaxEvents, tip.MaxBytes, enc) (container, stream) (pos, exp, n', eventsEncoded, unfoldsEncoded) with + | Sync.Result.ConflictUnknown -> return InternalSyncResult.ConflictUnknown + | Sync.Result.Written (etag', events, unfolds) -> + return InternalSyncResult.Written (Token.create (Position.fromElements (stream, n', events, unfolds, etag'))) } + + member _.Prune(log, stream, index) = + Prune.until log (container, stream) query.MaxItems index + +type internal Category<'event, 'state, 'context>(store : StoreClient, codec : IEventCodec<'event, EventBody, 'context>) = + member _.Load(log, stream, initial, checkUnfolds, fold, isOrigin) : Async = async { + let! token, events = store.Load(log, (stream, None), (codec.TryDecode, isOrigin), checkUnfolds) + return token, fold initial events } + member _.Reload(log, stream, (Token.Unpack pos as streamToken), state, fold, isOrigin, ?preloaded) : Async = async { + match! store.Reload(log, (stream, pos), (codec.TryDecode, isOrigin), ?preview = preloaded) with + | LoadFromTokenResult.Unchanged -> return streamToken, state + | LoadFromTokenResult.Found (token', events) -> return token', fold state events } + member cat.Sync(log, stream, (Token.Unpack pos as streamToken), state, events, mapUnfolds, fold, isOrigin, context): Async> = async { + let state' = fold state (Seq.ofList events) + let exp, events, eventsEncoded, unfoldsEncoded = + let encode e = codec.Encode(context, e) + let expVer = Position.toIndex >> Sync.Exp.Version + match mapUnfolds with + | Choice1Of3 () -> expVer, events, Seq.map encode events |> Array.ofSeq, Seq.empty + | Choice2Of3 unfold -> expVer, events, Seq.map encode events |> Array.ofSeq, Seq.map encode (unfold events state') + | Choice3Of3 transmute -> + let events', unfolds = transmute events state' + Position.toEtag >> Sync.Exp.Etag, events', Seq.map encode events' |> Array.ofSeq, Seq.map encode unfolds + let baseVer = Position.toIndex pos + int64 (List.length events) + match! store.Sync(log, stream, pos, exp, baseVer, eventsEncoded, Seq.toArray unfoldsEncoded) with + | InternalSyncResult.ConflictUnknown -> return SyncResult.Conflict (cat.Reload(log, stream, streamToken, state, fold, isOrigin)) + | InternalSyncResult.Written token' -> return SyncResult.Written (token', state') } + +module internal Caching = + + let applyCacheUpdatesWithSlidingExpiration (cache : ICache, prefix : string) (slidingExpiration : TimeSpan) = + let mkCacheEntry (initialToken : StreamToken, initialState : 'state) = CacheEntry<'state>(initialToken, initialState, Token.supersedes) + let options = CacheItemOptions.RelativeExpiration slidingExpiration + fun streamName value -> + cache.UpdateIfNewer(prefix + streamName, options, mkCacheEntry value) + + let applyCacheUpdatesWithFixedTimeSpan (cache : ICache, prefix : string) (period : TimeSpan) = + let mkCacheEntry (initialToken : StreamToken, initialState : 'state) = CacheEntry<'state>(initialToken, initialState, Token.supersedes) + fun streamName value -> + let expirationPoint = let creationDate = DateTimeOffset.UtcNow in creationDate.Add period + let options = CacheItemOptions.AbsoluteExpiration expirationPoint + cache.UpdateIfNewer(prefix + streamName, options, mkCacheEntry value) + + type CachingCategory<'event, 'state, 'context> + ( category : Category<'event, 'state, 'context>, + fold : 'state -> 'event seq -> 'state, initial : 'state, isOrigin : 'event -> bool, + tryReadCache, updateCache, + checkUnfolds, mapUnfolds : Choice 'state -> 'event seq, 'event list -> 'state -> 'event list * 'event list>) = + let cache streamName inner = async { + let! tokenAndState = inner + do! updateCache streamName tokenAndState + return tokenAndState } + interface ICategory<'event, 'state, string, 'context> with + member _.Load(log, streamName, allowStale) : Async = async { + match! tryReadCache streamName with + | None -> return! category.Load(log, streamName, initial, checkUnfolds, fold, isOrigin) |> cache streamName + | Some tokenAndState when allowStale -> return tokenAndState // read already updated TTL, no need to write + | Some (token, state) -> return! category.Reload(log, streamName, token, state, fold, isOrigin) |> cache streamName } + member _.TrySync(log : ILogger, streamName, streamToken, state, events : 'event list, context) : Async> = async { + match! category.Sync(log, streamName, streamToken, state, events, mapUnfolds, fold, isOrigin, context) with + | SyncResult.Conflict resync -> + return SyncResult.Conflict (cache streamName resync) + | SyncResult.Written (token', state') -> + do! updateCache streamName (token', state') + return SyncResult.Written (token', state') } + +namespace Equinox.DynamoStore + +open Equinox.Core +open Equinox.DynamoStore.Core +open System + +/// Manages Creation and configuration of an IAmazonDynamoDB connection +type DynamoStoreConnector(credentials : Amazon.Runtime.AWSCredentials, clientConfig : Amazon.DynamoDBv2.AmazonDynamoDBConfig) = + + /// maxRetries. AWS SDK Default: 10 + /// timeout. AWS SDK Default: 100s + new (serviceUrl, accessKey, secretKey, retries, timeout) = + let credentials = Amazon.Runtime.BasicAWSCredentials(accessKey, secretKey) + let mode, r, t = Amazon.Runtime.RequestRetryMode.Standard, retries, timeout + let clientConfig = Amazon.DynamoDBv2.AmazonDynamoDBConfig(ServiceURL = serviceUrl, RetryMode = mode, MaxErrorRetry = r, Timeout = t) + DynamoStoreConnector(credentials, clientConfig) + + member _.Options = clientConfig + member x.Retries = x.Options.MaxErrorRetry + member x.Timeout = let t = x.Options.Timeout in t.Value + member x.Endpoint = x.Options.ServiceURL + + member _.CreateClient() = new Amazon.DynamoDBv2.AmazonDynamoDBClient(credentials, clientConfig) :> Amazon.DynamoDBv2.IAmazonDynamoDB + +type ProvisionedThroughput = FSharp.AWS.DynamoDB.ProvisionedThroughput +type Throughput = FSharp.AWS.DynamoDB.Throughput + +type StreamViewType = Amazon.DynamoDBv2.StreamViewType +type Streaming = FSharp.AWS.DynamoDB.Streaming + +[] +type ConnectMode = + | SkipVerify + | Verify + | CreateIfNotExists of Throughput +module internal ConnectMode = + let apply client tableName = function + | SkipVerify -> async { () } + | Verify -> Initialization.verify client tableName + | CreateIfNotExists throughput -> Initialization.createIfNotExists client tableName (throughput, Initialization.StreamingMode.Default) + +/// Holds all relevant state for a Store. There should be a single one of these per process. +type DynamoStoreClient + ( // Facilitates custom mapping of Stream Category Name to underlying Table and Stream names + categoryAndStreamIdToTableAndStreamNames : string * string -> string * string, + createContainer : string -> Container, + createFallbackContainer : string -> Container option, + [] ?primaryTableToArchive : string -> string) = + let primaryTableToSecondary = defaultArg primaryTableToArchive id + new( client : Amazon.DynamoDBv2.IAmazonDynamoDB, tableName : string, + // Table name to use for archive store. Default: (if archiveClient specified) use same tableName but via archiveClient. + [] ?archiveTableName, + // Client to use for archive store. Default: (if archiveTableName specified) use same archiveTableName but via client. + [] ?archiveClient : Amazon.DynamoDBv2.IAmazonDynamoDB) = + let genStreamName (categoryName, streamId) = if categoryName = null then streamId else sprintf "%s-%s" categoryName streamId + let catAndStreamToTableStream (categoryName, streamId) = tableName, genStreamName (categoryName, streamId) + let primaryContainer t = Container.Create(client, t) + let fallbackContainer = + if Option.isNone archiveClient && Option.isNone archiveTableName then fun _ -> None + else fun t -> Some (Container.Create(defaultArg archiveClient client, defaultArg archiveTableName t)) + DynamoStoreClient(catAndStreamToTableStream, primaryContainer, fallbackContainer) + member internal _.ResolveContainerFallbackAndStreamName(categoryName, streamId) : Container * Container option * string = + let tableName, streamName = categoryAndStreamIdToTableAndStreamNames (categoryName, streamId) + let fallbackTableName = primaryTableToSecondary tableName + createContainer tableName, createFallbackContainer fallbackTableName, streamName + + /// Connect to an Equinox.DynamoStore in the specified Table + /// Events that have been archived and purged (and hence are missing from the primary) are retrieved from the archive where that is provided + static member Connect(client, tableName : string, [] ?archiveTableName, [] ?mode : ConnectMode) : Async = async { + let init t = ConnectMode.apply client t (defaultArg mode ConnectMode.Verify) + do! init tableName + match archiveTableName with None -> () | Some archiveTable-> do! init archiveTable + return DynamoStoreClient(client, tableName, ?archiveTableName = archiveTableName) } + +/// Defines a set of related access policies for a given Table, together with a Containers map defining mappings from (category, streamId) to (tableName, streamName) +type DynamoStoreContext(storeClient : DynamoStoreClient, tipOptions, queryOptions, ?encodingOptions) = + new( storeClient : DynamoStoreClient, + // Maximum serialized event size to permit to accumulate in Tip before they get moved out to a standalone Batch. Default: 32K. + [] ?maxBytes, + // Maximum number of events permitted in Tip. When this is exceeded, events are moved out to a standalone Batch. Default: limited by maxBytes + [] ?tipMaxEvents, + // Max number of Batches to return per paged query response. Default: 32. + [] ?queryMaxItems, + // Maximum number of trips to permit when slicing the work into multiple responses limited by `queryMaxItems`. Default: unlimited. + [] ?queryMaxRequests, + // Override the default encoding options for Event Bodies + [] ?encodingOptions : EncodingOptions, + // Inhibit throwing when events are missing, but no Archive Table has been supplied as a fallback + [] ?ignoreMissingEvents) = + let tipOptions = TipOptions(?maxBytes = maxBytes, ?maxEvents = tipMaxEvents) + let queryOptions = QueryOptions(?maxItems = queryMaxItems, ?maxRequests = queryMaxRequests, ?ignoreMissingEvents = ignoreMissingEvents) + DynamoStoreContext(storeClient, tipOptions, queryOptions, ?encodingOptions = encodingOptions) + member val StoreClient = storeClient + member val QueryOptions = queryOptions + member val TipOptions = tipOptions + member val EncodingOptions = defaultArg encodingOptions defaultEncodingOptions + member internal x.ResolveContainerClientAndStreamId(categoryName, streamId) = + let container, fallback, streamName = storeClient.ResolveContainerFallbackAndStreamName(categoryName, streamId) + StoreClient(container, fallback, x.QueryOptions, x.TipOptions, x.EncodingOptions), streamName + +/// For DynamoDB, caching is critical in order to reduce RU consumption. +/// As such, it can often be omitted, particularly if streams are short or there are snapshots being maintained +[] +type CachingStrategy = + /// Do not apply any caching strategy for this Stream. + /// NB opting not to leverage caching when using DynamoDB can have significant implications for the scalability + /// of your application, both in terms of latency and running costs. + /// While the cost of a cache miss can be ameliorated to varying degrees by employing an appropriate `AccessStrategy` + /// [that works well and has been validated for your scenario with real data], even a cache with a low Hit Rate provides + /// a direct benefit in terms of the number of Read and/or Write Request Charge Units (RCU)s that need to be provisioned for your Tables. + | NoCaching + /// Retain a single 'state per streamName, together with the associated etag. + /// Each cache hit for a stream renews the retention period for the defined window. + /// Upon expiration of the defined window from the point at which the cache was entry was last used, a full reload is triggered. + /// Unless LoadOption.AllowStale is used, each cache hit still incurs an etag-contingent Tip read (at a cost of a roundtrip with a 1RU charge if unmodified). + // NB while a strategy like EventStore.Caching.SlidingWindowPrefixed is obviously easy to implement, the recommended approach is to + // track all relevant data in the state, and/or have the `unfold` function ensure _all_ relevant events get held in the unfolds in Tip + | SlidingWindow of ICache * window : TimeSpan + /// Retain a single 'state per streamName, together with the associated etag. + /// Upon expiration of the defined period, a full reload is triggered. + /// Typically combined with `Equinox.LoadOption.AllowStale` to minimize loads. + /// Unless LoadOption.AllowStale is used, each cache hit still incurs an etag-contingent Tip read (at a cost of a roundtrip with a 1RU charge if unmodified). + | FixedTimeSpan of ICache * period : TimeSpan + +[] +type AccessStrategy<'event, 'state> = + /// Don't apply any optimized reading logic. Note this can be extremely RU cost prohibitive + /// and can severely impact system scalability. Should hence only be used with careful consideration. + | Unoptimized + /// Load only the single most recent event defined in 'event and trust that doing a fold from any such event + /// will yield a correct and complete state + /// In other words, the fold function should not need to consider either the preceding 'state or 'events. + /// + /// A copy of the event is also retained in the `Tip` document in order that the state of the stream can be + /// retrieved using a single (cached, etag-checked) point read. + /// isOrigin test) to be used to build the state + /// in lieu of folding all the events from the start of the stream, as a performance optimization. + /// toSnapshot is used to generate the unfold that will be held in the Tip document in order to + /// enable efficient reading without having to query the Event documents. + | Snapshot of isOrigin : ('event -> bool) * toSnapshot : ('state -> 'event) + /// Allow any events that pass the `isOrigin` test to be used in lieu of folding all the events from the start of the stream + /// When writing, uses `toSnapshots` to 'unfold' the 'state, representing it as one or more Event records to be stored in + /// the Tip with efficient read cost. + | MultiSnapshot of isOrigin : ('event -> bool) * toSnapshots : ('state -> 'event seq) + /// Instead of actually storing the events representing the decisions, only ever update a snapshot stored in the Tip document + /// In this mode, Optimistic Concurrency Control is necessarily based on the etag + | RollingState of toSnapshot : ('state -> 'event) + /// Allow produced events to be filtered, transformed or removed completely and/or to be transmuted to unfolds. + /// + /// In this mode, Optimistic Concurrency Control is based on the etag (rather than the normal Expected Version strategy) + /// in order that conflicting updates to the state not involving the writing of an event can trigger retries. + /// + | Custom of isOrigin : ('event -> bool) * transmute : ('event list -> 'state -> 'event list*'event list) + +type DynamoStoreCategory<'event, 'state, 'context>(context : DynamoStoreContext, codec, fold, initial, caching, access) = + let categories = System.Collections.Concurrent.ConcurrentDictionary>() + let resolveCategory (categoryName, container) = + let createCategory _name : ICategory<_, _, string, 'context> = + let tryReadCache, updateCache = + match caching with + | CachingStrategy.NoCaching -> (fun _ -> async { return None }), fun _ _ -> async { () } + | CachingStrategy.SlidingWindow (cache, window) -> cache.TryGet, Caching.applyCacheUpdatesWithSlidingExpiration (cache, null) window + | CachingStrategy.FixedTimeSpan (cache, period) -> cache.TryGet, Caching.applyCacheUpdatesWithFixedTimeSpan (cache, null) period + let isOrigin, checkUnfolds, mapUnfolds = + match access with + | AccessStrategy.Unoptimized -> (fun _ -> false), false, Choice1Of3 () + | AccessStrategy.LatestKnownEvent -> (fun _ -> true), true, Choice2Of3 (fun events _ -> Seq.last events |> Seq.singleton) + | AccessStrategy.Snapshot (isOrigin, toSnapshot) -> isOrigin, true, Choice2Of3 (fun _ state -> toSnapshot state |> Seq.singleton) + | AccessStrategy.MultiSnapshot (isOrigin, unfold) -> isOrigin, true, Choice2Of3 (fun _ state -> unfold state) + | AccessStrategy.RollingState toSnapshot -> (fun _ -> true), true, Choice3Of3 (fun _ state -> [], [toSnapshot state]) + | AccessStrategy.Custom (isOrigin, transmute) -> isOrigin, true, Choice3Of3 transmute + let cosmosCat = Category<'event, 'state, 'context>(container, codec) + Caching.CachingCategory<'event, 'state, 'context>(cosmosCat, fold, initial, isOrigin, tryReadCache, updateCache, checkUnfolds, mapUnfolds) :> _ + categories.GetOrAdd(categoryName, createCategory) + let resolve (FsCodec.StreamName.CategoryAndId (categoryName, streamId)) = + let container, streamName = context.ResolveContainerClientAndStreamId(categoryName, streamId) + resolveCategory (categoryName, container), streamName, None + let empty = Token.empty, initial + let storeCategory = StoreCategory(resolve, empty) + member _.Resolve(streamName, ?context) = storeCategory.Resolve(streamName, ?context = context) + +namespace Equinox.DynamoStore.Core + +open Equinox.Core +open FsCodec +open FSharp.Control + +/// Outcome of appending events, specifying the new and/or conflicting events, together with the updated Target write position +[] +type AppendResult<'t> = + | Ok of pos : 't + | ConflictUnknown + +/// Encapsulates the core facilities Equinox.DynamoStore offers for operating directly on Events in Streams. +type EventsContext internal + ( context : Equinox.DynamoStore.DynamoStoreContext, store : StoreClient, + // Logger to write to - see https://github.com/serilog/serilog/wiki/Provided-Sinks for how to wire to your logger + log : Serilog.ILogger) = + do if log = null then nullArg "log" + let maxCountPredicate count = + let acc = ref (max (count-1) 0) + fun _ -> + if acc.Value = 0 then true else + acc.Value <- acc.Value - 1 + false + + let yieldPositionAndData res = async { + let! Token.Unpack pos', data = res + return Position.flatten pos', data } + + new (context : Equinox.DynamoStore.DynamoStoreContext, log) = + let storeClient, _streamId = context.ResolveContainerClientAndStreamId(null, null) + EventsContext(context, storeClient, log) + + member x.StreamId(streamName) : string = context.ResolveContainerClientAndStreamId(null, streamName) |> snd + + member internal _.GetLazy(stream, ?queryMaxItems, ?direction, ?minIndex, ?maxIndex) : AsyncSeq array> = + let direction = defaultArg direction Direction.Forward + let batching = match queryMaxItems with Some qmi -> QueryOptions(qmi) | _ -> context.QueryOptions + store.ReadLazy(log, batching, stream, direction, (Some, fun _ -> false), ?minIndex = minIndex, ?maxIndex = maxIndex) + + member internal _.GetInternal(stream, ?minIndex, ?maxIndex, ?maxCount, ?direction) = async { + let direction = defaultArg direction Direction.Forward + if maxCount = Some 0 then + // Search semantics include the first hit so we need to special case this anyway + let startPos = (if direction = Direction.Backward then maxIndex else minIndex) |> Option.map Position.null_ + return Token.create (Position.flatten startPos), Array.empty + else + let isOrigin = + match maxCount with + | Some limit -> maxCountPredicate limit + | None -> fun _ -> false + let! token, events = store.Read(log, stream, direction, (Some, isOrigin), ?minIndex = minIndex, ?maxIndex = maxIndex) + if direction = Direction.Backward then System.Array.Reverse events + return token, events } + + /// Establishes the current position of the stream in as efficient a manner as possible + /// (The ideal situation is that the preceding token is supplied as input in order to avail of 1RU low latency state checks) + member _.Sync(stream, [] ?position : Position) : Async = async { + let! Token.Unpack pos' = store.GetPosition(log, stream, ?pos = position) + return Position.flatten pos' } + + /// Query (with MaxItems set to `queryMaxItems`) from the specified `Position`, allowing the reader to efficiently walk away from a running query + /// ... NB as long as they Dispose! + member x.Walk(stream, queryMaxItems, [] ?minIndex, [] ?maxIndex, [] ?direction) + : AsyncSeq array> = + x.GetLazy(stream, queryMaxItems, ?direction = direction, ?minIndex = minIndex, ?maxIndex = maxIndex) + + /// Reads all Events from a `Position` in a given `direction` + member x.Read(stream, [] ?minIndex, [] ?maxIndex, [] ?maxCount, [] ?direction) + : Async array> = + x.GetInternal(stream, ?minIndex = minIndex, ?maxIndex = maxIndex, ?maxCount = maxCount, ?direction = direction) |> yieldPositionAndData +#if APPEND_SUPPORT + + /// Appends the supplied batch of events, subject to a consistency check based on the `position` + /// Callers should implement appropriate idempotent handling, or use Equinox.Decider for that purpose + member x.Sync(stream, position, events : IEventData<_> array) : Async> = async { + match! store.Sync(log, stream, Some position, Position.toIndex >> Sync.Exp.Version, position.index, events, Seq.empty) with + | InternalSyncResult.Written (Token.Unpack pos) -> return AppendResult.Ok (Position.flatten pos) + | InternalSyncResult.ConflictUnknown -> return AppendResult.ConflictUnknown } +#endif + + member _.Prune(stream, index) : Async = + store.Prune(log, stream, index) + +/// Provides mechanisms for building `EventData` records to be supplied to the `Events` API +type EventData() = + /// Creates an Event record, suitable for supplying to Append et al + static member FromUtf8Bytes(eventType, data, ?meta) : IEventData<_> = FsCodec.Core.EventData.Create(eventType, data, ?meta = meta) :> _ + +/// Api as defined in the Equinox Specification +/// Note the DynamoContext APIs can yield better performance due to the fact that a Position tracks the etag of the Stream's Tip +module Events = + + let private (|PositionIndex|) (x : Position) = x.index + let private stripSyncResult (f : Async>) : Async> = async { + match! f with + | AppendResult.Ok (PositionIndex index)-> return AppendResult.Ok index + | AppendResult.ConflictUnknown -> return AppendResult.ConflictUnknown } + let private stripPosition (f : Async) : Async = async { + let! (PositionIndex index) = f + return index } + let private dropPosition (f : Async array>) : Async array> = async { + let! _, xs = f + return xs } + + /// Returns an async sequence of events in the stream starting at the specified sequence number, + /// reading in batches of the specified size. + /// Returns an empty sequence if the stream is empty or if the sequence number is larger than the largest + /// sequence number in the stream. + let getAll (ctx : EventsContext) (streamName : string) (index : int64) (batchSize : int) : AsyncSeq array> = + ctx.Walk(ctx.StreamId streamName, batchSize, minIndex = index) + + /// Returns an async array of events in the stream starting at the specified sequence number, + /// number of events to read is specified by batchSize + /// Returns an empty sequence if the stream is empty or if the sequence number is larger than the largest + /// sequence number in the stream. + let get (ctx : EventsContext) (streamName : string) (index : int64) (maxCount : int) : Async array> = + ctx.Read(ctx.StreamId streamName, ?minIndex = (if index = 0 then None else Some index), maxCount = maxCount) |> dropPosition + +#if APPEND_SUPPORT + /// Appends a batch of events to a stream at the specified expected sequence number. + /// If the specified expected sequence number does not match the stream, the events are not appended + /// and a failure is returned. + let append (ctx : EventsContext) (streamName : string) (index : int64) (events : IEventData<_> array) : Async> = + ctx.Sync(ctx.StreamId streamName, Sync.Exp.Version index, events) |> stripSyncResult + +#endif + /// Requests deletion of events up and including the specified index. + /// Due to the need to preserve ordering of data in the stream, only complete Batches will be removed. + /// If the index is within the Tip, events are removed via an etag-checked update. Does not alter the unfolds held in the Tip, or remove the Tip itself. + /// Returns count of events deleted this time, events that could not be deleted due to partial batches, and the stream's lowest remaining sequence number. + let pruneUntil (ctx : EventsContext) (streamName : string) (index : int64) : Async = + ctx.Prune(ctx.StreamId streamName, index) + + /// Returns an async sequence of events in the stream backwards starting from the specified sequence number, + /// reading in batches of the specified size. + /// Returns an empty sequence if the stream is empty or if the sequence number is smaller than the smallest + /// sequence number in the stream. + let getAllBackwards (ctx : EventsContext) (streamName : string) (index : int64) (batchSize : int) : AsyncSeq array> = + ctx.Walk(ctx.StreamId streamName, batchSize, maxIndex = index, direction = Direction.Backward) + + /// Returns an async array of events in the stream backwards starting from the specified sequence number, + /// number of events to read is specified by batchSize + /// Returns an empty sequence if the stream is empty or if the sequence number is smaller than the smallest + /// sequence number in the stream. + let getBackwards (ctx : EventsContext) (streamName : string) (index : int64) (maxCount : int) : Async array> = + ctx.Read(ctx.StreamId streamName, ?maxIndex = (match index with int64.MaxValue -> None | i -> Some (i + 1L)), maxCount = maxCount, direction = Direction.Backward) |> dropPosition + + /// Obtains the `index` from the current write Position + let getNextIndex (ctx : EventsContext) (streamName : string) : Async = + ctx.Sync(ctx.StreamId streamName) |> stripPosition diff --git a/src/Equinox.DynamoStore/Equinox.DynamoStore.fsproj b/src/Equinox.DynamoStore/Equinox.DynamoStore.fsproj new file mode 100644 index 000000000..afcb6058b --- /dev/null +++ b/src/Equinox.DynamoStore/Equinox.DynamoStore.fsproj @@ -0,0 +1,31 @@ + + + + netstandard2.1 + true + true + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/Equinox.CosmosStore.Integration/AccessStrategies.fs b/tests/Equinox.CosmosStore.Integration/AccessStrategies.fs index 5ad3590e3..95cd58f96 100644 --- a/tests/Equinox.CosmosStore.Integration/AccessStrategies.fs +++ b/tests/Equinox.CosmosStore.Integration/AccessStrategies.fs @@ -1,7 +1,12 @@ module Equinox.Store.Integration.AccessStrategies +#if STORE_DYNAMO +open Equinox.DynamoStore +open Equinox.DynamoStore.Integration.CosmosFixtures +#else open Equinox.CosmosStore open Equinox.CosmosStore.Integration.CosmosFixtures +#endif open Swensen.Unquote open System @@ -35,7 +40,11 @@ module SequenceCheck = type Event = | Add of {| value : int |} interface TypeShape.UnionContract.IUnionContract +#if STORE_DYNAMO + let codec = FsCodec.SystemTextJson.Codec.Create() +#else let codec = FsCodec.SystemTextJson.CodecJsonElement.Create() +#endif module Fold = diff --git a/tests/Equinox.CosmosStore.Integration/CosmosCoreIntegration.fs b/tests/Equinox.CosmosStore.Integration/CosmosCoreIntegration.fs index 02e408fd4..c92e18cef 100644 --- a/tests/Equinox.CosmosStore.Integration/CosmosCoreIntegration.fs +++ b/tests/Equinox.CosmosStore.Integration/CosmosCoreIntegration.fs @@ -108,7 +108,7 @@ type Tests(testOutputHelper) = let! pos = Events.getNextIndex ctx streamName test <@ [EqxAct.TipNotFound] = capture.ExternalCalls @> 0L =! pos - verifyRequestChargesMax 1 // for a 404 by definition + verifyRequestChargesMax 2 // was 1 for a 404 by definition, now 1.38 capture.Clear() let mutable pos = 0L @@ -327,7 +327,7 @@ type Tests(testOutputHelper) = let! deleted, deferred, trimmedPos = Events.pruneUntil ctx streamName -1L test <@ deleted = 0 && deferred = 0 && trimmedPos = 0L @> test <@ [EqxAct.PruneResponse; EqxAct.Prune] = capture.ExternalCalls @> - verifyRequestChargesMax 3 // 2.86 + verifyRequestChargesMax 4 // 3.62 // Trigger deletion of first batch, but as we're in the middle of the next Batch... capture.Clear() @@ -350,7 +350,7 @@ type Tests(testOutputHelper) = let! deleted, deferred, trimmedPos = Events.pruneUntil ctx streamName 4L test <@ deleted = 0 && deferred = (if eventsInTip then 0 else 1) && trimmedPos = pos @> test <@ [EqxAct.PruneResponse; EqxAct.Prune] = capture.ExternalCalls @> - verifyRequestChargesMax 3 // 2.86 + verifyRequestChargesMax 4 // 3.24 // We should still get the high-water mark even if we asked for less capture.Clear() @@ -378,7 +378,7 @@ type Tests(testOutputHelper) = let! deleted, deferred, trimmedPos = Events.pruneUntil ctx streamName 6L test <@ deleted = 0 && deferred = 0 && trimmedPos = 6L @> test <@ [EqxAct.PruneResponse; EqxAct.Prune] = capture.ExternalCalls @> - verifyRequestChargesMax 3 // 2.83 + verifyRequestChargesMax 4 // 3.21 } (* Fallback *) diff --git a/tests/Equinox.CosmosStore.Integration/CosmosFixtures.fs b/tests/Equinox.CosmosStore.Integration/CosmosFixtures.fs index de93abb7a..47a51be66 100644 --- a/tests/Equinox.CosmosStore.Integration/CosmosFixtures.fs +++ b/tests/Equinox.CosmosStore.Integration/CosmosFixtures.fs @@ -1,4 +1,66 @@ -[] +#if STORE_DYNAMO +[] +module Equinox.DynamoStore.Integration.CosmosFixtures + +open Amazon.DynamoDBv2 +open Equinox.DynamoStore +open System + +// docker compose up dynamodb-local will stand up a simulator instance that this wiring can connect to +let private tryRead env = Environment.GetEnvironmentVariable env |> Option.ofObj +let private tableName = tryRead "EQUINOX_DYNAMO_TABLE" |> Option.defaultValue "equinox-test" +let private archiveTableName = tryRead "EQUINOX_DYNAMO_TABLE_ARCHIVE" |> Option.defaultValue "equinox-test-archive" + +let discoverConnection () = + match tryRead "EQUINOX_DYNAMO_CONNECTION" with + | None -> "dynamodb-local", "http://localhost:8000" + | Some connectionString -> "EQUINOX_DYNAMO_CONNECTION", connectionString + +let createClient (log : Serilog.ILogger) name serviceUrl = + // See https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html#docker for details of how to deploy a simulator instance + let clientConfig = AmazonDynamoDBConfig(ServiceURL = serviceUrl) + log.Information("DynamoDB Connecting {name} to {endpoint}", name, serviceUrl) + // Credentials are not validated if connecting to local instance so anything will do (this avoids it looking for profiles to be configured) + let credentials = Amazon.Runtime.BasicAWSCredentials("A", "A") + new AmazonDynamoDBClient(credentials, clientConfig) :> IAmazonDynamoDB + +let connectPrimary log = + let name, serviceUrl = discoverConnection () + let client = createClient log name serviceUrl + DynamoStoreClient(client, tableName) + +let connectArchive log = + let name, serviceUrl = discoverConnection () + let client = createClient log name serviceUrl + DynamoStoreClient(client, archiveTableName) + +let connectWithFallback log = + let name, serviceUrl = discoverConnection () + let client = createClient log name serviceUrl + DynamoStoreClient(client, tableName, archiveTableName = archiveTableName) + +// Prepares the two required tables that the test lea on via connectPrimary/Archive/WithFallback +type DynamoTablesFixture() = + + interface Xunit.IAsyncLifetime with + member _.InitializeAsync() = + let name, serviceUrl = discoverConnection () + let client = createClient Serilog.Log.Logger name serviceUrl + let throughput = ProvisionedThroughput (100L, 100L) + let throughput = Throughput.Provisioned throughput + DynamoStoreClient.Connect(client, tableName, archiveTableName = archiveTableName, mode = CreateIfNotExists throughput) + |> Async.StartImmediateAsTask + :> System.Threading.Tasks.Task + member _.DisposeAsync() = task { () } + +[] +type DocStoreCollection() = + interface Xunit.ICollectionFixture + +type StoreContext = DynamoStoreContext +type StoreCategory<'E, 'S> = DynamoStoreCategory<'E, 'S, obj> +#else +[] module Equinox.CosmosStore.Integration.CosmosFixtures open Equinox.CosmosStore @@ -44,6 +106,7 @@ type DocStoreCollection() = type StoreContext = CosmosStoreContext type StoreCategory<'E, 'S> = CosmosStoreCategory<'E, 'S, obj> +#endif let createPrimaryContextIgnoreMissing client queryMaxItems tipMaxEvents ignoreMissing = StoreContext(client, tipMaxEvents = tipMaxEvents, queryMaxItems = queryMaxItems, ignoreMissingEvents = ignoreMissing) diff --git a/tests/Equinox.CosmosStore.Integration/CosmosFixturesInfrastructure.fs b/tests/Equinox.CosmosStore.Integration/CosmosFixturesInfrastructure.fs index 3226ac548..a7e2a56d5 100644 --- a/tests/Equinox.CosmosStore.Integration/CosmosFixturesInfrastructure.fs +++ b/tests/Equinox.CosmosStore.Integration/CosmosFixturesInfrastructure.fs @@ -19,8 +19,13 @@ module SerilogHelpers = let (|SerilogScalar|_|) : LogEventPropertyValue -> obj option = function | :? ScalarValue as x -> Some x.Value | _ -> None +#if STORE_DYNAMO + open Equinox.DynamoStore.Core + open Equinox.DynamoStore.Core.Log +#else open Equinox.CosmosStore.Core open Equinox.CosmosStore.Core.Log +#endif [] type EqxAct = | Tip | TipNotFound | TipNotModified @@ -39,14 +44,20 @@ module SerilogHelpers = | Metric.QueryResponse (Direction.Backward, _) -> EqxAct.ResponseBackward | Metric.SyncSuccess _ -> EqxAct.Append +#if !STORE_DYNAMO | Metric.SyncResync _ -> EqxAct.Resync +#endif | Metric.SyncConflict _ -> EqxAct.Conflict | Metric.Prune _ -> EqxAct.Prune | Metric.PruneResponse _ -> EqxAct.PruneResponse | Metric.Delete _ -> EqxAct.Delete | Metric.Trim _ -> EqxAct.Trim +#if !STORE_DYNAMO let (|Load|Write|Resync|Prune|Delete|Trim|Response|) = function +#else + let (|Load|Write|Prune|Delete|Trim|Response|) = function +#endif | Metric.Tip s | Metric.TipNotFound s | Metric.TipNotModified s @@ -56,7 +67,9 @@ module SerilogHelpers = | Metric.SyncSuccess s | Metric.SyncConflict s -> Write s +#if !STORE_DYNAMO | Metric.SyncResync s -> Resync s +#endif | Metric.Prune (_, s) -> Prune s | Metric.PruneResponse s -> Response s @@ -67,7 +80,9 @@ module SerilogHelpers = /// Facilitates splitting between events with direct charges vs synthetic events Equinox generates to avoid double counting let (|TotalRequestCharge|ResponseBreakdown|) = function | Load (Rc rc) | Write (Rc rc) +#if !STORE_DYNAMO | Resync (Rc rc) +#endif | Delete (Rc rc) | Trim (Rc rc) | Prune (Rc rc) as e -> TotalRequestCharge (e, rc) | Response _ -> ResponseBreakdown let (|EqxEvent|_|) (logEvent : LogEvent) : Metric option = diff --git a/tests/Equinox.CosmosStore.Integration/DocumentStoreIntegration.fs b/tests/Equinox.CosmosStore.Integration/DocumentStoreIntegration.fs index ce2fb75c9..80962567b 100644 --- a/tests/Equinox.CosmosStore.Integration/DocumentStoreIntegration.fs +++ b/tests/Equinox.CosmosStore.Integration/DocumentStoreIntegration.fs @@ -5,13 +5,22 @@ open FSharp.UMX open Swensen.Unquote open System open System.Threading +#if STORE_DYNAMO +open Equinox.DynamoStore +open Equinox.DynamoStore.Integration.CosmosFixtures +#else open Equinox.CosmosStore open Equinox.CosmosStore.Integration.CosmosFixtures +#endif module Cart = let fold, initial = Cart.Fold.fold, Cart.Fold.initial let snapshot = Cart.Fold.isOrigin, Cart.Fold.snapshot +#if STORE_DYNAMO + let codec = Cart.Events.codec +#else let codec = Cart.Events.codecJe +#endif let createServiceWithoutOptimization log context = let resolve = StoreCategory(context, codec, fold, initial, CachingStrategy.NoCaching, AccessStrategy.Unoptimized).Resolve Cart.create log resolve @@ -34,7 +43,11 @@ module Cart = module ContactPreferences = let fold, initial = ContactPreferences.Fold.fold, ContactPreferences.Fold.initial +#if STORE_DYNAMO + let codec = ContactPreferences.Events.codec +#else let codec = ContactPreferences.Events.codecJe +#endif let private createServiceWithLatestKnownEvent context log cachingStrategy = let resolveStream = StoreCategory(context, codec, fold, initial, cachingStrategy, AccessStrategy.LatestKnownEvent).Resolve ContactPreferences.create log resolveStream @@ -77,7 +90,11 @@ type Tests(testOutputHelper) = let service = Cart.createServiceWithoutOptimization log context let expectedResponses n = let tipItem = 1 +#if STORE_DYNAMO // For Cosmos, we supply a full query and it notices it is at the end - for Dynamo, another query is required + let finalEmptyPage = 1 +#else let finalEmptyPage = 0 +#endif let expectedItems = tipItem + (if eventsInTip then n / 2 else n) + finalEmptyPage max 1 (int (ceil (float expectedItems / float queryMaxItems))) @@ -87,7 +104,11 @@ type Tests(testOutputHelper) = for i in [1..transactions] do do! addAndThenRemoveItemsManyTimesExceptTheLastOne cartContext cartId skuId service addRemoveCount test <@ i = i && List.replicate (expectedResponses (i-1)) EqxAct.ResponseBackward @ [EqxAct.QueryBackward; EqxAct.Append] = capture.ExternalCalls @> +#if STORE_DYNAMO + if eventsInTip then verifyRequestChargesMax 181 // 180.5 [8.5; 172] +#else if eventsInTip then verifyRequestChargesMax 76 // 76.0 [3.72; 72.28] +#endif else verifyRequestChargesMax 79 // 78.37 [3.15; 75.22] capture.Clear() @@ -96,7 +117,11 @@ type Tests(testOutputHelper) = test <@ addRemoveCount = match state with { items = [{ quantity = quantity }] } -> quantity | _ -> failwith "nope" @> test <@ List.replicate (expectedResponses transactions) EqxAct.ResponseBackward @ [EqxAct.QueryBackward] = capture.ExternalCalls @> +#if STORE_DYNAMO + if eventsInTip then verifyRequestChargesMax 12 // 11.5 +#else if eventsInTip then verifyRequestChargesMax 9 // 8.05 +#endif else verifyRequestChargesMax 15 // 14.01 } @@ -172,10 +197,14 @@ type Tests(testOutputHelper) = && has sku21 21 && has sku22 22 @> // Intended conflicts arose let conflict = function EqxAct.Conflict | EqxAct.Resync as x -> Some x | _ -> None +#if !STORE_DYNAMO if eventsInTip then test <@ let c2 = List.choose conflict capture2.ExternalCalls [EqxAct.Resync] = List.choose conflict capture1.ExternalCalls && [EqxAct.Resync] = c2 @> +#else + if false then () +#endif else test <@ let c2 = List.choose conflict capture2.ExternalCalls [EqxAct.Conflict] = List.choose conflict capture1.ExternalCalls @@ -220,7 +249,11 @@ type Tests(testOutputHelper) = | Choice2Of2 e -> e.Message.StartsWith "Origin event not found; no Archive Container supplied" || e.Message.StartsWith "Origin event not found; no Archive Table supplied" | x -> failwithf "Unexpected %A" x @> +#if STORE_DYNAMO // Extra null query + test <@ [EqxAct.ResponseForward; EqxAct.ResponseForward; EqxAct.QueryForward] = capture.ExternalCalls @> +#else test <@ [EqxAct.ResponseForward; EqxAct.QueryForward] = capture.ExternalCalls @> +#endif verifyRequestChargesMax 3 // 2.99 // But not forgotten @@ -331,9 +364,15 @@ type Tests(testOutputHelper) = && has sku21 21 && has sku22 22 @> // Intended conflicts arose let conflict = function EqxAct.Conflict | EqxAct.Resync as x -> Some x | _ -> None +#if STORE_DYNAMO // Failed conditions do not yield the conflicting state, so it needs to be a separate load + test <@ let c2 = List.choose conflict capture2.ExternalCalls + [EqxAct.Conflict] = List.choose conflict capture1.ExternalCalls + && [EqxAct.Conflict] = c2 @> +#else test <@ let c2 = List.choose conflict capture2.ExternalCalls [EqxAct.Resync] = List.choose conflict capture1.ExternalCalls && [EqxAct.Resync] = c2 @> +#endif } [] diff --git a/tests/Equinox.DynamoStore.Integration/Equinox.DynamoStore.Integration.fsproj b/tests/Equinox.DynamoStore.Integration/Equinox.DynamoStore.Integration.fsproj new file mode 100644 index 000000000..df7ec8578 --- /dev/null +++ b/tests/Equinox.DynamoStore.Integration/Equinox.DynamoStore.Integration.fsproj @@ -0,0 +1,44 @@ + + + + net6.0 + true + $(DefineConstants);STORE_DYNAMO + + + + + + CosmosFixtures.fs + + + CosmosFixturesInfrastructure.fs + + + AutoDataAttribute.fs + + + DocumentStoreIntegration.fs + + + AccessStrategies.fs + + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + diff --git a/tests/Equinox.EventStoreDb.Integration/Equinox.EventStoreDb.Integration.fsproj b/tests/Equinox.EventStoreDb.Integration/Equinox.EventStoreDb.Integration.fsproj index d64eca76c..08607cb96 100644 --- a/tests/Equinox.EventStoreDb.Integration/Equinox.EventStoreDb.Integration.fsproj +++ b/tests/Equinox.EventStoreDb.Integration/Equinox.EventStoreDb.Integration.fsproj @@ -20,12 +20,11 @@ - - + - + all runtime; build; native; contentfiles; analyzers diff --git a/tools/Equinox.Tool/Program.fs b/tools/Equinox.Tool/Program.fs index ec97c082c..5dc10634a 100644 --- a/tools/Equinox.Tool/Program.fs +++ b/tools/Equinox.Tool/Program.fs @@ -25,6 +25,7 @@ type Arguments = | [] LogFile of string | [] Run of ParseResults | [] Init of ParseResults + | [] InitAws of ParseResults | [] Config of ParseResults | [] Stats of ParseResults | [] Dump of ParseResults @@ -36,6 +37,7 @@ type Arguments = | LogFile _ -> "specify a log file to write the result breakdown into (default: eqx.log)." | Run _ -> "Run a load test" | Init _ -> "Initialize Store/Container (supports `cosmos` stores; also handles RU/s provisioning adjustment)." + | InitAws _ -> "Initialize DynamoDB Table (supports `dynamo` stores; also handles RU/s provisioning adjustment)." | Config _ -> "Initialize Database Schema (supports `mssql`/`mysql`/`postgres` SqlStreamStore stores)." | Stats _ -> "inspect store to determine numbers of streams/documents/events (supports `cosmos` stores)." | Dump _ -> "Load and show events in a specified stream (supports all stores)." @@ -64,6 +66,25 @@ and CosmosInitInfo(args : ParseResults) = | CosmosModeType.Serverless -> if args.Contains Rus || args.Contains Autoscale then raise (Storage.MissingArg "Cannot specify RU/s or Autoscale in Serverless mode") CosmosInit.Provisioning.Serverless +and [] TableArguments = + | [] OnDemand + | [] Streaming of Equinox.DynamoStore.Core.Initialization.StreamingMode + | [] ReadCu of int64 + | [] WriteCu of int64 + | [] Dynamo of ParseResults + interface IArgParserTemplate with + member a.Usage = a |> function + | OnDemand -> "Specify On-Demand Capacity Mode." + | Streaming _ -> "Specify Streaming Mode. Default NEW_IMAGE" + | ReadCu _ -> "Specify Read Capacity Units to provision for the Table. (Ignored in On-Demand mode)" + | WriteCu _ -> "Specify Write Capacity Units to provision for the Table. (Ignored in On-Demand mode)" + | Dynamo _ -> "DynamoDB Connection parameters." +and DynamoInitInfo(args : ParseResults) = + let streaming = args.GetResult(Streaming, Equinox.DynamoStore.Core.Initialization.StreamingMode.Default) + let onDemand = args.Contains OnDemand + let readCu = args.GetResult ReadCu + let writeCu = args.GetResult WriteCu + member _.ProvisioningMode = streaming, if onDemand then None else Some (readCu, writeCu) and []ConfigArguments = | [] MsSql of ParseResults | [] MySql of ParseResults @@ -97,6 +118,7 @@ and []DumpArguments = | [] UnfoldsOnly | [] EventsOnly | [] Cosmos of ParseResults + | [] Dynamo of ParseResults | [] Es of ParseResults | [] MsSql of ParseResults | [] MySql of ParseResults @@ -114,6 +136,7 @@ and []DumpArguments = | EventsOnly -> "Exclude Unfolds/Snapshots. Default: show both Events and Unfolds." | Es _ -> "Parameters for EventStore." | Cosmos _ -> "Parameters for CosmosDB." + | Dynamo _ -> "Parameters for DynamoDB." | MsSql _ -> "Parameters for Sql Server." | MySql _ -> "Parameters for MySql." | Postgres _ -> "Parameters for Postgres." @@ -124,6 +147,9 @@ and DumpInfo(args: ParseResults) = | Some (DumpArguments.Cosmos sargs) -> let storeLog = createStoreLog <| sargs.Contains Storage.Cosmos.Arguments.VerboseStore storeLog, Storage.Cosmos.config log storeConfig (Storage.Cosmos.Info sargs) + | Some (DumpArguments.Dynamo sargs) -> + let storeLog = createStoreLog <| sargs.Contains Storage.Dynamo.Arguments.VerboseStore + storeLog, Storage.Dynamo.config log storeConfig (Storage.Dynamo.Info sargs) | Some (DumpArguments.Es sargs) -> let storeLog = createStoreLog <| sargs.Contains Storage.EventStore.Arguments.VerboseStore storeLog, Storage.EventStore.config (log, storeLog) storeConfig sargs @@ -136,7 +162,7 @@ and DumpInfo(args: ParseResults) = | Some (DumpArguments.Postgres sargs) -> let storeLog = createStoreLog false storeLog, Storage.Sql.Pg.config log storeConfig sargs - | _ -> failwith "please specify a `cosmos`,`es`,`ms`,`my` or `pg` endpoint" + | _ -> failwith "please specify a `cosmos`, `dynamo`, `es`,`ms`,`my` or `pg` endpoint" and []WebArguments = | [] Endpoint of string interface IArgParserTemplate with @@ -152,6 +178,7 @@ and []TestArguments = | [] ErrorCutoff of int64 | [] ReportIntervalS of int | [] Cosmos of ParseResults + | [] Dynamo of ParseResults | [] Es of ParseResults | [] Memory of ParseResults | [] MsSql of ParseResults @@ -170,6 +197,7 @@ and []TestArguments = | ReportIntervalS _ -> "specify reporting intervals in seconds (default: 10)." | Es _ -> "Run transactions in-process against EventStore." | Cosmos _ -> "Run transactions in-process against CosmosDB." + | Dynamo _ -> "Run transactions in-process against DynamoDb." | Memory _ -> "target in-process Transient Memory Store (Default if not other target specified)." | MsSql _ -> "Run transactions in-process against Sql Server." | MySql _ -> "Run transactions in-process against MySql." @@ -195,6 +223,10 @@ and TestInfo(args: ParseResults) = let storeLog = createStoreLog <| sargs.Contains Storage.Cosmos.Arguments.VerboseStore log.Information("Running transactions in-process against CosmosDB with storage options: {options:l}", x.Options) storeLog, Storage.Cosmos.config log (cache, x.Unfolds) (Storage.Cosmos.Info sargs) + | Some (Dynamo sargs) -> + let storeLog = createStoreLog <| sargs.Contains Storage.Dynamo.Arguments.VerboseStore + log.Information("Running transactions in-process against DynamoDB with storage options: {options:l}", x.Options) + storeLog, Storage.Dynamo.config log (cache, x.Unfolds) (Storage.Dynamo.Info sargs) | Some (Es sargs) -> let storeLog = createStoreLog <| sargs.Contains Storage.EventStore.Arguments.VerboseStore log.Information("Running transactions in-process against EventStore with storage options: {options:l}", x.Options) @@ -226,6 +258,7 @@ let createStoreLog verbose verboseConsole maybeSeqEndpoint = let c = LoggerConfiguration().Destructure.FSharpTypes() let c = if verbose then c.MinimumLevel.Debug() else c let c = c.WriteTo.Sink(Equinox.CosmosStore.Core.Log.InternalMetrics.Stats.LogSink()) + let c = c.WriteTo.Sink(Equinox.DynamoStore.Core.Log.InternalMetrics.Stats.LogSink()) let c = c.WriteTo.Sink(Equinox.EventStoreDb.Log.InternalMetrics.Stats.LogSink()) let c = c.WriteTo.Sink(Equinox.SqlStreamStore.Log.InternalMetrics.Stats.LogSink()) let level = @@ -242,6 +275,8 @@ let dumpStats storeConfig log = match storeConfig with | Some (Storage.StorageConfig.Cosmos _) -> Equinox.CosmosStore.Core.Log.InternalMetrics.dump log + | Some (Storage.StorageConfig.Dynamo _) -> + Equinox.DynamoStore.Core.Log.InternalMetrics.dump log | Some (Storage.StorageConfig.Es _) -> Equinox.EventStoreDb.Log.InternalMetrics.dump log | Some (Storage.StorageConfig.Sql _) -> @@ -301,6 +336,7 @@ module LoadTest = test, a.Duration, a.TestsPerSecond, clients.Length, a.ErrorCutoff, a.ReportingIntervals, reportFilename) // Reset the start time based on which the shared global metrics will be computed let _ = Equinox.CosmosStore.Core.Log.InternalMetrics.Stats.LogSink.Restart() + let _ = Equinox.DynamoStore.Core.Log.InternalMetrics.Stats.LogSink.Restart() let _ = Equinox.EventStoreDb.Log.InternalMetrics.Stats.LogSink.Restart() let _ = Equinox.SqlStreamStore.Log.InternalMetrics.Stats.LogSink.Restart() let results = runLoadTest log a.TestsPerSecond (duration.Add(TimeSpan.FromSeconds 5.)) a.ErrorCutoff a.ReportingIntervals clients runSingleTest |> Async.RunSynchronously @@ -315,6 +351,7 @@ let createDomainLog verbose verboseConsole maybeSeqEndpoint = let c = LoggerConfiguration().Destructure.FSharpTypes().Enrich.FromLogContext() let c = if verbose then c.MinimumLevel.Debug() else c let c = c.WriteTo.Sink(Equinox.CosmosStore.Core.Log.InternalMetrics.Stats.LogSink()) + let c = c.WriteTo.Sink(Equinox.DynamoStore.Core.Log.InternalMetrics.Stats.LogSink()) let c = c.WriteTo.Sink(Equinox.EventStoreDb.Log.InternalMetrics.Stats.LogSink()) let c = c.WriteTo.Sink(Equinox.SqlStreamStore.Log.InternalMetrics.Stats.LogSink()) let outputTemplate = "{Timestamp:T} {Level:u1} {Message:l} {Properties}{NewLine}{Exception}" @@ -346,6 +383,26 @@ module CosmosInit = return! CosmosInit.init log client (dName, cName) mode skipStoredProc | _ -> failwith "please specify a `cosmos` endpoint" } +module DynamoInit = + + open Equinox.DynamoStore + + let table (log : ILogger) (args : ParseResults) = async { + match args.TryGetSubCommand() with + | Some (TableArguments.Dynamo sa) -> + let info = Storage.Dynamo.Info sa + let client = info.Connector.CreateClient() + let streaming, throughput = (DynamoInitInfo args).ProvisioningMode + let tableName = info.Table + match throughput with + | Some (rcu, wcu) -> + log.Information("Provisioning `Equinox.DynamoStore` Table {table} with {read}/{write}CU; streaming {streaming}", tableName, rcu, wcu, streaming) + do! Core.Initialization.provision client tableName (Throughput.Provisioned (ProvisionedThroughput(rcu, wcu)), streaming) + | None -> + log.Information("Provisioning `Equinox.DynamoStore` Table {table} with On-Demand capacity management; streaming {streaming}", tableName, streaming) + do! Core.Initialization.provision client tableName (Throughput.OnDemand, streaming) + | _ -> failwith "please specify a `dynamo` endpoint" } + module SqlInit = let databaseOrSchema (log: ILogger) (iargs: ParseResults) = async { @@ -464,13 +521,14 @@ let main argv = use log = createDomainLog verbose verboseConsole maybeSeq try match args.GetSubCommand() with | Init iargs -> CosmosInit.containerAndOrDb log iargs |> Async.RunSynchronously + | InitAws targs -> DynamoInit.table log targs |> Async.RunSynchronously | Config cargs -> SqlInit.databaseOrSchema log cargs |> Async.RunSynchronously | Dump dargs -> Dump.run (log, verboseConsole, maybeSeq) dargs | Stats sargs -> CosmosStats.run (log, verboseConsole, maybeSeq) sargs |> Async.RunSynchronously | Run rargs -> let reportFilename = args.GetResult(LogFile, programName + ".log") |> fun n -> System.IO.FileInfo(n).FullName LoadTest.run log (verbose, verboseConsole, maybeSeq) reportFilename rargs - | _ -> failwith "Please specify a valid subcommand :- init, config, dump, stats or run" + | _ -> failwith "Please specify a valid subcommand :- init, initAws, config, dump, stats or run" 0 with e -> log.Debug(e, "Fatal error; exiting"); reraise () with :? ArguParseException as e -> eprintfn "%s" e.Message; 1