diff --git a/.gitignore b/.gitignore index 40a6b1e6..93c5dfdf 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,9 @@ erl_crash.dump .elixir_ls/** +**/**/_build/ +**/**/deps/ + # Temporary Sqlite3 database eigr-functions-db.db diff --git a/.tool-versions b/.tool-versions index dc8fa9ef..e70cf4f2 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ -elixir 1.15 -erlang 25.3.2.8 +elixir 1.17.2 +erlang 26.1.2 diff --git a/Makefile b/Makefile index 28e74698..dcb3d7d6 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,5 @@ -#version=1.4.3 -version=1.4.4-rc.38 -registry=eigr +version=1.4.3 +registry=ghcr.io/eigr CLUSTER_NAME=spawn-k8s K3D_KUBECONFIG_PATH?=./integration.yaml @@ -17,7 +16,7 @@ activator-simple-image=${registry}/spawn-activator-simple:0.1.3 spawn-sdk-example-image=${registry}/spawn-sdk-example:${version} ifeq "$(PROXY_DATABASE_TYPE)" "" - database:=mysql + database:=mariadb else database:=$(PROXY_DATABASE_TYPE) endif @@ -30,67 +29,184 @@ clean: mix deps.clean --all clean-all: - mix deps.clean --all && kind delete cluster --name default + mix deps.clean --all && \ + kind delete cluster --name default + +deps-get-all: + cd spawn_operator/spawn_operator && mix deps.get + cd spawn_sdk/spawn_sdk && mix deps.get + cd spawn_proxy/proxy && mix deps.get + cd spawn_activators/activator && mix deps.get + cd spawn_activators/activator_api && mix deps.get + cd spawn_activators/activator_pubsub && mix deps.get + cd spawn_activators/activator_kafka && mix deps.get + cd spawn_activators/activator_sqs && mix deps.get + cd spawn_activators/activator_rabbitmq && mix deps.get build: - mix deps.get && mix compile + mix deps.get && \ + mix compile clean-cli: - cd spawnctl/ && ./spawnctl maintenance uninstall + cd spawnctl/ && \ + ./spawnctl maintenance uninstall build-cli: - cd spawnctl/ && MIX_ENV=prod mix local.rebar --force && MIX_ENV=prod mix local.hex --force && MIX_ENV=prod mix deps.get && MIX_ENV=prod mix release.init && MIX_ENV=prod mix release spawnctl + cd spawnctl/ && \ + MIX_ENV=prod mix local.rebar --force && \ + MIX_ENV=prod mix local.hex --force && \ + MIX_ENV=prod mix deps.get && \ + MIX_ENV=prod mix release.init && \ + MIX_ENV=prod mix release spawnctl + +create-minikube-cluster: + minikube start + +create-kind-cluster: + kind create cluster -v 1 --name default --config kind-cluster-config.yaml + kubectl cluster-info --context kind-default + +delete-kind-cluster: + kind delete cluster --name default + +load-kind-images: + kind load docker-image ${operator-image} --name default + kind load docker-image ${proxy-image} --name default + kind load docker-image ${activator-api-image} --name default + kind load docker-image ${activator-http-image} --name default + kind load docker-image ${activator-kafka-image} --name default + kind load docker-image ${activator-pubsub-image} --name default + kind load docker-image ${activator-rabbitmq-image} --name default + kind load docker-image ${activator-sqs-image} --name default + +create-k8s-namespace: + kubectl create ns eigr-functions + +generate-k8s-manifests: + cd spawn_operator/spawn_operator && MIX_ENV=prod mix deps.get && MIX_ENV=prod mix bonny.gen.manifest --image ${operator-image} --namespace eigr-functions + +apply-k8s-manifests: + kubectl -n eigr-functions apply -f spawn_operator/spawn_operator/manifest.yaml build-proxy-image: - # When we migrate to new version of buildx we can do: docker buildx build -f Dockerfile-proxy --tag ${proxy-image} --attest type=provenance,mode=max . - docker build --no-cache -f Dockerfile-proxy -t ${proxy-image} . + # When we migrate to a new version of buildx, we can do: + # docker buildx build -f Dockerfile-proxy --tag ${proxy-image} --attest type=provenance,mode=max . + docker build --no-cache \ + -f Dockerfile-proxy \ + -t ${proxy-image} . build-operator-image: - docker build --no-cache -f Dockerfile-operator -t ${operator-image} . + docker build --no-cache \ + -f Dockerfile-operator \ + -t ${operator-image} . build-elixir-sdk-image: - docker build --no-cache -f Dockerfile-elixir-example -t ${spawn-sdk-example-image} . + docker build --no-cache \ + -f Dockerfile-elixir-example \ + -t ${spawn-sdk-example-image} . build-activator-simple-image: - docker build --no-cache -f Dockerfile-activator-simple -t ${activator-simple-image} . + docker build --no-cache \ + -f Dockerfile-activator-simple \ + -t ${activator-simple-image} . build-all-images: - docker build --no-cache -f Dockerfile-proxy -t ${proxy-image} . - docker build --no-cache -f Dockerfile-initializer -t ${proxy-initializer} . - docker build --no-cache -f Dockerfile-operator -t ${operator-image} . - #docker build --no-cache -f Dockerfile-activator-api -t ${activator-api-image} . - #docker build --no-cache -f Dockerfile-activator-kafka -t ${activator-kafka-image} . - #docker build --no-cache -f Dockerfile-activator-pubsub -t ${activator-pubsub-image} . - #docker build --no-cache -f Dockerfile-activator-rabbitmq -t ${activator-rabbitmq-image} . - #docker build --no-cache -f Dockerfile-activator-sqs -t ${activator-sqs-image} . - #docker build --no-cache -f Dockerfile-elixir-example -t ${spawn-sdk-example-image} . + docker build --no-cache \ + -f Dockerfile-proxy \ + -t ${proxy-image} . + docker build --no-cache \ + -f Dockerfile-initializer \ + -t ${proxy-initializer} . + docker build --no-cache \ + -f Dockerfile-operator \ + -t ${operator-image} . + # docker build --no-cache -f Dockerfile-activator-api -t ${activator-api-image} . + # docker build --no-cache -f Dockerfile-activator-kafka -t ${activator-kafka-image} . + # docker build --no-cache -f Dockerfile-activator-pubsub -t ${activator-pubsub-image} . + # docker build --no-cache -f Dockerfile-activator-rabbitmq -t ${activator-rabbitmq-image} . + # docker build --no-cache -f Dockerfile-activator-sqs -t ${activator-sqs-image} . + # docker build --no-cache -f Dockerfile-elixir-example -t ${spawn-sdk-example-image} . + +push-all-images: + docker push ${proxy-image} + docker push ${proxy-initializer} + docker push ${operator-image} + #docker push ${activator-api-image} + #docker push ${activator-http-image} + #docker push ${activator-kafka-image} + #docker push ${activator-pubsub-image} + #docker push ${activator-rabbitmq-image} + #docker push ${activator-sqs-image} + #docker push ${spawn-sdk-example-image} test-spawn: - MIX_ENV=test SPAWN_PROXY_LOGGER_LEVEL=info SPAWN_PROXY_TASK_CONFIG="PMRHIYLTNNAWG5DPOJZSEOS3PMRGCY3UN5ZE4YLNMURDUISKN5ZWKIRMEJ2G64DPNRXWO6JCHJ5SE3TPMRSVGZLMMVRXI33SEI5HWITHOB2SEORCMZQWY43FEJ6X27K5PU======" PROXY_DATABASE_TYPE=native PROXY_DATABASE_PORT=3307 PROXY_DATABASE_POOL_SIZE=15 PROXY_CLUSTER_STRATEGY=gossip PROXY_HTTP_PORT=9005 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= elixir --name spawn@127.0.0.1 -S mix test + MIX_ENV=test \ + PROXY_DATABASE_TYPE=mariadb \ + PROXY_DATABASE_PORT=3307 \ + PROXY_DATABASE_POOL_SIZE=15 \ + PROXY_CLUSTER_STRATEGY=gossip \ + PROXY_HTTP_PORT=9005 \ + SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \ + elixir --name spawn@127.0.0.1 -S mix test + +test-sdk: + cd spawn_sdk/spawn_sdk && \ + mix deps.get && \ + MIX_ENV=test \ + SPAWN_PROXY_LOGGER_LEVEL=info \ + PROXY_CLUSTER_STRATEGY=gossip \ + PROXY_DATABASE_TYPE=mariadb \ + PROXY_DATABASE_PORT=3307 \ + PROXY_DATABASE_POOL_SIZE=50 \ + SPAWN_USE_INTERNAL_NATS=true \ + SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \ + elixir --name spawn@127.0.0.1 -S mix test -test-statestores_mariadb: - cd spawn_statestores/statestores_mariadb && MIX_ENV=test mix deps.get && MIX_ENV=test PROXY_DATABASE_TYPE=mariadb PROXY_DATABASE_PORT=3307 PROXY_CLUSTER_STRATEGY=gossip PROXY_HTTP_PORT=9005 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= elixir --name spawn@127.0.0.1 -S mix test +test-operator: + cd spawn_operator/spawn_operator && MIX_ENV=test mix deps.get && MIX_ENV=test PROXY_DATABASE_TYPE=mysql PROXY_CLUSTER_STRATEGY=gossip PROXY_HTTP_PORT=9005 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= elixir --name spawn@127.0.0.1 -S mix test -test-statestores_postgres: - cd spawn_statestores/statestores_postgres && MIX_ENV=test mix deps.get && MIX_ENV=test PROXY_CLUSTER_STRATEGY=gossip PROXY_HTTP_PORT=9005 PROXY_DATABASE_USERNAME=postgres PROXY_DATABASE_SECRET=postgres SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= elixir --name spawn@127.0.0.1 -S mix test +test-proxy: + cd spawn_proxy/proxy && MIX_ENV=test mix deps.get && MIX_ENV=test SPAWN_PROXY_LOGGER_LEVEL=debug PROXY_DATABASE_TYPE=mariadb PROXY_DATABASE_PORT=3307 PROXY_CLUSTER_STRATEGY=gossip PROXY_HTTP_PORT=9005 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= elixir --name spawn@127.0.0.1 -S mix test test-statestores_native: cd spawn_statestores/statestores_native && MIX_ENV=test mix deps.get && MIX_ENV=test PROXY_CLUSTER_STRATEGY=gossip PROXY_HTTP_PORT=9005 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= elixir --name spawn@127.0.0.1 -S mix test -test-spawn-sdk: - cd spawn_sdk/spawn_sdk && MIX_ENV=test mix deps.get && MIX_ENV=test SPAWN_PROXY_USE_DEFAULT_FLAME_POOL=true PROXY_DATABASE_TYPE=native PROXY_DATABASE_PORT=3307 PROXY_CLUSTER_STRATEGY=gossip PROXY_HTTP_PORT=9005 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= PROXY_ACTOR_SYSTEM_NAME=spawn-system elixir --name spawn_test@127.0.0.1 -S mix test - -test-invoke-backpressure: - cd spawn_sdk/spawn_sdk && MIX_ENV=test mix deps.get && MIX_ENV=test PROXY_DATABASE_TYPE=native PROXY_DATABASE_POOL_SIZE=50 PROXY_DATABASE_PORT=3307 PROXY_CLUSTER_STRATEGY=gossip PROXY_HTTP_PORT=9005 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= PROXY_ACTOR_SYSTEM_NAME=spawn-system elixir --name spawn_test@127.0.0.1 -S mix test ./test/actor/actor_test.exs --only parallel:true - -test-operator: - cd spawn_operator/spawn_operator && MIX_ENV=test mix deps.get && MIX_ENV=test PROXY_DATABASE_TYPE=native PROXY_CLUSTER_STRATEGY=gossip PROXY_HTTP_PORT=9005 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= elixir --name spawn@127.0.0.1 -S mix test +test-statestores_mariadb: + cd spawn_statestores/statestores_mariadb && \ + MIX_ENV=test mix deps.get && \ + MIX_ENV=test \ + PROXY_DATABASE_TYPE=mariadb \ + PROXY_DATABASE_PORT=3307 \ + PROXY_DATABASE_USERNAME=admin \ + PROXY_DATABASE_SECRET=admin \ + PROXY_CLUSTER_STRATEGY=gossip \ + PROXY_HTTP_PORT=9005 \ + SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \ + elixir --name spawn@127.0.0.1 -S mix test -test-proxy: - cd spawn_proxy/proxy && MIX_ENV=test mix deps.get && MIX_ENV=test SPAWN_PROXY_LOGGER_LEVEL=debug PROXY_DATABASE_TYPE=native PROXY_DATABASE_PORT=3307 PROXY_CLUSTER_STRATEGY=gossip PROXY_HTTP_PORT=9005 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= elixir --name spawn@127.0.0.1 -S mix test +test-statestores_postgres: + cd spawn_statestores/statestores_postgres && \ + MIX_ENV=test mix deps.get && \ + MIX_ENV=test \ + PROXY_CLUSTER_STRATEGY=gossip \ + PROXY_HTTP_PORT=9005 \ + PROXY_DATABASE_TYPE=postgres \ + PROXY_DATABASE_PORT=5432 \ + PROXY_DATABASE_USERNAME=postgres \ + PROXY_DATABASE_SECRET=postgres \ + SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \ + elixir --name spawn@127.0.0.1 -S mix test -run-benchmark: - cd spawn_sdk/spawn_sdk_example && mix deps.get && SPAWN_PROXY_LOGGER_LEVEL=info PROXY_CLUSTER_STRATEGY=gossip PROXY_DATABASE_TYPE=mariadb PROXY_DATABASE_PORT=3307 PROXY_DATABASE_POOL_SIZE=50 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= iex --name spawn@127.0.0.1 -S mix run benchmark.exs +test-invoke-backpressure: + cd spawn_sdk/spawn_sdk_example && \ + mix deps.get && \ + SPAWN_PROXY_LOGGER_LEVEL=info \ + PROXY_CLUSTER_STRATEGY=gossip \ + PROXY_DATABASE_TYPE=mariadb \ + PROXY_DATABASE_PORT=3307 \ + PROXY_DATABASE_POOL_SIZE=50 \ + SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \ + iex --name spawn@127.0.0.1 -S mix test test/invoke_backpressure_test.exs integration.yaml: ## Create a k3d cluster - k3d cluster delete ${CLUSTER_NAME} @@ -98,122 +214,187 @@ integration.yaml: ## Create a k3d cluster k3d kubeconfig get ${CLUSTER_NAME} > ${K3D_KUBECONFIG_PATH} sleep 5 -.PHONY: test.integration - test.integration: integration.yaml test.integration: ## Run integration tests using k3d `make cluster` cd spawn_operator/spawn_operator && PROXY_CLUSTER_STRATEGY=gossip PROXY_DATABASE_TYPE=mysql PROXY_DATABASE_POOL_SIZE=10 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= TEST_KUBECONFIG=${K3D_KUBECONFIG_PATH} mix test --only integration -push-all-images: - docker push ${proxy-image} - docker push ${proxy-initializer} - docker push ${operator-image} - #docker push ${activator-api-image} - #docker push ${activator-http-image} - #docker push ${activator-kafka-image} - #docker push ${activator-pubsub-image} - #docker push ${activator-rabbitmq-image} - #docker push ${activator-sqs-image} - #docker push ${spawn-sdk-example-image} - -create-minikube-cluster: - minikube start - -create-kind-cluster: - kind create cluster -v 1 --name default --config kind-cluster-config.yaml - kubectl cluster-info --context kind-default - -delete-kind-cluster: - kind delete cluster --name default - -load-kind-images: - kind load docker-image ${operator-image} --name default - kind load docker-image ${proxy-image} --name default - kind load docker-image ${activator-api-image} --name default - kind load docker-image ${activator-http-image} --name default - kind load docker-image ${activator-kafka-image} --name default - kind load docker-image ${activator-pubsub-image} --name default - kind load docker-image ${activator-rabbitmq-image} --name default - kind load docker-image ${activator-sqs-image} --name default - -create-k8s-namespace: - kubectl create ns eigr-functions - -generate-k8s-manifests: - cd spawn_operator/spawn_operator && MIX_ENV=prod mix deps.get && MIX_ENV=prod mix bonny.gen.manifest --image ${operator-image} --namespace eigr-functions - -apply-k8s-manifests: - kubectl -n eigr-functions apply -f spawn_operator/spawn_operator/manifest.yaml +run-benchmark: + cd spawn_sdk/spawn_sdk_example && \ + mix deps.get && \ + SPAWN_PROXY_LOGGER_LEVEL=info \ + PROXY_CLUSTER_STRATEGY=gossip \ + PROXY_DATABASE_TYPE=mariadb \ + PROXY_DATABASE_PORT=3307 \ + PROXY_DATABASE_POOL_SIZE=50 \ + SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \ + iex --name spawn@127.0.0.1 -S mix run benchmark.exs run-proxy-local: - ERL_ZFLAGS='-proto_dist inet_tls -ssl_dist_optfile rel/overlays/local-mtls.ssl.conf' cd spawn_proxy/proxy && mix deps.get && PROXY_DATABASE_TYPE=$(database) PROXY_HTTP_PORT=9002 USER_FUNCTION_PORT=8091 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= iex --name spawn_a1@test.default.svc -S mix + ERL_ZFLAGS='-proto_dist inet_tls -ssl_dist_optfile rel/overlays/local-mtls.ssl.conf' \ + cd spawn_proxy/proxy && mix deps.get && \ + PROXY_DATABASE_TYPE=$(database) \ + PROXY_HTTP_PORT=9002 \ + USER_FUNCTION_PORT=8091 \ + SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \ + iex --name spawn_a1@test.default.svc -S mix run-proxy-local2: - ERL_ZFLAGS='-proto_dist inet_tls -ssl_dist_optfile rel/overlays/local-mtls.ssl.conf' cd spawn_proxy/proxy && mix deps.get && PROXY_DATABASE_TYPE=$(database) PROXY_HTTP_PORT=9003 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= iex --name spawn_a2@test.default.svc -S mix + ERL_ZFLAGS='-proto_dist inet_tls -ssl_dist_optfile rel/overlays/local-mtls.ssl.conf' \ + cd spawn_proxy/proxy && mix deps.get && \ + PROXY_DATABASE_TYPE=$(database) \ + PROXY_HTTP_PORT=9003 \ + SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \ + iex --name spawn_a2@test.default.svc -S mix run-proxy-local-3: - cd spawn_proxy/proxy && mix deps.get && OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4317 OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc OTEL_EXPORTER_OTLP_TRACES_COMPRESSION=gzip PROXY_CLUSTER_STRATEGY=epmd SPAWN_USE_INTERNAL_NATS=false SPAWN_PUBSUB_ADAPTER=native PROXY_DATABASE_PORT=3307 PROXY_DATABASE_TYPE=native PROXY_DATABASE_POOL_SIZE=30 PROXY_HTTP_PORT=9001 USER_FUNCTION_PORT=8090 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= iex --name spawn_a3@127.0.0.1 -S mix + cd spawn_proxy/proxy && mix deps.get && \ + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4317 \ + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc \ + OTEL_EXPORTER_OTLP_TRACES_COMPRESSION=gzip \ + SPAWN_PROXY_LOGGER_LEVEL=info \ + PROXY_CLUSTER_STRATEGY=epmd \ + SPAWN_USE_INTERNAL_NATS=false \ + SPAWN_PUBSUB_ADAPTER=native \ + PROXY_DATABASE_PORT=3307 \ + PROXY_DATABASE_TYPE=native \ + PROXY_DATABASE_POOL_SIZE=30 \ + PROXY_HTTP_PORT=9001 \ + USER_FUNCTION_PORT=8090 \ + SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \ + iex --name spawn_a3@127.0.0.1 -S mix + +run-proxy-with-postgres: + cd spawn_proxy/proxy && mix deps.get && \ + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4317 \ + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc \ + OTEL_EXPORTER_OTLP_TRACES_COMPRESSION=gzip \ + SPAWN_PROXY_LOGGER_LEVEL=info \ + PROXY_CLUSTER_STRATEGY=epmd \ + SPAWN_USE_INTERNAL_NATS=false \ + SPAWN_PUBSUB_ADAPTER=native \ + PROXY_DATABASE_PORT=5432 \ + PROXY_DATABASE_TYPE=postgres \ + PROXY_DATABASE_USERNAME=postgres \ + PROXY_DATABASE_SECRET=postgres \ + PROXY_DATABASE_POOL_SIZE=30 \ + PROXY_HTTP_PORT=9001 \ + USER_FUNCTION_PORT=8090 \ + SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \ + iex --name spawn_a3@127.0.0.1 -S mix run-proxy-local-nodejs-test: - ERL_ZFLAGS='-proto_dist inet_tls -ssl_dist_optfile rel/overlays/local-mtls.ssl.conf' cd spawn_proxy/proxy && mix deps.get && PROXY_DATABASE_TYPE=$(database) PROXY_HTTP_PORT=9001 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= PROXY_ACTOR_SYSTEM_NAME=SpawnSysTest SPAWN_SUPERVISORS_STATE_HANDOFF_CONTROLLER=crdt iex --name spawn_a1@test.default.svc -S mix + ERL_ZFLAGS='-proto_dist inet_tls -ssl_dist_optfile rel/overlays/local-mtls.ssl.conf' \ + cd spawn_proxy/proxy && mix deps.get && \ + PROXY_DATABASE_TYPE=$(database) \ + PROXY_HTTP_PORT=9001 \ + SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \ + PROXY_ACTOR_SYSTEM_NAME=SpawnSysTest \ + SPAWN_SUPERVISORS_STATE_HANDOFF_CONTROLLER=crdt \ + iex --name spawn_a1@test.default.svc -S mix run-proxy-local-dicegame: - cd spawn_proxy/proxy && mix deps.get && PROXY_DATABASE_TYPE=$(database) PROXY_CLUSTER_STRATEGY=epmd PROXY_HTTP_PORT=9001 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= PROXY_ACTOR_SYSTEM_NAME=game-system SPAWN_SUPERVISORS_STATE_HANDOFF_CONTROLLER=persistent SPAWN_USE_INTERNAL_NATS=true SPAWN_PUBSUB_ADAPTER=nats iex --name spawn_a4@127.0.0.1 -S mix + cd spawn_proxy/proxy && mix deps.get && \ + PROXY_DATABASE_TYPE=$(database) \ + PROXY_CLUSTER_STRATEGY=epmd \ + PROXY_HTTP_PORT=9001 \ + SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \ + PROXY_ACTOR_SYSTEM_NAME=game-system \ + SPAWN_SUPERVISORS_STATE_HANDOFF_CONTROLLER=persistent \ + SPAWN_USE_INTERNAL_NATS=true \ + SPAWN_PUBSUB_ADAPTER=nats \ + iex --name spawn_a4@127.0.0.1 -S mix run-sdk-local: - cd spawn_sdk/spawn_sdk_example && mix deps.get && PROXY_CLUSTER_STRATEGY=gossip PROXY_DATABASE_TYPE=$(database) SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= iex --name spawn_actors_node@127.0.0.1 -S mix + cd spawn_sdk/spawn_sdk_example && mix deps.get && \ + PROXY_CLUSTER_STRATEGY=gossip \ + PROXY_DATABASE_TYPE=$(database) \ + SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \ + iex --name spawn_actors_node@127.0.0.1 -S mix run-sdk-local2: - cd spawn_sdk/spawn_sdk_example && mix deps.get && PROXY_CLUSTER_STRATEGY=epmd PROXY_DATABASE_POOL_SIZE=5 SPAWN_USE_INTERNAL_NATS=true SPAWN_PUBSUB_ADAPTER=nats PROXY_DATABASE_TYPE=$(database) SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= iex --name spawn_a2@127.0.0.1 -S mix + cd spawn_sdk/spawn_sdk_example && mix deps.get && \ + PROXY_CLUSTER_STRATEGY=epmd \ + PROXY_DATABASE_POOL_SIZE=5 \ + SPAWN_USE_INTERNAL_NATS=true \ + SPAWN_PUBSUB_ADAPTER=nats \ + PROXY_DATABASE_TYPE=$(database) \ + SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \ + iex --name spawn_a2@127.0.0.1 -S mix run-sdk-local3: - cd spawn_sdk/spawn_sdk_example && mix deps.get && PROXY_CLUSTER_STRATEGY=epmd SPAWN_USE_INTERNAL_NATS=true SPAWN_PUBSUB_ADAPTER=nats PROXY_DATABASE_TYPE=mariadb PROXY_DATABASE_PORT=3307 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= iex --name spawn_a3@127.0.0.1 -S mix + cd spawn_sdk/spawn_sdk_example && mix deps.get && \ + PROXY_CLUSTER_STRATEGY=epmd \ + SPAWN_USE_INTERNAL_NATS=true \ + SPAWN_PUBSUB_ADAPTER=nats \ + PROXY_DATABASE_TYPE=mariadb \ + PROXY_DATABASE_PORT=3307 \ + SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \ + iex --name spawn_a3@127.0.0.1 -S mix run-sdk-local-with-mariadb: - cd spawn_sdk/spawn_sdk_example && mix deps.get && PROXY_CLUSTER_STRATEGY=epmd SPAWN_USE_INTERNAL_NATS=true SPAWN_PUBSUB_ADAPTER=nats PROXY_DATABASE_PORT=3307 PROXY_DATABASE_TYPE=mariadb SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= iex --name spawn_a3@127.0.0.1 -S mix + cd spawn_sdk/spawn_sdk_example && mix deps.get && \ + PROXY_CLUSTER_STRATEGY=epmd \ + SPAWN_USE_INTERNAL_NATS=true \ + SPAWN_PUBSUB_ADAPTER=nats \ + PROXY_DATABASE_PORT=3307 \ + PROXY_DATABASE_TYPE=mariadb \ + SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \ + iex --name spawn_a3@127.0.0.1 -S mix run-sdk-local-nats: - cd spawn_sdk/spawn_sdk_example && mix deps.get && PROXY_CLUSTER_STRATEGY=epmd PROXY_DATABASE_TYPE=$(database) SPAWN_USE_INTERNAL_NATS=true SPAWN_PUBSUB_ADAPTER=nats SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= iex --name spawn_a3@127.0.0.1 -S mix + cd spawn_sdk/spawn_sdk_example && mix deps.get && \ + PROXY_CLUSTER_STRATEGY=epmd \ + PROXY_DATABASE_TYPE=$(database) \ + SPAWN_USE_INTERNAL_NATS=true \ + SPAWN_PUBSUB_ADAPTER=nats \ + SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \ + iex --name spawn_a3@127.0.0.1 -S mix run-operator-local: - cd spawn_operator/spawn_operator && mix deps.get && MIX_ENV=dev BONNY_POD_NAME=spawn-operator iex --name operator@127.0.0.1 -S mix - + cd spawn_operator/spawn_operator && mix deps.get && \ + MIX_ENV=dev \ + BONNY_POD_NAME=spawn-operator \ + iex --name operator@127.0.0.1 -S mix + run-activator-local: - cd spawn_activators/activator && mix deps.get && MIX_ENV=dev iex --name activator@127.0.0.1 -S mix + cd spawn_activators/activator && mix deps.get && \ + MIX_ENV=dev \ + iex --name activator@127.0.0.1 -S mix run-activator-grpc-local: - cd spawn_activators/activator_api && mix deps.get && MIX_ENV=dev iex --name activator_api@127.0.0.1 -S mix + cd spawn_activators/activator_api && mix deps.get && \ + MIX_ENV=dev \ + iex --name activator_api@127.0.0.1 -S mix run-activator-kafka-local: - cd spawn_activators/activator_kafka && mix deps.get && MIX_ENV=dev iex --name activator_kafka@127.0.0.1 -S mix + cd spawn_activators/activator_kafka && mix deps.get && \ + MIX_ENV=dev \ + iex --name activator_kafka@127.0.0.1 -S mix run-activator-pubsub-local: - cd spawn_activators/activator_pubsub && mix deps.get && MIX_ENV=dev iex --name activator_pubsub@127.0.0.1 -S mix + cd spawn_activators/activator_pubsub && mix deps.get && \ + MIX_ENV=dev \ + iex --name activator_pubsub@127.0.0.1 -S mix run-activator-rabbitmq-local: - cd spawn_activators/activator_rabbitmq && mix deps.get && MIX_ENV=dev iex --name activator_rabbitmq@127.0.0.1 -S mix + cd spawn_activators/activator_rabbitmq && mix deps.get && \ + MIX_ENV=dev \ + iex --name activator_rabbitmq@127.0.0.1 -S mix run-activator-sqs-local: - cd spawn_activators/activator_sqs && mix deps.get && MIX_ENV=dev iex --name activator_sqs@127.0.0.1 -S mix - -run-deps-get-all: - cd spawn_operator/spawn_operator && mix deps.get - cd spawn_sdk/spawn_sdk && mix deps.get - cd spawn_proxy/proxy && mix deps.get - cd spawn_activators/activator && mix deps.get - cd spawn_activators/activator_api && mix deps.get - cd spawn_activators/activator_pubsub && mix deps.get - cd spawn_activators/activator_kafka && mix deps.get - cd spawn_activators/activator_sqs && mix deps.get - cd spawn_activators/activator_rabbitmq && mix deps.get + cd spawn_activators/activator_sqs && mix deps.get && \ + MIX_ENV=dev \ + iex --name activator_sqs@127.0.0.1 -S mix run-proxy-image: - docker run --rm --name=spawn-proxy -e PROXY_DATABASE_TYPE=mysql -e SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= --net=host ${proxy-image} + docker run --rm --name=spawn-proxy \ + -e PROXY_DATABASE_TYPE=mysql \ + -e SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \ + --net=host ${proxy-image} run-operator-image: docker run --rm --name=spawn-operator --net=host ${operator-image} run-initializer-image: docker run --rm --name=spawn-operator --net=host ${proxy-initializer} - diff --git a/compile-pb.sh b/compile-pb.sh index 8b87e3c2..9fc424fa 100755 --- a/compile-pb.sh +++ b/compile-pb.sh @@ -51,4 +51,5 @@ for file in $PROTOS; do --plugin=ProtobufGenerate.Plugins.GRPCWithOptions \ --one-file-per-module \ $BASE_PATH/$file -done \ No newline at end of file +done + diff --git a/config/config.exs b/config/config.exs index 1b2a855f..08886a79 100644 --- a/config/config.exs +++ b/config/config.exs @@ -38,19 +38,8 @@ config :logger, :console, config :protobuf, extensions: :enabled -# config :prometheus, MetricsEndpoint.Exporter, -# path: "/metrics", -# format: :auto, -# registry: :default, -# auth: false - config :opentelemetry, :resource, service: %{name: "spawn"} -# config :opentelemetry, -# span_processor: :batch, -# traces_exporter: {:otel_exporter_stdout, []} -# #traces_exporter: {:otel_exporter_stdout, []} - config :opentelemetry, :processors, otel_batch_processor: %{ diff --git a/config/runtime.exs b/config/runtime.exs index 7e26aa5b..17826087 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -4,3 +4,9 @@ if config_env() == :prod do config :logger, level: String.to_atom(System.get_env("SPAWN_PROXY_LOGGER_LEVEL", "info")) end + +# For OTLP set the following variables: + +# OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:55681 +# OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc +# OTEL_EXPORTER_OTLP_TRACES_COMPRESSION=gzip diff --git a/config/test.exs b/config/test.exs index aafc2ec7..beef3027 100644 --- a/config/test.exs +++ b/config/test.exs @@ -1,6 +1,14 @@ import Config -config :spawn_statestores, Statestores.Adapters.MySQLSnapshotAdapter, +config :spawn_statestores, Statestores.Adapters.MariaDBSnapshotAdapter, + pool: Ecto.Adapters.SQL.Sandbox, + ownership_timeout: :infinity, + pool_size: 24, + prepare: :unnamed, + queue_target: 5_000, + queue_interval: 500 + +config :spawn_statestores, Statestores.Adapters.MariaDBProjectionAdapter, pool: Ecto.Adapters.SQL.Sandbox, ownership_timeout: :infinity, pool_size: 24, diff --git a/docker-compose.yaml b/docker-compose.yaml index ad7ea9ed..b4ac5d51 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -8,6 +8,7 @@ services: ports: - 4317:4317 - 4318:4318 + - 55681:55681 volumes: - ./config/otel-collector-config.yaml:/conf/otel-collector-config.yaml links: @@ -57,6 +58,8 @@ services: - mariadb:/var/lib/mysql ports: - "3307:3306" + networks: + - mysql-compose-network adminer: image: adminer @@ -69,25 +72,24 @@ services: image: 'nats:latest' ports: - "4222:4222" - - spawn-proxy: - image: eigr/spawn-proxy:1.4.3 - restart: always - environment: - PROXY_APP_NAME: spawn - PROXY_HTTP_PORT: 9001 - PROXY_DATABASE_TYPE: postgres - PROXY_DATABASE_NAME: eigr-functions-db - PROXY_DATABASE_USERNAME: postgres - PROXY_DATABASE_SECRET: password - PROXY_DATABASE_HOST: localhost - PROXY_DATABASE_PORT: 5432 - SPAWN_STATESTORE_KEY: 3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= - USER_FUNCTION_HOST: 0.0.0.0 # Your NodeJS runtime host - USER_FUNCTION_PORT: 8090 # Your NodeJS runtime exposed port - # network_mode: host # only uncomment this if you're running your nodejs locally in Linux, check note below for Windows - ports: - - "9001:9001" + # spawn-proxy: + # image: eigr/spawn-proxy:1.4.3 + # restart: always + # environment: + # PROXY_APP_NAME: spawn + # PROXY_HTTP_PORT: 9001 + # PROXY_DATABASE_TYPE: postgres + # PROXY_DATABASE_NAME: eigr-functions-db + # PROXY_DATABASE_USERNAME: postgres + # PROXY_DATABASE_SECRET: password + # PROXY_DATABASE_HOST: localhost + # PROXY_DATABASE_PORT: 5432 + # SPAWN_STATESTORE_KEY: 3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= + # USER_FUNCTION_HOST: 0.0.0.0 # Your NodeJS runtime host + # USER_FUNCTION_PORT: 8090 # Your NodeJS runtime exposed port + # # network_mode: host # only uncomment this if you're running your nodejs locally in Linux, check note below for Windows + # ports: + # - "9001:9001" networks: mysql-compose-network: diff --git a/lib/actors/actor/entity/entity.ex b/lib/actors/actor/entity/entity.ex index 5d2e8e7f..c791c509 100644 --- a/lib/actors/actor/entity/entity.ex +++ b/lib/actors/actor/entity/entity.ex @@ -418,6 +418,14 @@ defmodule Actors.Actor.Entity do handle_invocation_request(invocation, opts, nil, state) |> reply_to_noreply() + {:process_projection_events, events} -> + Invocation.process_projection_events(events, state) + |> reply_to_noreply() + + {:replay, opts} -> + Invocation.replay(opts, state) + |> reply_to_noreply() + action -> do_handle_cast(action, state) end @@ -536,6 +544,19 @@ defmodule Actors.Actor.Entity do GenServer.call(via(ref), :get_state, timeout) end + @doc """ + When the Actor is a Projection the messages sent to the projection can be reprocessed. + See this for more information about this programming model. + """ + @spec replay(pid() | module(), Keyword.t()) :: {:error, term()} | :ok + def replay(ref, opts) when is_pid(ref) do + GenServer.cast(ref, {:replay, opts}) + end + + def replay(ref, opts) do + GenServer.cast(via(ref), {:replay, opts}) + end + @doc """ Retrieve the health check readiness status. """ diff --git a/lib/actors/actor/entity/invocation.ex b/lib/actors/actor/entity/invocation.ex index ade6b740..31ea9226 100644 --- a/lib/actors/actor/entity/invocation.ex +++ b/lib/actors/actor/entity/invocation.ex @@ -6,17 +6,22 @@ defmodule Actors.Actor.Entity.Invocation do require Logger require OpenTelemetry.Tracer, as: Tracer - alias Actors.Actor.Entity.{EntityState, Lifecycle} - alias Actors.Exceptions.NotAuthorizedException + alias Actors.Actor.Entity.EntityState + alias Actors.Actor.Entity.Lifecycle + alias Actors.Actor.Entity.Lifecycle.StreamInitiator alias Actors.Actor.InvocationScheduler + alias Actors.Exceptions.NotAuthorizedException + alias Actors.Actor.Pubsub alias Eigr.Functions.Protocol.Actors.{ Actor, ActorId, + ActorSettings, ActorSystem, ActorState, Action, - FixedTimerAction + FixedTimerAction, + Metadata } alias Eigr.Functions.Protocol.{ @@ -24,6 +29,7 @@ defmodule Actors.Actor.Entity.Invocation do ActorInvocationResponse, Broadcast, Context, + Fact, Forward, InvocationRequest, Pipe, @@ -32,8 +38,9 @@ defmodule Actors.Actor.Entity.Invocation do Noop } - alias Actors.Actor.Pubsub + alias Spawn.Utils.Nats + import Spawn.Utils.AnySerializer, only: [any_pack!: 1, unpack_any_bin: 1] import Spawn.Utils.Common, only: [return_and_maybe_hibernate: 1] @default_actions [ @@ -53,6 +60,79 @@ defmodule Actors.Actor.Entity.Invocation do @http_host_interface Actors.Actor.Interface.Http + def process_projection_events(messages, state) do + %EntityState{actor: %Actor{} = actor} = state + + invocations = + messages + |> Enum.map(fn %Broadway.Message{data: %Fact{} = message} -> + system_name = Map.get(message.metadata, "spawn-system") + parent = Map.get(message.metadata, "actor-parent") + name = Map.get(message.metadata, "actor-name") + action = Map.get(message.metadata, "actor-action") + + %InvocationRequest{ + async: true, + system: %ActorSystem{name: system_name}, + actor: %Actor{id: actor.id}, + metadata: message.metadata, + action_name: action, + payload: parse_payload(unpack_any_bin(message.state)), + caller: %ActorId{name: name, system: system_name, parent: parent} + } + end) + + spawn(fn -> + invocations + |> Flow.from_enumerable(min_demand: 1, max_demand: System.schedulers_online()) + |> Flow.map(fn invocation -> + try do + Actors.invoke(invocation, span_ctx: Tracer.current_span_ctx()) + catch + error -> + Logger.warning( + "Error during processing events on projection. Invocation: #{inspect(invocation)} Error: #{inspect(error)}" + ) + + :ok + end + end) + |> Flow.run() + end) + + {:noreply, state} + end + + defp parse_payload(response) do + case response do + nil -> {:noop, %Noop{}} + %Noop{} = noop -> {:noop, noop} + {:noop, %Noop{} = noop} -> {:noop, noop} + {_, nil} -> {:noop, %Noop{}} + {:value, response} -> {:value, any_pack!(response)} + response -> {:value, any_pack!(response)} + end + end + + def replay( + call_opts, + %EntityState{ + actor: + %Actor{ + settings: + %ActorSettings{ + kind: :PROJECTION + } = _settings + } = actor, + projection_stream_pid: stream_pid + } = state + ) do + {:ok, newpid} = StreamInitiator.replay(stream_pid, actor, call_opts) + {:noreply, %{state | projection_stream_pid: newpid}} + end + + def replay(_replaymsg, _call_opts, state), do: {:noreply, state} + def handle_timers([], _system, _actor), do: :ok def handle_timers(timers, system, actor) when is_list(timers) do @@ -289,12 +369,31 @@ defmodule Actors.Actor.Entity.Invocation do request, %ActorInvocationResponse{checkpoint: checkpoint} = response, %EntityState{ + actor: + %Actor{ + id: id, + settings: + %ActorSettings{ + kind: kind, + projection_settings: projection_settings + } = _settings + } = _actor, revision: revision } = state, opts ) do + response_params = %{ + actor_id: id, + kind: kind, + projection_settings: projection_settings, + request: request, + response: response, + state: state, + opts: opts + } + response = - case do_response(request, response, state, opts) do + case do_response(response_params) do :noreply -> {:noreply, state} |> return_and_maybe_hibernate() @@ -400,19 +499,59 @@ defmodule Actors.Actor.Entity.Invocation do end defp do_response( - _request, - %ActorInvocationResponse{workflow: workflow} = response, - _state, - _opts + %{ + actor_id: id, + kind: kind, + projection_settings: settings, + request: request, + response: %ActorInvocationResponse{workflow: workflow} = response, + state: state, + opts: _opts + } = _params ) when is_nil(workflow) or workflow == %{} do + :ok = do_handle_projection(id, request.action_name, settings, state) + response end - defp do_response(request, response, state, opts) do + defp do_response( + %{ + actor_id: id, + kind: kind, + projection_settings: settings, + request: request, + response: response, + state: state, + opts: opts + } = _params + ) do + :ok = do_handle_projection(id, request.action_name, settings, state) + do_run_workflow(request, response, state, opts) end + defp do_handle_projection(id, action, %{sourceable: true} = _settings, state) do + actor_name_or_parent = if id.parent == "", do: id.name, else: id.parent + + subject = "actors.#{actor_name_or_parent}.#{id.name}.#{action}" + payload = Google.Protobuf.Any.encode(state.actor.state.state) + + uuid = UUID.uuid4(:hex) + + Gnat.pub(Nats.connection_name(), subject, payload, + headers: [ + {"Nats-Msg-Id", uuid}, + {"Spawn-System", "#{id.system}"}, + {"Actor-Parent", "#{id.parent}"}, + {"Actor-Name", "#{id.name}"}, + {"Actor-Action", "#{action}"} + ] + ) + end + + defp do_handle_projection(_id, _action, _settings, _state), do: :ok + defp do_run_workflow( _request, %ActorInvocationResponse{workflow: workflow} = response, diff --git a/lib/actors/actor/entity/lifecycle.ex b/lib/actors/actor/entity/lifecycle.ex index 1b363bde..0ec7ab66 100644 --- a/lib/actors/actor/entity/lifecycle.ex +++ b/lib/actors/actor/entity/lifecycle.ex @@ -5,9 +5,12 @@ defmodule Actors.Actor.Entity.Lifecycle do """ require Logger - alias Actors.Actor.{Entity.EntityState, Entity.Invocation, StateManager} - alias Actors.Actor.Pubsub + alias Actors.Actor.Entity.EntityState + alias Actors.Actor.Entity.Invocation + alias Actors.Actor.Entity.Lifecycle.StreamInitiator alias Actors.Exceptions.NetworkPartitionException + alias Actors.Actor.Pubsub + alias Actors.Actor.StateManager alias Eigr.Functions.Protocol.Actors.{ Actor, @@ -32,18 +35,19 @@ defmodule Actors.Actor.Entity.Lifecycle do def init( %EntityState{ system: system, - actor: %Actor{ - id: %ActorId{name: name, parent: parent} = _id, - metadata: metadata, - settings: - %ActorSettings{ - stateful: stateful?, - snapshot_strategy: snapshot_strategy, - deactivation_strategy: deactivation_strategy, - kind: kind - } = _settings, - timer_actions: timer_actions - } + actor: + %Actor{ + id: %ActorId{name: name, parent: parent} = _id, + metadata: metadata, + settings: + %ActorSettings{ + stateful: stateful?, + snapshot_strategy: snapshot_strategy, + deactivation_strategy: deactivation_strategy, + kind: kind + } = _settings, + timer_actions: timer_actions + } = actor } = state ) do Process.flag(:trap_exit, true) @@ -70,7 +74,7 @@ defmodule Actors.Actor.Entity.Lifecycle do Actors.Actor.Entity, actor_name_key, self(), - state.actor.id + actor.id ) schedule_deactivate(deactivation_strategy, get_jitter()) @@ -84,7 +88,8 @@ defmodule Actors.Actor.Entity.Lifecycle do Keyword.merge(state.opts, timer: timer, split_brain_detector: split_brain_detector_mod - ) + ), + projection_stream_pid: maybe_init_projection(actor) } _ -> @@ -93,7 +98,8 @@ defmodule Actors.Actor.Entity.Lifecycle do | opts: Keyword.merge(state.opts, split_brain_detector: split_brain_detector_mod - ) + ), + projection_stream_pid: maybe_init_projection(actor) } end @@ -286,6 +292,30 @@ defmodule Actors.Actor.Entity.Lifecycle do # Private functions + defp maybe_init_projection(actor) do + case handle_projection(actor) do + {:ok, pid} when is_pid(pid) -> + pid + + _otherwise -> + nil + end + end + + defp handle_projection(%Actor{settings: %ActorSettings{kind: :PROJECTION}} = actor) do + StreamInitiator.init_projection_stream(actor) + end + + defp handle_projection( + %Actor{settings: %ActorSettings{kind: kind, projection_settings: %{sourceable: true}}} = + actor + ) + when kind in [:NAMED, :UNNAMED] do + StreamInitiator.init_sourceable_stream(actor) + end + + defp handle_projection(_actor), do: :ok + defp updated_state(%EntityState{actor: actor} = state, actual_state, revision) do %EntityState{state | actor: %Actor{actor | state: actual_state}, revision: revision} end diff --git a/lib/actors/actor/entity/lifecycle/stream_consumer.ex b/lib/actors/actor/entity/lifecycle/stream_consumer.ex new file mode 100644 index 00000000..b4414be7 --- /dev/null +++ b/lib/actors/actor/entity/lifecycle/stream_consumer.ex @@ -0,0 +1,94 @@ +defmodule Actors.Actor.Entity.Lifecycle.StreamConsumer do + @moduledoc false + use Broadway + + alias Broadway.Message + alias Spawn.Utils.Nats + alias Eigr.Functions.Protocol.Fact + alias Google.Protobuf.Timestamp + + @type fact :: %Fact{} + + @type opts :: %{ + projection_pid: pid(), + actor_name: String.t(), + strict_ordering: boolean() + } + + @spec start_link(opts :: opts()) :: :ignore | {:error, any()} | {:ok, pid()} + def start_link(opts) do + Broadway.start_link( + __MODULE__, + # there will be not a lot so probably fine to convert to atom + name: String.to_atom(opts.actor_name), + context: opts, + producer: [ + module: { + OffBroadway.Jetstream.Producer, + connection_name: Nats.connection_name(), + stream_name: opts.actor_name, + consumer_name: opts.actor_name + }, + concurrency: build_concurrency(opts) + ], + processors: [ + default: [concurrency: build_concurrency(opts)] + ], + batchers: [ + default: [ + concurrency: build_concurrency(opts), + # Avoi big batches, micro batches is better + batch_size: 10, + batch_timeout: 2_000 + ] + ] + ) + end + + @spec handle_message(any(), Message.t(), any()) :: Message.t() + def handle_message(_processor_name, message, _context) do + message + |> build_fact() + |> Message.configure_ack(on_success: :term) + end + + @spec handle_batch(any(), Message.t(), any(), opts()) :: list(Message.t()) + def handle_batch(_, messages, _, context) do + GenServer.cast(context.projection_pid, {:process_projection_events, messages}) + + messages + end + + @spec build_fact(Message.t()) :: Message.t() + defp build_fact(message) do + # %Broadway.Message{data: "{\"ACTION\":\"KEY_ADDED\",\"KEY\":\"MYKEY\",\"VALUE\":\"MYVALUE\"}", metadata: %{headers: [], topic: "actors.mike"}, acknowledger: {OffBroadway.Jetstream.Acknowledger, #Reference<0.743380651.807927811.227242>, %{on_success: :term, reply_to: "$JS.ACK.newtest.projectionviewertest.1.11.11.1725657673932595345.21"}}, batcher: :default, batch_key: :default, batch_mode: :bulk, status: :ok} + + message + |> Message.put_data(process_data(message)) + end + + @spec process_data(Message.t()) :: fact() + defp process_data(message) do + payload = message.data + + metadata = + Enum.reduce(message.metadata.headers, %{}, fn {key, value}, acc -> + Map.put(acc, key, value) + end) + |> Map.put("topic", message.metadata.topic) + + time = DateTime.utc_now() |> DateTime.to_unix(:seconds) + + %Fact{ + uuid: UUID.uuid4(:hex), + metadata: metadata, + state: payload, + timestamp: %Timestamp{seconds: time} + } + end + + # Projections are like long-lasting threads and therefore concurrency should be avoided + # if the intention is to have some notion of ordering. + defp build_concurrency(%{strict_ordering: true}), do: 1 + defp build_concurrency(%{strict_ordering: false}), do: System.schedulers_online() +end diff --git a/lib/actors/actor/entity/lifecycle/stream_initiator.ex b/lib/actors/actor/entity/lifecycle/stream_initiator.ex new file mode 100644 index 00000000..150a980b --- /dev/null +++ b/lib/actors/actor/entity/lifecycle/stream_initiator.ex @@ -0,0 +1,252 @@ +defmodule Actors.Actor.Entity.Lifecycle.StreamInitiator do + @moduledoc """ + Handles lifecycle functions for Actor Entity that interacts with Event Source mechanisms + """ + require Logger + + alias Actors.Actor.Entity.Lifecycle.StreamConsumer + + alias Eigr.Functions.Protocol.Actors.Actor + alias Eigr.Functions.Protocol.Actors.ProjectionSettings + alias Eigr.Functions.Protocol.Actors.ProjectionSubject + + alias Google.Protobuf.Timestamp + + alias Spawn.Utils.Nats + alias Gnat.Jetstream.API.Stream, as: NatsStream + alias Gnat.Jetstream.API.Consumer + + @consumer_not_found_code 10014 + @one_day_in_ms :timer.hours(24) + @stream_not_found_code 10059 + + @spec init_projection_stream(module()) :: :ignore | {:error, any()} | {:ok, pid()} + def init_projection_stream(%Actor{} = actor) do + name = "#{actor.id.system}:#{actor.id.name}" + + with {:create_stream, :ok} <- {:create_stream, create_stream(actor, true)}, + {:create_consumer, :ok} <- + {:create_consumer, create_consumer(actor, deliver_policy: :all)} do + start_pipeline(actor) + else + {:create_stream, error} -> + Logger.error( + "Error on start Projection #{name}. During phase [create_stream]. Details: #{inspect(error)}" + ) + + {:error, error} + + {:create_consumer, error} -> + Logger.error( + "Error on start Projection #{name}. During phase [create_consumer]. Details: #{inspect(error)}" + ) + + {:error, error} + + error -> + Logger.error( + "Error on start Projection #{name}. During phase [start_pipeline]. Details: #{inspect(error)}" + ) + + {:error, error} + end + end + + def init_sourceable_stream(%Actor{} = actor), do: create_stream(actor, false) + + def replay(stream_pid, actor, call_opts) do + # TODO: Necessary avoid naming conflicts using actor system and actor name to build name of stream + name = "#{actor.id.system}:#{actor.id.name}" + + with {:stop_pipeline, :ok} <- {:stop_pipeline, stop_pipeline(stream_pid)}, + {:destroy_consumer, :ok} <- {:destroy_consumer, destroy_consumer(actor)}, + {:recreate_consumer, :ok} <- {:recreate_consumer, create_consumer(actor, call_opts)}, + {:start_pipeline, {:ok, newpid}} <- {:start_pipeline, start_pipeline(actor)} do + {:ok, newpid} + else + {:stop_pipeline, error} -> + Logger.error( + "Error on start Projection #{name}. During phase [stop_pipeline]. Details: #{inspect(error)}" + ) + + {:error, error} + + {:destroy_consumer, error} -> + Logger.error( + "Error on start Projection #{name}. During phase [destroy_consumer]. Details: #{inspect(error)}" + ) + + {:error, error} + + {:recreate_consumer, error} -> + Logger.error( + "Error on start Projection #{name}. During phase [recreate_consumer]. Details: #{inspect(error)}" + ) + + {:error, error} + + {:start_pipeline, error} -> + Logger.error( + "Error on start Projection #{name}. During phase [start_pipeline]. Details: #{inspect(error)}" + ) + + {:error, error} + end + end + + defp build_consumer(stream_name, consumer_name, opts) do + deliver_policy = Keyword.get(opts, :deliver_policy, :all) + build_consumer_by_deliver(deliver_policy, stream_name, consumer_name, opts) + end + + defp build_consumer_by_deliver(:by_start_time, stream_name, consumer_name, opts) do + ten_minutes = + DateTime.utc_now() + |> DateTime.add(-@one_day_in_ms, :second) + + start_time = Keyword.get(opts, :opt_start_time, ten_minutes) + + %Consumer{ + stream_name: stream_name, + durable_name: consumer_name, + deliver_policy: :by_start_time, + opt_start_time: start_time + } + end + + defp build_consumer_by_deliver(:all, stream_name, consumer_name, _opts) do + %Consumer{stream_name: stream_name, durable_name: consumer_name, deliver_policy: :all} + end + + defp build_sources(%ProjectionSettings{} = settings) do + settings.subjects + |> Enum.map(fn %ProjectionSubject{start_time: %Timestamp{seconds: start_at}} = subject -> + %{ + name: subject.actor, + filter_subject: "actors.#{subject.actor}.*.#{subject.action}", + opt_start_time: DateTime.from_unix!(start_at, :second) + } + end) + end + + defp build_stream_max_age(%ProjectionSettings{} = settings) do + case Map.get(settings.events_retention_strategy, :strategy, {:time_in_ms, @one_day_in_ms}) do + {:infinite, true} -> 0 + # ms to ns + {:time_in_ms, %{time: max_age}} -> max_age * 1_000_000 + {:time_in_ms, max_age} -> max_age * 1_000_000 + end + end + + defp conn, do: Nats.connection_name() + + defp create_stream(actor, true) do + # TODO: Necessary avoid naming conflicts using actor system and actor name to build name of stream + stream_name = actor.id.name + max_age = build_stream_max_age(actor.settings.projection_settings) + + stream = + %NatsStream{ + name: stream_name, + subjects: [], + sources: build_sources(actor.settings.projection_settings), + duplicate_window: max_age, + max_age: max_age + } + + case NatsStream.info(conn(), stream_name) do + {:ok, _info} -> + # TODO: Make sure to update the stream if it already exists and sources is changed + :ok + + {:error, %{"code" => 404, "err_code" => @stream_not_found_code}} -> + {:ok, %{created: _}} = NatsStream.create(conn(), stream) + :ok + + error -> + error + end + end + + defp create_stream(actor, false) do + # TODO: Necessary avoid naming conflicts using actor system and actor name to build name of stream + stream_name = actor.id.name + + max_age = build_stream_max_age(actor.settings.projection_settings) + + stream = %NatsStream{ + name: stream_name, + subjects: ["actors.#{stream_name}.>"], + max_age: max_age, + duplicate_window: max_age + } + + case NatsStream.info(conn(), stream_name) do + {:ok, _info} -> + :ok + + {:error, %{"code" => 404, "err_code" => @stream_not_found_code}} -> + {:ok, %{created: _}} = NatsStream.create(conn(), stream) + :ok + + error -> + error + end + end + + defp create_consumer(actor, opts) do + # TODO: Necessary avoid naming conflicts using actor system and actor name to build name of stream + stream_name = actor.id.name + consumer_name = actor.id.name + + case Consumer.info(conn(), stream_name, consumer_name) do + {:ok, _info} -> + :ok + + {:error, + %{ + "code" => 404, + "description" => "consumer not found", + "err_code" => @consumer_not_found_code + }} -> + {:ok, %{created: _}} = + Consumer.create(conn(), build_consumer(stream_name, consumer_name, opts)) + + :ok + + error -> + error + end + end + + defp destroy_consumer(actor) do + stream_name = actor.id.name + consumer_name = actor.id.name + + case Consumer.info(conn(), stream_name, consumer_name) do + {:ok, _info} -> + Consumer.delete(conn(), stream_name, consumer_name) + + {:error, + %{ + "code" => 404, + "description" => "consumer not found", + "err_code" => @consumer_not_found_code + }} -> + :ok + + error -> + error + end + end + + defp start_pipeline(actor), + do: + StreamConsumer.start_link(%{ + actor_name: actor.id.name, + projection_pid: self(), + strict_ordering: actor.settings.projection_settings.strict_events_ordering + }) + + defp stop_pipeline(pid), do: Broadway.stop(pid) +end diff --git a/lib/actors/actor/entity/state.ex b/lib/actors/actor/entity/state.ex index 2aa0c153..bdf0f9f7 100644 --- a/lib/actors/actor/entity/state.ex +++ b/lib/actors/actor/entity/state.ex @@ -4,13 +4,19 @@ defmodule Actors.Actor.Entity.EntityState do """ alias Eigr.Functions.Protocol.Actors.Actor - defstruct system: nil, actor: nil, state_hash: nil, revision: 0, opts: [] + defstruct system: nil, + actor: nil, + state_hash: nil, + revision: 0, + projection_stream_pid: nil, + opts: [] @type t :: %__MODULE__{ system: String.t(), actor: Actor.t(), state_hash: binary(), revision: number(), + projection_stream_pid: pid(), opts: Keyword.t() } diff --git a/lib/spawn/actors/eigr/functions/protocol/actors/actor.pb.ex b/lib/spawn/actors/eigr/functions/protocol/actors/actor.pb.ex index deef8b15..20072c10 100644 --- a/lib/spawn/actors/eigr/functions/protocol/actors/actor.pb.ex +++ b/lib/spawn/actors/eigr/functions/protocol/actors/actor.pb.ex @@ -1,6 +1,6 @@ defmodule Eigr.Functions.Protocol.Actors.Kind do @moduledoc false - use Protobuf, enum: true, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, enum: true, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -42,6 +42,12 @@ defmodule Eigr.Functions.Protocol.Actors.Kind do number: 5, options: nil, __unknown_fields__: [] + }, + %Google.Protobuf.EnumValueDescriptorProto{ + name: "PROJECTION", + number: 6, + options: nil, + __unknown_fields__: [] } ], options: nil, @@ -57,11 +63,12 @@ defmodule Eigr.Functions.Protocol.Actors.Kind do field(:POOLED, 3) field(:PROXY, 4) field(:TASK, 5) + field(:PROJECTION, 6) end defmodule Eigr.Functions.Protocol.Actors.Registry.ActorsEntry do @moduledoc false - use Protobuf, map: true, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, map: true, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -124,7 +131,7 @@ end defmodule Eigr.Functions.Protocol.Actors.Registry do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -219,7 +226,7 @@ end defmodule Eigr.Functions.Protocol.Actors.ActorSystem do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -273,7 +280,7 @@ end defmodule Eigr.Functions.Protocol.Actors.ActorSnapshotStrategy do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -320,7 +327,7 @@ end defmodule Eigr.Functions.Protocol.Actors.ActorDeactivationStrategy do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -367,7 +374,7 @@ end defmodule Eigr.Functions.Protocol.Actors.TimeoutStrategy do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -406,7 +413,7 @@ end defmodule Eigr.Functions.Protocol.Actors.Action do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -445,7 +452,7 @@ end defmodule Eigr.Functions.Protocol.Actors.FixedTimerAction do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -499,7 +506,7 @@ end defmodule Eigr.Functions.Protocol.Actors.ActorState.TagsEntry do @moduledoc false - use Protobuf, map: true, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, map: true, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -562,7 +569,7 @@ end defmodule Eigr.Functions.Protocol.Actors.ActorState do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -673,7 +680,7 @@ end defmodule Eigr.Functions.Protocol.Actors.Metadata.TagsEntry do @moduledoc false - use Protobuf, map: true, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, map: true, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -736,7 +743,7 @@ end defmodule Eigr.Functions.Protocol.Actors.Metadata do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -851,7 +858,7 @@ end defmodule Eigr.Functions.Protocol.Actors.Channel do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -903,9 +910,273 @@ defmodule Eigr.Functions.Protocol.Actors.Channel do field(:action, 2, type: :string) end +defmodule Eigr.Functions.Protocol.Actors.ProjectionSubject do + @moduledoc false + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" + + def descriptor do + # credo:disable-for-next-line + %Google.Protobuf.DescriptorProto{ + name: "ProjectionSubject", + field: [ + %Google.Protobuf.FieldDescriptorProto{ + name: "actor", + extendee: nil, + number: 1, + label: :LABEL_OPTIONAL, + type: :TYPE_STRING, + type_name: nil, + default_value: nil, + options: nil, + oneof_index: nil, + json_name: "actor", + proto3_optional: nil, + __unknown_fields__: [] + }, + %Google.Protobuf.FieldDescriptorProto{ + name: "action", + extendee: nil, + number: 2, + label: :LABEL_OPTIONAL, + type: :TYPE_STRING, + type_name: nil, + default_value: nil, + options: nil, + oneof_index: nil, + json_name: "action", + proto3_optional: nil, + __unknown_fields__: [] + }, + %Google.Protobuf.FieldDescriptorProto{ + name: "start_time", + extendee: nil, + number: 3, + label: :LABEL_OPTIONAL, + type: :TYPE_MESSAGE, + type_name: ".google.protobuf.Timestamp", + default_value: nil, + options: nil, + oneof_index: nil, + json_name: "startTime", + proto3_optional: nil, + __unknown_fields__: [] + } + ], + nested_type: [], + enum_type: [], + extension_range: [], + extension: [], + options: nil, + oneof_decl: [], + reserved_range: [], + reserved_name: [], + __unknown_fields__: [] + } + end + + field(:actor, 1, type: :string) + field(:action, 2, type: :string) + field(:start_time, 3, type: Google.Protobuf.Timestamp, json_name: "startTime") +end + +defmodule Eigr.Functions.Protocol.Actors.EventsRetentionTime do + @moduledoc false + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" + + def descriptor do + # credo:disable-for-next-line + %Google.Protobuf.DescriptorProto{ + name: "EventsRetentionTime", + field: [ + %Google.Protobuf.FieldDescriptorProto{ + name: "time", + extendee: nil, + number: 1, + label: :LABEL_OPTIONAL, + type: :TYPE_INT64, + type_name: nil, + default_value: nil, + options: nil, + oneof_index: nil, + json_name: "time", + proto3_optional: nil, + __unknown_fields__: [] + } + ], + nested_type: [], + enum_type: [], + extension_range: [], + extension: [], + options: nil, + oneof_decl: [], + reserved_range: [], + reserved_name: [], + __unknown_fields__: [] + } + end + + field(:time, 1, type: :int64) +end + +defmodule Eigr.Functions.Protocol.Actors.EventsRetentionStrategy do + @moduledoc false + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" + + def descriptor do + # credo:disable-for-next-line + %Google.Protobuf.DescriptorProto{ + name: "EventsRetentionStrategy", + field: [ + %Google.Protobuf.FieldDescriptorProto{ + name: "time_in_ms", + extendee: nil, + number: 1, + label: :LABEL_OPTIONAL, + type: :TYPE_MESSAGE, + type_name: ".eigr.functions.protocol.actors.EventsRetentionTime", + default_value: nil, + options: nil, + oneof_index: 0, + json_name: "timeInMs", + proto3_optional: nil, + __unknown_fields__: [] + }, + %Google.Protobuf.FieldDescriptorProto{ + name: "infinite", + extendee: nil, + number: 2, + label: :LABEL_OPTIONAL, + type: :TYPE_BOOL, + type_name: nil, + default_value: nil, + options: nil, + oneof_index: 0, + json_name: "infinite", + proto3_optional: nil, + __unknown_fields__: [] + } + ], + nested_type: [], + enum_type: [], + extension_range: [], + extension: [], + options: nil, + oneof_decl: [ + %Google.Protobuf.OneofDescriptorProto{ + name: "strategy", + options: nil, + __unknown_fields__: [] + } + ], + reserved_range: [], + reserved_name: [], + __unknown_fields__: [] + } + end + + oneof(:strategy, 0) + + field(:time_in_ms, 1, + type: Eigr.Functions.Protocol.Actors.EventsRetentionTime, + json_name: "timeInMs", + oneof: 0 + ) + + field(:infinite, 2, type: :bool, oneof: 0) +end + +defmodule Eigr.Functions.Protocol.Actors.ProjectionSettings do + @moduledoc false + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" + + def descriptor do + # credo:disable-for-next-line + %Google.Protobuf.DescriptorProto{ + name: "ProjectionSettings", + field: [ + %Google.Protobuf.FieldDescriptorProto{ + name: "subjects", + extendee: nil, + number: 1, + label: :LABEL_REPEATED, + type: :TYPE_MESSAGE, + type_name: ".eigr.functions.protocol.actors.ProjectionSubject", + default_value: nil, + options: nil, + oneof_index: nil, + json_name: "subjects", + proto3_optional: nil, + __unknown_fields__: [] + }, + %Google.Protobuf.FieldDescriptorProto{ + name: "sourceable", + extendee: nil, + number: 2, + label: :LABEL_OPTIONAL, + type: :TYPE_BOOL, + type_name: nil, + default_value: nil, + options: nil, + oneof_index: nil, + json_name: "sourceable", + proto3_optional: nil, + __unknown_fields__: [] + }, + %Google.Protobuf.FieldDescriptorProto{ + name: "events_retention_strategy", + extendee: nil, + number: 3, + label: :LABEL_OPTIONAL, + type: :TYPE_MESSAGE, + type_name: ".eigr.functions.protocol.actors.EventsRetentionStrategy", + default_value: nil, + options: nil, + oneof_index: nil, + json_name: "eventsRetentionStrategy", + proto3_optional: nil, + __unknown_fields__: [] + }, + %Google.Protobuf.FieldDescriptorProto{ + name: "strict_events_ordering", + extendee: nil, + number: 4, + label: :LABEL_OPTIONAL, + type: :TYPE_BOOL, + type_name: nil, + default_value: nil, + options: nil, + oneof_index: nil, + json_name: "strictEventsOrdering", + proto3_optional: nil, + __unknown_fields__: [] + } + ], + nested_type: [], + enum_type: [], + extension_range: [], + extension: [], + options: nil, + oneof_decl: [], + reserved_range: [], + reserved_name: [], + __unknown_fields__: [] + } + end + + field(:subjects, 1, repeated: true, type: Eigr.Functions.Protocol.Actors.ProjectionSubject) + field(:sourceable, 2, type: :bool) + + field(:events_retention_strategy, 3, + type: Eigr.Functions.Protocol.Actors.EventsRetentionStrategy, + json_name: "eventsRetentionStrategy" + ) + + field(:strict_events_ordering, 4, type: :bool, json_name: "strictEventsOrdering") +end + defmodule Eigr.Functions.Protocol.Actors.ActorSettings do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -995,6 +1266,20 @@ defmodule Eigr.Functions.Protocol.Actors.ActorSettings do json_name: "maxPoolSize", proto3_optional: nil, __unknown_fields__: [] + }, + %Google.Protobuf.FieldDescriptorProto{ + name: "projection_settings", + extendee: nil, + number: 7, + label: :LABEL_OPTIONAL, + type: :TYPE_MESSAGE, + type_name: ".eigr.functions.protocol.actors.ProjectionSettings", + default_value: nil, + options: nil, + oneof_index: nil, + json_name: "projectionSettings", + proto3_optional: nil, + __unknown_fields__: [] } ], nested_type: [], @@ -1024,11 +1309,16 @@ defmodule Eigr.Functions.Protocol.Actors.ActorSettings do field(:min_pool_size, 5, type: :int32, json_name: "minPoolSize") field(:max_pool_size, 6, type: :int32, json_name: "maxPoolSize") + + field(:projection_settings, 7, + type: Eigr.Functions.Protocol.Actors.ProjectionSettings, + json_name: "projectionSettings" + ) end defmodule Eigr.Functions.Protocol.Actors.ActorId do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -1097,7 +1387,7 @@ end defmodule Eigr.Functions.Protocol.Actors.Actor do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line diff --git a/lib/spawn/actors/eigr/functions/protocol/actors/extensions.pb.ex b/lib/spawn/actors/eigr/functions/protocol/actors/extensions.pb.ex index 55549a42..1baf100e 100644 --- a/lib/spawn/actors/eigr/functions/protocol/actors/extensions.pb.ex +++ b/lib/spawn/actors/eigr/functions/protocol/actors/extensions.pb.ex @@ -1,6 +1,6 @@ defmodule Eigr.Functions.Protocol.Actors.PbExtension do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" extend(Google.Protobuf.FieldOptions, :actor_id, 9999, optional: true, diff --git a/lib/spawn/actors/eigr/functions/protocol/actors/healthcheck.pb.ex b/lib/spawn/actors/eigr/functions/protocol/actors/healthcheck.pb.ex index ff2b7d2d..94c7fc6b 100644 --- a/lib/spawn/actors/eigr/functions/protocol/actors/healthcheck.pb.ex +++ b/lib/spawn/actors/eigr/functions/protocol/actors/healthcheck.pb.ex @@ -1,6 +1,6 @@ defmodule Eigr.Functions.Protocol.Actors.Healthcheck.Status do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -69,7 +69,7 @@ end defmodule Eigr.Functions.Protocol.Actors.Healthcheck.HealthCheckReply do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line diff --git a/lib/spawn/actors/eigr/functions/protocol/actors/protocol.pb.ex b/lib/spawn/actors/eigr/functions/protocol/actors/protocol.pb.ex index aabfd02f..fd98cb0d 100644 --- a/lib/spawn/actors/eigr/functions/protocol/actors/protocol.pb.ex +++ b/lib/spawn/actors/eigr/functions/protocol/actors/protocol.pb.ex @@ -1,6 +1,6 @@ defmodule Eigr.Functions.Protocol.Status do @moduledoc false - use Protobuf, enum: true, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, enum: true, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -47,7 +47,7 @@ end defmodule Eigr.Functions.Protocol.Context.MetadataEntry do @moduledoc false - use Protobuf, map: true, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, map: true, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -110,7 +110,7 @@ end defmodule Eigr.Functions.Protocol.Context.TagsEntry do @moduledoc false - use Protobuf, map: true, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, map: true, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -173,7 +173,7 @@ end defmodule Eigr.Functions.Protocol.Context do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -381,7 +381,7 @@ end defmodule Eigr.Functions.Protocol.Noop do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -403,7 +403,7 @@ end defmodule Eigr.Functions.Protocol.JSONType do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -442,7 +442,7 @@ end defmodule Eigr.Functions.Protocol.RegistrationRequest do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -500,7 +500,7 @@ end defmodule Eigr.Functions.Protocol.RegistrationResponse do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -554,7 +554,7 @@ end defmodule Eigr.Functions.Protocol.ServiceInfo do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -683,7 +683,7 @@ end defmodule Eigr.Functions.Protocol.SpawnRequest do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -722,7 +722,7 @@ end defmodule Eigr.Functions.Protocol.SpawnResponse do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -761,7 +761,7 @@ end defmodule Eigr.Functions.Protocol.ProxyInfo do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -845,7 +845,7 @@ end defmodule Eigr.Functions.Protocol.SideEffect do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -884,7 +884,7 @@ end defmodule Eigr.Functions.Protocol.Broadcast do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -961,7 +961,7 @@ end defmodule Eigr.Functions.Protocol.Pipe do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -1015,7 +1015,7 @@ end defmodule Eigr.Functions.Protocol.Forward do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -1067,9 +1067,208 @@ defmodule Eigr.Functions.Protocol.Forward do field(:action_name, 2, type: :string, json_name: "actionName") end +defmodule Eigr.Functions.Protocol.Fact.MetadataEntry do + @moduledoc false + use Protobuf, map: true, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" + + def descriptor do + # credo:disable-for-next-line + %Google.Protobuf.DescriptorProto{ + name: "MetadataEntry", + field: [ + %Google.Protobuf.FieldDescriptorProto{ + name: "key", + extendee: nil, + number: 1, + label: :LABEL_OPTIONAL, + type: :TYPE_STRING, + type_name: nil, + default_value: nil, + options: nil, + oneof_index: nil, + json_name: "key", + proto3_optional: nil, + __unknown_fields__: [] + }, + %Google.Protobuf.FieldDescriptorProto{ + name: "value", + extendee: nil, + number: 2, + label: :LABEL_OPTIONAL, + type: :TYPE_STRING, + type_name: nil, + default_value: nil, + options: nil, + oneof_index: nil, + json_name: "value", + proto3_optional: nil, + __unknown_fields__: [] + } + ], + nested_type: [], + enum_type: [], + extension_range: [], + extension: [], + options: %Google.Protobuf.MessageOptions{ + message_set_wire_format: false, + no_standard_descriptor_accessor: false, + deprecated: false, + map_entry: true, + deprecated_legacy_json_field_conflicts: nil, + uninterpreted_option: [], + __pb_extensions__: %{}, + __unknown_fields__: [] + }, + oneof_decl: [], + reserved_range: [], + reserved_name: [], + __unknown_fields__: [] + } + end + + field(:key, 1, type: :string) + field(:value, 2, type: :string) +end + +defmodule Eigr.Functions.Protocol.Fact do + @moduledoc false + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" + + def descriptor do + # credo:disable-for-next-line + %Google.Protobuf.DescriptorProto{ + name: "Fact", + field: [ + %Google.Protobuf.FieldDescriptorProto{ + name: "uuid", + extendee: nil, + number: 1, + label: :LABEL_OPTIONAL, + type: :TYPE_STRING, + type_name: nil, + default_value: nil, + options: nil, + oneof_index: nil, + json_name: "uuid", + proto3_optional: nil, + __unknown_fields__: [] + }, + %Google.Protobuf.FieldDescriptorProto{ + name: "state", + extendee: nil, + number: 2, + label: :LABEL_OPTIONAL, + type: :TYPE_MESSAGE, + type_name: ".google.protobuf.Any", + default_value: nil, + options: nil, + oneof_index: nil, + json_name: "state", + proto3_optional: nil, + __unknown_fields__: [] + }, + %Google.Protobuf.FieldDescriptorProto{ + name: "metadata", + extendee: nil, + number: 3, + label: :LABEL_REPEATED, + type: :TYPE_MESSAGE, + type_name: ".eigr.functions.protocol.Fact.MetadataEntry", + default_value: nil, + options: nil, + oneof_index: nil, + json_name: "metadata", + proto3_optional: nil, + __unknown_fields__: [] + }, + %Google.Protobuf.FieldDescriptorProto{ + name: "timestamp", + extendee: nil, + number: 4, + label: :LABEL_OPTIONAL, + type: :TYPE_MESSAGE, + type_name: ".google.protobuf.Timestamp", + default_value: nil, + options: nil, + oneof_index: nil, + json_name: "timestamp", + proto3_optional: nil, + __unknown_fields__: [] + } + ], + nested_type: [ + %Google.Protobuf.DescriptorProto{ + name: "MetadataEntry", + field: [ + %Google.Protobuf.FieldDescriptorProto{ + name: "key", + extendee: nil, + number: 1, + label: :LABEL_OPTIONAL, + type: :TYPE_STRING, + type_name: nil, + default_value: nil, + options: nil, + oneof_index: nil, + json_name: "key", + proto3_optional: nil, + __unknown_fields__: [] + }, + %Google.Protobuf.FieldDescriptorProto{ + name: "value", + extendee: nil, + number: 2, + label: :LABEL_OPTIONAL, + type: :TYPE_STRING, + type_name: nil, + default_value: nil, + options: nil, + oneof_index: nil, + json_name: "value", + proto3_optional: nil, + __unknown_fields__: [] + } + ], + nested_type: [], + enum_type: [], + extension_range: [], + extension: [], + options: %Google.Protobuf.MessageOptions{ + message_set_wire_format: false, + no_standard_descriptor_accessor: false, + deprecated: false, + map_entry: true, + deprecated_legacy_json_field_conflicts: nil, + uninterpreted_option: [], + __pb_extensions__: %{}, + __unknown_fields__: [] + }, + oneof_decl: [], + reserved_range: [], + reserved_name: [], + __unknown_fields__: [] + } + ], + enum_type: [], + extension_range: [], + extension: [], + options: nil, + oneof_decl: [], + reserved_range: [], + reserved_name: [], + __unknown_fields__: [] + } + end + + field(:uuid, 1, type: :string) + field(:state, 2, type: Google.Protobuf.Any) + field(:metadata, 3, repeated: true, type: Eigr.Functions.Protocol.Fact.MetadataEntry, map: true) + field(:timestamp, 4, type: Google.Protobuf.Timestamp) +end + defmodule Eigr.Functions.Protocol.Workflow do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -1161,7 +1360,7 @@ end defmodule Eigr.Functions.Protocol.InvocationRequest.MetadataEntry do @moduledoc false - use Protobuf, map: true, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, map: true, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -1224,7 +1423,7 @@ end defmodule Eigr.Functions.Protocol.InvocationRequest do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -1479,7 +1678,7 @@ end defmodule Eigr.Functions.Protocol.ActorInvocation do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -1601,7 +1800,7 @@ end defmodule Eigr.Functions.Protocol.ActorInvocationResponse do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -1738,7 +1937,7 @@ end defmodule Eigr.Functions.Protocol.InvocationResponse do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -1845,7 +2044,7 @@ end defmodule Eigr.Functions.Protocol.RequestStatus do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line diff --git a/lib/spawn/actors/eigr/functions/protocol/actors/state.pb.ex b/lib/spawn/actors/eigr/functions/protocol/actors/state.pb.ex index 92915b44..f354c111 100644 --- a/lib/spawn/actors/eigr/functions/protocol/actors/state.pb.ex +++ b/lib/spawn/actors/eigr/functions/protocol/actors/state.pb.ex @@ -1,6 +1,6 @@ defmodule Eigr.Functions.Protocol.State.Revision do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line @@ -39,7 +39,7 @@ end defmodule Eigr.Functions.Protocol.State.Checkpoint do @moduledoc false - use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3 + use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0" def descriptor do # credo:disable-for-next-line diff --git a/mix.exs b/mix.exs index 63a66844..03367f78 100644 --- a/mix.exs +++ b/mix.exs @@ -89,9 +89,12 @@ defmodule Spawn.MixProject do {:phoenix_pubsub, "~> 2.1"}, {:phoenix_pubsub_nats, "~> 0.2"}, {:jason, "~> 1.3"}, - {:gnat, "~> 1.7"}, + {:gnat, "~> 1.9"}, + {:jetstream, "~> 0.0.9"}, {:k8s, "~> 2.2"}, {:k8s_webhoox, "~> 0.2"}, + {:uuid, "~> 1.1"}, + {:broadway, "~> 1.1"}, # temporary until bandit releases 1.5.4 {:hpax, "~> 0.1.1"}, # Metrics & Tracing deps diff --git a/mix.lock b/mix.lock index 27e4d4eb..9dc9068e 100644 --- a/mix.lock +++ b/mix.lock @@ -2,6 +2,7 @@ "acceptor_pool": {:hex, :acceptor_pool, "1.0.0", "43c20d2acae35f0c2bcd64f9d2bde267e459f0f3fd23dab26485bf518c281b21", [:rebar3], [], "hexpm", "0cbcd83fdc8b9ad2eee2067ef8b91a14858a5883cb7cd800e6fcd5803e158788"}, "bakeware": {:hex, :bakeware, "0.2.4", "0aaf49b34f4bab2aa433f9ff1485d9401e421603160abd6d269c469fc7b65212", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "7b97bcf6fbeee53bb32441d6c495bf478d26f9575633cfef6831e421e86ada6d"}, "bandit": {:hex, :bandit, "1.5.2", "ed0a41c43a9e529c670d0fd48371db4027e7b80d43b1942893e17deb8bed0540", [:mix], [{:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "35ddbdce7e8a2a3c6b5093f7299d70832a43ed2f4a1852885a61d334cab1b4ad"}, + "broadway": {:hex, :broadway, "1.1.0", "8ed3aea01fd6f5640b3e1515b90eca51c4fc1fac15fb954cdcf75dc054ae719c", [:mix], [{:gen_stage, "~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.3.7 or ~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "25e315ef1afe823129485d981dcc6d9b221cea30e625fd5439e9b05f44fb60e4"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "castore": {:hex, :castore, "1.0.10", "43bbeeac820f16c89f79721af1b3e092399b3a1ecc8df1a472738fd853574911", [:mix], [], "hexpm", "1b0b7ea14d889d9ea21202c43a4fa015eb913021cb535e8ed91946f4b77a8848"}, "cc_precompiler": {:hex, :cc_precompiler, "0.1.9", "e8d3364f310da6ce6463c3dd20cf90ae7bbecbf6c5203b98bf9b48035592649b", [:mix], [{:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "9dcab3d0f3038621f1601f13539e7a9ee99843862e66ad62827b0c42b2f58a54"}, @@ -47,6 +48,7 @@ "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"}, "iter": {:hex, :iter, "0.1.2", "bd5dbba48ba67e0f134889a4a29f2b377db6cdcee0661f3c29439e7b649e317a", [:mix], [], "hexpm", "e79f53ed36105ae72582fd3ef224ca2539ccc00cdc27e6e7fe69c49119c4e39b"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, + "jetstream": {:hex, :jetstream, "0.0.9", "f5943c992a98cedd11015436d054c14d1eec544884db0ba959f700363c60fa8f", [:mix], [{:broadway, "~> 1.0", [hex: :broadway, repo: "hexpm", optional: true]}, {:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:gnat, "~> 1.1", [hex: :gnat, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "ca519b254d8b0720865bcf371f50c6122c846d70d25d11fae648b46617577bc1"}, "k8s": {:hex, :k8s, "2.6.1", "ef949e268a65fc45e4481e1071b96c8aa7eae028f0f451052c301d72aae649a2", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: false]}, {:mint_web_socket, "~> 1.0", [hex: :mint_web_socket, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.8", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "a95bde29b60ea94b2ba341969f911e94ae38fc635a332549c39e726ba76ebe10"}, "k8s_webhoox": {:hex, :k8s_webhoox, "0.2.0", "5ef0968a426a0e5d168dd54db7075e0ee222dddfa5da2cf29f25f01a7d02ffd0", [:mix], [{:k8s, "~> 2.0", [hex: :k8s, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:pluggable, "~> 1.0", [hex: :pluggable, repo: "hexpm", optional: false]}, {:x509, "~> 0.8.5", [hex: :x509, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.0", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "4917e1bf43bcbae3c2fa53fa4206f444cc029e757dc4e2b7d550cb0ae8752543"}, "kcl": {:hex, :kcl, "1.4.2", "8b73a55a14899dc172fcb05a13a754ac171c8165c14f65043382d567922f44ab", [:mix], [{:curve25519, ">= 1.0.4", [hex: :curve25519, repo: "hexpm", optional: false]}, {:ed25519, "~> 1.3", [hex: :ed25519, repo: "hexpm", optional: false]}, {:poly1305, "~> 1.0", [hex: :poly1305, repo: "hexpm", optional: false]}, {:salsa20, "~> 1.0", [hex: :salsa20, repo: "hexpm", optional: false]}], "hexpm", "9f083dd3844d902df6834b258564a82b21a15eb9f6acdc98e8df0c10feeabf05"}, @@ -86,6 +88,8 @@ "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "recon": {:hex, :recon, "2.5.4", "05dd52a119ee4059fa9daa1ab7ce81bc7a8161a2f12e9d42e9d551ffd2ba901c", [:mix, :rebar3], [], "hexpm", "e9ab01ac7fc8572e41eb59385efeb3fb0ff5bf02103816535bacaedf327d0263"}, "retry": {:hex, :retry, "0.18.0", "dc58ebe22c95aa00bc2459f9e0c5400e6005541cf8539925af0aa027dc860543", [:mix], [], "hexpm", "9483959cc7bf69c9e576d9dfb2b678b71c045d3e6f39ab7c9aa1489df4492d73"}, + "scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"}, + "scrivener_ecto": {:hex, :scrivener_ecto, "2.7.1", "b8ca910c11429748d3c2d86f0e095abc6d0c49779c7fc5ac5db195e121c46a91", [:mix], [{:ecto, ">= 3.3.0 and < 3.12.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:scrivener, "~> 2.4", [hex: :scrivener, repo: "hexpm", optional: false]}], "hexpm", "f5c2f7db1fdcdfe9583ba378689c6ac20fd2af2c476378a017c03c950ac82c3e"}, "salsa20": {:hex, :salsa20, "1.0.4", "404cbea1fa8e68a41bcc834c0a2571ac175580fec01cc38cc70c0fb9ffc87e9b", [:mix], [], "hexpm", "745ddcd8cfa563ddb0fd61e7ce48d5146279a2cf7834e1da8441b369fdc58ac6"}, "shards": {:hex, :shards, "1.1.1", "8b42323457d185b26b15d05187784ce6c5d1e181b35c46fca36c45f661defe02", [:make, :rebar3], [], "hexpm", "169a045dae6668cda15fbf86d31bf433d0dbbaec42c8c23ca4f8f2d405ea8eda"}, "sobelow": {:hex, :sobelow, "0.13.0", "218afe9075904793f5c64b8837cc356e493d88fddde126a463839351870b8d1e", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "cd6e9026b85fc35d7529da14f95e85a078d9dd1907a9097b3ba6ac7ebbe34a0d"}, @@ -96,7 +100,8 @@ "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.1", "c9755987d7b959b557084e6990990cb96a50d6482c683fb9622a63837f3cd3d8", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5e2c599da4983c4f88a33e9571f1458bf98b0cf6ba930f1dc3a6e8cf45d5afb6"}, "telemetry_poller": {:hex, :telemetry_poller, "1.1.0", "58fa7c216257291caaf8d05678c8d01bd45f4bdbc1286838a28c4bb62ef32999", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9eb9d9cbfd81cbd7cdd24682f8711b6e2b691289a0de6826e58452f28c103c8f"}, "thousand_island": {:hex, :thousand_island, "1.3.5", "6022b6338f1635b3d32406ff98d68b843ba73b3aa95cfc27154223244f3a6ca5", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2be6954916fdfe4756af3239fb6b6d75d0b8063b5df03ba76fd8a4c87849e180"}, - "tls_certificate_check": {:hex, :tls_certificate_check, "1.24.0", "d00e2887551ff8cdae4d0340d90d9fcbc4943c7b5f49d32ed4bc23aff4db9a44", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "90b25a58ee433d91c17f036d4d354bf8859a089bfda60e68a86f8eecae45ef1b"}, + "tls_certificate_check": {:hex, :tls_certificate_check, "1.22.1", "0f450cc1568a67a65ce5e15df53c53f9a098c3da081c5f126199a72505858dc1", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "3092be0babdc0e14c2e900542351e066c0fa5a9cf4b3597559ad1e67f07938c0"}, + "uuid": {:hex, :uuid, "1.1.8", "e22fc04499de0de3ed1116b770c7737779f226ceefa0badb3592e64d5cfb4eb9", [:mix], [], "hexpm", "c790593b4c3b601f5dc2378baae7efaf5b3d73c4c6456ba85759905be792f2ac"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "x509": {:hex, :x509, "0.8.8", "aaf5e58b19a36a8e2c5c5cff0ad30f64eef5d9225f0fd98fb07912ee23f7aba3", [:mix], [], "hexpm", "ccc3bff61406e5bb6a63f06d549f3dba3a1bbb456d84517efaaa210d8a33750f"}, "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, diff --git a/priv/protos/eigr/functions/protocol/actors/actor.proto b/priv/protos/eigr/functions/protocol/actors/actor.proto index d49b3a60..9b79055e 100644 --- a/priv/protos/eigr/functions/protocol/actors/actor.proto +++ b/priv/protos/eigr/functions/protocol/actors/actor.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package eigr.functions.protocol.actors; import "google/protobuf/any.proto"; +import "google/protobuf/timestamp.proto"; option java_package = "io.eigr.functions.protocol.actors"; option go_package = "github.com/eigr/go-support/eigr/actors;actors"; @@ -118,6 +119,40 @@ enum Kind { PROXY = 4; TASK = 5; + + // Projection actors are used to project data from different actor streams. + PROJECTION = 6; +} + +message ProjectionSubject { + string actor = 1; + string action = 2; + google.protobuf.Timestamp start_time = 3; +} + +message EventsRetentionTime { + int64 time = 1; +} + +message EventsRetentionStrategy { + oneof strategy { + EventsRetentionTime time_in_ms = 1; + bool infinite = 2; + } +} + +message ProjectionSettings { + // Define this for projections that need to listen to events + repeated ProjectionSubject subjects = 1; + + // Define this for actors that can emit events + bool sourceable = 2; + + // The strategy for event store retention + EventsRetentionStrategy events_retention_strategy = 3; + + // Define how we consume events from subjects + bool strict_events_ordering = 4; } message ActorSettings { @@ -140,6 +175,9 @@ message ActorSettings { // When kind is POOLED this is used to define maximum actor instances int32 max_pool_size = 6; + + // Event source settings + ProjectionSettings projection_settings = 7; } message ActorId { @@ -172,4 +210,4 @@ message Actor { // The registered timer actions for an actor. repeated FixedTimerAction timer_actions = 5; -} \ No newline at end of file +} diff --git a/priv/protos/eigr/functions/protocol/actors/extensions.proto b/priv/protos/eigr/functions/protocol/actors/extensions.proto index 8df587ac..e4936ae7 100644 --- a/priv/protos/eigr/functions/protocol/actors/extensions.proto +++ b/priv/protos/eigr/functions/protocol/actors/extensions.proto @@ -6,4 +6,4 @@ import "google/protobuf/descriptor.proto"; extend google.protobuf.FieldOptions { bool actor_id = 9999; -} \ No newline at end of file +} diff --git a/priv/protos/eigr/functions/protocol/actors/protocol.proto b/priv/protos/eigr/functions/protocol/actors/protocol.proto index d3259825..912c18c7 100644 --- a/priv/protos/eigr/functions/protocol/actors/protocol.proto +++ b/priv/protos/eigr/functions/protocol/actors/protocol.proto @@ -2,14 +2,18 @@ // // Spawn is divided into two main parts namely: // -// 1. A sidecar proxy that exposes the server part of the Spawn Protocol in the form of an HTTP API. -// 2. A user function, written in any language that supports HTTP, that exposes the client part of the Spawn Protocol. +// 1. A sidecar proxy that exposes the server part of the Spawn Protocol in +// the form of an HTTP API. +// 2. A user function, written in any language that supports HTTP, that +// exposes the client part of the Spawn Protocol. // // Both are client and server of their counterparts. // -// In turn, the proxy exposes an HTTP endpoint for registering a user function a.k.a ActorSystem. -// -// A user function that wants to register actors in Proxy Spawn must proceed by making a POST request to the following endpoint: +// In turn, the proxy exposes an HTTP endpoint for registering a user function +// a.k.a ActorSystem. +// +// A user function that wants to register actors in Proxy Spawn must proceed by +// making a POST request to the following endpoint: // // ` // POST /api/v1/system HTTP 1.1 @@ -23,47 +27,38 @@ // // The general flow of a registration action is as follows: // -//╔═══════════════════╗ ╔═══════════════════╗ ╔═══════════════════╗ -//║ User Function ║ ║Local Spawn Sidecar║ ║ Actor ║ -//╚═══════════════════╝ ╚═══════════════════╝ ╚═══════════════════╝ -// ║ ║ ║ -// ║ ║ ║ -// ║ HTTP POST ║ ║ -// ║ Registration ║ ║ -// ║ Request ║ ║ -// ╠─────────────────────────────────────▶║ ║ -// ║ ║ Upfront start Actors with ║ -// ║ ╠───────BEAM Distributed Protocol─────▶║ -// ║ ║ ║ -// ║ ║ ╠───┐Initialize -// ║ ║ ║ │ State -// ║ ║ ║ │ Store -// ║ ║ ║◀──┘ -// ║ HTTP Registration ║ ║ -// ║ Response ║ ║ -// ║◀─────────────────────────────────────╣ ║ -// ║ ║ ║ -// ║ ║ ║ -// ║ ║ ║ -// ║ ║ ║ -// ║ ║ ║ -// ║ ║ ║ -// ║ ║ ║ -// ║ ║ ║ -// -// ███████████ ███████████ ███████████ +// ╔═══════════════════╗ ╔═══════════════════╗ +// ╔═══════════════════╗ ║ User Function ║ ║Local Spawn +// Sidecar║ ║ Actor ║ ╚═══════════════════╝ +// ╚═══════════════════╝ ╚═══════════════════╝ +// ║ ║ ║ ║ ║ ║ ║ HTTP +// POST ║ ║ ║ +// Registration ║ ║ +// ║ Request ║ ║ +// ╠─────────────────────────────────────▶║ ║ ║ ║ Upfront start +// Actors with ║ ║ ╠───────BEAM Distributed Protocol─────▶║ ║ ║ ║ +// ║ ║ ╠───┐Initialize ║ ║ ║ │ +// State ║ ║ ║ │ Store ║ ║ +// ║◀──┘ ║ HTTP Registration ║ ║ ║ Response ║ ║ +// ║◀─────────────────────────────────────╣ ║ ║ ║ ║ ║ ║ ║ ║ ║ ║ ║ ║ ║ +// ║ ║ ║ ║ ║ ║ ║ ║ ║ ║ ║ ║ +// +// ███████████ ███████████ ███████████ // // // ## Spawning Actors // -// Actors are usually created at the beginning of the SDK's communication flow with the Proxy by the registration step described above. -// However, some use cases require that Actors can be created ***on the fly***. -// In other words, Spawn is used to bring to life Actors previously registered as Unnameds, giving them a name and thus creating a concrete instance -// at runtime for that Actor. Actors created with the Spawn feature are generally used when you want to share a behavior while maintaining -// the isolation characteristics of the actors. -// For these situations we have the Spawning flow described below. +// Actors are usually created at the beginning of the SDK's communication flow +// with the Proxy by the registration step described above. However, some use +// cases require that Actors can be created ***on the fly***. In other words, +// Spawn is used to bring to life Actors previously registered as Unnameds, +// giving them a name and thus creating a concrete instance at runtime for that +// Actor. Actors created with the Spawn feature are generally used when you want +// to share a behavior while maintaining the isolation characteristics of the +// actors. For these situations we have the Spawning flow described below. // -// A user function that wants to Spawning new Actors in Proxy Spawn must proceed by making a POST request to the following endpoint: +// A user function that wants to Spawning new Actors in Proxy Spawn must proceed +// by making a POST request to the following endpoint: // // ``` // POST /system/:system_name/actors/spawn HTTP 1.1 @@ -78,31 +73,26 @@ // The general flow of a Spawning Actors is as follows: // // ``` -// +----------------+ +---------------------+ +-------+ -// | User Function | | Local Spawn Sidecar | | Actor | -// +----------------+ +---------------------+ +-------+ -// | | | -// | HTTP POST SpawnRequest | | -// |------------------------------------------------------>| | -// | | | -// | | Upfront start Actors with BEAM Distributed Protocol | -// | |---------------------------------------------------->| -// | | | -// | | |Initialize Statestore -// | | |---------------------- -// | | | | -// | | |<--------------------- -// | | | -// | HTTP SpawnResponse | | -// |<------------------------------------------------------| | -// | | | +// +----------------+ +---------------------+ +-------+ | User Function | | +// Local Spawn Sidecar | | Actor | +// +----------------+ +---------------------+ +-------+ +// | | | | HTTP +// POST SpawnRequest | | +// |------------------------------------------------------>| | | | | | +// | Upfront start Actors with BEAM Distributed Protocol | | +// |---------------------------------------------------->| | | | | | +// |Initialize Statestore | | |---------------------- | | | | | | +// |<--------------------- | | | | HTTP SpawnResponse | | +// |<------------------------------------------------------| | | | | // ``` // -// Once the system has been initialized, that is, the registration step has been successfully completed, -// then the user function will be able to make requests to the System Actors. -// This is done through a post request to the Proxy at the `/system/:name/actors/:actor_name/invoke` endpoint. +// Once the system has been initialized, that is, the registration step has been +// successfully completed, then the user function will be able to make requests +// to the System Actors. This is done through a post request to the Proxy at the +// `/system/:name/actors/:actor_name/invoke` endpoint. // -// A user function that wants to call actors in Proxy Spawn must proceed by making a POST request as the follow: +// A user function that wants to call actors in Proxy Spawn must proceed by +// making a POST request as the follow: // // ` // POST /system/:name/actors/:actor_name/invoke HTTP 1.1 @@ -113,43 +103,40 @@ // // invocation request type bytes encoded here :-) // ` -// -// Assuming that two user functions were registered in different separate Proxies, the above request would go the following way: -// -//╔═══════════════════╗ ╔═══════════════════╗ ╔═════════════════════════╗ ╔═════════════════════════════╗ -//║ User Function ║ ║Local Spawn Sidecar║ ║ Remote User Function B ║ ║Remote Spawn Sidecar/Actor B ║ -//╚═══════════════════╝ ╚═══════════════════╝ ╚═════════════════════════╝ ╚═════════════════════════════╝ -// ║ HTTP POST ║ ║ ║ -// ║ Registration ║ ║ ║ -// ║ Request ║ ║ ║ -// ╠─────────────────────────────────────▶║ ║ ║ -// ║ ╠───┐ ║ ║ -// ║ ║ │Lookup for ║ ║ -// ║ ║ │ Actor ║ ║ -// ║ ║◀──┘ ║ ║ -// ║ ║ ║ BEAM Distributed ║ -// ║ ╠─────────────────────────────────────╬────────────protocol call──────────▶║ -// ║ ║ ║ ║ -// ║ ║ ║ HTTP POST: ║ -// ║ ║ ║◀──────/api/v1/actors/actions───────╣ -// ║ ║ ║ ║ -// ║ ║ ╠───┐ ║ -// ║ ║ ║ │Handle request, ║ -// ║ ║ ║ │execute action ║ -// ║ ║ ║◀──┘ ║ -// ║ ║ ║ Reply with the ║ -// ║ ║ ╠────────────result and the ────────▶║ -// ║ ║ ║ new state of ║────┐ -// ║ ║ ║ ║ │ -// ║ ║ ║ ║ │Store new State -// ║ ║ Send response to the ║ ║ ◀──┘ -// ║ Respond to user with ║◀─────────Spawn Sidecar A────────────╬────────────────────────────────────╣ -// ║ result value ║ ║ ║ -// ║◀─────────────────────────────────────╣ ║ ║ -// ║ ║ ║ ║ -// ║ ║ ║ ║ -// -// ███████████ ████████████ ███████████ ███████████ +// +// Assuming that two user functions were registered in different separate +// Proxies, the above request would go the following way: +// +// ╔═══════════════════╗ ╔═══════════════════╗ +// ╔═════════════════════════╗ ╔═════════════════════════════╗ ║ User +// Function ║ ║Local Spawn Sidecar║ ║ Remote +// User Function B ║ ║Remote Spawn Sidecar/Actor B ║ +// ╚═══════════════════╝ ╚═══════════════════╝ +// ╚═════════════════════════╝ ╚═════════════════════════════╝ +// ║ HTTP POST ║ ║ ║ ║ Registration ║ ║ ║ ║ +// Request ║ ║ ║ +// ╠─────────────────────────────────────▶║ ║ ║ ║ ╠───┐ ║ ║ ║ ║ │Lookup +// for ║ ║ ║ +// ║ │ Actor ║ ║ ║ ║◀──┘ ║ ║ ║ ║ ║ BEAM +// Distributed ║ ║ +// ╠─────────────────────────────────────╬────────────protocol +// call──────────▶║ ║ ║ ║ ║ ║ ║ ║ +// HTTP POST: ║ ║ ║ +// ║◀──────/api/v1/actors/actions───────╣ ║ ║ ║ ║ ║ ║ ╠───┐ ║ ║ ║ ║ +// │Handle request, ║ ║ ║ ║ │execute action ║ ║ ║ +// ║◀──┘ ║ ║ ║ ║ Reply with +// the ║ ║ ║ +// ╠────────────result and the ────────▶║ ║ ║ ║ new state +// of ║────┐ ║ ║ ║ ║ │ +// ║ ║ ║ ║ │Store new State ║ +// ║ Send response to the ║ ║ ◀──┘ ║ Respond to +// user with ║◀─────────Spawn Sidecar +// A────────────╬────────────────────────────────────╣ ║ result value +// ║ ║ ║ +// ║◀─────────────────────────────────────╣ ║ ║ ║ ║ ║ ║ ║ ║ ║ ║ +// +// ███████████ ████████████ ███████████ +// ███████████ // // syntax = "proto3"; @@ -158,296 +145,328 @@ package eigr.functions.protocol; import "eigr/functions/protocol/actors/actor.proto"; import "google/protobuf/any.proto"; +import "google/protobuf/timestamp.proto"; option java_package = "io.eigr.functions.protocol"; option go_package = "github.com/eigr/go-support/eigr/protocol;protocol"; -// Context is where current and/or updated state is stored +// Context is where current and/or updated state is stored // to be transmitted to/from proxy and user function // // Params: -// * state: Actor state passed back and forth between proxy and user function. +// * state: Actor state passed back and forth between proxy and user function. // * metadata: Meta information that comes in invocations // * tags: Meta information stored in the actor // * caller: ActorId of who is calling target actor // * self: ActorId of itself message Context { - google.protobuf.Any state = 1; + google.protobuf.Any state = 1; - map metadata = 4; + map metadata = 4; - map tags = 5; + map tags = 5; - // Who is calling target actor - eigr.functions.protocol.actors.ActorId caller = 2; + // Who is calling target actor + eigr.functions.protocol.actors.ActorId caller = 2; - // The target actor itself - eigr.functions.protocol.actors.ActorId self = 3; + // The target actor itself + eigr.functions.protocol.actors.ActorId self = 3; } -// Noop is used when the input or output value of a function or method -// does not matter to the caller of a Workflow or when the user just wants to receive -// the Context in the request, that is, -// he does not care about the input value only with the state. +// Noop is used when the input or output value of a function or method +// does not matter to the caller of a Workflow or when the user just wants to +// receive the Context in the request, that is, he does not care about the input +// value only with the state. message Noop {} -// JSON is an alternative that some SDKs can opt in +// JSON is an alternative that some SDKs can opt in // it will bypass any type validation in spawn actors state / payloads -message JSONType { - string content = 1; -} +message JSONType { string content = 1; } message RegistrationRequest { - ServiceInfo service_info = 1; + ServiceInfo service_info = 1; - eigr.functions.protocol.actors.ActorSystem actor_system = 2; + eigr.functions.protocol.actors.ActorSystem actor_system = 2; } message RegistrationResponse { - RequestStatus status = 1; + RequestStatus status = 1; - ProxyInfo proxy_info = 2; + ProxyInfo proxy_info = 2; } message ServiceInfo { - // The name of the actor system, eg, "my-actor-system". - string service_name = 1; + // The name of the actor system, eg, "my-actor-system". + string service_name = 1; - // The version of the service. - string service_version = 2; + // The version of the service. + string service_version = 2; - // A description of the runtime for the service. Can be anything, but examples might be: - // - node v10.15.2 - // - OpenJDK Runtime Environment 1.8.0_192-b12 - string service_runtime = 3; + // A description of the runtime for the service. Can be anything, but examples + // might be: + // - node v10.15.2 + // - OpenJDK Runtime Environment 1.8.0_192-b12 + string service_runtime = 3; - // If using a support library, the name of that library, eg "spawn-jvm" - string support_library_name = 4; + // If using a support library, the name of that library, eg "spawn-jvm" + string support_library_name = 4; - // The version of the support library being used. - string support_library_version = 5; + // The version of the support library being used. + string support_library_version = 5; - // Spawn protocol major version accepted by the support library. - int32 protocol_major_version = 6; + // Spawn protocol major version accepted by the support library. + int32 protocol_major_version = 6; - // Spawn protocol minor version accepted by the support library. - int32 protocol_minor_version = 7; + // Spawn protocol minor version accepted by the support library. + int32 protocol_minor_version = 7; } message SpawnRequest { - repeated eigr.functions.protocol.actors.ActorId actors = 1; + repeated eigr.functions.protocol.actors.ActorId actors = 1; } -message SpawnResponse { - RequestStatus status = 1; -} +message SpawnResponse { RequestStatus status = 1; } message ProxyInfo { - int32 protocol_major_version = 1; + int32 protocol_major_version = 1; - int32 protocol_minor_version = 2; + int32 protocol_minor_version = 2; - string proxy_name = 3; - - string proxy_version = 4; + string proxy_name = 3; + + string proxy_version = 4; } -// When a Host Function is invoked it returns the updated state and return value to the call. -// It can also return a number of side effects to other Actors as a result of its computation. -// These side effects will be forwarded to the respective Actors asynchronously and should not affect the Host Function's response to its caller. -// Internally side effects is just a special kind of InvocationRequest. -// Useful for handle handle `recipient list` and `Composed Message Processor` patterns: +// When a Host Function is invoked it returns the updated state and return value +// to the call. It can also return a number of side effects to other Actors as a +// result of its computation. These side effects will be forwarded to the +// respective Actors asynchronously and should not affect the Host Function's +// response to its caller. Internally side effects is just a special kind of +// InvocationRequest. Useful for handle handle `recipient list` and `Composed +// Message Processor` patterns: // https://www.enterpriseintegrationpatterns.com/patterns/messaging/RecipientList.html // https://www.enterpriseintegrationpatterns.com/patterns/messaging/DistributionAggregate.html -message SideEffect { - InvocationRequest request = 1; -} +message SideEffect { InvocationRequest request = 1; } // Broadcast a message to many Actors -// Useful for handle `recipient list`, `publish-subscribe channel`, and `scatter-gatther` patterns: +// Useful for handle `recipient list`, `publish-subscribe channel`, and +// `scatter-gatther` patterns: // https://www.enterpriseintegrationpatterns.com/patterns/messaging/RecipientList.html // https://www.enterpriseintegrationpatterns.com/patterns/messaging/PublishSubscribeChannel.html // https://www.enterpriseintegrationpatterns.com/patterns/messaging/BroadcastAggregate.html message Broadcast { - // Target topic or channel - //Change this to channel - string channel_group = 1; - - // Payload - oneof payload { - google.protobuf.Any value = 3; - Noop noop = 4; - } + // Target topic or channel + // Change this to channel + string channel_group = 1; + + // Payload + oneof payload { + google.protobuf.Any value = 3; + Noop noop = 4; + } } -// Sends the output of a action of an Actor to the input of another action of an Actor -// Useful for handle `pipes` pattern: +// Sends the output of a action of an Actor to the input of another action of an +// Actor Useful for handle `pipes` pattern: // https://www.enterpriseintegrationpatterns.com/patterns/messaging/PipesAndFilters.html message Pipe { // Target Actor string actor = 1; - // Action. + // Action. string action_name = 2; } -// Sends the input of a action of an Actor to the input of another action of an Actor -// Useful for handle `content-basead router` pattern +// Sends the input of a action of an Actor to the input of another action of an +// Actor Useful for handle `content-basead router` pattern // https://www.enterpriseintegrationpatterns.com/patterns/messaging/ContentBasedRouter.html message Forward { // Target Actor string actor = 1; - // Action. + // Action. string action_name = 2; } -// Container for archicetural message patterns +// Facts are emitted by actions and represent the internal state of the moment +// at that moment. These are treated by Projections so that visualizations can +// be built around these states. +message Fact { + string uuid = 1; + google.protobuf.Any state = 2; + map metadata = 3; + google.protobuf.Timestamp timestamp = 4; +} + +// Container for archicetural message patterns message Workflow { - - Broadcast broadcast = 2; - repeated SideEffect effects = 1; + Broadcast broadcast = 2; + + repeated SideEffect effects = 1; - oneof routing { - Pipe pipe = 3; - Forward forward = 4; - } + oneof routing { + Pipe pipe = 3; + Forward forward = 4; + } } -// The user function when it wants to send a message to an Actor uses the InvocationRequest message type. +// The user function when it wants to send a message to an Actor uses the +// InvocationRequest message type. // // Params: // * system: See ActorSystem message. -// * actor: The target Actor, i.e. the one that the user function is calling to perform some computation. +// * actor: The target Actor, i.e. the one that the user function is calling +// to perform some computation. // * caller: The caller Actor -// * action_name: The function or method on the target Actor that will receive this request +// * action_name: The function or method on the target Actor that will receive +// this request // and perform some useful computation with the sent data. -// * value: This is the value sent by the user function to be computed by the request's target Actor action. -// * async: Indicates whether the action should be processed synchronously, where a response should be sent back to the user function, -// or whether the action should be processed asynchronously, i.e. no response sent to the caller and no waiting. +// * value: This is the value sent by the user function to be computed by the +// request's target Actor action. +// * async: Indicates whether the action should be processed synchronously, +// where a response should be sent back to the user function, +// or whether the action should be processed asynchronously, i.e. no +// response sent to the caller and no waiting. // * metadata: Meta information or headers -// * register_ref: If the invocation should register the specific actor with the given name without having to call register before +// * register_ref: If the invocation should register the specific actor with +// the given name without having to call register before message InvocationRequest { - eigr.functions.protocol.actors.ActorSystem system = 1; + eigr.functions.protocol.actors.ActorSystem system = 1; - eigr.functions.protocol.actors.Actor actor = 2; + eigr.functions.protocol.actors.Actor actor = 2; - string action_name = 3; + string action_name = 3; - oneof payload { - google.protobuf.Any value = 4; - Noop noop = 7; - } + oneof payload { + google.protobuf.Any value = 4; + Noop noop = 7; + } - bool async = 5; + bool async = 5; - eigr.functions.protocol.actors.ActorId caller = 6; + eigr.functions.protocol.actors.ActorId caller = 6; - map metadata = 8; + map metadata = 8; - int64 scheduled_to = 9; + int64 scheduled_to = 9; - bool pooled = 10; + bool pooled = 10; - string register_ref = 11; + string register_ref = 11; } -// ActorInvocation is a translation message between a local invocation made via InvocationRequest -// and the real Actor that intends to respond to this invocation and that can be located anywhere in the cluster. +// ActorInvocation is a translation message between a local invocation made via +// InvocationRequest and the real Actor that intends to respond to this +// invocation and that can be located anywhere in the cluster. // // Params: -// * actor: The ActorId handling the InvocationRequest request, also called the target Actor. -// * action_name: The function or method on the target Actor that will receive this request +// * actor: The ActorId handling the InvocationRequest request, also called +// the target Actor. +// * action_name: The function or method on the target Actor that will receive +// this request // and perform some useful computation with the sent data. -// * current_context: The current Context with current state value of the target Actor. -// That is, the same as found via matching in %Actor{name: target_actor, state: %ActorState{state: value} = actor_state}. -// In this case, the Context type will contain in the value attribute the same `value` as the matching above. -// * payload: The value to be passed to the function or method corresponding to action_name. +// * current_context: The current Context with current state value of the +// target Actor. +// That is, the same as found via matching in %Actor{name: +// target_actor, state: %ActorState{state: value} = +// actor_state}. In this case, the Context type will contain +// in the value attribute the same `value` as the matching +// above. +// * payload: The value to be passed to the function or method corresponding +// to action_name. message ActorInvocation { - eigr.functions.protocol.actors.ActorId actor = 1; + eigr.functions.protocol.actors.ActorId actor = 1; - string action_name = 2; + string action_name = 2; - Context current_context = 3; + Context current_context = 3; - oneof payload { - google.protobuf.Any value = 4; - Noop noop = 5; - } + oneof payload { + google.protobuf.Any value = 4; + Noop noop = 5; + } - eigr.functions.protocol.actors.ActorId caller = 6; + eigr.functions.protocol.actors.ActorId caller = 6; } -// The user function's response after executing the action originated by the local proxy request via ActorInvocation. +// The user function's response after executing the action originated by the +// local proxy request via ActorInvocation. // // Params: -// actor_name: The name of the Actor handling the InvocationRequest request, also called the target Actor. -// actor_system: The name of ActorSystem registered in Registration step. -// updated_context: The Context with updated state value of the target Actor after user function has processed a request. -// value: The value that the original request proxy will forward in response to the InvocationRequest type request. -// This is the final response from the point of view of the user who invoked the Actor call and its subsequent processing. +// actor_name: The name of the Actor handling the InvocationRequest request, +// also called the target Actor. actor_system: The name of ActorSystem +// registered in Registration step. updated_context: The Context with updated +// state value of the target Actor after user function has processed a +// request. value: The value that the original request proxy will forward in +// response to the InvocationRequest type request. +// This is the final response from the point of view of the user who +// invoked the Actor call and its subsequent processing. message ActorInvocationResponse { - string actor_name = 1; + string actor_name = 1; - string actor_system = 2; + string actor_system = 2; - Context updated_context = 3; + Context updated_context = 3; - oneof payload { - google.protobuf.Any value = 4; - Noop noop = 6; - } + oneof payload { + google.protobuf.Any value = 4; + Noop noop = 6; + } - Workflow workflow = 5; + Workflow workflow = 5; - bool checkpoint = 7; + bool checkpoint = 7; } -// InvocationResponse is the response that the proxy that received the InvocationRequest request will forward to the request's original user function. +// InvocationResponse is the response that the proxy that received the +// InvocationRequest request will forward to the request's original user +// function. // // Params: -// status: Status of request. Could be one of [UNKNOWN, OK, ACTOR_NOT_FOUND, ERROR]. -// system: The original ActorSystem of the InvocationRequest request. +// status: Status of request. Could be one of [UNKNOWN, OK, ACTOR_NOT_FOUND, +// ERROR]. system: The original ActorSystem of the InvocationRequest request. // actor: The target Actor originally sent in the InvocationRequest message. -// value: The value resulting from the request processing that the target Actor made. -// This value must be passed by the user function to the one who requested the initial request in InvocationRequest. +// value: The value resulting from the request processing that the target +// Actor made. +// This value must be passed by the user function to the one who +// requested the initial request in InvocationRequest. message InvocationResponse { - RequestStatus status = 1; + RequestStatus status = 1; - eigr.functions.protocol.actors.ActorSystem system = 2; + eigr.functions.protocol.actors.ActorSystem system = 2; - eigr.functions.protocol.actors.Actor actor = 3; + eigr.functions.protocol.actors.Actor actor = 3; - oneof payload { - google.protobuf.Any value = 4; - Noop noop = 5; - } + oneof payload { + google.protobuf.Any value = 4; + Noop noop = 5; + } } enum Status { - UNKNOWN = 0; + UNKNOWN = 0; - OK = 1; + OK = 1; - ACTOR_NOT_FOUND = 2; + ACTOR_NOT_FOUND = 2; - ERROR = 3; + ERROR = 3; } message RequestStatus { - Status status = 1; + Status status = 1; - string message = 2; + string message = 2; } diff --git a/spawn_proxy/proxy/mix.exs b/spawn_proxy/proxy/mix.exs index fa80adf7..2d146b69 100644 --- a/spawn_proxy/proxy/mix.exs +++ b/spawn_proxy/proxy/mix.exs @@ -21,7 +21,11 @@ defmodule Proxy.MixProject do # Run "mix help compile.app" to learn about applications. def application do [ - extra_applications: [:logger, :runtime_tools, :os_mon], + extra_applications: [ + :logger, + :runtime_tools, + :os_mon + ], mod: {Proxy.Application, []} ] end diff --git a/spawn_sdk/spawn_sdk/lib/actor.ex b/spawn_sdk/spawn_sdk/lib/actor.ex index 1cf7efe7..8dc720da 100644 --- a/spawn_sdk/spawn_sdk/lib/actor.ex +++ b/spawn_sdk/spawn_sdk/lib/actor.ex @@ -340,6 +340,10 @@ defmodule SpawnSdk.Actor do state_type = Keyword.get(opts, :state_type, :json) stateful = Keyword.get(opts, :stateful, true) + sourceable = Keyword.get(opts, :sourceable, false) + subjects = Keyword.get(opts, :subjects, []) + events_retention = Keyword.get(opts, :events_retention, 24 * 60 * 60) + strict_ordering = Keyword.get(opts, :strict_ordering, false) tags = Keyword.get(opts, :tags, nil) @@ -421,6 +425,10 @@ defmodule SpawnSdk.Actor do def __meta__(:max_pool_size), do: unquote(max_pool_size) def __meta__(:snapshot_timeout), do: unquote(snapshot_timeout) def __meta__(:deactivate_timeout), do: unquote(deactivate_timeout) + def __meta__(:sourceable), do: unquote(sourceable) + def __meta__(:subjects), do: unquote(subjects) + def __meta__(:events_retention), do: unquote(events_retention) + def __meta__(:strict_ordering), do: unquote(strict_ordering) def __meta__(:tags), do: Map.new(unquote(tags) || %{}) end end diff --git a/spawn_sdk/spawn_sdk/lib/system/spawn_system.ex b/spawn_sdk/spawn_sdk/lib/system/spawn_system.ex index 420ced8f..8ba2cbc8 100644 --- a/spawn_sdk/spawn_sdk/lib/system/spawn_system.ex +++ b/spawn_sdk/spawn_sdk/lib/system/spawn_system.ex @@ -23,7 +23,11 @@ defmodule SpawnSdk.System.SpawnSystem do Registry, TimeoutStrategy, ActorDeactivationStrategy, - Channel + Channel, + ProjectionSettings, + ProjectionSubject, + EventsRetentionStrategy, + EventsRetentionTime } alias Eigr.Functions.Protocol.{ @@ -39,6 +43,7 @@ defmodule SpawnSdk.System.SpawnSystem do Noop } + alias Google.Protobuf.Timestamp import Spawn.Utils.AnySerializer @app :spawn_sdk @@ -515,6 +520,10 @@ defmodule SpawnSdk.System.SpawnSystem do stateful = actor.__meta__(:stateful) snapshot_timeout = actor.__meta__(:snapshot_timeout) deactivate_timeout = actor.__meta__(:deactivate_timeout) + subjects = actor.__meta__(:subjects) + sourceable? = actor.__meta__(:sourceable) + events_retention = actor.__meta__(:events_retention) + strict_ordering? = actor.__meta__(:strict_ordering) timer_actions = actor.__meta__(:timers) min_pool_size = actor.__meta__(:min_pool_size) @@ -536,6 +545,41 @@ defmodule SpawnSdk.System.SpawnSystem do topic when is_binary(topic) -> %Channel{topic: topic, action: "receive"} end) + subjects = + Enum.map(subjects, fn subject -> + case subject do + {actor, action} -> + %ProjectionSubject{ + actor: actor, + action: action, + start_time: %Timestamp{seconds: 0} + } + + {actor, action, start_time} -> + %ProjectionSubject{ + actor: actor, + action: action, + start_time: %Timestamp{seconds: start_time} + } + end + end) + + events_retention_strategy = + case events_retention do + :infinite -> + %EventsRetentionStrategy{strategy: {:infinite, true}} + + value when is_number(value) -> + %EventsRetentionStrategy{strategy: {:time_in_ms, %EventsRetentionTime{time: value}}} + end + + projection_settings = %ProjectionSettings{ + subjects: subjects, + sourceable: sourceable?, + events_retention_strategy: events_retention_strategy, + strict_events_ordering: strict_ordering? + } + {name, %Actor{ id: %ActorId{system: system, name: name}, @@ -546,7 +590,8 @@ defmodule SpawnSdk.System.SpawnSystem do min_pool_size: min_pool_size, max_pool_size: max_pool_size, snapshot_strategy: snapshot_strategy, - deactivation_strategy: deactivation_strategy + deactivation_strategy: deactivation_strategy, + projection_settings: projection_settings }, actions: Enum.map(actions, fn action -> get_action(action) end), timer_actions: @@ -570,6 +615,8 @@ defmodule SpawnSdk.System.SpawnSystem do def decode_kind(:task), do: :TASK def decode_kind(:pooled), do: :POOLED def decode_kind(:POOLED), do: :POOLED + def decode_kind(:projection), do: :PROJECTION + def decode_kind(:PROJECTION), do: :PROJECTION def decode_kind(_), do: :UNKNOW_KIND defp get_action(action_atom) do diff --git a/spawn_sdk/spawn_sdk_example/lib/spawn_sdk_example/actors/clock_actor.ex b/spawn_sdk/spawn_sdk_example/lib/spawn_sdk_example/actors/clock_actor.ex index e68abb48..05f3a9d3 100644 --- a/spawn_sdk/spawn_sdk_example/lib/spawn_sdk_example/actors/clock_actor.ex +++ b/spawn_sdk/spawn_sdk_example/lib/spawn_sdk_example/actors/clock_actor.ex @@ -2,13 +2,14 @@ defmodule SpawnSdkExample.Actors.ClockActor do use SpawnSdk.Actor, name: "ClockActor", state_type: Io.Eigr.Spawn.Example.MyState, - deactivate_timeout: 15_000 + deactivate_timeout: 15_000, + sourceable: true require Logger alias Io.Eigr.Spawn.Example.MyState - action("Clock", [timer: 10_000], fn %Context{state: state} = ctx -> + action("Clock", [timer: 2_000], fn %Context{state: state} = ctx -> Logger.info("[clock] Clock Actor Received Request. Context: #{inspect(ctx)}") new_value = if is_nil(state), do: 0, else: state.value + 1 diff --git a/spawn_sdk/spawn_sdk_example/lib/spawn_sdk_example/actors/projection_actor.ex b/spawn_sdk/spawn_sdk_example/lib/spawn_sdk_example/actors/projection_actor.ex new file mode 100644 index 00000000..f0b46cc6 --- /dev/null +++ b/spawn_sdk/spawn_sdk_example/lib/spawn_sdk_example/actors/projection_actor.ex @@ -0,0 +1,36 @@ +defmodule SpawnSdkExample.Actors.ProjectionActor do + use SpawnSdk.Actor, + name: "projection_actor", + kind: :projection, + state_type: Io.Eigr.Spawn.Example.MyState, + deactivate_timeout: 60_000, + snapshot_timeout: 10_000, + subjects: [ + {"ClockActor", "SecondClock"}, + {"ClockActor", "Clock"} + ] + + require Logger + + alias Io.Eigr.Spawn.Example.MyState + + action("Clock", fn %Context{} = ctx, %MyState{} = payload -> + Logger.info("[projection] Projection Actor Received Request. Context: #{inspect(ctx)}") + + value = payload.value + new_value = (value || 0) + (Map.get(ctx.state || %{}, :value) || 0) + + Value.of() + |> Value.state(%MyState{value: new_value}) + end) + + action("SecondClock", fn %Context{} = ctx, %MyState{} = payload -> + Logger.info("[projection] Projection Actor Received Request. Context: #{inspect(ctx)}") + + value = payload.value + new_value = (value || 0) + (Map.get(ctx.state || %{}, :value) || 0) + + Value.of() + |> Value.state(%MyState{value: new_value}) + end) +end diff --git a/spawn_sdk/spawn_sdk_example/lib/spawn_sdk_example/application.ex b/spawn_sdk/spawn_sdk_example/lib/spawn_sdk_example/application.ex index e38067dc..e8a27048 100644 --- a/spawn_sdk/spawn_sdk_example/lib/spawn_sdk_example/application.ex +++ b/spawn_sdk/spawn_sdk_example/lib/spawn_sdk_example/application.ex @@ -13,7 +13,8 @@ defmodule SpawnSdkExample.Application do SpawnSdkExample.Actors.JsonActor, SpawnSdkExample.Actors.ClockActor, SpawnSdkExample.Actors.UnnamedActor, - SpawnSdkExample.Actors.PubSubActor + SpawnSdkExample.Actors.PubSubActor, + SpawnSdkExample.Actors.ProjectionActor ] # extenal_subscribers: [ # {SpawnSdkExample.Subscriber, []} diff --git a/spawn_sdk/spawn_sdk_example/mix.lock b/spawn_sdk/spawn_sdk_example/mix.lock index 982a5f07..fbfb0c8f 100644 --- a/spawn_sdk/spawn_sdk_example/mix.lock +++ b/spawn_sdk/spawn_sdk_example/mix.lock @@ -4,16 +4,20 @@ "benchee": {:hex, :benchee, "1.1.0", "f3a43817209a92a1fade36ef36b86e1052627fd8934a8b937ac9ab3a76c43062", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}], "hexpm", "7da57d545003165a012b587077f6ba90b89210fd88074ce3c60ce239eb5e6d93"}, "benchee_html": {:hex, :benchee_html, "1.0.0", "5b4d24effebd060f466fb460ec06576e7b34a00fc26b234fe4f12c4f05c95947", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:benchee_json, "~> 1.0", [hex: :benchee_json, repo: "hexpm", optional: false]}], "hexpm", "5280af9aac432ff5ca4216d03e8a93f32209510e925b60e7f27c33796f69e699"}, "benchee_json": {:hex, :benchee_json, "1.0.0", "cc661f4454d5995c08fe10dd1f2f72f229c8f0fb1c96f6b327a8c8fc96a91fe5", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "da05d813f9123505f870344d68fb7c86a4f0f9074df7d7b7e2bb011a63ec231c"}, + "broadway": {:hex, :broadway, "1.1.0", "8ed3aea01fd6f5640b3e1515b90eca51c4fc1fac15fb954cdcf75dc054ae719c", [:mix], [{:gen_stage, "~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.3.7 or ~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "25e315ef1afe823129485d981dcc6d9b221cea30e625fd5439e9b05f44fb60e4"}, "castore": {:hex, :castore, "1.0.7", "b651241514e5f6956028147fe6637f7ac13802537e895a724f90bf3e36ddd1dd", [:mix], [], "hexpm", "da7785a4b0d2a021cd1292a60875a784b6caef71e76bf4917bdee1f390455cf5"}, "cc_precompiler": {:hex, :cc_precompiler, "0.1.9", "e8d3364f310da6ce6463c3dd20cf90ae7bbecbf6c5203b98bf9b48035592649b", [:mix], [{:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "9dcab3d0f3038621f1601f13539e7a9ee99843862e66ad62827b0c42b2f58a54"}, + "chacha20": {:hex, :chacha20, "1.0.4", "0359d8f9a32269271044c1b471d5cf69660c362a7c61a98f73a05ef0b5d9eb9e", [:mix], [], "hexpm", "2027f5d321ae9903f1f0da7f51b0635ad6b8819bc7fe397837930a2011bc2349"}, "chatterbox": {:hex, :ts_chatterbox, "0.13.0", "6f059d97bcaa758b8ea6fffe2b3b81362bd06b639d3ea2bb088335511d691ebf", [:rebar3], [{:hpack, "~>0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "b93d19104d86af0b3f2566c4cba2a57d2e06d103728246ba1ac6c3c0ff010aa7"}, "cloak": {:hex, :cloak, "1.1.2", "7e0006c2b0b98d976d4f559080fabefd81f0e0a50a3c4b621f85ceeb563e80bb", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "940d5ac4fcd51b252930fd112e319ea5ae6ab540b722f3ca60a85666759b9585"}, "cloak_ecto": {:hex, :cloak_ecto, "1.2.0", "e86a3df3bf0dc8980f70406bcb0af2858bac247d55494d40bc58a152590bd402", [:mix], [{:cloak, "~> 1.1.1", [hex: :cloak, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "8bcc677185c813fe64b786618bd6689b1707b35cd95acaae0834557b15a0c62f"}, + "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, "cowboy": {:hex, :cowboy, "2.12.0", "f276d521a1ff88b2b9b4c54d0e753da6c66dd7be6c9fca3d9418b561828a3731", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "8a7abe6d183372ceb21caa2709bec928ab2b72e18a3911aa1771639bef82651e"}, "cowlib": {:hex, :cowlib, "2.13.0", "db8f7505d8332d98ef50a3ef34b34c1afddec7506e4ee4dd4a3a266285d282ca", [:make, :rebar3], [], "hexpm", "e1e1284dc3fc030a64b1ad0d8382ae7e99da46c3246b815318a4b848873800a4"}, "ctx": {:hex, :ctx, "0.6.0", "8ff88b70e6400c4df90142e7f130625b82086077a45364a78d208ed3ed53c7fe", [:rebar3], [], "hexpm", "a14ed2d1b67723dbebbe423b28d7615eb0bdcba6ff28f2d1f1b0a7e1d4aa5fc2"}, + "curve25519": {:hex, :curve25519, "1.0.5", "f801179424e4012049fcfcfcda74ac04f65d0ffceeb80e7ef1d3352deb09f5bb", [:mix], [], "hexpm", "0fba3ad55bf1154d4d5fc3ae5fb91b912b77b13f0def6ccb3a5d58168ff4192d"}, "db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"}, - "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, + "decimal": {:hex, :decimal, "2.2.0", "df3d06bb9517e302b1bd265c1e7f16cda51547ad9d99892049340841f3e15836", [:mix], [], "hexpm", "af8daf87384b51b7e611fb1a1f2c4d4876b65ef968fa8bd3adf44cff401c7f21"}, "decorator": {:hex, :decorator, "1.4.0", "a57ac32c823ea7e4e67f5af56412d12b33274661bb7640ec7fc882f8d23ac419", [:mix], [], "hexpm", "0a07cedd9083da875c7418dea95b78361197cf2bf3211d743f6f7ce39656597f"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "delta_crdt": {:hex, :delta_crdt, "0.6.4", "79d235eef82a58bb0cb668bc5b9558d2e65325ccb46b74045f20b36fd41671da", [:mix], [{:merkle_map, "~> 0.2.0", [hex: :merkle_map, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4a81f579c06aeeb625db54c6c109859a38aa00d837e3e7f8ac27b40cea34885a"}, @@ -22,13 +26,16 @@ "ecto_sqlite3": {:hex, :ecto_sqlite3, "0.13.0", "0c3dc8ff24f378ef108619fd5c18bbbea43cb86dc8733c1c596bd7e0a5bb9e28", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.11", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:exqlite, "~> 0.9", [hex: :exqlite, repo: "hexpm", optional: false]}], "hexpm", "8ab7d8bf6663b811b80c9fa8730780f7077106c40a3fdbae384fe8f82315b257"}, "ed25519": {:hex, :ed25519, "1.4.1", "479fb83c3e31987c9cad780e6aeb8f2015fb5a482618cdf2a825c9aff809afc4", [:mix], [], "hexpm", "0dacb84f3faa3d8148e81019ca35f9d8dcee13232c32c9db5c2fb8ff48c80ec7"}, "elixir_make": {:hex, :elixir_make, "0.7.7", "7128c60c2476019ed978210c245badf08b03dbec4f24d05790ef791da11aa17c", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "5bc19fff950fad52bbe5f211b12db9ec82c6b34a9647da0c2224b8b8464c7e6c"}, + "equivalex": {:hex, :equivalex, "1.0.3", "170d9a82ae066e0020dfe1cf7811381669565922eb3359f6c91d7e9a1124ff74", [:mix], [], "hexpm", "46fa311adb855117d36e461b9c0ad2598f72110ad17ad73d7533c78020e045fc"}, "exactor": {:hex, :exactor, "2.2.4", "5efb4ddeb2c48d9a1d7c9b465a6fffdd82300eb9618ece5d34c3334d5d7245b1", [:mix], [], "hexpm", "1222419f706e01bfa1095aec9acf6421367dcfab798a6f67c54cf784733cd6b5"}, "exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm", "32e95820a97cffea67830e91514a2ad53b888850442d6d395f53a1ac60c82e07"}, "exqlite": {:hex, :exqlite, "0.17.0", "865ab503debde7913ffa02b58838ab92885165978f4c88d8169ee8688c655d1e", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "719fa7986fed242839629a907d60f774000c1d2dc03ba6ba05fcd30579f2ab45"}, "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"}, + "flame": {:hex, :flame, "0.5.1", "339130ed9dff761efc1b2c001839e6d16aa9af291a1e155d8c14fa9b42c03caa", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, ">= 0.0.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "b09ac62b187f40fa7f959d6faca58aae0e575ff21435a8afd79727b8e5631085"}, + "flame_k8s_backend": {:hex, :flame_k8s_backend, "0.5.6", "91b9e2816366ed23020e05fead38a992b86230b112b6d4be2ef2776d586aea99", [:mix], [{:flame, "~> 0.4.0 or ~> 0.5.0", [hex: :flame, repo: "hexpm", optional: false]}], "hexpm", "e403571de233045a858ba9efcfcc0cfc68a5aa1e8c48b16428cdbad2f8dff72d"}, "flow": {:hex, :flow, "1.2.4", "1dd58918287eb286656008777cb32714b5123d3855956f29aa141ebae456922d", [:mix], [{:gen_stage, "~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}], "hexpm", "874adde96368e71870f3510b91e35bc31652291858c86c0e75359cbdd35eb211"}, "gen_stage": {:hex, :gen_stage, "1.2.1", "19d8b5e9a5996d813b8245338a28246307fd8b9c99d1237de199d21efc4c76a1", [:mix], [], "hexpm", "83e8be657fa05b992ffa6ac1e3af6d57aa50aace8f691fcf696ff02f8335b001"}, - "gnat": {:hex, :gnat, "1.7.1", "491144f9c3aec00e9941d69538e2fd2836271e220315c8d2d87907c20ca7abc8", [:mix], [{:cowlib, "~> 2.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:nkeys, "~> 0.2", [hex: :nkeys, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a5629088d9bdb16d982eb48fd431cf6c5a71e9b026281781983501237ab5b911"}, + "gnat": {:hex, :gnat, "1.9.0", "e73ad4279b02e4ad917b7d326342e830b1c6a1eb6677079d8b373faaa7d92f56", [:mix], [{:connection, "~> 1.1", [hex: :connection, repo: "hexpm", optional: false]}, {:cowlib, "~> 2.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:nkeys, "~> 0.2", [hex: :nkeys, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ea91a083dc11984cd9b9aa28c7cbb7a2ff597325ba3c2a400d9d73c2d893e3f3"}, "google_protos": {:hex, :google_protos, "0.4.0", "93e1be2c1a07517ffed761f69047776caf35e4acd385aac4f5ce4fedd07f3660", [:mix], [{:protobuf, "~> 0.10", [hex: :protobuf, repo: "hexpm", optional: false]}], "hexpm", "4c54983d78761a3643e2198adf0f5d40a5a8b08162f3fc91c50faa257f3fa19f"}, "gproc": {:hex, :gproc, "0.8.0", "cea02c578589c61e5341fce149ea36ccef236cc2ecac8691fba408e7ea77ec2f", [:rebar3], [], "hexpm", "580adafa56463b75263ef5a5df4c86af321f68694e7786cb057fd805d1e2a7de"}, "grpc": {:hex, :grpc, "0.8.1", "a8a5884a0d41fc30679c269d0332571cebb43cbfd6eb3d4819169778e866343e", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:cowboy, "~> 2.10", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowlib, "~> 2.12", [hex: :cowlib, repo: "hexpm", optional: false]}, {:gun, "~> 2.0", [hex: :gun, repo: "hexpm", optional: false]}, {:jason, ">= 0.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mint, "~> 1.5", [hex: :mint, repo: "hexpm", optional: false]}, {:protobuf, "~> 0.11", [hex: :protobuf, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "1cccd9fd83547a562f315cc0e1ee1879546f0a44193b5c8eb8d68dae0bb2065b"}, @@ -40,10 +47,12 @@ "hpack": {:hex, :hpack_erl, "0.2.3", "17670f83ff984ae6cd74b1c456edde906d27ff013740ee4d9efaa4f1bf999633", [:rebar3], [], "hexpm", "06f580167c4b8b8a6429040df36cc93bba6d571faeaec1b28816523379cbb23a"}, "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"}, "iter": {:hex, :iter, "0.1.2", "bd5dbba48ba67e0f134889a4a29f2b377db6cdcee0661f3c29439e7b649e317a", [:mix], [], "hexpm", "e79f53ed36105ae72582fd3ef224ca2539ccc00cdc27e6e7fe69c49119c4e39b"}, - "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, + "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, + "jetstream": {:hex, :jetstream, "0.0.9", "f5943c992a98cedd11015436d054c14d1eec544884db0ba959f700363c60fa8f", [:mix], [{:broadway, "~> 1.0", [hex: :broadway, repo: "hexpm", optional: true]}, {:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:gnat, "~> 1.1", [hex: :gnat, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "ca519b254d8b0720865bcf371f50c6122c846d70d25d11fae648b46617577bc1"}, "jsx": {:hex, :jsx, "2.8.3", "a05252d381885240744d955fbe3cf810504eb2567164824e19303ea59eef62cf", [:mix, :rebar3], [], "hexpm", "fc3499fed7a726995aa659143a248534adc754ebd16ccd437cd93b649a95091f"}, "k8s": {:hex, :k8s, "2.4.0", "0dd73cd5b623ff379802a0c8a4fb9f3129b7661d64eb5264bb17d5527122af37", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: false]}, {:mint_web_socket, "~> 1.0", [hex: :mint_web_socket, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.8", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "6f68b43fb5ff85ed66506d2db820dc322ca8e04181b0bae839b3be6111ba2553"}, "k8s_webhoox": {:hex, :k8s_webhoox, "0.2.0", "5ef0968a426a0e5d168dd54db7075e0ee222dddfa5da2cf29f25f01a7d02ffd0", [:mix], [{:k8s, "~> 2.0", [hex: :k8s, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:pluggable, "~> 1.0", [hex: :pluggable, repo: "hexpm", optional: false]}, {:x509, "~> 0.8.5", [hex: :x509, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.0", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "4917e1bf43bcbae3c2fa53fa4206f444cc029e757dc4e2b7d550cb0ae8752543"}, + "kcl": {:hex, :kcl, "1.4.2", "8b73a55a14899dc172fcb05a13a754ac171c8165c14f65043382d567922f44ab", [:mix], [{:curve25519, ">= 1.0.4", [hex: :curve25519, repo: "hexpm", optional: false]}, {:ed25519, "~> 1.3", [hex: :ed25519, repo: "hexpm", optional: false]}, {:poly1305, "~> 1.0", [hex: :poly1305, repo: "hexpm", optional: false]}, {:salsa20, "~> 1.0", [hex: :salsa20, repo: "hexpm", optional: false]}], "hexpm", "9f083dd3844d902df6834b258564a82b21a15eb9f6acdc98e8df0c10feeabf05"}, "libcluster": {:hex, :libcluster, "3.3.3", "a4f17721a19004cfc4467268e17cff8b1f951befe428975dd4f6f7b84d927fe0", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "7c0a2275a0bb83c07acd17dab3c3bfb4897b145106750eeccc62d302e3bdfee5"}, "libring": {:hex, :libring, "1.6.0", "d5dca4bcb1765f862ab59f175b403e356dec493f565670e0bacc4b35e109ce0d", [:mix], [], "hexpm", "5e91ece396af4bce99953d49ee0b02f698cd38326d93cd068361038167484319"}, "meck": {:hex, :meck, "0.9.2", "85ccbab053f1db86c7ca240e9fc718170ee5bda03810a6292b5306bf31bae5f5", [:rebar3], [], "hexpm", "81344f561357dc40a8344afa53767c32669153355b626ea9fcbc8da6b3045826"}, @@ -57,34 +66,41 @@ "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, - "nkeys": {:hex, :nkeys, "0.2.2", "b1ab3324ed4f3a2c9658d7e80feeef86b4d15fbfd12ca5c8cf068289f582fcfa", [:mix], [{:ed25519, "~> 1.3", [hex: :ed25519, repo: "hexpm", optional: false]}], "hexpm", "3578802427b8d1d11ea6dd785c2ab774f527e2c3e449e67bd34612ab71ca471d"}, + "nkeys": {:hex, :nkeys, "0.3.0", "837add5261a3cdd8ff75b54e0475062313093929ab5e042fa48e010f33b10d16", [:mix], [{:ed25519, "~> 1.3", [hex: :ed25519, repo: "hexpm", optional: false]}, {:kcl, "~> 1.4", [hex: :kcl, repo: "hexpm", optional: false]}], "hexpm", "b5af773a296620ee8eeb1ec6dc5b68f716386f7e53f7bda8c4ac23515823dfe4"}, "norm": {:hex, :norm, "0.13.0", "2c562113f3205e3f195ee288d3bd1ab903743e7e9f3282562c56c61c4d95dec4", [:mix], [{:stream_data, "~> 0.5", [hex: :stream_data, repo: "hexpm", optional: true]}], "hexpm", "447cc96dd2d0e19dcb37c84b5fc0d6842aad69386e846af048046f95561d46d7"}, "opentelemetry": {:hex, :opentelemetry, "1.3.0", "988ac3c26acac9720a1d4fb8d9dc52e95b45ecfec2d5b5583276a09e8936bc5e", [:rebar3], [{:opentelemetry_api, "~> 1.2.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "8e09edc26aad11161509d7ecad854a3285d88580f93b63b0b1cf0bac332bfcc0"}, "opentelemetry_api": {:hex, :opentelemetry_api, "1.2.1", "7b69ed4f40025c005de0b74fce8c0549625d59cb4df12d15c32fe6dc5076ff42", [:mix, :rebar3], [{:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "6d7a27b7cad2ad69a09cabf6670514cafcec717c8441beb5c96322bac3d05350"}, + "opentelemetry_ecto": {:hex, :opentelemetry_ecto, "1.2.0", "2382cb47ddc231f953d3b8263ed029d87fbf217915a1da82f49159d122b64865", [:mix], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.2", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "70dfa2e79932e86f209df00e36c980b17a32f82d175f0068bf7ef9a96cf080cf"}, "opentelemetry_exporter": {:hex, :opentelemetry_exporter, "1.6.0", "f4fbf69aa9f1541b253813221b82b48a9863bc1570d8ecc517bc510c0d1d3d8c", [:rebar3], [{:grpcbox, ">= 0.0.0", [hex: :grpcbox, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.3", [hex: :opentelemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.2", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:tls_certificate_check, "~> 1.18", [hex: :tls_certificate_check, repo: "hexpm", optional: false]}], "hexpm", "1802d1dca297e46f21e5832ecf843c451121e875f73f04db87355a6cb2ba1710"}, + "opentelemetry_process_propagator": {:hex, :opentelemetry_process_propagator, "0.3.0", "ef5b2059403a1e2b2d2c65914e6962e56371570b8c3ab5323d7a8d3444fb7f84", [:mix, :rebar3], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "7243cb6de1523c473cba5b1aefa3f85e1ff8cc75d08f367104c1e11919c8c029"}, "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "0.2.0", "b67fe459c2938fcab341cb0951c44860c62347c005ace1b50f8402576f241435", [:mix, :rebar3], [], "hexpm", "d61fa1f5639ee8668d74b527e6806e0503efc55a42db7b5f39939d84c07d6895"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"}, "phoenix_pubsub_nats": {:hex, :phoenix_pubsub_nats, "0.2.1", "9a9e1d75e80fe4b26c8549a76668a193dd656c0bc23c6f8badfc7e9cc4736cb5", [:mix], [{:gnat, "~> 1.6", [hex: :gnat, repo: "hexpm", optional: false]}, {:jason, "~> 1.3", [hex: :jason, repo: "hexpm", optional: false]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}], "hexpm", "39099672b6ef50c055006c8814d6b621d0055e93f96acd267ced92e6cfb94ef7"}, "plug": {:hex, :plug, "1.14.2", "cff7d4ec45b4ae176a227acd94a7ab536d9b37b942c8e8fa6dfc0fff98ff4d80", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "842fc50187e13cf4ac3b253d47d9474ed6c296a8732752835ce4a86acdf68d13"}, "plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"}, "pluggable": {:hex, :pluggable, "1.0.1", "ffd91303879d0ccfde2cbf2b5609f4f602608653e6165c44f5867c32e645e337", [:mix], [], "hexpm", "bce3403fe24dd5e14846b97e64ffa424b7ccda327829a4f6d1067cfc7a87d4a2"}, + "poly1305": {:hex, :poly1305, "1.0.4", "7cdc8961a0a6e00a764835918cdb8ade868044026df8ef5d718708ea6cc06611", [:mix], [{:chacha20, "~> 1.0", [hex: :chacha20, repo: "hexpm", optional: false]}, {:equivalex, "~> 1.0", [hex: :equivalex, repo: "hexpm", optional: false]}], "hexpm", "e14e684661a5195e149b3139db4a1693579d4659d65bba115a307529c47dbc3b"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"}, "postgrex": {:hex, :postgrex, "0.17.4", "5777781f80f53b7c431a001c8dad83ee167bcebcf3a793e3906efff680ab62b3", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "6458f7d5b70652bc81c3ea759f91736c16a31be000f306d3c64bcdfe9a18b3cc"}, "protobuf": {:hex, :protobuf, "0.12.0", "58c0dfea5f929b96b5aa54ec02b7130688f09d2de5ddc521d696eec2a015b223", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "75fa6cbf262062073dd51be44dd0ab940500e18386a6c4e87d5819a58964dc45"}, "protobuf_generate": {:hex, :protobuf_generate, "0.1.2", "45b9a9ae8606333cdea993ceaaecd799d206cdfe23348d37c06207eac76cbee6", [:mix], [{:protobuf, "~> 0.12", [hex: :protobuf, repo: "hexpm", optional: false]}], "hexpm", "55b0ff8385703317ca90e1bd30a2ece99e80ae0c73e6ebcfb374e84e57870d61"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "retry": {:hex, :retry, "0.18.0", "dc58ebe22c95aa00bc2459f9e0c5400e6005541cf8539925af0aa027dc860543", [:mix], [], "hexpm", "9483959cc7bf69c9e576d9dfb2b678b71c045d3e6f39ab7c9aa1489df4492d73"}, + "salsa20": {:hex, :salsa20, "1.0.4", "404cbea1fa8e68a41bcc834c0a2571ac175580fec01cc38cc70c0fb9ffc87e9b", [:mix], [], "hexpm", "745ddcd8cfa563ddb0fd61e7ce48d5146279a2cf7834e1da8441b369fdc58ac6"}, "sbroker": {:hex, :sbroker, "1.0.0", "28ff1b5e58887c5098539f236307b36fe1d3edaa2acff9d6a3d17c2dcafebbd0", [:rebar3], [], "hexpm", "ba952bfa35b374e1e5d84bc5f5efe8360c6f99dc93b3118f714a9a2dff6c9e19"}, + "scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"}, + "scrivener_ecto": {:hex, :scrivener_ecto, "2.7.1", "b8ca910c11429748d3c2d86f0e095abc6d0c49779c7fc5ac5db195e121c46a91", [:mix], [{:ecto, ">= 3.3.0 and < 3.12.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:scrivener, "~> 2.4", [hex: :scrivener, repo: "hexpm", optional: false]}], "hexpm", "f5c2f7db1fdcdfe9583ba378689c6ac20fd2af2c476378a017c03c950ac82c3e"}, "shards": {:hex, :shards, "1.1.0", "ed3032e63ae99f0eaa6d012b8b9f9cead48b9a810b3f91aeac266cfc4118eff6", [:make, :rebar3], [], "hexpm", "1d188e565a54a458a7a601c2fd1e74f5cfeba755c5a534239266d28b7ff124c7"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"}, "tds": {:hex, :tds, "2.3.4", "534749dd9ef61af960fcafa9cbb7186d6d7b9f92ea0133fb25da07b121c8295c", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.9 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "bb9a53d4688a85fd566f342f76b50d39adfc4b410062886ef908365ead24ba3f"}, - "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, + "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, "telemetry_metrics": {:hex, :telemetry_metrics, "1.0.0", "29f5f84991ca98b8eb02fc208b2e6de7c95f8bb2294ef244a176675adc7775df", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f23713b3847286a534e005126d4c959ebcca68ae9582118ce436b521d1d47d5d"}, "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.1", "c9755987d7b959b557084e6990990cb96a50d6482c683fb9622a63837f3cd3d8", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5e2c599da4983c4f88a33e9571f1458bf98b0cf6ba930f1dc3a6e8cf45d5afb6"}, "telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"}, "tls_certificate_check": {:hex, :tls_certificate_check, "1.19.0", "c76c4c5d79ee79a2b11c84f910c825d6f024a78427c854f515748e9bd025e987", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "4083b4a298add534c96125337cb01161c358bb32dd870d5a893aae685fd91d70"}, "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"}, + "uuid": {:hex, :uuid, "1.1.8", "e22fc04499de0de3ed1116b770c7737779f226ceefa0badb3592e64d5cfb4eb9", [:mix], [], "hexpm", "c790593b4c3b601f5dc2378baae7efaf5b3d73c4c6456ba85759905be792f2ac"}, "vapor": {:hex, :vapor, "0.10.0", "547a94b381093dea61a4ca2200109385b6e44b86d72d1ebf93e5ac1a8873bc3c", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:norm, "~> 0.9", [hex: :norm, repo: "hexpm", optional: false]}, {:toml, "~> 0.3", [hex: :toml, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.1", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "ee6d089a71309647a0a2a2ae6cf3bea61739a983e8c1310d53ff04b1493afbc1"}, "x509": {:hex, :x509, "0.8.7", "576130ad83731613fdb5787917f2fadbe308b6f5a48ed6e865a4b6be3fa802d0", [:mix], [], "hexpm", "3604125d6a0171da6e8a935810b58c999fccab0e3d20b2ed28d97fa2d9e2f6b4"}, "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, diff --git a/spawn_sdk/spawn_sdk_example/priv/protos/actors/example.proto b/spawn_sdk/spawn_sdk_example/priv/protos/actors/example.proto deleted file mode 100644 index f7d2c34c..00000000 --- a/spawn_sdk/spawn_sdk_example/priv/protos/actors/example.proto +++ /dev/null @@ -1,29 +0,0 @@ -syntax = "proto3"; - -package io.eigr.spawn.example; - -option java_multiple_files = true; -option java_package = "io.eigr.spawn.example"; -option java_outer_classname = "ExampleProtos"; - -import "google/api/annotations.proto"; -import "google/protobuf/empty.proto"; - -message MyState { int32 value = 1; } - -message MyBusinessMessage { int32 value = 1; } - -service Joe { - rpc Ping(google.protobuf.Empty) returns (MyBusinessMessage) { - option (google.api.http) = { - get : "/v1/ping" - }; - } - - rpc Sum(MyBusinessMessage) returns (MyBusinessMessage) { - option (google.api.http) = { - post : "/v1/sum" - body : "*" - }; - } -} diff --git a/spawn_sdk/spawn_sdk_example/priv/protos/actors/inside_folder/clock.proto b/spawn_sdk/spawn_sdk_example/priv/protos/actors/inside_folder/clock.proto deleted file mode 100644 index ebc414b4..00000000 --- a/spawn_sdk/spawn_sdk_example/priv/protos/actors/inside_folder/clock.proto +++ /dev/null @@ -1,25 +0,0 @@ -syntax = "proto3"; - -package io.eigr.spawn.example; - -option java_multiple_files = true; -option java_package = "io.eigr.spawn.example"; -option java_outer_classname = "ExampleProtos"; - -import "google/api/annotations.proto"; -import "google/protobuf/empty.proto"; - -service ClockActor { - rpc Clock(google.protobuf.Empty) returns (google.protobuf.Empty) { - option (google.api.http) = { - get : "/v1/clock" - }; - } - - rpc SecondClock(google.protobuf.Empty) returns (google.protobuf.Empty) { - option (google.api.http) = { - post : "/v1/second_clock" - body : "*" - }; - } -} \ No newline at end of file diff --git a/spawn_sdk/spawn_sdk_example/priv/protos/actors/inside_folder/nested/clock.proto b/spawn_sdk/spawn_sdk_example/priv/protos/actors/inside_folder/nested/clock.proto deleted file mode 100644 index ebc414b4..00000000 --- a/spawn_sdk/spawn_sdk_example/priv/protos/actors/inside_folder/nested/clock.proto +++ /dev/null @@ -1,25 +0,0 @@ -syntax = "proto3"; - -package io.eigr.spawn.example; - -option java_multiple_files = true; -option java_package = "io.eigr.spawn.example"; -option java_outer_classname = "ExampleProtos"; - -import "google/api/annotations.proto"; -import "google/protobuf/empty.proto"; - -service ClockActor { - rpc Clock(google.protobuf.Empty) returns (google.protobuf.Empty) { - option (google.api.http) = { - get : "/v1/clock" - }; - } - - rpc SecondClock(google.protobuf.Empty) returns (google.protobuf.Empty) { - option (google.api.http) = { - post : "/v1/second_clock" - body : "*" - }; - } -} \ No newline at end of file diff --git a/spawn_statestores/statestores/lib/statestores/adapters/projection_behaviour.ex b/spawn_statestores/statestores/lib/statestores/adapters/projection_behaviour.ex new file mode 100644 index 00000000..84776ef3 --- /dev/null +++ b/spawn_statestores/statestores/lib/statestores/adapters/projection_behaviour.ex @@ -0,0 +1,93 @@ +defmodule Statestores.Adapters.ProjectionBehaviour do + @moduledoc """ + Defines the default behavior for each Statestore Provider. + """ + alias Scrivener.Page + alias Statestores.Schemas.Projection + + @type metadata_key :: String.t() + + @type metadata_value :: String.t() + + @type page :: integer() + + @type page_size :: integer() + + @type page_data :: Page.t() + + @type projection :: Projection.t() + + @type projections :: list(Projection.t()) + + @type projection_name :: String.t() + + @type projection_id :: String.t() + + @type revision :: integer() + + @type time_start :: String.t() + + @type time_end :: String.t() + + @callback create_table(projection_name()) :: {:ok, String.t()} + + @callback get_last(projection_name()) :: {:error, any} | {:ok, projection()} + + @callback get_last_by_projection_id(projection_name(), projection_id()) :: + {:error, any} | {:ok, projection()} + + @callback get_all(projection_name(), page(), page_size()) :: {:error, any} | {:ok, page_data()} + + @callback get_all_by_projection_id(projection_name(), projection_id(), page(), page_size()) :: + {:error, any} | {:ok, page_data()} + + @callback get_by_interval( + projection_name(), + time_start(), + time_end(), + page(), + page_size() + ) :: {:error, any} | {:ok, page_data()} + + @callback get_by_projection_id_and_interval( + projection_name(), + projection_id(), + time_start(), + time_end(), + page(), + page_size() + ) :: {:error, any} | {:ok, page_data()} + + @callback search_by_metadata( + projection_name(), + metadata_key(), + metadata_value(), + page(), + page_size() + ) :: + {:error, any} | {:ok, page_data()} + + @callback search_by_projection_id_and_metadata( + projection_name(), + projection_id(), + metadata_key(), + metadata_value(), + page(), + page_size() + ) :: {:error, any} | {:ok, page_data()} + + @callback save(projection()) :: {:error, any} | {:ok, projection()} + + @callback default_port :: <<_::32>> + + defmacro __using__(_opts) do + quote do + alias Statestores.Adapters.ProjectionBehaviour + import Statestores.Util, only: [init_config: 1, generate_key: 1] + + @behaviour Statestores.Adapters.ProjectionBehaviour + + def init(_type, config), do: init_config(config) + end + end +end diff --git a/spawn_statestores/statestores/lib/statestores/schemas/projection.ex b/spawn_statestores/statestores/lib/statestores/schemas/projection.ex new file mode 100644 index 00000000..59f7fc2c --- /dev/null +++ b/spawn_statestores/statestores/lib/statestores/schemas/projection.ex @@ -0,0 +1,38 @@ +defmodule Statestores.Schemas.Projection do + use Ecto.Schema + import Ecto.Changeset + + @primary_key {:id, :string, autogenerate: false} + schema "projection_placeholder" do + field :projection_id, :string + field :projection_name, :string + field :system, :string + field :metadata, :map + field :data_type, :string + field :data, Statestores.Types.Binary + field :inserted_at, :naive_datetime + field :updated_at, :naive_datetime + end + + @doc false + def changeset(projection, attrs) do + projection + |> cast(attrs, [:id, :projection_id, :projection_name, :system, :metadata, :data_type, :data]) + |> validate_required([ + :id, + :projection_id, + :projection_name, + :system, + :metadata, + :data_type, + :data + ]) + |> case do + %{valid?: false, changes: changes} = changeset when changes == %{} -> + %{changeset | action: :ignore} + + changeset -> + changeset + end + end +end diff --git a/spawn_statestores/statestores/lib/statestores/supervisor.ex b/spawn_statestores/statestores/lib/statestores/supervisor.ex index 3d9ea819..2a747e22 100644 --- a/spawn_statestores/statestores/lib/statestores/supervisor.ex +++ b/spawn_statestores/statestores/lib/statestores/supervisor.ex @@ -5,7 +5,12 @@ defmodule Statestores.Supervisor do @shutdown_timeout_ms 330_000 import Statestores.Util, - only: [load_lookup_adapter: 0, load_snapshot_adapter: 0, supervisor_process_logger: 1] + only: [ + load_lookup_adapter: 0, + load_snapshot_adapter: 0, + load_projection_adapter: 0, + supervisor_process_logger: 1 + ] def start_link(args) do Supervisor.start_link(__MODULE__, args, name: __MODULE__, shutdown: @shutdown_timeout_ms) @@ -22,6 +27,7 @@ defmodule Statestores.Supervisor do def init(_args) do lookup_adapter = load_lookup_adapter() snapshot_adapter = load_snapshot_adapter() + projection_adapter = load_projection_adapter() Statestores.Migrator.migrate(snapshot_adapter) Statestores.Migrator.migrate(lookup_adapter) @@ -30,6 +36,7 @@ defmodule Statestores.Supervisor do supervisor_process_logger(__MODULE__), Statestores.Vault, snapshot_adapter, + projection_adapter, lookup_adapter ] |> maybe_add_native_children(snapshot_adapter) diff --git a/spawn_statestores/statestores/lib/statestores/util.ex b/spawn_statestores/statestores/lib/statestores/util.ex index 691f630e..9f365bae 100644 --- a/spawn_statestores/statestores/lib/statestores/util.ex +++ b/spawn_statestores/statestores/lib/statestores/util.ex @@ -89,6 +89,22 @@ defmodule Statestores.Util do end end + @spec load_projection_adapter :: adapter() + def load_projection_adapter() do + case Application.fetch_env(@otp_app, :projection_adapter) do + {:ok, value} -> + value + + :error -> + type = + String.to_existing_atom( + System.get_env("PROXY_DATABASE_TYPE", get_default_database_type()) + ) + + load_projection_adapter_by_type(type) + end + end + def get_default_database_type do cond do Code.ensure_loaded?(Statestores.Adapters.PostgresSnapshotAdapter) -> "postgres" @@ -197,6 +213,16 @@ defmodule Statestores.Util do key end + def normalize_table_name(nil), do: {:error, "Table name cannot be nil"} + + def normalize_table_name(name) when is_binary(name) do + name + # Converts "CamelCase" to "snake_case" + |> Macro.underscore() + # Ensures the name is all lowercase + |> String.downcase() + end + # Lookup Adapters defp load_lookup_adapter_by_type(:mariadb), do: Statestores.Adapters.MariaDBLookupAdapter @@ -210,4 +236,14 @@ defmodule Statestores.Util do defp load_snapshot_adapter_by_type(:postgres), do: Statestores.Adapters.PostgresSnapshotAdapter defp load_snapshot_adapter_by_type(:native), do: Statestores.Adapters.NativeSnapshotAdapter + + # Projections Adapters + defp load_projection_adapter_by_type(:mariadb), + do: Statestores.Adapters.MariaDBProjectionAdapter + + defp load_projection_adapter_by_type(:postgres), + do: Statestores.Adapters.PostgresProjectionAdapter + + defp load_projection_adapter_by_type(:native), + do: Statestores.Adapters.NativeProjectionAdapter end diff --git a/spawn_statestores/statestores/mix.exs b/spawn_statestores/statestores/mix.exs index f1e27b67..b4f46660 100644 --- a/spawn_statestores/statestores/mix.exs +++ b/spawn_statestores/statestores/mix.exs @@ -12,10 +12,10 @@ defmodule Statestores.MixProject do description: "Spawn Statestores is the storage lib for the Spawn Actors System", source_url: @source_url, homepage_url: "https://eigr.io/", - build_path: "../../_build", + build_path: "./_build", config_path: "../../config/config.exs", - deps_path: "../../deps", - lockfile: "../../mix.lock", + deps_path: "./deps", + lockfile: "./mix.lock", elixir: "~> 1.15", start_permanent: Mix.env() == :prod, deps: deps(), @@ -58,6 +58,7 @@ defmodule Statestores.MixProject do {:castore, "~> 1.0"}, {:cloak_ecto, "~> 1.2"}, {:ecto_sql, "~> 3.10"}, + {:scrivener_ecto, "~> 2.0"}, {:jason, "~> 1.3"}, {:credo, "~> 1.6", only: [:dev, :test], runtime: false}, {:ex_doc, ">= 0.0.0", only: :dev, runtime: false} diff --git a/spawn_statestores/statestores/priv/sqlite_lookup_adapter/migrations/20220521162411_create_lookups_table.exs b/spawn_statestores/statestores/priv/sqlite_lookup_adapter/migrations/20220521162411_create_lookups_table.exs deleted file mode 100644 index 5f4b124c..00000000 --- a/spawn_statestores/statestores/priv/sqlite_lookup_adapter/migrations/20220521162411_create_lookups_table.exs +++ /dev/null @@ -1,22 +0,0 @@ -defmodule Statestores.Adapters.SQLite3LookupAdapter.Migrations.CreateLookupsTable do - use Ecto.Migration - - def up do - execute """ - CREATE TABLE IF NOT EXISTS lookups ( - id INTEGER, - node TEXT, - actor TEXT, - system TEXT, - data BLOB, - inserted_at TEXT_DATETIME, - updated_at TEXT_DATETIME, - PRIMARY KEY (id, node) - ) - """ - end - - def down do - drop table(:lookups) - end -end diff --git a/spawn_statestores/statestores/priv/sqlite_snapshot_adapter/migrations/20220521162410_create_snapshots_table.exs b/spawn_statestores/statestores/priv/sqlite_snapshot_adapter/migrations/20220521162410_create_snapshots_table.exs deleted file mode 100644 index 3df1bb91..00000000 --- a/spawn_statestores/statestores/priv/sqlite_snapshot_adapter/migrations/20220521162410_create_snapshots_table.exs +++ /dev/null @@ -1,25 +0,0 @@ -defmodule Statestores.Adapters.SQLite3SnapshotAdapter.Migrations.CreateSnapshotsTable do - use Ecto.Migration - - def up do - execute """ - CREATE TABLE IF NOT EXISTS snapshots ( - id INTEGER PRIMARY KEY, - actor TEXT, - system TEXT, - status TEXT, - node TEXT, - revision INTEGER DEFAULT 0, - tags JSON, - data_type TEXT NOT NULL, - data BLOB, - inserted_at TEXT_DATETIME, - updated_at TEXT_DATETIME - ) - """ - end - - def down do - drop table(:snapshots) - end -end diff --git a/spawn_statestores/statestores_mariadb/lib/statestores/adapters/mariadb_projection_adapter.ex b/spawn_statestores/statestores_mariadb/lib/statestores/adapters/mariadb_projection_adapter.ex new file mode 100644 index 00000000..b0535c7a --- /dev/null +++ b/spawn_statestores/statestores_mariadb/lib/statestores/adapters/mariadb_projection_adapter.ex @@ -0,0 +1,305 @@ +defmodule Statestores.Adapters.MariaDBProjectionAdapter do + @moduledoc """ + Implements the ProjectionBehaviour for MariaDB, with dynamic table name support. + """ + use Statestores.Adapters.ProjectionBehaviour + use Ecto.Repo, otp_app: :spawn_statestores, adapter: Ecto.Adapters.MyXQL + use Scrivener, page_size: 50 + + alias Statestores.Schemas.{Projection, ValueObjectSchema} + + import Ecto.Query + import Statestores.Util, only: [normalize_table_name: 1] + + @impl true + def create_table(nil), do: {:error, "Projection name cannot be nil."} + + def create_table(projection_name) do + table_name = normalize_table_name(projection_name) + + query = """ + CREATE TABLE IF NOT EXISTS #{table_name} ( + id VARCHAR(255) PRIMARY KEY, + projection_id VARCHAR(255), + projection_name VARCHAR(255), + system VARCHAR(255), + metadata JSON, + data_type VARCHAR(255), + data BLOB, + inserted_at DATETIME, + updated_at DATETIME + ); + """ + + case Ecto.Adapters.SQL.query(__MODULE__, query) do + {:ok, _result} -> + {:ok, "Table #{table_name} created or already exists."} + + {:error, ex} -> + {:error, "Error during creation of table #{table_name}.: #{inspect(ex)}"} + end + end + + @impl true + def get_last(nil), do: {:error, "No record found"} + + def get_last(projection_name) do + table_name = normalize_table_name(projection_name) + + query = + from(p in {table_name, Projection}, + order_by: [desc: p.updated_at], + limit: 1 + ) + + fetch_single_record(query) + end + + @impl true + def get_last_by_projection_id(nil, _projection_id), do: {:error, "No record found"} + def get_last_by_projection_id(_projection_name, nil), do: {:error, "No record found"} + + def get_last_by_projection_id(projection_name, projection_id) do + table_name = normalize_table_name(projection_name) + + query = + from(p in {table_name, Projection}, + where: p.projection_id == ^projection_id, + order_by: [desc: p.updated_at], + limit: 1 + ) + + fetch_single_record(query) + end + + @impl true + def get_all(nil, _page, _page_size), do: {:error, "No records found"} + + def get_all(projection_name, page \\ 1, page_size \\ 50) do + table_name = normalize_table_name(projection_name) + + query = + from(p in {table_name, Projection}, + order_by: [asc: p.inserted_at] + ) + + paginate_query(query, page, page_size) + end + + @impl true + def get_all_by_projection_id(nil, _projection_id, _page, _page_size), + do: {:error, "No records found"} + + def get_all_by_projection_id(_projection_name, nil, _page, _page_size), + do: {:error, "No records found"} + + def get_all_by_projection_id(projection_name, projection_id, page \\ 1, page_size \\ 50) do + table_name = normalize_table_name(projection_name) + + query = + from(p in {table_name, Projection}, + where: p.projection_id == ^projection_id, + order_by: [asc: p.inserted_at] + ) + + paginate_query(query, page, page_size) + end + + @impl true + def get_by_interval(nil, _time_start, _time_end, _page, _page_size), + do: {:error, "No records found"} + + def get_by_interval(_projection_name, nil, _time_end, _page, _page_size), + do: {:error, "No records found"} + + def get_by_interval(_projection_name, _time_start, nil, _page, _page_size), + do: {:error, "No records found"} + + def get_by_interval(projection_name, time_start, time_end, page \\ 1, page_size \\ 50) do + table_name = normalize_table_name(projection_name) + + query = + from(p in {table_name, Projection}, + where: p.inserted_at >= ^time_start and p.inserted_at <= ^time_end, + order_by: [asc: p.inserted_at] + ) + + paginate_query(query, page, page_size) + end + + @impl true + def get_by_projection_id_and_interval( + nil, + _projection_id, + _time_start, + _time_end, + _page, + _page_size + ), + do: {:error, "No records found"} + + def get_by_projection_id_and_interval( + _projection_name, + nil, + _time_start, + _time_end, + _page, + _page_size + ), + do: {:error, "No records found"} + + def get_by_projection_id_and_interval( + _projection_name, + _projection_id, + nil, + _time_end, + _page, + _page_size + ), + do: {:error, "No records found"} + + def get_by_projection_id_and_interval( + _projection_name, + _projection_id, + _time_start, + nil, + _page, + _page_size + ), + do: {:error, "No records found"} + + def get_by_projection_id_and_interval( + projection_name, + projection_id, + time_start, + time_end, + page \\ 1, + page_size \\ 50 + ) do + table_name = normalize_table_name(projection_name) + + query = + from(p in {table_name, Projection}, + where: + p.projection_id == ^projection_id and p.inserted_at >= ^time_start and + p.inserted_at <= ^time_end, + order_by: [asc: p.inserted_at] + ) + + paginate_query(query, page, page_size) + end + + @impl true + def search_by_metadata( + projection_name, + metadata_key, + metadata_value, + page \\ 1, + page_size \\ 50 + ) do + table_name = normalize_table_name(projection_name) + key = "$.#{metadata_key}" + + query = + from(p in {table_name, Projection}, + where: + fragment("JSON_UNQUOTE(JSON_EXTRACT(?, ?)) = ?", p.metadata, ^key, ^metadata_value), + order_by: [asc: p.inserted_at] + ) + + paginate_query(query, page, page_size) + end + + @impl true + def search_by_projection_id_and_metadata( + projection_name, + projection_id, + metadata_key, + metadata_value, + page \\ 1, + page_size \\ 50 + ) do + table_name = normalize_table_name(projection_name) + key = "$.#{metadata_key}" + + query = + from(p in {table_name, Projection}, + where: + p.projection_id == ^projection_id and + fragment("JSON_UNQUOTE(JSON_EXTRACT(?, ?)) = ?", p.metadata, ^key, ^metadata_value), + order_by: [asc: p.inserted_at] + ) + + paginate_query(query, page, page_size) + end + + @impl true + def save(%Projection{} = projection) do + table_name = normalize_table_name(projection.projection_name) + record = ValueObjectSchema.to_map(projection) + {:ok, data} = Statestores.Vault.encrypt(record.data) + + query = """ + INSERT INTO #{table_name} + (id, projection_id, projection_name, system, metadata, data_type, data, inserted_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + projection_id = VALUES(projection_id), + projection_name = VALUES(projection_name), + system = VALUES(system), + metadata = VALUES(metadata), + data_type = VALUES(data_type), + data = VALUES(data), + inserted_at = VALUES(inserted_at), + updated_at = VALUES(updated_at) + """ + + bindings = [ + record.id, + record.projection_id, + record.projection_name, + record.system, + to_json(record.metadata), + record.data_type, + data, + record.inserted_at, + record.updated_at + ] + + case Ecto.Adapters.SQL.query(__MODULE__, query, bindings) do + {:ok, _result} -> + {:ok, projection} + + {:error, reason} -> + {:error, reason} + end + end + + @impl true + def default_port, do: "3306" + + defp to_json(nil), do: Jason.encode!(%{}) + defp to_json(map), do: Jason.encode!(map) + + # Private helper to fetch a single record from the database + defp fetch_single_record(query) do + case __MODULE__.one(query) do + nil -> + {:error, "No record found"} + + projection -> + {:ok, projection} + end + end + + # Private helper to handle pagination + defp paginate_query(query, page, page_size) do + case __MODULE__.paginate(query, page: page, page_size: page_size) do + %Scrivener.Page{} = page_data -> + {:ok, page_data} + + _ -> + {:error, "No records found"} + end + end +end diff --git a/spawn_statestores/statestores_mariadb/mix.exs b/spawn_statestores/statestores_mariadb/mix.exs index caa7a4f7..f67f6510 100644 --- a/spawn_statestores/statestores_mariadb/mix.exs +++ b/spawn_statestores/statestores_mariadb/mix.exs @@ -12,10 +12,10 @@ defmodule StatestoresMysql.MixProject do description: "Spawn Statestores Mysql is a storage lib for the Spawn Actors System", source_url: @source_url, homepage_url: "https://eigr.io/", - build_path: "../../_build", + build_path: "./_build", config_path: "../../config/config.exs", - deps_path: "../../deps", - lockfile: "../../mix.lock", + deps_path: "./deps", + lockfile: "./mix.lock", elixir: "~> 1.15", start_permanent: Mix.env() == :prod, deps: deps(), diff --git a/spawn_statestores/statestores_mariadb/mix.lock b/spawn_statestores/statestores_mariadb/mix.lock new file mode 100644 index 00000000..d11fb77f --- /dev/null +++ b/spawn_statestores/statestores_mariadb/mix.lock @@ -0,0 +1,20 @@ +%{ + "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"}, + "cloak": {:hex, :cloak, "1.1.4", "aba387b22ea4d80d92d38ab1890cc528b06e0e7ef2a4581d71c3fdad59e997e7", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "92b20527b9aba3d939fab0dd32ce592ff86361547cfdc87d74edce6f980eb3d7"}, + "cloak_ecto": {:hex, :cloak_ecto, "1.3.0", "0de127c857d7452ba3c3367f53fb814b0410ff9c680a8d20fbe8b9a3c57a1118", [:mix], [{:cloak, "~> 1.1.1", [hex: :cloak, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "314beb0c123b8a800418ca1d51065b27ba3b15f085977e65c0f7b2adab2de1cc"}, + "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, + "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"}, + "ecto": {:hex, :ecto, "3.12.3", "1a9111560731f6c3606924c81c870a68a34c819f6d4f03822f370ea31a582208", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9efd91506ae722f95e48dc49e70d0cb632ede3b7a23896252a60a14ac6d59165"}, + "ecto_sql": {:hex, :ecto_sql, "3.12.0", "73cea17edfa54bde76ee8561b30d29ea08f630959685006d9c6e7d1e59113b7d", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.12", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dc9e4d206f274f3947e96142a8fdc5f69a2a6a9abb4649ef5c882323b6d512f0"}, + "ex_doc": {:hex, :ex_doc, "0.34.2", "13eedf3844ccdce25cfd837b99bea9ad92c4e511233199440488d217c92571e8", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "5ce5f16b41208a50106afed3de6a2ed34f4acfd65715b82a0b84b49d995f95c1"}, + "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, + "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, + "makeup_erlang": {:hex, :makeup_erlang, "1.0.1", "c7f58c120b2b5aa5fd80d540a89fdf866ed42f1f3994e4fe189abebeab610839", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "8a89a1eeccc2d798d6ea15496a6e4870b75e014d1af514b1b71fa33134f57814"}, + "myxql": {:hex, :myxql, "0.7.1", "7c7b75aa82227cd2bc9b7fbd4de774fb19a1cdb309c219f411f82ca8860f8e01", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:geo, "~> 3.4", [hex: :geo, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a491cdff53353a09b5850ac2d472816ebe19f76c30b0d36a43317a67c9004936"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, + "scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"}, + "scrivener_ecto": {:hex, :scrivener_ecto, "2.7.0", "cf64b8cb8a96cd131cdbcecf64e7fd395e21aaa1cb0236c42a7c2e34b0dca580", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:scrivener, "~> 2.4", [hex: :scrivener, repo: "hexpm", optional: false]}], "hexpm", "e809f171687806b0031129034352f5ae44849720c48dd839200adeaf0ac3e260"}, + "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, +} diff --git a/spawn_statestores/statestores_mariadb/test/mariadb_projection_adapter_test.exs b/spawn_statestores/statestores_mariadb/test/mariadb_projection_adapter_test.exs new file mode 100644 index 00000000..15b22273 --- /dev/null +++ b/spawn_statestores/statestores_mariadb/test/mariadb_projection_adapter_test.exs @@ -0,0 +1,262 @@ +defmodule Statestores.Adapters.MariaDBProjectionAdapterTest do + use Statestores.DataCase + alias Statestores.Adapters.MariaDBProjectionAdapter, as: Adapter + alias Statestores.Schemas.Projection + + import Statestores.Util, only: [load_projection_adapter: 0] + + setup do + repo = load_projection_adapter() + %{repo: repo} + end + + describe "create_table/1" do + test "creates a table if it does not exist" do + projection_name = "test_projections" + + assert {:ok, message} = Adapter.create_table(projection_name) + assert message == "Table #{projection_name} created or already exists." + end + end + + describe "get_last/1" do + test "returns the last inserted projection", ctx do + repo = ctx.repo + IO.inspect(repo) + projection_name = "test_projections" + + {:ok, _} = + repo.save(%Projection{ + id: "123", + projection_id: "proj_1", + projection_name: projection_name, + system: "test_system", + metadata: %{"key" => "value"}, + data_type: "type.googleapis.com/io.eigr.spawn.example.MyState", + data: <<1, 2, 3>>, + inserted_at: DateTime.utc_now(), + updated_at: DateTime.utc_now() + }) + + assert {:ok, projection} = Adapter.get_last(projection_name) + assert projection.projection_id == "proj_1" + end + end + + describe "get_last_by_projection_id/2" do + test "returns the last inserted projection for a specific projection_id", ctx do + repo = ctx.repo + projection_name = "test_projections" + projection_id = "proj_1" + + {:ok, _} = + repo.save(%Projection{ + id: "123", + projection_id: projection_id, + projection_name: projection_name, + system: "test_system", + metadata: %{"key" => "value"}, + data_type: "type.googleapis.com/io.eigr.spawn.example.MyState", + data: <<1, 2, 3>>, + inserted_at: DateTime.utc_now(), + updated_at: DateTime.utc_now() + }) + + assert {:ok, projection} = + Adapter.get_last_by_projection_id(projection_name, projection_id) + + assert projection.projection_id == projection_id + end + end + + describe "get_all/3" do + test "returns paginated projections", ctx do + repo = ctx.repo + projection_name = "test_projections" + + Enum.each(1..20, fn n -> + repo.save(%Projection{ + id: "#{n}", + projection_id: "proj_#{n}", + projection_name: projection_name, + system: "test_system", + metadata: %{"key" => "value#{n}"}, + data_type: "type.googleapis.com/io.eigr.spawn.example.MyState", + data: <<1, 2, 3>>, + inserted_at: DateTime.utc_now(), + updated_at: DateTime.utc_now() + }) + end) + + {:ok, result} = Adapter.get_all(projection_name, 1, 10) + IO.inspect(result, label: "Pagination Result -----------") + assert length(result.entries) == 10 + assert result.page_number == 1 + end + end + + describe "search_by_metadata/5" do + test "returns projections matching metadata key and value", ctx do + repo = ctx.repo + projection_name = "test_projections" + metadata_key = "key" + metadata_value = "value1" + + repo.save(%Projection{ + id: "1", + projection_id: "proj_1", + projection_name: projection_name, + system: "test_system", + metadata: %{"key" => "value1"}, + data_type: "type.googleapis.com/io.eigr.spawn.example.MyState", + data: <<1, 2, 3>>, + inserted_at: DateTime.utc_now(), + updated_at: DateTime.utc_now() + }) + + repo.save(%Projection{ + id: "2", + projection_id: "proj_2", + projection_name: projection_name, + system: "test_system", + metadata: %{"key" => "value2"}, + data_type: "type.googleapis.com/io.eigr.spawn.example.MyState", + data: <<1, 2, 3>>, + inserted_at: DateTime.utc_now(), + updated_at: DateTime.utc_now() + }) + + {:ok, result} = Adapter.get_all(projection_name) + assert length(result.entries) == 2 + + {:ok, result} = + Adapter.search_by_metadata( + projection_name, + metadata_key, + metadata_value + ) + + assert length(result.entries) == 1 + assert result.entries |> Enum.at(0) |> Map.get(:projection_id) == "proj_1" + end + end + + describe "search_by_projection_id_and_metadata/6" do + test "returns projections matching projection_id and metadata", ctx do + repo = ctx.repo + projection_name = "test_projections" + projection_id = "proj_1" + metadata_key = "key" + metadata_value = "value1" + + repo.save(%Projection{ + id: "1", + projection_id: projection_id, + projection_name: projection_name, + system: "test_system", + metadata: %{"key" => "value1"}, + data_type: "type.googleapis.com/io.eigr.spawn.example.MyState", + data: <<1, 2, 3>>, + inserted_at: DateTime.utc_now(), + updated_at: DateTime.utc_now() + }) + + {:ok, result} = + Adapter.search_by_projection_id_and_metadata( + projection_name, + projection_id, + metadata_key, + metadata_value + ) + + assert length(result.entries) == 1 + assert result.entries |> Enum.at(0) |> Map.get(:projection_id) == projection_id + end + end + + describe "fail for get_last_by_projection_id/2" do + test "returns error if no matching projection_id found", _ctx do + projection_name = "test_projections" + non_existing_projection_id = "non_existing_proj" + + assert {:error, _error_msg} = + Adapter.get_last_by_projection_id( + projection_name, + non_existing_projection_id + ) + end + end + + describe "fail get_last/1" do + test "returns error if no projections exist", ctx do + projection_name = "empty_projections_table" + + assert_raise MyXQL.Error, fn -> + Adapter.get_last(projection_name) + end + end + end + + describe "fail search_by_metadata/5" do + test "returns no results for non-existing metadata key", ctx do + repo = ctx.repo + projection_name = "test_projections" + invalid_metadata_key = "non_existing_key" + metadata_value = "value1" + + {:ok, result} = + Adapter.search_by_metadata( + projection_name, + invalid_metadata_key, + metadata_value + ) + + assert length(result.entries) == 0 + end + + test "returns no results for non-existing metadata value", ctx do + repo = ctx.repo + projection_name = "test_projections" + metadata_key = "key" + invalid_metadata_value = "non_existing_value" + + {:ok, result} = + Adapter.search_by_metadata( + projection_name, + metadata_key, + invalid_metadata_value + ) + + assert length(result.entries) == 0 + end + end + + describe "fail get_last_by_projection_id/2" do + test "returns error if invalid parameters are provided", ctx do + projection_name = "test_projections" + invalid_projection_id = nil + + assert {:error, _message} = + Adapter.get_last_by_projection_id( + projection_name, + invalid_projection_id + ) + end + end + + describe "fail get_all/3" do + test "returns empty results if requested page is out of bounds", ctx do + projection_name = "test_projections" + + {:ok, result} = Adapter.get_all(projection_name, 5, 10) + + IO.inspect(result, + label: "fail get_all/3 ----------------------------------------------------------------" + ) + + assert length(result.entries) == 0 + # this totall create on test + assert result.page_number == 1 + end + end +end diff --git a/spawn_statestores/statestores_mariadb/test/support/data_case.ex b/spawn_statestores/statestores_mariadb/test/support/data_case.ex index 98e14ef3..fa18c83c 100644 --- a/spawn_statestores/statestores_mariadb/test/support/data_case.ex +++ b/spawn_statestores/statestores_mariadb/test/support/data_case.ex @@ -5,7 +5,11 @@ defmodule Statestores.DataCase do using do quote do - use Statestores.SandboxHelper, repos: [Statestores.Util.load_snapshot_adapter()] + use Statestores.SandboxHelper, + repos: [ + # Statestores.Util.load_snapshot_adapter(), + Statestores.Util.load_projection_adapter() + ] import Statestores.DataCase end diff --git a/spawn_statestores/statestores_mariadb/test/test_helper.exs b/spawn_statestores/statestores_mariadb/test/test_helper.exs index 16f8bc05..4a6ce7f9 100644 --- a/spawn_statestores/statestores_mariadb/test/test_helper.exs +++ b/spawn_statestores/statestores_mariadb/test/test_helper.exs @@ -10,6 +10,12 @@ Application.put_env( Statestores.Adapters.MariaDBLookupAdapter ) +Application.put_env( + :spawn_statestores, + :database_projection_adapter, + Statestores.Adapters.MariaDBProjectionAdapter +) + ExUnit.start() Statestores.Supervisor.start_link(%{}) diff --git a/spawn_statestores/statestores_native/lib/statestores/adapters/native_projection_adapter.ex b/spawn_statestores/statestores_native/lib/statestores/adapters/native_projection_adapter.ex new file mode 100644 index 00000000..1b3b7ad2 --- /dev/null +++ b/spawn_statestores/statestores_native/lib/statestores/adapters/native_projection_adapter.ex @@ -0,0 +1,148 @@ +defmodule Statestores.Adapters.NativeProjectionAdapter do + @moduledoc """ + Implements the ProjectionBehaviour for Mnesia, with dynamic table name support. + """ + use Statestores.Adapters.ProjectionBehaviour + + alias Statestores.Schemas.Projection + + import Statestores.Util, only: [normalize_table_name: 1] + + @impl true + def create_table(nil), do: {:error, "Projection name cannot be nil."} + + def create_table(projection_name) do + table_name = normalize_table_name(projection_name) + {:ok, "Table #{table_name} created or already exists."} + end + + @impl true + def get_last(nil), do: {:error, "No record found"} + + def get_last(projection_name) do + nil + end + + @impl true + def get_last_by_projection_id(nil, _projection_id), do: {:error, "No record found"} + def get_last_by_projection_id(_projection_name, nil), do: {:error, "No record found"} + + def get_last_by_projection_id(projection_name, projection_id) do + nil + end + + @impl true + def get_all(nil, _page, _page_size), do: {:error, "No records found"} + + def get_all(projection_name, page \\ 1, page_size \\ 50) do + nil + end + + @impl true + def get_all_by_projection_id(nil, _projection_id, _page, _page_size), + do: {:error, "No records found"} + + def get_all_by_projection_id(_projection_name, nil, _page, _page_size), + do: {:error, "No records found"} + + def get_all_by_projection_id(projection_name, projection_id, page \\ 1, page_size \\ 50) do + nil + end + + @impl true + def get_by_interval(nil, _time_start, _time_end, _page, _page_size), + do: {:error, "No records found"} + + def get_by_interval(_projection_name, nil, _time_end, _page, _page_size), + do: {:error, "No records found"} + + def get_by_interval(_projection_name, _time_start, nil, _page, _page_size), + do: {:error, "No records found"} + + def get_by_interval(projection_name, time_start, time_end, page \\ 1, page_size \\ 50) do + nil + end + + @impl true + def get_by_projection_id_and_interval( + nil, + _projection_id, + _time_start, + _time_end, + _page, + _page_size + ), + do: {:error, "No records found"} + + def get_by_projection_id_and_interval( + _projection_name, + nil, + _time_start, + _time_end, + _page, + _page_size + ), + do: {:error, "No records found"} + + def get_by_projection_id_and_interval( + _projection_name, + _projection_id, + nil, + _time_end, + _page, + _page_size + ), + do: {:error, "No records found"} + + def get_by_projection_id_and_interval( + _projection_name, + _projection_id, + _time_start, + nil, + _page, + _page_size + ), + do: {:error, "No records found"} + + def get_by_projection_id_and_interval( + projection_name, + projection_id, + time_start, + time_end, + page \\ 1, + page_size \\ 50 + ) do + nil + end + + @impl true + def search_by_metadata( + projection_name, + metadata_key, + metadata_value, + page \\ 1, + page_size \\ 50 + ) do + nil + end + + @impl true + def search_by_projection_id_and_metadata( + projection_name, + projection_id, + metadata_key, + metadata_value, + page \\ 1, + page_size \\ 50 + ) do + nil + end + + @impl true + def save(%Projection{} = projection) do + {:ok, projection} + end + + @impl true + def default_port, do: "0000" +end diff --git a/spawn_statestores/statestores_native/mix.exs b/spawn_statestores/statestores_native/mix.exs index fa0b41f1..689a7fac 100644 --- a/spawn_statestores/statestores_native/mix.exs +++ b/spawn_statestores/statestores_native/mix.exs @@ -13,10 +13,10 @@ defmodule StatestoresNative.MixProject do "Spawn Statestores Native is a storage lib for the Spawn Actors System using Mnesia", source_url: @source_url, homepage_url: "https://eigr.io/", - build_path: "../../_build", + build_path: "./_build", config_path: "../../config/config.exs", - deps_path: "../../deps", - lockfile: "../../mix.lock", + deps_path: "./deps", + lockfile: "./mix.lock", elixir: "~> 1.15", start_permanent: Mix.env() == :prod, deps: deps(), diff --git a/spawn_statestores/statestores_native/mix.lock b/spawn_statestores/statestores_native/mix.lock new file mode 100644 index 00000000..cd209616 --- /dev/null +++ b/spawn_statestores/statestores_native/mix.lock @@ -0,0 +1,20 @@ +%{ + "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"}, + "cloak": {:hex, :cloak, "1.1.4", "aba387b22ea4d80d92d38ab1890cc528b06e0e7ef2a4581d71c3fdad59e997e7", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "92b20527b9aba3d939fab0dd32ce592ff86361547cfdc87d74edce6f980eb3d7"}, + "cloak_ecto": {:hex, :cloak_ecto, "1.3.0", "0de127c857d7452ba3c3367f53fb814b0410ff9c680a8d20fbe8b9a3c57a1118", [:mix], [{:cloak, "~> 1.1.1", [hex: :cloak, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "314beb0c123b8a800418ca1d51065b27ba3b15f085977e65c0f7b2adab2de1cc"}, + "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, + "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"}, + "ecto": {:hex, :ecto, "3.12.3", "1a9111560731f6c3606924c81c870a68a34c819f6d4f03822f370ea31a582208", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9efd91506ae722f95e48dc49e70d0cb632ede3b7a23896252a60a14ac6d59165"}, + "ecto_sql": {:hex, :ecto_sql, "3.12.0", "73cea17edfa54bde76ee8561b30d29ea08f630959685006d9c6e7d1e59113b7d", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.12", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dc9e4d206f274f3947e96142a8fdc5f69a2a6a9abb4649ef5c882323b6d512f0"}, + "ex_doc": {:hex, :ex_doc, "0.34.2", "13eedf3844ccdce25cfd837b99bea9ad92c4e511233199440488d217c92571e8", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "5ce5f16b41208a50106afed3de6a2ed34f4acfd65715b82a0b84b49d995f95c1"}, + "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, + "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, + "makeup_erlang": {:hex, :makeup_erlang, "1.0.1", "c7f58c120b2b5aa5fd80d540a89fdf866ed42f1f3994e4fe189abebeab610839", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "8a89a1eeccc2d798d6ea15496a6e4870b75e014d1af514b1b71fa33134f57814"}, + "mnesiac": {:hex, :mnesiac, "0.3.14", "5ea3f1f3e615073629d0822bcf2297be73149beee2d1f7e482c1943894f59b53", [:mix], [{:libcluster, "~> 3.3", [hex: :libcluster, repo: "hexpm", optional: true]}], "hexpm", "e51b38bf983b9320aba56d5dce79dbf50cbff07f7495e70b89eb45461b8d32fa"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, + "scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"}, + "scrivener_ecto": {:hex, :scrivener_ecto, "2.7.0", "cf64b8cb8a96cd131cdbcecf64e7fd395e21aaa1cb0236c42a7c2e34b0dca580", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:scrivener, "~> 2.4", [hex: :scrivener, repo: "hexpm", optional: false]}], "hexpm", "e809f171687806b0031129034352f5ae44849720c48dd839200adeaf0ac3e260"}, + "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, +} diff --git a/spawn_statestores/statestores_postgres/lib/statestores/adapters/postgres_projection_adapter.ex b/spawn_statestores/statestores_postgres/lib/statestores/adapters/postgres_projection_adapter.ex new file mode 100644 index 00000000..a29bf562 --- /dev/null +++ b/spawn_statestores/statestores_postgres/lib/statestores/adapters/postgres_projection_adapter.ex @@ -0,0 +1,308 @@ +defmodule Statestores.Adapters.PostgresProjectionAdapter do + @moduledoc """ + Implements the ProjectionBehaviour for Postgres, with dynamic table name support. + """ + use Statestores.Adapters.ProjectionBehaviour + + use Ecto.Repo, + otp_app: :spawn_statestores, + adapter: Ecto.Adapters.Postgres + + use Scrivener, page_size: 50 + + alias Statestores.Schemas.{Projection, ValueObjectSchema} + + import Ecto.Query + import Statestores.Util, only: [normalize_table_name: 1] + + @impl true + def create_table(nil), do: {:error, "Projection name cannot be nil."} + + def create_table(projection_name) do + table_name = normalize_table_name(projection_name) + + query = """ + CREATE TABLE IF NOT EXISTS #{table_name} ( + id VARCHAR(255) PRIMARY KEY, + projection_id VARCHAR(255), + projection_name VARCHAR(255), + system VARCHAR(255), + metadata JSON, + data_type VARCHAR(255), + data BYTEA, + inserted_at TIMESTAMP, + updated_at TIMESTAMP + ); + """ + + case Ecto.Adapters.SQL.query(__MODULE__, query) do + {:ok, _result} -> + {:ok, "Table #{table_name} created or already exists."} + + {:error, ex} -> + {:error, "Error during creation of table #{table_name}.: #{inspect(ex)}"} + end + end + + @impl true + def get_last(nil), do: {:error, "No record found"} + + def get_last(projection_name) do + table_name = normalize_table_name(projection_name) + + query = + from(p in {table_name, Projection}, + order_by: [desc: p.updated_at], + limit: 1 + ) + + fetch_single_record(query) + end + + @impl true + def get_last_by_projection_id(nil, _projection_id), do: {:error, "No record found"} + def get_last_by_projection_id(_projection_name, nil), do: {:error, "No record found"} + + def get_last_by_projection_id(projection_name, projection_id) do + table_name = normalize_table_name(projection_name) + + query = + from(p in {table_name, Projection}, + where: p.projection_id == ^projection_id, + order_by: [desc: p.updated_at], + limit: 1 + ) + + fetch_single_record(query) + end + + @impl true + def get_all(nil, _page, _page_size), do: {:error, "No records found"} + + def get_all(projection_name, page \\ 1, page_size \\ 50) do + table_name = normalize_table_name(projection_name) + + query = + from(p in {table_name, Projection}, + order_by: [asc: p.inserted_at] + ) + + paginate_query(query, page, page_size) + end + + @impl true + def get_all_by_projection_id(nil, _projection_id, _page, _page_size), + do: {:error, "No records found"} + + def get_all_by_projection_id(_projection_name, nil, _page, _page_size), + do: {:error, "No records found"} + + def get_all_by_projection_id(projection_name, projection_id, page \\ 1, page_size \\ 50) do + table_name = normalize_table_name(projection_name) + + query = + from(p in {table_name, Projection}, + where: p.projection_id == ^projection_id, + order_by: [asc: p.inserted_at] + ) + + paginate_query(query, page, page_size) + end + + @impl true + def get_by_interval(nil, _time_start, _time_end, _page, _page_size), + do: {:error, "No records found"} + + def get_by_interval(_projection_name, nil, _time_end, _page, _page_size), + do: {:error, "No records found"} + + def get_by_interval(_projection_name, _time_start, nil, _page, _page_size), + do: {:error, "No records found"} + + def get_by_interval(projection_name, time_start, time_end, page \\ 1, page_size \\ 50) do + table_name = normalize_table_name(projection_name) + + query = + from(p in {table_name, Projection}, + where: p.inserted_at >= ^time_start and p.inserted_at <= ^time_end, + order_by: [asc: p.inserted_at] + ) + + paginate_query(query, page, page_size) + end + + @impl true + def get_by_projection_id_and_interval( + nil, + _projection_id, + _time_start, + _time_end, + _page, + _page_size + ), + do: {:error, "No records found"} + + def get_by_projection_id_and_interval( + _projection_name, + nil, + _time_start, + _time_end, + _page, + _page_size + ), + do: {:error, "No records found"} + + def get_by_projection_id_and_interval( + _projection_name, + _projection_id, + nil, + _time_end, + _page, + _page_size + ), + do: {:error, "No records found"} + + def get_by_projection_id_and_interval( + _projection_name, + _projection_id, + _time_start, + nil, + _page, + _page_size + ), + do: {:error, "No records found"} + + def get_by_projection_id_and_interval( + projection_name, + projection_id, + time_start, + time_end, + page \\ 1, + page_size \\ 50 + ) do + table_name = normalize_table_name(projection_name) + + query = + from(p in {table_name, Projection}, + where: + p.projection_id == ^projection_id and p.inserted_at >= ^time_start and + p.inserted_at <= ^time_end, + order_by: [asc: p.inserted_at] + ) + + paginate_query(query, page, page_size) + end + + @impl true + def search_by_metadata( + projection_name, + metadata_key, + metadata_value, + page \\ 1, + page_size \\ 50 + ) do + table_name = normalize_table_name(projection_name) + + query = + from(p in {table_name, Projection}, + where: fragment("?->>? = ?", p.metadata, ^metadata_key, ^metadata_value), + order_by: [asc: p.inserted_at] + ) + + paginate_query(query, page, page_size) + end + + @impl true + def search_by_projection_id_and_metadata( + projection_name, + projection_id, + metadata_key, + metadata_value, + page \\ 1, + page_size \\ 50 + ) do + table_name = normalize_table_name(projection_name) + + query = + from(p in {table_name, Projection}, + where: + p.projection_id == ^projection_id and + fragment("?->>? = ?", p.metadata, ^metadata_key, ^metadata_value), + order_by: [asc: p.inserted_at] + ) + + paginate_query(query, page, page_size) + end + + @impl true + def save(%Projection{} = projection) do + record = ValueObjectSchema.to_map(projection) + {:ok, data} = Statestores.Vault.encrypt(record.data) + + # Prepare the SQL query using `ON CONFLICT` + query = """ + INSERT INTO #{projection.projection_name} + (id, projection_id, projection_name, system, metadata, data_type, data, inserted_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (id) DO UPDATE + SET + projection_id = EXCLUDED.projection_id, + projection_name = EXCLUDED.projection_name, + system = EXCLUDED.system, + metadata = EXCLUDED.metadata, + data_type = EXCLUDED.data_type, + data = EXCLUDED.data, + inserted_at = EXCLUDED.inserted_at, + updated_at = EXCLUDED.updated_at + """ + + bindings = [ + record.id, + record.projection_id, + record.projection_name, + record.system, + record.metadata, + record.data_type, + data, + record.inserted_at, + record.updated_at + ] + + # Execute the query using Ecto.Adapters.SQL.query/4 + case Ecto.Adapters.SQL.query(__MODULE__, query, bindings) do + {:ok, _result} -> + {:ok, projection} + + {:error, reason} -> + {:error, reason} + end + end + + @impl true + def default_port, do: "5432" + + defp to_json(nil), do: Jason.encode!(%{}) + defp to_json(map), do: Jason.encode!(map) + + # Private helper to fetch a single record from the database + defp fetch_single_record(query) do + case __MODULE__.one(query) do + nil -> + {:error, "No record found"} + + projection -> + {:ok, projection} + end + end + + # Private helper to handle pagination + defp paginate_query(query, page, page_size) do + case __MODULE__.paginate(query, page: page, page_size: page_size) do + %Scrivener.Page{} = page_data -> + {:ok, page_data} + + _ -> + {:error, "No records found"} + end + end +end diff --git a/spawn_statestores/statestores_postgres/mix.exs b/spawn_statestores/statestores_postgres/mix.exs index 0e313c62..e546ac40 100644 --- a/spawn_statestores/statestores_postgres/mix.exs +++ b/spawn_statestores/statestores_postgres/mix.exs @@ -12,10 +12,10 @@ defmodule StatestoresPostgres.MixProject do description: "Spawn Statestores Postgres is a storage lib for the Spawn Actors System", source_url: @source_url, homepage_url: "https://eigr.io/", - build_path: "../../_build", + build_path: "./_build", config_path: "../../config/config.exs", - deps_path: "../../deps", - lockfile: "../../mix.lock", + deps_path: "./deps", + lockfile: "./mix.lock", elixir: "~> 1.15", start_permanent: Mix.env() == :prod, deps: deps(), diff --git a/spawn_statestores/statestores_postgres/mix.lock b/spawn_statestores/statestores_postgres/mix.lock new file mode 100644 index 00000000..cd3b6adc --- /dev/null +++ b/spawn_statestores/statestores_postgres/mix.lock @@ -0,0 +1,20 @@ +%{ + "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"}, + "cloak": {:hex, :cloak, "1.1.4", "aba387b22ea4d80d92d38ab1890cc528b06e0e7ef2a4581d71c3fdad59e997e7", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "92b20527b9aba3d939fab0dd32ce592ff86361547cfdc87d74edce6f980eb3d7"}, + "cloak_ecto": {:hex, :cloak_ecto, "1.3.0", "0de127c857d7452ba3c3367f53fb814b0410ff9c680a8d20fbe8b9a3c57a1118", [:mix], [{:cloak, "~> 1.1.1", [hex: :cloak, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "314beb0c123b8a800418ca1d51065b27ba3b15f085977e65c0f7b2adab2de1cc"}, + "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, + "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"}, + "ecto": {:hex, :ecto, "3.12.3", "1a9111560731f6c3606924c81c870a68a34c819f6d4f03822f370ea31a582208", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9efd91506ae722f95e48dc49e70d0cb632ede3b7a23896252a60a14ac6d59165"}, + "ecto_sql": {:hex, :ecto_sql, "3.12.0", "73cea17edfa54bde76ee8561b30d29ea08f630959685006d9c6e7d1e59113b7d", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.12", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dc9e4d206f274f3947e96142a8fdc5f69a2a6a9abb4649ef5c882323b6d512f0"}, + "ex_doc": {:hex, :ex_doc, "0.34.2", "13eedf3844ccdce25cfd837b99bea9ad92c4e511233199440488d217c92571e8", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "5ce5f16b41208a50106afed3de6a2ed34f4acfd65715b82a0b84b49d995f95c1"}, + "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, + "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, + "makeup_erlang": {:hex, :makeup_erlang, "1.0.1", "c7f58c120b2b5aa5fd80d540a89fdf866ed42f1f3994e4fe189abebeab610839", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "8a89a1eeccc2d798d6ea15496a6e4870b75e014d1af514b1b71fa33134f57814"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, + "postgrex": {:hex, :postgrex, "0.19.1", "73b498508b69aded53907fe48a1fee811be34cc720e69ef4ccd568c8715495ea", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "8bac7885a18f381e091ec6caf41bda7bb8c77912bb0e9285212829afe5d8a8f8"}, + "scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"}, + "scrivener_ecto": {:hex, :scrivener_ecto, "2.7.0", "cf64b8cb8a96cd131cdbcecf64e7fd395e21aaa1cb0236c42a7c2e34b0dca580", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:scrivener, "~> 2.4", [hex: :scrivener, repo: "hexpm", optional: false]}], "hexpm", "e809f171687806b0031129034352f5ae44849720c48dd839200adeaf0ac3e260"}, + "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, +} diff --git a/spawn_statestores/statestores_postgres/test/postgres_projection_adapter_test.exs b/spawn_statestores/statestores_postgres/test/postgres_projection_adapter_test.exs new file mode 100644 index 00000000..37d2f20b --- /dev/null +++ b/spawn_statestores/statestores_postgres/test/postgres_projection_adapter_test.exs @@ -0,0 +1,258 @@ +defmodule Statestores.Adapters.PostgresProjectionAdapterTest do + use Statestores.DataCase + alias Statestores.Adapters.PostgresProjectionAdapter, as: Adapter + alias Statestores.Schemas.Projection + + import Statestores.Util, only: [load_projection_adapter: 0] + + setup do + repo = load_projection_adapter() + projection_name = "test_projections" + _ = Adapter.create_table(projection_name) + %{repo: repo, projection_name: projection_name} + end + + describe "create_table/1" do + test "creates a table if it does not exist", ctx do + projection_name = ctx.projection_name + + assert {:ok, message} = Adapter.create_table(projection_name) + assert {:ok, result} = Adapter.get_all(projection_name, 1, 10) + assert message == "Table #{projection_name} created or already exists." + end + end + + describe "get_last/1" do + test "returns the last inserted projection", ctx do + repo = ctx.repo + IO.inspect(repo) + projection_name = ctx.projection_name + + {:ok, _} = + repo.save(%Projection{ + id: "123", + projection_id: "proj_1", + projection_name: projection_name, + system: "test_system", + metadata: %{"key" => "value"}, + data_type: "type.googleapis.com/io.eigr.spawn.example.MyState", + data: <<1, 2, 3>>, + inserted_at: DateTime.utc_now(), + updated_at: DateTime.utc_now() + }) + + assert {:ok, projection} = Adapter.get_last(projection_name) + assert projection.projection_id == "proj_1" + end + end + + describe "get_last_by_projection_id/2" do + test "returns the last inserted projection for a specific projection_id", ctx do + repo = ctx.repo + projection_name = ctx.projection_name + projection_id = "proj_1" + + {:ok, _} = + repo.save(%Projection{ + id: "123", + projection_id: projection_id, + projection_name: projection_name, + system: "test_system", + metadata: %{"key" => "value"}, + data_type: "type.googleapis.com/io.eigr.spawn.example.MyState", + data: <<1, 2, 3>>, + inserted_at: DateTime.utc_now(), + updated_at: DateTime.utc_now() + }) + + assert {:ok, projection} = + Adapter.get_last_by_projection_id(projection_name, projection_id) + + assert projection.projection_id == projection_id + end + end + + describe "get_all/3" do + test "returns paginated projections", ctx do + repo = ctx.repo + projection_name = ctx.projection_name + + Enum.each(1..20, fn n -> + repo.save(%Projection{ + id: "#{n}", + projection_id: "proj_#{n}", + projection_name: projection_name, + system: "test_system", + metadata: %{"key" => "value#{n}"}, + data_type: "type.googleapis.com/io.eigr.spawn.example.MyState", + data: <<1, 2, 3>>, + inserted_at: DateTime.utc_now(), + updated_at: DateTime.utc_now() + }) + end) + + {:ok, result} = Adapter.get_all(projection_name, 1, 10) + IO.inspect(result, label: "Pagination Result -----------") + assert length(result.entries) == 10 + assert result.page_number == 1 + end + end + + describe "search_by_metadata/5" do + test "returns projections matching metadata key and value", ctx do + repo = ctx.repo + projection_name = ctx.projection_name + metadata_key = "key" + metadata_value = "value1" + + repo.save(%Projection{ + id: "1", + projection_id: "proj_1", + projection_name: projection_name, + system: "test_system", + metadata: %{"key" => "value1"}, + data_type: "type.googleapis.com/io.eigr.spawn.example.MyState", + data: <<1, 2, 3>>, + inserted_at: DateTime.utc_now(), + updated_at: DateTime.utc_now() + }) + + repo.save(%Projection{ + id: "2", + projection_id: "proj_2", + projection_name: projection_name, + system: "test_system", + metadata: %{"key" => "value2"}, + data_type: "type.googleapis.com/io.eigr.spawn.example.MyState", + data: <<1, 2, 3>>, + inserted_at: DateTime.utc_now(), + updated_at: DateTime.utc_now() + }) + + {:ok, result} = Adapter.get_all(projection_name) + assert length(result.entries) == 2 + + {:ok, result} = + Adapter.search_by_metadata( + projection_name, + metadata_key, + metadata_value + ) + + assert length(result.entries) == 1 + assert result.entries |> Enum.at(0) |> Map.get(:projection_id) == "proj_1" + end + end + + describe "search_by_projection_id_and_metadata/6" do + test "returns projections matching projection_id and metadata", ctx do + repo = ctx.repo + projection_name = ctx.projection_name + projection_id = "proj_1" + metadata_key = "key" + metadata_value = "value1" + + repo.save(%Projection{ + id: "1", + projection_id: projection_id, + projection_name: projection_name, + system: "test_system", + metadata: %{"key" => "value1"}, + data_type: "type.googleapis.com/io.eigr.spawn.example.MyState", + data: <<1, 2, 3>>, + inserted_at: DateTime.utc_now(), + updated_at: DateTime.utc_now() + }) + + {:ok, result} = + Adapter.search_by_projection_id_and_metadata( + projection_name, + projection_id, + metadata_key, + metadata_value + ) + + assert length(result.entries) == 1 + assert result.entries |> Enum.at(0) |> Map.get(:projection_id) == projection_id + end + end + + describe "fail for get_last_by_projection_id/2" do + test "returns error if no matching projection_id found", ctx do + projection_name = ctx.projection_name + non_existing_projection_id = "non_existing_proj" + + assert {:error, _error_msg} = + Adapter.get_last_by_projection_id( + projection_name, + non_existing_projection_id + ) + end + end + + describe "fail get_last/1" do + test "returns error if no projections exist", _ctx do + projection_name = "empty_projections_table" + + assert_raise Postgrex.Error, fn -> + Adapter.get_last(projection_name) + end + end + end + + describe "fail search_by_metadata/5" do + test "returns no results for non-existing metadata key", ctx do + projection_name = ctx.projection_name + invalid_metadata_key = "non_existing_key" + metadata_value = "value1" + + {:ok, result} = + Adapter.search_by_metadata( + projection_name, + invalid_metadata_key, + metadata_value + ) + + assert length(result.entries) == 0 + end + + test "returns no results for non-existing metadata value", _ctx do + projection_name = "test_projections" + metadata_key = "key" + invalid_metadata_value = "non_existing_value" + + {:ok, result} = + Adapter.search_by_metadata( + projection_name, + metadata_key, + invalid_metadata_value + ) + + assert length(result.entries) == 0 + end + end + + describe "fail get_last_by_projection_id/2" do + test "returns error if invalid parameters are provided", ctx do + projection_name = ctx.projection_name + invalid_projection_id = nil + + assert {:error, _message} = + Adapter.get_last_by_projection_id( + projection_name, + invalid_projection_id + ) + end + end + + describe "fail get_all/3" do + test "returns empty results if requested page is out of bounds", ctx do + projection_name = ctx.projection_name + + {:ok, result} = Adapter.get_all(projection_name, 5, 10) + + assert length(result.entries) == 0 + assert result.page_number == 1 + end + end +end diff --git a/spawn_statestores/statestores_postgres/test/repo_test.bkp b/spawn_statestores/statestores_postgres/test/repo_test.bkp deleted file mode 100644 index 8bedc614..00000000 --- a/spawn_statestores/statestores_postgres/test/repo_test.bkp +++ /dev/null @@ -1,69 +0,0 @@ -defmodule StatestoresPostgresTest.RepoTest do - use Statestores.DataCase - alias Statestores.Schemas.Snapshot - import Statestores.Util, only: [load_snapshot_adapter: 0, generate_key: 1] - - setup do - %{system: "test-system"} - end - - test "insert! should persist an valid Event", ctx do - %{system: system} = ctx - - actor = "mike" - id = %{name: actor, system: system} - key = generate_key(id) - repo = load_snapshot_adapter() - - event = %Snapshot{ - id: key, - system: system, - actor: actor, - status: "ACTIVATED", - node: Atom.to_string(Node.self()), - revision: 0, - tags: %{}, - data_type: "type.googleapis.com/io.eigr.spawn.example.MyState", - data: "Hello Joe" - } - - _result = repo.save(event) - actor_state = repo.get_by_key(key) - - assert actor_state.data == "Hello Joe" - end - - test "insert! should update when inserted before", ctx do - %{system: system} = ctx - - actor = "mike" - id = %{name: actor, system: system} - key = generate_key(id) - repo = load_snapshot_adapter() - - event = %Snapshot{ - id: key, - system: system, - actor: actor, - status: "ACTIVATED", - node: Atom.to_string(Node.self()), - revision: 0, - tags: %{}, - data_type: "type.googleapis.com/io.eigr.spawn.example.MyState", - data: "Hello Joe" - } - - _result = repo.save(event) - actor_state = repo.get_by_key(key) - - Process.sleep(1000) - event = %{event | data: "new joe"} - _result = repo.save(event) - actor_state2 = repo.get_by_key(key) - - refute is_nil(actor_state.updated_at) - refute is_nil(actor_state2.updated_at) - assert actor_state.updated_at != actor_state2.updated_at - assert actor_state2.data == "new joe" - end -end diff --git a/spawn_statestores/statestores_postgres/test/support/data_case.ex b/spawn_statestores/statestores_postgres/test/support/data_case.ex index 98e14ef3..c60cd65e 100644 --- a/spawn_statestores/statestores_postgres/test/support/data_case.ex +++ b/spawn_statestores/statestores_postgres/test/support/data_case.ex @@ -5,7 +5,11 @@ defmodule Statestores.DataCase do using do quote do - use Statestores.SandboxHelper, repos: [Statestores.Util.load_snapshot_adapter()] + use Statestores.SandboxHelper, + repos: [ + Statestores.Util.load_snapshot_adapter(), + Statestores.Util.load_projection_adapter() + ] import Statestores.DataCase end diff --git a/spawn_statestores/statestores_postgres/test/test_helper.exs b/spawn_statestores/statestores_postgres/test/test_helper.exs index 2d28e7c0..e766a9f1 100644 --- a/spawn_statestores/statestores_postgres/test/test_helper.exs +++ b/spawn_statestores/statestores_postgres/test/test_helper.exs @@ -10,6 +10,12 @@ Application.put_env( Statestores.Adapters.PostgresLookupAdapter ) +Application.put_env( + :spawn_statestores, + :database_projection_adapter, + Statestores.Adapters.PostgresProjectionAdapter +) + ExUnit.start() Statestores.Supervisor.start_link(%{}) diff --git a/spawnctl/lib/spawnctl/commands/dev/run.ex b/spawnctl/lib/spawnctl/commands/dev/run.ex index 33d77bf5..3a9328d4 100644 --- a/spawnctl/lib/spawnctl/commands/dev/run.ex +++ b/spawnctl/lib/spawnctl/commands/dev/run.ex @@ -24,7 +24,9 @@ defmodule SpawnCtl.Commands.Dev.Run do - Statestore Key: "myfake-key-3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE=" - Logger level: "info" - Proxy instance name: "proxy" - - Use Nats for cross ActorSystem communication: false + - Nats image: "nats" + - Nats http port: 8222 + - Nats port: 4222 ### Example 2: Running with Custom Actor System and Database Host @@ -63,7 +65,9 @@ defmodule SpawnCtl.Commands.Dev.Run do --database-pool 50 \ --statestore-key "custom-key" \ --log-level "debug" \ - --enable-nats true \ + --nats-image "nats" \ + --nats-http-port 8222 \ + --nats-port 4222 \ --name "custom-proxy" """ use DoIt.Command, @@ -93,7 +97,9 @@ defmodule SpawnCtl.Commands.Dev.Run do statestore_key: "3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE=", log_level: "info", name: "proxy", - enable_nats: false + nats_image: "nats", + nats_http_port: 8222, + nats_port: 4222 } option(:actor_system, :string, "Defines the name of the ActorSystem.", @@ -181,11 +187,21 @@ defmodule SpawnCtl.Commands.Dev.Run do default: @default_opts.name ) - option(:enable_nats, :boolean, "Use or not Nats for cross ActorSystem communication", + option(:nats_image, :string, "Nats test image", alias: :N, - default: @default_opts.enable_nats + default: @default_opts.nats_image ) + option(:nats_http_port, :integer, "Nats http port", default: @default_opts.nats_http_port) + + option(:nats_port, :integer, "Nats port", default: @default_opts.nats_port) + + # Only supports one restartable atm + @containers [ + %{image: :nats_image, restartable: false}, + %{image: :proxy_image, restartable: true} + ] + @doc """ Runs the Spawn proxy in development mode. @@ -212,40 +228,58 @@ defmodule SpawnCtl.Commands.Dev.Run do defp do_run(_, opts, ctx) do log(:info, Emoji.runner(), "[#{get_time()}] Starting Spawn Proxy in dev mode...") + {:ok, _pid} = SpawnCtl.GroupExecAfter.start_link() + + {:ok, _pid} = + case Testcontainers.start_link() do + {:ok, pid} -> + {:ok, pid} + + {:error, {:error, {:failed_to_register_ryuk_filter, :closed}}} -> + Process.sleep(500) + Testcontainers.start_link() + end + if opts.proto_changes_watcher do - paths = - if File.exists?(opts.manifest_path), - do: [opts.protos, opts.manifest_path], - else: [opts.protos] - - {:ok, pid} = FileSystem.start_link(dirs: paths) - FileSystem.subscribe(pid) - watch(nil, opts, ctx) + @containers + |> Enum.reject(& &1.restartable) + |> Enum.each(fn container_params -> + {:ok, _} = start_container(container_params, opts, ctx) + end) + + container_params = Enum.find(@containers, & &1.restartable) + + spawn(fn -> + watch(container_params, nil, opts, ctx) + end) + + Process.sleep(:infinity) else - case Testcontainers.start_link() do - {:ok, _docker_pid} -> - case start_container(opts, ctx) do - {:ok, _container} -> - Process.sleep(:infinity) - - {:error, error} -> - log_failure(error) - System.stop(1) - end - - error -> - log_failure(error) - System.stop(1) + for container <- @containers do + {:ok, _} = start_container(container, opts, ctx) end + + Process.sleep(:infinity) end + rescue + error -> + log_failure(error) + System.stop(1) end - defp start_container(opts, _ctx) do + defp start_container(%{image: :proxy_image} = params, opts, _ctx) do opts |> parse_inputs() |> build_proxy_container() |> Testcontainers.start_container() - |> handle_container_start_result(opts) + |> handle_container_start_result(params, opts) + end + + defp start_container(%{image: :nats_image} = params, opts, _ctx) do + opts + |> build_nats_container() + |> Testcontainers.start_container() + |> handle_container_start_result(params, opts) end defp parse_inputs(opts) do @@ -259,79 +293,85 @@ defmodule SpawnCtl.Commands.Dev.Run do |> log_success(container, opts) setup_exit_handler(container) + {:ok, container} end - defp handle_container_start_result(error, _opts) do + defp handle_container_start_result({:error, error}, _container_params, _opts) do log_failure(error) + {:error, error} end - defp watch(nil, opts, ctx) do - if opts[:docker_pid] == nil do - with {:ok, docker_pid} <- Testcontainers.start_link(), - {:ok, container} <- start_container(Map.put(opts, :docker_pid, docker_pid), ctx) do - await_file_events(container, opts, ctx) - else - {:error, {:already_started, pid}} -> - log(:info, Emoji.winking(), "Stopping docker setup...") - stop_existing_docker_process(pid) - watch(nil, opts, ctx) + defp watch(%{image: :proxy_image} = params, nil = _container, opts, ctx) do + start_and_watch_container(params, opts, ctx) + end - {:error, {:error, {:failed_to_register_ryuk_filter, :closed}}} -> - log_transient_fault() - watch(nil, opts, ctx) + defp watch(%{image: :proxy_image} = params, container, opts, ctx), + do: await_file_events(params, container, opts, ctx) - error -> - log_failure(error) - end - else - start_and_watch_container(opts, ctx) - end - end + defp await_file_events(%{image: :proxy_image} = params, container, opts, ctx) do + paths = + if File.exists?(opts.manifest_path), + do: [opts.protos, opts.manifest_path], + else: [opts.protos] + + {:ok, pid} = FileSystem.start_link(dirs: paths) - defp watch(container, opts, ctx), do: await_file_events(container, opts, ctx) + FileSystem.subscribe(pid) - defp await_file_events(container, opts, ctx) do receive do - {:file_event, _worker_pid, {path, events}} = _evt -> + {:file_event, _worker_pid, {path, events} = event} = _evt -> main_evt = List.first(events) if is_valid?(path) && Enum.member?([:created, :modified, :deleted], main_evt) do - log( - :info, - Emoji.floppy_disk(), - "Detected #{inspect(List.first(events))} change in file [#{inspect(path)}]. Restarting Spawn proxy now..." - ) - - restart_container(container, opts, ctx) + # this will create a process in background and kill this listener + restart_container(params, container, event, opts, ctx) else - watch(container, opts, ctx) + watch(params, container, opts, ctx) end _other -> - watch(container, opts, ctx) + watch(params, container, opts, ctx) end end - defp restart_container(container, opts, ctx) do - Testcontainers.stop_container(container.container_id) - Process.sleep(500) - watch(nil, opts, ctx) + defp restart_container( + %{image: :proxy_image, restartable: true} = params, + container, + {path, events}, + opts, + ctx + ) do + SpawnCtl.GroupExecAfter.exec( + fn -> + log( + :info, + Emoji.floppy_disk(), + "Detected #{inspect(List.first(events))} change in file [#{inspect(path)}]. Restarting Spawn proxy now..." + ) + + Testcontainers.stop_container(container.container_id) + + watch(params, nil, opts, ctx) + end, + 500 + ) end - defp start_and_watch_container(opts, ctx) do - with {:ok, container} <- start_container(opts, ctx) do - await_file_events(container, opts, ctx) - else + defp start_and_watch_container(%{image: :proxy_image} = params, opts, ctx) do + case start_container(params, opts, ctx) do + {:ok, container} -> + await_file_events(params, container, opts, ctx) + {:error, {:already_started, pid}} -> log(:info, Emoji.winking(), "Stopping docker setup...") stop_existing_docker_process(pid) - watch(nil, opts, ctx) + watch(params, nil, opts, ctx) - {:error, {:error, :failed_to_register_ryuk_filter, :closed}} -> + {:error, {:error, {:failed_to_register_ryuk_filter, :closed}}} -> log_transient_fault() - watch(nil, opts, ctx) + watch(params, nil, opts, ctx) {:error, error} -> log_failure(error) @@ -368,7 +408,7 @@ defmodule SpawnCtl.Commands.Dev.Run do |> Container.with_environment("PROXY_HTTP_PORT", "#{opts.proxy_bind_port}") |> Container.with_environment("PROXY_GRPC_PORT", "#{opts.proxy_bind_grpc_port}") |> Container.with_environment("PROXY_ACTOR_SYSTEM_NAME", "#{opts.actor_system}") - |> Container.with_environment("SPAWN_USE_INTERNAL_NATS", "#{opts.enable_nats}") + |> Container.with_environment("SPAWN_USE_INTERNAL_NATS", "true") |> Container.with_environment("SPAWN_PROXY_LOGGER_LEVEL", opts.log_level) |> Container.with_environment("SPAWN_STATESTORE_KEY", opts.statestore_key) |> Container.with_environment("USER_FUNCTION_PORT", "#{opts.actor_host_port}") @@ -383,6 +423,13 @@ defmodule SpawnCtl.Commands.Dev.Run do |> Container.with_label("spawn.proxy.logger.level", opts.log_level) end + defp build_nats_container(opts) do + Container.new(opts.nats_image) + |> maybe_use_host_network(opts) + |> Container.with_exposed_ports([opts.nats_port, opts.nats_http_port]) + |> Container.with_cmd(["--jetstream", "--http_port=#{opts.nats_http_port}"]) + end + defp maybe_use_host_network(container, _opts) do case :os.type() do {:win32, _} -> @@ -411,14 +458,10 @@ defmodule SpawnCtl.Commands.Dev.Run do end end - defp log_success({:win32, _}, container, opts), do: log_sucess_with_ports(container, opts) - - defp log_success({:unix, :darwin}, container, opts), do: log_sucess_with_ports(container, opts) - - defp log_success({:unix, _}, container, opts) do - log(:info, Emoji.exclamation(), "Spawn Proxy using host network. Exposed ports: [ - Proxy HTTP: #{opts.proxy_bind_port}, - Proxy gRPC: #{opts.proxy_bind_grpc_port} + defp log_success(container, %{image: :proxy_image}, opts) do + log(:info, Emoji.exclamation(), "Spawn Proxy uses the following mapped ports: [ + Proxy HTTP: #{inspect(Container.mapped_port(container, opts.proxy_bind_port) || opts.proxy_bind_port)}:#{opts.proxy_bind_port}, + Proxy gRPC: #{inspect(Container.mapped_port(container, opts.proxy_bind_grpc_port) || opts.proxy_bind_grpc_port)}:#{opts.proxy_bind_grpc_port} ]") log( @@ -428,16 +471,16 @@ defmodule SpawnCtl.Commands.Dev.Run do ) end - defp log_sucess_with_ports(container, opts) do - log(:info, Emoji.exclamation(), "Spawn Proxy uses the following mapped ports: [ - Proxy HTTP: #{inspect(Container.mapped_port(container, opts.proxy_bind_port))}:#{opts.proxy_bind_port}, - Proxy gRPC: #{inspect(Container.mapped_port(container, opts.proxy_bind_grpc_port))}:#{opts.proxy_bind_grpc_port} + defp log_success(container, %{image: :nats_image}, opts) do + log(:info, Emoji.exclamation(), "Nats uses the following mapped ports: [ + Nats: #{inspect(Container.mapped_port(container, opts.nats_port) || opts.nats_port)}:#{opts.nats_port}, + Nats HTTP: #{inspect(Container.mapped_port(container, opts.nats_port) || opts.nats_http_port)}:#{opts.nats_http_port} ]") log( :info, Emoji.rocket(), - "[#{get_time()}] Spawn Proxy started successfully. Container Id: #{container.container_id}" + "Nats started successfully. Container Id: #{container.container_id}" ) end @@ -460,7 +503,7 @@ defmodule SpawnCtl.Commands.Dev.Run do log( :info, Emoji.winking(), - "Stopping Spawn Proxy in dev mode with status: #{inspect(status)}. Container Id: #{container.container_id}" + "Stopping Spawn Proxy in dev mode with status: #{inspect(status)}. ContainerId: #{inspect(container.container_id)}" ) end) end diff --git a/spawnctl/lib/spawnctl/exec_after.ex b/spawnctl/lib/spawnctl/exec_after.ex new file mode 100644 index 00000000..ee766546 --- /dev/null +++ b/spawnctl/lib/spawnctl/exec_after.ex @@ -0,0 +1,35 @@ +defmodule SpawnCtl.GroupExecAfter do + @moduledoc false + + use GenServer + + @impl true + def init(_init_arg) do + {:ok, %{timer_ref: nil}} + end + + @impl true + def handle_call({:exec, callback, wait_for}, _from, state) do + if state.timer_ref do + {:reply, :ok, state} + else + ref = Process.send_after(self(), {:callback, callback}, wait_for) + + {:reply, :ok, %{state | timer_ref: ref}} + end + end + + @impl true + def handle_info({:callback, callback}, state) do + spawn(fn -> callback.() end) + + {:noreply, %{state | timer_ref: nil}} + end + + def exec(callback, wait_for \\ 500) do + GenServer.call(__MODULE__, {:exec, callback, wait_for}) + end + + @doc false + def start_link, do: GenServer.start_link(__MODULE__, [], name: __MODULE__) +end