diff --git a/.github/workflows/auto_deploy.yml b/.github/workflows/auto_deploy.yml index ed7fc609..cc06746c 100644 --- a/.github/workflows/auto_deploy.yml +++ b/.github/workflows/auto_deploy.yml @@ -19,6 +19,8 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 + submodules: recursive + token: ${{ secrets.RELEASE_PAT }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index dcad3c73..01aea517 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -47,6 +47,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + with: + submodules: recursive + token: ${{ secrets.RELEASE_PAT }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 @@ -156,6 +159,7 @@ jobs: TF_VAR_jwt_secret: ${{ secrets.PROD_JWT_SECRET }} TF_VAR_image_version: ${{ inputs.image_tag }} TF_VAR_relay_public_key: ${{ secrets.RELAY_PUBLIC_KEY }} + TF_VAR_notification_channels: NNOynGwVz with: environment: "prod" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fafbaf4a..7641066a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,6 +110,9 @@ jobs: # Checkout code - name: "Git checkout" uses: actions/checkout@v2 + with: + submodules: recursive + token: ${{ secrets.RELEASE_PAT }} # Install sccache - name: "Install sccache" diff --git a/.github/workflows/ci_terraform.yml b/.github/workflows/ci_terraform.yml index 1295ff4b..ee2a5111 100644 --- a/.github/workflows/ci_terraform.yml +++ b/.github/workflows/ci_terraform.yml @@ -23,6 +23,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + with: + submodules: recursive + token: ${{ secrets.RELEASE_PAT }} - name: Setup Terraform uses: hashicorp/setup-terraform@v2 @@ -88,6 +91,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + with: + submodules: recursive + token: ${{ secrets.RELEASE_PAT }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 769921c7..238082e4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,6 +27,8 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 + submodules: recursive + token: ${{ secrets.RELEASE_PAT }} - name: Install lld and llvm run: sudo apt-get install -y lld llvm diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..b7368fba --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "terraform/monitoring/grafonnet-lib"] + path = terraform/monitoring/grafonnet-lib + url = git@github.com:WalletConnect/grafonnet-lib.git diff --git a/src/handlers/register_client.rs b/src/handlers/register_client.rs index 9b826e91..161adb7d 100644 --- a/src/handlers/register_client.rs +++ b/src/handlers/register_client.rs @@ -96,6 +96,7 @@ pub async fn handler( token: body.token, always_raw, }, + state.metrics.as_ref(), ) .await?; diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index ec2f348e..61ce3118 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -1,4 +1,13 @@ -use wc::metrics::{otel::metrics::Counter, ServiceMetrics}; +use { + std::time::Instant, + wc::metrics::{ + otel::{ + metrics::{Counter, Histogram}, + KeyValue, + }, + ServiceMetrics, + }, +}; #[derive(Clone)] pub struct Metrics { @@ -16,6 +25,9 @@ pub struct Metrics { pub tenant_suspensions: Counter, pub client_suspensions: Counter, + + postgres_queries: Counter, + postgres_query_latency: Histogram, } impl Default for Metrics { @@ -84,6 +96,16 @@ impl Metrics { .with_description("The number of clients that have been suspended") .init(); + let postgres_queries: Counter = meter + .u64_counter("postgres_queries") + .with_description("The number of Postgres queries executed") + .init(); + + let postgres_query_latency: Histogram = meter + .u64_histogram("postgres_query_latency") + .with_description("The latency Postgres queries") + .init(); + Metrics { registered_clients: clients_counter, received_notifications: received_notification_counter, @@ -96,6 +118,17 @@ impl Metrics { tenant_fcm_v1_updates: tenant_fcm_v1_updates_counter, tenant_suspensions: tenant_suspensions_counter, client_suspensions: client_suspensions_counter, + postgres_queries, + postgres_query_latency, } } + + pub fn postgres_query(&self, query_name: &'static str, start: Instant) { + let elapsed = start.elapsed(); + + let attributes = [KeyValue::new("name", query_name)]; + self.postgres_queries.add(1, &attributes); + self.postgres_query_latency + .record(elapsed.as_millis() as u64, &attributes); + } } diff --git a/src/stores/client.rs b/src/stores/client.rs index 52d597ae..8aefa7a9 100644 --- a/src/stores/client.rs +++ b/src/stores/client.rs @@ -1,10 +1,12 @@ use { crate::{ + metrics::Metrics, providers::ProviderKind, stores::{self, StoreError::NotFound}, }, async_trait::async_trait, sqlx::Executor, + std::time::Instant, tracing::{debug, instrument}, }; @@ -19,15 +21,27 @@ pub struct Client { #[async_trait] pub trait ClientStore { - async fn create_client(&self, tenant_id: &str, id: &str, client: Client) -> stores::Result<()>; + async fn create_client( + &self, + tenant_id: &str, + id: &str, + client: Client, + metrics: Option<&Metrics>, + ) -> stores::Result<()>; async fn get_client(&self, tenant_id: &str, id: &str) -> stores::Result; async fn delete_client(&self, tenant_id: &str, id: &str) -> stores::Result<()>; } #[async_trait] impl ClientStore for sqlx::PgPool { - #[instrument(skip(self, client))] - async fn create_client(&self, tenant_id: &str, id: &str, client: Client) -> stores::Result<()> { + #[instrument(skip(self, client, metrics))] + async fn create_client( + &self, + tenant_id: &str, + id: &str, + client: Client, + metrics: Option<&Metrics>, + ) -> stores::Result<()> { debug!( "ClientStore::create_client tenant_id={tenant_id} id={id} token={} with locking", client.token @@ -37,6 +51,7 @@ impl ClientStore for sqlx::PgPool { // Statement for locking based on the client id to prevent an issue #230 // and locking based on the token to prevent an issue #292 + let start = Instant::now(); sqlx::query( "SELECT pg_advisory_xact_lock(abs(hashtext($1::text))), @@ -46,13 +61,21 @@ impl ClientStore for sqlx::PgPool { .bind(client.token.clone()) .execute(&mut transaction) .await?; + if let Some(metrics) = metrics { + metrics.postgres_query("create_client_pg_advisory_xact_lock", start); + } + let start = Instant::now(); sqlx::query("DELETE FROM public.clients WHERE id = $1 OR device_token = $2") .bind(id) .bind(client.token.clone()) .execute(&mut transaction) .await?; + if let Some(metrics) = metrics { + metrics.postgres_query("create_client_delete", start); + } + let start = Instant::now(); let mut insert_query = sqlx::QueryBuilder::new( "INSERT INTO public.clients (id, tenant_id, push_type, device_token, always_raw)", ); @@ -73,7 +96,15 @@ impl ClientStore for sqlx::PgPool { }, ); insert_query.build().execute(&mut transaction).await?; + if let Some(metrics) = metrics { + metrics.postgres_query("create_client_insert", start); + } + + let start = Instant::now(); transaction.commit().await?; + if let Some(metrics) = metrics { + metrics.postgres_query("create_client_commit", start); + } Ok(()) } diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl index 645a440a..d25f06f0 100644 --- a/terraform/.terraform.lock.hcl +++ b/terraform/.terraform.lock.hcl @@ -1,6 +1,28 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. +provider "registry.terraform.io/alxrem/jsonnet" { + version = "2.3.1" + constraints = "~> 2.3.0" + hashes = [ + "h1:hhgft+XkSDE5ayFLJEGeyQtH8SDnVIh96XcLISmSP9U=", + "zh:07479550eb7604605d943cc31fa96b002624be63a1c0c04d3bed350af9ef3543", + "zh:0b8eba2be9a162f0cc78a5911f760dfefdd78b50ecc785710a98e86f933b53a4", + "zh:0e320d005a8730a0c1ed65a3bf3bb4bb8c35d4a19005f43e0442cfa3b62622e5", + "zh:10861809d729f74acb773d71d251160449b1acc728a3710d43ddc2b31b3f09f7", + "zh:24fad4bd096d8386b1dc27e6d4296086fb275c0dc6f25814077eb6cdc8146389", + "zh:2a9bca30f084fa992ef614ebf0ceaf83145890fdef005220541d4c889e935cc0", + "zh:3602b01d339a322d8366f23ff1836ca9438091c6bc3de12a0f7679b9650ceaa6", + "zh:402adf2f5bb2d6724405b22f2a81a6cff3059598fb8dbc93aa1d2a4cf09aa888", + "zh:70768655d9f24aba48ccacdbd1ea4318c0dc1f0077f8f7c4e7edb6f2edb90b6d", + "zh:73d84e10ce8211c18342aebd89eab8f3085ddcb48ec846d7368c80d94b8ebe4e", + "zh:775540195cb6e94b23459dd51e6b9d33c2992040be86d4db227a295ca0b5023a", + "zh:bc350b6529bf07e72479ef3098382e89464a68d956af397414a8ef6c66482048", + "zh:e82ffb1c7aff7bc9cf8f4ff7452207c690a19294844653900b6a3b7e73bf68ea", + "zh:ff9c40d558b7782370bf4147a78600e7a430fbf44a4bdde89c33475336bc9a53", + ] +} + provider "registry.terraform.io/bwoznicki/assert" { version = "0.0.1" hashes = [ @@ -21,24 +43,24 @@ provider "registry.terraform.io/bwoznicki/assert" { } provider "registry.terraform.io/grafana/grafana" { - version = "1.43.0" - constraints = "~> 1.28" + version = "2.19.0" + constraints = "~> 2.0, >= 2.1.0" hashes = [ - "h1:hPUCqxfmDcx9ZEb68TUE6RGvTj9AsHcy1wu6p2joSrE=", - "zh:008b7190066f5d5c82879f5d7b4ff7574881ca6bb92e9bf961c3a4cc1925a5a5", - "zh:0412eabd3339b2d354dbc2827be8a0fbcf13ade804c508125b211154ce4cc89c", - "zh:0886093d1ed3e81dc145bb9e3382da7669e5222213c1d322635803e429dee442", - "zh:12bba5fbc0127f55f10283efe3ce5fe6ba291c52d1053b464e4a4d9e369c6095", - "zh:3c14c67109188cee2ba382c2df144554f0006fe1048a7081162f0c01d1b4ab3b", - "zh:6094dc57bff0108e5d246831ad7a7c404b6e8556ba6cb5a9d8e9415fb78e652c", - "zh:7742e34025d9dd8dda787332e04e230cc467579a7f38d06f2077d5e531b725e8", - "zh:852a92cd17e2063894720ad9ba5041ac3ff81d3a5b05c86946cc76c8dd7d1435", - "zh:a2a179972db737136a2a60b5bb55151b58b318fd18d98fc850e05b402a12bcf3", - "zh:a5d24cb6d5d0c0a1a950d264fecda18a7c69389f2eadeedbfb73c4c5a49b1a53", - "zh:b07fc16abbc4992fa7d673560129e509600916c8d764f81c163f004d797251a5", - "zh:c02df4cbee1cdbd2745140775be54c8b6785618de327fc18a3a47d2f2d82ef30", - "zh:c558944fb09092516f598a7640100b89f669df54e2a5fd652c56aaa8ad335fee", - "zh:d0f30f234c7002d7c09052c5221f03e7d0be3c3d3c11e90a539f2d56786c6a75", + "h1:Rqn7cIJs9cU+V7Dus2cQwl5HaFhnUAZoZrly3oz9UlM=", + "zh:0f85e0602ccf119a203f47ee18524a8172457e82b8cbffca7b6b4d7f474639ad", + "zh:1a626e0d04dedd48cb5b0bdf8a68f428fdf0f7080128928dfaa293a4db78b9b3", + "zh:2a4de32569e070de38faa3d3224ab7fe0a837809d70d75dedc8f1f1f8d9dc227", + "zh:3c3b77436da2fc8ff428ee7dae44d0bc820ebf835c951de4ed1052b68431081f", + "zh:404de4f0e0bb7fe9b6f95187a69e4d9a9e6ba452bba7a62c62c15f154136c2b2", + "zh:6a247dea12ade810c5f74528bac5ffeb7a4c9d54089e1a42134e34b193fddc32", + "zh:90d795e190f1374bde468e90014dcd818e65f66b21cd518487f84cec7bc842c9", + "zh:9e2e846bb354fe8b041913e55211f95bb9d8246ec426e7c589c87f1b4e0573c5", + "zh:a43e2f0cc5fb76dbf508b3f66061d67b469bd248a557f05d3c7f43b7a45041c0", + "zh:b90e3120face56f753933ecabd5b80131679c444b72ab53d0e40263be684eab6", + "zh:bdb35128710a6f45c8e473b9c0171f6cb7ae5cf572ca72561d0543a4dca3e609", + "zh:c786b6e95597c677f6d6e5e994376889b17635df512fe2798902b0ee3fd63367", + "zh:cb3d7f28a1eb696607b3d799a6b6b2ac5914240e61b578c7b45e42eb76979481", + "zh:f106de3485bcd1af0dbf45ffd475ec60c452cb39df51e2e550eb4ab4cf1e0f73", ] } diff --git a/terraform/backend.tf b/terraform/backend.tf index c24c5377..62003bda 100644 --- a/terraform/backend.tf +++ b/terraform/backend.tf @@ -20,7 +20,7 @@ terraform { } grafana = { source = "grafana/grafana" - version = "~> 1.28" + version = ">= 2.1" } random = { source = "hashicorp/random" @@ -31,4 +31,4 @@ terraform { version = "5.7.0" } } -} \ No newline at end of file +} diff --git a/terraform/main.tf b/terraform/main.tf index da8e7b24..0749ea2c 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -191,6 +191,7 @@ module "monitoring" { prometheus_workspace_id = aws_prometheus_workspace.prometheus.id load_balancer_arn = module.ecs.load_balancer_arn environment = local.environment + notification_channels = var.notification_channels } data "aws_ecr_repository" "repository" { diff --git a/terraform/monitoring/backend.tf b/terraform/monitoring/backend.tf deleted file mode 100644 index 6e2b2a6e..00000000 --- a/terraform/monitoring/backend.tf +++ /dev/null @@ -1,9 +0,0 @@ -terraform { - required_version = "~> 1.0" - required_providers { - grafana = { - source = "grafana/grafana" - version = "~> 1.28" - } - } -} \ No newline at end of file diff --git a/terraform/monitoring/dashboard.jsonnet b/terraform/monitoring/dashboard.jsonnet new file mode 100644 index 00000000..d94fe237 --- /dev/null +++ b/terraform/monitoring/dashboard.jsonnet @@ -0,0 +1,55 @@ +local grafana = import 'grafonnet-lib/grafana.libsonnet'; +local panels = import 'panels/panels.libsonnet'; + +local dashboard = grafana.dashboard; +local row = grafana.row; +local annotation = grafana.annotation; +local layout = grafana.layout; + +local ds = { + prometheus: { + type: 'prometheus', + uid: std.extVar('prometheus_uid'), + }, + cloudwatch: { + type: 'cloudwatch', + uid: std.extVar('cloudwatch_uid'), + }, +}; +local vars = { + namespace: 'Push', + environment: std.extVar('environment'), + notifications: std.parseJson(std.extVar('notifications')), +}; + +//////////////////////////////////////////////////////////////////////////////// + +local height = 8; +local pos = grafana.layout.pos(height); + +//////////////////////////////////////////////////////////////////////////////// + +dashboard.new( + title = std.extVar('dashboard_title'), + uid = std.extVar('dashboard_uid'), + editable = true, + graphTooltip = dashboard.graphTooltips.sharedCrosshair, + timezone = dashboard.timezones.utc, +) +.addAnnotation( + annotation.new( + target = { + limit: 100, + matchAny: false, + tags: [], + type: 'dashboard', + }, + ) +) + +.addPanels(layout.generate_grid([ + ////////////////////////////////////////////////////////////////////////////// + row.new('Application'), + panels.app.postgres_query_rate(ds, vars) { gridPos: pos._6 }, + panels.app.postgres_query_latency(ds, vars) { gridPos: pos._6 }, +])) diff --git a/terraform/monitoring/grafonnet-lib b/terraform/monitoring/grafonnet-lib new file mode 160000 index 00000000..aa25c4dc --- /dev/null +++ b/terraform/monitoring/grafonnet-lib @@ -0,0 +1 @@ +Subproject commit aa25c4dcf091270f943ea8e79986514447efb93d diff --git a/terraform/monitoring/main.tf b/terraform/monitoring/main.tf index a7f640b9..6c27eb60 100644 --- a/terraform/monitoring/main.tf +++ b/terraform/monitoring/main.tf @@ -2,13 +2,7 @@ locals { # Turns the arn into the format expected by # the Grafana provider e.g. # net/prod-relay-load-balancer/e9a51c46020a0f85 - load_balancer = join("/", slice(split("/", var.load_balancer_arn), 1, 4)) - opsgenie_notification_channel = "NNOynGwVz" - notifications = ( - var.environment == "prod" ? - [{ uid = local.opsgenie_notification_channel }] : - [] - ) + load_balancer = join("/", slice(split("/", var.load_balancer_arn), 1, 4)) } resource "grafana_data_source" "prometheus" { @@ -34,7 +28,29 @@ resource "grafana_data_source" "cloudwatch" { }) } +data "jsonnet_file" "dashboard" { + source = "${path.module}/dashboard.jsonnet" + + ext_str = { + dashboard_title = "Push Server - ${title(var.environment)}" + dashboard_uid = "push-${var.environment}" + + prometheus_uid = grafana_data_source.prometheus.uid + cloudwatch_uid = grafana_data_source.cloudwatch.uid + + environment = var.environment + notifications = jsonencode(var.notification_channels) + } +} + resource "grafana_dashboard" "at_a_glance" { + overwrite = true + message = "Updated by Terraform" + config_json = data.jsonnet_file.dashboard.rendered +} + + +resource "grafana_dashboard" "at_a_glance_old" { overwrite = true message = "Updated by Terraform" config_json = jsonencode({ @@ -352,7 +368,7 @@ resource "grafana_dashboard" "at_a_glance" { "uid" : grafana_data_source.prometheus.uid }, "exemplar" : true, - "expr" : "sum(rate(registered_clients{}[$__rate_interval]))", + "expr" : "sum(rate(registered_clients_total{}[$__rate_interval]))", "interval" : "", "legendFormat" : "", "refId" : "Clients" @@ -533,7 +549,7 @@ resource "grafana_dashboard" "at_a_glance" { "name" : "${var.environment} Echo Server 5XX alert", "noDataState" : "no_data", "message" : "Echo server - Prod - 5XX error", - "notifications" : local.notifications + "notifications" : var.notification_channels }, "datasource" : { "type" : "cloudwatch", @@ -804,8 +820,8 @@ resource "grafana_dashboard" "at_a_glance" { }, "timepicker" : {}, "timezone" : "", - "title" : var.app_name, - "uid" : var.app_name, + "title" : "${var.app_name} - old", + "uid" : "${var.app_name}-old", "version" : 13, "weekStart" : "" }) diff --git a/terraform/monitoring/panels/app/postgres_query_latency.libsonnet b/terraform/monitoring/panels/app/postgres_query_latency.libsonnet new file mode 100644 index 00000000..fe863d42 --- /dev/null +++ b/terraform/monitoring/panels/app/postgres_query_latency.libsonnet @@ -0,0 +1,25 @@ +local grafana = import '../../grafonnet-lib/grafana.libsonnet'; +local defaults = import '../../grafonnet-lib/defaults.libsonnet'; + +local panels = grafana.panels; +local targets = grafana.targets; + +{ + new(ds, vars):: + panels.timeseries( + title = 'Postgres Query Latency', + datasource = ds.prometheus, + ) + .configure( + defaults.configuration.timeseries + .withUnit('ms') + ) + + .addTarget(targets.prometheus( + datasource = ds.prometheus, + expr = 'sum by (aws_ecs_task_revision, name) (rate(postgres_query_latency_sum[$__rate_interval])) / sum by (aws_ecs_task_revision, name) (rate(postgres_query_latency_count[$__rate_interval]))', + legendFormat = '{{name}} r{{aws_ecs_task_revision}}', + exemplar = false, + refId = 'PostgresQueryLatency', + )) +} diff --git a/terraform/monitoring/panels/app/postgres_query_rate.libsonnet b/terraform/monitoring/panels/app/postgres_query_rate.libsonnet new file mode 100644 index 00000000..2183cd4f --- /dev/null +++ b/terraform/monitoring/panels/app/postgres_query_rate.libsonnet @@ -0,0 +1,33 @@ +local grafana = import '../../grafonnet-lib/grafana.libsonnet'; +local defaults = import '../../grafonnet-lib/defaults.libsonnet'; + +local panels = grafana.panels; +local targets = grafana.targets; + +{ + new(ds, vars):: + panels.timeseries( + title = 'Postgres Query Rate', + datasource = ds.prometheus, + ) + .configure( + defaults.configuration.timeseries + .withUnit('cps') + ) + + .addTarget(targets.prometheus( + datasource = ds.prometheus, + expr = 'sum by (aws_ecs_task_revision, name) (rate(postgres_queries_total[$__rate_interval]))', + legendFormat = '{{name}} r{{aws_ecs_task_revision}}', + exemplar = true, + refId = 'PostgresQueryRate', + )) + + .addTarget(targets.prometheus( + datasource = ds.prometheus, + expr = 'sum(rate(postgres_queries_total[$__rate_interval]))', + legendFormat = 'r{{aws_ecs_task_revision}}', + exemplar = true, + refId = 'PostgresQueryRateTotal', + )) +} diff --git a/terraform/monitoring/panels/panels.libsonnet b/terraform/monitoring/panels/panels.libsonnet new file mode 100644 index 00000000..15364c63 --- /dev/null +++ b/terraform/monitoring/panels/panels.libsonnet @@ -0,0 +1,8 @@ +local panels = (import '../grafonnet-lib/defaults.libsonnet').panels; + +{ + app: { + postgres_query_rate: (import 'app/postgres_query_rate.libsonnet' ).new, + postgres_query_latency: (import 'app/postgres_query_latency.libsonnet' ).new, + }, +} diff --git a/terraform/monitoring/terraform.tf b/terraform/monitoring/terraform.tf new file mode 100644 index 00000000..f875768d --- /dev/null +++ b/terraform/monitoring/terraform.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + grafana = { + source = "grafana/grafana" + version = "~> 2.0" + } + jsonnet = { + source = "alxrem/jsonnet" + version = "~> 2.3.0" + } + } +} diff --git a/terraform/monitoring/variables.tf b/terraform/monitoring/variables.tf index 168f542f..536318aa 100644 --- a/terraform/monitoring/variables.tf +++ b/terraform/monitoring/variables.tf @@ -13,3 +13,8 @@ variable "prometheus_workspace_id" { variable "load_balancer_arn" { type = string } + +variable "notification_channels" { + description = "The notification channels to send alerts to" + type = list(any) +} diff --git a/terraform/variables.tf b/terraform/variables.tf index 8f8c756f..ca7c3f2b 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -41,3 +41,12 @@ variable "relay_public_key" { type = string sensitive = true } + +#------------------------------------------------------------------------------- +# Alerting / Monitoring + +variable "notification_channels" { + description = "The notification channels to send alerts to" + type = list(any) + default = [] +} diff --git a/tests/functional/stores/client.rs b/tests/functional/stores/client.rs index d9666725..2fcf1e25 100644 --- a/tests/functional/stores/client.rs +++ b/tests/functional/stores/client.rs @@ -24,6 +24,7 @@ async fn client_creation(ctx: &mut StoreContext) { token, always_raw: false, }, + None, ) .await .unwrap(); @@ -46,6 +47,7 @@ async fn client_creation_fcm(ctx: &mut StoreContext) { token, always_raw: false, }, + None, ) .await .unwrap(); @@ -68,6 +70,7 @@ async fn client_creation_apns(ctx: &mut StoreContext) { token, always_raw: false, }, + None, ) .await .unwrap(); @@ -92,6 +95,7 @@ async fn client_upsert_token(ctx: &mut StoreContext) { token: token.clone(), always_raw: false, }, + None, ) .await .unwrap(); @@ -133,6 +137,7 @@ async fn client_upsert_token(ctx: &mut StoreContext) { token: updated_token.clone(), always_raw: true, }, + None, ) .await .unwrap(); @@ -164,6 +169,7 @@ async fn client_upsert_id(ctx: &mut StoreContext) { token: token.clone(), always_raw: false, }, + None, ) .await .unwrap(); @@ -205,6 +211,7 @@ async fn client_upsert_id(ctx: &mut StoreContext) { token: token.clone(), always_raw: false, }, + None, ) .await .unwrap(); @@ -239,6 +246,7 @@ async fn client_create_same_id_and_token(ctx: &mut StoreContext) { token: token.clone(), always_raw: false, }, + None, ) .await .unwrap(); @@ -280,6 +288,7 @@ async fn client_create_same_id_and_token(ctx: &mut StoreContext) { token: token.clone(), always_raw: false, }, + None, ) .await .unwrap(); @@ -310,6 +319,7 @@ async fn client_deletion(ctx: &mut StoreContext) { token, always_raw: false, }, + None, ) .await .unwrap(); @@ -332,6 +342,7 @@ async fn client_fetch(ctx: &mut StoreContext) { token: token.clone(), always_raw: false, }, + None, ) .await .unwrap(); diff --git a/tests/functional/stores/notification.rs b/tests/functional/stores/notification.rs index 8319e8fe..113cf0dd 100644 --- a/tests/functional/stores/notification.rs +++ b/tests/functional/stores/notification.rs @@ -24,6 +24,7 @@ pub async fn create_client(client_store: &ClientStoreArc) -> String { token, always_raw: false, }, + None, ) .await .expect("failed to create client for notification test");