diff --git a/Cargo.lock b/Cargo.lock index 4c18b2312cff7..a3be0022631c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10927,6 +10927,7 @@ dependencies = [ "risingwave_rpc_client", "risingwave_storage", "serde", + "tokio-retry", "tracing", "workspace-hack", ] @@ -10971,6 +10972,7 @@ dependencies = [ "tempfile", "thiserror-ext", "tikv-jemalloc-ctl", + "tokio-retry", "tokio-stream 0.1.15", "tower 0.5.0", "tracing", diff --git a/src/compute/Cargo.toml b/src/compute/Cargo.toml index d6a14aa04cd9e..f1f940f92b4a3 100644 --- a/src/compute/Cargo.toml +++ b/src/compute/Cargo.toml @@ -62,6 +62,7 @@ futures-async-stream = { workspace = true } rand = { workspace = true } risingwave_hummock_sdk = { workspace = true } tempfile = "3" +tokio-retry = "0.3" [lints] workspace = true diff --git a/src/compute/src/telemetry.rs b/src/compute/src/telemetry.rs index 4e1041384afbe..1eb52ed29294a 100644 --- a/src/compute/src/telemetry.rs +++ b/src/compute/src/telemetry.rs @@ -104,7 +104,13 @@ mod test { let pb_report = report.to_pb_bytes(); let url = (TELEMETRY_REPORT_URL.to_owned() + "/" + TELEMETRY_COMPUTE_REPORT_TYPE).to_owned(); - let post_res = post_telemetry_report_pb(&url, pb_report).await; - assert!(post_res.is_ok()); + + // Retry 3 times to mitigate occasional failures on CI. + tokio_retry::Retry::spawn( + tokio_retry::strategy::ExponentialBackoff::from_millis(10).take(3), + || post_telemetry_report_pb(&url, pb_report.clone()), + ) + .await + .unwrap(); } } diff --git a/src/meta/src/telemetry.rs b/src/meta/src/telemetry.rs index 2c15a8251df34..d845dcd76062f 100644 --- a/src/meta/src/telemetry.rs +++ b/src/meta/src/telemetry.rs @@ -282,7 +282,13 @@ mod test { let pb_bytes = report.to_pb_bytes(); let url = (TELEMETRY_REPORT_URL.to_owned() + "/" + TELEMETRY_META_REPORT_TYPE).to_owned(); - let result = post_telemetry_report_pb(&url, pb_bytes).await; - assert!(result.is_ok()); + + // Retry 3 times to mitigate occasional failures on CI. + tokio_retry::Retry::spawn( + tokio_retry::strategy::ExponentialBackoff::from_millis(10).take(3), + || post_telemetry_report_pb(&url, pb_bytes.clone()), + ) + .await + .unwrap(); } } diff --git a/src/storage/compactor/Cargo.toml b/src/storage/compactor/Cargo.toml index d94da125ca3ec..89ebc01f7d0a3 100644 --- a/src/storage/compactor/Cargo.toml +++ b/src/storage/compactor/Cargo.toml @@ -34,6 +34,9 @@ tokio = { version = "0.2", package = "madsim-tokio", features = [ tonic = { workspace = true } tracing = "0.1" +[dev-dependencies] +tokio-retry = "0.3" + [target.'cfg(not(madsim))'.dependencies] workspace-hack = { path = "../../workspace-hack" } diff --git a/src/storage/compactor/src/telemetry.rs b/src/storage/compactor/src/telemetry.rs index a4f4ab4b185dc..d343c92599ef0 100644 --- a/src/storage/compactor/src/telemetry.rs +++ b/src/storage/compactor/src/telemetry.rs @@ -126,7 +126,13 @@ mod test { let pb_report = report.to_pb_bytes(); let url = (TELEMETRY_REPORT_URL.to_owned() + "/" + TELEMETRY_COMPACTOR_REPORT_TYPE).to_owned(); - let post_res = post_telemetry_report_pb(&url, pb_report).await; - assert!(post_res.is_ok()); + + // Retry 3 times to mitigate occasional failures on CI. + tokio_retry::Retry::spawn( + tokio_retry::strategy::ExponentialBackoff::from_millis(10).take(3), + || post_telemetry_report_pb(&url, pb_report.clone()), + ) + .await + .unwrap(); } }