From fbaed43b58a7f27ca223d63ad479c6c7626e74ae Mon Sep 17 00:00:00 2001 From: Heiko Seeberger Date: Wed, 22 Nov 2023 08:30:00 +0100 Subject: [PATCH] feat: add optional shutdown timeout to servers --- Cargo.lock | 18 +++++++++++++ Cargo.toml | 1 + hello-tracing-backend/Cargo.toml | 1 + hello-tracing-backend/config/default.yaml | 1 + hello-tracing-backend/src/api/mod.rs | 30 ++++++++++++--------- hello-tracing-gateway/Cargo.toml | 1 + hello-tracing-gateway/config/default.yaml | 1 + hello-tracing-gateway/src/api/mod.rs | 32 ++++++++++++++--------- justfile | 4 +-- 9 files changed, 63 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ddb9e38..90a52b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -523,6 +523,7 @@ dependencies = [ "axum", "configured", "hello-tracing-common", + "humantime-serde", "opentelemetry", "opentelemetry-http", "opentelemetry-otlp", @@ -572,6 +573,7 @@ dependencies = [ "axum", "configured", "hello-tracing-common", + "humantime-serde", "opentelemetry", "opentelemetry-http", "opentelemetry-otlp", @@ -644,6 +646,22 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "humantime-serde" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c" +dependencies = [ + "humantime", + "serde", +] + [[package]] name = "hyper" version = "0.14.27" diff --git a/Cargo.toml b/Cargo.toml index 2fd18a7..f2e0e2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ publish = false anyhow = { version = "1.0", features = [ "backtrace" ] } axum = { version = "0.6", features = [ "http2", "json" ] } configured = { version = "0.7" } +humantime-serde = { version = "1.1" } opentelemetry = { version = "0.21" } opentelemetry_sdk = { version = "0.21", features = [ "rt-tokio" ] } opentelemetry-http = { version = "0.10", features = [ "reqwest" ] } diff --git a/hello-tracing-backend/Cargo.toml b/hello-tracing-backend/Cargo.toml index d165e51..cb6d4b6 100644 --- a/hello-tracing-backend/Cargo.toml +++ b/hello-tracing-backend/Cargo.toml @@ -16,6 +16,7 @@ hello-tracing-common = { path = "../hello-tracing-common" } anyhow = { workspace = true } axum = { workspace = true } configured = { workspace = true } +humantime-serde = { workspace = true } opentelemetry = { workspace = true } opentelemetry-http = { workspace = true } opentelemetry-otlp = { workspace = true } diff --git a/hello-tracing-backend/config/default.yaml b/hello-tracing-backend/config/default.yaml index 422cee0..f93265d 100644 --- a/hello-tracing-backend/config/default.yaml +++ b/hello-tracing-backend/config/default.yaml @@ -1,6 +1,7 @@ api: addr: 0.0.0.0 port: 8080 + shutdown-timeout: 3s tracing: service-name: hello-tracing-backend diff --git a/hello-tracing-backend/src/api/mod.rs b/hello-tracing-backend/src/api/mod.rs index bccd7ad..2ad68cf 100644 --- a/hello-tracing-backend/src/api/mod.rs +++ b/hello-tracing-backend/src/api/mod.rs @@ -4,8 +4,11 @@ use anyhow::{Context, Result}; use axum::{body::Body, http::Request}; use hello_tracing_common::otel::http::{accept_trace, record_trace_id}; use serde::Deserialize; -use std::net::{IpAddr, SocketAddr}; -use tokio::signal::unix::{signal, SignalKind}; +use std::{net::IpAddr, time::Duration}; +use tokio::{ + signal::unix::{signal, SignalKind}, + time, +}; use tonic::transport::Server; use tower::ServiceBuilder; use tower_http::trace::TraceLayer; @@ -16,16 +19,16 @@ use tracing::{field, info_span, Span}; pub struct Config { addr: IpAddr, port: u16, -} - -impl Config { - fn socket_addr(&self) -> SocketAddr { - SocketAddr::new(self.addr, self.port) - } + #[serde(with = "humantime_serde")] + shutdown_timeout: Option, } pub async fn serve(config: Config) -> Result<()> { - let socket_addr = config.socket_addr(); + let Config { + addr, + port, + shutdown_timeout, + } = config; let app = Server::builder() .layer( @@ -36,9 +39,9 @@ pub async fn serve(config: Config) -> Result<()> { ) .add_service(v0::hello()); - app.serve_with_shutdown(socket_addr, shutdown_signal()) + app.serve_with_shutdown((addr, port).into(), shutdown_signal(shutdown_timeout)) .await - .context("serving the api") + .context("run server") } fn make_span(request: &Request) -> Span { @@ -46,9 +49,12 @@ fn make_span(request: &Request) -> Span { info_span!("incoming request", ?headers, trace_id = field::Empty) } -async fn shutdown_signal() { +async fn shutdown_signal(shutdown_timeout: Option) { signal(SignalKind::terminate()) .expect("install SIGTERM handler") .recv() .await; + if let Some(shutdown_timeout) = shutdown_timeout { + time::sleep(shutdown_timeout).await; + } } diff --git a/hello-tracing-gateway/Cargo.toml b/hello-tracing-gateway/Cargo.toml index 21c1042..50f8026 100644 --- a/hello-tracing-gateway/Cargo.toml +++ b/hello-tracing-gateway/Cargo.toml @@ -16,6 +16,7 @@ hello-tracing-common = { path = "../hello-tracing-common" } anyhow = { workspace = true } axum = { workspace = true } configured = { workspace = true } +humantime-serde = { workspace = true } opentelemetry = { workspace = true } opentelemetry-http = { workspace = true } opentelemetry-otlp = { workspace = true } diff --git a/hello-tracing-gateway/config/default.yaml b/hello-tracing-gateway/config/default.yaml index 80c83ec..97707f6 100644 --- a/hello-tracing-gateway/config/default.yaml +++ b/hello-tracing-gateway/config/default.yaml @@ -1,6 +1,7 @@ api: addr: 0.0.0.0 port: 8080 + shutdown-timeout: 3s # backend: # endpoint: ... diff --git a/hello-tracing-gateway/src/api/mod.rs b/hello-tracing-gateway/src/api/mod.rs index 97fd843..45d8d0d 100644 --- a/hello-tracing-gateway/src/api/mod.rs +++ b/hello-tracing-gateway/src/api/mod.rs @@ -11,8 +11,11 @@ use axum::{ }; use hello_tracing_common::otel::http::{accept_trace, record_trace_id}; use serde::Deserialize; -use std::net::{IpAddr, SocketAddr}; -use tokio::signal::unix::{signal, SignalKind}; +use std::{net::IpAddr, time::Duration}; +use tokio::{ + signal::unix::{signal, SignalKind}, + time, +}; use tower::ServiceBuilder; use tower_http::trace::TraceLayer; use tracing::{field, info_span, Span}; @@ -22,15 +25,17 @@ use tracing::{field, info_span, Span}; pub struct Config { addr: IpAddr, port: u16, -} - -impl Config { - fn socket_addr(&self) -> SocketAddr { - SocketAddr::new(self.addr, self.port) - } + #[serde(with = "humantime_serde")] + shutdown_timeout: Option, } pub async fn serve(config: Config, backend: Backend) -> Result<()> { + let Config { + addr, + port, + shutdown_timeout, + } = config; + let app = Router::new() .route("/", get(ready)) .nest("/v0", v0::app(backend)) @@ -41,11 +46,11 @@ pub async fn serve(config: Config, backend: Backend) -> Result<()> { .map_request(record_trace_id), ); - Server::bind(&config.socket_addr()) + Server::bind(&(addr, port).into()) .serve(app.into_make_service()) - .with_graceful_shutdown(shutdown_signal()) + .with_graceful_shutdown(shutdown_signal(shutdown_timeout)) .await - .context("serving api") + .context("run server") } async fn ready() -> impl IntoResponse { @@ -57,9 +62,12 @@ fn make_span(request: &Request) -> Span { info_span!("incoming request", ?headers, trace_id = field::Empty) } -async fn shutdown_signal() { +async fn shutdown_signal(shutdown_timeout: Option) { signal(SignalKind::terminate()) .expect("install SIGTERM handler") .recv() .await; + if let Some(shutdown_timeout) = shutdown_timeout { + time::sleep(shutdown_timeout).await; + } } diff --git a/justfile b/justfile index 9f82a7f..0971222 100644 --- a/justfile +++ b/justfile @@ -32,14 +32,14 @@ run-gateway port="8080" backend_port="8081": APP__API__PORT={{port}} \ APP__BACKEND__ENDPOINT=http://localhost:{{backend_port}} \ cargo run -p hello-tracing-gateway \ - > $HOME/tmp/hello-tracing-gateway.log + | jq run-backend port="8081": RUST_LOG=hello_tracing_backend=debug,info \ CONFIG_DIR=hello-tracing-backend/config \ APP__API__PORT={{port}} \ cargo run -p hello-tracing-backend \ - > $HOME/tmp/hello-tracing-backend.log + | jq docker tag="latest": docker build \