Adds Daprd option --dapr-block-shutdown-duration (dapr#7268)

* Adds Daprd option `--block-shutdown-seconds` Closes dapr#4313 Docs: dapr/docs#3893 PR adds the `--block-shutdown-seconds` CLI flag and corresponding `dapr.io/block-shutdown-seconds` Kubernetes annotation which configures Daprd to block the graceful shutdown procedure until _either_, the block shutdown seconds has elapsed _or_ the application has become unhealthy, as according to the normal app health status. By default, this option is unset, and therefore there is no effect to the current behaviour of graceful shutdown. When set, Daprd will block the interrupt signal cascading into runtime until the above requirements have been met. The framework process `Cleanup` order has been reversed to mimic `t.Cleanup` and allow the `logline` process to function correctly. Signed-off-by: joshvanl <[email protected]> * Revert if check on killing process exec proc cleanup Signed-off-by: joshvanl <[email protected]> * Revert error ignore of processes already killed in unix Signed-off-by: joshvanl <[email protected]> * Skip shutdown/graceful/block/healthy on windows. * Skip shutdown/block/unhealthy test on windows. * Linting Signed-off-by: joshvanl <[email protected]> * Updates `dapr-block-shutdown-seconds` to `dapr-block-shutdown-duration` Signed-off-by: joshvanl <[email protected]> --------- Signed-off-by: joshvanl <[email protected]> Co-authored-by: Loong Dai <[email protected]>
JasonChen86899 · Jun 18, 2024 · 5d8950f · 5d8950f
1 parent 1f616bc
commit 5d8950f
Show file tree

Hide file tree

Showing 26 changed files with 1,029 additions and 84 deletions.
diff --git a/cmd/daprd/main.go b/cmd/daprd/main.go
@@ -150,6 +150,7 @@ func main() {
 				UnixDomainSocket:             opts.UnixDomainSocket,
 				DaprHTTPReadBufferSize:       opts.DaprHTTPReadBufferSize,
 				DaprGracefulShutdownSeconds:  opts.DaprGracefulShutdownSeconds,
+				DaprBlockShutdownDuration:    opts.DaprBlockShutdownDuration,
 				DisableBuiltinK8sSecretStore: opts.DisableBuiltinK8sSecretStore,
 				EnableAppHealthCheck:         opts.EnableAppHealthCheck,
 				AppHealthCheckPath:           opts.AppHealthCheckPath,
@@ -175,4 +176,5 @@ func main() {
 	if err != nil {
 		log.Fatalf("Fatal error from runtime: %s", err)
 	}
+	log.Info("Daprd shutdown gracefully")
 }
diff --git a/cmd/daprd/options/options.go b/cmd/daprd/options/options.go
@@ -57,6 +57,7 @@ type Options struct {
 	DaprPublicPort               string
 	AppPort                      string
 	DaprGracefulShutdownSeconds  int
+	DaprBlockShutdownDuration    *time.Duration
 	PlacementServiceHostAddr     string
 	DaprAPIListenAddresses       string
 	AppHealthProbeInterval       int
@@ -79,6 +80,8 @@ func New(args []string) *Options {
 		EnableAPILogging: new(bool),
 	}
 
+	var blockShutdownDuration time.Duration
+
 	flag.StringVar(&opts.Mode, "mode", string(modes.StandaloneMode), "Runtime mode for Dapr")
 	flag.StringVar(&opts.DaprHTTPPort, "dapr-http-port", strconv.Itoa(runtime.DefaultDaprHTTPPort), "HTTP port for Dapr API to listen on")
 	flag.StringVar(&opts.DaprAPIListenAddresses, "dapr-listen-addresses", runtime.DefaultAPIListenAddress, "One or more addresses for the Dapr API to listen on, CSV limited")
@@ -109,6 +112,7 @@ func New(args []string) *Options {
 	flag.StringVar(&opts.UnixDomainSocket, "unix-domain-socket", "", "Path to a unix domain socket dir mount. If specified, Dapr API servers will use Unix Domain Sockets")
 	flag.IntVar(&opts.DaprHTTPReadBufferSize, "dapr-http-read-buffer-size", runtime.DefaultReadBufferSize, "Increasing max size of read buffer in KB to handle sending multi-KB headers")
 	flag.IntVar(&opts.DaprGracefulShutdownSeconds, "dapr-graceful-shutdown-seconds", int(runtime.DefaultGracefulShutdownDuration/time.Second), "Graceful shutdown time in seconds")
+	flag.DurationVar(&blockShutdownDuration, "dapr-block-shutdown-duration", 0, "If enabled, will block graceful shutdown after terminate signal is received until either the given duration has elapsed or the app reports unhealthy. Disabled by default")
 	flag.BoolVar(opts.EnableAPILogging, "enable-api-logging", false, "Enable API logging for API calls")
 	flag.BoolVar(&opts.DisableBuiltinK8sSecretStore, "disable-builtin-k8s-secret-store", false, "Disable the built-in Kubernetes Secret Store")
 	flag.BoolVar(&opts.EnableAppHealthCheck, "enable-app-health-check", false, "Enable health checks for the application using the protocol defined with app-protocol")
@@ -150,6 +154,10 @@ func New(args []string) *Options {
 		}
 	}
 
+	if isFlagPassed("dapr-block-shutdown-duration") {
+		opts.DaprBlockShutdownDuration = &blockShutdownDuration
+	}
+
 	return &opts
 }
 

diff --git a/pkg/injector/annotations/annotations.go b/pkg/injector/annotations/annotations.go
@@ -53,6 +53,7 @@ const (
 	KeyHTTPMaxRequestSize               = "dapr.io/http-max-request-size"
 	KeyHTTPReadBufferSize               = "dapr.io/http-read-buffer-size"
 	KeyGracefulShutdownSeconds          = "dapr.io/graceful-shutdown-seconds"
+	KeyBlockShutdownDuration            = "dapr.io/block-shutdown-duration"
 	KeyEnableAPILogging                 = "dapr.io/enable-api-logging"
 	KeyUnixDomainSocketPath             = "dapr.io/unix-domain-socket-path"
 	KeyVolumeMountsReadOnly             = "dapr.io/volume-mounts"

diff --git a/pkg/injector/patcher/sidecar.go b/pkg/injector/patcher/sidecar.go
@@ -56,57 +56,58 @@ type SidecarConfig struct {
 	SidecarInternalGRPCPort     int32 `default:"50002"`
 	SidecarPublicPort           int32 `default:"3501"`
 
-	Enabled                             bool   `annotation:"dapr.io/enabled"`
-	AppPort                             int32  `annotation:"dapr.io/app-port"`
-	Config                              string `annotation:"dapr.io/config"`
-	AppProtocol                         string `annotation:"dapr.io/app-protocol"                            default:"http"`
-	AppSSL                              bool   `annotation:"dapr.io/app-ssl"` // TODO: Deprecated in Dapr 1.11; remove in a future Dapr version
-	AppID                               string `annotation:"dapr.io/app-id"`
-	EnableProfiling                     bool   `annotation:"dapr.io/enable-profiling"`
-	LogLevel                            string `annotation:"dapr.io/log-level"                               default:"info"`
-	APITokenSecret                      string `annotation:"dapr.io/api-token-secret"`
-	AppTokenSecret                      string `annotation:"dapr.io/app-token-secret"`
-	LogAsJSON                           bool   `annotation:"dapr.io/log-as-json"`
-	AppMaxConcurrency                   *int   `annotation:"dapr.io/app-max-concurrency"`
-	EnableMetrics                       bool   `annotation:"dapr.io/enable-metrics"                          default:"true"`
-	SidecarMetricsPort                  int32  `annotation:"dapr.io/metrics-port"                            default:"9090"`
-	EnableDebug                         bool   `annotation:"dapr.io/enable-debug"                            default:"false"`
-	SidecarDebugPort                    int32  `annotation:"dapr.io/debug-port"                              default:"40000"`
-	Env                                 string `annotation:"dapr.io/env"`
-	SidecarCPURequest                   string `annotation:"dapr.io/sidecar-cpu-request"`
-	SidecarCPULimit                     string `annotation:"dapr.io/sidecar-cpu-limit"`
-	SidecarMemoryRequest                string `annotation:"dapr.io/sidecar-memory-request"`
-	SidecarMemoryLimit                  string `annotation:"dapr.io/sidecar-memory-limit"`
-	SidecarListenAddresses              string `annotation:"dapr.io/sidecar-listen-addresses"                default:"[::1],127.0.0.1"`
-	SidecarLivenessProbeDelaySeconds    int32  `annotation:"dapr.io/sidecar-liveness-probe-delay-seconds"    default:"3"`
-	SidecarLivenessProbeTimeoutSeconds  int32  `annotation:"dapr.io/sidecar-liveness-probe-timeout-seconds"  default:"3"`
-	SidecarLivenessProbePeriodSeconds   int32  `annotation:"dapr.io/sidecar-liveness-probe-period-seconds"   default:"6"`
-	SidecarLivenessProbeThreshold       int32  `annotation:"dapr.io/sidecar-liveness-probe-threshold"        default:"3"`
-	SidecarReadinessProbeDelaySeconds   int32  `annotation:"dapr.io/sidecar-readiness-probe-delay-seconds"   default:"3"`
-	SidecarReadinessProbeTimeoutSeconds int32  `annotation:"dapr.io/sidecar-readiness-probe-timeout-seconds" default:"3"`
-	SidecarReadinessProbePeriodSeconds  int32  `annotation:"dapr.io/sidecar-readiness-probe-period-seconds"  default:"6"`
-	SidecarReadinessProbeThreshold      int32  `annotation:"dapr.io/sidecar-readiness-probe-threshold"       default:"3"`
-	SidecarImage                        string `annotation:"dapr.io/sidecar-image"`
-	SidecarSeccompProfileType           string `annotation:"dapr.io/sidecar-seccomp-profile-type"`
-	HTTPMaxRequestSize                  *int   `annotation:"dapr.io/http-max-request-size"`
-	HTTPReadBufferSize                  *int   `annotation:"dapr.io/http-read-buffer-size"`
-	GracefulShutdownSeconds             int    `annotation:"dapr.io/graceful-shutdown-seconds"               default:"-1"`
-	EnableAPILogging                    *bool  `annotation:"dapr.io/enable-api-logging"`
-	UnixDomainSocketPath                string `annotation:"dapr.io/unix-domain-socket-path"`
-	VolumeMounts                        string `annotation:"dapr.io/volume-mounts"`
-	VolumeMountsRW                      string `annotation:"dapr.io/volume-mounts-rw"`
-	DisableBuiltinK8sSecretStore        bool   `annotation:"dapr.io/disable-builtin-k8s-secret-store"`
-	EnableAppHealthCheck                bool   `annotation:"dapr.io/enable-app-health-check"`
-	AppHealthCheckPath                  string `annotation:"dapr.io/app-health-check-path"                   default:"/healthz"`
-	AppHealthProbeInterval              int32  `annotation:"dapr.io/app-health-probe-interval"               default:"5"`   // In seconds
-	AppHealthProbeTimeout               int32  `annotation:"dapr.io/app-health-probe-timeout"                default:"500"` // In milliseconds
-	AppHealthThreshold                  int32  `annotation:"dapr.io/app-health-threshold"                    default:"3"`
-	PlacementAddress                    string `annotation:"dapr.io/placement-host-address"`
-	PluggableComponents                 string `annotation:"dapr.io/pluggable-components"`
-	PluggableComponentsSocketsFolder    string `annotation:"dapr.io/pluggable-components-sockets-folder"`
-	ComponentContainer                  string `annotation:"dapr.io/component-container"`
-	InjectPluggableComponents           bool   `annotation:"dapr.io/inject-pluggable-components"`
-	AppChannelAddress                   string `annotation:"dapr.io/app-channel-address"`
+	Enabled                             bool    `annotation:"dapr.io/enabled"`
+	AppPort                             int32   `annotation:"dapr.io/app-port"`
+	Config                              string  `annotation:"dapr.io/config"`
+	AppProtocol                         string  `annotation:"dapr.io/app-protocol"                            default:"http"`
+	AppSSL                              bool    `annotation:"dapr.io/app-ssl"` // TODO: Deprecated in Dapr 1.11; remove in a future Dapr version
+	AppID                               string  `annotation:"dapr.io/app-id"`
+	EnableProfiling                     bool    `annotation:"dapr.io/enable-profiling"`
+	LogLevel                            string  `annotation:"dapr.io/log-level"                               default:"info"`
+	APITokenSecret                      string  `annotation:"dapr.io/api-token-secret"`
+	AppTokenSecret                      string  `annotation:"dapr.io/app-token-secret"`
+	LogAsJSON                           bool    `annotation:"dapr.io/log-as-json"`
+	AppMaxConcurrency                   *int    `annotation:"dapr.io/app-max-concurrency"`
+	EnableMetrics                       bool    `annotation:"dapr.io/enable-metrics"                          default:"true"`
+	SidecarMetricsPort                  int32   `annotation:"dapr.io/metrics-port"                            default:"9090"`
+	EnableDebug                         bool    `annotation:"dapr.io/enable-debug"                            default:"false"`
+	SidecarDebugPort                    int32   `annotation:"dapr.io/debug-port"                              default:"40000"`
+	Env                                 string  `annotation:"dapr.io/env"`
+	SidecarCPURequest                   string  `annotation:"dapr.io/sidecar-cpu-request"`
+	SidecarCPULimit                     string  `annotation:"dapr.io/sidecar-cpu-limit"`
+	SidecarMemoryRequest                string  `annotation:"dapr.io/sidecar-memory-request"`
+	SidecarMemoryLimit                  string  `annotation:"dapr.io/sidecar-memory-limit"`
+	SidecarListenAddresses              string  `annotation:"dapr.io/sidecar-listen-addresses"                default:"[::1],127.0.0.1"`
+	SidecarLivenessProbeDelaySeconds    int32   `annotation:"dapr.io/sidecar-liveness-probe-delay-seconds"    default:"3"`
+	SidecarLivenessProbeTimeoutSeconds  int32   `annotation:"dapr.io/sidecar-liveness-probe-timeout-seconds"  default:"3"`
+	SidecarLivenessProbePeriodSeconds   int32   `annotation:"dapr.io/sidecar-liveness-probe-period-seconds"   default:"6"`
+	SidecarLivenessProbeThreshold       int32   `annotation:"dapr.io/sidecar-liveness-probe-threshold"        default:"3"`
+	SidecarReadinessProbeDelaySeconds   int32   `annotation:"dapr.io/sidecar-readiness-probe-delay-seconds"   default:"3"`
+	SidecarReadinessProbeTimeoutSeconds int32   `annotation:"dapr.io/sidecar-readiness-probe-timeout-seconds" default:"3"`
+	SidecarReadinessProbePeriodSeconds  int32   `annotation:"dapr.io/sidecar-readiness-probe-period-seconds"  default:"6"`
+	SidecarReadinessProbeThreshold      int32   `annotation:"dapr.io/sidecar-readiness-probe-threshold"       default:"3"`
+	SidecarImage                        string  `annotation:"dapr.io/sidecar-image"`
+	SidecarSeccompProfileType           string  `annotation:"dapr.io/sidecar-seccomp-profile-type"`
+	HTTPMaxRequestSize                  *int    `annotation:"dapr.io/http-max-request-size"`
+	HTTPReadBufferSize                  *int    `annotation:"dapr.io/http-read-buffer-size"`
+	GracefulShutdownSeconds             int     `annotation:"dapr.io/graceful-shutdown-seconds"               default:"-1"`
+	BlockShutdownDuration               *string `annotation:"dapr.io/block-shutdown-duration"`
+	EnableAPILogging                    *bool   `annotation:"dapr.io/enable-api-logging"`
+	UnixDomainSocketPath                string  `annotation:"dapr.io/unix-domain-socket-path"`
+	VolumeMounts                        string  `annotation:"dapr.io/volume-mounts"`
+	VolumeMountsRW                      string  `annotation:"dapr.io/volume-mounts-rw"`
+	DisableBuiltinK8sSecretStore        bool    `annotation:"dapr.io/disable-builtin-k8s-secret-store"`
+	EnableAppHealthCheck                bool    `annotation:"dapr.io/enable-app-health-check"`
+	AppHealthCheckPath                  string  `annotation:"dapr.io/app-health-check-path"                   default:"/healthz"`
+	AppHealthProbeInterval              int32   `annotation:"dapr.io/app-health-probe-interval"               default:"5"`   // In seconds
+	AppHealthProbeTimeout               int32   `annotation:"dapr.io/app-health-probe-timeout"                default:"500"` // In milliseconds
+	AppHealthThreshold                  int32   `annotation:"dapr.io/app-health-threshold"                    default:"3"`
+	PlacementAddress                    string  `annotation:"dapr.io/placement-host-address"`
+	PluggableComponents                 string  `annotation:"dapr.io/pluggable-components"`
+	PluggableComponentsSocketsFolder    string  `annotation:"dapr.io/pluggable-components-sockets-folder"`
+	ComponentContainer                  string  `annotation:"dapr.io/component-container"`
+	InjectPluggableComponents           bool    `annotation:"dapr.io/inject-pluggable-components"`
+	AppChannelAddress                   string  `annotation:"dapr.io/app-channel-address"`
 
 	pod *corev1.Pod
 }

diff --git a/pkg/injector/patcher/sidecar_container.go b/pkg/injector/patcher/sidecar_container.go
@@ -169,6 +169,10 @@ func (c *SidecarConfig) getSidecarContainer(opts getSidecarContainerOpts) (*core
 		args = append(args, "--unix-domain-socket", injectorConsts.UnixDomainSocketDaprdPath)
 	}
 
+	if c.BlockShutdownDuration != nil {
+		args = append(args, "--dapr-block-shutdown-duration", *c.BlockShutdownDuration)
+	}
+
 	// When debugging is enabled, we need to override the command and the flags
 	if c.EnableDebug {
 		ports = append(ports, corev1.ContainerPort{

diff --git a/pkg/injector/patcher/sidecar_container_test.go b/pkg/injector/patcher/sidecar_container_test.go
@@ -576,6 +576,27 @@ func TestGetSidecarContainer(t *testing.T) {
 		},
 	}))
 
+	t.Run("block shutdown duration", testSuiteGenerator([]testCase{
+		{
+			name:        "default to empty",
+			annotations: map[string]string{},
+			assertFn: func(t *testing.T, container *corev1.Container) {
+				args := strings.Join(container.Args, " ")
+				assert.NotContains(t, args, "--dapr-block-shutdown-duration")
+			},
+		},
+		{
+			name: "add a block shutdown duration",
+			annotations: map[string]string{
+				"dapr.io/block-shutdown-duration": "3s",
+			},
+			assertFn: func(t *testing.T, container *corev1.Container) {
+				args := strings.Join(container.Args, " ")
+				assert.Contains(t, args, "--dapr-block-shutdown-duration 3s")
+			},
+		},
+	}))
+
 	t.Run("sidecar image", testSuiteGenerator([]testCase{
 		{
 			name:        "no annotation",

diff --git a/pkg/runtime/config.go b/pkg/runtime/config.go
@@ -90,6 +90,7 @@ type Config struct {
 	DaprPublicPort               string
 	ApplicationPort              string
 	DaprGracefulShutdownSeconds  int
+	DaprBlockShutdownDuration    *time.Duration
 	PlacementServiceHostAddr     string
 	DaprAPIListenAddresses       string
 	AppHealthProbeInterval       int
@@ -129,6 +130,7 @@ type internalConfig struct {
 	unixDomainSocket             string
 	readBufferSize               int
 	gracefulShutdownDuration     time.Duration
+	blockShutdownDuration        *time.Duration
 	enableAPILogging             *bool
 	disableBuiltinK8sSecretStore bool
 	config                       []string
@@ -284,8 +286,9 @@ func (c *Config) toInternal() (*internalConfig, error) {
 			HealthCheckHTTPPath: c.AppHealthCheckPath,
 			MaxConcurrency:      c.AppMaxConcurrency,
 		},
-		registry:        registry.New(c.Registry),
-		metricsExporter: metrics.NewExporterWithOptions(log, metrics.DefaultMetricNamespace, c.Metrics),
+		registry:              registry.New(c.Registry),
+		metricsExporter:       metrics.NewExporterWithOptions(log, metrics.DefaultMetricNamespace, c.Metrics),
+		blockShutdownDuration: c.DaprBlockShutdownDuration,
 	}
 
 	if len(intc.standalone.ResourcesPath) == 0 && c.ComponentsPath != "" {

diff --git a/pkg/runtime/config_test.go b/pkg/runtime/config_test.go
@@ -55,6 +55,8 @@ func TestParsePlacementAddr(t *testing.T) {
 func Test_toInternal(t *testing.T) {
 	cfg := defaultTestConfig()
 
+	var nilDuration *time.Duration
+
 	intc, err := cfg.toInternal()
 	require.NoError(t, err)
 
@@ -81,6 +83,7 @@ func Test_toInternal(t *testing.T) {
 	assert.Equal(t, "", intc.unixDomainSocket)
 	assert.Equal(t, 4, intc.readBufferSize)
 	assert.Equal(t, time.Second, intc.gracefulShutdownDuration)
+	assert.Equal(t, nilDuration, intc.blockShutdownDuration)
 	assert.Equal(t, ptr.Of(true), intc.enableAPILogging)
 	assert.True(t, intc.disableBuiltinK8sSecretStore)
 	assert.Equal(t, "1.1.1.1", intc.appConnectionConfig.ChannelAddress)