Skip to content

Commit 9c5dbca

Browse files
ferantiveroferantivero
committed
Merged PR 875: add readiness/liveness probes to workflow
add healthz file creation based on health check status now workflow adds a readiness/liveness health check, and also a health check publisher. This way, the health check system periodically executes this check. - create file when reporting healthy - delete file when reporting unhealthy - add health check - add health check publisher to o the service container, - add publisher unit tests - add readiness/liveness probes cfg solved: #133010
1 parent 78cd88b commit 9c5dbca

File tree

6 files changed

+288
-1
lines changed

6 files changed

+288
-1
lines changed

‎charts/workflow/templates/workflow-deploy.yaml

+38
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,42 @@ spec:
4646
- name: fabrikam-workflow
4747
image: {{ .Values.dockerregistry }}{{ .Values.dockerregistrynamespace }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}
4848
imagePullPolicy: {{ .Values.image.pullPolicy }}
49+
readinessProbe:
50+
exec:
51+
command:
52+
{{- range .Values.readinessProbe.exec.command }}
53+
- {{ . | quote }}
54+
{{- end }}
55+
{{- if .Values.readinessProbe.initialDelaySeconds }}
56+
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
57+
{{- end }}
58+
{{- if .Values.readinessProbe.periodSeconds }}
59+
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
60+
{{- end }}
61+
{{- if .Values.readinessProbe.timeoutSeconds }}
62+
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
63+
{{- end }}
64+
{{- if .Values.readinessProbe.failureThreshold }}
65+
failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
66+
{{- end }}
67+
livenessProbe:
68+
exec:
69+
command:
70+
{{- range .Values.livenessProbe.exec.command }}
71+
- {{ . | quote }}
72+
{{- end }}
73+
{{- if .Values.livenessProbe.initialDelaySeconds }}
74+
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
75+
{{- end }}
76+
{{- if .Values.livenessProbe.periodSeconds }}
77+
periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
78+
{{- end }}
79+
{{- if .Values.livenessProbe.timeoutSeconds }}
80+
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
81+
{{- end }}
82+
{{- if .Values.livenessProbe.failureThreshold }}
83+
failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
84+
{{- end }}
4985
resources:
5086
requests:
5187
cpu: {{ required "A valid .Values.resources.requests.cpu entry required!" .Values.resources.requests.cpu }}
@@ -60,6 +96,8 @@ spec:
6096
env:
6197
- name: CONFIGURATION_FOLDER
6298
value: /kvmnt
99+
- name: HEALTHCHECK_INITIAL_DELAY
100+
value: {{ default "30000" .Values.healthcheck.delay | quote }}
63101
- name: SERVICE_URI_DELIVERY
64102
value: {{ .Values.serviceuri.delivery }}
65103
- name: SERVICE_URI_DRONE

‎charts/workflow/values.yaml

+21-1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,26 @@ servicerequest:
2323
circuitbreakerbreakduration: 30
2424
maxbulkheadsize: 100
2525
maxbulkheadqueuesize: 25
26+
healthcheck:
27+
delay:
28+
readinessProbe:
29+
exec:
30+
command:
31+
- cat
32+
- /app/healthz
33+
initialDelaySeconds: 40
34+
periodSeconds: 15
35+
timeoutSeconds: 2
36+
failureThreshold: 5
37+
livenessProbe:
38+
exec:
39+
command:
40+
- find
41+
- /app/healthz
42+
- -mmin
43+
- -1
44+
initialDelaySeconds: 50
45+
periodSeconds: 30
2646
keyvault:
2747
name:
2848
resourcegroup:
@@ -34,4 +54,4 @@ tags:
3454
dev: false
3555
prod: false
3656
qa: false
37-
staging: false
57+
staging: false

‎src/shipping/workflow/Fabrikam.Workflow.Service.Tests/ReadinessLivenessPublisherTests.cs

+120
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
// ------------------------------------------------------------
2+
// Copyright (c) Microsoft Corporation. All rights reserved.
3+
// Licensed under the MIT License (MIT). See License.txt in the repo root for license information.
4+
// ------------------------------------------------------------
5+
6+
using System;
7+
using System.Collections.Generic;
8+
using System.IO;
9+
using System.Threading;
10+
using System.Threading.Tasks;
11+
using Microsoft.Extensions.DependencyInjection;
12+
using Microsoft.Extensions.Diagnostics.HealthChecks;
13+
using Microsoft.Extensions.Logging;
14+
using Fabrikam.Workflow.Service.Services;
15+
using Moq;
16+
using Xunit;
17+
18+
namespace Fabrikam.Workflow.Service.Tests
19+
{
20+
public class ReadinessLivenessPublisherTests
21+
{
22+
private const int DelayCompletionMs = 1000;
23+
24+
private readonly ReadinessLivenessPublisher _publisher;
25+
26+
public ReadinessLivenessPublisherTests()
27+
{
28+
var servicesBuilder = new ServiceCollection();
29+
servicesBuilder.AddLogging(logging => logging.AddDebug());
30+
var services = servicesBuilder.BuildServiceProvider();
31+
32+
_publisher =
33+
new ReadinessLivenessPublisher(
34+
services.GetService<ILogger<ReadinessLivenessPublisher>>());
35+
}
36+
37+
[Fact]
38+
public async Task WhenPublishingAndReportIsHealthy_FileExists()
39+
{
40+
// Arrange
41+
var healthReportEntries = new Dictionary<string, HealthReportEntry>()
42+
{
43+
{"healthy", new HealthReportEntry(HealthStatus.Healthy, null,TimeSpan.MinValue, null, null) }
44+
};
45+
46+
// Act
47+
await _publisher.PublishAsync(
48+
new HealthReport(healthReportEntries, TimeSpan.MinValue),
49+
new CancellationTokenSource().Token);
50+
51+
// Arrange
52+
Assert.True(File.Exists(ReadinessLivenessPublisher.FilePath));
53+
}
54+
55+
[Fact]
56+
public async Task WhenPublishingAndReportIsUnhealthy_FileDateTimeIsNotModified()
57+
{
58+
// Arrange
59+
var healthReportEntries = new Dictionary<string, HealthReportEntry>()
60+
{
61+
{"healthy", new HealthReportEntry(HealthStatus.Healthy, null,TimeSpan.MinValue, null, null) }
62+
};
63+
64+
await _publisher.PublishAsync(
65+
new HealthReport(
66+
healthReportEntries,
67+
TimeSpan.MinValue),
68+
new CancellationTokenSource().Token);
69+
70+
healthReportEntries.Add(
71+
"unhealthy",
72+
new HealthReportEntry(
73+
HealthStatus.Unhealthy,
74+
null,TimeSpan.MinValue, null, null));
75+
76+
// Act
77+
DateTime healthyWriteTime = File.GetLastWriteTime(ReadinessLivenessPublisher.FilePath);
78+
await _publisher.PublishAsync(
79+
new HealthReport(healthReportEntries, TimeSpan.MinValue),
80+
new CancellationTokenSource().Token);
81+
82+
// Arrange
83+
Assert.True(File.Exists(ReadinessLivenessPublisher.FilePath));
84+
Assert.Equal(healthyWriteTime, File.GetLastWriteTime(ReadinessLivenessPublisher.FilePath));
85+
}
86+
87+
[Fact(Timeout = DelayCompletionMs * 3)]
88+
public async Task WhenPublishingAndReportIsHealthyTwice_FileDateTimeIsModified()
89+
{
90+
// Arrange
91+
Func<Task> emulatePeriodicHealthCheckAsync =
92+
() => Task.Delay(DelayCompletionMs);
93+
94+
var healthReportEntries = new Dictionary<string, HealthReportEntry>()
95+
{
96+
{"healthy", new HealthReportEntry(HealthStatus.Healthy, null,TimeSpan.MinValue, null, null) }
97+
};
98+
99+
// Act
100+
await _publisher.PublishAsync(
101+
new HealthReport(
102+
healthReportEntries,
103+
TimeSpan.MinValue),
104+
new CancellationTokenSource().Token);
105+
106+
DateTime firstTimehealthyWriteTime = File.GetLastWriteTime(ReadinessLivenessPublisher.FilePath);
107+
108+
await emulatePeriodicHealthCheckAsync();
109+
110+
await _publisher.PublishAsync(
111+
new HealthReport(healthReportEntries, TimeSpan.MinValue),
112+
new CancellationTokenSource().Token);
113+
114+
DateTime sencondTimehealthyWriteTime = File.GetLastWriteTime(ReadinessLivenessPublisher.FilePath);
115+
116+
// Arrange
117+
Assert.True(firstTimehealthyWriteTime < sencondTimehealthyWriteTime);
118+
}
119+
}
120+
}

‎src/shipping/workflow/Fabrikam.Workflow.Service/Fabrikam.Workflow.Service.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
<ItemGroup>
1010
<PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.6.1" />
11+
<PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="2.2.0" />
1112
<PackageReference Include="Microsoft.Extensions.Logging.ApplicationInsights" Version="2.9.1" />
1213
<PackageReference Include="Microsoft.ApplicationInsights.Kubernetes" Version="1.0.2" />
1314
<PackageReference Include="Microsoft.AspNet.WebApi.Client" Version="5.2.7" />

‎src/shipping/workflow/Fabrikam.Workflow.Service/ServiceStartup.cs

+27
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55

66
using System;
77
using Microsoft.Extensions.DependencyInjection;
8+
using Microsoft.Extensions.DependencyInjection.Extensions;
9+
using Microsoft.Extensions.Diagnostics.HealthChecks;
810
using Microsoft.Extensions.Hosting;
911
using Fabrikam.Workflow.Service.RequestProcessing;
1012
using Fabrikam.Workflow.Service.Services;
@@ -13,6 +15,9 @@ namespace Fabrikam.Workflow.Service
1315
{
1416
public static class ServiceStartup
1517
{
18+
private const string HealthCheckName = "ReadinessLiveness";
19+
private const string HealthCheckServiceAssembly = "Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckPublisherHostedService";
20+
1621
public static void ConfigureServices(HostBuilderContext context, IServiceCollection services)
1722
{
1823
services.AddOptions();
@@ -26,6 +31,20 @@ public static void ConfigureServices(HostBuilderContext context, IServiceCollect
2631

2732
services.AddTransient<IRequestProcessor, RequestProcessor>();
2833

34+
// Add health check │
35+
services.AddHealthChecks().AddCheck(
36+
HealthCheckName,
37+
() => HealthCheckResult.Healthy("OK"));
38+
39+
if (context.Configuration["HEALTHCHECK_INITIAL_DELAY"] is var configuredDelay &&
40+
double.TryParse(configuredDelay, out double delay))
41+
{
42+
services.Configure<HealthCheckPublisherOptions>(options =>
43+
{
44+
options.Delay = TimeSpan.FromMilliseconds(delay);
45+
});
46+
}
47+
2948
services
3049
.AddHttpClient<IPackageServiceCaller, PackageServiceCaller>(c =>
3150
{
@@ -46,6 +65,14 @@ public static void ConfigureServices(HostBuilderContext context, IServiceCollect
4665
c.BaseAddress = new Uri(context.Configuration["SERVICE_URI_DELIVERY"]);
4766
})
4867
.AddResiliencyPolicies(context.Configuration);
68+
69+
// workaround .NET Core 2.2: for more info https://github.com/aspnet/AspNetCore.Docs/blob/master/aspnetcore/host-and-deploy/health-checks/samples/2.x/HealthChecksSample/LivenessProbeStartup.cs#L51
70+
services.TryAddEnumerable(
71+
ServiceDescriptor.Singleton(typeof(IHostedService),
72+
typeof(HealthCheckPublisherOptions).Assembly
73+
.GetType(HealthCheckServiceAssembly)));
74+
75+
services.AddSingleton<IHealthCheckPublisher, ReadinessLivenessPublisher>();
4976
}
5077
}
5178
}

‎src/shipping/workflow/Fabrikam.Workflow.Service/Services/ReadinessLivenessPublisher.cs

+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
// ------------------------------------------------------------
2+
// Copyright (c) Microsoft Corporation. All rights reserved.
3+
// Licensed under the MIT License (MIT). See License.txt in the repo root for license information.
4+
// ------------------------------------------------------------
5+
6+
using System;
7+
using System.IO;
8+
using System.Threading;
9+
using System.Threading.Tasks;
10+
using Microsoft.Extensions.Diagnostics.HealthChecks;
11+
using Microsoft.Extensions.Logging;
12+
13+
namespace Fabrikam.Workflow.Service.Services
14+
{
15+
public class ReadinessLivenessPublisher : IHealthCheckPublisher
16+
{
17+
public const string FilePath = "healthz";
18+
19+
private readonly ILogger _logger;
20+
21+
public ReadinessLivenessPublisher(ILogger<ReadinessLivenessPublisher> logger)
22+
{
23+
this._logger = logger;
24+
}
25+
26+
public Task PublishAsync(HealthReport report,
27+
CancellationToken cancellationToken)
28+
{
29+
switch (report.Status)
30+
{
31+
case HealthStatus.Healthy:
32+
{
33+
this._logger.LogInformation(
34+
"{Timestamp} Readiness/Liveness Probe Status: {Result}",
35+
DateTime.UtcNow,
36+
report.Status);
37+
38+
CreateOrUpdateHealthz();
39+
40+
break;
41+
}
42+
43+
case HealthStatus.Degraded:
44+
{
45+
this._logger.LogWarning(
46+
"{Timestamp} Readiness/Liveness Probe Status: {Result}",
47+
DateTime.UtcNow,
48+
report.Status);
49+
50+
break;
51+
}
52+
53+
case HealthStatus.Unhealthy:
54+
{
55+
this._logger.LogError(
56+
"{Timestamp} Readiness Probe/Liveness Status: {Result}",
57+
DateTime.UtcNow,
58+
report.Status);
59+
60+
break;
61+
}
62+
}
63+
64+
cancellationToken.ThrowIfCancellationRequested();
65+
66+
return Task.CompletedTask;
67+
}
68+
69+
private static void CreateOrUpdateHealthz()
70+
{
71+
if (File.Exists(FilePath))
72+
{
73+
File.SetLastWriteTimeUtc(FilePath, DateTime.UtcNow);
74+
}
75+
else
76+
{
77+
File.AppendText(FilePath).Close();
78+
}
79+
}
80+
}
81+
}

0 commit comments

Comments
 (0)