Integration tests occasionally fail for "ERROR cannot start service: exited quickly with code 2"

[sed-i]

Bug Description

This may indicate an issue in charm code relating to service restart.

To Reproduce

Run integration tests in CI. Recent example.

Environment

Github CI runner.

Relevant log output

unit-grafana-0: 06:21:03 ERROR unit.grafana/0.juju-log Could not restart grafana at this time: cannot perform the following tasks:
- Start service "grafana" (cannot start service: exited quickly with code 2)
----- Logs from task 0 -----
2025-01-19T06:21:03Z INFO Most recent service output:
    (...)
    github.com/grafana/grafana/pkg/services/ngalert/notifier.(*MultiOrgAlertmanager).LoadAndSyncAlertmanagersForOrgs(0xc000a2ef00, {0x41a9090, 0xc0001365a0})
    	/root/parts/grafana/build/pkg/services/ngalert/notifier/multiorg_alertmanager.go:141 +0xd8
    github.com/grafana/grafana/pkg/services/ngalert.(*AlertNG).init(0xc0008d2ea0)
    	/root/parts/grafana/build/pkg/services/ngalert/ngalert.go:177 +0x466
    github.com/grafana/grafana/pkg/services/ngalert.ProvideService(_, {_, _}, {_, _}, {_, _}, {_, _}, {0x41c39e0, ...}, ...)
    	/root/parts/grafana/build/pkg/services/ngalert/ngalert.go:101 +0x5a9
    github.com/grafana/grafana/pkg/server.Initialize({{0x7ffe12c3d757, 0x1f}, {0x0, 0x0}, {0xc000a28b90, 0x1, 0x1}}, {{0x0, 0x0}, {0x0, ...}, ...}, ...)
    	/root/parts/grafana/build/pkg/server/wire_gen.go:441 +0x4c9e
    github.com/grafana/grafana/pkg/cmd/grafana-server/commands.RunServer({{0x416c7b4, 0x5}, {0x416d9f8, 0x7}, {0x416c7ac, 0x4}, {0x416daa8, 0xa}, 0xc000a935c0})
    	/root/parts/grafana/build/pkg/cmd/grafana-server/commands/cli.go:228 +0x100a
    github.com/grafana/grafana/pkg/cmd/grafana-server/commands.ServerCommand.func1(0xc0004eb680?)
    	/root/parts/grafana/build/pkg/cmd/grafana-server/commands/cli.go:97 +0xc5
    github.com/urfave/cli/v2.(*Command).Run(0xc0004806c0, 0xc000a931c0)
    	/root/go/pkg/mod/github.com/urfave/cli/[email protected]/command.go:163 +0x5dc
    github.com/urfave/cli/v2.(*App).RunContext(0xc0001036c0, {0x41a9058?, 0xc000060040}, {0xc000052080, 0x4, 0x4})
    	/root/go/pkg/mod/github.com/urfave/cli/[email protected]/app.go:313 +0xb7d
    github.com/urfave/cli/v2.(*App).Run(...)
    	/root/go/pkg/mod/github.com/urfave/cli/[email protected]/app.go:224
    main.main()
    	/root/parts/grafana/build/pkg/cmd/grafana/main.go:39 +0x27d
2025-01-19T06:21:03Z ERROR cannot start service: exited quickly with code 2
-----
unit-grafana-0: 06:21:07 ERROR unit.grafana/0.juju-log Could not restart grafana at this time: cannot perform the following tasks:
- Start service "grafana" (cannot start service: exited quickly with code 2)
----- Logs from task 0 -----
2025-01-19T06:21:07Z INFO Most recent service output:
    (...)
    github.com/grafana/grafana/pkg/services/ngalert/notifier.(*MultiOrgAlertmanager).LoadAndSyncAlertmanagersForOrgs(0xc000f46480, {0x41a9090, 0xc0011243c0})
    	/root/parts/grafana/build/pkg/services/ngalert/notifier/multiorg_alertmanager.go:141 +0xd8
    github.com/grafana/grafana/pkg/services/ngalert.(*AlertNG).init(0xc000d7cd00)
    	/root/parts/grafana/build/pkg/services/ngalert/ngalert.go:177 +0x466
    github.com/grafana/grafana/pkg/services/ngalert.ProvideService(_, {_, _}, {_, _}, {_, _}, {_, _}, {0x41c39e0, ...}, ...)
    	/root/parts/grafana/build/pkg/services/ngalert/ngalert.go:101 +0x5a9
    github.com/grafana/grafana/pkg/server.Initialize({{0x7ffed09f1757, 0x1f}, {0x0, 0x0}, {0xc000376860, 0x1, 0x1}}, {{0x0, 0x0}, {0x0, ...}, ...}, ...)
    	/root/parts/grafana/build/pkg/server/wire_gen.go:441 +0x4c9e
    github.com/grafana/grafana/pkg/cmd/grafana-server/commands.RunServer({{0x416c7b4, 0x5}, {0x416d9f8, 0x7}, {0x416c7ac, 0x4}, {0x416daa8, 0xa}, 0xc0002ecc80})
    	/root/parts/grafana/build/pkg/cmd/grafana-server/commands/cli.go:228 +0x100a
-----
unit-grafana-0: 06:21:37 ERROR unit.grafana/0.juju-log certificates:2: Could not restart grafana at this time: cannot perform the following tasks:
- Start service "grafana" (cannot start service: exited quickly with code 2)
----- Logs from task 0 -----
2025-01-19T06:21:37Z INFO Most recent service output:
    (...)
    github.com/grafana/grafana/pkg/services/ngalert/notifier.(*MultiOrgAlertmanager).LoadAndSyncAlertmanagersForOrgs(0xc0000e3b00, {0x41a9090, 0xc0008e69c0})
    	/root/parts/grafana/build/pkg/services/ngalert/notifier/multiorg_alertmanager.go:141 +0xd8
    github.com/grafana/grafana/pkg/services/ngalert.(*AlertNG).init(0xc00096c1a0)
    	/root/parts/grafana/build/pkg/services/ngalert/ngalert.go:177 +0x466
    github.com/grafana/grafana/pkg/services/ngalert.ProvideService(_, {_, _}, {_, _}, {_, _}, {_, _}, {0x41c39e0, ...}, ...)
    	/root/parts/grafana/build/pkg/services/ngalert/ngalert.go:101 +0x5a9
    github.com/grafana/grafana/pkg/server.Initialize({{0x7ffdb197f757, 0x1f}, {0x0, 0x0}, {0xc0009c1620, 0x1, 0x1}}, {{0x0, 0x0}, {0x0, ...}, ...}, ...)
    	/root/parts/grafana/build/pkg/server/wire_gen.go:441 +0x4c9e
    github.com/grafana/grafana/pkg/cmd/grafana-server/commands.RunServer({{0x416c7b4, 0x5}, {0x416d9f8, 0x7}, {0x416c7ac, 0x4}, {0x416daa8, 0xa}, 0xc00072c800})
    	/root/parts/grafana/build/pkg/cmd/grafana-server/commands/cli.go:228 +0x100a
    github.com/grafana/grafana/pkg/cmd/grafana-server/commands.ServerCommand.func1(0xc0009f4000?)
    	/root/parts/grafana/build/pkg/cmd/grafana-server/commands/cli.go:97 +0xc5
    github.com/urfave/cli/v2.(*Command).Run(0xc0009e4120, 0xc00072c400)
    	/root/go/pkg/mod/github.com/urfave/cli/[email protected]/command.go:163 +0x5dc
    github.com/urfave/cli/v2.(*App).RunContext(0xc00067cb60, {0x41a9058?, 0xc000128000}, {0xc000122000, 0x4, 0x4})
    	/root/go/pkg/mod/github.com/urfave/cli/[email protected]/app.go:313 +0xb7d
    github.com/urfave/cli/v2.(*App).Run(...)
    	/root/go/pkg/mod/github.com/urfave/cli/[email protected]/app.go:224
    main.main()
    	/root/parts/grafana/build/pkg/cmd/grafana/main.go:39 +0x27d
2025-01-19T06:21:37Z ERROR cannot start service: exited quickly with code 2

Additional context

No response