Skip to content

Commit

Permalink
[cli] Print goroutine stacktrace on SIGTERM
Browse files Browse the repository at this point in the history
This will help debug current and future timeouts in infrastructure, e.g.
commit 612779b had to be reverted
because it was causing shac to apparently hang.

Change-Id: I9c95b0e5088a967abfc384a82f1b2acf56278b8b
Reviewed-on: https://fuchsia-review.googlesource.com/c/shac-project/shac/+/929634
Fuchsia-Auto-Submit: Oliver Newman <[email protected]>
Reviewed-by: Ina Huh <[email protected]>
Commit-Queue: Auto-Submit <[email protected]>
  • Loading branch information
orn688 authored and CQ Bot committed Oct 11, 2023
1 parent 0dae038 commit 9a096d6
Show file tree
Hide file tree
Showing 3 changed files with 34 additions and 12 deletions.
4 changes: 1 addition & 3 deletions internal/cli/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,7 @@ type subcommand interface {
}

// Main implements shac executable.
func Main(args []string) error {
ctx := context.Background()

func Main(ctx context.Context, args []string) error {
subcommands := [...]subcommand{
// Ordered roughly by importance, because ordering here corresponds to
// the order in which subcommands will be listed in `shac help`.
Expand Down
5 changes: 3 additions & 2 deletions internal/cli/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package cli

import (
"bytes"
"context"
"fmt"
"os"
"path/filepath"
Expand Down Expand Up @@ -43,7 +44,7 @@ func TestMainHelp(t *testing.T) {
for i, line := range data {
t.Run(strconv.Itoa(i), func(t *testing.T) {
b := getBuf(t)
if Main(line.args) == nil {
if Main(context.Background(), line.args) == nil {
t.Fatal("expected error")
}
if s := b.String(); !strings.HasPrefix(s, line.want) {
Expand Down Expand Up @@ -128,7 +129,7 @@ func TestMainErr(t *testing.T) {
t.Parallel()
args, wantErr := f(t)
cmd := append([]string{"shac"}, args...)
err := Main(cmd)
err := Main(context.Background(), cmd)
if err == nil {
t.Fatalf("Expected error from running %s", cmd)
}
Expand Down
37 changes: 30 additions & 7 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,13 @@
package main

import (
"context"
"errors"
"fmt"
"os"
"os/signal"
"runtime/pprof"
"syscall"

"github.com/mattn/go-isatty"
flag "github.com/spf13/pflag"
Expand All @@ -27,17 +31,36 @@ import (
)

func main() {
if err := cli.Main(os.Args); err != nil && !errors.Is(err, flag.ErrHelp) {
signalChannel := make(chan os.Signal, 2)
signal.Notify(signalChannel, syscall.SIGTERM, syscall.SIGINT)
ctx, cancel := context.WithCancel(context.Background())
go func() {
sig := <-signalChannel
cancel()
// Print a goroutine stacktrace only on SIGTERM - we only want to see a
// stack trace when shac gets canceled by automation, which may indicate
// a timeout due to a hang. If shac gets Ctrl-C'd (SIGINT) by a human
// user it's not helpful to print a stacktrace.
if sig == syscall.SIGTERM {
_ = pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
}
}()

if err := cli.Main(ctx, os.Args); err != nil && !errors.Is(err, flag.ErrHelp) {
var stackerr engine.BacktraceableError
if errors.As(err, &stackerr) {
_, _ = os.Stderr.WriteString(stackerr.Backtrace())
}
// If a check failed and stderr is a terminal, appropriate information
// should have already been emitted by the reporter. If stderr is not a
// terminal then it may still be useful to print the "check failed"
// error message since the reporter output may not show up in the same
// stream as stderr.
if !errors.Is(err, engine.ErrCheckFailed) || !isatty.IsTerminal(os.Stderr.Fd()) {
// If stderr is not a terminal, always print the error.
//
// If stderr is a terminal:
// - If a check failed, appropriate information should have already been
// emitted by the reporter.
// - A context cancellation will likely be because the user Ctrl-C'd
// shac, so the exit will be expected and there's no need to print
// anything.
if !isatty.IsTerminal(os.Stderr.Fd()) ||
(!errors.Is(err, engine.ErrCheckFailed) && !errors.Is(err, context.Canceled)) {
_, _ = fmt.Fprintf(os.Stderr, "shac: %s\n", err)
}
os.Exit(1)
Expand Down

0 comments on commit 9a096d6

Please sign in to comment.