From 0c0bdd1081cb4a1fcfe722a8e6e0512284782db3 Mon Sep 17 00:00:00 2001 From: Serge Smertin Date: Wed, 27 Mar 2024 23:26:58 +0100 Subject: [PATCH] Create issues on failures of integration tests --- .github/workflows/build.sh | 4 +-- .github/workflows/nightly-go-libs.yml | 36 +++++++++++++++++++++++++++ acceptance/action.yml | 4 +++ acceptance/boilerplate/actions.go | 34 +++++++++++++++++++++++++ acceptance/ecosystem/report.go | 20 +++++++++++---- acceptance/main.go | 34 +++++++++++++++++++++---- acceptance/shim.js | 2 +- go-libs/github/github.go | 12 +++++++++ go-libs/github/issues.go | 7 ++++++ go-libs/sqlexec/sqlexec_test.go | 2 +- go.work.sum | 2 ++ 11 files changed, 143 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/nightly-go-libs.yml diff --git a/.github/workflows/build.sh b/.github/workflows/build.sh index 23be212..514b370 100755 --- a/.github/workflows/build.sh +++ b/.github/workflows/build.sh @@ -3,8 +3,8 @@ app=$1 # Define platforms and architectures -platforms=("windows" "linux" "darwin") -architectures=("amd64" "arm64") +platforms=("linux") +architectures=("amd64") mkdir -p dist # Build binaries for each platform and architecture combination diff --git a/.github/workflows/nightly-go-libs.yml b/.github/workflows/nightly-go-libs.yml new file mode 100644 index 0000000..2f63e9d --- /dev/null +++ b/.github/workflows/nightly-go-libs.yml @@ -0,0 +1,36 @@ +name: acceptance + +on: + workflow_dispatch: + # ... + +permissions: + id-token: write + contents: read + pull-requests: write + issues: write + +jobs: + acceptance: + environment: tool + runs-on: larger + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: 1.21 + + - name: Acceptance + uses: databrickslabs/sandbox/acceptance@acceptance/v0.2.0 + with: + directory: go-libs + vault_uri: ${{ secrets.VAULT_URI }} + create_issues: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} + ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} + \ No newline at end of file diff --git a/acceptance/action.yml b/acceptance/action.yml index 06c8b70..55230e7 100644 --- a/acceptance/action.yml +++ b/acceptance/action.yml @@ -20,6 +20,10 @@ inputs: description: 'Maximum suite execution time. Defaults to 1h' required: false default: 1h + create_issues: + description: 'Create issues in the repository for failed tests' + required: false + default: false outputs: sample: description: 'Sample output' diff --git a/acceptance/boilerplate/actions.go b/acceptance/boilerplate/actions.go index cd50124..12d7380 100644 --- a/acceptance/boilerplate/actions.go +++ b/acceptance/boilerplate/actions.go @@ -92,6 +92,40 @@ func (a *boilerplate) RunURL(ctx context.Context) (string, error) { return "", fmt.Errorf("id not found for current run: %s", a.context.Job) } +func (a *boilerplate) CreateIssueIfNotOpen(ctx context.Context, newIssue github.NewIssue) error { + org, repo := a.context.Repo() + it := a.GitHub.ListRepositoryIssues(ctx, org, repo, &github.ListIssues{ + State: "open", + }) + created := map[string]bool{} + for it.HasNext(ctx) { + issue, err := it.Next(ctx) + if err != nil { + return fmt.Errorf("issue: %w", err) + } + created[issue.Title] = true + } + if created[newIssue.Title] { + return nil + } + body, err := a.taggedComment(ctx, newIssue.Body) + if err != nil { + return fmt.Errorf("tagged comment: %w", err) + } + // with the tagged comment, which has the workflow ref, we can link to the run + issue, err := a.GitHub.CreateIssue(ctx, org, repo, github.NewIssue{ + Title: newIssue.Title, + Assignees: newIssue.Assignees, + Labels: newIssue.Labels, + Body: body, + }) + if err != nil { + return fmt.Errorf("new issue: %w", err) + } + logger.Infof(ctx, "Created issue: https://github.com/%s/%s/issues/%d", issue.Number) + return nil +} + func (a *boilerplate) tag() string { // The ref path to the workflow. For example, // octocat/hello-world/.github/workflows/my-workflow.yml@refs/heads/my_branch. diff --git a/acceptance/ecosystem/report.go b/acceptance/ecosystem/report.go index 7ee796d..1f808cf 100644 --- a/acceptance/ecosystem/report.go +++ b/acceptance/ecosystem/report.go @@ -25,6 +25,19 @@ func (tr TestResult) Duration() time.Duration { return time.Duration(tr.Elapsed * float64(time.Second)) } +func (tr TestResult) Failed() bool { + return !tr.Pass && !tr.Skip +} + +func (tr TestResult) Summary() string { + res := []string{} + res = append(res, "
") + res = append(res, fmt.Sprintf("%s", tr)) + res = append(res, fmt.Sprintf("\n```\n%s\n```\n", tr.Output)) + res = append(res, "
") + return strings.Join(res, "\n") +} + func (tr TestResult) String() string { summary := "" if !tr.Pass { @@ -138,13 +151,10 @@ func (r TestReport) String() string { func (r TestReport) StepSummary() string { res := []string{r.String()} for _, v := range r { - if v.Pass || v.Skip { + if !v.Failed() { continue } - res = append(res, "
") - res = append(res, fmt.Sprintf("%s", v)) - res = append(res, fmt.Sprintf("\n```\n%s\n```\n", v.Output)) - res = append(res, "
") + res = append(res, v.Summary()) } if r.Flaky() { res = append(res, "\nFlaky tests:\n") diff --git a/acceptance/main.go b/acceptance/main.go index 8ba977e..60bea96 100644 --- a/acceptance/main.go +++ b/acceptance/main.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" "github.com/databrickslabs/sandbox/acceptance/boilerplate" @@ -12,6 +13,7 @@ import ( "github.com/databrickslabs/sandbox/acceptance/notify" "github.com/databrickslabs/sandbox/acceptance/testenv" "github.com/databrickslabs/sandbox/go-libs/env" + "github.com/databrickslabs/sandbox/go-libs/github" "github.com/sethvargo/go-githubactions" ) @@ -79,21 +81,43 @@ func run(ctx context.Context, opts ...githubactions.Option) error { return fmt.Errorf("upload artifact: %w", err) } slackWebhook := b.Action.GetInput("slack_webhook") - if !report.Pass() && slackWebhook != "" { + createIssues := strings.ToLower(b.Action.GetInput("create_issues")) + needsSlack := slackWebhook != "" + needsIssues := createIssues == "true" || createIssues == "yes" + needsNotification := needsSlack || needsIssues + if !report.Pass() && needsNotification { runUrl, err := b.RunURL(ctx) if err != nil { return fmt.Errorf("run url: %w", err) } - err = notify.Notification{ + alert := notify.Notification{ Project: project, Report: report, Cloud: loaded.Cloud(), RunName: b.WorkflowRunName(), WebHook: slackWebhook, RunURL: runUrl, - }.ToSlack() - if err != nil { - return fmt.Errorf("slack: %w", err) + } + if needsSlack { + err = alert.ToSlack() + if err != nil { + return fmt.Errorf("slack: %w", err) + } + } + if needsIssues { + for _, v := range report { + if !v.Failed() { + continue + } + err = b.CreateIssueIfNotOpen(ctx, github.NewIssue{ + Title: fmt.Sprintf("Test failure: `%s`", v.Name), + Body: v.Summary(), + Labels: []string{"bug"}, + }) + if err != nil { + return fmt.Errorf("create issue: %w", err) + } + } } } return report.Failed() diff --git a/acceptance/shim.js b/acceptance/shim.js index f511809..4b0da06 100644 --- a/acceptance/shim.js +++ b/acceptance/shim.js @@ -1,4 +1,4 @@ -const version = 'v0.1.4'; +const version = 'v0.2.0'; const { createWriteStream, chmodSync } = require('fs'); const { createGunzip } = require('zlib'); diff --git a/go-libs/github/github.go b/go-libs/github/github.go index 62080d2..8bcc5d6 100644 --- a/go-libs/github/github.go +++ b/go-libs/github/github.go @@ -292,6 +292,18 @@ func (c *GitHubClient) GetIssueComments(ctx context.Context, org, repo string, n }) } +func (c *GitHubClient) CreateIssue(ctx context.Context, org, repo string, body NewIssue) (*Issue, error) { + path := fmt.Sprintf("%s/repos/%s/%s/issues", gitHubAPI, org, repo) + var res Issue + err := c.api.Do(ctx, "POST", path, + httpclient.WithRequestData(body), + httpclient.WithResponseUnmarshal(&res)) + if err != nil { + return nil, err + } + return &res, nil +} + func (c *GitHubClient) CreateIssueComment(ctx context.Context, org, repo string, number int, body string) (*IssueComment, error) { path := fmt.Sprintf("%s/repos/%s/%s/issues/%d/comments", gitHubAPI, org, repo, number) var res IssueComment diff --git a/go-libs/github/issues.go b/go-libs/github/issues.go index f80453b..1404679 100644 --- a/go-libs/github/issues.go +++ b/go-libs/github/issues.go @@ -15,6 +15,13 @@ type ListIssues struct { PageOptions } +type NewIssue struct { + Title string `json:"title,omitempty"` + Body string `json:"body,omitempty"` + Assignees []string `json:"assignees,omitempty"` + Labels []string `json:"labels,omitempty"` +} + type Issue struct { ID int64 `json:"id,omitempty"` Number int `json:"number,omitempty"` diff --git a/go-libs/sqlexec/sqlexec_test.go b/go-libs/sqlexec/sqlexec_test.go index 6728e0a..8cceefc 100644 --- a/go-libs/sqlexec/sqlexec_test.go +++ b/go-libs/sqlexec/sqlexec_test.go @@ -82,7 +82,7 @@ func TestAccQueryTypes(t *testing.T) { require.NoError(t, result.Err()) require.Equal(t, []int{1, 2}, arrayType) - require.Equal(t, []byte("a"), binaryType) + require.Equal(t, []byte("a..."), binaryType) require.Equal(t, true, booleanType) require.Equal(t, time.Now().UTC().Truncate(24*time.Hour), dateType) require.InDelta(t, 3, decimalType, 0.01) diff --git a/go.work.sum b/go.work.sum index c747dcf..2d13f36 100644 --- a/go.work.sum +++ b/go.work.sum @@ -445,6 +445,7 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -615,6 +616,7 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014/go. google.golang.org/genproto/googleapis/bytestream v0.0.0-20231127180814-3a041ad873d4/go.mod h1:o8b+u5ZiOSKuCwaZNjqXDJtJ0CmB9NtUPgCfO4rbakw= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:ZSvZ8l+AWJwXw91DoTjWjaVLpWU6o0eZ4YLYpH8aLeQ= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:om8Bj876Z0v9ei+RD1LnEWig7vpHQ371PUqsgjmLQEA= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20240304161311-37d4d3c04a78/go.mod h1:vh/N7795ftP0AkN1w8XKqN4w1OdUKXW5Eummda+ofv8= google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc=