From f9b76d7732c86d6cb0e16f519342df402c9b87a9 Mon Sep 17 00:00:00 2001 From: mlosicki Date: Sun, 23 Jan 2022 12:57:15 +0000 Subject: [PATCH 01/14] feat: bitbucketServer for pull request generator Discover pull requests from repositories in a Bitbucket Server (not the same as Bitbucket Cloud). Private repos can be accessed via Basic auth (password or personal access token). Signed-off-by: mlosicki --- api/v1alpha1/applicationset_types.go | 25 +- api/v1alpha1/zz_generated.deepcopy.go | 50 +++ docs/Generators-Pull-Request.md | 37 +++ go.mod | 1 + go.sum | 3 + .../crds/argoproj.io_applicationsets.yaml | 99 ++++++ manifests/install.yaml | 99 ++++++ pkg/generators/pull_request.go | 12 + pkg/services/pull_request/bitbucket_server.go | 97 ++++++ .../pull_request/bitbucket_server_test.go | 304 ++++++++++++++++++ 10 files changed, 726 insertions(+), 1 deletion(-) create mode 100644 pkg/services/pull_request/bitbucket_server.go create mode 100644 pkg/services/pull_request/bitbucket_server_test.go diff --git a/api/v1alpha1/applicationset_types.go b/api/v1alpha1/applicationset_types.go index b6dc6bc1..3ce69dbb 100644 --- a/api/v1alpha1/applicationset_types.go +++ b/api/v1alpha1/applicationset_types.go @@ -347,7 +347,8 @@ type SCMProviderGeneratorFilter struct { // PullRequestGenerator defines a generator that scrapes a PullRequest API to find candidate pull requests. type PullRequestGenerator struct { // Which provider to use and config for it. - Github *PullRequestGeneratorGithub `json:"github,omitempty"` + Github *PullRequestGeneratorGithub `json:"github,omitempty"` + BitbucketServer *PullRequestGeneratorBitbucketServer `json:"bitbucketServer,omitempty"` // Standard parameters. RequeueAfterSeconds *int64 `json:"requeueAfterSeconds,omitempty"` Template ApplicationSetTemplate `json:"template,omitempty"` @@ -367,6 +368,28 @@ type PullRequestGeneratorGithub struct { Labels []string `json:"labels,omitempty"` } +// PullRequestGenerator defines a connection info specific to BitbucketServer. +type PullRequestGeneratorBitbucketServer struct { + // Project to scan. Required. + Project string `json:"project"` + // Repo name to scan. Required. + Repo string `json:"repo"` + // The Bitbucket REST API URL to talk to e.g. https://bitbucket.org/rest Required. + API string `json:"api"` + // A regex which must match the branch name. + BranchMatch *string `json:"branchMatch,omitempty"` + // Credentials for Basic auth + BasicAuth *BasicAuthBitbucketServer `json:"basicAuth,omitempty"` +} + +// BasicAuthBitbucketServer defines the username/(password or personal access token) for Basic auth. +type BasicAuthBitbucketServer struct { + // Username for Basic auth + Username string `json:"username"` + // Password (or personal access token) reference. + PasswordRef *SecretRef `json:"passwordRef"` +} + // ApplicationSetStatus defines the observed state of ApplicationSet type ApplicationSetStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 4248a934..28664db8 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -415,6 +415,26 @@ func (in ApplicationSetTerminalGenerators) DeepCopy() ApplicationSetTerminalGene return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BasicAuthBitbucketServer) DeepCopyInto(out *BasicAuthBitbucketServer) { + *out = *in + if in.PasswordRef != nil { + in, out := &in.PasswordRef, &out.PasswordRef + *out = new(SecretRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BasicAuthBitbucketServer. +func (in *BasicAuthBitbucketServer) DeepCopy() *BasicAuthBitbucketServer { + if in == nil { + return nil + } + out := new(BasicAuthBitbucketServer) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterGenerator) DeepCopyInto(out *ClusterGenerator) { *out = *in @@ -660,6 +680,11 @@ func (in *PullRequestGenerator) DeepCopyInto(out *PullRequestGenerator) { *out = new(PullRequestGeneratorGithub) (*in).DeepCopyInto(*out) } + if in.BitbucketServer != nil { + in, out := &in.BitbucketServer, &out.BitbucketServer + *out = new(PullRequestGeneratorBitbucketServer) + (*in).DeepCopyInto(*out) + } if in.RequeueAfterSeconds != nil { in, out := &in.RequeueAfterSeconds, &out.RequeueAfterSeconds *out = new(int64) @@ -678,6 +703,31 @@ func (in *PullRequestGenerator) DeepCopy() *PullRequestGenerator { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PullRequestGeneratorBitbucketServer) DeepCopyInto(out *PullRequestGeneratorBitbucketServer) { + *out = *in + if in.BranchMatch != nil { + in, out := &in.BranchMatch, &out.BranchMatch + *out = new(string) + **out = **in + } + if in.BasicAuth != nil { + in, out := &in.BasicAuth, &out.BasicAuth + *out = new(BasicAuthBitbucketServer) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PullRequestGeneratorBitbucketServer. +func (in *PullRequestGeneratorBitbucketServer) DeepCopy() *PullRequestGeneratorBitbucketServer { + if in == nil { + return nil + } + out := new(PullRequestGeneratorBitbucketServer) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PullRequestGeneratorGithub) DeepCopyInto(out *PullRequestGeneratorGithub) { *out = *in diff --git a/docs/Generators-Pull-Request.md b/docs/Generators-Pull-Request.md index 6cad00ff..1e7b551b 100644 --- a/docs/Generators-Pull-Request.md +++ b/docs/Generators-Pull-Request.md @@ -53,6 +53,43 @@ spec: * `tokenRef`: A `Secret` name and key containing the GitHub access token to use for requests. If not specified, will make anonymous requests which have a lower rate limit and can only see public repositories. (Optional) * `labels`: Labels is used to filter the PRs that you want to target. (Optional) +## Bitbucket Server + +Fetch pull requests from a repo hosted on a Bitbucket Server (not to same as Bitbucket Cloud). + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: myapps +spec: + generators: + - pullRequest: + bitbucketServer: + project: myproject + repo: myrepository + # URL of the Bitbucket Server. Required. + api: https://mycompany.bitbucket.org + # Credentials for Basic authentication. Required for private repositories. + basicAuth: + # The username to authenticate with + username: myuser + # Reference to a Secret containing the password or personal access token. + passwordRef: + secretName: mypassword + key: password + # Labels are not supported by Bitbucket Server + template: + # ... +``` + +* `project`: Required name of the Bitbucket project +* `repo`: Required name of the Bitbucket repository. +* `api`: Required URL to access the Bitbucket REST API. For the example above, an API request would be made to `https://mycompany.bitbucket.org/rest/api/1.0/projects/myproject/repos/myrepository/pull-requests` +If you want to access a private repository, you must also provide the credentials for Basic auth (this is the only auth supported currently): +* `username`: The username to authenticate with. It only needs read access to the relevant repo. +* `passwordRef`: A `Secret` name and key containing the password or personal access token to use for requests. + ## Template As with all generators, several keys are available for replacement in the generated application. diff --git a/go.mod b/go.mod index 81f7868a..b0102dc3 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/argoproj/argo-cd/v2 v2.3.0-rc5.0.20220206192056-4b04a3918029 github.com/argoproj/gitops-engine v0.5.1-0.20220126184517-b0c5e00ccfa5 github.com/argoproj/pkg v0.11.1-0.20211203175135-36c59d8fafe0 + github.com/gfleury/go-bitbucket-v1 v0.0.0-20210826163055-dff2223adeac github.com/go-logr/logr v1.2.2 github.com/google/go-github/v35 v35.0.0 github.com/imdario/mergo v0.3.12 diff --git a/go.sum b/go.sum index 9474b218..77098644 100644 --- a/go.sum +++ b/go.sum @@ -281,6 +281,8 @@ github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/gfleury/go-bitbucket-v1 v0.0.0-20210826163055-dff2223adeac h1:0hlPRK2IeBbM3zab+3+vLh+vgepD+gsx9Ra9CIUeX84= +github.com/gfleury/go-bitbucket-v1 v0.0.0-20210826163055-dff2223adeac/go.mod h1:LB3osS9X2JMYmTzcCArHHLrndBAfcVLQAvUddfs+ONs= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= @@ -671,6 +673,7 @@ github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUb github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= diff --git a/manifests/crds/argoproj.io_applicationsets.yaml b/manifests/crds/argoproj.io_applicationsets.yaml index 7eda3253..008a0949 100644 --- a/manifests/crds/argoproj.io_applicationsets.yaml +++ b/manifests/crds/argoproj.io_applicationsets.yaml @@ -2459,6 +2459,39 @@ spec: x-kubernetes-preserve-unknown-fields: true pullRequest: properties: + bitbucketServer: + properties: + api: + type: string + basicAuth: + properties: + passwordRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + username: + type: string + required: + - passwordRef + - username + type: object + branchMatch: + type: string + project: + type: string + repo: + type: string + required: + - api + - project + - repo + type: object github: properties: api: @@ -4601,6 +4634,39 @@ spec: x-kubernetes-preserve-unknown-fields: true pullRequest: properties: + bitbucketServer: + properties: + api: + type: string + basicAuth: + properties: + passwordRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + username: + type: string + required: + - passwordRef + - username + type: object + branchMatch: + type: string + project: + type: string + repo: + type: string + required: + - api + - project + - repo + type: object github: properties: api: @@ -5532,6 +5598,39 @@ spec: type: object pullRequest: properties: + bitbucketServer: + properties: + api: + type: string + basicAuth: + properties: + passwordRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + username: + type: string + required: + - passwordRef + - username + type: object + branchMatch: + type: string + project: + type: string + repo: + type: string + required: + - api + - project + - repo + type: object github: properties: api: diff --git a/manifests/install.yaml b/manifests/install.yaml index fec1e96d..796916f4 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -2458,6 +2458,39 @@ spec: x-kubernetes-preserve-unknown-fields: true pullRequest: properties: + bitbucketServer: + properties: + api: + type: string + basicAuth: + properties: + passwordRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + username: + type: string + required: + - passwordRef + - username + type: object + branchMatch: + type: string + project: + type: string + repo: + type: string + required: + - api + - project + - repo + type: object github: properties: api: @@ -4600,6 +4633,39 @@ spec: x-kubernetes-preserve-unknown-fields: true pullRequest: properties: + bitbucketServer: + properties: + api: + type: string + basicAuth: + properties: + passwordRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + username: + type: string + required: + - passwordRef + - username + type: object + branchMatch: + type: string + project: + type: string + repo: + type: string + required: + - api + - project + - repo + type: object github: properties: api: @@ -5531,6 +5597,39 @@ spec: type: object pullRequest: properties: + bitbucketServer: + properties: + api: + type: string + basicAuth: + properties: + passwordRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + username: + type: string + required: + - passwordRef + - username + type: object + branchMatch: + type: string + project: + type: string + repo: + type: string + required: + - api + - project + - repo + type: object github: properties: api: diff --git a/pkg/generators/pull_request.go b/pkg/generators/pull_request.go index 2461cccb..f7e60bbc 100644 --- a/pkg/generators/pull_request.go +++ b/pkg/generators/pull_request.go @@ -86,6 +86,18 @@ func (g *PullRequestGenerator) selectServiceProvider(ctx context.Context, genera } return pullrequest.NewGithubService(ctx, token, providerConfig.API, providerConfig.Owner, providerConfig.Repo, providerConfig.Labels) } + if generatorConfig.BitbucketServer != nil { + providerConfig := generatorConfig.BitbucketServer + if providerConfig.BasicAuth != nil { + password, err := g.getSecretRef(ctx, providerConfig.BasicAuth.PasswordRef, applicationSetInfo.Namespace) + if err != nil { + return nil, fmt.Errorf("error fetching Secret token: %v", err) + } + return pullrequest.NewBitbucketServiceBasicAuth(ctx, providerConfig.BasicAuth.Username, password, providerConfig.API, providerConfig.Project, providerConfig.Repo, providerConfig.BranchMatch) + } else { + return pullrequest.NewBitbucketServiceNoAuth(ctx, providerConfig.API, providerConfig.Project, providerConfig.Repo, providerConfig.BranchMatch) + } + } return nil, fmt.Errorf("no Pull Request provider implementation configured") } diff --git a/pkg/services/pull_request/bitbucket_server.go b/pkg/services/pull_request/bitbucket_server.go new file mode 100644 index 00000000..40c1bce7 --- /dev/null +++ b/pkg/services/pull_request/bitbucket_server.go @@ -0,0 +1,97 @@ +package pull_request + +import ( + "context" + "fmt" + "regexp" + "strings" + + bitbucketv1 "github.com/gfleury/go-bitbucket-v1" +) + +type BitbucketService struct { + client *bitbucketv1.APIClient + projectKey string + repositorySlug string + branchMatch *regexp.Regexp + // Not supported for PRs by Bitbucket Server + // labels []string +} + +var _ PullRequestService = (*BitbucketService)(nil) + +func NewBitbucketServiceBasicAuth(ctx context.Context, username, password, url, projectKey, repositorySlug string, branchMatch *string) (PullRequestService, error) { + bitbucketConfig := bitbucketv1.NewConfiguration(url) + // Avoid the XSRF check + bitbucketConfig.AddDefaultHeader("x-atlassian-token", "no-check") + bitbucketConfig.AddDefaultHeader("x-requested-with", "XMLHttpRequest") + + ctx = context.WithValue(ctx, bitbucketv1.ContextBasicAuth, bitbucketv1.BasicAuth{ + UserName: username, + Password: password, + }) + return newBitbucketService(ctx, bitbucketConfig, projectKey, repositorySlug, branchMatch) +} + +func NewBitbucketServiceNoAuth(ctx context.Context, url, projectKey, repositorySlug string, branchMatch *string) (PullRequestService, error) { + return newBitbucketService(ctx, bitbucketv1.NewConfiguration(url), projectKey, repositorySlug, branchMatch) +} + +func newBitbucketService(ctx context.Context, bitbucketConfig *bitbucketv1.Configuration, projectKey, repositorySlug string, branchMatch *string) (PullRequestService, error) { + if !strings.HasSuffix(bitbucketConfig.BasePath, "/rest") { + bitbucketConfig.BasePath = bitbucketConfig.BasePath + "/rest" + } + bitbucketClient := bitbucketv1.NewAPIClient(ctx, bitbucketConfig) + + var branchMatchRegexp *regexp.Regexp + if branchMatch != nil { + var err error + branchMatchRegexp, err = regexp.Compile(*branchMatch) + if err != nil { + return nil, fmt.Errorf("error compiling BranchMatch regexp %q: %v", *branchMatch, err) + } + } + + return &BitbucketService{ + client: bitbucketClient, + projectKey: projectKey, + repositorySlug: repositorySlug, + branchMatch: branchMatchRegexp, + }, nil +} + +func (b *BitbucketService) List(_ context.Context) ([]*PullRequest, error) { + paged := map[string]interface{}{ + "limit": 100, + } + + pullRequests := []*PullRequest{} + for { + response, err := b.client.DefaultApi.GetPullRequestsPage(b.projectKey, b.repositorySlug, paged) + if err != nil { + return nil, fmt.Errorf("error listing pull requests for %s/%s: %v", b.projectKey, b.repositorySlug, err) + } + pulls, err := bitbucketv1.GetPullRequestsResponse(response) + if err != nil { + return nil, fmt.Errorf("error parsing pull request response %s: %v", response.Values, err) + } + + for _, pull := range pulls { + if b.branchMatch != nil && !b.branchMatch.MatchString(pull.FromRef.DisplayID) { + continue + } + pullRequests = append(pullRequests, &PullRequest{ + Number: pull.ID, + Branch: pull.FromRef.DisplayID, // ID: refs/heads/main DisplayID: main + HeadSHA: pull.FromRef.LatestCommit, // This is not defined in the official docs, but works in practice + }) + } + + hasNextPage, nextPageStart := bitbucketv1.HasNextPage(response) + if !hasNextPage { + break + } + paged["start"] = nextPageStart + } + return pullRequests, nil +} diff --git a/pkg/services/pull_request/bitbucket_server_test.go b/pkg/services/pull_request/bitbucket_server_test.go new file mode 100644 index 00000000..3da13e10 --- /dev/null +++ b/pkg/services/pull_request/bitbucket_server_test.go @@ -0,0 +1,304 @@ +package pull_request + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func defaultHandler(t *testing.T) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + var err error + switch r.RequestURI { + case "/rest/api/1.0/projects/PROJECT/repos/REPO/pull-requests?limit=100": + _, err = io.WriteString(w, `{ + "size": 1, + "limit": 100, + "isLastPage": true, + "values": [ + { + "id": 101, + "fromRef": { + "id": "refs/heads/feature-ABC-123", + "displayId": "feature-ABC-123", + "latestCommit": "cb3cf2e4d1517c83e720d2585b9402dbef71f992" + } + } + ], + "start": 0 + }`) + default: + t.Fail() + } + if err != nil { + t.Fail() + } + } +} + +func TestListPullRequestNoAuth(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.Header.Get("Authorization")) + defaultHandler(t)(w, r) + })) + defer ts.Close() + svc, err := NewBitbucketServiceNoAuth(context.TODO(), ts.URL, "PROJECT", "REPO", nil) + assert.Nil(t, err) + pullRequests, err := svc.List(context.TODO()) + assert.Nil(t, err) + assert.Equal(t, 1, len(pullRequests)) + assert.Equal(t, 101, pullRequests[0].Number) + assert.Equal(t, "feature-ABC-123", pullRequests[0].Branch) + assert.Equal(t, "cb3cf2e4d1517c83e720d2585b9402dbef71f992", pullRequests[0].HeadSHA) +} + +func TestListPullRequestPagination(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + var err error + switch r.RequestURI { + case "/rest/api/1.0/projects/PROJECT/repos/REPO/pull-requests?limit=100": + _, err = io.WriteString(w, `{ + "size": 2, + "limit": 2, + "isLastPage": false, + "values": [ + { + "id": 101, + "fromRef": { + "id": "refs/heads/feature-101", + "displayId": "feature-101", + "latestCommit": "ab3cf2e4d1517c83e720d2585b9402dbef71f992" + } + }, + { + "id": 102, + "fromRef": { + "id": "refs/heads/feature-102", + "displayId": "feature-102", + "latestCommit": "bb3cf2e4d1517c83e720d2585b9402dbef71f992" + } + } + ], + "nextPageStart": 200 + }`) + case "/rest/api/1.0/projects/PROJECT/repos/REPO/pull-requests?limit=100&start=200": + _, err = io.WriteString(w, `{ + "size": 1, + "limit": 2, + "isLastPage": true, + "values": [ + { + "id": 200, + "fromRef": { + "id": "refs/heads/feature-200", + "displayId": "feature-200", + "latestCommit": "cb3cf2e4d1517c83e720d2585b9402dbef71f992" + } + } + ], + "start": 200 + }`) + default: + t.Fail() + } + if err != nil { + t.Fail() + } + })) + defer ts.Close() + svc, err := NewBitbucketServiceNoAuth(context.TODO(), ts.URL, "PROJECT", "REPO", nil) + assert.Nil(t, err) + pullRequests, err := svc.List(context.TODO()) + assert.Nil(t, err) + assert.Equal(t, 3, len(pullRequests)) + assert.Equal(t, PullRequest{ + Number: 101, + Branch: "feature-101", + HeadSHA: "ab3cf2e4d1517c83e720d2585b9402dbef71f992", + }, *pullRequests[0]) + assert.Equal(t, PullRequest{ + Number: 102, + Branch: "feature-102", + HeadSHA: "bb3cf2e4d1517c83e720d2585b9402dbef71f992", + }, *pullRequests[1]) + assert.Equal(t, PullRequest{ + Number: 200, + Branch: "feature-200", + HeadSHA: "cb3cf2e4d1517c83e720d2585b9402dbef71f992", + }, *pullRequests[2]) +} + +func TestListPullRequestBasicAuth(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // base64(user:password) + assert.Equal(t, "Basic dXNlcjpwYXNzd29yZA==", r.Header.Get("Authorization")) + assert.Equal(t, "no-check", r.Header.Get("X-Atlassian-Token")) + defaultHandler(t)(w, r) + })) + defer ts.Close() + svc, err := NewBitbucketServiceBasicAuth(context.TODO(), "user", "password", ts.URL, "PROJECT", "REPO", nil) + assert.Nil(t, err) + pullRequests, err := svc.List(context.TODO()) + assert.Nil(t, err) + assert.Equal(t, 1, len(pullRequests)) + assert.Equal(t, 101, pullRequests[0].Number) + assert.Equal(t, "feature-ABC-123", pullRequests[0].Branch) + assert.Equal(t, "cb3cf2e4d1517c83e720d2585b9402dbef71f992", pullRequests[0].HeadSHA) +} + +func TestListResponseError(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + })) + defer ts.Close() + svc, _ := NewBitbucketServiceNoAuth(context.TODO(), ts.URL, "PROJECT", "REPO", nil) + _, err := svc.List(context.TODO()) + assert.NotNil(t, err, err) +} + +func TestListResponseMalformed(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.RequestURI { + case "/rest/api/1.0/projects/PROJECT/repos/REPO/pull-requests?limit=100": + _, err := io.WriteString(w, `{ + "size": 1, + "limit": 100, + "isLastPage": true, + "values": { "id": 101 }, + "start": 0 + }`) + if err != nil { + t.Fail() + } + default: + t.Fail() + } + })) + defer ts.Close() + svc, _ := NewBitbucketServiceNoAuth(context.TODO(), ts.URL, "PROJECT", "REPO", nil) + _, err := svc.List(context.TODO()) + assert.NotNil(t, err, err) +} + +func TestListResponseEmpty(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.RequestURI { + case "/rest/api/1.0/projects/PROJECT/repos/REPO/pull-requests?limit=100": + _, err := io.WriteString(w, `{ + "size": 0, + "limit": 100, + "isLastPage": true, + "values": [], + "start": 0 + }`) + if err != nil { + t.Fail() + } + default: + t.Fail() + } + })) + defer ts.Close() + svc, err := NewBitbucketServiceNoAuth(context.TODO(), ts.URL, "PROJECT", "REPO", nil) + assert.Nil(t, err) + pullRequests, err := svc.List(context.TODO()) + assert.Nil(t, err) + assert.Empty(t, pullRequests) +} + +func TestListPullRequestBranchMatch(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + var err error + switch r.RequestURI { + case "/rest/api/1.0/projects/PROJECT/repos/REPO/pull-requests?limit=100": + _, err = io.WriteString(w, `{ + "size": 2, + "limit": 2, + "isLastPage": false, + "values": [ + { + "id": 101, + "fromRef": { + "id": "refs/heads/feature-101", + "displayId": "feature-101", + "latestCommit": "ab3cf2e4d1517c83e720d2585b9402dbef71f992" + } + }, + { + "id": 102, + "fromRef": { + "id": "refs/heads/feature-102", + "displayId": "feature-102", + "latestCommit": "bb3cf2e4d1517c83e720d2585b9402dbef71f992" + } + } + ], + "nextPageStart": 200 + }`) + case "/rest/api/1.0/projects/PROJECT/repos/REPO/pull-requests?limit=100&start=200": + _, err = io.WriteString(w, `{ + "size": 1, + "limit": 2, + "isLastPage": true, + "values": [ + { + "id": 200, + "fromRef": { + "id": "refs/heads/feature-200", + "displayId": "feature-200", + "latestCommit": "cb3cf2e4d1517c83e720d2585b9402dbef71f992" + } + } + ], + "start": 200 + }`) + default: + t.Fail() + } + if err != nil { + t.Fail() + } + })) + defer ts.Close() + regexp := `feature-1[\d]{2}` + svc, err := NewBitbucketServiceNoAuth(context.TODO(), ts.URL, "PROJECT", "REPO", ®exp) + assert.Nil(t, err) + pullRequests, err := svc.List(context.TODO()) + assert.Nil(t, err) + assert.Equal(t, 2, len(pullRequests)) + assert.Equal(t, PullRequest{ + Number: 101, + Branch: "feature-101", + HeadSHA: "ab3cf2e4d1517c83e720d2585b9402dbef71f992", + }, *pullRequests[0]) + assert.Equal(t, PullRequest{ + Number: 102, + Branch: "feature-102", + HeadSHA: "bb3cf2e4d1517c83e720d2585b9402dbef71f992", + }, *pullRequests[1]) + + regexp = `.*2$` + svc, err = NewBitbucketServiceNoAuth(context.TODO(), ts.URL, "PROJECT", "REPO", ®exp) + assert.Nil(t, err) + pullRequests, err = svc.List(context.TODO()) + assert.Nil(t, err) + assert.Equal(t, 1, len(pullRequests)) + assert.Equal(t, PullRequest{ + Number: 102, + Branch: "feature-102", + HeadSHA: "bb3cf2e4d1517c83e720d2585b9402dbef71f992", + }, *pullRequests[0]) + + regexp = `[\d{2}` + _, err = NewBitbucketServiceNoAuth(context.TODO(), ts.URL, "PROJECT", "REPO", ®exp) + assert.NotNil(t, err) +} From 7f25b015da37ba4c15614d36a08f2df2a920ef4d Mon Sep 17 00:00:00 2001 From: mlosicki Date: Mon, 24 Jan 2022 22:59:38 +0100 Subject: [PATCH 02/14] feat: bitbucketServer for scmProvider Signed-off-by: mlosicki --- api/v1alpha1/applicationset_types.go | 17 +- api/v1alpha1/zz_generated.deepcopy.go | 25 + docs/Generators-SCM-Provider.md | 40 ++ go.mod | 2 +- go.sum | 4 +- .../crds/argoproj.io_applicationsets.yaml | 90 ++++ manifests/install.yaml | 90 ++++ pkg/generators/scm_provider.go | 15 + pkg/services/scm_provider/bitbucket_server.go | 178 +++++++ .../scm_provider/bitbucket_server_test.go | 437 ++++++++++++++++++ 10 files changed, 893 insertions(+), 5 deletions(-) create mode 100644 pkg/services/scm_provider/bitbucket_server.go create mode 100644 pkg/services/scm_provider/bitbucket_server_test.go diff --git a/api/v1alpha1/applicationset_types.go b/api/v1alpha1/applicationset_types.go index 3ce69dbb..b5cd3921 100644 --- a/api/v1alpha1/applicationset_types.go +++ b/api/v1alpha1/applicationset_types.go @@ -292,8 +292,9 @@ type GitFileGeneratorItem struct { // SCMProviderGenerator defines a generator that scrapes a SCMaaS API to find candidate repos. type SCMProviderGenerator struct { // Which provider to use and config for it. - Github *SCMProviderGeneratorGithub `json:"github,omitempty"` - Gitlab *SCMProviderGeneratorGitlab `json:"gitlab,omitempty"` + Github *SCMProviderGeneratorGithub `json:"github,omitempty"` + Gitlab *SCMProviderGeneratorGitlab `json:"gitlab,omitempty"` + BitbucketServer *SCMProviderGeneratorBitbucketServer `json:"bitbucketServer,omitempty"` // Filters for which repos should be considered. Filters []SCMProviderGeneratorFilter `json:"filters,omitempty"` // Which protocol to use for the SCM URL. Default is provider-specific but ssh if possible. Not all providers @@ -330,6 +331,18 @@ type SCMProviderGeneratorGitlab struct { AllBranches bool `json:"allBranches,omitempty"` } +// SCMProviderGeneratorBitbucketServer defines a connection info specific to Bitbucket Server. +type SCMProviderGeneratorBitbucketServer struct { + // Project to scan. Required. + Project string `json:"project"` + // The Bitbucket Server REST API URL to talk to. + API string `json:"api"` + // Credentials for Basic auth + BasicAuth *BasicAuthBitbucketServer `json:"basicAuth,omitempty"` + // Scan all branches instead of just the default branch. + AllBranches bool `json:"allBranches,omitempty"` +} + // SCMProviderGeneratorFilter is a single repository filter. // If multiple filter types are set on a single struct, they will be AND'd together. All filters must // pass for a repo to be included. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 28664db8..ec1d6389 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -766,6 +766,11 @@ func (in *SCMProviderGenerator) DeepCopyInto(out *SCMProviderGenerator) { *out = new(SCMProviderGeneratorGitlab) (*in).DeepCopyInto(*out) } + if in.BitbucketServer != nil { + in, out := &in.BitbucketServer, &out.BitbucketServer + *out = new(SCMProviderGeneratorBitbucketServer) + (*in).DeepCopyInto(*out) + } if in.Filters != nil { in, out := &in.Filters, &out.Filters *out = make([]SCMProviderGeneratorFilter, len(*in)) @@ -791,6 +796,26 @@ func (in *SCMProviderGenerator) DeepCopy() *SCMProviderGenerator { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SCMProviderGeneratorBitbucketServer) DeepCopyInto(out *SCMProviderGeneratorBitbucketServer) { + *out = *in + if in.BasicAuth != nil { + in, out := &in.BasicAuth, &out.BasicAuth + *out = new(BasicAuthBitbucketServer) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SCMProviderGeneratorBitbucketServer. +func (in *SCMProviderGeneratorBitbucketServer) DeepCopy() *SCMProviderGeneratorBitbucketServer { + if in == nil { + return nil + } + out := new(SCMProviderGeneratorBitbucketServer) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SCMProviderGeneratorFilter) DeepCopyInto(out *SCMProviderGeneratorFilter) { *out = *in diff --git a/docs/Generators-SCM-Provider.md b/docs/Generators-SCM-Provider.md index 4107e4ac..a2465377 100644 --- a/docs/Generators-SCM-Provider.md +++ b/docs/Generators-SCM-Provider.md @@ -94,6 +94,46 @@ For label filtering, the repository tags are used. Available clone protocols are `ssh` and `https`. +## Bitbucket Server + +Use the Bitbucket Server API (1.0) to scan repos in a project. Note that Bitbucket Server is not to same as Bitbucket Cloud (API 2.0) + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: myapps +spec: + generators: + - scmProvider: + bitbucketServer: + project: myproject + # URL of the Bitbucket Server. Required. + api: https://mycompany.bitbucket.org + # If true, scan every branch of every repository. If false, scan only the default branch. Defaults to false. + allBranches: true + # Credentials for Basic authentication. Required for private repositories. + basicAuth: + # The username to authenticate with + username: myuser + # Reference to a Secret containing the password or personal access token. + passwordRef: + secretName: mypassword + key: password + # Support for labels is TODO. Bitbucket server labels are not supported for PRs, but they are for repos + template: + # ... +``` + +* `project`: Required name of the Bitbucket project +* `api`: Required URL to access the Bitbucket REST api. +* `allBranches`: By default (false) the template will only be evaluated for the default branch of each repo. If this is true, every branch of every repository will be passed to the filters. If using this flag, you likely want to use a `branchMatch` filter. +If you want to access a private repository, you must also provide the credentials for Basic auth (this is the only auth supported currently): +* `username`: The username to authenticate with. It only needs read access to the relevant repo. +* `passwordRef`: A `Secret` name and key containing the password or personal access token to use for requests. + +Available clone protocols are `ssh` and `https`. + ## Filters Filters allow selecting which repositories to generate for. Each filter can declare one or more conditions, all of which must pass. If multiple filters are present, any can match for a repository to be included. If no filters are specified, all repositories will be processed. diff --git a/go.mod b/go.mod index b0102dc3..19b34980 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/argoproj/argo-cd/v2 v2.3.0-rc5.0.20220206192056-4b04a3918029 github.com/argoproj/gitops-engine v0.5.1-0.20220126184517-b0c5e00ccfa5 github.com/argoproj/pkg v0.11.1-0.20211203175135-36c59d8fafe0 - github.com/gfleury/go-bitbucket-v1 v0.0.0-20210826163055-dff2223adeac + github.com/gfleury/go-bitbucket-v1 v0.0.0-20220125132502-90a950f9bcba github.com/go-logr/logr v1.2.2 github.com/google/go-github/v35 v35.0.0 github.com/imdario/mergo v0.3.12 diff --git a/go.sum b/go.sum index 77098644..7b560a97 100644 --- a/go.sum +++ b/go.sum @@ -281,8 +281,8 @@ github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/gfleury/go-bitbucket-v1 v0.0.0-20210826163055-dff2223adeac h1:0hlPRK2IeBbM3zab+3+vLh+vgepD+gsx9Ra9CIUeX84= -github.com/gfleury/go-bitbucket-v1 v0.0.0-20210826163055-dff2223adeac/go.mod h1:LB3osS9X2JMYmTzcCArHHLrndBAfcVLQAvUddfs+ONs= +github.com/gfleury/go-bitbucket-v1 v0.0.0-20220125132502-90a950f9bcba h1:JstvmY1XmxDNHsCqxzjNLbB+mUJDT/wQIinmCmnU0yM= +github.com/gfleury/go-bitbucket-v1 v0.0.0-20220125132502-90a950f9bcba/go.mod h1:LB3osS9X2JMYmTzcCArHHLrndBAfcVLQAvUddfs+ONs= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= diff --git a/manifests/crds/argoproj.io_applicationsets.yaml b/manifests/crds/argoproj.io_applicationsets.yaml index 008a0949..c2d16a5e 100644 --- a/manifests/crds/argoproj.io_applicationsets.yaml +++ b/manifests/crds/argoproj.io_applicationsets.yaml @@ -2798,6 +2798,36 @@ spec: type: object scmProvider: properties: + bitbucketServer: + properties: + allBranches: + type: boolean + api: + type: string + basicAuth: + properties: + passwordRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + username: + type: string + required: + - passwordRef + - username + type: object + project: + type: string + required: + - api + - project + type: object cloneProtocol: type: string filters: @@ -4973,6 +5003,36 @@ spec: type: object scmProvider: properties: + bitbucketServer: + properties: + allBranches: + type: boolean + api: + type: string + basicAuth: + properties: + passwordRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + username: + type: string + required: + - passwordRef + - username + type: object + project: + type: string + required: + - api + - project + type: object cloneProtocol: type: string filters: @@ -5937,6 +5997,36 @@ spec: type: object scmProvider: properties: + bitbucketServer: + properties: + allBranches: + type: boolean + api: + type: string + basicAuth: + properties: + passwordRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + username: + type: string + required: + - passwordRef + - username + type: object + project: + type: string + required: + - api + - project + type: object cloneProtocol: type: string filters: diff --git a/manifests/install.yaml b/manifests/install.yaml index 796916f4..28b671b5 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -2797,6 +2797,36 @@ spec: type: object scmProvider: properties: + bitbucketServer: + properties: + allBranches: + type: boolean + api: + type: string + basicAuth: + properties: + passwordRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + username: + type: string + required: + - passwordRef + - username + type: object + project: + type: string + required: + - api + - project + type: object cloneProtocol: type: string filters: @@ -4972,6 +5002,36 @@ spec: type: object scmProvider: properties: + bitbucketServer: + properties: + allBranches: + type: boolean + api: + type: string + basicAuth: + properties: + passwordRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + username: + type: string + required: + - passwordRef + - username + type: object + project: + type: string + required: + - api + - project + type: object cloneProtocol: type: string filters: @@ -5936,6 +5996,36 @@ spec: type: object scmProvider: properties: + bitbucketServer: + properties: + allBranches: + type: boolean + api: + type: string + basicAuth: + properties: + passwordRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + username: + type: string + required: + - passwordRef + - username + type: object + project: + type: string + required: + - api + - project + type: object cloneProtocol: type: string filters: diff --git a/pkg/generators/scm_provider.go b/pkg/generators/scm_provider.go index b4c145b9..c69445c6 100644 --- a/pkg/generators/scm_provider.go +++ b/pkg/generators/scm_provider.go @@ -77,6 +77,21 @@ func (g *SCMProviderGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha if err != nil { return nil, fmt.Errorf("error initializing Gitlab service: %v", err) } + } else if providerConfig.BitbucketServer != nil { + providerConfig := providerConfig.BitbucketServer + var scmError error + if providerConfig.BasicAuth != nil { + password, err := g.getSecretRef(ctx, providerConfig.BasicAuth.PasswordRef, applicationSetInfo.Namespace) + if err != nil { + return nil, fmt.Errorf("error fetching Secret token: %v", err) + } + provider, scmError = scm_provider.NewBitbucketServerProviderBasicAuth(ctx, providerConfig.BasicAuth.Username, password, providerConfig.API, providerConfig.Project, providerConfig.AllBranches) + } else { + provider, scmError = scm_provider.NewBitbucketServerProviderNoAuth(ctx, providerConfig.API, providerConfig.Project, providerConfig.AllBranches) + } + if scmError != nil { + return nil, fmt.Errorf("error initializing Bitbucket Server service: %v", scmError) + } } else { return nil, fmt.Errorf("no SCM provider implementation configured") } diff --git a/pkg/services/scm_provider/bitbucket_server.go b/pkg/services/scm_provider/bitbucket_server.go new file mode 100644 index 00000000..6d449eaa --- /dev/null +++ b/pkg/services/scm_provider/bitbucket_server.go @@ -0,0 +1,178 @@ +package scm_provider + +import ( + "context" + "fmt" + "strings" + + bitbucketv1 "github.com/gfleury/go-bitbucket-v1" +) + +type BitbucketServerProvider struct { + client *bitbucketv1.APIClient + projectKey string + allBranches bool +} + +var _ SCMProviderService = &BitbucketServerProvider{} + +func NewBitbucketServerProviderBasicAuth(ctx context.Context, username, password, url, projectKey string, allBranches bool) (*BitbucketServerProvider, error) { + bitbucketConfig := bitbucketv1.NewConfiguration(url) + // Avoid the XSRF check + bitbucketConfig.AddDefaultHeader("x-atlassian-token", "no-check") + bitbucketConfig.AddDefaultHeader("x-requested-with", "XMLHttpRequest") + + ctx = context.WithValue(ctx, bitbucketv1.ContextBasicAuth, bitbucketv1.BasicAuth{ + UserName: username, + Password: password, + }) + return newBitbucketServerProvider(ctx, bitbucketConfig, projectKey, allBranches) +} + +func NewBitbucketServerProviderNoAuth(ctx context.Context, url, projectKey string, allBranches bool) (*BitbucketServerProvider, error) { + return newBitbucketServerProvider(ctx, bitbucketv1.NewConfiguration(url), projectKey, allBranches) +} + +func newBitbucketServerProvider(ctx context.Context, bitbucketConfig *bitbucketv1.Configuration, projectKey string, allBranches bool) (*BitbucketServerProvider, error) { + if !strings.HasSuffix(bitbucketConfig.BasePath, "/rest") { + bitbucketConfig.BasePath = bitbucketConfig.BasePath + "/rest" + } + bitbucketClient := bitbucketv1.NewAPIClient(ctx, bitbucketConfig) + + return &BitbucketServerProvider{ + client: bitbucketClient, + projectKey: projectKey, + allBranches: allBranches, + }, nil +} + +func (b *BitbucketServerProvider) ListRepos(_ context.Context, cloneProtocol string) ([]*Repository, error) { + paged := map[string]interface{}{ + "limit": 100, + } + repos := []*Repository{} + for { + response, err := b.client.DefaultApi.GetRepositoriesWithOptions(b.projectKey, paged) + if err != nil { + return nil, fmt.Errorf("error listing repositories for %s: %v", b.projectKey, err) + } + repositories, err := bitbucketv1.GetRepositoriesResponse(response) + if err != nil { + return nil, fmt.Errorf("error parsing repositories response %s: %v", response.Values, err) + } + for _, bitbucketRepo := range repositories { + var url string + switch cloneProtocol { + // Default to SSH if unspecified (i.e. if ""). + case "", "ssh": + url = getCloneURLFromLinks(bitbucketRepo.Links.Clone, "ssh") + case "https": + url = getCloneURLFromLinks(bitbucketRepo.Links.Clone, "http") + default: + return nil, fmt.Errorf("unknown clone protocol for Bitbucket Server %v", cloneProtocol) + } + + branches, err := b.listBranches(&bitbucketRepo) + if err != nil { + return nil, fmt.Errorf("error listing branches for %s/%s: %v", b.projectKey, bitbucketRepo.Name, err) + } + + for _, branch := range branches { + repos = append(repos, &Repository{ + Organization: bitbucketRepo.Project.Key, + Repository: bitbucketRepo.Name, + URL: url, + Branch: branch.DisplayID, + SHA: branch.LatestCommit, + Labels: []string{}, + }) + } + } + hasNextPage, nextPageStart := bitbucketv1.HasNextPage(response) + if !hasNextPage { + break + } + paged["start"] = nextPageStart + } + return repos, nil +} + +func (b *BitbucketServerProvider) RepoHasPath(_ context.Context, repo *Repository, path string) (bool, error) { + opts := map[string]interface{}{ + "limit": 100, + "at": repo.Branch, + } + // No need to query for all pages here + response, err := b.client.DefaultApi.StreamFiles_42(repo.Organization, repo.Repository, path, opts) + if response != nil && response.StatusCode == 404 { + // The path requested does not exist at the supplied commit. + return false, nil + } + if response != nil && response.StatusCode == 400 { + // If the path is a file, the first call will return 400: The path requested is not a directory at the supplied commit. + // Simply retry with an API call that works with files and expect a 200 return code + opts["type_"] = true // Only request the type, we don't need the content + response, err := b.client.DefaultApi.GetContent_0(repo.Organization, repo.Repository, path, opts) + if response != nil && response.StatusCode == 404 { + // File not found + return false, nil + } + if err != nil { + return false, err + } + // 200 ok + return true, nil + } + if err != nil { + return false, err + } + return true, nil +} + +func (b *BitbucketServerProvider) listBranches(repo *bitbucketv1.Repository) ([]bitbucketv1.Branch, error) { + // If we don't specifically want to query for all branches, just use the default branch and call it a day. + if !b.allBranches { + response, err := b.client.DefaultApi.GetDefaultBranch(repo.Project.Key, repo.Name) + if err != nil { + return nil, err + } + branch, err := bitbucketv1.GetBranchResponse(response) + if err != nil { + return nil, err + } + return []bitbucketv1.Branch{branch}, nil + } + // Otherwise, scrape the GetBranches API. + branches := []bitbucketv1.Branch{} + paged := map[string]interface{}{ + "limit": 100, + } + for { + response, err := b.client.DefaultApi.GetBranches(repo.Project.Key, repo.Name, paged) + if err != nil { + return nil, fmt.Errorf("error listing branches for %s/%s: %v", repo.Project.Key, repo.Name, err) + } + bitbucketBranches, err := bitbucketv1.GetBranchesResponse(response) + if err != nil { + return nil, fmt.Errorf("error parsing branches response %s: %v", response.Values, err) + } + + branches = append(branches, bitbucketBranches...) + + hasNextPage, nextPageStart := bitbucketv1.HasNextPage(response) + if !hasNextPage { + break + } + paged["start"] = nextPageStart + } + return branches, nil +} + +func getCloneURLFromLinks(links []bitbucketv1.CloneLink, name string) string { + for _, link := range links { + if link.Name == name { + return link.Href + } + } + return "" +} diff --git a/pkg/services/scm_provider/bitbucket_server_test.go b/pkg/services/scm_provider/bitbucket_server_test.go new file mode 100644 index 00000000..1a731d4a --- /dev/null +++ b/pkg/services/scm_provider/bitbucket_server_test.go @@ -0,0 +1,437 @@ +package scm_provider + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func defaultHandler(t *testing.T) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + var err error + switch r.RequestURI { + case "/rest/api/1.0/projects/PROJECT/repos?limit=100": + _, err = io.WriteString(w, `{ + "size": 1, + "limit": 100, + "isLastPage": true, + "values": [ + { + "name": "REPO", + "project": { + "key": "PROJECT" + }, + "links": { + "clone": [ + { + "href": "ssh://git@mycompany.bitbucket.org/PROJECT/REPO.git", + "name": "ssh" + }, + { + "href": "https://mycompany.bitbucket.org/scm/PROJECT/REPO.git", + "name": "http" + } + ] + } + } + ], + "start": 0 + }`) + case "/rest/api/1.0/projects/PROJECT/repos/REPO/branches?limit=100": + _, err = io.WriteString(w, `{ + "size": 1, + "limit": 100, + "isLastPage": true, + "values": [ + { + "id": "refs/heads/main", + "displayId": "main", + "type": "BRANCH", + "latestCommit": "8d51122def5632836d1cb1026e879069e10a1e13", + "latestChangeset": "8d51122def5632836d1cb1026e879069e10a1e13", + "isDefault": true + } + ], + "start": 0 + }`) + default: + t.Fail() + } + if err != nil { + t.Fail() + } + } +} + +func verifyDefaultRepo(t *testing.T, err error, repos []*Repository) { + assert.Nil(t, err) + assert.Equal(t, 1, len(repos)) + assert.Equal(t, Repository{ + Organization: "PROJECT", + Repository: "REPO", + URL: "ssh://git@mycompany.bitbucket.org/PROJECT/REPO.git", + Branch: "main", + SHA: "8d51122def5632836d1cb1026e879069e10a1e13", + Labels: []string{}, + }, *repos[0]) +} + +func TestListReposNoAuth(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.Header.Get("Authorization")) + defaultHandler(t)(w, r) + })) + defer ts.Close() + provider, err := NewBitbucketServerProviderNoAuth(context.TODO(), ts.URL, "PROJECT", true) + assert.Nil(t, err) + repos, err := provider.ListRepos(context.TODO(), "ssh") + verifyDefaultRepo(t, err, repos) +} + +func TestListReposPagination(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.Header.Get("Authorization")) + var err error + switch r.RequestURI { + case "/rest/api/1.0/projects/PROJECT/repos?limit=100": + _, err = io.WriteString(w, `{ + "size": 1, + "limit": 100, + "isLastPage": false, + "values": [ + { + "name": "REPO", + "project": { + "key": "PROJECT" + }, + "links": { + "clone": [ + { + "href": "ssh://git@mycompany.bitbucket.org/PROJECT/REPO.git", + "name": "ssh" + }, + { + "href": "https://mycompany.bitbucket.org/scm/PROJECT/REPO.git", + "name": "http" + } + ] + } + } + ], + "start": 0, + "nextPageStart": 200 + }`) + case "/rest/api/1.0/projects/PROJECT/repos?limit=100&start=200": + _, err = io.WriteString(w, `{ + "size": 1, + "limit": 100, + "isLastPage": true, + "values": [ + { + "name": "REPO2", + "project": { + "key": "PROJECT" + }, + "links": { + "clone": [ + { + "href": "ssh://git@mycompany.bitbucket.org/PROJECT/REPO2.git", + "name": "ssh" + }, + { + "href": "https://mycompany.bitbucket.org/scm/PROJECT/REPO2.git", + "name": "http" + } + ] + } + } + ], + "start": 200 + }`) + case "/rest/api/1.0/projects/PROJECT/repos/REPO/branches?limit=100": + _, err = io.WriteString(w, `{ + "size": 1, + "limit": 100, + "isLastPage": true, + "values": [ + { + "id": "refs/heads/main", + "displayId": "main", + "type": "BRANCH", + "latestCommit": "8d51122def5632836d1cb1026e879069e10a1e13", + "latestChangeset": "8d51122def5632836d1cb1026e879069e10a1e13", + "isDefault": true + } + ], + "start": 0 + }`) + case "/rest/api/1.0/projects/PROJECT/repos/REPO2/branches?limit=100": + _, err = io.WriteString(w, `{ + "size": 1, + "limit": 100, + "isLastPage": true, + "values": [ + { + "id": "refs/heads/development", + "displayId": "development", + "type": "BRANCH", + "latestCommit": "2d51122def5632836d1cb1026e879069e10a1e13", + "latestChangeset": "2d51122def5632836d1cb1026e879069e10a1e13", + "isDefault": true + } + ], + "start": 0 + }`) + default: + t.Fail() + } + if err != nil { + t.Fail() + } + })) + defer ts.Close() + provider, err := NewBitbucketServerProviderNoAuth(context.TODO(), ts.URL, "PROJECT", true) + assert.Nil(t, err) + repos, err := provider.ListRepos(context.TODO(), "ssh") + assert.Nil(t, err) + assert.Equal(t, 2, len(repos)) + assert.Equal(t, Repository{ + Organization: "PROJECT", + Repository: "REPO", + URL: "ssh://git@mycompany.bitbucket.org/PROJECT/REPO.git", + Branch: "main", + SHA: "8d51122def5632836d1cb1026e879069e10a1e13", + Labels: []string{}, + }, *repos[0]) + + assert.Equal(t, Repository{ + Organization: "PROJECT", + Repository: "REPO2", + URL: "ssh://git@mycompany.bitbucket.org/PROJECT/REPO2.git", + Branch: "development", + SHA: "2d51122def5632836d1cb1026e879069e10a1e13", + Labels: []string{}, + }, *repos[1]) +} + +func TestListReposBranchPagination(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.Header.Get("Authorization")) + switch r.RequestURI { + case "/rest/api/1.0/projects/PROJECT/repos/REPO/branches?limit=100": + _, err := io.WriteString(w, `{ + "size": 1, + "limit": 100, + "isLastPage": false, + "values": [ + { + "id": "refs/heads/main", + "displayId": "main", + "type": "BRANCH", + "latestCommit": "8d51122def5632836d1cb1026e879069e10a1e13", + "latestChangeset": "8d51122def5632836d1cb1026e879069e10a1e13", + "isDefault": true + } + ], + "start": 0, + "nextPageStart": 200 + }`) + if err != nil { + t.Fail() + } + return + case "/rest/api/1.0/projects/PROJECT/repos/REPO/branches?limit=100&start=200": + _, err := io.WriteString(w, `{ + "size": 1, + "limit": 100, + "isLastPage": true, + "values": [ + { + "id": "refs/heads/feature", + "displayId": "feature", + "type": "BRANCH", + "latestCommit": "9d51122def5632836d1cb1026e879069e10a1e13", + "latestChangeset": "9d51122def5632836d1cb1026e879069e10a1e13", + "isDefault": true + } + ], + "start": 200 + }`) + if err != nil { + t.Fail() + } + return + } + defaultHandler(t)(w, r) + })) + defer ts.Close() + provider, err := NewBitbucketServerProviderNoAuth(context.TODO(), ts.URL, "PROJECT", true) + assert.Nil(t, err) + repos, err := provider.ListRepos(context.TODO(), "ssh") + assert.Nil(t, err) + assert.Equal(t, 2, len(repos)) + assert.Equal(t, Repository{ + Organization: "PROJECT", + Repository: "REPO", + URL: "ssh://git@mycompany.bitbucket.org/PROJECT/REPO.git", + Branch: "main", + SHA: "8d51122def5632836d1cb1026e879069e10a1e13", + Labels: []string{}, + }, *repos[0]) + + assert.Equal(t, Repository{ + Organization: "PROJECT", + Repository: "REPO", + URL: "ssh://git@mycompany.bitbucket.org/PROJECT/REPO.git", + Branch: "feature", + SHA: "9d51122def5632836d1cb1026e879069e10a1e13", + Labels: []string{}, + }, *repos[1]) +} + +func TestListReposBasicAuth(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "Basic dXNlcjpwYXNzd29yZA==", r.Header.Get("Authorization")) + assert.Equal(t, "no-check", r.Header.Get("X-Atlassian-Token")) + defaultHandler(t)(w, r) + })) + defer ts.Close() + provider, err := NewBitbucketServerProviderBasicAuth(context.TODO(), "user", "password", ts.URL, "PROJECT", true) + assert.Nil(t, err) + repos, err := provider.ListRepos(context.TODO(), "ssh") + verifyDefaultRepo(t, err, repos) +} + +func TestListReposDefaultBranch(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.Header.Get("Authorization")) + switch r.RequestURI { + case "/rest/api/1.0/projects/PROJECT/repos/REPO/branches/default": + _, err := io.WriteString(w, `{ + "id": "refs/heads/default", + "displayId": "default", + "type": "BRANCH", + "latestCommit": "1d51122def5632836d1cb1026e879069e10a1e13", + "latestChangeset": "1d51122def5632836d1cb1026e879069e10a1e13", + "isDefault": true + }`) + if err != nil { + t.Fail() + } + return + } + defaultHandler(t)(w, r) + })) + defer ts.Close() + provider, err := NewBitbucketServerProviderNoAuth(context.TODO(), ts.URL, "PROJECT", false) + assert.Nil(t, err) + repos, err := provider.ListRepos(context.TODO(), "ssh") + assert.Nil(t, err) + assert.Equal(t, 1, len(repos)) + assert.Equal(t, Repository{ + Organization: "PROJECT", + Repository: "REPO", + URL: "ssh://git@mycompany.bitbucket.org/PROJECT/REPO.git", + Branch: "default", + SHA: "1d51122def5632836d1cb1026e879069e10a1e13", + Labels: []string{}, + }, *repos[0]) +} + +func TestListReposCloneProtocol(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.Header.Get("Authorization")) + defaultHandler(t)(w, r) + })) + defer ts.Close() + provider, err := NewBitbucketServerProviderNoAuth(context.TODO(), ts.URL, "PROJECT", true) + assert.Nil(t, err) + repos, err := provider.ListRepos(context.TODO(), "https") + assert.Nil(t, err) + assert.Equal(t, 1, len(repos)) + assert.Equal(t, Repository{ + Organization: "PROJECT", + Repository: "REPO", + URL: "https://mycompany.bitbucket.org/scm/PROJECT/REPO.git", + Branch: "main", + SHA: "8d51122def5632836d1cb1026e879069e10a1e13", + Labels: []string{}, + }, *repos[0]) +} + +func TestListReposUnknownProtocol(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.Header.Get("Authorization")) + defaultHandler(t)(w, r) + })) + defer ts.Close() + provider, err := NewBitbucketServerProviderNoAuth(context.TODO(), ts.URL, "PROJECT", true) + assert.Nil(t, err) + _, errProtocol := provider.ListRepos(context.TODO(), "http") + assert.NotNil(t, errProtocol) +} + +func TestBitbucketServerHasPath(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var err error + switch r.RequestURI { + case "/rest/api/1.0/projects/PROJECT/repos/REPO/files/pkg/?at=main&limit=100": + _, err = io.WriteString(w, `{ + "size": 1, + "limit": 100, + "isLastPage": true, + "values": [ + "pkg/file.txt" + ], + "start": 0 + }`) + + case "/rest/api/1.0/projects/PROJECT/repos/REPO/files/anotherpkg/file.txt?at=main&limit=100": + http.Error(w, "The path requested is not a directory at the supplied commit.", 400) + case "/rest/api/1.0/projects/PROJECT/repos/REPO/browse/anotherpkg/file.txt?at=main&limit=100&type=true": + _, err = io.WriteString(w, `{"type":"FILE"}`) + case "/rest/api/1.0/projects/PROJECT/repos/REPO/files/anotherpkg/missing.txt?at=main&limit=100": + http.Error(w, "The path requested is not a directory at the supplied commit.", 400) + case "/rest/api/1.0/projects/PROJECT/repos/REPO/browse/anotherpkg/missing.txt?at=main&limit=100&type=true": + http.Error(w, "The path \"anotherpkg/missing.txt\" does not exist at revision \"main\"", 404) + + case "/rest/api/1.0/projects/PROJECT/repos/REPO/files/notathing/?at=main&limit=100": + http.Error(w, "The path requested does not exist at the supplied commit.", 404) + + default: + t.Fail() + } + if err != nil { + t.Fail() + } + })) + defer ts.Close() + provider, err := NewBitbucketServerProviderNoAuth(context.TODO(), ts.URL, "PROJECT", true) + assert.Nil(t, err) + repo := &Repository{ + Organization: "PROJECT", + Repository: "REPO", + Branch: "main", + } + ok, err := provider.RepoHasPath(context.Background(), repo, "pkg/") + assert.Nil(t, err) + assert.True(t, ok) + + ok, err = provider.RepoHasPath(context.Background(), repo, "anotherpkg/file.txt") + assert.Nil(t, err) + assert.True(t, ok) + + ok, err = provider.RepoHasPath(context.Background(), repo, "anotherpkg/missing.txt") + assert.Nil(t, err) + assert.False(t, ok) + + ok, err = provider.RepoHasPath(context.Background(), repo, "notathing/") + assert.Nil(t, err) + assert.False(t, ok) + +} From 3479c439761883e2bc124706f676ae7f8e9156e1 Mon Sep 17 00:00:00 2001 From: mlosicki Date: Wed, 16 Feb 2022 07:51:20 +0000 Subject: [PATCH 03/14] fix: use 1.16 for deepcopy Signed-off-by: mlosicki --- api/v1alpha1/zz_generated.deepcopy.go | 1 - 1 file changed, 1 deletion(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ec1d6389..8a597eb5 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1,4 +1,3 @@ -//go:build !ignore_autogenerated // +build !ignore_autogenerated /* From b2f0ccda83d5bac3f6ea47b5a072a3b86baf3df6 Mon Sep 17 00:00:00 2001 From: mlosicki Date: Wed, 16 Feb 2022 09:01:57 +0100 Subject: [PATCH 04/14] docs: branchName for bitbucketServer PR + formatting Signed-off-by: mlosicki --- docs/Generators-Pull-Request.md | 4 ++++ docs/Generators-SCM-Provider.md | 1 + 2 files changed, 5 insertions(+) diff --git a/docs/Generators-Pull-Request.md b/docs/Generators-Pull-Request.md index 1e7b551b..09d0c9a8 100644 --- a/docs/Generators-Pull-Request.md +++ b/docs/Generators-Pull-Request.md @@ -70,6 +70,8 @@ spec: repo: myrepository # URL of the Bitbucket Server. Required. api: https://mycompany.bitbucket.org + # Filter PRs using the source branch name. (optional) + branchMatch: ".*-argocd" # Credentials for Basic authentication. Required for private repositories. basicAuth: # The username to authenticate with @@ -86,6 +88,8 @@ spec: * `project`: Required name of the Bitbucket project * `repo`: Required name of the Bitbucket repository. * `api`: Required URL to access the Bitbucket REST API. For the example above, an API request would be made to `https://mycompany.bitbucket.org/rest/api/1.0/projects/myproject/repos/myrepository/pull-requests` +* `branchMatch`: Optional regexp filter which should match the source branch name. This is an alternative to labels which are not supported by Bitbucket Server. + If you want to access a private repository, you must also provide the credentials for Basic auth (this is the only auth supported currently): * `username`: The username to authenticate with. It only needs read access to the relevant repo. * `passwordRef`: A `Secret` name and key containing the password or personal access token to use for requests. diff --git a/docs/Generators-SCM-Provider.md b/docs/Generators-SCM-Provider.md index a2465377..61fa4d1b 100644 --- a/docs/Generators-SCM-Provider.md +++ b/docs/Generators-SCM-Provider.md @@ -128,6 +128,7 @@ spec: * `project`: Required name of the Bitbucket project * `api`: Required URL to access the Bitbucket REST api. * `allBranches`: By default (false) the template will only be evaluated for the default branch of each repo. If this is true, every branch of every repository will be passed to the filters. If using this flag, you likely want to use a `branchMatch` filter. + If you want to access a private repository, you must also provide the credentials for Basic auth (this is the only auth supported currently): * `username`: The username to authenticate with. It only needs read access to the relevant repo. * `passwordRef`: A `Secret` name and key containing the password or personal access token to use for requests. From 51d3b6ff948e5432be377450a06b2bf800f92b93 Mon Sep 17 00:00:00 2001 From: mlosicki Date: Thu, 17 Feb 2022 16:45:56 +0000 Subject: [PATCH 05/14] docs: pr comments Signed-off-by: mlosicki --- api/v1alpha1/applicationset_types.go | 12 ++++++------ docs/Generators-Pull-Request.md | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/v1alpha1/applicationset_types.go b/api/v1alpha1/applicationset_types.go index b5cd3921..00185acc 100644 --- a/api/v1alpha1/applicationset_types.go +++ b/api/v1alpha1/applicationset_types.go @@ -305,7 +305,7 @@ type SCMProviderGenerator struct { Template ApplicationSetTemplate `json:"template,omitempty"` } -// SCMProviderGeneratorGithub defines a connection info specific to GitHub. +// SCMProviderGeneratorGithub defines connection info specific to GitHub. type SCMProviderGeneratorGithub struct { // GitHub org to scan. Required. Organization string `json:"organization"` @@ -317,7 +317,7 @@ type SCMProviderGeneratorGithub struct { AllBranches bool `json:"allBranches,omitempty"` } -// SCMProviderGeneratorGitlab defines a connection info specific to Gitlab. +// SCMProviderGeneratorGitlab defines connection info specific to Gitlab. type SCMProviderGeneratorGitlab struct { // Gitlab group to scan. Required. You can use either the project id (recommended) or the full namespaced path. Group string `json:"group"` @@ -331,11 +331,11 @@ type SCMProviderGeneratorGitlab struct { AllBranches bool `json:"allBranches,omitempty"` } -// SCMProviderGeneratorBitbucketServer defines a connection info specific to Bitbucket Server. +// SCMProviderGeneratorBitbucketServer defines connection info specific to Bitbucket Server. type SCMProviderGeneratorBitbucketServer struct { // Project to scan. Required. Project string `json:"project"` - // The Bitbucket Server REST API URL to talk to. + // The Bitbucket Server REST API URL to talk to. Required. API string `json:"api"` // Credentials for Basic auth BasicAuth *BasicAuthBitbucketServer `json:"basicAuth,omitempty"` @@ -367,7 +367,7 @@ type PullRequestGenerator struct { Template ApplicationSetTemplate `json:"template,omitempty"` } -// PullRequestGenerator defines a connection info specific to GitHub. +// PullRequestGenerator defines connection info specific to GitHub. type PullRequestGeneratorGithub struct { // GitHub org or user to scan. Required. Owner string `json:"owner"` @@ -381,7 +381,7 @@ type PullRequestGeneratorGithub struct { Labels []string `json:"labels,omitempty"` } -// PullRequestGenerator defines a connection info specific to BitbucketServer. +// PullRequestGenerator defines connection info specific to BitbucketServer. type PullRequestGeneratorBitbucketServer struct { // Project to scan. Required. Project string `json:"project"` diff --git a/docs/Generators-Pull-Request.md b/docs/Generators-Pull-Request.md index 09d0c9a8..4b0bd919 100644 --- a/docs/Generators-Pull-Request.md +++ b/docs/Generators-Pull-Request.md @@ -55,7 +55,7 @@ spec: ## Bitbucket Server -Fetch pull requests from a repo hosted on a Bitbucket Server (not to same as Bitbucket Cloud). +Fetch pull requests from a repo hosted on a Bitbucket Server (not the same as Bitbucket Cloud). ```yaml apiVersion: argoproj.io/v1alpha1 @@ -80,7 +80,7 @@ spec: passwordRef: secretName: mypassword key: password - # Labels are not supported by Bitbucket Server + # Labels are not supported by Bitbucket Server, so filtering by label is not possible. template: # ... ``` From f95bf8a73fb90b48ff8c5addb73154c67a26a962 Mon Sep 17 00:00:00 2001 From: mlosicki Date: Thu, 17 Feb 2022 17:33:19 +0000 Subject: [PATCH 06/14] fix: pr comments context + noError Signed-off-by: mlosicki --- .../pull_request/bitbucket_server_test.go | 58 ++++++++--------- .../scm_provider/bitbucket_server_test.go | 64 +++++++++---------- 2 files changed, 61 insertions(+), 61 deletions(-) diff --git a/pkg/services/pull_request/bitbucket_server_test.go b/pkg/services/pull_request/bitbucket_server_test.go index 3da13e10..21eab859 100644 --- a/pkg/services/pull_request/bitbucket_server_test.go +++ b/pkg/services/pull_request/bitbucket_server_test.go @@ -47,10 +47,10 @@ func TestListPullRequestNoAuth(t *testing.T) { defaultHandler(t)(w, r) })) defer ts.Close() - svc, err := NewBitbucketServiceNoAuth(context.TODO(), ts.URL, "PROJECT", "REPO", nil) - assert.Nil(t, err) - pullRequests, err := svc.List(context.TODO()) - assert.Nil(t, err) + svc, err := NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO", nil) + assert.NoError(t, err) + pullRequests, err := svc.List(context.Background()) + assert.NoError(t, err) assert.Equal(t, 1, len(pullRequests)) assert.Equal(t, 101, pullRequests[0].Number) assert.Equal(t, "feature-ABC-123", pullRequests[0].Branch) @@ -112,10 +112,10 @@ func TestListPullRequestPagination(t *testing.T) { } })) defer ts.Close() - svc, err := NewBitbucketServiceNoAuth(context.TODO(), ts.URL, "PROJECT", "REPO", nil) - assert.Nil(t, err) - pullRequests, err := svc.List(context.TODO()) - assert.Nil(t, err) + svc, err := NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO", nil) + assert.NoError(t, err) + pullRequests, err := svc.List(context.Background()) + assert.NoError(t, err) assert.Equal(t, 3, len(pullRequests)) assert.Equal(t, PullRequest{ Number: 101, @@ -142,10 +142,10 @@ func TestListPullRequestBasicAuth(t *testing.T) { defaultHandler(t)(w, r) })) defer ts.Close() - svc, err := NewBitbucketServiceBasicAuth(context.TODO(), "user", "password", ts.URL, "PROJECT", "REPO", nil) - assert.Nil(t, err) - pullRequests, err := svc.List(context.TODO()) - assert.Nil(t, err) + svc, err := NewBitbucketServiceBasicAuth(context.Background(), "user", "password", ts.URL, "PROJECT", "REPO", nil) + assert.NoError(t, err) + pullRequests, err := svc.List(context.Background()) + assert.NoError(t, err) assert.Equal(t, 1, len(pullRequests)) assert.Equal(t, 101, pullRequests[0].Number) assert.Equal(t, "feature-ABC-123", pullRequests[0].Branch) @@ -157,8 +157,8 @@ func TestListResponseError(t *testing.T) { w.WriteHeader(500) })) defer ts.Close() - svc, _ := NewBitbucketServiceNoAuth(context.TODO(), ts.URL, "PROJECT", "REPO", nil) - _, err := svc.List(context.TODO()) + svc, _ := NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO", nil) + _, err := svc.List(context.Background()) assert.NotNil(t, err, err) } @@ -182,8 +182,8 @@ func TestListResponseMalformed(t *testing.T) { } })) defer ts.Close() - svc, _ := NewBitbucketServiceNoAuth(context.TODO(), ts.URL, "PROJECT", "REPO", nil) - _, err := svc.List(context.TODO()) + svc, _ := NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO", nil) + _, err := svc.List(context.Background()) assert.NotNil(t, err, err) } @@ -207,10 +207,10 @@ func TestListResponseEmpty(t *testing.T) { } })) defer ts.Close() - svc, err := NewBitbucketServiceNoAuth(context.TODO(), ts.URL, "PROJECT", "REPO", nil) - assert.Nil(t, err) - pullRequests, err := svc.List(context.TODO()) - assert.Nil(t, err) + svc, err := NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO", nil) + assert.NoError(t, err) + pullRequests, err := svc.List(context.Background()) + assert.NoError(t, err) assert.Empty(t, pullRequests) } @@ -270,10 +270,10 @@ func TestListPullRequestBranchMatch(t *testing.T) { })) defer ts.Close() regexp := `feature-1[\d]{2}` - svc, err := NewBitbucketServiceNoAuth(context.TODO(), ts.URL, "PROJECT", "REPO", ®exp) - assert.Nil(t, err) - pullRequests, err := svc.List(context.TODO()) - assert.Nil(t, err) + svc, err := NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO", ®exp) + assert.NoError(t, err) + pullRequests, err := svc.List(context.Background()) + assert.NoError(t, err) assert.Equal(t, 2, len(pullRequests)) assert.Equal(t, PullRequest{ Number: 101, @@ -287,10 +287,10 @@ func TestListPullRequestBranchMatch(t *testing.T) { }, *pullRequests[1]) regexp = `.*2$` - svc, err = NewBitbucketServiceNoAuth(context.TODO(), ts.URL, "PROJECT", "REPO", ®exp) - assert.Nil(t, err) - pullRequests, err = svc.List(context.TODO()) - assert.Nil(t, err) + svc, err = NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO", ®exp) + assert.NoError(t, err) + pullRequests, err = svc.List(context.Background()) + assert.NoError(t, err) assert.Equal(t, 1, len(pullRequests)) assert.Equal(t, PullRequest{ Number: 102, @@ -299,6 +299,6 @@ func TestListPullRequestBranchMatch(t *testing.T) { }, *pullRequests[0]) regexp = `[\d{2}` - _, err = NewBitbucketServiceNoAuth(context.TODO(), ts.URL, "PROJECT", "REPO", ®exp) + _, err = NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO", ®exp) assert.NotNil(t, err) } diff --git a/pkg/services/scm_provider/bitbucket_server_test.go b/pkg/services/scm_provider/bitbucket_server_test.go index 1a731d4a..de73c6e7 100644 --- a/pkg/services/scm_provider/bitbucket_server_test.go +++ b/pkg/services/scm_provider/bitbucket_server_test.go @@ -69,7 +69,7 @@ func defaultHandler(t *testing.T) func(http.ResponseWriter, *http.Request) { } func verifyDefaultRepo(t *testing.T, err error, repos []*Repository) { - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, 1, len(repos)) assert.Equal(t, Repository{ Organization: "PROJECT", @@ -87,9 +87,9 @@ func TestListReposNoAuth(t *testing.T) { defaultHandler(t)(w, r) })) defer ts.Close() - provider, err := NewBitbucketServerProviderNoAuth(context.TODO(), ts.URL, "PROJECT", true) - assert.Nil(t, err) - repos, err := provider.ListRepos(context.TODO(), "ssh") + provider, err := NewBitbucketServerProviderNoAuth(context.Background(), ts.URL, "PROJECT", true) + assert.NoError(t, err) + repos, err := provider.ListRepos(context.Background(), "ssh") verifyDefaultRepo(t, err, repos) } @@ -195,10 +195,10 @@ func TestListReposPagination(t *testing.T) { } })) defer ts.Close() - provider, err := NewBitbucketServerProviderNoAuth(context.TODO(), ts.URL, "PROJECT", true) - assert.Nil(t, err) - repos, err := provider.ListRepos(context.TODO(), "ssh") - assert.Nil(t, err) + provider, err := NewBitbucketServerProviderNoAuth(context.Background(), ts.URL, "PROJECT", true) + assert.NoError(t, err) + repos, err := provider.ListRepos(context.Background(), "ssh") + assert.NoError(t, err) assert.Equal(t, 2, len(repos)) assert.Equal(t, Repository{ Organization: "PROJECT", @@ -270,10 +270,10 @@ func TestListReposBranchPagination(t *testing.T) { defaultHandler(t)(w, r) })) defer ts.Close() - provider, err := NewBitbucketServerProviderNoAuth(context.TODO(), ts.URL, "PROJECT", true) - assert.Nil(t, err) - repos, err := provider.ListRepos(context.TODO(), "ssh") - assert.Nil(t, err) + provider, err := NewBitbucketServerProviderNoAuth(context.Background(), ts.URL, "PROJECT", true) + assert.NoError(t, err) + repos, err := provider.ListRepos(context.Background(), "ssh") + assert.NoError(t, err) assert.Equal(t, 2, len(repos)) assert.Equal(t, Repository{ Organization: "PROJECT", @@ -301,9 +301,9 @@ func TestListReposBasicAuth(t *testing.T) { defaultHandler(t)(w, r) })) defer ts.Close() - provider, err := NewBitbucketServerProviderBasicAuth(context.TODO(), "user", "password", ts.URL, "PROJECT", true) - assert.Nil(t, err) - repos, err := provider.ListRepos(context.TODO(), "ssh") + provider, err := NewBitbucketServerProviderBasicAuth(context.Background(), "user", "password", ts.URL, "PROJECT", true) + assert.NoError(t, err) + repos, err := provider.ListRepos(context.Background(), "ssh") verifyDefaultRepo(t, err, repos) } @@ -328,10 +328,10 @@ func TestListReposDefaultBranch(t *testing.T) { defaultHandler(t)(w, r) })) defer ts.Close() - provider, err := NewBitbucketServerProviderNoAuth(context.TODO(), ts.URL, "PROJECT", false) - assert.Nil(t, err) - repos, err := provider.ListRepos(context.TODO(), "ssh") - assert.Nil(t, err) + provider, err := NewBitbucketServerProviderNoAuth(context.Background(), ts.URL, "PROJECT", false) + assert.NoError(t, err) + repos, err := provider.ListRepos(context.Background(), "ssh") + assert.NoError(t, err) assert.Equal(t, 1, len(repos)) assert.Equal(t, Repository{ Organization: "PROJECT", @@ -349,10 +349,10 @@ func TestListReposCloneProtocol(t *testing.T) { defaultHandler(t)(w, r) })) defer ts.Close() - provider, err := NewBitbucketServerProviderNoAuth(context.TODO(), ts.URL, "PROJECT", true) - assert.Nil(t, err) - repos, err := provider.ListRepos(context.TODO(), "https") - assert.Nil(t, err) + provider, err := NewBitbucketServerProviderNoAuth(context.Background(), ts.URL, "PROJECT", true) + assert.NoError(t, err) + repos, err := provider.ListRepos(context.Background(), "https") + assert.NoError(t, err) assert.Equal(t, 1, len(repos)) assert.Equal(t, Repository{ Organization: "PROJECT", @@ -370,9 +370,9 @@ func TestListReposUnknownProtocol(t *testing.T) { defaultHandler(t)(w, r) })) defer ts.Close() - provider, err := NewBitbucketServerProviderNoAuth(context.TODO(), ts.URL, "PROJECT", true) - assert.Nil(t, err) - _, errProtocol := provider.ListRepos(context.TODO(), "http") + provider, err := NewBitbucketServerProviderNoAuth(context.Background(), ts.URL, "PROJECT", true) + assert.NoError(t, err) + _, errProtocol := provider.ListRepos(context.Background(), "http") assert.NotNil(t, errProtocol) } @@ -411,27 +411,27 @@ func TestBitbucketServerHasPath(t *testing.T) { } })) defer ts.Close() - provider, err := NewBitbucketServerProviderNoAuth(context.TODO(), ts.URL, "PROJECT", true) - assert.Nil(t, err) + provider, err := NewBitbucketServerProviderNoAuth(context.Background(), ts.URL, "PROJECT", true) + assert.NoError(t, err) repo := &Repository{ Organization: "PROJECT", Repository: "REPO", Branch: "main", } ok, err := provider.RepoHasPath(context.Background(), repo, "pkg/") - assert.Nil(t, err) + assert.NoError(t, err) assert.True(t, ok) ok, err = provider.RepoHasPath(context.Background(), repo, "anotherpkg/file.txt") - assert.Nil(t, err) + assert.NoError(t, err) assert.True(t, ok) ok, err = provider.RepoHasPath(context.Background(), repo, "anotherpkg/missing.txt") - assert.Nil(t, err) + assert.NoError(t, err) assert.False(t, ok) ok, err = provider.RepoHasPath(context.Background(), repo, "notathing/") - assert.Nil(t, err) + assert.NoError(t, err) assert.False(t, ok) } From 4690633dd14db3a2e294fb72748f790254ce2eeb Mon Sep 17 00:00:00 2001 From: mlosicki Date: Mon, 28 Feb 2022 15:46:35 +0100 Subject: [PATCH 07/14] fix: refactor for PR#472 Signed-off-by: mlosicki --- api/v1alpha1/zz_generated.deepcopy.go | 1 + go.mod | 2 + go.sum | 2 +- pkg/services/scm_provider/bitbucket_server.go | 73 +++++++---- .../scm_provider/bitbucket_server_test.go | 114 +++++++++++++----- 5 files changed, 138 insertions(+), 54 deletions(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 8a597eb5..ec1d6389 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/go.mod b/go.mod index 19b34980..b8de6287 100644 --- a/go.mod +++ b/go.mod @@ -90,6 +90,7 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-wordwrap v1.0.0 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -98,6 +99,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.11.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect diff --git a/go.sum b/go.sum index 7b560a97..796fe452 100644 --- a/go.sum +++ b/go.sum @@ -673,8 +673,8 @@ github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUb github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/ipvs v1.0.1/go.mod h1:2pngiyseZbIKXNv7hsKj3O9UEz30c53MT9005gt2hxQ= diff --git a/pkg/services/scm_provider/bitbucket_server.go b/pkg/services/scm_provider/bitbucket_server.go index 6d449eaa..1f4be192 100644 --- a/pkg/services/scm_provider/bitbucket_server.go +++ b/pkg/services/scm_provider/bitbucket_server.go @@ -72,21 +72,23 @@ func (b *BitbucketServerProvider) ListRepos(_ context.Context, cloneProtocol str return nil, fmt.Errorf("unknown clone protocol for Bitbucket Server %v", cloneProtocol) } - branches, err := b.listBranches(&bitbucketRepo) + org := bitbucketRepo.Project.Key + repo := bitbucketRepo.Name + // Bitbucket doesn't return the default branch in the repo query, fetch it here + branch, err := b.getDefaultBranch(org, repo) if err != nil { - return nil, fmt.Errorf("error listing branches for %s/%s: %v", b.projectKey, bitbucketRepo.Name, err) + return nil, err } - for _, branch := range branches { - repos = append(repos, &Repository{ - Organization: bitbucketRepo.Project.Key, - Repository: bitbucketRepo.Name, - URL: url, - Branch: branch.DisplayID, - SHA: branch.LatestCommit, - Labels: []string{}, - }) - } + repos = append(repos, &Repository{ + Organization: org, + Repository: repo, + URL: url, + Branch: branch.DisplayID, + SHA: branch.LatestCommit, + Labels: []string{}, // Not supported by library + RepositoryId: bitbucketRepo.ID, + }) } hasNextPage, nextPageStart := bitbucketv1.HasNextPage(response) if !hasNextPage { @@ -129,18 +131,35 @@ func (b *BitbucketServerProvider) RepoHasPath(_ context.Context, repo *Repositor return true, nil } -func (b *BitbucketServerProvider) listBranches(repo *bitbucketv1.Repository) ([]bitbucketv1.Branch, error) { +func (b *BitbucketServerProvider) GetBranches(_ context.Context, repo *Repository) ([]*Repository, error) { + repos := []*Repository{} + branches, err := b.listBranches(repo) + if err != nil { + return nil, fmt.Errorf("error listing branches for %s/%s: %v", repo.Organization, repo.Repository, err) + } + + for _, branch := range branches { + repos = append(repos, &Repository{ + Organization: repo.Organization, + Repository: repo.Repository, + URL: repo.URL, + Branch: branch.DisplayID, + SHA: branch.LatestCommit, + Labels: repo.Labels, + RepositoryId: repo.RepositoryId, + }) + } + return repos, nil +} + +func (b *BitbucketServerProvider) listBranches(repo *Repository) ([]bitbucketv1.Branch, error) { // If we don't specifically want to query for all branches, just use the default branch and call it a day. if !b.allBranches { - response, err := b.client.DefaultApi.GetDefaultBranch(repo.Project.Key, repo.Name) + branch, err := b.getDefaultBranch(repo.Organization, repo.Repository) if err != nil { return nil, err } - branch, err := bitbucketv1.GetBranchResponse(response) - if err != nil { - return nil, err - } - return []bitbucketv1.Branch{branch}, nil + return []bitbucketv1.Branch{*branch}, nil } // Otherwise, scrape the GetBranches API. branches := []bitbucketv1.Branch{} @@ -148,9 +167,9 @@ func (b *BitbucketServerProvider) listBranches(repo *bitbucketv1.Repository) ([] "limit": 100, } for { - response, err := b.client.DefaultApi.GetBranches(repo.Project.Key, repo.Name, paged) + response, err := b.client.DefaultApi.GetBranches(repo.Organization, repo.Repository, paged) if err != nil { - return nil, fmt.Errorf("error listing branches for %s/%s: %v", repo.Project.Key, repo.Name, err) + return nil, fmt.Errorf("error listing branches for %s/%s: %v", repo.Organization, repo.Repository, err) } bitbucketBranches, err := bitbucketv1.GetBranchesResponse(response) if err != nil { @@ -168,6 +187,18 @@ func (b *BitbucketServerProvider) listBranches(repo *bitbucketv1.Repository) ([] return branches, nil } +func (b *BitbucketServerProvider) getDefaultBranch(org string, repo string) (*bitbucketv1.Branch, error) { + response, err := b.client.DefaultApi.GetDefaultBranch(org, repo) + if err != nil { + return nil, err + } + branch, err := bitbucketv1.GetBranchResponse(response) + if err != nil { + return nil, err + } + return &branch, nil +} + func getCloneURLFromLinks(links []bitbucketv1.CloneLink, name string) string { for _, link := range links { if link.Name == name { diff --git a/pkg/services/scm_provider/bitbucket_server_test.go b/pkg/services/scm_provider/bitbucket_server_test.go index de73c6e7..df5bd1cd 100644 --- a/pkg/services/scm_provider/bitbucket_server_test.go +++ b/pkg/services/scm_provider/bitbucket_server_test.go @@ -22,6 +22,7 @@ func defaultHandler(t *testing.T) func(http.ResponseWriter, *http.Request) { "isLastPage": true, "values": [ { + "id": 1, "name": "REPO", "project": { "key": "PROJECT" @@ -59,6 +60,15 @@ func defaultHandler(t *testing.T) func(http.ResponseWriter, *http.Request) { ], "start": 0 }`) + case "/rest/api/1.0/projects/PROJECT/repos/REPO/branches/default": + _, err = io.WriteString(w, `{ + "id": "refs/heads/main", + "displayId": "main", + "type": "BRANCH", + "latestCommit": "8d51122def5632836d1cb1026e879069e10a1e13", + "latestChangeset": "8d51122def5632836d1cb1026e879069e10a1e13", + "isDefault": true + }`) default: t.Fail() } @@ -78,6 +88,7 @@ func verifyDefaultRepo(t *testing.T, err error, repos []*Repository) { Branch: "main", SHA: "8d51122def5632836d1cb1026e879069e10a1e13", Labels: []string{}, + RepositoryId: 1, }, *repos[0]) } @@ -105,6 +116,7 @@ func TestListReposPagination(t *testing.T) { "isLastPage": false, "values": [ { + "id": 100, "name": "REPO", "project": { "key": "PROJECT" @@ -133,6 +145,7 @@ func TestListReposPagination(t *testing.T) { "isLastPage": true, "values": [ { + "id": 200, "name": "REPO2", "project": { "key": "PROJECT" @@ -153,39 +166,21 @@ func TestListReposPagination(t *testing.T) { ], "start": 200 }`) - case "/rest/api/1.0/projects/PROJECT/repos/REPO/branches?limit=100": + case "/rest/api/1.0/projects/PROJECT/repos/REPO/branches/default": _, err = io.WriteString(w, `{ - "size": 1, - "limit": 100, - "isLastPage": true, - "values": [ - { - "id": "refs/heads/main", - "displayId": "main", - "type": "BRANCH", - "latestCommit": "8d51122def5632836d1cb1026e879069e10a1e13", - "latestChangeset": "8d51122def5632836d1cb1026e879069e10a1e13", - "isDefault": true - } - ], - "start": 0 + "id": "refs/heads/main", + "displayId": "main", + "type": "BRANCH", + "latestCommit": "8d51122def5632836d1cb1026e879069e10a1e13", + "isDefault": true }`) - case "/rest/api/1.0/projects/PROJECT/repos/REPO2/branches?limit=100": + case "/rest/api/1.0/projects/PROJECT/repos/REPO2/branches/default": _, err = io.WriteString(w, `{ - "size": 1, - "limit": 100, - "isLastPage": true, - "values": [ - { - "id": "refs/heads/development", - "displayId": "development", - "type": "BRANCH", - "latestCommit": "2d51122def5632836d1cb1026e879069e10a1e13", - "latestChangeset": "2d51122def5632836d1cb1026e879069e10a1e13", - "isDefault": true - } - ], - "start": 0 + "id": "refs/heads/development", + "displayId": "development", + "type": "BRANCH", + "latestCommit": "2d51122def5632836d1cb1026e879069e10a1e13", + "isDefault": true }`) default: t.Fail() @@ -207,6 +202,7 @@ func TestListReposPagination(t *testing.T) { Branch: "main", SHA: "8d51122def5632836d1cb1026e879069e10a1e13", Labels: []string{}, + RepositoryId: 100, }, *repos[0]) assert.Equal(t, Repository{ @@ -216,10 +212,11 @@ func TestListReposPagination(t *testing.T) { Branch: "development", SHA: "2d51122def5632836d1cb1026e879069e10a1e13", Labels: []string{}, + RepositoryId: 200, }, *repos[1]) } -func TestListReposBranchPagination(t *testing.T) { +func TestGetBranchesBranchPagination(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assert.Empty(t, r.Header.Get("Authorization")) switch r.RequestURI { @@ -272,7 +269,13 @@ func TestListReposBranchPagination(t *testing.T) { defer ts.Close() provider, err := NewBitbucketServerProviderNoAuth(context.Background(), ts.URL, "PROJECT", true) assert.NoError(t, err) - repos, err := provider.ListRepos(context.Background(), "ssh") + repos, err := provider.GetBranches(context.Background(), &Repository{ + Organization: "PROJECT", + Repository: "REPO", + URL: "ssh://git@mycompany.bitbucket.org/PROJECT/REPO.git", + Labels: []string{}, + RepositoryId: 1, + }) assert.NoError(t, err) assert.Equal(t, 2, len(repos)) assert.Equal(t, Repository{ @@ -282,6 +285,7 @@ func TestListReposBranchPagination(t *testing.T) { Branch: "main", SHA: "8d51122def5632836d1cb1026e879069e10a1e13", Labels: []string{}, + RepositoryId: 1, }, *repos[0]) assert.Equal(t, Repository{ @@ -291,9 +295,53 @@ func TestListReposBranchPagination(t *testing.T) { Branch: "feature", SHA: "9d51122def5632836d1cb1026e879069e10a1e13", Labels: []string{}, + RepositoryId: 1, }, *repos[1]) } +func TestGetBranchesDefaultOnly(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.Header.Get("Authorization")) + switch r.RequestURI { + case "/rest/api/1.0/projects/PROJECT/repos/REPO/branches/default": + _, err := io.WriteString(w, `{ + "id": "refs/heads/default", + "displayId": "default", + "type": "BRANCH", + "latestCommit": "ab51122def5632836d1cb1026e879069e10a1e13", + "latestChangeset": "ab51122def5632836d1cb1026e879069e10a1e13", + "isDefault": true + }`) + if err != nil { + t.Fail() + } + return + } + defaultHandler(t)(w, r) + })) + defer ts.Close() + provider, err := NewBitbucketServerProviderNoAuth(context.Background(), ts.URL, "PROJECT", false) + assert.NoError(t, err) + repos, err := provider.GetBranches(context.Background(), &Repository{ + Organization: "PROJECT", + Repository: "REPO", + URL: "ssh://git@mycompany.bitbucket.org/PROJECT/REPO.git", + Labels: []string{}, + RepositoryId: 1, + }) + assert.NoError(t, err) + assert.Equal(t, 1, len(repos)) + assert.Equal(t, Repository{ + Organization: "PROJECT", + Repository: "REPO", + URL: "ssh://git@mycompany.bitbucket.org/PROJECT/REPO.git", + Branch: "default", + SHA: "ab51122def5632836d1cb1026e879069e10a1e13", + Labels: []string{}, + RepositoryId: 1, + }, *repos[0]) +} + func TestListReposBasicAuth(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, "Basic dXNlcjpwYXNzd29yZA==", r.Header.Get("Authorization")) @@ -340,6 +388,7 @@ func TestListReposDefaultBranch(t *testing.T) { Branch: "default", SHA: "1d51122def5632836d1cb1026e879069e10a1e13", Labels: []string{}, + RepositoryId: 1, }, *repos[0]) } @@ -361,6 +410,7 @@ func TestListReposCloneProtocol(t *testing.T) { Branch: "main", SHA: "8d51122def5632836d1cb1026e879069e10a1e13", Labels: []string{}, + RepositoryId: 1, }, *repos[0]) } From 216d0eeba17b2931de244f387c458165a0d2525f Mon Sep 17 00:00:00 2001 From: mlosicki Date: Mon, 28 Feb 2022 16:22:26 +0100 Subject: [PATCH 08/14] fix: pr comments, refactor bitbucket BasePath Signed-off-by: mlosicki --- pkg/services/pull_request/bitbucket_server.go | 6 ++--- pkg/services/scm_provider/bitbucket_server.go | 6 ++--- pkg/utils/util.go | 10 +++++++ pkg/utils/util_test.go | 27 +++++++++++++++++++ 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/pkg/services/pull_request/bitbucket_server.go b/pkg/services/pull_request/bitbucket_server.go index 40c1bce7..2dd790c7 100644 --- a/pkg/services/pull_request/bitbucket_server.go +++ b/pkg/services/pull_request/bitbucket_server.go @@ -4,8 +4,8 @@ import ( "context" "fmt" "regexp" - "strings" + "github.com/argoproj/applicationset/pkg/utils" bitbucketv1 "github.com/gfleury/go-bitbucket-v1" ) @@ -38,9 +38,7 @@ func NewBitbucketServiceNoAuth(ctx context.Context, url, projectKey, repositoryS } func newBitbucketService(ctx context.Context, bitbucketConfig *bitbucketv1.Configuration, projectKey, repositorySlug string, branchMatch *string) (PullRequestService, error) { - if !strings.HasSuffix(bitbucketConfig.BasePath, "/rest") { - bitbucketConfig.BasePath = bitbucketConfig.BasePath + "/rest" - } + bitbucketConfig.BasePath = utils.NormalizeBitbucketBasePath(bitbucketConfig.BasePath) bitbucketClient := bitbucketv1.NewAPIClient(ctx, bitbucketConfig) var branchMatchRegexp *regexp.Regexp diff --git a/pkg/services/scm_provider/bitbucket_server.go b/pkg/services/scm_provider/bitbucket_server.go index 1f4be192..287d6c00 100644 --- a/pkg/services/scm_provider/bitbucket_server.go +++ b/pkg/services/scm_provider/bitbucket_server.go @@ -3,8 +3,8 @@ package scm_provider import ( "context" "fmt" - "strings" + "github.com/argoproj/applicationset/pkg/utils" bitbucketv1 "github.com/gfleury/go-bitbucket-v1" ) @@ -34,9 +34,7 @@ func NewBitbucketServerProviderNoAuth(ctx context.Context, url, projectKey strin } func newBitbucketServerProvider(ctx context.Context, bitbucketConfig *bitbucketv1.Configuration, projectKey string, allBranches bool) (*BitbucketServerProvider, error) { - if !strings.HasSuffix(bitbucketConfig.BasePath, "/rest") { - bitbucketConfig.BasePath = bitbucketConfig.BasePath + "/rest" - } + bitbucketConfig.BasePath = utils.NormalizeBitbucketBasePath(bitbucketConfig.BasePath) bitbucketClient := bitbucketv1.NewAPIClient(ctx, bitbucketConfig) return &BitbucketServerProvider{ diff --git a/pkg/utils/util.go b/pkg/utils/util.go index 1436cb23..a0b6dc25 100644 --- a/pkg/utils/util.go +++ b/pkg/utils/util.go @@ -176,3 +176,13 @@ func addInvalidGeneratorNames(names map[string]bool, applicationSetInfo *argopro break } } + +func NormalizeBitbucketBasePath(basePath string) string { + if strings.HasSuffix(basePath, "/rest/") { + return strings.TrimSuffix(basePath, "/") + } + if !strings.HasSuffix(basePath, "/rest") { + return basePath + "/rest" + } + return basePath +} diff --git a/pkg/utils/util_test.go b/pkg/utils/util_test.go index d56b25c7..ca5fd1fb 100644 --- a/pkg/utils/util_test.go +++ b/pkg/utils/util_test.go @@ -649,3 +649,30 @@ func TestInvalidGenerators(t *testing.T) { assert.Equal(t, c.expectedNames, names, c.testName) } } + +func TestNormalizeBitbucketBasePath(t *testing.T) { + for _, c := range []struct { + testName string + basePath string + expectedBasePath string + }{ + { + testName: "default api url", + basePath: "https://company.bitbucket.com", + expectedBasePath: "https://company.bitbucket.com/rest", + }, + { + testName: "with /rest suffix", + basePath: "https://company.bitbucket.com/rest", + expectedBasePath: "https://company.bitbucket.com/rest", + }, + { + testName: "with /rest/ suffix", + basePath: "https://company.bitbucket.com/rest/", + expectedBasePath: "https://company.bitbucket.com/rest", + }, + } { + result := NormalizeBitbucketBasePath(c.basePath) + assert.Equal(t, c.expectedBasePath, result, c.testName) + } +} From f1981505c591ed6555e1883092a29f66f822943f Mon Sep 17 00:00:00 2001 From: mlosicki Date: Mon, 28 Feb 2022 16:32:55 +0100 Subject: [PATCH 09/14] fix: pr comment, do not put full response in err Signed-off-by: mlosicki --- pkg/services/pull_request/bitbucket_server.go | 4 +++- pkg/services/scm_provider/bitbucket_server.go | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/services/pull_request/bitbucket_server.go b/pkg/services/pull_request/bitbucket_server.go index 2dd790c7..a63b4439 100644 --- a/pkg/services/pull_request/bitbucket_server.go +++ b/pkg/services/pull_request/bitbucket_server.go @@ -7,6 +7,7 @@ import ( "github.com/argoproj/applicationset/pkg/utils" bitbucketv1 "github.com/gfleury/go-bitbucket-v1" + log "github.com/sirupsen/logrus" ) type BitbucketService struct { @@ -71,7 +72,8 @@ func (b *BitbucketService) List(_ context.Context) ([]*PullRequest, error) { } pulls, err := bitbucketv1.GetPullRequestsResponse(response) if err != nil { - return nil, fmt.Errorf("error parsing pull request response %s: %v", response.Values, err) + log.Errorf("error parsing pull request response '%v'", response.Values) + return nil, fmt.Errorf("error parsing pull request response for %s/%s: %v", b.projectKey, b.repositorySlug, err) } for _, pull := range pulls { diff --git a/pkg/services/scm_provider/bitbucket_server.go b/pkg/services/scm_provider/bitbucket_server.go index 287d6c00..8790d86a 100644 --- a/pkg/services/scm_provider/bitbucket_server.go +++ b/pkg/services/scm_provider/bitbucket_server.go @@ -6,6 +6,7 @@ import ( "github.com/argoproj/applicationset/pkg/utils" bitbucketv1 "github.com/gfleury/go-bitbucket-v1" + log "github.com/sirupsen/logrus" ) type BitbucketServerProvider struct { @@ -56,7 +57,8 @@ func (b *BitbucketServerProvider) ListRepos(_ context.Context, cloneProtocol str } repositories, err := bitbucketv1.GetRepositoriesResponse(response) if err != nil { - return nil, fmt.Errorf("error parsing repositories response %s: %v", response.Values, err) + log.Errorf("error parsing repositories response '%v'", response.Values) + return nil, fmt.Errorf("error parsing repositories response %s: %v", b.projectKey, err) } for _, bitbucketRepo := range repositories { var url string @@ -171,7 +173,8 @@ func (b *BitbucketServerProvider) listBranches(repo *Repository) ([]bitbucketv1. } bitbucketBranches, err := bitbucketv1.GetBranchesResponse(response) if err != nil { - return nil, fmt.Errorf("error parsing branches response %s: %v", response.Values, err) + log.Errorf("error parsing branches response '%v'", response.Values) + return nil, fmt.Errorf("error parsing branches response for %s/%s: %v", repo.Organization, repo.Repository, err) } branches = append(branches, bitbucketBranches...) From 0915a90139408bcaf22ccebb4c467aa04b1f2dea Mon Sep 17 00:00:00 2001 From: mlosicki Date: Sat, 5 Mar 2022 16:55:05 +0100 Subject: [PATCH 10/14] docs: pr comment Signed-off-by: mlosicki --- docs/Generators-SCM-Provider.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Generators-SCM-Provider.md b/docs/Generators-SCM-Provider.md index 61fa4d1b..9aaae459 100644 --- a/docs/Generators-SCM-Provider.md +++ b/docs/Generators-SCM-Provider.md @@ -120,7 +120,7 @@ spec: passwordRef: secretName: mypassword key: password - # Support for labels is TODO. Bitbucket server labels are not supported for PRs, but they are for repos + # Support for filtering by labels is TODO. Bitbucket server labels are not supported for PRs, but they are for repos template: # ... ``` From 39dff309c407f98fa1f94b15e9214dda6d33e21d Mon Sep 17 00:00:00 2001 From: mlosicki Date: Thu, 10 Mar 2022 20:52:46 +0000 Subject: [PATCH 11/14] fix: existing copy paste error Signed-off-by: mlosicki --- pkg/services/scm_provider/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/scm_provider/utils.go b/pkg/services/scm_provider/utils.go index 07f29a59..67eccb94 100644 --- a/pkg/services/scm_provider/utils.go +++ b/pkg/services/scm_provider/utils.go @@ -34,7 +34,7 @@ func compileFilters(filters []argoprojiov1alpha1.SCMProviderGeneratorFilter) ([] if filter.BranchMatch != nil { outFilter.BranchMatch, err = regexp.Compile(*filter.BranchMatch) if err != nil { - return nil, fmt.Errorf("error compiling BranchMatch regexp %q: %v", *filter.LabelMatch, err) + return nil, fmt.Errorf("error compiling BranchMatch regexp %q: %v", *filter.BranchMatch, err) } outFilter.FilterType = FilterTypeBranch } From 4b1acb57926ed141fe4c405d10813e13170de17d Mon Sep 17 00:00:00 2001 From: mlosicki Date: Thu, 10 Mar 2022 22:09:40 +0000 Subject: [PATCH 12/14] refactor: filters based on pr comments Signed-off-by: mlosicki --- api/v1alpha1/applicationset_types.go | 11 +- api/v1alpha1/zz_generated.deepcopy.go | 32 +++- docs/Generators-Pull-Request.md | 30 +++- .../crds/argoproj.io_applicationsets.yaml | 27 +++- manifests/install.yaml | 27 +++- pkg/generators/pull_request.go | 7 +- pkg/services/pull_request/bitbucket_server.go | 25 +--- .../pull_request/bitbucket_server_test.go | 55 ++++--- pkg/services/pull_request/interface.go | 9 +- pkg/services/pull_request/utils.go | 62 ++++++++ pkg/services/pull_request/utils_test.go | 139 ++++++++++++++++++ 11 files changed, 358 insertions(+), 66 deletions(-) create mode 100644 pkg/services/pull_request/utils.go create mode 100644 pkg/services/pull_request/utils_test.go diff --git a/api/v1alpha1/applicationset_types.go b/api/v1alpha1/applicationset_types.go index 00185acc..05804258 100644 --- a/api/v1alpha1/applicationset_types.go +++ b/api/v1alpha1/applicationset_types.go @@ -362,6 +362,8 @@ type PullRequestGenerator struct { // Which provider to use and config for it. Github *PullRequestGeneratorGithub `json:"github,omitempty"` BitbucketServer *PullRequestGeneratorBitbucketServer `json:"bitbucketServer,omitempty"` + // Filters for which pull requests should be considered. + Filters []PullRequestGeneratorFilter `json:"filters,omitempty"` // Standard parameters. RequeueAfterSeconds *int64 `json:"requeueAfterSeconds,omitempty"` Template ApplicationSetTemplate `json:"template,omitempty"` @@ -389,8 +391,6 @@ type PullRequestGeneratorBitbucketServer struct { Repo string `json:"repo"` // The Bitbucket REST API URL to talk to e.g. https://bitbucket.org/rest Required. API string `json:"api"` - // A regex which must match the branch name. - BranchMatch *string `json:"branchMatch,omitempty"` // Credentials for Basic auth BasicAuth *BasicAuthBitbucketServer `json:"basicAuth,omitempty"` } @@ -403,6 +403,13 @@ type BasicAuthBitbucketServer struct { PasswordRef *SecretRef `json:"passwordRef"` } +// PullRequestGeneratorFilter is a single pull request filter. +// If multiple filter types are set on a single struct, they will be AND'd together. All filters must +// pass for a pull request to be included. +type PullRequestGeneratorFilter struct { + BranchMatch *string `json:"branchMatch,omitempty"` +} + // ApplicationSetStatus defines the observed state of ApplicationSet type ApplicationSetStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ec1d6389..ff281ef6 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -685,6 +685,13 @@ func (in *PullRequestGenerator) DeepCopyInto(out *PullRequestGenerator) { *out = new(PullRequestGeneratorBitbucketServer) (*in).DeepCopyInto(*out) } + if in.Filters != nil { + in, out := &in.Filters, &out.Filters + *out = make([]PullRequestGeneratorFilter, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.RequeueAfterSeconds != nil { in, out := &in.RequeueAfterSeconds, &out.RequeueAfterSeconds *out = new(int64) @@ -706,11 +713,6 @@ func (in *PullRequestGenerator) DeepCopy() *PullRequestGenerator { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PullRequestGeneratorBitbucketServer) DeepCopyInto(out *PullRequestGeneratorBitbucketServer) { *out = *in - if in.BranchMatch != nil { - in, out := &in.BranchMatch, &out.BranchMatch - *out = new(string) - **out = **in - } if in.BasicAuth != nil { in, out := &in.BasicAuth, &out.BasicAuth *out = new(BasicAuthBitbucketServer) @@ -728,6 +730,26 @@ func (in *PullRequestGeneratorBitbucketServer) DeepCopy() *PullRequestGeneratorB return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PullRequestGeneratorFilter) DeepCopyInto(out *PullRequestGeneratorFilter) { + *out = *in + if in.BranchMatch != nil { + in, out := &in.BranchMatch, &out.BranchMatch + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PullRequestGeneratorFilter. +func (in *PullRequestGeneratorFilter) DeepCopy() *PullRequestGeneratorFilter { + if in == nil { + return nil + } + out := new(PullRequestGeneratorFilter) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PullRequestGeneratorGithub) DeepCopyInto(out *PullRequestGeneratorGithub) { *out = *in diff --git a/docs/Generators-Pull-Request.md b/docs/Generators-Pull-Request.md index 4b0bd919..cad482bc 100644 --- a/docs/Generators-Pull-Request.md +++ b/docs/Generators-Pull-Request.md @@ -70,8 +70,6 @@ spec: repo: myrepository # URL of the Bitbucket Server. Required. api: https://mycompany.bitbucket.org - # Filter PRs using the source branch name. (optional) - branchMatch: ".*-argocd" # Credentials for Basic authentication. Required for private repositories. basicAuth: # The username to authenticate with @@ -80,7 +78,10 @@ spec: passwordRef: secretName: mypassword key: password - # Labels are not supported by Bitbucket Server, so filtering by label is not possible. + # Labels are not supported by Bitbucket Server, so filtering by label is not possible. + # Filter PRs using the source branch name. (optional) + filters: + - branchMatch: ".*-argocd" template: # ... ``` @@ -94,6 +95,29 @@ If you want to access a private repository, you must also provide the credential * `username`: The username to authenticate with. It only needs read access to the relevant repo. * `passwordRef`: A `Secret` name and key containing the password or personal access token to use for requests. +## Filters + +Filters allow selecting which pull requests to generate for. Each filter can declare one or more conditions, all of which must pass. If multiple filters are present, any can match for a repository to be included. If no filters are specified, all pull requests will be processed. +Currently, only a subset of filters is available when comparing with SCM provider filters. + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: myapps +spec: + generators: + - scmProvider: + # ... + # Include any pull request ending with "argocd". (optional) + filters: + - branchMatch: ".*-argocd" + template: + # ... +``` + +* `branchMatch`: A regexp matched against source branch names. + ## Template As with all generators, several keys are available for replacement in the generated application. diff --git a/manifests/crds/argoproj.io_applicationsets.yaml b/manifests/crds/argoproj.io_applicationsets.yaml index c2d16a5e..6b8d463c 100644 --- a/manifests/crds/argoproj.io_applicationsets.yaml +++ b/manifests/crds/argoproj.io_applicationsets.yaml @@ -2481,8 +2481,6 @@ spec: - passwordRef - username type: object - branchMatch: - type: string project: type: string repo: @@ -2492,6 +2490,13 @@ spec: - project - repo type: object + filters: + items: + properties: + branchMatch: + type: string + type: object + type: array github: properties: api: @@ -4686,8 +4691,6 @@ spec: - passwordRef - username type: object - branchMatch: - type: string project: type: string repo: @@ -4697,6 +4700,13 @@ spec: - project - repo type: object + filters: + items: + properties: + branchMatch: + type: string + type: object + type: array github: properties: api: @@ -5680,8 +5690,6 @@ spec: - passwordRef - username type: object - branchMatch: - type: string project: type: string repo: @@ -5691,6 +5699,13 @@ spec: - project - repo type: object + filters: + items: + properties: + branchMatch: + type: string + type: object + type: array github: properties: api: diff --git a/manifests/install.yaml b/manifests/install.yaml index 28b671b5..bf8de52e 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -2480,8 +2480,6 @@ spec: - passwordRef - username type: object - branchMatch: - type: string project: type: string repo: @@ -2491,6 +2489,13 @@ spec: - project - repo type: object + filters: + items: + properties: + branchMatch: + type: string + type: object + type: array github: properties: api: @@ -4685,8 +4690,6 @@ spec: - passwordRef - username type: object - branchMatch: - type: string project: type: string repo: @@ -4696,6 +4699,13 @@ spec: - project - repo type: object + filters: + items: + properties: + branchMatch: + type: string + type: object + type: array github: properties: api: @@ -5679,8 +5689,6 @@ spec: - passwordRef - username type: object - branchMatch: - type: string project: type: string repo: @@ -5690,6 +5698,13 @@ spec: - project - repo type: object + filters: + items: + properties: + branchMatch: + type: string + type: object + type: array github: properties: api: diff --git a/pkg/generators/pull_request.go b/pkg/generators/pull_request.go index f7e60bbc..22d71186 100644 --- a/pkg/generators/pull_request.go +++ b/pkg/generators/pull_request.go @@ -10,6 +10,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" argoprojiov1alpha1 "github.com/argoproj/applicationset/api/v1alpha1" + "github.com/argoproj/applicationset/pkg/services/pull_request" pullrequest "github.com/argoproj/applicationset/pkg/services/pull_request" ) @@ -61,7 +62,7 @@ func (g *PullRequestGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha return nil, fmt.Errorf("failed to select pull request service provider: %v", err) } - pulls, err := svc.List(ctx) + pulls, err := pull_request.ListPullRequests(ctx, svc, appSetGenerator.PullRequest.Filters) if err != nil { return nil, fmt.Errorf("error listing repos: %v", err) } @@ -93,9 +94,9 @@ func (g *PullRequestGenerator) selectServiceProvider(ctx context.Context, genera if err != nil { return nil, fmt.Errorf("error fetching Secret token: %v", err) } - return pullrequest.NewBitbucketServiceBasicAuth(ctx, providerConfig.BasicAuth.Username, password, providerConfig.API, providerConfig.Project, providerConfig.Repo, providerConfig.BranchMatch) + return pullrequest.NewBitbucketServiceBasicAuth(ctx, providerConfig.BasicAuth.Username, password, providerConfig.API, providerConfig.Project, providerConfig.Repo) } else { - return pullrequest.NewBitbucketServiceNoAuth(ctx, providerConfig.API, providerConfig.Project, providerConfig.Repo, providerConfig.BranchMatch) + return pullrequest.NewBitbucketServiceNoAuth(ctx, providerConfig.API, providerConfig.Project, providerConfig.Repo) } } return nil, fmt.Errorf("no Pull Request provider implementation configured") diff --git a/pkg/services/pull_request/bitbucket_server.go b/pkg/services/pull_request/bitbucket_server.go index a63b4439..2523cd70 100644 --- a/pkg/services/pull_request/bitbucket_server.go +++ b/pkg/services/pull_request/bitbucket_server.go @@ -3,7 +3,6 @@ package pull_request import ( "context" "fmt" - "regexp" "github.com/argoproj/applicationset/pkg/utils" bitbucketv1 "github.com/gfleury/go-bitbucket-v1" @@ -14,14 +13,13 @@ type BitbucketService struct { client *bitbucketv1.APIClient projectKey string repositorySlug string - branchMatch *regexp.Regexp // Not supported for PRs by Bitbucket Server // labels []string } var _ PullRequestService = (*BitbucketService)(nil) -func NewBitbucketServiceBasicAuth(ctx context.Context, username, password, url, projectKey, repositorySlug string, branchMatch *string) (PullRequestService, error) { +func NewBitbucketServiceBasicAuth(ctx context.Context, username, password, url, projectKey, repositorySlug string) (PullRequestService, error) { bitbucketConfig := bitbucketv1.NewConfiguration(url) // Avoid the XSRF check bitbucketConfig.AddDefaultHeader("x-atlassian-token", "no-check") @@ -31,31 +29,21 @@ func NewBitbucketServiceBasicAuth(ctx context.Context, username, password, url, UserName: username, Password: password, }) - return newBitbucketService(ctx, bitbucketConfig, projectKey, repositorySlug, branchMatch) + return newBitbucketService(ctx, bitbucketConfig, projectKey, repositorySlug) } -func NewBitbucketServiceNoAuth(ctx context.Context, url, projectKey, repositorySlug string, branchMatch *string) (PullRequestService, error) { - return newBitbucketService(ctx, bitbucketv1.NewConfiguration(url), projectKey, repositorySlug, branchMatch) +func NewBitbucketServiceNoAuth(ctx context.Context, url, projectKey, repositorySlug string) (PullRequestService, error) { + return newBitbucketService(ctx, bitbucketv1.NewConfiguration(url), projectKey, repositorySlug) } -func newBitbucketService(ctx context.Context, bitbucketConfig *bitbucketv1.Configuration, projectKey, repositorySlug string, branchMatch *string) (PullRequestService, error) { +func newBitbucketService(ctx context.Context, bitbucketConfig *bitbucketv1.Configuration, projectKey, repositorySlug string) (PullRequestService, error) { bitbucketConfig.BasePath = utils.NormalizeBitbucketBasePath(bitbucketConfig.BasePath) bitbucketClient := bitbucketv1.NewAPIClient(ctx, bitbucketConfig) - var branchMatchRegexp *regexp.Regexp - if branchMatch != nil { - var err error - branchMatchRegexp, err = regexp.Compile(*branchMatch) - if err != nil { - return nil, fmt.Errorf("error compiling BranchMatch regexp %q: %v", *branchMatch, err) - } - } - return &BitbucketService{ client: bitbucketClient, projectKey: projectKey, repositorySlug: repositorySlug, - branchMatch: branchMatchRegexp, }, nil } @@ -77,9 +65,6 @@ func (b *BitbucketService) List(_ context.Context) ([]*PullRequest, error) { } for _, pull := range pulls { - if b.branchMatch != nil && !b.branchMatch.MatchString(pull.FromRef.DisplayID) { - continue - } pullRequests = append(pullRequests, &PullRequest{ Number: pull.ID, Branch: pull.FromRef.DisplayID, // ID: refs/heads/main DisplayID: main diff --git a/pkg/services/pull_request/bitbucket_server_test.go b/pkg/services/pull_request/bitbucket_server_test.go index 21eab859..09f5cf52 100644 --- a/pkg/services/pull_request/bitbucket_server_test.go +++ b/pkg/services/pull_request/bitbucket_server_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "testing" + "github.com/argoproj/applicationset/api/v1alpha1" "github.com/stretchr/testify/assert" ) @@ -47,9 +48,9 @@ func TestListPullRequestNoAuth(t *testing.T) { defaultHandler(t)(w, r) })) defer ts.Close() - svc, err := NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO", nil) + svc, err := NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO") assert.NoError(t, err) - pullRequests, err := svc.List(context.Background()) + pullRequests, err := ListPullRequests(context.Background(), svc, []v1alpha1.PullRequestGeneratorFilter{}) assert.NoError(t, err) assert.Equal(t, 1, len(pullRequests)) assert.Equal(t, 101, pullRequests[0].Number) @@ -112,9 +113,9 @@ func TestListPullRequestPagination(t *testing.T) { } })) defer ts.Close() - svc, err := NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO", nil) + svc, err := NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO") assert.NoError(t, err) - pullRequests, err := svc.List(context.Background()) + pullRequests, err := ListPullRequests(context.Background(), svc, []v1alpha1.PullRequestGeneratorFilter{}) assert.NoError(t, err) assert.Equal(t, 3, len(pullRequests)) assert.Equal(t, PullRequest{ @@ -142,9 +143,9 @@ func TestListPullRequestBasicAuth(t *testing.T) { defaultHandler(t)(w, r) })) defer ts.Close() - svc, err := NewBitbucketServiceBasicAuth(context.Background(), "user", "password", ts.URL, "PROJECT", "REPO", nil) + svc, err := NewBitbucketServiceBasicAuth(context.Background(), "user", "password", ts.URL, "PROJECT", "REPO") assert.NoError(t, err) - pullRequests, err := svc.List(context.Background()) + pullRequests, err := ListPullRequests(context.Background(), svc, []v1alpha1.PullRequestGeneratorFilter{}) assert.NoError(t, err) assert.Equal(t, 1, len(pullRequests)) assert.Equal(t, 101, pullRequests[0].Number) @@ -157,9 +158,9 @@ func TestListResponseError(t *testing.T) { w.WriteHeader(500) })) defer ts.Close() - svc, _ := NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO", nil) - _, err := svc.List(context.Background()) - assert.NotNil(t, err, err) + svc, _ := NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO") + _, err := ListPullRequests(context.Background(), svc, []v1alpha1.PullRequestGeneratorFilter{}) + assert.Error(t, err) } func TestListResponseMalformed(t *testing.T) { @@ -182,9 +183,9 @@ func TestListResponseMalformed(t *testing.T) { } })) defer ts.Close() - svc, _ := NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO", nil) - _, err := svc.List(context.Background()) - assert.NotNil(t, err, err) + svc, _ := NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO") + _, err := ListPullRequests(context.Background(), svc, []v1alpha1.PullRequestGeneratorFilter{}) + assert.Error(t, err) } func TestListResponseEmpty(t *testing.T) { @@ -207,9 +208,9 @@ func TestListResponseEmpty(t *testing.T) { } })) defer ts.Close() - svc, err := NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO", nil) + svc, err := NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO") assert.NoError(t, err) - pullRequests, err := svc.List(context.Background()) + pullRequests, err := ListPullRequests(context.Background(), svc, []v1alpha1.PullRequestGeneratorFilter{}) assert.NoError(t, err) assert.Empty(t, pullRequests) } @@ -270,9 +271,13 @@ func TestListPullRequestBranchMatch(t *testing.T) { })) defer ts.Close() regexp := `feature-1[\d]{2}` - svc, err := NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO", ®exp) + svc, err := NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO") assert.NoError(t, err) - pullRequests, err := svc.List(context.Background()) + pullRequests, err := ListPullRequests(context.Background(), svc, []v1alpha1.PullRequestGeneratorFilter{ + { + BranchMatch: ®exp, + }, + }) assert.NoError(t, err) assert.Equal(t, 2, len(pullRequests)) assert.Equal(t, PullRequest{ @@ -287,9 +292,13 @@ func TestListPullRequestBranchMatch(t *testing.T) { }, *pullRequests[1]) regexp = `.*2$` - svc, err = NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO", ®exp) + svc, err = NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO") assert.NoError(t, err) - pullRequests, err = svc.List(context.Background()) + pullRequests, err = ListPullRequests(context.Background(), svc, []v1alpha1.PullRequestGeneratorFilter{ + { + BranchMatch: ®exp, + }, + }) assert.NoError(t, err) assert.Equal(t, 1, len(pullRequests)) assert.Equal(t, PullRequest{ @@ -299,6 +308,12 @@ func TestListPullRequestBranchMatch(t *testing.T) { }, *pullRequests[0]) regexp = `[\d{2}` - _, err = NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO", ®exp) - assert.NotNil(t, err) + svc, err = NewBitbucketServiceNoAuth(context.Background(), ts.URL, "PROJECT", "REPO") + assert.NoError(t, err) + _, err = ListPullRequests(context.Background(), svc, []v1alpha1.PullRequestGeneratorFilter{ + { + BranchMatch: ®exp, + }, + }) + assert.Error(t, err) } diff --git a/pkg/services/pull_request/interface.go b/pkg/services/pull_request/interface.go index bc67681c..c55fa5ef 100644 --- a/pkg/services/pull_request/interface.go +++ b/pkg/services/pull_request/interface.go @@ -1,6 +1,9 @@ package pull_request -import "context" +import ( + "context" + "regexp" +) type PullRequest struct { // Number is a number that will be the ID of the pull request. @@ -15,3 +18,7 @@ type PullRequestService interface { // List gets a list of pull requests. List(ctx context.Context) ([]*PullRequest, error) } + +type Filter struct { + BranchMatch *regexp.Regexp +} diff --git a/pkg/services/pull_request/utils.go b/pkg/services/pull_request/utils.go new file mode 100644 index 00000000..59fd18e2 --- /dev/null +++ b/pkg/services/pull_request/utils.go @@ -0,0 +1,62 @@ +package pull_request + +import ( + "context" + "fmt" + "regexp" + + argoprojiov1alpha1 "github.com/argoproj/applicationset/api/v1alpha1" +) + +func compileFilters(filters []argoprojiov1alpha1.PullRequestGeneratorFilter) ([]*Filter, error) { + outFilters := make([]*Filter, 0, len(filters)) + for _, filter := range filters { + outFilter := &Filter{} + var err error + if filter.BranchMatch != nil { + outFilter.BranchMatch, err = regexp.Compile(*filter.BranchMatch) + if err != nil { + return nil, fmt.Errorf("error compiling BranchMatch regexp %q: %v", *filter.BranchMatch, err) + } + } + outFilters = append(outFilters, outFilter) + } + return outFilters, nil +} + +func matchFilter(pullRequest *PullRequest, filter *Filter) bool { + if filter.BranchMatch != nil && !filter.BranchMatch.MatchString(pullRequest.Branch) { + return false + } + + return true +} + +func ListPullRequests(ctx context.Context, provider PullRequestService, filters []argoprojiov1alpha1.PullRequestGeneratorFilter) ([]*PullRequest, error) { + compiledFilters, err := compileFilters(filters) + if err != nil { + return nil, err + } + + pullRequests, err := provider.List(ctx) + if err != nil { + return nil, err + } + + if len(compiledFilters) == 0 { + return pullRequests, nil + } + + filteredPullRequests := make([]*PullRequest, 0, len(pullRequests)) + for _, pullRequest := range pullRequests { + for _, filter := range compiledFilters { + matches := matchFilter(pullRequest, filter) + if matches { + filteredPullRequests = append(filteredPullRequests, pullRequest) + break + } + } + } + + return filteredPullRequests, nil +} diff --git a/pkg/services/pull_request/utils_test.go b/pkg/services/pull_request/utils_test.go new file mode 100644 index 00000000..2f65b9cc --- /dev/null +++ b/pkg/services/pull_request/utils_test.go @@ -0,0 +1,139 @@ +package pull_request + +import ( + "context" + "testing" + + argoprojiov1alpha1 "github.com/argoproj/applicationset/api/v1alpha1" + "github.com/stretchr/testify/assert" +) + +func strp(s string) *string { + return &s +} +func TestFilterBranchMatchBadRegexp(t *testing.T) { + provider, _ := NewFakeService( + context.Background(), + []*PullRequest{ + { + Number: 1, + Branch: "branch1", + HeadSHA: "089d92cbf9ff857a39e6feccd32798ca700fb958", + }, + }, + nil, + ) + filters := []argoprojiov1alpha1.PullRequestGeneratorFilter{ + { + BranchMatch: strp("("), + }, + } + _, err := ListPullRequests(context.Background(), provider, filters) + assert.Error(t, err) +} + +func TestFilterBranchMatch(t *testing.T) { + provider, _ := NewFakeService( + context.Background(), + []*PullRequest{ + { + Number: 1, + Branch: "one", + HeadSHA: "189d92cbf9ff857a39e6feccd32798ca700fb958", + }, + { + Number: 2, + Branch: "two", + HeadSHA: "289d92cbf9ff857a39e6feccd32798ca700fb958", + }, + { + Number: 3, + Branch: "three", + HeadSHA: "389d92cbf9ff857a39e6feccd32798ca700fb958", + }, + { + Number: 4, + Branch: "four", + HeadSHA: "489d92cbf9ff857a39e6feccd32798ca700fb958", + }, + }, + nil, + ) + filters := []argoprojiov1alpha1.PullRequestGeneratorFilter{ + { + BranchMatch: strp("w"), + }, + } + pullRequests, err := ListPullRequests(context.Background(), provider, filters) + assert.NoError(t, err) + assert.Len(t, pullRequests, 1) + assert.Equal(t, "two", pullRequests[0].Branch) +} + +func TestMultiFilterOr(t *testing.T) { + provider, _ := NewFakeService( + context.Background(), + []*PullRequest{ + { + Number: 1, + Branch: "one", + HeadSHA: "189d92cbf9ff857a39e6feccd32798ca700fb958", + }, + { + Number: 2, + Branch: "two", + HeadSHA: "289d92cbf9ff857a39e6feccd32798ca700fb958", + }, + { + Number: 3, + Branch: "three", + HeadSHA: "389d92cbf9ff857a39e6feccd32798ca700fb958", + }, + { + Number: 4, + Branch: "four", + HeadSHA: "489d92cbf9ff857a39e6feccd32798ca700fb958", + }, + }, + nil, + ) + filters := []argoprojiov1alpha1.PullRequestGeneratorFilter{ + { + BranchMatch: strp("w"), + }, + { + BranchMatch: strp("r"), + }, + } + pullRequests, err := ListPullRequests(context.Background(), provider, filters) + assert.NoError(t, err) + assert.Len(t, pullRequests, 3) + assert.Equal(t, "two", pullRequests[0].Branch) + assert.Equal(t, "three", pullRequests[1].Branch) + assert.Equal(t, "four", pullRequests[2].Branch) +} + +func TestNoFilters(t *testing.T) { + provider, _ := NewFakeService( + context.Background(), + []*PullRequest{ + { + Number: 1, + Branch: "one", + HeadSHA: "189d92cbf9ff857a39e6feccd32798ca700fb958", + }, + { + Number: 2, + Branch: "two", + HeadSHA: "289d92cbf9ff857a39e6feccd32798ca700fb958", + }, + }, + nil, + ) + filters := []argoprojiov1alpha1.PullRequestGeneratorFilter{} + repos, err := ListPullRequests(context.Background(), provider, filters) + assert.NoError(t, err) + assert.Len(t, repos, 2) + assert.Equal(t, "one", repos[0].Branch) + assert.Equal(t, "two", repos[1].Branch) +} From c121f441a810ec6067771c86999eaf4c4e0c0795 Mon Sep 17 00:00:00 2001 From: mlosicki Date: Thu, 10 Mar 2022 22:48:57 +0000 Subject: [PATCH 13/14] fix: handle no default branch Signed-off-by: mlosicki --- pkg/services/scm_provider/bitbucket_server.go | 11 +++ .../scm_provider/bitbucket_server_test.go | 78 +++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/pkg/services/scm_provider/bitbucket_server.go b/pkg/services/scm_provider/bitbucket_server.go index 8790d86a..823cf85f 100644 --- a/pkg/services/scm_provider/bitbucket_server.go +++ b/pkg/services/scm_provider/bitbucket_server.go @@ -79,6 +79,10 @@ func (b *BitbucketServerProvider) ListRepos(_ context.Context, cloneProtocol str if err != nil { return nil, err } + if branch == nil { + log.Debugf("%s/%s does not have a default branch, skipping", org, repo) + continue + } repos = append(repos, &Repository{ Organization: org, @@ -159,6 +163,9 @@ func (b *BitbucketServerProvider) listBranches(repo *Repository) ([]bitbucketv1. if err != nil { return nil, err } + if branch == nil { + return []bitbucketv1.Branch{}, nil + } return []bitbucketv1.Branch{*branch}, nil } // Otherwise, scrape the GetBranches API. @@ -190,6 +197,10 @@ func (b *BitbucketServerProvider) listBranches(repo *Repository) ([]bitbucketv1. func (b *BitbucketServerProvider) getDefaultBranch(org string, repo string) (*bitbucketv1.Branch, error) { response, err := b.client.DefaultApi.GetDefaultBranch(org, repo) + if response != nil && response.StatusCode == 404 { + // There's no default branch i.e. empty repo, not an error + return nil, nil + } if err != nil { return nil, err } diff --git a/pkg/services/scm_provider/bitbucket_server_test.go b/pkg/services/scm_provider/bitbucket_server_test.go index df5bd1cd..f60bd366 100644 --- a/pkg/services/scm_provider/bitbucket_server_test.go +++ b/pkg/services/scm_provider/bitbucket_server_test.go @@ -342,6 +342,51 @@ func TestGetBranchesDefaultOnly(t *testing.T) { }, *repos[0]) } +func TestGetBranchesMissingDefault(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.Header.Get("Authorization")) + switch r.RequestURI { + case "/rest/api/1.0/projects/PROJECT/repos/REPO/branches/default": + http.Error(w, "Not found", 404) + } + defaultHandler(t)(w, r) + })) + defer ts.Close() + provider, err := NewBitbucketServerProviderNoAuth(context.Background(), ts.URL, "PROJECT", false) + assert.NoError(t, err) + repos, err := provider.GetBranches(context.Background(), &Repository{ + Organization: "PROJECT", + Repository: "REPO", + URL: "ssh://git@mycompany.bitbucket.org/PROJECT/REPO.git", + Labels: []string{}, + RepositoryId: 1, + }) + assert.NoError(t, err) + assert.Empty(t, repos) +} + +func TestGetBranchesErrorDefaultBranch(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.Header.Get("Authorization")) + switch r.RequestURI { + case "/rest/api/1.0/projects/PROJECT/repos/REPO/branches/default": + http.Error(w, "Internal server error", 500) + } + defaultHandler(t)(w, r) + })) + defer ts.Close() + provider, err := NewBitbucketServerProviderNoAuth(context.Background(), ts.URL, "PROJECT", false) + assert.NoError(t, err) + _, err = provider.GetBranches(context.Background(), &Repository{ + Organization: "PROJECT", + Repository: "REPO", + URL: "ssh://git@mycompany.bitbucket.org/PROJECT/REPO.git", + Labels: []string{}, + RepositoryId: 1, + }) + assert.Error(t, err) +} + func TestListReposBasicAuth(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, "Basic dXNlcjpwYXNzd29yZA==", r.Header.Get("Authorization")) @@ -392,6 +437,39 @@ func TestListReposDefaultBranch(t *testing.T) { }, *repos[0]) } +func TestListReposMissingDefaultBranch(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.Header.Get("Authorization")) + switch r.RequestURI { + case "/rest/api/1.0/projects/PROJECT/repos/REPO/branches/default": + http.Error(w, "Not found", 404) + } + defaultHandler(t)(w, r) + })) + defer ts.Close() + provider, err := NewBitbucketServerProviderNoAuth(context.Background(), ts.URL, "PROJECT", false) + assert.NoError(t, err) + repos, err := provider.ListRepos(context.Background(), "ssh") + assert.NoError(t, err) + assert.Empty(t, repos) +} + +func TestListReposErrorDefaultBranch(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.Header.Get("Authorization")) + switch r.RequestURI { + case "/rest/api/1.0/projects/PROJECT/repos/REPO/branches/default": + http.Error(w, "Internal server error", 500) + } + defaultHandler(t)(w, r) + })) + defer ts.Close() + provider, err := NewBitbucketServerProviderNoAuth(context.Background(), ts.URL, "PROJECT", false) + assert.NoError(t, err) + _, err = provider.ListRepos(context.Background(), "ssh") + assert.Error(t, err) +} + func TestListReposCloneProtocol(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assert.Empty(t, r.Header.Get("Authorization")) From fb62af0e3ed0993f1186bfdc86aa94ac3562e7b6 Mon Sep 17 00:00:00 2001 From: mlosicki Date: Sun, 13 Mar 2022 15:35:33 +0100 Subject: [PATCH 14/14] fix: refactor RepoHasPath Signed-off-by: mlosicki --- pkg/services/scm_provider/bitbucket_server.go | 20 ++------- .../scm_provider/bitbucket_server_test.go | 45 +++++++++++-------- 2 files changed, 29 insertions(+), 36 deletions(-) diff --git a/pkg/services/scm_provider/bitbucket_server.go b/pkg/services/scm_provider/bitbucket_server.go index 823cf85f..6ab72a2a 100644 --- a/pkg/services/scm_provider/bitbucket_server.go +++ b/pkg/services/scm_provider/bitbucket_server.go @@ -107,28 +107,14 @@ func (b *BitbucketServerProvider) RepoHasPath(_ context.Context, repo *Repositor opts := map[string]interface{}{ "limit": 100, "at": repo.Branch, + "type_": true, } // No need to query for all pages here - response, err := b.client.DefaultApi.StreamFiles_42(repo.Organization, repo.Repository, path, opts) + response, err := b.client.DefaultApi.GetContent_0(repo.Organization, repo.Repository, path, opts) if response != nil && response.StatusCode == 404 { - // The path requested does not exist at the supplied commit. + // File/directory not found return false, nil } - if response != nil && response.StatusCode == 400 { - // If the path is a file, the first call will return 400: The path requested is not a directory at the supplied commit. - // Simply retry with an API call that works with files and expect a 200 return code - opts["type_"] = true // Only request the type, we don't need the content - response, err := b.client.DefaultApi.GetContent_0(repo.Organization, repo.Repository, path, opts) - if response != nil && response.StatusCode == 404 { - // File not found - return false, nil - } - if err != nil { - return false, err - } - // 200 ok - return true, nil - } if err != nil { return false, err } diff --git a/pkg/services/scm_provider/bitbucket_server_test.go b/pkg/services/scm_provider/bitbucket_server_test.go index f60bd366..986e03a8 100644 --- a/pkg/services/scm_provider/bitbucket_server_test.go +++ b/pkg/services/scm_provider/bitbucket_server_test.go @@ -508,28 +508,25 @@ func TestBitbucketServerHasPath(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var err error switch r.RequestURI { - case "/rest/api/1.0/projects/PROJECT/repos/REPO/files/pkg/?at=main&limit=100": - _, err = io.WriteString(w, `{ - "size": 1, - "limit": 100, - "isLastPage": true, - "values": [ - "pkg/file.txt" - ], - "start": 0 - }`) - - case "/rest/api/1.0/projects/PROJECT/repos/REPO/files/anotherpkg/file.txt?at=main&limit=100": - http.Error(w, "The path requested is not a directory at the supplied commit.", 400) + case "/rest/api/1.0/projects/PROJECT/repos/REPO/browse/pkg?at=main&limit=100&type=true": + _, err = io.WriteString(w, `{"type":"DIRECTORY"}`) + case "/rest/api/1.0/projects/PROJECT/repos/REPO/browse/pkg/?at=main&limit=100&type=true": + _, err = io.WriteString(w, `{"type":"DIRECTORY"}`) case "/rest/api/1.0/projects/PROJECT/repos/REPO/browse/anotherpkg/file.txt?at=main&limit=100&type=true": _, err = io.WriteString(w, `{"type":"FILE"}`) - case "/rest/api/1.0/projects/PROJECT/repos/REPO/files/anotherpkg/missing.txt?at=main&limit=100": - http.Error(w, "The path requested is not a directory at the supplied commit.", 400) + case "/rest/api/1.0/projects/PROJECT/repos/REPO/browse/anotherpkg/missing.txt?at=main&limit=100&type=true": http.Error(w, "The path \"anotherpkg/missing.txt\" does not exist at revision \"main\"", 404) + case "/rest/api/1.0/projects/PROJECT/repos/REPO/browse/notathing?at=main&limit=100&type=true": + http.Error(w, "The path \"notathing\" does not exist at revision \"main\"", 404) + + case "/rest/api/1.0/projects/PROJECT/repos/REPO/browse/return-redirect?at=main&limit=100&type=true": + http.Redirect(w, r, "http://"+r.Host+"/rest/api/1.0/projects/PROJECT/repos/REPO/browse/redirected?at=main&limit=100&type=true", 301) + case "/rest/api/1.0/projects/PROJECT/repos/REPO/browse/redirected?at=main&limit=100&type=true": + _, err = io.WriteString(w, `{"type":"DIRECTORY"}`) - case "/rest/api/1.0/projects/PROJECT/repos/REPO/files/notathing/?at=main&limit=100": - http.Error(w, "The path requested does not exist at the supplied commit.", 404) + case "/rest/api/1.0/projects/PROJECT/repos/REPO/browse/unauthorized-response?at=main&limit=100&type=true": + http.Error(w, "Authentication failed", 401) default: t.Fail() @@ -546,7 +543,11 @@ func TestBitbucketServerHasPath(t *testing.T) { Repository: "REPO", Branch: "main", } - ok, err := provider.RepoHasPath(context.Background(), repo, "pkg/") + ok, err := provider.RepoHasPath(context.Background(), repo, "pkg") + assert.NoError(t, err) + assert.True(t, ok) + + ok, err = provider.RepoHasPath(context.Background(), repo, "pkg/") assert.NoError(t, err) assert.True(t, ok) @@ -558,8 +559,14 @@ func TestBitbucketServerHasPath(t *testing.T) { assert.NoError(t, err) assert.False(t, ok) - ok, err = provider.RepoHasPath(context.Background(), repo, "notathing/") + ok, err = provider.RepoHasPath(context.Background(), repo, "notathing") assert.NoError(t, err) assert.False(t, ok) + ok, err = provider.RepoHasPath(context.Background(), repo, "return-redirect") + assert.NoError(t, err) + assert.True(t, ok) + + _, err = provider.RepoHasPath(context.Background(), repo, "unauthorized-response") + assert.Error(t, err) }