diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..c0763f74e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,88 @@ +--- +name: Bug Report +about: You're experiencing an issue with the Consul API Gateway that is different than the documented behavior. +labels: bug + +--- + + + +### Community Note + +* Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request. Searching for pre-existing feature requests helps us consolidate datapoints for identical requirements into a single place, thank you! +* Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request. +* If you are interested in working on this issue or have submitted a pull request, please leave a comment. + + + +--- + + + +### Overview of the Issue + + + +### Reproduction Steps + + + +### Logs + + + +### Expected behavior + + + +### Environment details + + + + +### Additional Context + + diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..2c9726afd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Consul Discuss Forum + url: https://discuss.hashicorp.com/c/consul + about: Please check out our discussion forum. Ask a question or see if yours has already been answered there. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..7c38499a0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,34 @@ +--- +name: Feature Request +about: If you have something you think the Consul API Gateway could improve or add support for. +labels: enhancement + +--- + + + +### Community Note + +* Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request. Searching for pre-existing feature requests helps us consolidate datapoints for identical requirements into a single place, thank you! +* Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request. +* If you are interested in working on this issue or have submitted a pull request, please leave a comment. + + + +--- + +#### Is your feature request related to a problem? Please describe. + + + +#### Feature Description + + + +#### Use Case(s) + + + +#### Contributions + + \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..4cbef6480 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,14 @@ +Changes proposed in this PR: +- +- + +How I've tested this PR: + +How I expect reviewers to test this PR: + + +Checklist: +- [ ] Tests added +- [ ] CHANGELOG entry added + > HashiCorp engineers only, community PRs should not add a changelog entry. + > Entries should use present tense (e.g. Add support for...) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3dbf39f92..f158b2610 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -98,4 +98,4 @@ jobs: target: default arch: ${{matrix.arch}} tags: | - docker.io/hashicorp/${{env.repo}}:${{env.version}} \ No newline at end of file + docker.io/hashicorp/${{env.repo}}:${{env.version}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0690d1b2f..f6a3b4d8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,8 @@ name: ci on: - push: + push: branches: ["main"] - pull_request: + pull_request: branches: ["main", "release/**"] env: GO_VERSION: '1.16' @@ -31,7 +31,7 @@ jobs: name: unit test (consul-version=${{ matrix.consul-version }}) strategy: matrix: - consul-version: [1.11.0-beta2, 1.11.0+ent-beta2] + consul-version: [1.11.2, 1.11.2+ent] runs-on: ubuntu-latest env: TEST_RESULTS_DIR: /tmp/test-results/consul@${{ matrix.consul-version }} diff --git a/.release/ci.hcl b/.release/ci.hcl index 5528c80a6..3063efc2c 100644 --- a/.release/ci.hcl +++ b/.release/ci.hcl @@ -40,8 +40,36 @@ event "upload-dev" { } } -event "sign" { +event "security-scan-binaries" { depends = ["upload-dev"] + action "security-scan-binaries" { + organization = "hashicorp" + repository = "crt-workflows-common" + workflow = "security-scan-binaries" + config = "security-scan.hcl" + } + + notification { + on = "fail" + } +} + +event "security-scan-containers" { + depends = ["security-scan-binaries"] + action "security-scan-containers" { + organization = "hashicorp" + repository = "crt-workflows-common" + workflow = "security-scan-containers" + config = "security-scan.hcl" + } + + notification { + on = "fail" + } +} + +event "sign" { + depends = ["security-scan-containers"] action "sign" { organization = "hashicorp" repository = "crt-workflows-common" @@ -64,4 +92,4 @@ event "verify" { notification { on = "always" } -} \ No newline at end of file +} diff --git a/.release/security-scan.hcl b/.release/security-scan.hcl new file mode 100644 index 000000000..3fd4ef388 --- /dev/null +++ b/.release/security-scan.hcl @@ -0,0 +1,13 @@ +container { + dependencies = true + alpine_secdb = true + secrets = true +} + +binary { + secrets = true + go_modules = true + osv = true + oss_index = true + nvd = true +} diff --git a/README.md b/README.md index 12c585667..651733825 100644 --- a/README.md +++ b/README.md @@ -2,24 +2,27 @@ # Overview -The Consul API Gateway implements a North/South managed gateway that integrates natively with the Consul Service Mesh. Currently this -is implemented as a Kubernetes Gateway Controller, but is meant to eventually work across multiple scheduler and runtime ecosystems. +The Consul API Gateway is a dedicated ingress solution for intelligently routing traffic to applications +running on a Consul Service Mesh. Currently it only runs on Kubernetes and is implemented as a +Kubernetes Gateway Controller but, in future releases, it will work across multiple scheduler and +runtime ecosystems. # Usage -The Consul API Gateway project Kubernetes integration leverages connect-injected services managed by the -[Consul K8s](https://github.com/hashicorp/consul-k8s) project. To use this project, make sure you have a running Kubernetes cluster and -Consul 1.11 or greater installed [via Helm](https://github.com/hashicorp/consul-k8s#usage) with Connect injection support enabled. +## Prerequisites -Our default `kustomization` manifests also assume that the Consul helm chart has TLS enabled. To install a compatible Consul instance via -Helm, you can run the following commands: +The Consul API Gateway must be installed on a Kubernetes cluster with the [Consul K8s](https://github.com/hashicorp/consul-k8s) service +mesh deployed on it. The installed version of Consul must be `v1.11-beta2` or greater. + +The Consul Helm chart must be used, with specific settings, to install Consul on the Kubernetes +cluster. This can be done with the following commands: ```bash helm repo add hashicorp https://helm.releases.hashicorp.com -cat <&1 > /dev/null + helm install consul hashicorp/consul --version 0.39.0 -f dev/config/helm/consul.yaml 2>&1 > /dev/null echo "Waiting for consul to stabilize" sleep 10 kubectl wait --for=condition=ready pod --selector=app=consul,component=server,release=consul --timeout=90s diff --git a/go.mod b/go.mod index 488383a64..aa6757dd4 100644 --- a/go.mod +++ b/go.mod @@ -13,8 +13,8 @@ require ( github.com/golang/mock v1.6.0 github.com/google/uuid v1.2.0 github.com/gorilla/mux v1.8.0 // indirect - github.com/hashicorp/consul/api v1.10.1-0.20210924170522-581357c32a29 - github.com/hashicorp/consul/sdk v0.7.0 + github.com/hashicorp/consul/api v1.12.1-0.20220111183205-dcf1cd485363 + github.com/hashicorp/consul/sdk v0.8.0 github.com/hashicorp/go-hclog v0.16.2 github.com/hashicorp/go-multierror v1.1.0 github.com/mitchellh/cli v1.1.2 diff --git a/go.sum b/go.sum index 9dc0a1281..90f0d255a 100644 --- a/go.sum +++ b/go.sum @@ -485,9 +485,13 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.10.1-0.20210924170522-581357c32a29 h1:Ty8MSJh/nwRGYMlFaRq+/vPVJ4ez6HznxZlOxbgESdE= github.com/hashicorp/consul/api v1.10.1-0.20210924170522-581357c32a29/go.mod h1:sDjTOq0yUyv5G4h+BqSea7Fn6BU+XbolEz1952UB+mk= +github.com/hashicorp/consul/api v1.12.1-0.20220111183205-dcf1cd485363 h1:MHyZwMbb1vGW/Xbt1lFlZy7NJhQQ+64bwMu9y29V20s= +github.com/hashicorp/consul/api v1.12.1-0.20220111183205-dcf1cd485363/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.7.0 h1:H6R9d008jDcHPQPAqPNuydAshJ4v5/8URdFnUvK/+sc= github.com/hashicorp/consul/sdk v0.7.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM= +github.com/hashicorp/consul/sdk v0.8.0 h1:OJtKBtEjboEZvG6AOUdh4Z1Zbyu0WcxQ0qatRrZHTVU= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -524,12 +528,16 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/memberlist v0.2.2 h1:5+RffWKwqJ71YPu9mWsF7ZOscZmwfasdA8kbdC7AO2g= github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.9.5 h1:EBWvyu9tcRszt3Bxp3KNssBMP1KuHWyO51lz9+786iM= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hashicorp/serf v0.9.6 h1:uuEX1kLR6aoda1TBttmJQKDLZE1Ob7KN0NPdE7EtCDc= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= @@ -613,6 +621,7 @@ github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88J github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -1015,6 +1024,7 @@ golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a h1:bRuuGXV8wwSdGTB+CtJf+FjgO1APK1CoO39T4BN/XBw= @@ -1126,6 +1136,7 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/internal/adapters/consul/http.go b/internal/adapters/consul/http.go new file mode 100644 index 000000000..1bfe69e31 --- /dev/null +++ b/internal/adapters/consul/http.go @@ -0,0 +1,262 @@ +package consul + +import ( + "fmt" + "hash/crc32" + "sort" + "strconv" + + "github.com/hashicorp/consul-api-gateway/internal/core" + "github.com/hashicorp/consul/api" +) + +// httpRouteToServiceDiscoChain will convert a k8s HTTPRoute to a Consul service-router config entry and 0 or +// more service-splitter config entries. A prefix can be given to prefix all config entry names with. +func httpRouteDiscoveryChain(route core.HTTPRoute) (*api.ServiceRouterConfigEntry, []*api.ServiceSplitterConfigEntry) { + router := &api.ServiceRouterConfigEntry{ + Kind: api.ServiceRouter, + Name: route.GetName(), + Meta: route.GetMeta(), + Namespace: route.GetNamespace(), + } + var splitters []*api.ServiceSplitterConfigEntry + + for idx, rule := range route.Rules { + modifier := httpRouteFiltersToServiceRouteHeaderModifier(rule.Filters) + + var destination core.ResolvedService + if len(rule.Services) == 1 { + destination = rule.Services[0].Service + serviceModifier := httpRouteFiltersToServiceRouteHeaderModifier(rule.Services[0].Filters) + modifier.Add = mergeMaps(modifier.Add, serviceModifier.Add) + modifier.Set = mergeMaps(modifier.Set, serviceModifier.Set) + modifier.Remove = append(modifier.Remove, serviceModifier.Remove...) + } else { + // create a virtual service to split + destination = core.ResolvedService{ + Service: fmt.Sprintf("%s-%d", route.GetName(), idx), + ConsulNamespace: route.GetNamespace(), + } + splitter := &api.ServiceSplitterConfigEntry{ + Kind: api.ServiceSplitter, + Name: destination.Service, + Namespace: destination.ConsulNamespace, + Splits: []api.ServiceSplit{}, + Meta: route.GetMeta(), + } + + totalWeight := int32(0) + for _, service := range rule.Services { + totalWeight += service.Weight + } + + for _, service := range rule.Services { + if service.Weight == 0 { + continue + } + + modifier := httpRouteFiltersToServiceRouteHeaderModifier(service.Filters) + + weightPercentage := float32(service.Weight) / float32(totalWeight) + split := api.ServiceSplit{ + RequestHeaders: modifier, + Weight: weightPercentage * 100, + } + split.Service = service.Service.Service + split.Namespace = service.Service.ConsulNamespace + splitter.Splits = append(splitter.Splits, split) + } + if len(splitter.Splits) > 0 { + splitters = append(splitters, splitter) + } + } + + // for each match rule a ServiceRoute is created for the service-router + // if there are no rules a single route with the destination is set + if len(rule.Matches) == 0 { + router.Routes = append(router.Routes, api.ServiceRoute{ + Destination: &api.ServiceRouteDestination{ + Service: destination.Service, + RequestHeaders: modifier, + Namespace: destination.ConsulNamespace, + }, + }) + } + for _, match := range rule.Matches { + router.Routes = append(router.Routes, api.ServiceRoute{ + Match: &api.ServiceRouteMatch{HTTP: httpRouteMatchToServiceRouteHTTPMatch(match)}, + Destination: &api.ServiceRouteDestination{ + Service: destination.Service, + RequestHeaders: modifier, + Namespace: destination.ConsulNamespace, + }, + }) + } + } + + return router, splitters +} + +func httpRouteFiltersToServiceRouteHeaderModifier(filters []core.HTTPFilter) *api.HTTPHeaderModifiers { + modifier := &api.HTTPHeaderModifiers{ + Add: make(map[string]string), + Set: make(map[string]string), + } + for _, filter := range filters { + switch filter.Type { + case core.HTTPHeaderFilterType: + // If we have multiple filters specified, then we can potentially clobber + // "Add" and "Set" here -- as far as K8S gateway spec is concerned, this + // is all implmentation-specific behavior and undefined by the spec. + modifier.Add = mergeMaps(modifier.Add, filter.Header.Add) + modifier.Set = mergeMaps(modifier.Set, filter.Header.Set) + modifier.Remove = append(modifier.Remove, filter.Header.Remove...) + } + } + return modifier +} + +func mergeMaps(a, b map[string]string) map[string]string { + for k, v := range b { + a[k] = v + } + return a +} + +func httpRouteMatchToServiceRouteHTTPMatch(match core.HTTPMatch) *api.ServiceRouteHTTPMatch { + var consulMatch api.ServiceRouteHTTPMatch + switch match.Path.Type { + case core.HTTPPathMatchExactType: + consulMatch.PathExact = match.Path.Value + case core.HTTPPathMatchPrefixType: + consulMatch.PathPrefix = match.Path.Value + case core.HTTPPathMatchRegularExpressionType: + consulMatch.PathRegex = match.Path.Value + } + + for _, header := range match.Headers { + switch header.Type { + case core.HTTPHeaderMatchExactType: + consulMatch.Header = append(consulMatch.Header, api.ServiceRouteHTTPMatchHeader{ + Name: header.Name, + Exact: header.Value, + }) + case core.HTTPHeaderMatchPrefixType: + consulMatch.Header = append(consulMatch.Header, api.ServiceRouteHTTPMatchHeader{ + Name: header.Name, + Prefix: header.Value, + }) + case core.HTTPHeaderMatchSuffixType: + consulMatch.Header = append(consulMatch.Header, api.ServiceRouteHTTPMatchHeader{ + Name: header.Name, + Suffix: header.Value, + }) + case core.HTTPHeaderMatchPresentType: + consulMatch.Header = append(consulMatch.Header, api.ServiceRouteHTTPMatchHeader{ + Name: header.Name, + Present: true, + }) + case core.HTTPHeaderMatchRegularExpressionType: + consulMatch.Header = append(consulMatch.Header, api.ServiceRouteHTTPMatchHeader{ + Name: header.Name, + Regex: header.Value, + }) + } + } + + for _, query := range match.Query { + switch query.Type { + case core.HTTPQueryMatchExactType: + consulMatch.QueryParam = append(consulMatch.QueryParam, api.ServiceRouteHTTPMatchQueryParam{ + Name: query.Name, + Exact: query.Value, + }) + case core.HTTPQueryMatchPresentType: + consulMatch.QueryParam = append(consulMatch.QueryParam, api.ServiceRouteHTTPMatchQueryParam{ + Name: query.Name, + Present: true, + }) + case core.HTTPQueryMatchRegularExpressionType: + consulMatch.QueryParam = append(consulMatch.QueryParam, api.ServiceRouteHTTPMatchQueryParam{ + Name: query.Name, + Regex: query.Value, + }) + } + } + + switch match.Method { + case core.HTTPMethodConnect: + consulMatch.Methods = append(consulMatch.Methods, "CONNECT") + case core.HTTPMethodDelete: + consulMatch.Methods = append(consulMatch.Methods, "DELETE") + case core.HTTPMethodGet: + consulMatch.Methods = append(consulMatch.Methods, "GET") + case core.HTTPMethodHead: + consulMatch.Methods = append(consulMatch.Methods, "HEAD") + case core.HTTPMethodOptions: + consulMatch.Methods = append(consulMatch.Methods, "OPTIONS") + case core.HTTPMethodPatch: + consulMatch.Methods = append(consulMatch.Methods, "PATCH") + case core.HTTPMethodPost: + consulMatch.Methods = append(consulMatch.Methods, "POST") + case core.HTTPMethodPut: + consulMatch.Methods = append(consulMatch.Methods, "PUT") + case core.HTTPMethodTrace: + consulMatch.Methods = append(consulMatch.Methods, "TRACE") + } + + return &consulMatch +} + +func hostsKey(hosts []string) string { + sort.Strings(hosts) + hostsHash := crc32.NewIEEE() + for _, h := range hosts { + if _, err := hostsHash.Write([]byte(h)); err != nil { + continue + } + } + return strconv.FormatUint(uint64(hostsHash.Sum32()), 16) +} + +func compareHTTPRules(ruleA, ruleB core.HTTPRouteRule) bool { + matchesA := ruleA.Matches + matchesB := ruleB.Matches + + // this tries to implement some of the logic specified by the K8S gateway API spec + + // Proxy or Load Balancer routing configuration generated from HTTPRoutes MUST prioritize + // rules based on the following criteria, continuing on ties. Precedence must be given + // to the the Rule with the largest number of: + // Characters in a matching non-wildcard hostname. + // Characters in a matching hostname. + // Characters in a matching path. + // Header matches. + // Query param matches. + + var longestPathMatchA int + for _, match := range matchesA { + pathLength := len(match.Path.Value) + if longestPathMatchA < pathLength { + longestPathMatchA = pathLength + } + } + var longestPathMatchB int + for _, match := range matchesB { + pathLength := len(match.Path.Value) + if longestPathMatchB < pathLength { + longestPathMatchB = pathLength + } + } + return longestPathMatchA > longestPathMatchB +} + +func httpServiceDefault(entry api.ConfigEntry, meta map[string]string) *api.ServiceConfigEntry { + return &api.ServiceConfigEntry{ + Kind: api.ServiceDefaults, + Name: entry.GetName(), + Namespace: entry.GetNamespace(), + Protocol: "http", + Meta: meta, + } +} diff --git a/internal/adapters/consul/sync.go b/internal/adapters/consul/sync.go index 3519d9ce0..960837978 100644 --- a/internal/adapters/consul/sync.go +++ b/internal/adapters/consul/sync.go @@ -4,12 +4,11 @@ import ( "context" "encoding/json" "fmt" - "hash/crc32" "sort" - "strconv" "sync" "time" + "github.com/hashicorp/consul-api-gateway/internal/common" "github.com/hashicorp/consul-api-gateway/internal/consul" "github.com/hashicorp/consul-api-gateway/internal/core" "github.com/hashicorp/consul/api" @@ -65,214 +64,6 @@ func (a *ConsulSyncAdapter) deleteConfigEntries(ctx context.Context, entries ... return result } -// httpRouteToServiceDiscoChain will convert a k8s HTTPRoute to a Consul service-router config entry and 0 or -// more service-splitter config entries. A prefix can be given to prefix all config entry names with. -func httpRouteDiscoveryChain(route core.HTTPRoute) (*api.ServiceRouterConfigEntry, []*api.ServiceSplitterConfigEntry) { - router := &api.ServiceRouterConfigEntry{ - Kind: api.ServiceRouter, - Name: route.GetName(), - Meta: route.GetMeta(), - Namespace: route.GetNamespace(), - } - var splitters []*api.ServiceSplitterConfigEntry - - for idx, rule := range route.Rules { - modifier := httpRouteFiltersToServiceRouteHeaderModifier(rule.Filters) - - var destination core.ResolvedService - if len(rule.Services) == 1 { - destination = rule.Services[0].Service - serviceModifier := httpRouteFiltersToServiceRouteHeaderModifier(rule.Services[0].Filters) - modifier.Add = mergeMaps(modifier.Add, serviceModifier.Add) - modifier.Set = mergeMaps(modifier.Set, serviceModifier.Set) - modifier.Remove = append(modifier.Remove, serviceModifier.Remove...) - } else { - // create a virtual service to split - destination = core.ResolvedService{ - Service: fmt.Sprintf("%s-%d", route.GetName(), idx), - ConsulNamespace: route.GetNamespace(), - } - splitter := &api.ServiceSplitterConfigEntry{ - Kind: api.ServiceSplitter, - Name: destination.Service, - Namespace: destination.ConsulNamespace, - Splits: []api.ServiceSplit{}, - Meta: route.GetMeta(), - } - - totalWeight := int32(0) - for _, service := range rule.Services { - totalWeight += service.Weight - } - - for _, service := range rule.Services { - if service.Weight == 0 { - continue - } - - modifier := httpRouteFiltersToServiceRouteHeaderModifier(service.Filters) - - weightPercentage := float32(service.Weight) / float32(totalWeight) - split := api.ServiceSplit{ - RequestHeaders: modifier, - Weight: weightPercentage * 100, - } - split.Service = service.Service.Service - split.Namespace = service.Service.ConsulNamespace - splitter.Splits = append(splitter.Splits, split) - } - if len(splitter.Splits) > 0 { - splitters = append(splitters, splitter) - } - } - - // for each match rule a ServiceRoute is created for the service-router - // if there are no rules a single route with the destination is set - if len(rule.Matches) == 0 { - router.Routes = append(router.Routes, api.ServiceRoute{ - Destination: &api.ServiceRouteDestination{ - Service: destination.Service, - RequestHeaders: modifier, - Namespace: destination.ConsulNamespace, - }, - }) - } - for _, match := range rule.Matches { - router.Routes = append(router.Routes, api.ServiceRoute{ - Match: &api.ServiceRouteMatch{HTTP: httpRouteMatchToServiceRouteHTTPMatch(match)}, - Destination: &api.ServiceRouteDestination{ - Service: destination.Service, - RequestHeaders: modifier, - Namespace: destination.ConsulNamespace, - }, - }) - } - } - - return router, splitters -} - -func httpRouteFiltersToServiceRouteHeaderModifier(filters []core.HTTPFilter) *api.HTTPHeaderModifiers { - modifier := &api.HTTPHeaderModifiers{ - Add: make(map[string]string), - Set: make(map[string]string), - } - for _, filter := range filters { - switch filter.Type { - case core.HTTPHeaderFilterType: - // If we have multiple filters specified, then we can potentially clobber - // "Add" and "Set" here -- as far as K8S gateway spec is concerned, this - // is all implmentation-specific behavior and undefined by the spec. - modifier.Add = mergeMaps(modifier.Add, filter.Header.Add) - modifier.Set = mergeMaps(modifier.Set, filter.Header.Set) - modifier.Remove = append(modifier.Remove, filter.Header.Remove...) - } - } - return modifier -} - -func mergeMaps(a, b map[string]string) map[string]string { - for k, v := range b { - a[k] = v - } - return a -} - -func httpRouteMatchToServiceRouteHTTPMatch(match core.HTTPMatch) *api.ServiceRouteHTTPMatch { - var consulMatch api.ServiceRouteHTTPMatch - switch match.Path.Type { - case core.HTTPPathMatchExactType: - consulMatch.PathExact = match.Path.Value - case core.HTTPPathMatchPrefixType: - consulMatch.PathPrefix = match.Path.Value - case core.HTTPPathMatchRegularExpressionType: - consulMatch.PathRegex = match.Path.Value - } - - for _, header := range match.Headers { - switch header.Type { - case core.HTTPHeaderMatchExactType: - consulMatch.Header = append(consulMatch.Header, api.ServiceRouteHTTPMatchHeader{ - Name: header.Name, - Exact: header.Value, - }) - case core.HTTPHeaderMatchPrefixType: - consulMatch.Header = append(consulMatch.Header, api.ServiceRouteHTTPMatchHeader{ - Name: header.Name, - Prefix: header.Value, - }) - case core.HTTPHeaderMatchSuffixType: - consulMatch.Header = append(consulMatch.Header, api.ServiceRouteHTTPMatchHeader{ - Name: header.Name, - Suffix: header.Value, - }) - case core.HTTPHeaderMatchPresentType: - consulMatch.Header = append(consulMatch.Header, api.ServiceRouteHTTPMatchHeader{ - Name: header.Name, - Present: true, - }) - case core.HTTPHeaderMatchRegularExpressionType: - consulMatch.Header = append(consulMatch.Header, api.ServiceRouteHTTPMatchHeader{ - Name: header.Name, - Regex: header.Value, - }) - } - } - - for _, query := range match.Query { - switch query.Type { - case core.HTTPQueryMatchExactType: - consulMatch.QueryParam = append(consulMatch.QueryParam, api.ServiceRouteHTTPMatchQueryParam{ - Name: query.Name, - Exact: query.Value, - }) - case core.HTTPQueryMatchPresentType: - consulMatch.QueryParam = append(consulMatch.QueryParam, api.ServiceRouteHTTPMatchQueryParam{ - Name: query.Name, - Present: true, - }) - case core.HTTPQueryMatchRegularExpressionType: - consulMatch.QueryParam = append(consulMatch.QueryParam, api.ServiceRouteHTTPMatchQueryParam{ - Name: query.Name, - Regex: query.Value, - }) - } - } - - switch match.Method { - case core.HTTPMethodConnect: - consulMatch.Methods = append(consulMatch.Methods, "CONNECT") - case core.HTTPMethodDelete: - consulMatch.Methods = append(consulMatch.Methods, "DELETE") - case core.HTTPMethodGet: - consulMatch.Methods = append(consulMatch.Methods, "GET") - case core.HTTPMethodHead: - consulMatch.Methods = append(consulMatch.Methods, "HEAD") - case core.HTTPMethodOptions: - consulMatch.Methods = append(consulMatch.Methods, "OPTIONS") - case core.HTTPMethodPatch: - consulMatch.Methods = append(consulMatch.Methods, "PATCH") - case core.HTTPMethodPost: - consulMatch.Methods = append(consulMatch.Methods, "POST") - case core.HTTPMethodPut: - consulMatch.Methods = append(consulMatch.Methods, "PUT") - case core.HTTPMethodTrace: - consulMatch.Methods = append(consulMatch.Methods, "TRACE") - } - - return &consulMatch -} - -func httpServiceDefault(entry api.ConfigEntry, meta map[string]string) *api.ServiceConfigEntry { - return &api.ServiceConfigEntry{ - Kind: api.ServiceDefaults, - Name: entry.GetName(), - Namespace: entry.GetNamespace(), - Protocol: "http", - Meta: meta, - } -} - func routeDiscoveryChain(route core.ResolvedRoute) (*api.IngressService, *api.ServiceRouterConfigEntry, *consul.ConfigEntryIndex, *consul.ConfigEntryIndex) { meta := route.GetMeta() splitters := consul.NewConfigEntryIndex(api.ServiceSplitter) @@ -296,12 +87,18 @@ func routeDiscoveryChain(route core.ResolvedRoute) (*api.IngressService, *api.Se Hosts: httpRoute.Hostnames, Namespace: httpRoute.GetNamespace(), }, router, splitters, defaults + case core.ResolvedTCPRouteType: + tcpRoute := route.(core.TCPRoute) + return &api.IngressService{ + Name: tcpRoute.Service.Service, + Namespace: tcpRoute.Service.ConsulNamespace, + }, nil, nil, nil default: return nil, nil, nil, nil } } -func mergeRoutes(gateway core.ResolvedGateway, routes []core.ResolvedRoute) []core.ResolvedRoute { +func mergeHTTPRoutes(gateway core.ResolvedGateway, routes []core.ResolvedRoute) []core.ResolvedRoute { merged := map[string]core.HTTPRoute{} unmerged := []core.ResolvedRoute{} for _, route := range routes { @@ -332,47 +129,29 @@ func mergeRoutes(gateway core.ResolvedGateway, routes []core.ResolvedRoute) []co return unmerged } -func hostsKey(hosts []string) string { - sort.Strings(hosts) - hostsHash := crc32.NewIEEE() - for _, h := range hosts { - if _, err := hostsHash.Write([]byte(h)); err != nil { - continue +// filterTCPRoutes makes sure we only have a single TCPRoute for a given listener +func filterTCPRoutes(routes []core.ResolvedRoute) []core.ResolvedRoute { + filtered := []core.ResolvedRoute{} + + found := false + for _, route := range routes { + switch route.GetType() { + case core.ResolvedTCPRouteType: + if !found { + found = true + filtered = append(filtered, route) + } + default: + filtered = append(filtered, route) } } - return strconv.FormatUint(uint64(hostsHash.Sum32()), 16) + + return filtered } -func compareHTTPRules(ruleA, ruleB core.HTTPRouteRule) bool { - matchesA := ruleA.Matches - matchesB := ruleB.Matches - - // this tries to implement some of the logic specified by the K8S gateway API spec - - // Proxy or Load Balancer routing configuration generated from HTTPRoutes MUST prioritize - // rules based on the following criteria, continuing on ties. Precedence must be given - // to the the Rule with the largest number of: - // Characters in a matching non-wildcard hostname. - // Characters in a matching hostname. - // Characters in a matching path. - // Header matches. - // Query param matches. - - var longestPathMatchA int - for _, match := range matchesA { - pathLength := len(match.Path.Value) - if longestPathMatchA < pathLength { - longestPathMatchA = pathLength - } - } - var longestPathMatchB int - for _, match := range matchesB { - pathLength := len(match.Path.Value) - if longestPathMatchB < pathLength { - longestPathMatchB = pathLength - } - } - return longestPathMatchA > longestPathMatchB +func mergeRoutes(gateway core.ResolvedGateway, routes []core.ResolvedRoute) []core.ResolvedRoute { + routes = mergeHTTPRoutes(gateway, routes) + return filterTCPRoutes(routes) } func discoveryChain(gateway core.ResolvedGateway) (*api.IngressGatewayConfigEntry, *consul.ConfigEntryIndex, *consul.ConfigEntryIndex, *consul.ConfigEntryIndex) { @@ -394,7 +173,9 @@ func discoveryChain(gateway core.ResolvedGateway) (*api.IngressGatewayConfigEntr service, router, splits, serviceDefaults := routeDiscoveryChain(route) if service != nil { services = append(services, *service) - routers.Add(router) + if router != nil { + routers.Add(router) + } splitters.Merge(splits) defaults.Merge(serviceDefaults) } @@ -402,12 +183,32 @@ func discoveryChain(gateway core.ResolvedGateway) (*api.IngressGatewayConfigEntr if len(services) > 0 { tls := &api.GatewayTLSConfig{} - if len(listener.Certificates) > 0 { + + if listener.TLS.MinVersion != "" { + tls.TLSMinVersion = listener.TLS.MinVersion + } else { + // set secure default instead of Envoy's TLS 1.0 default + tls.TLSMinVersion = "TLSv1_2" + } + + if listener.TLS.MaxVersion != "" { + tls.TLSMaxVersion = listener.TLS.MaxVersion + } + + if len(listener.TLS.CipherSuites) > 0 { + tls.CipherSuites = listener.TLS.CipherSuites + } else { + // set secure defaults excluding insecure RSA and SHA-1 ciphers pending removal from Envoy + tls.CipherSuites = common.DefaultTLSCipherSuites() + } + + if len(listener.TLS.Certificates) > 0 { tls.SDS = &api.GatewayTLSSDSConfig{ ClusterName: "sds-cluster", - CertResource: listener.Certificates[0], + CertResource: listener.TLS.Certificates[0], } } + ingress.Listeners = append(ingress.Listeners, api.IngressListener{ Port: listener.Port, Protocol: listener.Protocol, diff --git a/internal/adapters/consul/sync_test.go b/internal/adapters/consul/sync_test.go index fd0664686..3f32eec6e 100644 --- a/internal/adapters/consul/sync_test.go +++ b/internal/adapters/consul/sync_test.go @@ -1,14 +1,19 @@ package consul import ( + "context" "encoding/json" "fmt" "os" "path" + "reflect" "testing" + "time" + "github.com/hashicorp/consul-api-gateway/internal/common" "github.com/hashicorp/consul-api-gateway/internal/core" "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/sdk/testutil" "github.com/stretchr/testify/require" ) @@ -143,3 +148,61 @@ func TestHTTPRouteDiscoveryChain(t *testing.T) { }) } } + +func TestConsulSyncAdapter_Sync(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + consulSrv, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) { + c.Connect = map[string]interface{}{"enabled": true} + }) + require.NoError(t, err) + + t.Cleanup(func() { + cancel() + _ = consulSrv.Stop() + }) + + cfg := api.DefaultConfig() + cfg.Address = consulSrv.HTTPAddr + consul, err := api.NewClient(cfg) + require.NoError(t, err) + + adapter := NewConsulSyncAdapter(testutil.Logger(t), consul) + + route := core.NewTCPRouteBuilder(). + WithName("tcp-default/route1"). + WithService(core.ResolvedService{ + Service: "tcp-default/service1", + }). + Build() + + gateway := core.ResolvedGateway{ + ID: core.GatewayID{ + Service: "name1", + }, + Listeners: []core.ResolvedListener{{ + TLS: core.TLSParams{ + MinVersion: "TLSv1_2", + }, + Routes: []core.ResolvedRoute{route}, + }}, + } + + err = adapter.Sync(ctx, gateway) + require.NoError(t, err) + + require.Eventually(t, func() bool { + entry, _, err := consul.ConfigEntries().Get(api.IngressGateway, "name1", nil) + if err != nil { + return false + } + + ingress, ok := entry.(*api.IngressGatewayConfigEntry) + require.True(t, ok) + require.NotNil(t, ingress) + + return ingress.Listeners[0].TLS.TLSMinVersion == "TLSv1_2" && + reflect.DeepEqual(ingress.Listeners[0].TLS.CipherSuites, common.DefaultTLSCipherSuites()) + }, 30*time.Second, 1*time.Second, "listener TLS config not synced in the allotted time") +} diff --git a/internal/commands/exec/command.go b/internal/commands/exec/command.go index 48594699f..89426c0bd 100644 --- a/internal/commands/exec/command.go +++ b/internal/commands/exec/command.go @@ -141,9 +141,11 @@ func (c *Command) Run(args []string) (ret int) { cfg := api.DefaultConfig() cfg.Address = hostPort if c.flagConsulCACertFile != "" { - cfg.Scheme = "https" cfg.TLSConfig.CAFile = c.flagConsulCACertFile } + if cfg.TLSConfig.CAFile != "" { + cfg.Scheme = "https" + } consulClient, err := api.NewClient(cfg) if err != nil { logger.Error("error creating consul client", "error", err) @@ -177,7 +179,7 @@ func (c *Command) Run(args []string) (ret int) { Namespace: c.flagGatewayNamespace, }, EnvoyConfig: EnvoyConfig{ - CACertificateFile: c.flagConsulCACertFile, + CACertificateFile: cfg.TLSConfig.CAFile, XDSAddress: c.flagConsulHTTPAddress, XDSPort: c.flagConsulXDSPort, SDSAddress: c.flagSDSServerAddress, diff --git a/internal/commands/server/command.go b/internal/commands/server/command.go index 78677b85e..b9bfc389b 100644 --- a/internal/commands/server/command.go +++ b/internal/commands/server/command.go @@ -12,6 +12,7 @@ import ( "github.com/mitchellh/cli" "sigs.k8s.io/controller-runtime/pkg/client/config" + "github.com/hashicorp/consul-api-gateway/internal/k8s/utils" "github.com/hashicorp/consul/api" "github.com/hashicorp/go-hclog" @@ -35,13 +36,15 @@ type Command struct { flagCAFile string // CA File for CA for Consul server flagCASecret string // CA Secret for Consul server flagCASecretNamespace string // CA Secret namespace for Consul server - flagConsulAddress string // Consul server address - flagSDSServerHost string // SDS server host - flagSDSServerPort int // SDS server port - flagMetricsPort int // Port for prometheus metrics - flagPprofPort int // Port for pprof profiling - flagK8sContext string // context to use - flagK8sNamespace string // namespace we're run in + + flagConsulAddress string // Consul server address + + flagSDSServerHost string // SDS server host + flagSDSServerPort int // SDS server port + flagMetricsPort int // Port for prometheus metrics + flagPprofPort int // Port for pprof profiling + flagK8sContext string // context to use + flagK8sNamespace string // namespace we're run in // Logging flagLogLevel string @@ -60,7 +63,7 @@ func (c *Command) init() { c.flagSet = flag.NewFlagSet("", flag.ContinueOnError) c.flagSet.StringVar(&c.flagCAFile, "ca-file", "", "Path to CA for Consul server.") c.flagSet.StringVar(&c.flagCASecret, "ca-secret", "", "CA Secret for Consul server.") - c.flagSet.StringVar(&c.flagCASecretNamespace, "ca-secret-namespace", "", "CA Secret namespace for Consul server.") + c.flagSet.StringVar(&c.flagCASecretNamespace, "ca-secret-namespace", "default", "CA Secret namespace for Consul server.") c.flagSet.StringVar(&c.flagConsulAddress, "consul-address", "", "Consul Address.") c.flagSet.StringVar(&c.flagSDSServerHost, "sds-server-host", defaultSDSServerHost, "SDS Server Host.") c.flagSet.StringVar(&c.flagK8sContext, "k8s-context", "", "Kubernetes context to use.") @@ -93,21 +96,23 @@ func (c *Command) Run(args []string) int { IncludeLocation: true, }).Named("consul-api-gateway-server") - consulCfg := api.DefaultConfig() cfg := k8s.Defaults() + restConfig, err := config.GetConfigWithContext(c.flagK8sContext) + if err != nil { + logger.Error("error getting kubernetes configuration", "error", err) + return 1 + } + cfg.RestConfig = restConfig + cfg.SDSServerHost = c.flagSDSServerHost + cfg.SDSServerPort = c.flagSDSServerPort + cfg.Namespace = c.flagK8sNamespace + consulCfg := api.DefaultConfig() if c.flagCAFile != "" { consulCfg.TLSConfig.CAFile = c.flagCAFile - cfg.CACertFile = c.flagCAFile - consulCfg.Scheme = "https" } if c.flagCASecret != "" { - cfg.CACertSecret = c.flagCASecret - if c.flagCASecretNamespace != "" { - cfg.CACertSecretNamespace = c.flagCASecretNamespace - } - // if we're pulling the cert from a secret, then we override the location // where we store it file, err := ioutil.TempFile("", "consul-api-gateway") @@ -116,25 +121,29 @@ func (c *Command) Run(args []string) int { return 1 } defer os.Remove(file.Name()) - cfg.CACertFile = file.Name() consulCfg.TLSConfig.CAFile = file.Name() + + if err := utils.WriteSecretCertFile(restConfig, c.flagCASecret, file.Name(), c.flagCASecretNamespace); err != nil { + logger.Error("error creating the kubernetes controller", "error", err) + return 1 + } + } + // CA file can be set by cli flag or 'CONSUL_CACERT' env var + if consulCfg.TLSConfig.CAFile != "" { consulCfg.Scheme = "https" + consulCA, err := ioutil.ReadFile(consulCfg.TLSConfig.CAFile) + if err != nil { + logger.Error("error creating the kubernetes controller", "error", err) + return 1 + } + + cfg.CACert = string(consulCA) } if c.flagConsulAddress != "" { consulCfg.Address = c.flagConsulAddress } - restConfig, err := config.GetConfigWithContext(c.flagK8sContext) - if err != nil { - logger.Error("error getting kubernetes configuration", "error", err) - return 1 - } - cfg.RestConfig = restConfig - cfg.SDSServerHost = c.flagSDSServerHost - cfg.SDSServerPort = c.flagSDSServerPort - cfg.Namespace = c.flagK8sNamespace - return RunServer(ServerConfig{ Context: context.Background(), Logger: logger, diff --git a/internal/commands/server/k8s_e2e_test.go b/internal/commands/server/k8s_e2e_test.go index 496b6859d..e8fa3953b 100644 --- a/internal/commands/server/k8s_e2e_test.go +++ b/internal/commands/server/k8s_e2e_test.go @@ -1,4 +1,5 @@ -//+build e2e +//go:build e2e +// +build e2e package server @@ -7,6 +8,7 @@ import ( "crypto/tls" "fmt" "io" + "net" "net/http" "os" "strings" @@ -139,9 +141,10 @@ func TestGatewayBasic(t *testing.T) { return status == "passing" }, 30*time.Second, 1*time.Second, "no healthy consul service found in the allotted time") + require.Eventually(t, gatewayStatusCheck(ctx, resources, gatewayName, namespace, conditionReady), 30*time.Second, 1*time.Second, "no gateway found in the allotted time") + err = resources.Delete(ctx, created) require.NoError(t, err) - require.Eventually(t, func() bool { services, _, err := client.Catalog().Service(gatewayName, "", nil) if err != nil { @@ -234,18 +237,18 @@ func TestServiceListeners(t *testing.T) { testenv.Test(t, feature.Feature()) } -func TestMeshService(t *testing.T) { +func TestHTTPMeshService(t *testing.T) { feature := features.New("mesh service routing"). Assess("basic routing", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - serviceOne, err := e2e.DeployMeshService(ctx, cfg) + serviceOne, err := e2e.DeployHTTPMeshService(ctx, cfg) require.NoError(t, err) - serviceTwo, err := e2e.DeployMeshService(ctx, cfg) + serviceTwo, err := e2e.DeployHTTPMeshService(ctx, cfg) require.NoError(t, err) - serviceThree, err := e2e.DeployMeshService(ctx, cfg) + serviceThree, err := e2e.DeployHTTPMeshService(ctx, cfg) require.NoError(t, err) - serviceFour, err := e2e.DeployMeshService(ctx, cfg) + serviceFour, err := e2e.DeployHTTPMeshService(ctx, cfg) require.NoError(t, err) - serviceFive, err := e2e.DeployMeshService(ctx, cfg) + serviceFive, err := e2e.DeployHTTPMeshService(ctx, cfg) require.NoError(t, err) namespace := e2e.Namespace(ctx) @@ -255,7 +258,6 @@ func TestMeshService(t *testing.T) { routeOneName := envconf.RandomName("route", 16) routeTwoName := envconf.RandomName("route", 16) routeThreeName := envconf.RandomName("route", 16) - routeFourName := envconf.RandomName("route", 16) gatewayNamespace := gateway.Namespace(namespace) resources := cfg.Client().Resources(namespace) @@ -312,7 +314,7 @@ func TestMeshService(t *testing.T) { GatewayClassName: gateway.ObjectName(gc.Name), Listeners: []gateway.Listener{{ Name: "https", - Port: gateway.PortNumber(e2e.ExtraPort(ctx)), + Port: gateway.PortNumber(e2e.HTTPPort(ctx)), Protocol: gateway.HTTPSProtocolType, TLS: &gateway.GatewayTLSConfig{ CertificateRefs: []*gateway.SecretObjectReference{{ @@ -325,7 +327,7 @@ func TestMeshService(t *testing.T) { } err = resources.Create(ctx, gw) require.NoError(t, err) - require.Eventually(t, gatewayStatusCheck(ctx, resources, gatewayName, namespace, gatewayReady), 30*time.Second, 1*time.Second, "no gateway found in the allotted time") + require.Eventually(t, gatewayStatusCheck(ctx, resources, gatewayName, namespace, conditionReady), 30*time.Second, 1*time.Second, "no gateway found in the allotted time") // route 1 port := gateway.PortNumber(serviceOne.Spec.Ports[0].Port) @@ -365,6 +367,8 @@ func TestMeshService(t *testing.T) { // route 2 port = gateway.PortNumber(serviceTwo.Spec.Ports[0].Port) + portFour := gateway.PortNumber(serviceFour.Spec.Ports[0].Port) + portFive := gateway.PortNumber(serviceFive.Spec.Ports[0].Port) path = "/v2" route := &gateway.HTTPRoute{ ObjectMeta: meta.ObjectMeta{ @@ -392,6 +396,22 @@ func TestMeshService(t *testing.T) { }, }, }}, + }, { + BackendRefs: []gateway.HTTPBackendRef{{ + BackendRef: gateway.BackendRef{ + BackendObjectReference: gateway.BackendObjectReference{ + Name: gateway.ObjectName(serviceFour.Name), + Port: &portFour, + }, + }, + }, { + BackendRef: gateway.BackendRef{ + BackendObjectReference: gateway.BackendObjectReference{ + Name: gateway.ObjectName(serviceFive.Name), + Port: &portFive, + }, + }, + }}, }}, }, } @@ -440,34 +460,171 @@ func TestMeshService(t *testing.T) { err = resources.Create(ctx, route) require.NoError(t, err) - // route 4 - fallback - portFour := gateway.PortNumber(serviceFour.Spec.Ports[0].Port) - portFive := gateway.PortNumber(serviceFive.Spec.Ports[0].Port) - route = &gateway.HTTPRoute{ + checkPort := e2e.HTTPPort(ctx) + checkRoute(t, checkPort, "/v1", serviceOne.Name, nil, "service one not routable in allotted time") + checkRoute(t, checkPort, "/v2", serviceTwo.Name, nil, "service two not routable in allotted time") + checkRoute(t, checkPort, "/v3", serviceThree.Name, map[string]string{ + "x-v3": "v3", + "Host": "test.host", + }, "service three not routable in allotted time") + checkRoute(t, checkPort, "/v3", serviceFour.Name, nil, "service four not routable in allotted time") + checkRoute(t, checkPort, "/v3", serviceFive.Name, nil, "service five not routable in allotted time") + + err = resources.Delete(ctx, routeOne) + require.NoError(t, err) + + checkRoute(t, checkPort, "/v1", serviceFour.Name, nil, "after route deletion service four not routable in allotted time") + checkRoute(t, checkPort, "/v1", serviceFive.Name, nil, "after route deletion service five not routable in allotted time") + + require.Eventually(t, gatewayStatusCheck(ctx, resources, gatewayName, namespace, conditionInSync), 30*time.Second, 1*time.Second, "gateway not synced in the allotted time") + return ctx + }) + + testenv.Test(t, feature.Feature()) +} + +func TestTCPMeshService(t *testing.T) { + feature := features.New("mesh service tcp routing"). + Assess("basic routing", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + serviceOne, err := e2e.DeployTCPMeshService(ctx, cfg) + require.NoError(t, err) + serviceTwo, err := e2e.DeployTCPMeshService(ctx, cfg) + require.NoError(t, err) + serviceThree, err := e2e.DeployTCPMeshService(ctx, cfg) + require.NoError(t, err) + serviceFour, err := e2e.DeployTCPMeshService(ctx, cfg) + require.NoError(t, err) + + namespace := e2e.Namespace(ctx) + configName := envconf.RandomName("gcc", 16) + className := envconf.RandomName("gc", 16) + gatewayName := envconf.RandomName("gw", 16) + routeOneName := envconf.RandomName("route", 16) + routeTwoName := envconf.RandomName("route", 16) + + resources := cfg.Client().Resources(namespace) + + gcc := &apigwv1alpha1.GatewayClassConfig{ + ObjectMeta: meta.ObjectMeta{ + Name: configName, + }, + Spec: apigwv1alpha1.GatewayClassConfigSpec{ + ImageSpec: apigwv1alpha1.ImageSpec{ + ConsulAPIGateway: e2e.DockerImage(ctx), + }, + UseHostPorts: true, + LogLevel: "trace", + ConsulSpec: apigwv1alpha1.ConsulSpec{ + Address: hostRoute, + Scheme: "https", + PortSpec: apigwv1alpha1.PortSpec{ + GRPC: e2e.ConsulGRPCPort(ctx), + HTTP: e2e.ConsulHTTPPort(ctx), + }, + AuthSpec: apigwv1alpha1.AuthSpec{ + Method: "consul-api-gateway", + Account: "consul-api-gateway", + }, + }, + }, + } + err = resources.Create(ctx, gcc) + require.NoError(t, err) + + gc := &gateway.GatewayClass{ ObjectMeta: meta.ObjectMeta{ - Name: routeFourName, + Name: className, + }, + Spec: gateway.GatewayClassSpec{ + ControllerName: k8s.ControllerName, + ParametersRef: &gateway.ParametersReference{ + Group: apigwv1alpha1.Group, + Kind: apigwv1alpha1.GatewayClassConfigKind, + Name: configName, + }, + }, + } + err = resources.Create(ctx, gc) + require.NoError(t, err) + + gw := &gateway.Gateway{ + ObjectMeta: meta.ObjectMeta{ + Name: gatewayName, Namespace: namespace, }, - Spec: gateway.HTTPRouteSpec{ + Spec: gateway.GatewaySpec{ + GatewayClassName: gateway.ObjectName(gc.Name), + Listeners: []gateway.Listener{{ + Name: "tcp", + Port: gateway.PortNumber(e2e.TCPPort(ctx)), + Protocol: gateway.TCPProtocolType, + }}, + }, + } + err = resources.Create(ctx, gw) + require.NoError(t, err) + require.Eventually(t, gatewayStatusCheck(ctx, resources, gatewayName, namespace, conditionReady), 30*time.Second, 1*time.Second, "no gateway found in the allotted time") + + // route 1 + portOne := gateway.PortNumber(serviceOne.Spec.Ports[0].Port) + portTwo := gateway.PortNumber(serviceTwo.Spec.Ports[0].Port) + portThree := gateway.PortNumber(serviceThree.Spec.Ports[0].Port) + routeOne := &gateway.TCPRoute{ + ObjectMeta: meta.ObjectMeta{ + Name: routeOneName, + Namespace: namespace, + }, + Spec: gateway.TCPRouteSpec{ CommonRouteSpec: gateway.CommonRouteSpec{ ParentRefs: []gateway.ParentRef{{ Name: gateway.ObjectName(gatewayName), }}, }, - Rules: []gateway.HTTPRouteRule{{ - BackendRefs: []gateway.HTTPBackendRef{{ - BackendRef: gateway.BackendRef{ - BackendObjectReference: gateway.BackendObjectReference{ - Name: gateway.ObjectName(serviceFour.Name), - Port: &portFour, - }, + Rules: []gateway.TCPRouteRule{{ + BackendRefs: []gateway.BackendRef{{ + BackendObjectReference: gateway.BackendObjectReference{ + Name: gateway.ObjectName(serviceOne.Name), + Port: &portOne, }, }, { - BackendRef: gateway.BackendRef{ - BackendObjectReference: gateway.BackendObjectReference{ - Name: gateway.ObjectName(serviceFive.Name), - Port: &portFive, - }, + BackendObjectReference: gateway.BackendObjectReference{ + Name: gateway.ObjectName(serviceTwo.Name), + Port: &portTwo, + }, + }}, + }, { + BackendRefs: []gateway.BackendRef{{ + BackendObjectReference: gateway.BackendObjectReference{ + Name: gateway.ObjectName(serviceThree.Name), + Port: &portThree, + }, + }}, + }}, + }, + } + err = resources.Create(ctx, routeOne) + require.NoError(t, err) + + require.Eventually(t, tcpRouteStatusCheck(ctx, resources, gatewayName, routeOneName, namespace, routeRefErrors), 30*time.Second, 1*time.Second, "route status not set in allotted time") + + // route 2 + portFour := gateway.PortNumber(serviceFour.Spec.Ports[0].Port) + route := &gateway.TCPRoute{ + ObjectMeta: meta.ObjectMeta{ + Name: routeTwoName, + Namespace: namespace, + }, + Spec: gateway.TCPRouteSpec{ + CommonRouteSpec: gateway.CommonRouteSpec{ + ParentRefs: []gateway.ParentRef{{ + Name: gateway.ObjectName(gatewayName), + }}, + }, + Rules: []gateway.TCPRouteRule{{ + BackendRefs: []gateway.BackendRef{{ + BackendObjectReference: gateway.BackendObjectReference{ + Name: gateway.ObjectName(serviceFour.Name), + Port: &portFour, }, }}, }}, @@ -476,23 +633,151 @@ func TestMeshService(t *testing.T) { err = resources.Create(ctx, route) require.NoError(t, err) - checkPort := e2e.ExtraPort(ctx) - checkRoute(t, checkPort, "/v1", serviceOne.Name, nil, "service one not routable in allotted time") - checkRoute(t, checkPort, "/v2", serviceTwo.Name, nil, "service two not routable in allotted time") - checkRoute(t, checkPort, "/v3", serviceThree.Name, map[string]string{ - "x-v3": "v3", - "Host": "test.host", - }, "service three not routable in allotted time") - checkRoute(t, checkPort, "/v3", serviceFour.Name, nil, "service four not routable in allotted time") - checkRoute(t, checkPort, "/v3", serviceFive.Name, nil, "service five not routable in allotted time") + checkPort := e2e.TCPPort(ctx) - err = resources.Delete(ctx, routeOne) + // only service 4 should be routable as we don't support routes with multiple rules or backend refs for TCP + checkTCPRoute(t, checkPort, serviceFour.Name, "service four not routable in allotted time") + + require.Eventually(t, gatewayStatusCheck(ctx, resources, gatewayName, namespace, conditionInSync), 30*time.Second, 1*time.Second, "gateway not synced in the allotted time") + return ctx + }). + Assess("tls routing", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + serviceOne, err := e2e.DeployTCPMeshService(ctx, cfg) + serviceTwo, err := e2e.DeployTCPMeshService(ctx, cfg) require.NoError(t, err) - checkRoute(t, checkPort, "/v1", serviceFour.Name, nil, "after route deletion service four not routable in allotted time") - checkRoute(t, checkPort, "/v1", serviceFive.Name, nil, "after route deletion service five not routable in allotted time") + namespace := e2e.Namespace(ctx) + configName := envconf.RandomName("gcc", 16) + className := envconf.RandomName("gc", 16) + gatewayName := envconf.RandomName("gw", 16) + routeOneName := envconf.RandomName("route", 16) + routeTwoName := envconf.RandomName("route", 16) + listenerOneName := "tcp" + listenerTwoName := "insecure" + listenerOnePort := e2e.TCPTLSPort(ctx) + listenerTwoPort := e2e.ExtraTCPTLSPort(ctx) + + gatewayNamespace := gateway.Namespace(namespace) + resources := cfg.Client().Resources(namespace) + + gcc := &apigwv1alpha1.GatewayClassConfig{ + ObjectMeta: meta.ObjectMeta{ + Name: configName, + }, + Spec: apigwv1alpha1.GatewayClassConfigSpec{ + ImageSpec: apigwv1alpha1.ImageSpec{ + ConsulAPIGateway: e2e.DockerImage(ctx), + }, + UseHostPorts: true, + LogLevel: "trace", + ConsulSpec: apigwv1alpha1.ConsulSpec{ + Address: hostRoute, + Scheme: "https", + PortSpec: apigwv1alpha1.PortSpec{ + GRPC: e2e.ConsulGRPCPort(ctx), + HTTP: e2e.ConsulHTTPPort(ctx), + }, + AuthSpec: apigwv1alpha1.AuthSpec{ + Method: "consul-api-gateway", + Account: "consul-api-gateway", + }, + }, + }, + } + err = resources.Create(ctx, gcc) + require.NoError(t, err) + + gc := &gateway.GatewayClass{ + ObjectMeta: meta.ObjectMeta{ + Name: className, + }, + Spec: gateway.GatewayClassSpec{ + ControllerName: k8s.ControllerName, + ParametersRef: &gateway.ParametersReference{ + Group: apigwv1alpha1.Group, + Kind: apigwv1alpha1.GatewayClassConfigKind, + Name: configName, + }, + }, + } + err = resources.Create(ctx, gc) + require.NoError(t, err) + + gw := &gateway.Gateway{ + ObjectMeta: meta.ObjectMeta{ + Name: gatewayName, + Namespace: namespace, + }, + Spec: gateway.GatewaySpec{ + GatewayClassName: gateway.ObjectName(gc.Name), + Listeners: []gateway.Listener{ + { + Name: gateway.SectionName(listenerOneName), + Port: gateway.PortNumber(listenerOnePort), + Protocol: gateway.TCPProtocolType, + TLS: &gateway.GatewayTLSConfig{ + CertificateRefs: []*gateway.SecretObjectReference{{ + Name: "consul-server-cert", + Namespace: &gatewayNamespace, + }}, + }, + }, + { + Name: gateway.SectionName(listenerTwoName), + Port: gateway.PortNumber(listenerTwoPort), + Protocol: gateway.TCPProtocolType, + TLS: &gateway.GatewayTLSConfig{ + CertificateRefs: []*gateway.SecretObjectReference{{ + Name: "consul-server-cert", + Namespace: &gatewayNamespace, + }}, + Options: map[gateway.AnnotationKey]gateway.AnnotationValue{ + "api-gateway.consul.hashicorp.com/tls_min_version": "TLSv1_1", + "api-gateway.consul.hashicorp.com/tls_cipher_suites": "TLS_RSA_WITH_AES_128_CBC_SHA", + }, + }, + }, + }, + }, + } + err = resources.Create(ctx, gw) + require.NoError(t, err) + require.Eventually(t, gatewayStatusCheck(ctx, resources, gatewayName, namespace, conditionReady), 30*time.Second, 1*time.Second, "no gateway found in the allotted time") + + createTCPRoute(ctx, t, resources, namespace, gatewayName, gateway.SectionName(listenerOneName), routeOneName, serviceOne.Name, gateway.PortNumber(serviceOne.Spec.Ports[0].Port)) + createTCPRoute(ctx, t, resources, namespace, gatewayName, gateway.SectionName(listenerTwoName), routeTwoName, serviceTwo.Name, gateway.PortNumber(serviceTwo.Spec.Ports[0].Port)) + + checkTCPTLSRoute(t, listenerOnePort, &tls.Config{ + InsecureSkipVerify: true, + }, serviceOne.Name, "service not routable in allotted time") + + // Force insecure cipher suite excluded from Consul API Gateway default + // cipher suites, but supported by Envoy defaults, limit max version to + // TLS 1.2 to ensure cipher suite config is applicable. + checkTCPTLSRoute(t, listenerOnePort, &tls.Config{ + InsecureSkipVerify: true, + MaxVersion: tls.VersionTLS12, + CipherSuites: []uint16{tls.TLS_RSA_WITH_AES_128_CBC_SHA}, + }, "remote error: tls: handshake failure", "connection not rejected with expected error in allotted time") + + // Force TLS max version below Consul API Gateway default min version, but + // supported by Envoy defaults + checkTCPTLSRoute(t, listenerOnePort, &tls.Config{ + InsecureSkipVerify: true, + MaxVersion: tls.VersionTLS11, + }, "remote error: tls: protocol version not supported", "connection not rejected with expected error in allotted time") + + // Service two listener overrides default config + checkTCPTLSRoute(t, listenerTwoPort, &tls.Config{ + InsecureSkipVerify: true, + CipherSuites: []uint16{tls.TLS_RSA_WITH_AES_128_CBC_SHA}, + MaxVersion: tls.VersionTLS11, + }, serviceTwo.Name, "service not routable in allotted time") + + require.Eventually(t, gatewayStatusCheck(ctx, resources, gatewayName, namespace, conditionInSync), 30*time.Second, 1*time.Second, "gateway not synced in the allotted time") + + require.Eventually(t, listenerStatusCheck(ctx, resources, gatewayName, namespace, conditionReady), 30*time.Second, 1*time.Second, "listeners not ready in the allotted time") - require.Eventually(t, gatewayStatusCheck(ctx, resources, gatewayName, namespace, gatewayInSync), 30*time.Second, 1*time.Second, "gateway not synced in the allotted time") return ctx }) @@ -505,13 +790,57 @@ func gatewayStatusCheck(ctx context.Context, resources *resources.Resources, gat if err := resources.Get(ctx, gatewayName, namespace, updated); err != nil { return false } + return checkFn(updated.Status.Conditions) } } -func gatewayReady(conditions []meta.Condition) bool { +func listenerStatusCheck(ctx context.Context, resources *resources.Resources, gatewayName, namespace string, checkFn func([]meta.Condition) bool) func() bool { + return func() bool { + updated := &gateway.Gateway{} + if err := resources.Get(ctx, gatewayName, namespace, updated); err != nil { + return false + } + + for _, listener := range updated.Status.Listeners { + if ok := checkFn(listener.Conditions); !ok { + return false + } + } + + return true + } +} + +func tcpRouteStatusCheck(ctx context.Context, resources *resources.Resources, gatewayName, routeName, namespace string, checkFn func([]meta.Condition) bool) func() bool { + return func() bool { + updated := &gateway.TCPRoute{} + if err := resources.Get(ctx, routeName, namespace, updated); err != nil { + return false + } + for _, status := range updated.Status.Parents { + if string(status.ParentRef.Name) == gatewayName { + return checkFn(status.Conditions) + } + } + return false + } +} + +func routeRefErrors(conditions []meta.Condition) bool { for _, condition := range conditions { - if condition.Type == "Accepted" || + if condition.Type == "ResolvedRefs" && + condition.Status == "False" && + condition.Reason == "Errors" { + return true + } + } + return false +} + +func conditionReady(conditions []meta.Condition) bool { + for _, condition := range conditions { + if condition.Type == "Ready" && condition.Status == "True" { return true } @@ -519,9 +848,9 @@ func gatewayReady(conditions []meta.Condition) bool { return false } -func gatewayInSync(conditions []meta.Condition) bool { +func conditionInSync(conditions []meta.Condition) bool { for _, condition := range conditions { - if condition.Type == "InSync" || + if condition.Type == "InSync" && condition.Status == "True" { return true } @@ -584,6 +913,36 @@ func createGatewayClass(ctx context.Context, t *testing.T, cfg *envconf.Config) return gcc, gc } +func createTCPRoute(ctx context.Context, t *testing.T, resources *resources.Resources, namespace string, gatewayName string, listenerName gateway.SectionName, routeName string, serviceName string, port gateway.PortNumber) { + t.Helper() + + route := &gateway.TCPRoute{ + ObjectMeta: meta.ObjectMeta{ + Name: routeName, + Namespace: namespace, + }, + Spec: gateway.TCPRouteSpec{ + CommonRouteSpec: gateway.CommonRouteSpec{ + ParentRefs: []gateway.ParentRef{{ + Name: gateway.ObjectName(gatewayName), + SectionName: &listenerName, + }}, + }, + Rules: []gateway.TCPRouteRule{{ + BackendRefs: []gateway.BackendRef{{ + BackendObjectReference: gateway.BackendObjectReference{ + Name: gateway.ObjectName(serviceName), + Port: &port, + }, + }}, + }}, + }, + } + + err := resources.Create(ctx, route) + require.NoError(t, err) +} + func checkRoute(t *testing.T, port int, path, expected string, headers map[string]string, message string) { t.Helper() @@ -622,3 +981,45 @@ func checkRoute(t *testing.T, port int, path, expected string, headers map[strin return strings.HasPrefix(string(data), expected) }, 30*time.Second, 1*time.Second, message) } + +func checkTCPRoute(t *testing.T, port int, expected string, message string) { + t.Helper() + + require.Eventually(t, func() bool { + conn, err := net.DialTCP("tcp", nil, &net.TCPAddr{ + IP: net.IPv4(127, 0, 0, 1), + Port: port, + }) + if err != nil { + return false + } + data, err := io.ReadAll(conn) + if err != nil { + return false + } + return strings.HasPrefix(string(data), expected) + }, 30*time.Second, 1*time.Second, message) +} + +func checkTCPTLSRoute(t *testing.T, port int, config *tls.Config, expected string, message string) { + t.Helper() + + require.Eventually(t, func() bool { + conn, err := net.DialTCP("tcp", nil, &net.TCPAddr{ + IP: net.IPv4(127, 0, 0, 1), + Port: port, + }) + if err != nil { + return false + } + tlsConn := tls.Client(conn, config) + data, err := io.ReadAll(tlsConn) + + if err != nil { + t.Log(err) + return strings.HasPrefix(err.Error(), expected) + } + + return strings.HasPrefix(string(data), expected) + }, 30*time.Second, 1*time.Second, message) +} diff --git a/internal/common/tls.go b/internal/common/tls.go new file mode 100644 index 000000000..0b2998c7e --- /dev/null +++ b/internal/common/tls.go @@ -0,0 +1,45 @@ +package common + +var defaultTLSCipherSuites = []string{ + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", +} + +func DefaultTLSCipherSuites() []string { + return defaultTLSCipherSuites +} + +// NOTE: the following cipher suites are currently supported by Envoy but insecure and +// pending removal +var extraTLSCipherSuites = []string{ + // https://github.com/envoyproxy/envoy/issues/5399 + "TLS_RSA_WITH_AES_128_GCM_SHA256", + "TLS_RSA_WITH_AES_128_CBC_SHA", + "TLS_RSA_WITH_AES_256_GCM_SHA384", + "TLS_RSA_WITH_AES_256_CBC_SHA", + + // https://github.com/envoyproxy/envoy/issues/5400 + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", + "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA", + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", +} + +var supportedTLSCipherSuites = (func() map[string]struct{} { + cipherSuites := make(map[string]struct{}) + + for _, c := range append(defaultTLSCipherSuites, extraTLSCipherSuites...) { + cipherSuites[c] = struct{}{} + } + + return cipherSuites +})() + +func SupportedTLSCipherSuite(cipherSuite string) bool { + _, ok := supportedTLSCipherSuites[cipherSuite] + return ok +} diff --git a/internal/consul/certmanager.go b/internal/consul/certmanager.go index 9748587e0..eaac6b211 100644 --- a/internal/consul/certmanager.go +++ b/internal/consul/certmanager.go @@ -13,6 +13,7 @@ import ( "path" "sync" "text/template" + "time" "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/api/watch" @@ -121,8 +122,9 @@ type CertManager struct { rootWatch *watch.Plan leafWatch *watch.Plan - // this can be overwritten to check retry logic in testing - writeCerts certWriter + // these can be overwritten to modify retry logic in testing + writeCerts certWriter + skipExtraFetch bool } // NewCertManager creates a new CertManager instance. @@ -146,6 +148,7 @@ func NewCertManager(logger hclog.Logger, consul *api.Client, service string, opt func (c *CertManager) handleRootWatch(blockParam watch.BlockingParamVal, raw interface{}) { if raw == nil { + c.logger.Error("received nil interface") return } v, ok := raw.(*api.CARootList) @@ -187,6 +190,7 @@ func (c *CertManager) handleRootWatch(blockParam watch.BlockingParamVal, raw int func (c *CertManager) handleLeafWatch(blockParam watch.BlockingParamVal, raw interface{}) { if raw == nil { + c.logger.Error("received nil interface") return // ignore } v, ok := raw.(*api.LeafCert) @@ -246,6 +250,25 @@ func (c *CertManager) Manage(ctx context.Context) error { if err := w.RunWithClientAndHclog(c.consul, c.logger); err != nil { c.logger.Error("consul watch.Plan returned unexpectedly", "error", err) } + c.logger.Trace("consul watch.Plan stopped") + } + + // Consul 1.11 has a bug where blocking queries on the leaf certificate endpoint + // cause all subsequent non-blocking queries to unexpectedly block. The problem + // is that this means that, on restart, the query for a leaf certificate with + // the given service id will never return until the previous leaf certificate + // expires/is rotated. Adding a wait here causes the API to return once the timeout has + // been hit -- allowing us to short-circuit the buggy blocking. The subsequent + // goroutines can then be leveraged to pick up any certificate rotations. + if !c.skipExtraFetch { + leafCert, _, err := c.consul.Agent().ConnectCALeaf(c.service, &api.QueryOptions{ + WaitTime: 1 * time.Second, + }) + if err != nil { + c.logger.Error("error grabbing leaf certificate", "error", err) + return err + } + c.handleLeafWatch(nil, leafCert) } go wrapWatch(c.rootWatch) go wrapWatch(c.leafWatch) @@ -265,17 +288,23 @@ func (c *CertManager) persist() error { if c.directory != "" { if c.ca != nil { + c.logger.Trace("writing root CA file", "file", rootCAFile) if err := os.WriteFile(rootCAFile, c.ca, 0600); err != nil { + c.logger.Error("error writing root CA file", "error", err) return fmt.Errorf("error writing root CA fiile: %w", err) } } if c.certificate != nil { + c.logger.Trace("writing client cert file", "file", clientCertFile) if err := os.WriteFile(clientCertFile, c.certificate, 0600); err != nil { + c.logger.Error("error writing client cert file", "error", err) return fmt.Errorf("error writing client cert fiile: %w", err) } } if c.privateKey != nil { + c.logger.Trace("writing client private key file", "file", clientPrivateKeyFile) if err := os.WriteFile(clientPrivateKeyFile, c.privateKey, 0600); err != nil { + c.logger.Error("error writing client private key file", "error", err) return fmt.Errorf("error writing client private key fiile: %w", err) } } diff --git a/internal/consul/certmanager_test.go b/internal/consul/certmanager_test.go index 792c08d62..e6516dd81 100644 --- a/internal/consul/certmanager_test.go +++ b/internal/consul/certmanager_test.go @@ -50,6 +50,7 @@ func TestManage(t *testing.T) { options.Directory = directory manager := NewCertManager(hclog.NewNullLogger(), server.consul, service, options) + manager.skipExtraFetch = true ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -108,6 +109,7 @@ func TestManage_Refresh(t *testing.T) { options := DefaultCertManagerOptions() manager := NewCertManager(hclog.NewNullLogger(), server.consul, service, options) + manager.skipExtraFetch = true writes := int32(0) manager.writeCerts = func() error { diff --git a/internal/consul/config_entries.go b/internal/consul/config_entries.go index b1bf54988..079a02089 100644 --- a/internal/consul/config_entries.go +++ b/internal/consul/config_entries.go @@ -17,6 +17,9 @@ func NewConfigEntryIndex(kind string) *ConfigEntryIndex { } func (i *ConfigEntryIndex) Add(entry api.ConfigEntry) { + if entry == nil { + return + } if entry.GetKind() != i.kind { return } @@ -24,6 +27,9 @@ func (i *ConfigEntryIndex) Add(entry api.ConfigEntry) { } func (i *ConfigEntryIndex) Merge(other *ConfigEntryIndex) { + if other == nil { + return + } if i.kind != other.kind { return } diff --git a/internal/consul/intentions_test.go b/internal/consul/intentions_test.go index 75940254d..73b019dfe 100644 --- a/internal/consul/intentions_test.go +++ b/internal/consul/intentions_test.go @@ -251,8 +251,8 @@ func TestIntentionsReconciler_Reconcile(t *testing.T) { require.NoError(err) ok, _, err := c.ConfigEntries().Set(igw, nil) - require.True(ok) require.NoError(err) + require.True(ok) err = c.Agent().ServiceRegister(&api.AgentServiceRegistration{ Kind: api.ServiceKindIngressGateway, Name: r.gatewayName.Name, diff --git a/internal/core/resolved.go b/internal/core/resolved.go index c7a84066f..4d1aa8029 100644 --- a/internal/core/resolved.go +++ b/internal/core/resolved.go @@ -21,13 +21,21 @@ type ResolvedRoute interface { GetNamespace() string } -type ResolvedListener struct { - Name string - Hostname string - Port int - Protocol string +type TLSParams struct { + Enabled bool + MinVersion string + MaxVersion string + CipherSuites []string Certificates []string - Routes []ResolvedRoute +} + +type ResolvedListener struct { + Name string + Hostname string + Port int + Protocol string + TLS TLSParams + Routes []ResolvedRoute } type GatewayID struct { diff --git a/internal/core/tcp.go b/internal/core/tcp.go new file mode 100644 index 000000000..ee37cde94 --- /dev/null +++ b/internal/core/tcp.go @@ -0,0 +1,52 @@ +package core + +type TCPRoute struct { + CommonRoute + Service ResolvedService +} + +func (r TCPRoute) GetType() ResolvedRouteType { + return ResolvedTCPRouteType +} + +type TCPRouteBuilder struct { + meta map[string]string + name string + namespace string + service ResolvedService +} + +func (b *TCPRouteBuilder) WithMeta(meta map[string]string) *TCPRouteBuilder { + b.meta = meta + return b +} + +func (b *TCPRouteBuilder) WithName(name string) *TCPRouteBuilder { + b.name = name + return b +} + +func (b *TCPRouteBuilder) WithNamespace(namespace string) *TCPRouteBuilder { + b.namespace = namespace + return b +} + +func (b *TCPRouteBuilder) WithService(service ResolvedService) *TCPRouteBuilder { + b.service = service + return b +} + +func (b *TCPRouteBuilder) Build() ResolvedRoute { + return TCPRoute{ + CommonRoute: CommonRoute{ + Meta: b.meta, + Name: b.name, + Namespace: b.namespace, + }, + Service: b.service, + } +} + +func NewTCPRouteBuilder() *TCPRouteBuilder { + return &TCPRouteBuilder{} +} diff --git a/internal/k8s/builder/builder.go b/internal/k8s/builder/builder.go new file mode 100644 index 000000000..10715aba0 --- /dev/null +++ b/internal/k8s/builder/builder.go @@ -0,0 +1,65 @@ +package builder + +import ( + "fmt" + "strconv" + + v1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + + "github.com/hashicorp/consul-api-gateway/internal/version" +) + +var ( + defaultImage string + defaultServiceAnnotations = []string{ + "external-dns.alpha.kubernetes.io/hostname", + } +) + +func init() { + imageVersion := version.Version + if version.VersionPrerelease != "" { + imageVersion += "-" + version.VersionPrerelease + } + defaultImage = fmt.Sprintf("hashicorp/consul-api-gateway:%s", imageVersion) +} + +const ( + defaultEnvoyImage = "envoyproxy/envoy:v1.19-latest" + defaultLogLevel = "info" + defaultConsulAddress = "$(HOST_IP)" + defaultConsulHTTPPort = "8500" + defaultConsulXDSPort = "8502" + + consulCALocalPath = "/consul/tls" + consulCALocalFile = consulCALocalPath + "/ca.pem" +) + +type Builder interface { + Validate() error +} + +type DeploymentBuilder interface { + Builder + Build() *v1.Deployment +} + +type ServiceBuilder interface { + Builder + Build() *corev1.Service +} + +func orDefault(value, defaultValue string) string { + if value != "" { + return value + } + return defaultValue +} + +func orDefaultIntString(value int, defaultValue string) string { + if value != 0 { + return strconv.Itoa(value) + } + return defaultValue +} diff --git a/internal/k8s/builder/gateway.go b/internal/k8s/builder/gateway.go new file mode 100644 index 000000000..498a3a017 --- /dev/null +++ b/internal/k8s/builder/gateway.go @@ -0,0 +1,330 @@ +package builder + +import ( + "bytes" + "fmt" + "strings" + "text/template" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + gw "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "github.com/hashicorp/consul-api-gateway/internal/k8s/utils" + "github.com/hashicorp/consul-api-gateway/pkg/apis/v1alpha1" +) + +type GatewayServiceBuilder struct { + gateway *gw.Gateway + gwConfig *v1alpha1.GatewayClassConfig +} + +func NewGatewayService(gw *gw.Gateway) *GatewayServiceBuilder { + return &GatewayServiceBuilder{gateway: gw} +} + +func (b *GatewayServiceBuilder) WithClassConfig(cfg v1alpha1.GatewayClassConfig) { + b.gwConfig = &cfg +} + +func (b *GatewayServiceBuilder) Validate() error { + if b.gwConfig == nil { + return fmt.Errorf("GatewayClassConfig must be set") + } + + return nil +} +func (b *GatewayServiceBuilder) Build() *corev1.Service { + if b.gwConfig.Spec.ServiceType == nil { + return nil + } + ports := []corev1.ServicePort{} + for _, listener := range b.gateway.Spec.Listeners { + ports = append(ports, corev1.ServicePort{ + Name: string(listener.Name), + Protocol: "TCP", + Port: int32(listener.Port), + }) + } + labels := utils.LabelsForGateway(b.gateway) + allowedAnnotations := b.gwConfig.Spec.CopyAnnotations.Service + if allowedAnnotations == nil { + allowedAnnotations = defaultServiceAnnotations + } + + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: b.gateway.Name, + Namespace: b.gateway.Namespace, + Labels: labels, + Annotations: filterAnnotations(b.gateway.Annotations, allowedAnnotations), + }, + Spec: corev1.ServiceSpec{ + Selector: labels, + Type: *b.gwConfig.Spec.ServiceType, + Ports: ports, + }, + } +} + +func filterAnnotations(annotations map[string]string, allowed []string) map[string]string { + filtered := make(map[string]string) + for _, annotation := range allowed { + if value, found := annotations[annotation]; found { + filtered[annotation] = value + } + } + return filtered +} + +type GatewayDeploymentBuilder struct { + gateway *gw.Gateway + gwConfig *v1alpha1.GatewayClassConfig + sdsHost string + sdsPort int + consulCAData string +} + +func NewGatewayDeployment(gw *gw.Gateway) *GatewayDeploymentBuilder { + return &GatewayDeploymentBuilder{gateway: gw} +} + +func (b *GatewayDeploymentBuilder) WithClassConfig(cfg v1alpha1.GatewayClassConfig) { + b.gwConfig = &cfg +} + +func (b *GatewayDeploymentBuilder) WithSDS(host string, port int) { + b.sdsHost = host + b.sdsPort = port +} + +func (b *GatewayDeploymentBuilder) WithConsulCA(caData string) { + b.consulCAData = caData +} + +func (b *GatewayDeploymentBuilder) Validate() error { + if b.gwConfig == nil { + return fmt.Errorf("GatewayClassConfig must be set") + } + + if b.sdsHost == "" || b.sdsPort == 0 { + return fmt.Errorf("SDS must be set") + } + + if b.requiresCA() && b.consulCAData == "" { + return fmt.Errorf("ConsulCA must be set") + } + return nil +} + +func (b *GatewayDeploymentBuilder) Build() *v1.Deployment { + labels := utils.LabelsForGateway(b.gateway) + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: b.gateway.Name, + Namespace: b.gateway.Namespace, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Annotations: map[string]string{ + "consul.hashicorp.com/connect-inject": "false", + }, + }, + Spec: b.podSpec(), + }, + }, + } +} + +func (b *GatewayDeploymentBuilder) podSpec() corev1.PodSpec { + volumes, mounts := b.volumes() + defaultServiceAccount := "" + if b.gwConfig.Spec.ConsulSpec.AuthSpec.Managed { + defaultServiceAccount = b.gateway.Name + } + return corev1.PodSpec{ + NodeSelector: b.gwConfig.Spec.NodeSelector, + ServiceAccountName: orDefault(b.gwConfig.Spec.ConsulSpec.AuthSpec.Account, defaultServiceAccount), + // the init container copies the binary into the + // next envoy container so we can decouple the envoy + // versions from our version of consul-api-gateway. + InitContainers: []corev1.Container{{ + Image: orDefault(b.gwConfig.Spec.ImageSpec.ConsulAPIGateway, defaultImage), + Name: "consul-api-gateway-init", + VolumeMounts: mounts, + Command: []string{ + "cp", "/bin/consul-api-gateway", "/bootstrap/consul-api-gateway", + }, + }}, + Containers: []corev1.Container{{ + Image: orDefault(b.gwConfig.Spec.ImageSpec.Envoy, defaultEnvoyImage), + Name: "consul-api-gateway", + VolumeMounts: mounts, + Ports: b.containerPorts(), + Env: []corev1.EnvVar{ + { + Name: "IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + { + Name: "HOST_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.hostIP", + }, + }, + }, + }, + Command: b.execCommand(), + ReadinessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/ready", + Port: intstr.FromInt(20000), + }, + }, + }, + }}, + Volumes: volumes, + } +} + +func (b *GatewayDeploymentBuilder) execCommand() []string { + // Render the command + data := gwContainerCommandData{ + ACLAuthMethod: b.gwConfig.Spec.ConsulSpec.AuthSpec.Method, + ConsulHTTPAddr: orDefault(b.gwConfig.Spec.ConsulSpec.Address, defaultConsulAddress), + ConsulHTTPPort: orDefaultIntString(b.gwConfig.Spec.ConsulSpec.PortSpec.HTTP, defaultConsulHTTPPort), + ConsulGRPCPort: orDefaultIntString(b.gwConfig.Spec.ConsulSpec.PortSpec.GRPC, defaultConsulXDSPort), + LogLevel: orDefault(b.gwConfig.Spec.LogLevel, defaultLogLevel), + GatewayHost: "$(IP)", + GatewayName: b.gateway.Name, + SDSHost: b.sdsHost, + SDSPort: b.sdsPort, + } + if b.requiresCA() { + data.ConsulCAFile = consulCALocalFile + data.ConsulCAData = b.consulCAData + } + if method := b.gwConfig.Spec.ConsulSpec.AuthSpec.Method; method != "" { + data.ACLAuthMethod = method + } + var buf bytes.Buffer + err := template.Must(template.New("root").Parse(strings.TrimSpace( + gwContainerCommandTpl))).Execute(&buf, &data) + if err != nil { + return nil + } + + return []string{"/bin/sh", "-ec", buf.String()} +} + +func (b *GatewayDeploymentBuilder) volumes() ([]corev1.Volume, []corev1.VolumeMount) { + volumes := []corev1.Volume{{ + Name: "bootstrap", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, { + Name: "certs", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }} + mounts := []corev1.VolumeMount{{ + Name: "bootstrap", + MountPath: "/bootstrap", + }, { + Name: "certs", + MountPath: "/certs", + }} + if b.requiresCA() { + volumes = append(volumes, corev1.Volume{ + Name: "ca", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }) + mounts = append(mounts, corev1.VolumeMount{ + Name: "ca", + MountPath: consulCALocalPath, + }) + } + return volumes, mounts +} + +func (b *GatewayDeploymentBuilder) containerPorts() []corev1.ContainerPort { + ports := []corev1.ContainerPort{{ + Name: "ready", + Protocol: "TCP", + ContainerPort: 20000, + }} + for _, listener := range b.gateway.Spec.Listeners { + port := corev1.ContainerPort{ + Name: string(listener.Name), + Protocol: "TCP", + ContainerPort: int32(listener.Port), + } + if b.gwConfig.Spec.UseHostPorts { + port.HostPort = int32(listener.Port) + } + ports = append(ports, port) + } + return ports +} + +func (b *GatewayDeploymentBuilder) requiresCA() bool { + return b.gwConfig.Spec.ConsulSpec.Scheme == "https" +} + +type gwContainerCommandData struct { + ConsulCAFile string + ConsulCAData string + ConsulHTTPAddr string + ConsulHTTPPort string + ConsulGRPCPort string + ACLAuthMethod string + LogLevel string + GatewayHost string + GatewayName string + SDSHost string + SDSPort int +} + +// gwContainerCommandTpl is the template for the command executed by +// the exec container. +const gwContainerCommandTpl = ` +{{- if .ConsulCAFile}} +export CONSUL_CACERT={{ .ConsulCAFile }} +cat <{{ .ConsulCAFile }} +{{ .ConsulCAData }} +EOF +{{- end}} + +exec /bootstrap/consul-api-gateway exec -log-json \ + -log-level {{ .LogLevel }} \ + -gateway-host "{{ .GatewayHost }}" \ + -gateway-name {{ .GatewayName }} \ + -consul-http-address {{ .ConsulHTTPAddr }} \ + -consul-http-port {{ .ConsulHTTPPort }} \ + -consul-xds-port {{ .ConsulGRPCPort }} \ +{{- if .ACLAuthMethod }} + -acl-auth-method {{ .ACLAuthMethod }} \ +{{- end }} + -envoy-bootstrap-path /bootstrap/envoy.json \ + -envoy-sds-address {{ .SDSHost }} \ + -envoy-sds-port {{ .SDSPort }} +` diff --git a/internal/k8s/builder/gateway_test.go b/internal/k8s/builder/gateway_test.go new file mode 100644 index 000000000..0f9411d51 --- /dev/null +++ b/internal/k8s/builder/gateway_test.go @@ -0,0 +1,132 @@ +package builder + +import ( + "bytes" + "fmt" + "os" + "path" + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer/json" + "k8s.io/apimachinery/pkg/util/yaml" + gateway "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "github.com/hashicorp/consul-api-gateway/pkg/apis/v1alpha1" +) + +var ( + generate bool + fixtures = []string{ + "tls-cert", + "static-mapping", + "clusterip", + "loadbalancer", + } +) + +func init() { + if os.Getenv("GENERATE") == "true" { + generate = true + } +} + +type gatewayTestConfig struct { + gatewayClassConfig *v1alpha1.GatewayClassConfig + gatewayClass *gateway.GatewayClass + gateway *gateway.Gateway +} + +func newGatewayTestConfig() *gatewayTestConfig { + return &gatewayTestConfig{ + gatewayClassConfig: &v1alpha1.GatewayClassConfig{}, + gatewayClass: &gateway.GatewayClass{}, + gateway: &gateway.Gateway{}, + } +} + +func (g *gatewayTestConfig) EncodeDeployment() runtime.Object { + b := NewGatewayDeployment(g.gateway) + b.WithSDS("consul-api-gateway-controller.default.svc.cluster.local", 9090) + b.WithClassConfig(*g.gatewayClassConfig) + b.WithConsulCA("CONSUL_CA_MOCKED") + return b.Build() +} + +func (g *gatewayTestConfig) EncodeService() runtime.Object { + b := NewGatewayService(g.gateway) + b.WithClassConfig(*g.gatewayClassConfig) + return b.Build() +} + +func TestGatewayDeploymentBuilder(t *testing.T) { + t.Parallel() + + for _, name := range fixtures { + t.Run(name, func(t *testing.T) { + config := newGatewayTestConfig() + fixtureTest(t, name, "deployment", config, func() runtime.Object { + return config.EncodeDeployment() + }) + }) + } +} + +func TestGatewayServiceBuilder(t *testing.T) { + t.Parallel() + + for _, name := range fixtures { + t.Run(name, func(t *testing.T) { + config := newGatewayTestConfig() + fixtureTest(t, name, "service", config, func() runtime.Object { + return config.EncodeService() + }) + }) + } +} + +func fixtureTest(t *testing.T, name, suffix string, into *gatewayTestConfig, encode func() runtime.Object) { + t.Helper() + + file, err := os.OpenFile(path.Join("testdata", fmt.Sprintf("%s.yaml", name)), os.O_RDONLY, 0644) + require.NoError(t, err) + defer file.Close() + + stat, err := file.Stat() + require.NoError(t, err) + + decoder := yaml.NewYAMLOrJSONDecoder(file, int(stat.Size())) + err = decoder.Decode(into.gatewayClassConfig) + require.NoError(t, err) + err = decoder.Decode(into.gatewayClass) + require.NoError(t, err) + err = decoder.Decode(into.gateway) + require.NoError(t, err) + + var buffer bytes.Buffer + serializer := json.NewSerializerWithOptions( + json.DefaultMetaFactory, nil, nil, + json.SerializerOptions{ + Yaml: true, + Pretty: true, + Strict: true, + }, + ) + err = serializer.Encode(encode(), &buffer) + require.NoError(t, err) + + var expected string + expectedFileName := fmt.Sprintf("%s.%s.golden.yaml", name, suffix) + if generate { + expected = buffer.String() + err := os.WriteFile(path.Join("testdata", expectedFileName), buffer.Bytes(), 0644) + require.NoError(t, err) + } else { + data, err := os.ReadFile(path.Join("testdata", expectedFileName)) + require.NoError(t, err) + expected = string(data) + } + + require.Equal(t, expected, buffer.String()) +} diff --git a/pkg/apis/v1alpha1/testdata/clusterip.deployment.golden.yaml b/internal/k8s/builder/testdata/clusterip.deployment.golden.yaml similarity index 78% rename from pkg/apis/v1alpha1/testdata/clusterip.deployment.golden.yaml rename to internal/k8s/builder/testdata/clusterip.deployment.golden.yaml index 1f6ca59a8..1be04735b 100644 --- a/pkg/apis/v1alpha1/testdata/clusterip.deployment.golden.yaml +++ b/internal/k8s/builder/testdata/clusterip.deployment.golden.yaml @@ -27,27 +27,21 @@ spec: spec: containers: - command: - - /bootstrap/consul-api-gateway - - exec - - -log-json - - -log-level - - info - - -gateway-host - - $(IP) - - -gateway-name - - test-clusterip - - -consul-http-address - - $(HOST_IP) - - -consul-http-port - - "8500" - - -consul-xds-port - - "8502" - - -envoy-bootstrap-path - - /bootstrap/envoy.json - - -envoy-sds-address - - consul-api-gateway-controller.default.svc.cluster.local - - -envoy-sds-port - - "9090" + - /bin/sh + - -ec + - |2- + + + exec /bootstrap/consul-api-gateway exec -log-json \ + -log-level info \ + -gateway-host "$(IP)" \ + -gateway-name test-clusterip \ + -consul-http-address $(HOST_IP) \ + -consul-http-port 8500 \ + -consul-xds-port 8502 \ + -envoy-bootstrap-path /bootstrap/envoy.json \ + -envoy-sds-address consul-api-gateway-controller.default.svc.cluster.local \ + -envoy-sds-port 9090 env: - name: IP valueFrom: @@ -84,7 +78,7 @@ spec: - cp - /bin/consul-api-gateway - /bootstrap/consul-api-gateway - image: hashicorp/consul-api-gateway:0.1.0-techpreview + image: hashicorp/consul-api-gateway:0.1.0-beta name: consul-api-gateway-init resources: {} volumeMounts: diff --git a/pkg/apis/v1alpha1/testdata/clusterip.service.golden.yaml b/internal/k8s/builder/testdata/clusterip.service.golden.yaml similarity index 100% rename from pkg/apis/v1alpha1/testdata/clusterip.service.golden.yaml rename to internal/k8s/builder/testdata/clusterip.service.golden.yaml diff --git a/pkg/apis/v1alpha1/testdata/clusterip.yaml b/internal/k8s/builder/testdata/clusterip.yaml similarity index 100% rename from pkg/apis/v1alpha1/testdata/clusterip.yaml rename to internal/k8s/builder/testdata/clusterip.yaml diff --git a/pkg/apis/v1alpha1/testdata/loadbalancer.deployment.golden.yaml b/internal/k8s/builder/testdata/loadbalancer.deployment.golden.yaml similarity index 78% rename from pkg/apis/v1alpha1/testdata/loadbalancer.deployment.golden.yaml rename to internal/k8s/builder/testdata/loadbalancer.deployment.golden.yaml index 5b3100cec..6c37e3666 100644 --- a/pkg/apis/v1alpha1/testdata/loadbalancer.deployment.golden.yaml +++ b/internal/k8s/builder/testdata/loadbalancer.deployment.golden.yaml @@ -27,27 +27,21 @@ spec: spec: containers: - command: - - /bootstrap/consul-api-gateway - - exec - - -log-json - - -log-level - - info - - -gateway-host - - $(IP) - - -gateway-name - - test-loadbalancer - - -consul-http-address - - $(HOST_IP) - - -consul-http-port - - "8500" - - -consul-xds-port - - "8502" - - -envoy-bootstrap-path - - /bootstrap/envoy.json - - -envoy-sds-address - - consul-api-gateway-controller.default.svc.cluster.local - - -envoy-sds-port - - "9090" + - /bin/sh + - -ec + - |2- + + + exec /bootstrap/consul-api-gateway exec -log-json \ + -log-level info \ + -gateway-host "$(IP)" \ + -gateway-name test-loadbalancer \ + -consul-http-address $(HOST_IP) \ + -consul-http-port 8500 \ + -consul-xds-port 8502 \ + -envoy-bootstrap-path /bootstrap/envoy.json \ + -envoy-sds-address consul-api-gateway-controller.default.svc.cluster.local \ + -envoy-sds-port 9090 env: - name: IP valueFrom: @@ -84,7 +78,7 @@ spec: - cp - /bin/consul-api-gateway - /bootstrap/consul-api-gateway - image: hashicorp/consul-api-gateway:0.1.0-techpreview + image: hashicorp/consul-api-gateway:0.1.0-beta name: consul-api-gateway-init resources: {} volumeMounts: diff --git a/pkg/apis/v1alpha1/testdata/loadbalancer.service.golden.yaml b/internal/k8s/builder/testdata/loadbalancer.service.golden.yaml similarity index 100% rename from pkg/apis/v1alpha1/testdata/loadbalancer.service.golden.yaml rename to internal/k8s/builder/testdata/loadbalancer.service.golden.yaml diff --git a/pkg/apis/v1alpha1/testdata/loadbalancer.yaml b/internal/k8s/builder/testdata/loadbalancer.yaml similarity index 100% rename from pkg/apis/v1alpha1/testdata/loadbalancer.yaml rename to internal/k8s/builder/testdata/loadbalancer.yaml diff --git a/pkg/apis/v1alpha1/testdata/static-mapping.deployment.golden.yaml b/internal/k8s/builder/testdata/static-mapping.deployment.golden.yaml similarity index 77% rename from pkg/apis/v1alpha1/testdata/static-mapping.deployment.golden.yaml rename to internal/k8s/builder/testdata/static-mapping.deployment.golden.yaml index 6903e7335..1db5eb9c8 100644 --- a/pkg/apis/v1alpha1/testdata/static-mapping.deployment.golden.yaml +++ b/internal/k8s/builder/testdata/static-mapping.deployment.golden.yaml @@ -27,29 +27,22 @@ spec: spec: containers: - command: - - /bootstrap/consul-api-gateway - - exec - - -log-json - - -log-level - - info - - -gateway-host - - $(IP) - - -gateway-name - - test-static-mapping - - -consul-http-address - - host.docker.internal - - -consul-http-port - - "443" - - -consul-xds-port - - "8502" - - -envoy-bootstrap-path - - /bootstrap/envoy.json - - -envoy-sds-address - - consul-api-gateway-controller.default.svc.cluster.local - - -envoy-sds-port - - "9090" - - -acl-auth-method - - consul-api-gateway + - /bin/sh + - -ec + - |2- + + + exec /bootstrap/consul-api-gateway exec -log-json \ + -log-level info \ + -gateway-host "$(IP)" \ + -gateway-name test-static-mapping \ + -consul-http-address host.docker.internal \ + -consul-http-port 443 \ + -consul-xds-port 8502 \ + -acl-auth-method consul-api-gateway \ + -envoy-bootstrap-path /bootstrap/envoy.json \ + -envoy-sds-address consul-api-gateway-controller.default.svc.cluster.local \ + -envoy-sds-port 9090 env: - name: IP valueFrom: @@ -88,7 +81,7 @@ spec: - cp - /bin/consul-api-gateway - /bootstrap/consul-api-gateway - image: hashicorp/consul-api-gateway:0.1.0-techpreview + image: hashicorp/consul-api-gateway:0.1.0-beta name: consul-api-gateway-init resources: {} volumeMounts: diff --git a/pkg/apis/v1alpha1/testdata/static-mapping.service.golden.yaml b/internal/k8s/builder/testdata/static-mapping.service.golden.yaml similarity index 100% rename from pkg/apis/v1alpha1/testdata/static-mapping.service.golden.yaml rename to internal/k8s/builder/testdata/static-mapping.service.golden.yaml diff --git a/pkg/apis/v1alpha1/testdata/static-mapping.yaml b/internal/k8s/builder/testdata/static-mapping.yaml similarity index 100% rename from pkg/apis/v1alpha1/testdata/static-mapping.yaml rename to internal/k8s/builder/testdata/static-mapping.yaml diff --git a/pkg/apis/v1alpha1/testdata/tls-cert.deployment.golden.yaml b/internal/k8s/builder/testdata/tls-cert.deployment.golden.yaml similarity index 72% rename from pkg/apis/v1alpha1/testdata/tls-cert.deployment.golden.yaml rename to internal/k8s/builder/testdata/tls-cert.deployment.golden.yaml index 34afd11f6..9b37d8e3a 100644 --- a/pkg/apis/v1alpha1/testdata/tls-cert.deployment.golden.yaml +++ b/internal/k8s/builder/testdata/tls-cert.deployment.golden.yaml @@ -27,29 +27,25 @@ spec: spec: containers: - command: - - /bootstrap/consul-api-gateway - - exec - - -log-json - - -log-level - - info - - -gateway-host - - $(IP) - - -gateway-name - - tls-cert-test - - -consul-http-address - - $(HOST_IP) - - -consul-http-port - - "8500" - - -consul-xds-port - - "8502" - - -envoy-bootstrap-path - - /bootstrap/envoy.json - - -envoy-sds-address - - consul-api-gateway-controller.default.svc.cluster.local - - -envoy-sds-port - - "9090" - - -consul-ca-cert-file - - /ca/tls.crt + - /bin/sh + - -ec + - |2- + + export CONSUL_CACERT=/consul/tls/ca.pem + cat </consul/tls/ca.pem + CONSUL_CA_MOCKED + EOF + + exec /bootstrap/consul-api-gateway exec -log-json \ + -log-level info \ + -gateway-host "$(IP)" \ + -gateway-name tls-cert-test \ + -consul-http-address $(HOST_IP) \ + -consul-http-port 8500 \ + -consul-xds-port 8502 \ + -envoy-bootstrap-path /bootstrap/envoy.json \ + -envoy-sds-address consul-api-gateway-controller.default.svc.cluster.local \ + -envoy-sds-port 9090 env: - name: IP valueFrom: @@ -78,15 +74,14 @@ spec: name: bootstrap - mountPath: /certs name: certs - - mountPath: /ca + - mountPath: /consul/tls name: ca - readOnly: true initContainers: - command: - cp - /bin/consul-api-gateway - /bootstrap/consul-api-gateway - image: hashicorp/consul-api-gateway:0.1.0-techpreview + image: hashicorp/consul-api-gateway:0.1.0-beta name: consul-api-gateway-init resources: {} volumeMounts: @@ -94,15 +89,13 @@ spec: name: bootstrap - mountPath: /certs name: certs - - mountPath: /ca + - mountPath: /consul/tls name: ca - readOnly: true volumes: - emptyDir: {} name: bootstrap - emptyDir: {} name: certs - - name: ca - secret: - secretName: super-secret + - emptyDir: {} + name: ca status: {} diff --git a/pkg/apis/v1alpha1/testdata/tls-cert.service.golden.yaml b/internal/k8s/builder/testdata/tls-cert.service.golden.yaml similarity index 100% rename from pkg/apis/v1alpha1/testdata/tls-cert.service.golden.yaml rename to internal/k8s/builder/testdata/tls-cert.service.golden.yaml diff --git a/pkg/apis/v1alpha1/testdata/tls-cert.yaml b/internal/k8s/builder/testdata/tls-cert.yaml similarity index 100% rename from pkg/apis/v1alpha1/testdata/tls-cert.yaml rename to internal/k8s/builder/testdata/tls-cert.yaml diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index de3324acb..b798e6ffd 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -19,7 +19,6 @@ import ( "github.com/hashicorp/consul-api-gateway/internal/k8s/controllers" "github.com/hashicorp/consul-api-gateway/internal/k8s/gatewayclient" "github.com/hashicorp/consul-api-gateway/internal/k8s/reconciler" - "github.com/hashicorp/consul-api-gateway/internal/k8s/utils" "github.com/hashicorp/consul-api-gateway/internal/store" apigwv1alpha1 "github.com/hashicorp/consul-api-gateway/pkg/apis/v1alpha1" ) @@ -45,37 +44,32 @@ func init() { } type Kubernetes struct { - sDSServerHost string - sDSServerPort int - k8sManager ctrl.Manager - consul *api.Client - store store.Store - logger hclog.Logger + config *Config + k8sManager ctrl.Manager + consul *api.Client + store store.Store + logger hclog.Logger } type Config struct { - CACertSecretNamespace string - CACertSecret string - CACertFile string - SDSServerHost string - SDSServerPort int - MetricsBindAddr string - HealthProbeBindAddr string - WebhookPort int - RestConfig *rest.Config - Namespace string + CACert string + SDSServerHost string + SDSServerPort int + MetricsBindAddr string + HealthProbeBindAddr string + WebhookPort int + RestConfig *rest.Config + Namespace string } func Defaults() *Config { return &Config{ - CACertSecretNamespace: "default", - CACertSecret: "", - CACertFile: "", - SDSServerHost: "consul-api-gateway-controller.default.svc.cluster.local", - SDSServerPort: 9090, - MetricsBindAddr: ":8080", - HealthProbeBindAddr: ":8081", - WebhookPort: 8443, + CACert: "", + SDSServerHost: "consul-api-gateway-controller.default.svc.cluster.local", + SDSServerPort: 9090, + MetricsBindAddr: ":8080", + HealthProbeBindAddr: ":8081", + WebhookPort: 8443, } } @@ -111,17 +105,10 @@ func New(logger hclog.Logger, config *Config) (*Kubernetes, error) { return nil, fmt.Errorf("unable to set up ready check: %w", err) } - if config.CACertSecret != "" && config.CACertFile != "" { - if err := utils.WriteSecretCertFile(config.RestConfig, config.CACertSecret, config.CACertFile, config.CACertSecretNamespace); err != nil { - return nil, fmt.Errorf("unable to write CA cert file: %w", err) - } - } - return &Kubernetes{ - k8sManager: mgr, - sDSServerHost: config.SDSServerHost, - sDSServerPort: config.SDSServerPort, - logger: logger.Named("k8s"), + k8sManager: mgr, + config: config, + logger: logger.Named("k8s"), }, nil } @@ -146,12 +133,11 @@ func (k *Kubernetes) Start(ctx context.Context) error { ControllerName: ControllerName, Client: gwClient, Consul: k.consul, - SDSConfig: apigwv1alpha1.SDSConfig{ - Host: k.sDSServerHost, - Port: k.sDSServerPort, - }, - Logger: k.logger.Named("Reconciler"), - Store: k.store, + ConsulCA: k.config.CACert, + SDSHost: k.config.SDSServerHost, + SDSPort: k.config.SDSServerPort, + Logger: k.logger.Named("Reconciler"), + Store: k.store, }) err := (&controllers.GatewayClassConfigReconciler{ @@ -192,5 +178,15 @@ func (k *Kubernetes) Start(ctx context.Context) error { return fmt.Errorf("failed to create http route controller: %w", err) } + err = (&controllers.TCPRouteReconciler{ + Client: gwClient, + Log: k.logger.Named("TCPRoute"), + Manager: reconcileManager, + ControllerName: ControllerName, + }).SetupWithManager(k.k8sManager) + if err != nil { + return fmt.Errorf("failed to create tcp route controller: %w", err) + } + return k.k8sManager.Start(ctx) } diff --git a/internal/k8s/controllers/gateway_controller.go b/internal/k8s/controllers/gateway_controller.go index 63b2da367..ff112a42c 100644 --- a/internal/k8s/controllers/gateway_controller.go +++ b/internal/k8s/controllers/gateway_controller.go @@ -38,6 +38,7 @@ type GatewayReconciler struct { //+kubebuilder:rbac:groups=core,resources=pods,verbs=list;watch //+kubebuilder:rbac:groups=apps,resources=deployments,verbs=list;get;create;watch //+kubebuilder:rbac:groups=core,resources=services,verbs=list;get;create;watch +//+kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=list;get;create;watch //+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch // Reconcile is part of the main kubernetes reconciliation loop which aims to @@ -82,6 +83,7 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&gateway.Gateway{}). Owns(&appsv1.Deployment{}). Owns(&corev1.Service{}). + Owns(&corev1.ServiceAccount{}). Watches( &source.Kind{Type: &corev1.Pod{}}, handler.EnqueueRequestsFromMapFunc(podToGatewayRequest), diff --git a/internal/k8s/controllers/http_route_controller.go b/internal/k8s/controllers/http_route_controller.go index 8050aa47a..a2cdc9bdd 100644 --- a/internal/k8s/controllers/http_route_controller.go +++ b/internal/k8s/controllers/http_route_controller.go @@ -28,7 +28,7 @@ type HTTPRouteReconciler struct { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.8.3/pkg/reconcile func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := r.Log.With("route", req.NamespacedName) + logger := r.Log.With("http-route", req.NamespacedName) route, err := r.Client.GetHTTPRoute(ctx, req.NamespacedName) if err != nil { diff --git a/internal/k8s/controllers/tcp_route_controller.go b/internal/k8s/controllers/tcp_route_controller.go new file mode 100644 index 000000000..8d691069d --- /dev/null +++ b/internal/k8s/controllers/tcp_route_controller.go @@ -0,0 +1,57 @@ +package controllers + +import ( + "context" + + ctrl "sigs.k8s.io/controller-runtime" + gateway "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "github.com/hashicorp/consul-api-gateway/internal/k8s/gatewayclient" + "github.com/hashicorp/consul-api-gateway/internal/k8s/reconciler" + "github.com/hashicorp/go-hclog" +) + +// TCPRouteReconciler reconciles a HTTPRoute object +type TCPRouteReconciler struct { + Client gatewayclient.Client + Log hclog.Logger + ControllerName string + Manager reconciler.ReconcileManager +} + +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tcproutes,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tcproutes/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tcproutes/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.8.3/pkg/reconcile +func (r *TCPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := r.Log.With("tcp-route", req.NamespacedName) + + route, err := r.Client.GetTCPRoute(ctx, req.NamespacedName) + if err != nil { + logger.Error("failed to get tcp route", "error", err) + return ctrl.Result{}, err + } + + if route == nil { + // clean up cached resources + err := r.Manager.DeleteTCPRoute(ctx, req.NamespacedName) + return ctrl.Result{}, err + } + + // let the route get upserted so long as there's a single gateway we control + // that it's managed by -- the underlying reconciliation code will handle the + // validation of gateway attachment + err = r.Manager.UpsertTCPRoute(ctx, route) + return ctrl.Result{}, err +} + +// SetupWithManager sets up the controller with the Manager. +func (r *TCPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&gateway.TCPRoute{}). + Complete(gatewayclient.NewRequeueingMiddleware(r.Log, r)) +} diff --git a/internal/k8s/gatewayclient/gatewayclient.go b/internal/k8s/gatewayclient/gatewayclient.go index 4948e2335..6f44db460 100644 --- a/internal/k8s/gatewayclient/gatewayclient.go +++ b/internal/k8s/gatewayclient/gatewayclient.go @@ -2,6 +2,7 @@ package gatewayclient import ( "context" + "errors" apps "k8s.io/api/apps/v1" core "k8s.io/api/core/v1" @@ -32,6 +33,7 @@ type Client interface { GetSecret(ctx context.Context, key types.NamespacedName) (*core.Secret, error) GetService(ctx context.Context, key types.NamespacedName) (*core.Service, error) GetHTTPRoute(ctx context.Context, key types.NamespacedName) (*gateway.HTTPRoute, error) + GetTCPRoute(ctx context.Context, key types.NamespacedName) (*gateway.TCPRoute, error) // finalizer helpers @@ -65,6 +67,7 @@ type Client interface { CreateOrUpdateDeployment(ctx context.Context, deployment *apps.Deployment, mutators ...func() error) (bool, error) CreateOrUpdateService(ctx context.Context, service *core.Service, mutators ...func() error) (bool, error) DeleteService(ctx context.Context, service *core.Service) error + EnsureServiceAccount(ctx context.Context, owner *gateway.Gateway, serviceAccount *core.ServiceAccount) error } type gatewayClient struct { @@ -213,6 +216,17 @@ func (g *gatewayClient) GetHTTPRoute(ctx context.Context, key types.NamespacedNa return route, nil } +func (g *gatewayClient) GetTCPRoute(ctx context.Context, key types.NamespacedName) (*gateway.TCPRoute, error) { + route := &gateway.TCPRoute{} + if err := g.Client.Get(ctx, key, route); err != nil { + if k8serrors.IsNotFound(err) { + return nil, nil + } + return nil, NewK8sError(err) + } + return route, nil +} + func (g *gatewayClient) EnsureFinalizer(ctx context.Context, object client.Object, finalizer string) (bool, error) { finalizers := object.GetFinalizers() for _, f := range finalizers { @@ -306,6 +320,31 @@ func (g *gatewayClient) DeleteService(ctx context.Context, service *core.Service return nil } +func (g *gatewayClient) EnsureServiceAccount(ctx context.Context, owner *gateway.Gateway, serviceAccount *core.ServiceAccount) error { + created := &core.ServiceAccount{} + key := types.NamespacedName{Name: serviceAccount.Name, Namespace: serviceAccount.Namespace} + if err := g.Client.Get(ctx, key, created); err != nil { + if k8serrors.IsNotFound(err) { + if err := g.SetControllerOwnership(owner, serviceAccount); err != nil { + return err + } + if err := g.Client.Create(ctx, serviceAccount); err != nil { + return NewK8sError(err) + } + return nil + } + return NewK8sError(err) + } + for _, ref := range created.GetOwnerReferences() { + if ref.UID == owner.GetUID() && ref.Name == owner.GetName() { + // we found proper ownership + return nil + } + } + // we found the object, but we're not the owner of it, return an error + return errors.New("service account not owned by the gateway") +} + func (g *gatewayClient) SetControllerOwnership(owner, object client.Object) error { if err := ctrl.SetControllerReference(owner, object, g.scheme); err != nil { return NewK8sError(err) diff --git a/internal/k8s/gatewayclient/mocks/gatewayclient.go b/internal/k8s/gatewayclient/mocks/gatewayclient.go index c83a23afc..3654af61a 100644 --- a/internal/k8s/gatewayclient/mocks/gatewayclient.go +++ b/internal/k8s/gatewayclient/mocks/gatewayclient.go @@ -124,6 +124,20 @@ func (mr *MockClientMockRecorder) EnsureFinalizer(ctx, object, finalizer interfa return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureFinalizer", reflect.TypeOf((*MockClient)(nil).EnsureFinalizer), ctx, object, finalizer) } +// EnsureServiceAccount mocks base method. +func (m *MockClient) EnsureServiceAccount(ctx context.Context, owner *v1alpha2.Gateway, serviceAccount *v10.ServiceAccount) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureServiceAccount", ctx, owner, serviceAccount) + ret0, _ := ret[0].(error) + return ret0 +} + +// EnsureServiceAccount indicates an expected call of EnsureServiceAccount. +func (mr *MockClientMockRecorder) EnsureServiceAccount(ctx, owner, serviceAccount interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureServiceAccount", reflect.TypeOf((*MockClient)(nil).EnsureServiceAccount), ctx, owner, serviceAccount) +} + // GatewayClassConfigInUse mocks base method. func (m *MockClient) GatewayClassConfigInUse(ctx context.Context, gcc *v1alpha1.GatewayClassConfig) (bool, error) { m.ctrl.T.Helper() @@ -260,6 +274,21 @@ func (mr *MockClientMockRecorder) GetService(ctx, key interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetService", reflect.TypeOf((*MockClient)(nil).GetService), ctx, key) } +// GetTCPRoute mocks base method. +func (m *MockClient) GetTCPRoute(ctx context.Context, key types.NamespacedName) (*v1alpha2.TCPRoute, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTCPRoute", ctx, key) + ret0, _ := ret[0].(*v1alpha2.TCPRoute) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTCPRoute indicates an expected call of GetTCPRoute. +func (mr *MockClientMockRecorder) GetTCPRoute(ctx, key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTCPRoute", reflect.TypeOf((*MockClient)(nil).GetTCPRoute), ctx, key) +} + // HasManagedDeployment mocks base method. func (m *MockClient) HasManagedDeployment(ctx context.Context, gw *v1alpha2.Gateway) (bool, error) { m.ctrl.T.Helper() diff --git a/internal/k8s/reconciler/config/errors.yaml b/internal/k8s/reconciler/config/errors.yaml index aeeb2f215..57555c492 100644 --- a/internal/k8s/reconciler/config/errors.yaml +++ b/internal/k8s/reconciler/config/errors.yaml @@ -1,4 +1,4 @@ - name: CertificateResolution types: ["NotFound","Unsupported"] - name: Bind - types: ["RouteKind","ListenerNamespacePolicy","HostnameMismatch"] \ No newline at end of file + types: ["RouteKind","ListenerNamespacePolicy","HostnameMismatch","RouteInvalid"] \ No newline at end of file diff --git a/internal/k8s/reconciler/gateway.go b/internal/k8s/reconciler/gateway.go index a0ecd964f..09cf3d63a 100644 --- a/internal/k8s/reconciler/gateway.go +++ b/internal/k8s/reconciler/gateway.go @@ -8,23 +8,26 @@ import ( "reflect" "strings" + corev1 "k8s.io/api/core/v1" + gw "sigs.k8s.io/gateway-api/apis/v1alpha2" + "github.com/hashicorp/consul-api-gateway/internal/core" + "github.com/hashicorp/consul-api-gateway/internal/k8s/builder" "github.com/hashicorp/consul-api-gateway/internal/k8s/gatewayclient" "github.com/hashicorp/consul-api-gateway/internal/k8s/utils" "github.com/hashicorp/consul-api-gateway/internal/store" apigwv1alpha1 "github.com/hashicorp/consul-api-gateway/pkg/apis/v1alpha1" "github.com/hashicorp/go-hclog" - corev1 "k8s.io/api/core/v1" - gw "sigs.k8s.io/gateway-api/apis/v1alpha2" ) type K8sGateway struct { - consulNamespace string - logger hclog.Logger - client gatewayclient.Client - gateway *gw.Gateway - config apigwv1alpha1.GatewayClassConfig - sdsConfig apigwv1alpha1.SDSConfig + consulNamespace string + logger hclog.Logger + client gatewayclient.Client + gateway *gw.Gateway + config apigwv1alpha1.GatewayClassConfig + deploymentBuilder builder.DeploymentBuilder + serviceBuilder builder.ServiceBuilder status GatewayStatus podReady bool @@ -36,7 +39,9 @@ var _ store.StatusTrackingGateway = &K8sGateway{} type K8sGatewayConfig struct { ConsulNamespace string - SDSConfig apigwv1alpha1.SDSConfig + ConsulCA string + SDSHost string + SDSPort int Config apigwv1alpha1.GatewayClassConfig Logger hclog.Logger Client gatewayclient.Client @@ -54,21 +59,29 @@ func NewK8sGateway(gateway *gw.Gateway, config K8sGatewayConfig) *K8sGateway { listeners[k8sListener.ID()] = k8sListener } + deployment := builder.NewGatewayDeployment(gateway) + deployment.WithSDS(config.SDSHost, config.SDSPort) + deployment.WithClassConfig(config.Config) + deployment.WithConsulCA(config.ConsulCA) + service := builder.NewGatewayService(gateway) + service.WithClassConfig(config.Config) + return &K8sGateway{ - config: config.Config, - sdsConfig: config.SDSConfig, - consulNamespace: config.ConsulNamespace, - logger: gatewayLogger, - client: config.Client, - gateway: gateway, - listeners: listeners, + config: config.Config, + deploymentBuilder: deployment, + serviceBuilder: service, + consulNamespace: config.ConsulNamespace, + logger: gatewayLogger, + client: config.Client, + gateway: gateway, + listeners: listeners, } } func (g *K8sGateway) certificates() []string { certificates := []string{} for _, listener := range g.listeners { - certificates = append(certificates, listener.Certificates()...) + certificates = append(certificates, listener.Config().TLS.Certificates...) } return certificates } @@ -274,11 +287,6 @@ func (g *K8sGateway) ShouldBind(route store.Route) bool { return false } - if !k8sRoute.IsValid() { - g.logger.Trace("route is invalid, should not bind", "route", route.ID()) - return false - } - for _, ref := range k8sRoute.CommonRouteSpec().ParentRefs { if namespacedName, isGateway := utils.ReferencesGateway(k8sRoute.GetNamespace(), ref); isGateway { if utils.NamespacedName(g.gateway) == namespacedName { @@ -286,7 +294,6 @@ func (g *K8sGateway) ShouldBind(route store.Route) bool { } } } - g.logger.Trace("route does not reference gateway, should not bind", "route", route.ID()) return false } @@ -370,7 +377,14 @@ func (g *K8sGateway) TrackSync(ctx context.Context, sync func() (bool, error)) e } func (g *K8sGateway) ensureDeploymentExists(ctx context.Context) error { - deployment := g.config.DeploymentFor(g.gateway, g.sdsConfig) + // Create service account for the gateway + if serviceAccount := g.config.ServiceAccountFor(g.gateway); serviceAccount != nil { + if err := g.client.EnsureServiceAccount(ctx, g.gateway, serviceAccount); err != nil { + return err + } + } + + deployment := g.deploymentBuilder.Build() mutated := deployment.DeepCopy() if updated, err := g.client.CreateOrUpdateDeployment(ctx, mutated, func() error { mutated = apigwv1alpha1.MergeDeployment(deployment, mutated) @@ -387,7 +401,7 @@ func (g *K8sGateway) ensureDeploymentExists(ctx context.Context) error { } // Create service for the gateway - if service := g.config.ServiceFor(g.gateway); service != nil { + if service := g.serviceBuilder.Build(); service != nil { mutated := service.DeepCopy() if updated, err := g.client.CreateOrUpdateService(ctx, mutated, func() error { mutated = apigwv1alpha1.MergeService(service, mutated) diff --git a/internal/k8s/reconciler/gateway_test.go b/internal/k8s/reconciler/gateway_test.go index 5d770edfb..8b6b99f2a 100644 --- a/internal/k8s/reconciler/gateway_test.go +++ b/internal/k8s/reconciler/gateway_test.go @@ -451,10 +451,10 @@ func TestGatewayCompare(t *testing.T) { }, K8sGatewayConfig{ Logger: hclog.NewNullLogger(), }) - gateway.listeners["1"].certificates = []string{"other"} + gateway.listeners["1"].tls.Certificates = []string{"other"} require.Equal(t, store.CompareResultNotEqual, gateway.Compare(other)) - gateway.listeners["1"].certificates = nil + gateway.listeners["1"].tls.Certificates = []string{} gateway.status.Scheduled.Unknown = errors.New("") require.Equal(t, store.CompareResultNotEqual, gateway.Compare(other)) diff --git a/internal/k8s/reconciler/http_route.go b/internal/k8s/reconciler/http_route.go index 96ab8fc96..7130d123a 100644 --- a/internal/k8s/reconciler/http_route.go +++ b/internal/k8s/reconciler/http_route.go @@ -51,11 +51,13 @@ var methodMappings = map[gw.HTTPMethod]core.HTTPMethod{ var pathMappings = map[gw.PathMatchType]core.HTTPPathMatchType{ gw.PathMatchExact: core.HTTPPathMatchExactType, + gw.PathMatchPathPrefix: core.HTTPPathMatchPrefixType, gw.PathMatchRegularExpression: core.HTTPPathMatchRegularExpressionType, } var queryMappings = map[gw.QueryParamMatchType]core.HTTPQueryMatchType{ - gw.QueryParamMatchExact: core.HTTPQueryMatchExactType, + gw.QueryParamMatchExact: core.HTTPQueryMatchExactType, + gw.QueryParamMatchRegularExpression: core.HTTPQueryMatchRegularExpressionType, } var headerMappings = map[gw.HeaderMatchType]core.HTTPHeaderMatchType{ diff --git a/internal/k8s/reconciler/listener.go b/internal/k8s/reconciler/listener.go index 581983579..44ef86622 100644 --- a/internal/k8s/reconciler/listener.go +++ b/internal/k8s/reconciler/listener.go @@ -4,8 +4,11 @@ import ( "context" "errors" "fmt" + "strings" "sync/atomic" + "github.com/hashicorp/consul-api-gateway/internal/common" + "github.com/hashicorp/consul-api-gateway/internal/core" "github.com/hashicorp/consul-api-gateway/internal/k8s/gatewayclient" "github.com/hashicorp/consul-api-gateway/internal/k8s/utils" "github.com/hashicorp/consul-api-gateway/internal/store" @@ -25,11 +28,19 @@ var ( Group: (*gw.Group)(&gw.GroupVersion.Group), Kind: "HTTPRoute", }}, + gw.TCPProtocolType: {{ + Group: (*gw.Group)(&gw.GroupVersion.Group), + Kind: "TCPRoute", + }}, } ) const ( - defaultListenerName = "default" + defaultListenerName = "default" + annotationKeyPrefix = "api-gateway.consul.hashicorp.com/" + tlsMinVersionAnnotationKey = annotationKeyPrefix + "tls_min_version" + tlsMaxVersionAnnotationKey = annotationKeyPrefix + "tls_max_version" + tlsCipherSuitesAnnotationKey = annotationKeyPrefix + "tls_cipher_suites" ) type K8sListener struct { @@ -40,8 +51,8 @@ type K8sListener struct { client gatewayclient.Client status ListenerStatus + tls core.TLSParams routeCount int32 - certificates []string supportedKinds []gw.RouteGroupKind } @@ -69,10 +80,6 @@ func (l *K8sListener) ID() string { return string(l.listener.Name) } -func (l *K8sListener) Certificates() []string { - return l.certificates -} - func (l *K8sListener) Validate(ctx context.Context) error { l.validateUnsupported() l.validateProtocols() @@ -91,7 +98,8 @@ func (l *K8sListener) Validate(ctx context.Context) error { func (l *K8sListener) validateTLS(ctx context.Context) error { if l.listener.TLS == nil { - if l.Config().TLS { + // TODO: should this struct field be "Required" instead of "Enabled"? + if l.Config().TLS.Enabled { // we are using a protocol that requires TLS but has no TLS // configured l.status.Ready.Invalid = errors.New("tls configuration required for the given protocol") @@ -119,12 +127,81 @@ func (l *K8sListener) validateTLS(ctx context.Context) error { } l.status.ResolvedRefs.InvalidCertificateRef = certificateErr } else { - l.certificates = []string{resource} + l.tls.Certificates = []string{resource} + } + + if l.listener.TLS.Options != nil { + tlsMinVersion := l.listener.TLS.Options[tlsMinVersionAnnotationKey] + tlsMaxVersion := l.listener.TLS.Options[tlsMaxVersionAnnotationKey] + tlsCipherSuitesStr := l.listener.TLS.Options[tlsCipherSuitesAnnotationKey] + + if tlsMinVersion != "" { + if _, ok := supportedTlsVersions[string(tlsMinVersion)]; !ok { + l.status.Ready.Invalid = errors.New("unrecognized TLS min version") + return nil + } + + if tlsCipherSuitesStr != "" { + if _, ok := tlsVersionsWithConfigurableCipherSuites[string(tlsMinVersion)]; !ok { + l.status.Ready.Invalid = errors.New("configuring TLS cipher suites is only supported for TLS 1.2 and earlier") + return nil + } + } + + l.tls.MinVersion = string(tlsMinVersion) + } + + if tlsMaxVersion != "" { + if _, ok := supportedTlsVersions[string(tlsMaxVersion)]; !ok { + l.status.Ready.Invalid = errors.New("unrecognized TLS max version") + return nil + } + + l.tls.MaxVersion = string(tlsMaxVersion) + } + + if tlsCipherSuitesStr != "" { + // split comma delimited string into string array and trim whitespace + tlsCipherSuitesUntrimmed := strings.Split(string(tlsCipherSuitesStr), ",") + tlsCipherSuites := tlsCipherSuitesUntrimmed[:0] + for _, c := range tlsCipherSuitesUntrimmed { + tlsCipherSuites = append(tlsCipherSuites, strings.TrimSpace(c)) + } + + // validate each cipher suite in array + for _, c := range tlsCipherSuites { + if ok := common.SupportedTLSCipherSuite(c); !ok { + l.status.Ready.Invalid = fmt.Errorf("unrecognized or unsupported TLS cipher suite: %s", c) + return nil + } + } + + // set cipher suites on listener TLS params + l.tls.CipherSuites = tlsCipherSuites + } } return nil } +var supportedTlsVersions = map[string]struct{}{ + "TLS_AUTO": {}, + "TLSv1_0": {}, + "TLSv1_1": {}, + "TLSv1_2": {}, + "TLSv1_3": {}, +} + +var tlsVersionsWithConfigurableCipherSuites = map[string]struct{}{ + // Remove these two if Envoy ever sets TLS 1.3 as default minimum + "": {}, + "TLS_AUTO": {}, + + "TLSv1_0": {}, + "TLSv1_1": {}, + "TLSv1_2": {}, +} + func (l *K8sListener) validateUnsupported() { // seems weird that we're looking at gateway fields for listener status // but that's the weirdness of the spec @@ -214,12 +291,16 @@ func (l *K8sListener) Config() store.ListenerConfig { hostname = string(*l.listener.Hostname) } protocol, tls := utils.ProtocolToConsul(l.listener.Protocol) + + // Update listener TLS config to specify whether TLS is required by the protocol + l.tls.Enabled = tls + return store.ListenerConfig{ Name: name, Hostname: hostname, Port: int(l.listener.Port), Protocol: protocol, - TLS: tls, + TLS: l.tls, } } @@ -260,6 +341,8 @@ func (l *K8sListener) canBind(ref gw.ParentRef, route *K8sRoute) (bool, error) { return false, nil } + l.logger.Trace("checking listener match", "expected", l.listener.Name, "found", ref.SectionName) + // must is only true if there's a ref with a specific listener name // meaning if we must attach, but cannot, it's an error allowed, must := routeMatchesListener(l.listener.Name, ref.SectionName) @@ -291,6 +374,10 @@ func (l *K8sListener) canBind(ref gw.ParentRef, route *K8sRoute) (bool, error) { return false, nil } + // check if the route is valid, if not, then return a status about it being rejected + if !route.IsValid() { + return false, NewBindErrorRouteInvalid("route is in an invalid state and cannot bind") + } return true, nil } @@ -307,10 +394,28 @@ func (l *K8sListener) OnRouteRemoved(_ string) { } func (l *K8sListener) Status() gw.ListenerStatus { + routeCount := atomic.LoadInt32(&l.routeCount) + if l.listener.Protocol == gw.TCPProtocolType { + if routeCount > 1 { + l.status.Conflicted.RouteConflict = errors.New("only a single TCP route can be bound to a TCP listener") + } else { + l.status.Conflicted.RouteConflict = nil + } + } return gw.ListenerStatus{ Name: l.listener.Name, SupportedKinds: l.supportedKinds, - AttachedRoutes: atomic.LoadInt32(&l.routeCount), + AttachedRoutes: routeCount, Conditions: l.status.Conditions(l.gateway.Generation), } } + +func (l *K8sListener) IsValid() bool { + routeCount := atomic.LoadInt32(&l.routeCount) + if l.listener.Protocol == gw.TCPProtocolType { + if routeCount > 1 { + return false + } + } + return l.status.Valid() +} diff --git a/internal/k8s/reconciler/listener_test.go b/internal/k8s/reconciler/listener_test.go index e37b3eb6e..d79dfdd99 100644 --- a/internal/k8s/reconciler/listener_test.go +++ b/internal/k8s/reconciler/listener_test.go @@ -6,12 +6,13 @@ import ( "testing" "github.com/golang/mock/gomock" + "github.com/hashicorp/consul-api-gateway/internal/core" "github.com/hashicorp/consul-api-gateway/internal/k8s/gatewayclient/mocks" "github.com/hashicorp/consul-api-gateway/internal/store" storeMocks "github.com/hashicorp/consul-api-gateway/internal/store/mocks" "github.com/hashicorp/go-hclog" "github.com/stretchr/testify/require" - core "k8s.io/api/core/v1" + k8s "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" gw "sigs.k8s.io/gateway-api/apis/v1alpha2" @@ -150,14 +151,14 @@ func TestListenerValidate(t *testing.T) { Logger: hclog.NewNullLogger(), Client: client, }) - client.EXPECT().GetSecret(gomock.Any(), gomock.Any()).Return(&core.Secret{ + client.EXPECT().GetSecret(gomock.Any(), gomock.Any()).Return(&k8s.Secret{ ObjectMeta: meta.ObjectMeta{ Name: "secret", }, }, nil) - require.Len(t, listener.Certificates(), 0) + require.Len(t, listener.Config().TLS.Certificates, 0) require.NoError(t, listener.Validate(context.Background())) - require.Len(t, listener.Certificates(), 1) + require.Len(t, listener.Config().TLS.Certificates, 1) group := gw.Group("group") kind := gw.Kind("kind") @@ -179,6 +180,127 @@ func TestListenerValidate(t *testing.T) { require.NoError(t, listener.Validate(context.Background())) condition = listener.status.ResolvedRefs.Condition(0) require.Equal(t, ListenerConditionReasonInvalidCertificateRef, condition.Reason) + + listener = NewK8sListener(&gw.Gateway{}, gw.Listener{ + Protocol: gw.HTTPSProtocolType, + TLS: &gw.GatewayTLSConfig{ + CertificateRefs: []*gw.SecretObjectReference{{ + Name: "secret", + }}, + Options: map[gw.AnnotationKey]gw.AnnotationValue{ + "api-gateway.consul.hashicorp.com/tls_min_version": "TLSv1_2", + }, + }, + }, K8sListenerConfig{ + Logger: hclog.NewNullLogger(), + Client: client, + }) + client.EXPECT().GetSecret(gomock.Any(), gomock.Any()).Return(&k8s.Secret{ + ObjectMeta: meta.ObjectMeta{ + Name: "secret", + }, + }, nil) + require.NoError(t, listener.Validate(context.Background())) + condition = listener.status.Ready.Condition(0) + require.Equal(t, ListenerConditionReasonReady, condition.Reason) + require.Equal(t, "TLSv1_2", listener.tls.MinVersion) + + listener = NewK8sListener(&gw.Gateway{}, gw.Listener{ + Protocol: gw.HTTPSProtocolType, + TLS: &gw.GatewayTLSConfig{ + CertificateRefs: []*gw.SecretObjectReference{{ + Name: "secret", + }}, + Options: map[gw.AnnotationKey]gw.AnnotationValue{ + "api-gateway.consul.hashicorp.com/tls_min_version": "foo", + }, + }, + }, K8sListenerConfig{ + Logger: hclog.NewNullLogger(), + Client: client, + }) + client.EXPECT().GetSecret(gomock.Any(), gomock.Any()).Return(&k8s.Secret{ + ObjectMeta: meta.ObjectMeta{ + Name: "secret", + }, + }, nil) + require.NoError(t, listener.Validate(context.Background())) + condition = listener.status.Ready.Condition(0) + require.Equal(t, ListenerConditionReasonInvalid, condition.Reason) + require.Equal(t, "unrecognized TLS min version", condition.Message) + + listener = NewK8sListener(&gw.Gateway{}, gw.Listener{ + Protocol: gw.HTTPSProtocolType, + TLS: &gw.GatewayTLSConfig{ + CertificateRefs: []*gw.SecretObjectReference{{ + Name: "secret", + }}, + Options: map[gw.AnnotationKey]gw.AnnotationValue{ + "api-gateway.consul.hashicorp.com/tls_cipher_suites": "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + }, + }, + }, K8sListenerConfig{ + Logger: hclog.NewNullLogger(), + Client: client, + }) + client.EXPECT().GetSecret(gomock.Any(), gomock.Any()).Return(&k8s.Secret{ + ObjectMeta: meta.ObjectMeta{ + Name: "secret", + }, + }, nil) + require.NoError(t, listener.Validate(context.Background())) + condition = listener.status.Ready.Condition(0) + require.Equal(t, ListenerConditionReasonReady, condition.Reason) + require.Equal(t, []string{"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"}, listener.tls.CipherSuites) + + listener = NewK8sListener(&gw.Gateway{}, gw.Listener{ + Protocol: gw.HTTPSProtocolType, + TLS: &gw.GatewayTLSConfig{ + CertificateRefs: []*gw.SecretObjectReference{{ + Name: "secret", + }}, + Options: map[gw.AnnotationKey]gw.AnnotationValue{ + "api-gateway.consul.hashicorp.com/tls_min_version": "TLSv1_3", + "api-gateway.consul.hashicorp.com/tls_cipher_suites": "foo", + }, + }, + }, K8sListenerConfig{ + Logger: hclog.NewNullLogger(), + Client: client, + }) + client.EXPECT().GetSecret(gomock.Any(), gomock.Any()).Return(&k8s.Secret{ + ObjectMeta: meta.ObjectMeta{ + Name: "secret", + }, + }, nil) + require.NoError(t, listener.Validate(context.Background())) + condition = listener.status.Ready.Condition(0) + require.Equal(t, ListenerConditionReasonInvalid, condition.Reason) + require.Equal(t, "configuring TLS cipher suites is only supported for TLS 1.2 and earlier", condition.Message) + + listener = NewK8sListener(&gw.Gateway{}, gw.Listener{ + Protocol: gw.HTTPSProtocolType, + TLS: &gw.GatewayTLSConfig{ + CertificateRefs: []*gw.SecretObjectReference{{ + Name: "secret", + }}, + Options: map[gw.AnnotationKey]gw.AnnotationValue{ + "api-gateway.consul.hashicorp.com/tls_cipher_suites": "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, foo", + }, + }, + }, K8sListenerConfig{ + Logger: hclog.NewNullLogger(), + Client: client, + }) + client.EXPECT().GetSecret(gomock.Any(), gomock.Any()).Return(&k8s.Secret{ + ObjectMeta: meta.ObjectMeta{ + Name: "secret", + }, + }, nil) + require.NoError(t, listener.Validate(context.Background())) + condition = listener.status.Ready.Condition(0) + require.Equal(t, ListenerConditionReasonInvalid, condition.Reason) + require.Equal(t, "unrecognized or unsupported TLS cipher suite: foo", condition.Message) } func TestIsKindInSet(t *testing.T) { @@ -208,7 +330,7 @@ func TestListenerConfig(t *testing.T) { require.Equal(t, store.ListenerConfig{ Name: "listener", - TLS: false, + TLS: core.TLSParams{Enabled: false}, }, NewK8sListener(&gw.Gateway{}, gw.Listener{ Name: gw.SectionName("listener"), }, K8sListenerConfig{ @@ -219,7 +341,7 @@ func TestListenerConfig(t *testing.T) { require.Equal(t, store.ListenerConfig{ Name: "default", Hostname: "hostname", - TLS: false, + TLS: core.TLSParams{Enabled: false}, }, NewK8sListener(&gw.Gateway{}, gw.Listener{ Hostname: &hostname, }, K8sListenerConfig{ diff --git a/internal/k8s/reconciler/manager.go b/internal/k8s/reconciler/manager.go index 525e0aba8..7b4e1d91e 100644 --- a/internal/k8s/reconciler/manager.go +++ b/internal/k8s/reconciler/manager.go @@ -45,7 +45,9 @@ type GatewayReconcileManager struct { logger hclog.Logger client gatewayclient.Client consul *api.Client - sdsConfig apigwv1alpha1.SDSConfig + consulCA string + sdsHost string + sdsPort int store store.Store gatewayClasses *K8sGatewayClasses @@ -61,7 +63,9 @@ type ManagerConfig struct { ControllerName string Client gatewayclient.Client Consul *api.Client - SDSConfig apigwv1alpha1.SDSConfig + ConsulCA string + SDSHost string + SDSPort int Store store.Store Logger hclog.Logger } @@ -72,7 +76,9 @@ func NewReconcileManager(config ManagerConfig) *GatewayReconcileManager { logger: config.Logger, client: config.Client, consul: config.Consul, - sdsConfig: config.SDSConfig, + consulCA: config.ConsulCA, + sdsHost: config.SDSHost, + sdsPort: config.SDSPort, gatewayClasses: NewK8sGatewayClasses(config.Logger.Named("gatewayclasses"), config.Client), namespaceMap: make(map[types.NamespacedName]string), store: config.Store, @@ -153,9 +159,11 @@ func (m *GatewayReconcileManager) UpsertGateway(ctx context.Context, g *gw.Gatew m.namespaceMap[utils.NamespacedName(g)] = consulNamespace gateway := NewK8sGateway(g, K8sGatewayConfig{ ConsulNamespace: consulNamespace, + ConsulCA: m.consulCA, Logger: m.logger, Client: m.client, - SDSConfig: m.sdsConfig, + SDSHost: m.sdsHost, + SDSPort: m.sdsPort, Config: config, }) diff --git a/internal/k8s/reconciler/route.go b/internal/k8s/reconciler/route.go index bf5ee5d9d..cd60f5a9b 100644 --- a/internal/k8s/reconciler/route.go +++ b/internal/k8s/reconciler/route.go @@ -188,10 +188,24 @@ func (r *K8sRoute) OnBindFailed(err error, gateway store.Gateway) { status.Accepted.ListenerNamespacePolicy = err case BindErrorTypeRouteKind: status.Accepted.InvalidRouteKind = err + case BindErrorTypeRouteInvalid: + status.Accepted.BindError = err } - return + } else { + status.Accepted.BindError = err + } + // set resolution errors - we can do this here because + // a route with resolution errors will always fail to bind + errorType, err := r.resolutionErrors.Flatten() + switch errorType { + case service.GenericResolutionErrorType: + status.ResolvedRefs.Errors = err + case service.ConsulServiceResolutionErrorType: + status.ResolvedRefs.ConsulServiceNotFound = err + case service.K8sServiceResolutionErrorType: + status.ResolvedRefs.ServiceNotFound = err } - status.Accepted.BindError = err + r.parentStatuses[id] = status } } @@ -202,9 +216,10 @@ func (r *K8sRoute) OnBound(gateway store.Gateway) { if ok { id, found := r.parentKeyForGateway(utils.NamespacedName(k8sGateway.gateway)) if found { - // clear out any existing errors on the accepted status + // clear out any existing errors on our statuses if status, statusFound := r.parentStatuses[id]; statusFound { status.Accepted = RouteAcceptedStatus{} + status.ResolvedRefs = RouteResolvedRefsStatus{} } else { r.parentStatuses[id] = &RouteStatus{} } @@ -312,6 +327,14 @@ func (r *K8sRoute) Resolve(listener store.Listener) *core.ResolvedRoute { "consul-api-gateway/k8s/HTTPRoute.Name": r.GetName(), "consul-api-gateway/k8s/HTTPRoute.Namespace": r.GetNamespace(), }, route, r) + case *gw.TCPRoute: + return convertTCPRoute(namespace, prefix, map[string]string{ + "external-source": "consul-api-gateway", + "consul-api-gateway/k8s/Gateway.Name": k8sListener.gateway.Name, + "consul-api-gateway/k8s/Gateway.Namespace": k8sListener.gateway.Namespace, + "consul-api-gateway/k8s/TCPRoute.Name": r.GetName(), + "consul-api-gateway/k8s/TCPRoute.Namespace": r.GetNamespace(), + }, route, r) default: // TODO: add other route types return nil @@ -336,9 +359,11 @@ func (r *K8sRoute) Parents() []gw.ParentRef { func (r *K8sRoute) Validate(ctx context.Context) error { switch route := r.Route.(type) { case *gw.HTTPRoute: - for _, rule := range route.Spec.Rules { + for _, httpRule := range route.Spec.Rules { + rule := httpRule routeRule := service.NewRouteRule(&rule) - for _, ref := range rule.BackendRefs { + for _, backendRef := range rule.BackendRefs { + ref := backendRef reference, err := r.resolver.Resolve(ctx, ref.BackendObjectReference) if err != nil { var resolutionError service.ResolutionError @@ -352,6 +377,36 @@ func (r *K8sRoute) Validate(ctx context.Context) error { r.references.Add(routeRule, *reference) } } + case *gw.TCPRoute: + if len(route.Spec.Rules) != 1 { + err := service.NewResolutionError("a single tcp rule is required") + r.resolutionErrors.Add(err) + return nil + } + + rule := route.Spec.Rules[0] + + if len(rule.BackendRefs) != 1 { + err := service.NewResolutionError("a single backendRef per tcp rule is required") + r.resolutionErrors.Add(err) + return nil + } + + routeRule := service.NewRouteRule(rule) + + ref := rule.BackendRefs[0] + reference, err := r.resolver.Resolve(ctx, ref.BackendObjectReference) + if err != nil { + var resolutionError service.ResolutionError + if !errors.As(err, &resolutionError) { + return err + } + r.resolutionErrors.Add(resolutionError) + return nil + } + + reference.Reference.Set(&ref) + r.references.Add(routeRule, *reference) } return nil diff --git a/internal/k8s/reconciler/tcp_route.go b/internal/k8s/reconciler/tcp_route.go index c1e869ce3..6f9667d61 100644 --- a/internal/k8s/reconciler/tcp_route.go +++ b/internal/k8s/reconciler/tcp_route.go @@ -1,7 +1,46 @@ package reconciler -import "k8s.io/apimachinery/pkg/types" +import ( + "github.com/hashicorp/consul-api-gateway/internal/core" + "github.com/hashicorp/consul-api-gateway/internal/k8s/service" + "k8s.io/apimachinery/pkg/types" + gw "sigs.k8s.io/gateway-api/apis/v1alpha2" +) func TCPRouteID(namespacedName types.NamespacedName) string { return "tcp-" + namespacedName.String() } + +func convertTCPRoute(namespace, prefix string, meta map[string]string, route *gw.TCPRoute, k8sRoute *K8sRoute) *core.ResolvedRoute { + name := prefix + route.Name + + resolved := core.NewTCPRouteBuilder(). + WithName(name). + // we always use the listener namespace for the routes + // themselves, while the services they route to might + // be in different namespaces + WithNamespace(namespace). + WithMeta(meta). + WithService(tcpReferencesToService(k8sRoute.references)). + Build() + return &resolved +} + +func tcpReferencesToService(referenceMap service.RouteRuleReferenceMap) core.ResolvedService { + for _, references := range referenceMap { + for _, reference := range references { + switch reference.Type { + case service.ConsulServiceReference: + // at this point there should only be a single resolved service in the reference map + return core.ResolvedService{ + ConsulNamespace: reference.Consul.Namespace, + Service: reference.Consul.Name, + } + default: + // TODO: support other reference types + continue + } + } + } + return core.ResolvedService{} +} diff --git a/internal/k8s/reconciler/zz_generated_errors.go b/internal/k8s/reconciler/zz_generated_errors.go index fa0d458c2..c0e15c529 100644 --- a/internal/k8s/reconciler/zz_generated_errors.go +++ b/internal/k8s/reconciler/zz_generated_errors.go @@ -35,6 +35,7 @@ const ( BindErrorTypeRouteKind BindErrorType = iota BindErrorTypeListenerNamespacePolicy BindErrorTypeHostnameMismatch + BindErrorTypeRouteInvalid ) type BindError struct { @@ -51,6 +52,9 @@ func NewBindErrorListenerNamespacePolicy(inner string) BindError { func NewBindErrorHostnameMismatch(inner string) BindError { return BindError{inner, BindErrorTypeHostnameMismatch} } +func NewBindErrorRouteInvalid(inner string) BindError { + return BindError{inner, BindErrorTypeRouteInvalid} +} func (r BindError) Error() string { return r.inner diff --git a/internal/k8s/reconciler/zz_generated_errors_test.go b/internal/k8s/reconciler/zz_generated_errors_test.go index 1951473e1..0ef99a5df 100644 --- a/internal/k8s/reconciler/zz_generated_errors_test.go +++ b/internal/k8s/reconciler/zz_generated_errors_test.go @@ -30,4 +30,6 @@ func TestBindErrorType(t *testing.T) { require.Equal(t, BindErrorTypeListenerNamespacePolicy, NewBindErrorListenerNamespacePolicy(expected).Kind()) require.Equal(t, expected, NewBindErrorHostnameMismatch(expected).Error()) require.Equal(t, BindErrorTypeHostnameMismatch, NewBindErrorHostnameMismatch(expected).Kind()) + require.Equal(t, expected, NewBindErrorRouteInvalid(expected).Error()) + require.Equal(t, BindErrorTypeRouteInvalid, NewBindErrorRouteInvalid(expected).Kind()) } diff --git a/internal/k8s/service/resolver.go b/internal/k8s/service/resolver.go index 57b1f3894..2d8266016 100644 --- a/internal/k8s/service/resolver.go +++ b/internal/k8s/service/resolver.go @@ -97,7 +97,7 @@ func (r *ResolutionErrors) String() string { } if len(r.genericErrors) > 0 { - genericErrs := "k8s: " + genericErrs := "" for i, err := range r.genericErrors { if i != 0 { genericErrs += ", " diff --git a/internal/store/interfaces.go b/internal/store/interfaces.go index 172c20db2..97f9413ea 100644 --- a/internal/store/interfaces.go +++ b/internal/store/interfaces.go @@ -45,7 +45,7 @@ type ListenerConfig struct { Hostname string Port int Protocol string - TLS bool + TLS core.TLSParams } // RouteTrackingListener is an optional extension @@ -63,8 +63,8 @@ type RouteTrackingListener interface { type Listener interface { ID() string CanBind(route Route) (bool, error) - Certificates() []string Config() ListenerConfig + IsValid() bool } // StatusTrackingRoute is an optional extension diff --git a/internal/store/memory/gateway.go b/internal/store/memory/gateway.go index ec08e73f9..539a754bf 100644 --- a/internal/store/memory/gateway.go +++ b/internal/store/memory/gateway.go @@ -26,7 +26,7 @@ func newGatewayState(logger hclog.Logger, gateway store.Gateway, adapter core.Sy gatewayLogger := logger.With("gateway.consul.namespace", id.ConsulNamespace, "gateway.consul.service", id.Service) listeners := make(map[string]*listenerState) for _, listener := range gateway.Listeners() { - for _, cert := range listener.Certificates() { + for _, cert := range listener.Config().TLS.Certificates { secrets[cert] = struct{}{} } listeners[listener.ID()] = newListenerState(gatewayLogger, gateway, listener) @@ -116,7 +116,9 @@ func (g *gatewayState) sync(ctx context.Context) error { func (g *gatewayState) Resolve() core.ResolvedGateway { listeners := []core.ResolvedListener{} for _, listener := range g.listeners { - listeners = append(listeners, listener.Resolve()) + if listener.Listener.IsValid() { + listeners = append(listeners, listener.Resolve()) + } } return core.ResolvedGateway{ ID: g.ID(), diff --git a/internal/store/memory/listener.go b/internal/store/memory/listener.go index 374118d4d..7c4c923e7 100644 --- a/internal/store/memory/listener.go +++ b/internal/store/memory/listener.go @@ -12,7 +12,7 @@ const ( defaultListenerName = "default" ) -// boundListener wraps a lstener and its set of routes +// boundListener wraps a listener and its set of routes type listenerState struct { store.Listener @@ -101,11 +101,11 @@ func (l *listenerState) Resolve() core.ResolvedListener { routes = append(routes, route) } return core.ResolvedListener{ - Name: l.name, - Hostname: l.hostname, - Port: l.port, - Protocol: l.protocol, - Certificates: l.Listener.Certificates(), - Routes: routes, + Name: l.name, + Hostname: l.hostname, + Port: l.port, + Protocol: l.protocol, + TLS: l.Listener.Config().TLS, + Routes: routes, } } diff --git a/internal/testing/e2e/consul.go b/internal/testing/e2e/consul.go index 7663816c2..1386a9907 100644 --- a/internal/testing/e2e/consul.go +++ b/internal/testing/e2e/consul.go @@ -24,7 +24,7 @@ import ( ) const ( - consulImage = "hashicorpdev/consul:581357c32" + consulImage = "hashicorp/consul:1.11.2" configTemplateString = ` { "log_level": "trace", @@ -70,14 +70,17 @@ func init() { } type consulTestEnvironment struct { - ca []byte - consulClient *api.Client - token string - policy *api.ACLPolicy - httpPort int - grpcPort int - extraPort int - ip string + ca []byte + consulClient *api.Client + token string + policy *api.ACLPolicy + httpPort int + grpcPort int + extraHTTPPort int + extraTCPPort int + extraTCPTLSPort int + extraTCPTLSPortTwo int + ip string } func CreateTestConsulContainer(name, namespace string) env.Func { @@ -91,7 +94,10 @@ func CreateTestConsulContainer(name, namespace string) env.Func { cluster := clusterVal.(*kindCluster) httpsPort := cluster.httpsPort grpcPort := cluster.grpcPort - extraPort := cluster.extraPort + extraTCPPort := cluster.extraTCPPort + extraTCPTLSPort := cluster.extraTCPTLSPort + extraTCPTLSPortTwo := cluster.extraTCPTLSPortTwo + extraHTTPPort := cluster.extraHTTPPort rootCA, err := testing.GenerateSignedCertificate(testing.GenerateCertificateOptions{ IsCA: true, @@ -175,12 +181,15 @@ func CreateTestConsulContainer(name, namespace string) env.Func { } env := &consulTestEnvironment{ - ca: rootCA.CertBytes, - consulClient: consulClient, - httpPort: httpsPort, - grpcPort: grpcPort, - extraPort: extraPort, - ip: ip, + ca: rootCA.CertBytes, + consulClient: consulClient, + httpPort: httpsPort, + grpcPort: grpcPort, + extraHTTPPort: extraHTTPPort, + extraTCPPort: extraTCPPort, + extraTCPTLSPort: extraTCPTLSPort, + extraTCPTLSPortTwo: extraTCPTLSPortTwo, + ip: ip, } return context.WithValue(ctx, consulTestContextKey, env), nil @@ -380,6 +389,15 @@ func ConsulClient(ctx context.Context) *api.Client { return consulEnvironment.(*consulTestEnvironment).consulClient } +func ConsulCA(ctx context.Context) []byte { + consulEnvironment := ctx.Value(consulTestContextKey) + if consulEnvironment == nil { + panic("must run this with an integration test that has called CreateTestConsul") + } + return consulEnvironment.(*consulTestEnvironment).ca + +} + func ConsulMasterToken(ctx context.Context) string { consulEnvironment := ctx.Value(consulTestContextKey) if consulEnvironment == nil { @@ -404,12 +422,36 @@ func ConsulGRPCPort(ctx context.Context) int { return consulEnvironment.(*consulTestEnvironment).grpcPort } -func ExtraPort(ctx context.Context) int { +func TCPPort(ctx context.Context) int { + consulEnvironment := ctx.Value(consulTestContextKey) + if consulEnvironment == nil { + panic("must run this with an integration test that has called CreateTestConsul") + } + return consulEnvironment.(*consulTestEnvironment).extraTCPPort +} + +func TCPTLSPort(ctx context.Context) int { + consulEnvironment := ctx.Value(consulTestContextKey) + if consulEnvironment == nil { + panic("must run this with an integration test that has called CreateTestConsul") + } + return consulEnvironment.(*consulTestEnvironment).extraTCPTLSPort +} + +func ExtraTCPTLSPort(ctx context.Context) int { + consulEnvironment := ctx.Value(consulTestContextKey) + if consulEnvironment == nil { + panic("must run this with an integration test that has called CreateTestConsul") + } + return consulEnvironment.(*consulTestEnvironment).extraTCPTLSPortTwo +} + +func HTTPPort(ctx context.Context) int { consulEnvironment := ctx.Value(consulTestContextKey) if consulEnvironment == nil { panic("must run this with an integration test that has called CreateTestConsul") } - return consulEnvironment.(*consulTestEnvironment).extraPort + return consulEnvironment.(*consulTestEnvironment).extraHTTPPort } func ConsulHTTPPort(ctx context.Context) int { diff --git a/internal/testing/e2e/gateway.go b/internal/testing/e2e/gateway.go index 0a7bbafd2..07fbed1e2 100644 --- a/internal/testing/e2e/gateway.go +++ b/internal/testing/e2e/gateway.go @@ -52,11 +52,10 @@ func (p *gatewayTestEnvironment) run(ctx context.Context, namespace string, cfg k8sSecretClient.AddToMultiClient(secretClient) controller, err := k8s.New(nullLogger, &k8s.Config{ - CACertSecretNamespace: namespace, - CACertSecret: "consul-ca-cert", - SDSServerHost: HostRoute(ctx), - SDSServerPort: 9090, - RestConfig: cfg.Client().RESTConfig(), + SDSServerHost: HostRoute(ctx), + SDSServerPort: 9090, + RestConfig: cfg.Client().RESTConfig(), + CACert: string(ConsulCA(ctx)), }) if err != nil { return err diff --git a/internal/testing/e2e/kind.go b/internal/testing/e2e/kind.go index a755a14eb..f3600df46 100644 --- a/internal/testing/e2e/kind.go +++ b/internal/testing/e2e/kind.go @@ -40,8 +40,17 @@ nodes: - containerPort: {{ .GRPCPort }} hostPort: {{ .GRPCPort }} protocol: TCP - - containerPort: {{ .ExtraPort }} - hostPort: {{ .ExtraPort }} + - containerPort: {{ .ExtraTCPPort }} + hostPort: {{ .ExtraTCPPort }} + protocol: TCP + - containerPort: {{ .ExtraHTTPPort }} + hostPort: {{ .ExtraHTTPPort }} + protocol: TCP + - containerPort: {{ .ExtraTCPTLSPort }} + hostPort: {{ .ExtraTCPTLSPort }} + protocol: TCP + - containerPort: {{ .ExtraTCPTLSPortTwo }} + hostPort: {{ .ExtraTCPTLSPortTwo }} protocol: TCP ` ) @@ -54,32 +63,50 @@ func init() { // based off github.com/kubernetes-sigs/e2e-framework/support/kind type kindCluster struct { - name string - e *gexe.Echo - kubecfgFile string - config string - httpsPort int - extraPort int - grpcPort int + name string + e *gexe.Echo + kubecfgFile string + config string + httpsPort int + grpcPort int + extraHTTPPort int + extraTCPPort int + extraTCPTLSPort int + extraTCPTLSPortTwo int } func newKindCluster(name string) *kindCluster { - ports := freeport.MustTake(3) - return &kindCluster{name: name, e: gexe.New(), httpsPort: ports[0], grpcPort: ports[1], extraPort: ports[2]} + ports := freeport.MustTake(6) + return &kindCluster{ + name: name, + e: gexe.New(), + httpsPort: ports[0], + grpcPort: ports[1], + extraHTTPPort: ports[2], + extraTCPPort: ports[3], + extraTCPTLSPort: ports[4], + extraTCPTLSPortTwo: ports[5], + } } func (k *kindCluster) Create() (string, error) { - log.Println("Creating kind cluster ", k.name) + log.Println("Creating kind cluster", k.name) var kindConfig bytes.Buffer err := kindTemplate.Execute(&kindConfig, &struct { - HTTPSPort int - GRPCPort int - ExtraPort int + HTTPSPort int + GRPCPort int + ExtraTCPPort int + ExtraTCPTLSPort int + ExtraTCPTLSPortTwo int + ExtraHTTPPort int }{ - HTTPSPort: k.httpsPort, - GRPCPort: k.grpcPort, - ExtraPort: k.extraPort, + HTTPSPort: k.httpsPort, + GRPCPort: k.grpcPort, + ExtraTCPPort: k.extraTCPPort, + ExtraTCPTLSPort: k.extraTCPTLSPort, + ExtraTCPTLSPortTwo: k.extraTCPTLSPortTwo, + ExtraHTTPPort: k.extraHTTPPort, }) if err != nil { return "", err diff --git a/internal/testing/e2e/kubernetes.go b/internal/testing/e2e/kubernetes.go index ec3addc55..78fea3374 100644 --- a/internal/testing/e2e/kubernetes.go +++ b/internal/testing/e2e/kubernetes.go @@ -49,6 +49,8 @@ func InstallGatewayCRDs(ctx context.Context, cfg *envconf.Config) (context.Conte &gateway.GatewayList{}, &gateway.HTTPRoute{}, &gateway.HTTPRouteList{}, + &gateway.TCPRoute{}, + &gateway.TCPRouteList{}, ) meta.AddToGroupVersion(scheme.Scheme, gateway.SchemeGroupVersion) diff --git a/internal/testing/e2e/service.go b/internal/testing/e2e/service.go index b3872f728..deea3ba2d 100644 --- a/internal/testing/e2e/service.go +++ b/internal/testing/e2e/service.go @@ -18,8 +18,8 @@ import ( ) const ( - envoyImage = "envoyproxy/envoy:v1.19-latest" - bootstrapJSONTemplate = `{ + envoyImage = "envoyproxy/envoy:v1.19-latest" + httpBootstrapJSONTemplate = `{ "admin": { "access_log_path": "/dev/null", "address": { @@ -141,14 +141,120 @@ const ( } } ` + tcpBootstrapJSONTemplate = `{ + "admin": { + "access_log_path": "/dev/null", + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 19000 + } + } + }, + "node": { + "cluster": "{{ .ID }}", + "id": "{{ .ID }}" + }, + "static_resources": { + "listeners": [{ + "name": "static", + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 19001 + } + }, + "filter_chains": [{ + "filters": [{ + "name": "envoy.filters.network.direct_response", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.direct_response.v3.Config", + "response": { + "inline_string": "{{ .ID }}" + } + } + }] + }] + }], + "clusters": [ + { + "name": "consul-server", + "ignore_health_on_host_removal": false, + "connect_timeout": "1s", + "type": "{{ .AddressType }}", + "transport_socket": { + "name": "tls", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "common_tls_context": { + "validation_context": { + "trusted_ca": { + "filename": "/ca/tls.crt" + } + } + } + } + }, + "http2_protocol_options": {}, + "loadAssignment": { + "clusterName": "consul-server", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "{{ .ConsulAddress }}", + "port_value": {{ .ConsulXDSPort }} + } + } + } + } + ] + } + ] + } + } + ] + }, + "dynamic_resources": { + "lds_config": { + "ads": {}, + "resource_api_version": "V3" + }, + "cds_config": { + "ads": {}, + "resource_api_version": "V3" + }, + "ads_config": { + "api_type": "DELTA_GRPC", + "transport_api_version": "V3", + "grpc_services": { + "initial_metadata": [ + { + "key": "x-consul-token", + "value": "{{ .Token }}" + } + ], + "envoy_grpc": { + "cluster_name": "consul-server" + } + } + } + } + } + ` ) var ( - bootstrapTemplate *template.Template + tcpBootstrapTemplate *template.Template + httpBootstrapTemplate *template.Template ) func init() { - bootstrapTemplate = template.Must(template.New("bootstrap").Parse(bootstrapJSONTemplate)) + tcpBootstrapTemplate = template.Must(template.New("tcp-bootstrap").Parse(tcpBootstrapJSONTemplate)) + httpBootstrapTemplate = template.Must(template.New("http-bootstrap").Parse(httpBootstrapJSONTemplate)) } type bootstrapArgs struct { @@ -159,9 +265,19 @@ type bootstrapArgs struct { ConsulXDSPort int } -// DeployMeshService deploys an envoy proxy with roughly the same logic that consul-k8s uses +// DeployHTTPMeshService deploys an envoy proxy with roughly the same logic that consul-k8s uses +// in its connect-inject registration +func DeployHTTPMeshService(ctx context.Context, cfg *envconf.Config) (*core.Service, error) { + return deployMeshService(ctx, cfg, "http", httpBootstrapTemplate) +} + +// DeployTCPMeshService deploys an envoy proxy with roughly the same logic that consul-k8s uses // in its connect-inject registration -func DeployMeshService(ctx context.Context, cfg *envconf.Config) (*core.Service, error) { +func DeployTCPMeshService(ctx context.Context, cfg *envconf.Config) (*core.Service, error) { + return deployMeshService(ctx, cfg, "tcp", tcpBootstrapTemplate) +} + +func deployMeshService(ctx context.Context, cfg *envconf.Config, protocol string, template *template.Template) (*core.Service, error) { servicePort := 8080 namespace := Namespace(ctx) name := envconf.RandomName("mesh", 16) @@ -173,7 +289,7 @@ func DeployMeshService(ctx context.Context, cfg *envconf.Config) (*core.Service, resourcesClient := cfg.Client().Resources(namespace) - configMap, err := meshServiceConfigMap(name, namespace, proxyServiceName, token, consulAddress, consulPort) + configMap, err := meshServiceConfigMap(template, name, namespace, proxyServiceName, token, consulAddress, consulPort) if err != nil { return nil, err } @@ -233,7 +349,7 @@ func DeployMeshService(ctx context.Context, cfg *envconf.Config) (*core.Service, _, _, err = client.ConfigEntries().Set(&api.ServiceConfigEntry{ Kind: api.ServiceDefaults, Name: name, - Protocol: "http", + Protocol: protocol, }, nil) if err != nil { return nil, err @@ -262,8 +378,8 @@ func DeployMeshService(ctx context.Context, cfg *envconf.Config) (*core.Service, return service, nil } -func meshServiceConfigMap(name, namespace, proxyServiceName, token, consulAddress string, consulPort int) (*core.ConfigMap, error) { - config, err := meshServiceConfig(proxyServiceName, token, consulAddress, consulPort) +func meshServiceConfigMap(template *template.Template, name, namespace, proxyServiceName, token, consulAddress string, consulPort int) (*core.ConfigMap, error) { + config, err := meshServiceConfig(template, proxyServiceName, token, consulAddress, consulPort) if err != nil { return nil, err } @@ -279,9 +395,9 @@ func meshServiceConfigMap(name, namespace, proxyServiceName, token, consulAddres }, nil } -func meshServiceConfig(name, token, consulAddress string, consulPort int) (string, error) { - var template bytes.Buffer - if err := bootstrapTemplate.Execute(&template, &bootstrapArgs{ +func meshServiceConfig(template *template.Template, name, token, consulAddress string, consulPort int) (string, error) { + var data bytes.Buffer + if err := template.Execute(&data, &bootstrapArgs{ ID: name, AddressType: common.AddressTypeForAddress(consulAddress), Token: token, @@ -290,7 +406,7 @@ func meshServiceConfig(name, token, consulAddress string, consulPort int) (strin }); err != nil { return "", err } - return template.String(), nil + return data.String(), nil } func meshService(deployment *apps.Deployment, port int) *core.Service { diff --git a/internal/version/version.go b/internal/version/version.go index 8662da365..ebb193a32 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -19,7 +19,7 @@ var ( // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. - VersionPrerelease = "techpreview" + VersionPrerelease = "beta" ) // GetHumanVersion composes the parts of the version in a way that's suitable diff --git a/pkg/apis/v1alpha1/types.go b/pkg/apis/v1alpha1/types.go index 2261d9f21..3563d85b1 100644 --- a/pkg/apis/v1alpha1/types.go +++ b/pkg/apis/v1alpha1/types.go @@ -1,44 +1,17 @@ package v1alpha1 import ( - "fmt" - "strconv" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" gateway "sigs.k8s.io/gateway-api/apis/v1alpha2" "github.com/hashicorp/consul-api-gateway/internal/k8s/utils" - "github.com/hashicorp/consul-api-gateway/internal/version" ) -var ( - defaultImage string - defaultServiceAnnotations = []string{ - "external-dns.alpha.kubernetes.io/hostname", - } -) - -func init() { - imageVersion := version.Version - if version.VersionPrerelease != "" { - imageVersion += "-" + version.VersionPrerelease - } - defaultImage = fmt.Sprintf("hashicorp/consul-api-gateway:%s", imageVersion) -} - const ( GatewayClassConfigKind = "GatewayClassConfig" - - defaultEnvoyImage = "envoyproxy/envoy:v1.19-latest" - defaultLogLevel = "info" - defaultCASecret = "consul-ca-cert" - defaultConsulAddress = "$(HOST_IP)" - defaultConsulHTTPPort = "8500" - defaultConsulXDSPort = "8502" ) // +genclient @@ -91,8 +64,6 @@ type ConsulSpec struct { Address string `json:"address,omitempty"` // The information about Consul's ports PortSpec PortSpec `json:"ports,omitempty"` - // The location of a secret to mount with the Consul root CA. - CASecret string `json:"caSecret,omitempty"` } type PortSpec struct { @@ -117,6 +88,8 @@ type CopyAnnotationsSpec struct { } type AuthSpec struct { + // Whether deployments should be run with "managed" service accounts created by the gateway controller. + Managed bool `json:"managed,omitempty"` // The Consul auth method used for initial authentication by consul-api-gateway. Method string `json:"method,omitempty"` // The Kubernetes service account to authenticate as. @@ -135,54 +108,17 @@ type GatewayClassConfigList struct { Items []GatewayClassConfig `json:"items"` } -type SDSConfig struct { - Host string - Port int -} - -// EmptyServiceFor returns an empty service definition for ensuring deletion -func (c *GatewayClassConfig) EmptyServiceFor(gw *gateway.Gateway) *corev1.Service { - return &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: gw.Name, - Namespace: gw.Namespace, - }, - } -} - -// ServicesFor returns the service configuration for the given gateway. -// The gateway should be marked with the api-gateway.consul.hashicorp.com/service-type -// annotation and marked with 'ClusterIP', `NodePort` or `LoadBalancer` to -// expose the gateway listeners. Any other value does not expose the gateway. -func (c *GatewayClassConfig) ServiceFor(gw *gateway.Gateway) *corev1.Service { - if c.Spec.ServiceType == nil { +// ServicesAccountFor returns the service account to be created for the given gateway. +func (c *GatewayClassConfig) ServiceAccountFor(gw *gateway.Gateway) *corev1.ServiceAccount { + if !c.Spec.ConsulSpec.AuthSpec.Managed { return nil } - ports := []corev1.ServicePort{} - for _, listener := range gw.Spec.Listeners { - ports = append(ports, corev1.ServicePort{ - Name: string(listener.Name), - Protocol: "TCP", - Port: int32(listener.Port), - }) - } labels := utils.LabelsForGateway(gw) - allowedAnnotations := c.Spec.CopyAnnotations.Service - if allowedAnnotations == nil { - allowedAnnotations = defaultServiceAnnotations - } - - return &corev1.Service{ + return &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ - Name: gw.Name, - Namespace: gw.Namespace, - Labels: labels, - Annotations: getAnnotations(gw.Annotations, allowedAnnotations), - }, - Spec: corev1.ServiceSpec{ - Selector: labels, - Type: *c.Spec.ServiceType, - Ports: ports, + Name: gw.Name, + Namespace: gw.Namespace, + Labels: labels, }, } } @@ -222,42 +158,6 @@ func compareServices(a, b *corev1.Service) bool { return true } -func getAnnotations(annotations map[string]string, allowed []string) map[string]string { - filtered := make(map[string]string) - for _, annotation := range allowed { - if value, found := annotations[annotation]; found { - filtered[annotation] = value - } - } - return filtered -} - -// DeploymentsFor returns the deployment configuration for the given gateway. -func (c *GatewayClassConfig) DeploymentFor(gw *gateway.Gateway, sds SDSConfig) *appsv1.Deployment { - labels := utils.LabelsForGateway(gw) - return &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: gw.Name, - Namespace: gw.Namespace, - Labels: labels, - }, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - Annotations: map[string]string{ - "consul.hashicorp.com/connect-inject": "false", - }, - }, - Spec: c.podSpecFor(gw, sds), - }, - }, - } -} - // MergeDeploymentmerges a gateway deployment a onto b and returns b, overriding all of // the fields that we'd normally set for a service deployment. It does not attempt // to change the service type @@ -293,157 +193,3 @@ func compareDeployments(a, b *appsv1.Deployment) bool { } return true } - -func (c *GatewayClassConfig) podSpecFor(gw *gateway.Gateway, sds SDSConfig) corev1.PodSpec { - volumes, mounts := c.volumesFor(gw) - return corev1.PodSpec{ - NodeSelector: c.Spec.NodeSelector, - ServiceAccountName: orDefault(c.Spec.ConsulSpec.AuthSpec.Account, ""), - // the init container copies the binary into the - // next envoy container so we can decouple the envoy - // versions from our version of consul-api-gateway. - InitContainers: []corev1.Container{{ - Image: orDefault(c.Spec.ImageSpec.ConsulAPIGateway, defaultImage), - Name: "consul-api-gateway-init", - VolumeMounts: mounts, - Command: []string{ - "cp", "/bin/consul-api-gateway", "/bootstrap/consul-api-gateway", - }, - }}, - Containers: []corev1.Container{{ - Image: orDefault(c.Spec.ImageSpec.Envoy, defaultEnvoyImage), - Name: "consul-api-gateway", - VolumeMounts: mounts, - Ports: c.containerPortsFor(gw), - Env: []corev1.EnvVar{ - { - Name: "IP", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "status.podIP", - }, - }, - }, - { - Name: "HOST_IP", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "status.hostIP", - }, - }, - }, - }, - Command: c.execCommandFor(gw, sds), - ReadinessProbe: &corev1.Probe{ - Handler: corev1.Handler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/ready", - Port: intstr.FromInt(20000), - }, - }, - }, - }}, - Volumes: volumes, - } -} - -func (c *GatewayClassConfig) execCommandFor(gw *gateway.Gateway, sds SDSConfig) []string { - initCommand := []string{ - "/bootstrap/consul-api-gateway", "exec", - "-log-json", - "-log-level", orDefault(c.Spec.LogLevel, defaultLogLevel), - "-gateway-host", "$(IP)", - "-gateway-name", gw.Name, - "-consul-http-address", orDefault(c.Spec.ConsulSpec.Address, defaultConsulAddress), - "-consul-http-port", orDefaultIntString(c.Spec.ConsulSpec.PortSpec.HTTP, defaultConsulHTTPPort), - "-consul-xds-port", orDefaultIntString(c.Spec.ConsulSpec.PortSpec.GRPC, defaultConsulXDSPort), - "-envoy-bootstrap-path", "/bootstrap/envoy.json", - "-envoy-sds-address", sds.Host, - "-envoy-sds-port", strconv.Itoa(sds.Port), - } - - if method := c.Spec.ConsulSpec.AuthSpec.Method; method != "" { - initCommand = append(initCommand, "-acl-auth-method", method) - } - - if c.requiresCA(gw) { - initCommand = append(initCommand, "-consul-ca-cert-file", "/ca/tls.crt") - } - return initCommand -} - -func (c *GatewayClassConfig) volumesFor(gw *gateway.Gateway) ([]corev1.Volume, []corev1.VolumeMount) { - volumes := []corev1.Volume{{ - Name: "bootstrap", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, { - Name: "certs", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }} - mounts := []corev1.VolumeMount{{ - Name: "bootstrap", - MountPath: "/bootstrap", - }, { - Name: "certs", - MountPath: "/certs", - }} - if c.requiresCA(gw) { - caCertSecret := orDefault(c.Spec.ConsulSpec.CASecret, defaultCASecret) - volumes = append(volumes, corev1.Volume{ - Name: "ca", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: caCertSecret, - }, - }, - }) - mounts = append(mounts, corev1.VolumeMount{ - Name: "ca", - MountPath: "/ca", - ReadOnly: true, - }) - } - return volumes, mounts -} - -func orDefault(value, defaultValue string) string { - if value != "" { - return value - } - return defaultValue -} - -func orDefaultIntString(value int, defaultValue string) string { - if value != 0 { - return strconv.Itoa(value) - } - return defaultValue -} - -func (c *GatewayClassConfig) containerPortsFor(gw *gateway.Gateway) []corev1.ContainerPort { - ports := []corev1.ContainerPort{{ - Name: "ready", - Protocol: "TCP", - ContainerPort: 20000, - }} - for _, listener := range gw.Spec.Listeners { - port := corev1.ContainerPort{ - Name: string(listener.Name), - Protocol: "TCP", - ContainerPort: int32(listener.Port), - } - if c.Spec.UseHostPorts { - port.HostPort = int32(listener.Port) - } - ports = append(ports, port) - } - return ports -} - -func (c *GatewayClassConfig) requiresCA(gw *gateway.Gateway) bool { - return c.Spec.ConsulSpec.Scheme == "https" -} diff --git a/pkg/apis/v1alpha1/types_test.go b/pkg/apis/v1alpha1/types_test.go index 78f656a99..5485017b9 100644 --- a/pkg/apis/v1alpha1/types_test.go +++ b/pkg/apis/v1alpha1/types_test.go @@ -1,133 +1,13 @@ package v1alpha1 import ( - "bytes" - "fmt" - "os" - "path" "testing" "github.com/stretchr/testify/require" core "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer/json" - "k8s.io/apimachinery/pkg/util/yaml" - gateway "sigs.k8s.io/gateway-api/apis/v1alpha2" ) -var ( - generate bool - fixtures = []string{ - "tls-cert", - "static-mapping", - "clusterip", - "loadbalancer", - } -) - -func init() { - if os.Getenv("GENERATE") == "true" { - generate = true - } -} - -type gatewayTestConfig struct { - gatewayClassConfig *GatewayClassConfig - gatewayClass *gateway.GatewayClass - gateway *gateway.Gateway -} - -func newGatewayTestConfig() *gatewayTestConfig { - return &gatewayTestConfig{ - gatewayClassConfig: &GatewayClassConfig{}, - gatewayClass: &gateway.GatewayClass{}, - gateway: &gateway.Gateway{}, - } -} - -func (g *gatewayTestConfig) EncodeDeployment() runtime.Object { - return g.gatewayClassConfig.DeploymentFor(g.gateway, SDSConfig{ - Host: "consul-api-gateway-controller.default.svc.cluster.local", - Port: 9090, - }) -} - -func (g *gatewayTestConfig) EncodeService() runtime.Object { - return g.gatewayClassConfig.ServiceFor(g.gateway) -} - -func TestDeploymentFor(t *testing.T) { - t.Parallel() - - for _, name := range fixtures { - t.Run(name, func(t *testing.T) { - config := newGatewayTestConfig() - fixtureTest(t, name, "deployment", config, func() runtime.Object { - return config.EncodeDeployment() - }) - }) - } -} - -func TestServiceFor(t *testing.T) { - t.Parallel() - - for _, name := range fixtures { - t.Run(name, func(t *testing.T) { - config := newGatewayTestConfig() - fixtureTest(t, name, "service", config, func() runtime.Object { - return config.EncodeService() - }) - }) - } -} - -func fixtureTest(t *testing.T, name, suffix string, into *gatewayTestConfig, encode func() runtime.Object) { - t.Helper() - - file, err := os.OpenFile(path.Join("testdata", fmt.Sprintf("%s.yaml", name)), os.O_RDONLY, 0644) - require.NoError(t, err) - defer file.Close() - - stat, err := file.Stat() - require.NoError(t, err) - - decoder := yaml.NewYAMLOrJSONDecoder(file, int(stat.Size())) - err = decoder.Decode(into.gatewayClassConfig) - require.NoError(t, err) - err = decoder.Decode(into.gatewayClass) - require.NoError(t, err) - err = decoder.Decode(into.gateway) - require.NoError(t, err) - - var buffer bytes.Buffer - serializer := json.NewSerializerWithOptions( - json.DefaultMetaFactory, nil, nil, - json.SerializerOptions{ - Yaml: true, - Pretty: true, - Strict: true, - }, - ) - err = serializer.Encode(encode(), &buffer) - require.NoError(t, err) - - var expected string - expectedFileName := fmt.Sprintf("%s.%s.golden.yaml", name, suffix) - if generate { - expected = buffer.String() - err := os.WriteFile(path.Join("testdata", expectedFileName), buffer.Bytes(), 0644) - require.NoError(t, err) - } else { - data, err := os.ReadFile(path.Join("testdata", expectedFileName)) - require.NoError(t, err) - expected = string(data) - } - - require.Equal(t, expected, buffer.String()) -} - func TestGatewayClassConfigDeepCopy(t *testing.T) { var nilConfig *GatewayClassConfig require.Nil(t, nilConfig.DeepCopy()) diff --git a/pkg/apis/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/v1alpha1/zz_generated.deepcopy.go index 11a29ac16..d8faa0fe2 100644 --- a/pkg/apis/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/v1alpha1/zz_generated.deepcopy.go @@ -1,4 +1,3 @@ -//go:build !ignore_autogenerated // +build !ignore_autogenerated // Code generated by controller-gen. DO NOT EDIT.