From a75da134f7595a6600dbe3c48f97bb172a295988 Mon Sep 17 00:00:00 2001 From: craig Date: Wed, 25 Oct 2023 16:38:28 +0100 Subject: [PATCH] add new main integration tests split and working add ocm enabled flag add image build action add deployment to kustomize fix metrics ports for local-setup e2e update readme and some make changes --- .github/workflows/ci-e2e.yaml | 6 +- .../workflows/policy-controller-image.yaml | 62 ++++++++ .gitignore | 3 +- .vscode/launch.json | 2 +- Dockerfile | 17 +- Makefile | 5 +- README.md | 15 +- ...eway-controller.clusterserviceversion.yaml | 60 ++++++- cmd/gateway_controller/main.go | 148 ++++++++++++++++++ cmd/{controller => policy_controller}/main.go | 93 ++--------- config/default/kustomization.yaml | 2 + config/{manager => default}/namespace.yaml | 0 config/manager/kustomization.yaml | 1 - config/policy-controller/kustomization.yaml | 8 + config/policy-controller/manager.yaml | 63 ++++++++ go.mod | 3 +- go.sum | 6 - hack/.deployUtils | 4 +- hack/make/controller.make | 51 ------ hack/make/gateway_controller.make | 51 ++++++ hack/make/policy_controller.make | 51 ++++++ .../dnspolicy/dnspolicy_controller.go | 19 ++- .../dnspolicy/dnspolicy_dnsrecords.go | 32 ++-- .../gateway_controller_test.go | 2 +- .../placement_fake.go} | 34 +--- test/gateway_integration/suite_test.go | 148 ++++++++++++++++++ .../dnsheathcheckprobe_controller_test.go | 2 +- .../dnspolicy_controller_test.go | 2 +- test/policy_integration/helper_test.go | 61 ++++++++ .../managedzone_controller_test.go | 2 +- .../suite_test.go | 22 +-- .../tlspolicy_controller_test.go | 2 +- test/policy_integration/util.go | 52 ++++++ 33 files changed, 796 insertions(+), 233 deletions(-) create mode 100644 .github/workflows/policy-controller-image.yaml create mode 100644 cmd/gateway_controller/main.go rename cmd/{controller => policy_controller}/main.go (62%) rename config/{manager => default}/namespace.yaml (100%) create mode 100644 config/policy-controller/kustomization.yaml create mode 100644 config/policy-controller/manager.yaml delete mode 100644 hack/make/controller.make create mode 100644 hack/make/gateway_controller.make create mode 100644 hack/make/policy_controller.make rename test/{integration => gateway_integration}/gateway_controller_test.go (99%) rename test/{integration/helper_test.go => gateway_integration/placement_fake.go} (90%) create mode 100644 test/gateway_integration/suite_test.go rename test/{integration => policy_integration}/dnsheathcheckprobe_controller_test.go (99%) rename test/{integration => policy_integration}/dnspolicy_controller_test.go (99%) create mode 100644 test/policy_integration/helper_test.go rename test/{integration => policy_integration}/managedzone_controller_test.go (99%) rename test/{integration => policy_integration}/suite_test.go (90%) rename test/{integration => policy_integration}/tlspolicy_controller_test.go (99%) create mode 100644 test/policy_integration/util.go diff --git a/.github/workflows/ci-e2e.yaml b/.github/workflows/ci-e2e.yaml index 9dcc61a1a..d3d875c35 100644 --- a/.github/workflows/ci-e2e.yaml +++ b/.github/workflows/ci-e2e.yaml @@ -65,11 +65,13 @@ jobs: run: | export OCM_SINGLE=1 make local-setup - - name: Deploy MGC + - name: Deploy run: | - make docker-build-controller kind-load-controller deploy-controller + make docker-build-gateway-controller kind-load-gateway-controller docker-build-policy-controller kind-load-policy-controller deploy-gateway-controller kubectl --context kind-mgc-control-plane -n multicluster-gateway-controller-system wait --timeout=300s --for=condition=Available deployment/mgc-controller-manager + kubectl --context kind-mgc-control-plane -n multicluster-gateway-controller-system wait --timeout=300s --for=condition=Available deployment/mgc-policy-controller-manager kubectl --context kind-mgc-control-plane logs --all-containers --ignore-errors deployment/mgc-controller-manager -n multicluster-gateway-controller-system + kubectl --context kind-mgc-control-plane logs --all-containers --ignore-errors deployment/mgc-policy-controller-manager -n multicluster-gateway-controller-system kubectl get managedzones -n multi-cluster-gateways mgc-dev-mz-aws -o yaml kubectl --context kind-mgc-control-plane -n multi-cluster-gateways wait --timeout=60s --for=condition=Ready managedzone/mgc-dev-mz-aws kubectl get managedzones -n multi-cluster-gateways mgc-dev-mz-gcp -o yaml diff --git a/.github/workflows/policy-controller-image.yaml b/.github/workflows/policy-controller-image.yaml new file mode 100644 index 000000000..f48418dda --- /dev/null +++ b/.github/workflows/policy-controller-image.yaml @@ -0,0 +1,62 @@ +name: Build and Publish Policy Controller Image + +on: + push: + branches: + - main + - "release-*" + tags: + - "v[0-9]+.[0-9]+.[0-9]+" + +env: + IMG_REGISTRY_HOST: quay.io + IMG_REGISTRY_ORG: kuadrant + IMG_REGISTRY_REPO: policy-controller + MAIN_BRANCH_NAME: main + +jobs: + controller: + if: github.repository_owner == 'kuadrant' + name: Build controller image + runs-on: ubuntu-22.04 + outputs: + sha_short: ${{ steps.vars.outputs.sha_short }} + controller_image: ${{ steps.vars.outputs.base_image }}:${{ steps.vars.outputs.sha_short }} + steps: + - uses: actions/checkout@v3 + + - name: Calculate vars + id: vars + run: | + echo "sha_short=$(echo ${{ github.sha }} | cut -b -7)" >> $GITHUB_OUTPUT + echo "base_image=${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/${{ env.IMG_REGISTRY_REPO }}" >> $GITHUB_OUTPUT + + - name: Add image tags + id: add-tags + run: echo "IMG_TAGS=${{ steps.vars.outputs.base_image }}:${{ steps.vars.outputs.sha_short }},${{ steps.vars.outputs.base_image }}:${{ github.ref_name }}" >> $GITHUB_ENV + + - name: Add latest tag + if: ${{ github.ref_name == env.MAIN_BRANCH_NAME }} + id: add-latest-tag + run: echo "IMG_TAGS=${{ steps.vars.outputs.base_image }}:latest,${{ env.IMG_TAGS }}" >> $GITHUB_ENV + + - name: Login to Quay.io + uses: docker/login-action@v2 + id: registry-login + with: + registry: ${{ env.IMG_REGISTRY_HOST }} + username: ${{ secrets.IMG_REGISTRY_USERNAME }} + password: ${{ secrets.IMG_REGISTRY_TOKEN }} + + - name: Build and push Controller Image + id: build-and-push + uses: docker/build-push-action@v4 + with: + push: true + tags: ${{ env.IMG_TAGS }} + target: policy-controller + + - name: Print Image URL + run: | + echo "Image pushed to ${{ env.IMG_TAGS }}" + echo "Image digest: ${{ steps.build-and-push.outputs.digest }}" \ No newline at end of file diff --git a/.gitignore b/.gitignore index f7f0ed39f..389a5409d 100644 --- a/.gitignore +++ b/.gitignore @@ -90,7 +90,8 @@ config/deploy/**/_* /control-plane.yaml /workload*.yaml -cmd/controller/__debug_bin +cmd/gateway_controller/__debug_bin +cmd/policy_controller/__debug_bin # Submariner broker config /broker-info.subm* diff --git a/.vscode/launch.json b/.vscode/launch.json index ab430d9cc..411bd65a5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,7 +9,7 @@ "type": "go", "request": "launch", "mode": "auto", - "program": "./cmd/controller/main.go", + "program": "./cmd/gateway_controller/main.go", "args": [ "--metrics-bind-address=:8080", "--health-probe-bind-address=:8081" diff --git a/Dockerfile b/Dockerfile index e7369cd5c..e3b9cf172 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,10 @@ COPY pkg/ pkg/ # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. FROM builder as controller_builder -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o controller cmd/controller/main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o controller cmd/gateway_controller/main.go + +FROM builder as policy_builder +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o policy_controller cmd/policy_controller/main.go FROM builder as addon_builder RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o add-on-manager cmd/ocm/main.go @@ -42,4 +45,14 @@ WORKDIR / COPY --from=addon_builder /workspace/add-on-manager . USER 65532:65532 -ENTRYPOINT ["/add-on-manager"] \ No newline at end of file +ENTRYPOINT ["/add-on-manager"] + + +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM gcr.io/distroless/static:nonroot as policy-controller +WORKDIR / +COPY --from=policy_builder /workspace/policy_controller . +USER 65532:65532 + +ENTRYPOINT ["/policy_controller"] \ No newline at end of file diff --git a/Makefile b/Makefile index 43147cbfc..d55e3b20d 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,8 @@ test-unit: manifests generate fmt vet envtest ## Run unit tests. .PHONY: test-integration test-integration: ginkgo manifests generate fmt vet envtest ## Run integration tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GINKGO) -tags=integration -v --focus "${FOCUS}" ./test/integration + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GINKGO) -tags=integration -v --focus "${FOCUS}" ./test/policy_integration + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GINKGO) -tags=integration -v --focus "${FOCUS}" ./test/gateway_integration .PHONY: test test: test-unit test-integration ## Run tests. @@ -109,7 +110,7 @@ local-cleanup-mgc: ## Cleanup MGC from kind clusters ./hack/local-cleanup-mgc.sh .PHONY: build -build: build-controller ## Build all binaries. +build: build-gateway-controller build-policy-controller ## Build all binaries. ##@ Deployment ifndef ignore-not-found diff --git a/README.md b/README.md index 3d7cc306c..cdf0b564f 100644 --- a/README.md +++ b/README.md @@ -43,12 +43,16 @@ When deploying the multicluster gateway controller using the make targets, the f 1. Build the controller image and load it into the control plane ```sh kubectl config use-context kind-mgc-control-plane - make kind-load-controller + make kind-load-policy-controller + + kubectl config use-context kind-mgc-control-plane + make kind-load-gateway-controller ``` -1. Deploy the controller to the control plane cluster +1. Deploy the controller(s) to the control plane cluster ```sh - make deploy-controller + make deploy-policy-controller + make deploy-gateway-controller ``` 1. (Optional) View the logs of the deployed controller @@ -68,7 +72,10 @@ When deploying the multicluster gateway controller using the make targets, the f 1. Run the controller locally: ```sh kubectl config use-context kind-mgc-control-plane - make build-controller install run-controller + make build-policy-controller install run-policy-controller + + kubectl config use-context kind-mgc-control-plane + make build-gateway-controller install run-gatewway-controller ``` ## 3. Running the agent in the cluster: diff --git a/bundle/manifests/multicluster-gateway-controller.clusterserviceversion.yaml b/bundle/manifests/multicluster-gateway-controller.clusterserviceversion.yaml index ef6b0b06d..de772a979 100644 --- a/bundle/manifests/multicluster-gateway-controller.clusterserviceversion.yaml +++ b/bundle/manifests/multicluster-gateway-controller.clusterserviceversion.yaml @@ -4,7 +4,7 @@ metadata: annotations: alm-examples: '[]' capabilities: Basic Install - createdAt: "2023-10-10T15:03:19Z" + createdAt: "2023-10-27T14:36:31Z" operators.operatorframework.io/builder: operator-sdk-v1.28.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v3 name: multicluster-gateway-controller.v0.0.0 @@ -571,6 +571,64 @@ spec: runAsNonRoot: true serviceAccountName: mgc-add-on-manager terminationGracePeriodSeconds: 10 + - label: + app.kubernetes.io/component: manager + app.kubernetes.io/created-by: policy-controller + app.kubernetes.io/instance: policy-controller + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: deployment + app.kubernetes.io/part-of: kuadrant + control-plane: controller-manager + name: mgc-policy-controller-manager + spec: + replicas: 1 + selector: + matchLabels: + control-plane: policy-controller + strategy: {} + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: policy-controller + spec: + containers: + - args: + - --leader-elect + command: + - /policy_controller + image: quay.io/kuadrant/policy-controller:main + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: policy-controller + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 10m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + securityContext: + runAsNonRoot: true + serviceAccountName: mgc-controller-manager + terminationGracePeriodSeconds: 10 permissions: - rules: - apiGroups: diff --git a/cmd/gateway_controller/main.go b/cmd/gateway_controller/main.go new file mode 100644 index 000000000..b1738fd15 --- /dev/null +++ b/cmd/gateway_controller/main.go @@ -0,0 +1,148 @@ +/* +Copyright 2022 The MultiCluster Traffic Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "os" + + certmanv1 "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1" + clusterv1 "open-cluster-management.io/api/cluster/v1" + clusterv1beta2 "open-cluster-management.io/api/cluster/v1beta1" + workv1 "open-cluster-management.io/api/work/v1" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamicinformer" + "k8s.io/client-go/kubernetes/scheme" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/client-go/tools/cache" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + gatewayapi "sigs.k8s.io/gateway-api/apis/v1beta1" + + "github.com/Kuadrant/multicluster-gateway-controller/pkg/apis/v1alpha1" + "github.com/Kuadrant/multicluster-gateway-controller/pkg/controllers/gateway" + "github.com/Kuadrant/multicluster-gateway-controller/pkg/placement" + "github.com/Kuadrant/multicluster-gateway-controller/pkg/policysync" + //+kubebuilder:scaffold:imports +) + +var ( + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme.Scheme)) + + utilruntime.Must(v1alpha1.AddToScheme(scheme.Scheme)) + utilruntime.Must(certmanv1.AddToScheme(scheme.Scheme)) + utilruntime.Must(gatewayapi.AddToScheme(scheme.Scheme)) + utilruntime.Must(clusterv1beta2.AddToScheme(scheme.Scheme)) + utilruntime.Must(workv1.AddToScheme(scheme.Scheme)) + utilruntime.Must(clusterv1.AddToScheme(scheme.Scheme)) + + //+kubebuilder:scaffold:scheme +} + +func main() { + var metricsAddr string + var enableLeaderElection bool + var probeAddr string + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + ctx := ctrl.SetupSignalHandler() + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme.Scheme, + MetricsBindAddress: metricsAddr, + Port: 9443, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "fb80029c-controller.kuadrant.io", + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + placer := placement.NewOCMPlacer(mgr.GetClient()) + if err = (&gateway.GatewayClassReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "GatewayClass") + os.Exit(1) + } + + dynamicClient := dynamic.NewForConfigOrDie(mgr.GetConfig()) + dynamicInformerFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory( + dynamicClient, + 0, + corev1.NamespaceAll, + nil, + ) + + policyInformersManager := policysync.NewPolicyInformersManager(dynamicInformerFactory) + if err := policyInformersManager.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to start policy informers manager") + os.Exit(1) + } + + if err = (&gateway.GatewayReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Placement: placer, + PolicyInformersManager: policyInformersManager, + DynamicClient: dynamicClient, + WatchedPolicies: map[schema.GroupVersionResource]cache.ResourceEventHandlerRegistration{}, + }).SetupWithManager(mgr, ctx); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Gateway") + os.Exit(1) + } + + //+kubebuilder:scaffold:builder + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + + setupLog.Info("starting manager") + if err := mgr.Start(ctx); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} diff --git a/cmd/controller/main.go b/cmd/policy_controller/main.go similarity index 62% rename from cmd/controller/main.go rename to cmd/policy_controller/main.go index 029327615..a5f69059b 100644 --- a/cmd/controller/main.go +++ b/cmd/policy_controller/main.go @@ -19,22 +19,13 @@ package main import ( "flag" "os" - "time" certmanv1 "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1" clusterv1 "open-cluster-management.io/api/cluster/v1" - clusterv1beta2 "open-cluster-management.io/api/cluster/v1beta1" - workv1 "open-cluster-management.io/api/work/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/client-go/kubernetes/scheme" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - _ "k8s.io/client-go/plugin/pkg/client/auth" - "k8s.io/client-go/tools/cache" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log" @@ -44,17 +35,11 @@ import ( "github.com/kuadrant/kuadrant-operator/pkg/reconcilers" "github.com/Kuadrant/multicluster-gateway-controller/pkg/apis/v1alpha1" - "github.com/Kuadrant/multicluster-gateway-controller/pkg/controllers/dnshealthcheckprobe" "github.com/Kuadrant/multicluster-gateway-controller/pkg/controllers/dnspolicy" "github.com/Kuadrant/multicluster-gateway-controller/pkg/controllers/dnsrecord" - "github.com/Kuadrant/multicluster-gateway-controller/pkg/controllers/gateway" "github.com/Kuadrant/multicluster-gateway-controller/pkg/controllers/managedzone" "github.com/Kuadrant/multicluster-gateway-controller/pkg/controllers/tlspolicy" "github.com/Kuadrant/multicluster-gateway-controller/pkg/dns/dnsprovider" - "github.com/Kuadrant/multicluster-gateway-controller/pkg/health" - "github.com/Kuadrant/multicluster-gateway-controller/pkg/placement" - "github.com/Kuadrant/multicluster-gateway-controller/pkg/policysync" - //+kubebuilder:scaffold:imports ) var ( @@ -63,28 +48,27 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme.Scheme)) - + utilruntime.Must(gatewayapi.AddToScheme(scheme.Scheme)) utilruntime.Must(v1alpha1.AddToScheme(scheme.Scheme)) utilruntime.Must(certmanv1.AddToScheme(scheme.Scheme)) - utilruntime.Must(gatewayapi.AddToScheme(scheme.Scheme)) - utilruntime.Must(clusterv1beta2.AddToScheme(scheme.Scheme)) - utilruntime.Must(workv1.AddToScheme(scheme.Scheme)) + //this is need for now but will be removed soon utilruntime.Must(clusterv1.AddToScheme(scheme.Scheme)) - - //+kubebuilder:scaffold:scheme } func main() { var metricsAddr string var enableLeaderElection bool var probeAddr string - var certProvider string + var ocmHub bool flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") - flag.StringVar(&certProvider, "cert-provider", "glbc-ca", "The name of the certificate provider to use") + + flag.BoolVar(&ocmHub, "ocm-hub", true, + "tells the controller if it is in an ocm hub"+ + "setting this to false will cause the controller to stop watching for managedcluster resources") opts := zap.Options{ Development: true, } @@ -100,29 +84,14 @@ func main() { Port: 9443, HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, - LeaderElectionID: "fb80029c-controller.kuadrant.io", + LeaderElectionID: "fb80029c-policy-controller.kuadrant.io", }) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) } - - placer := placement.NewOCMPlacer(mgr.GetClient()) provider := dnsprovider.NewProvider(mgr.GetClient()) - healthMonitor := health.NewMonitor() - healthCheckQueue := health.NewRequestQueue(time.Second * 5) - - if err := mgr.Add(healthMonitor); err != nil { - setupLog.Error(err, "unable to start health monitor") - os.Exit(1) - } - - if err := mgr.Add(healthCheckQueue); err != nil { - setupLog.Error(err, "unable to start health check queue") - os.Exit(1) - } - if err = (&dnsrecord.DNSRecordReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), @@ -143,7 +112,7 @@ func main() { BaseReconciler: dnsPolicyBaseReconciler, }, DNSProvider: provider.DNSProviderFactory, - }).SetupWithManager(mgr); err != nil { + }).SetupWithManager(mgr, ocmHub); err != nil { setupLog.Error(err, "unable to create controller", "controller", "DNSPolicy") os.Exit(1) } @@ -173,49 +142,6 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "ManagedZone") os.Exit(1) } - if err = (&gateway.GatewayClassReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "GatewayClass") - os.Exit(1) - } - - dynamicClient := dynamic.NewForConfigOrDie(mgr.GetConfig()) - dynamicInformerFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory( - dynamicClient, - 0, - corev1.NamespaceAll, - nil, - ) - - policyInformersManager := policysync.NewPolicyInformersManager(dynamicInformerFactory) - if err := policyInformersManager.SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to start policy informers manager") - os.Exit(1) - } - - if err = (&gateway.GatewayReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Placement: placer, - PolicyInformersManager: policyInformersManager, - DynamicClient: dynamicClient, - WatchedPolicies: map[schema.GroupVersionResource]cache.ResourceEventHandlerRegistration{}, - }).SetupWithManager(mgr, ctx); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Gateway") - os.Exit(1) - } - - if err = (&dnshealthcheckprobe.DNSHealthCheckProbeReconciler{ - Client: mgr.GetClient(), - HealthMonitor: healthMonitor, - Queue: healthCheckQueue, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "DNSHealthCheckProbe") - os.Exit(1) - } - //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") @@ -231,4 +157,5 @@ func main() { setupLog.Error(err, "problem running manager") os.Exit(1) } + } diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 3777e64da..cad637732 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -31,7 +31,9 @@ namePrefix: mgc- resources: - ../crd - ../rbac +- namespace.yaml - ../manager - ../add-on-manager +- ../policy-controller patches: - path: manager_metrics_patch.yaml diff --git a/config/manager/namespace.yaml b/config/default/namespace.yaml similarity index 100% rename from config/manager/namespace.yaml rename to config/default/namespace.yaml diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index fe5263851..730f91846 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,5 +1,4 @@ resources: -- namespace.yaml - manager.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/config/policy-controller/kustomization.yaml b/config/policy-controller/kustomization.yaml new file mode 100644 index 000000000..4fe55410a --- /dev/null +++ b/config/policy-controller/kustomization.yaml @@ -0,0 +1,8 @@ +resources: +- manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: policy-controller + newName: quay.io/kuadrant/policy-controller + newTag: main diff --git a/config/policy-controller/manager.yaml b/config/policy-controller/manager.yaml new file mode 100644 index 000000000..ab13c4927 --- /dev/null +++ b/config/policy-controller/manager.yaml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: policy-controller-manager + namespace: system + labels: + control-plane: controller-manager + app.kubernetes.io/name: deployment + app.kubernetes.io/instance: policy-controller + app.kubernetes.io/component: manager + app.kubernetes.io/created-by: policy-controller + app.kubernetes.io/part-of: kuadrant + app.kubernetes.io/managed-by: kustomize +spec: + selector: + matchLabels: + control-plane: policy-controller + replicas: 1 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: policy-controller + spec: + securityContext: + runAsNonRoot: true + containers: + - command: + - /policy_controller + args: + - --leader-elect + image: policy-controller:latest + imagePullPolicy: Always + name: policy-controller + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + # TODO(user): Configure the resources accordingly based on the project requirements. + # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 10m + memory: 64Mi + serviceAccountName: controller-manager + terminationGracePeriodSeconds: 10 diff --git a/go.mod b/go.mod index 10a40d016..c599e1736 100644 --- a/go.mod +++ b/go.mod @@ -8,9 +8,9 @@ require ( github.com/google/uuid v1.3.0 github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e github.com/jetstack/cert-manager v1.7.1 + github.com/kuadrant/authorino v0.10.0 github.com/kuadrant/kuadrant-operator v0.1.1-0.20230323151616-58593d01833a github.com/martinlindhe/base36 v1.1.1 - github.com/miekg/dns v1.1.34 github.com/onsi/ginkgo/v2 v2.6.1 github.com/onsi/gomega v1.24.2 github.com/operator-framework/api v0.17.5 @@ -64,7 +64,6 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kuadrant/authorino v0.10.0 // indirect github.com/kuadrant/authorino-operator v0.4.1 // indirect github.com/kuadrant/limitador-operator v0.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/go.sum b/go.sum index 25d2e25ca..c482c1383 100644 --- a/go.sum +++ b/go.sum @@ -262,8 +262,6 @@ github.com/martinlindhe/base36 v1.1.1/go.mod h1:vMS8PaZ5e/jV9LwFKlm0YLnXl/hpOihi github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.1.34 h1:SgTzfkN+oLoIHF1bgUP+C71mzuDl3AhLApHzCCIAMWM= -github.com/miekg/dns v1.1.34/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= @@ -441,7 +439,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -488,7 +485,6 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -501,7 +497,6 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -580,7 +575,6 @@ golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= diff --git a/hack/.deployUtils b/hack/.deployUtils index 22c47c303..298d1b44a 100644 --- a/hack/.deployUtils +++ b/hack/.deployUtils @@ -50,7 +50,7 @@ deployOLM(){ kubectl config use-context kind-${clusterName} kubectl config --kubeconfig=${TMP_DIR}/kubeconfig use-context kind-${clusterName} echo "Installing OLM in ${clusterName}" - ${OPERATOR_SDK_BIN} olm install --timeout 6m0s + ${OPERATOR_SDK_BIN} olm install --timeout 10m0s } @@ -110,7 +110,7 @@ deployMetalLB () { echo "Deploying MetalLB to ${clusterName}" ${KUSTOMIZE_BIN} build ${METALLB_KUSTOMIZATION_DIR} | kubectl apply -f - echo "Waiting for deployments to be ready ..." - kubectl -n metallb-system wait --for=condition=ready pod --selector=app=metallb --timeout=300s + kubectl -n metallb-system wait --for=condition=ready pod --selector=app=metallb --timeout=600s configureMetalLB ${clusterName} ${metalLBSubnet} } diff --git a/hack/make/controller.make b/hack/make/controller.make deleted file mode 100644 index 80156a8ad..000000000 --- a/hack/make/controller.make +++ /dev/null @@ -1,51 +0,0 @@ -##@ Controller - -CONTROLLER_IMG ?= controller:$(TAG) -LOG_LEVEL ?= 3 - -.PHONY: build-controller -build-controller: manifests generate fmt vet ## Build controller binary. - go build -o bin/controller ./cmd/controller/main.go - -.PHONY: run-controller -run-controller: manifests generate fmt vet install - go run ./cmd/controller/main.go \ - --metrics-bind-address=:8080 \ - --health-probe-bind-address=:8081 \ - --zap-log-level=$(LOG_LEVEL) - -.PHONY: docker-build-controller -docker-build-controller: ## Build docker image with the controller. - docker build --target controller -t ${CONTROLLER_IMG} . - docker image prune -f --filter label=stage=mgc-builder - -.PHONY: kind-load-controller -kind-load-controller: docker-build-controller - kind load docker-image ${CONTROLLER_IMG} --name mgc-control-plane --nodes mgc-control-plane-control-plane - -.PHONY: docker-push-controller -docker-push-controller: ## Push docker image with the controller. - docker push ${CONTROLLER_IMG} - -.PHONY: deploy-controller -deploy-controller: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${CONTROLLER_IMG} - $(KUSTOMIZE) --load-restrictor LoadRestrictionsNone build config/deploy/local | kubectl apply -f - - @if [ $(METRICS) = "true" ]; then\ - $(KUSTOMIZE) build config/prometheus | kubectl apply -f -;\ - fi - -.PHONY: undeploy-controller -undeploy-controller: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - @if [ $(METRICS) = "true" ]; then\ - $(KUSTOMIZE) build config/prometheus | kubectl delete --ignore-not-found=$(ignore-not-found) -f -;\ - fi - $(KUSTOMIZE) --load-restrictor LoadRestrictionsNone build config/deploy/local | kubectl delete --ignore-not-found=$(ignore-not-found) -f - - -.PHONY: restart-controller -restart-controller: - kubectl rollout restart deployment mgc-controller-manager -n multicluster-gateway-controller-system - -.PHONY: tail-controller-logs -tail-controller-logs: - kubectl logs -f deployment/mgc-controller-manager -n multicluster-gateway-controller-system \ No newline at end of file diff --git a/hack/make/gateway_controller.make b/hack/make/gateway_controller.make new file mode 100644 index 000000000..8b78bcaf4 --- /dev/null +++ b/hack/make/gateway_controller.make @@ -0,0 +1,51 @@ +##@ Controller + +CONTROLLER_IMG ?= controller:$(TAG) +LOG_LEVEL ?= 3 + +.PHONY: build-gateway-controller +build-gateway-controller: manifests generate fmt vet ## Build controller binary. + go build -o bin/controller ./cmd/gateway_controller/main.go + +.PHONY: run-gateway-controller +run-gateway-controller: manifests generate fmt vet install + go run ./cmd/gateway_controller/main.go \ + --metrics-bind-address=:8080 \ + --health-probe-bind-address=:8081 \ + --zap-log-level=$(LOG_LEVEL) + +.PHONY: docker-build-gateway-controller +docker-build-gateway-controller: ## Build docker image with the controller. + docker build --target controller -t ${CONTROLLER_IMG} . + docker image prune -f --filter label=stage=mgc-builder + +.PHONY: kind-load-gateway-controller +kind-load-gateway-controller: docker-build-gateway-controller + kind load docker-image ${CONTROLLER_IMG} --name mgc-control-plane --nodes mgc-control-plane-control-plane + +.PHONY: docker-push-gateway-controller +docker-push-gateway-controller: ## Push docker image with the controller. + docker push ${CONTROLLER_IMG} + +.PHONY: deploy-gateway-controller +deploy-gateway-controller: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && $(KUSTOMIZE) edit set image controller=${CONTROLLER_IMG} + $(KUSTOMIZE) --load-restrictor LoadRestrictionsNone build config/deploy/local | kubectl apply -f - + @if [ "$(METRICS)" = "true" ]; then\ + $(KUSTOMIZE) build config/prometheus | kubectl apply -f -;\ + fi + +.PHONY: undeploy-gateway-controller +undeploy-gateway-controller: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + @if [ $(METRICS) = "true" ]; then\ + $(KUSTOMIZE) build config/prometheus | kubectl delete --ignore-not-found=$(ignore-not-found) -f -;\ + fi + $(KUSTOMIZE) --load-restrictor LoadRestrictionsNone build config/deploy/local | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + +.PHONY: restart-gateway-controller +restart-gateway-controller: + kubectl rollout restart deployment mgc-controller-manager -n multicluster-gateway-controller-system + +.PHONY: tail-gateway-controller-logs +tail-gateway-controller-logs: + kubectl logs -f deployment/mgc-controller-manager -n multicluster-gateway-controller-system \ No newline at end of file diff --git a/hack/make/policy_controller.make b/hack/make/policy_controller.make new file mode 100644 index 000000000..ecc24ae58 --- /dev/null +++ b/hack/make/policy_controller.make @@ -0,0 +1,51 @@ +##@ PolicyController + +POLICY_CONTROLLER_IMG ?= policy-controller:$(TAG) +LOG_LEVEL ?= 3 + +.PHONY: build-policy-controller +build-policy-controller: manifests generate fmt vet ## Build controller binary. + go build -o bin/policy_controller ./cmd/policy_controller/main.go + +.PHONY: run-policy-controller +run-policy-controller: manifests generate fmt vet install + go run ./cmd/policy_controller/main.go \ + --metrics-bind-address=:8090 \ + --health-probe-bind-address=:8091 \ + --zap-log-level=$(LOG_LEVEL) + +.PHONY: docker-build-policy-controller +docker-build-policy-controller: ## Build docker image with the controller. + docker build --target policy-controller -t ${POLICY_CONTROLLER_IMG} . + docker image prune -f --filter label=stage=mgc-builder + +.PHONY: kind-load-policy-controller +kind-load-policy-controller: docker-build-policy-controller + kind load docker-image ${POLICY_CONTROLLER_IMG} --name mgc-control-plane --nodes mgc-control-plane-control-plane + +.PHONY: docker-push-policy-controller +docker-push-policy-controller: ## Push docker image with the controller. + docker push ${POLICY_CONTROLLER_IMG} + +.PHONY: deploy-policy-controller +deploy-policy-controller: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/policy-controller && $(KUSTOMIZE) edit set image policy-controller=${POLICY_CONTROLLER_IMG} + $(KUSTOMIZE) --load-restrictor LoadRestrictionsNone build config/deploy/local | kubectl apply -f - + @if [ "$(METRICS)" = "true" ]; then\ + $(KUSTOMIZE) build config/prometheus | kubectl apply -f -;\ + fi + +.PHONY: undeploy-policy-controller +undeploy-policy-controller: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + @if [ $(METRICS) = "true" ]; then\ + $(KUSTOMIZE) build config/prometheus | kubectl delete --ignore-not-found=$(ignore-not-found) -f -;\ + fi + $(KUSTOMIZE) --load-restrictor LoadRestrictionsNone build config/deploy/local | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + +.PHONY: restart-policy-controller +restart-policy-controller: + kubectl rollout restart deployment policy-controller-manager -n multicluster-gateway-controller-system + +.PHONY: tail-policy-controller-logs +tail-policy-controller-logs: + kubectl logs -f deployment/policy-controller-manager -n multicluster-gateway-controller-system \ No newline at end of file diff --git a/pkg/controllers/dnspolicy/dnspolicy_controller.go b/pkg/controllers/dnspolicy/dnspolicy_controller.go index b03a1f69a..334217e2c 100644 --- a/pkg/controllers/dnspolicy/dnspolicy_controller.go +++ b/pkg/controllers/dnspolicy/dnspolicy_controller.go @@ -305,24 +305,27 @@ func (r *DNSPolicyReconciler) updateGatewayCondition(ctx context.Context, condit return nil } -func (r *DNSPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { +func (r *DNSPolicyReconciler) SetupWithManager(mgr ctrl.Manager, ocmHub bool) error { gatewayEventMapper := events.NewGatewayEventMapper(r.Logger(), &DNSPolicyRefsConfig{}, "dnspolicy") clusterEventMapper := events.NewClusterEventMapper(r.Logger(), r.Client(), &DNSPolicyRefsConfig{}, "dnspolicy") probeEventMapper := events.NewProbeEventMapper(r.Logger(), DNSPolicyBackRefAnnotation, "dnspolicy") r.dnsHelper = dnsHelper{Client: r.Client()} - return ctrl.NewControllerManagedBy(mgr). + ctrlr := ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.DNSPolicy{}). Watches( &source.Kind{Type: &gatewayapiv1beta1.Gateway{}}, handler.EnqueueRequestsFromMapFunc(gatewayEventMapper.MapToPolicy), ). - Watches( - &source.Kind{Type: &clusterv1.ManagedCluster{}}, - handler.EnqueueRequestsFromMapFunc(clusterEventMapper.MapToPolicy), - ). Watches( &source.Kind{Type: &v1alpha1.DNSHealthCheckProbe{}}, handler.EnqueueRequestsFromMapFunc(probeEventMapper.MapToPolicy), - ). - Complete(r) + ) + if ocmHub { + r.Logger().Info("ocm enabled turning on managed cluster watch") + ctrlr.Watches( + &source.Kind{Type: &clusterv1.ManagedCluster{}}, + handler.EnqueueRequestsFromMapFunc(clusterEventMapper.MapToPolicy), + ) + } + return ctrlr.Complete(r) } diff --git a/pkg/controllers/dnspolicy/dnspolicy_dnsrecords.go b/pkg/controllers/dnspolicy/dnspolicy_dnsrecords.go index 53d09025c..af5affa70 100644 --- a/pkg/controllers/dnspolicy/dnspolicy_dnsrecords.go +++ b/pkg/controllers/dnspolicy/dnspolicy_dnsrecords.go @@ -20,6 +20,10 @@ import ( "github.com/Kuadrant/multicluster-gateway-controller/pkg/dns" ) +const ( + singleCluster = "kudarant.io/single" +) + func (r *DNSPolicyReconciler) reconcileDNSRecords(ctx context.Context, dnsPolicy *v1alpha1.DNSPolicy, gwDiffObj *reconcilers.GatewayDiff) error { log := crlog.FromContext(ctx) @@ -76,7 +80,7 @@ func (r *DNSPolicyReconciler) reconcileGatewayDNSRecords(ctx context.Context, ga } log.V(3).Info("hostHasAttachedRoutes", "host", listener.Name, "hostHasAttachedRoutes", attached) - cg, err := r.buildClusterGateway(ctx, clusterName, clusterAddress) + cg, err := r.buildClusterGateway(ctx, clusterName, clusterAddress, gateway) if err != nil { return fmt.Errorf("get cluster gateway failed: %s", err) } @@ -147,28 +151,36 @@ func (r *DNSPolicyReconciler) deleteDNSRecordsWithLabels(ctx context.Context, lb return nil } -func (r *DNSPolicyReconciler) buildClusterGateway(ctx context.Context, downstreamClusterName string, clusterAddress []gatewayv1beta1.GatewayAddress) (dns.ClusterGateway, error) { +func (r *DNSPolicyReconciler) buildClusterGateway(ctx context.Context, downstreamClusterName string, clusterAddress []gatewayv1beta1.GatewayAddress, targetGW *gatewayv1beta1.Gateway) (dns.ClusterGateway, error) { var target dns.ClusterGateway singleClusterAddresses := make([]gatewayv1beta1.GatewayAddress, len(clusterAddress)) - mc := &clusterv1.ManagedCluster{} - if err := r.Client().Get(ctx, client.ObjectKey{Name: downstreamClusterName}, mc, &client.GetOptions{}); err != nil { - return target, err + var metaObj client.Object + if downstreamClusterName != singleCluster { + mc := &clusterv1.ManagedCluster{} + if err := r.Client().Get(ctx, client.ObjectKey{Name: downstreamClusterName}, mc, &client.GetOptions{}); err != nil { + return target, err + } + metaObj = mc + } else { + metaObj = targetGW } for i, addr := range clusterAddress { - addrType := gatewayv1beta1.IPAddressType - if *addr.Type == gateway.MultiClusterHostnameAddressType { + addrType := *addr.Type + if addrType == gateway.MultiClusterHostnameAddressType { addrType = gatewayv1beta1.HostnameAddressType } + if addrType == gateway.MultiClusterIPAddressType { + addrType = gatewayv1beta1.IPAddressType + } singleClusterAddresses[i] = gatewayv1beta1.GatewayAddress{ Type: &addrType, Value: addr.Value, } } - - target = *dns.NewClusterGateway(mc, singleClusterAddresses) + target = *dns.NewClusterGateway(metaObj, singleClusterAddresses) return target, nil } @@ -180,7 +192,7 @@ func getClusterGatewayAddresses(gw *gatewayv1beta1.Gateway) map[string][]gateway var gatewayAddresses []gatewayv1beta1.GatewayAddress //Default to Single Cluster (Normal Gateway Status) - cluster := "none" + cluster := singleCluster addressValue := address.Value //Check for Multi Cluster (MGC Gateway Status) diff --git a/test/integration/gateway_controller_test.go b/test/gateway_integration/gateway_controller_test.go similarity index 99% rename from test/integration/gateway_controller_test.go rename to test/gateway_integration/gateway_controller_test.go index 57e15e2f3..9aadae1fe 100644 --- a/test/integration/gateway_controller_test.go +++ b/test/gateway_integration/gateway_controller_test.go @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package integration +package gateway_integration import ( "encoding/json" diff --git a/test/integration/helper_test.go b/test/gateway_integration/placement_fake.go similarity index 90% rename from test/integration/helper_test.go rename to test/gateway_integration/placement_fake.go index 9fdd49f2d..c3b9f50f3 100644 --- a/test/integration/helper_test.go +++ b/test/gateway_integration/placement_fake.go @@ -1,11 +1,9 @@ //go:build integration -package integration +package gateway_integration import ( "context" - "fmt" - "net/http" "time" "github.com/google/uuid" @@ -171,33 +169,3 @@ func DeleteNamespace(namespace *string) { return nil }, 3*time.Minute, 2*time.Second).Should(BeNil()) } - -type testHealthServer struct { - Port int -} - -func (s *testHealthServer) Start(ctx context.Context) error { - mux := http.NewServeMux() - - endpoint := func(expectedCode int) func(http.ResponseWriter, *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(expectedCode) - } - } - - mux.HandleFunc("/healthy", endpoint(200)) - mux.HandleFunc("/unhealthy", endpoint(500)) - - errCh := make(chan error) - - go func() { - errCh <- http.ListenAndServe(fmt.Sprintf(":%d", s.Port), mux) - }() - - select { - case <-ctx.Done(): - return ctx.Err() - case err := <-errCh: - return err - } -} diff --git a/test/gateway_integration/suite_test.go b/test/gateway_integration/suite_test.go new file mode 100644 index 000000000..209cac7b1 --- /dev/null +++ b/test/gateway_integration/suite_test.go @@ -0,0 +1,148 @@ +//go:build integration + +/* +Copyright 2023 The MultiCluster Traffic Controller Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gateway_integration + +import ( + "context" + "path/filepath" + "testing" + + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + ocmclusterv1 "open-cluster-management.io/api/cluster/v1" + ocmclusterv1beta1 "open-cluster-management.io/api/cluster/v1beta1" + ocmworkv1 "open-cluster-management.io/api/work/v1" + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + + "github.com/Kuadrant/multicluster-gateway-controller/pkg/apis/v1alpha1" + . "github.com/Kuadrant/multicluster-gateway-controller/pkg/controllers/gateway" + "github.com/Kuadrant/multicluster-gateway-controller/pkg/dns" + "github.com/Kuadrant/multicluster-gateway-controller/pkg/placement" + //+kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + cfg *rest.Config + k8sClient client.Client + testEnv *envtest.Environment + ctx context.Context + cancel context.CancelFunc + logger logr.Logger + providerFactory = func(ctx context.Context, managedZone *v1alpha1.ManagedZone) (dns.Provider, error) { + return &dns.FakeProvider{}, nil + } +) + +func testClient() client.Client { return k8sClient } + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func() { + logger = zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)) + logger.WithName("suite_test") + logf.SetLogger(logger) + ctx, cancel = context.WithCancel(context.TODO()) + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join("../../", "config", "crd", "bases"), + filepath.Join("../../", "config", "gateway-api", "crd", "standard"), + filepath.Join("../../", "config", "cert-manager", "crd", "latest"), + filepath.Join("../../", "config", "ocm", "crd"), + }, + ErrorIfCRDPathMissing: true, + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + err = v1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + err = gatewayv1beta1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + // err = certman.AddToScheme(scheme.Scheme) + // Expect(err).NotTo(HaveOccurred()) + + err = ocmworkv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + err = ocmclusterv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + err = ocmclusterv1beta1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + HealthProbeBindAddress: "0", + MetricsBindAddress: "0", + }) + Expect(err).ToNot(HaveOccurred()) + + plc := placement.NewOCMPlacer(k8sManager.GetClient()) + + err = (&GatewayClassReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + }).SetupWithManager(k8sManager) + Expect(err).ToNot(HaveOccurred()) + + err = (&GatewayReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + Placement: plc, + }).SetupWithManager(k8sManager, ctx) + Expect(err).ToNot(HaveOccurred()) + + go func() { + defer GinkgoRecover() + err = k8sManager.Start(ctx) + Expect(err).ToNot(HaveOccurred(), "failed to run manager") + }() +}) + +var _ = AfterSuite(func() { + cancel() + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/test/integration/dnsheathcheckprobe_controller_test.go b/test/policy_integration/dnsheathcheckprobe_controller_test.go similarity index 99% rename from test/integration/dnsheathcheckprobe_controller_test.go rename to test/policy_integration/dnsheathcheckprobe_controller_test.go index a5692e0d6..bbc2a548b 100644 --- a/test/integration/dnsheathcheckprobe_controller_test.go +++ b/test/policy_integration/dnsheathcheckprobe_controller_test.go @@ -1,6 +1,6 @@ //go:build integration -package integration +package policy_integration import ( "context" diff --git a/test/integration/dnspolicy_controller_test.go b/test/policy_integration/dnspolicy_controller_test.go similarity index 99% rename from test/integration/dnspolicy_controller_test.go rename to test/policy_integration/dnspolicy_controller_test.go index b6fb8f055..3d240b25c 100644 --- a/test/integration/dnspolicy_controller_test.go +++ b/test/policy_integration/dnspolicy_controller_test.go @@ -1,6 +1,6 @@ //go:build integration -package integration +package policy_integration import ( "encoding/json" diff --git a/test/policy_integration/helper_test.go b/test/policy_integration/helper_test.go new file mode 100644 index 000000000..53626b3e1 --- /dev/null +++ b/test/policy_integration/helper_test.go @@ -0,0 +1,61 @@ +//go:build integration + +package policy_integration + +import ( + "context" + "fmt" + "net/http" + "time" +) + +const ( + TestTimeoutMedium = time.Second * 10 + TestTimeoutLong = time.Second * 30 + ConsistentlyTimeoutMedium = time.Second * 60 + TestRetryIntervalMedium = time.Millisecond * 250 + TestPlacedGatewayName = "test-placed-gateway" + TestPlacedClusterControlName = "test-placed-control" + TestPlaceClusterWorkloadName = "test-placed-workload-1" + TestAttachedRouteName = "test.example.com" + OtherAttachedRouteName = "other.example.com" + TestWildCardListenerName = "wildcard" + TestWildCardListenerHost = "*.example.com" + TestAttachedRouteAddressOne = "172.0.0.1" + TestAttachedRouteAddressTwo = "172.0.0.2" + nsSpoke1Name = "test-spoke-cluster-1" + nsSpoke2Name = "test-spoke-cluster-2" + defaultNS = "default" + gatewayFinalizer = "kuadrant.io/gateway" + providerCredential = "secretname" +) + +type testHealthServer struct { + Port int +} + +func (s *testHealthServer) Start(ctx context.Context) error { + mux := http.NewServeMux() + + endpoint := func(expectedCode int) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(expectedCode) + } + } + + mux.HandleFunc("/healthy", endpoint(200)) + mux.HandleFunc("/unhealthy", endpoint(500)) + + errCh := make(chan error) + + go func() { + errCh <- http.ListenAndServe(fmt.Sprintf(":%d", s.Port), mux) + }() + + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-errCh: + return err + } +} diff --git a/test/integration/managedzone_controller_test.go b/test/policy_integration/managedzone_controller_test.go similarity index 99% rename from test/integration/managedzone_controller_test.go rename to test/policy_integration/managedzone_controller_test.go index 77af7fc37..d3e55ffff 100644 --- a/test/integration/managedzone_controller_test.go +++ b/test/policy_integration/managedzone_controller_test.go @@ -13,7 +13,7 @@ // limitations under the License. // */ -package integration +package policy_integration import ( . "github.com/onsi/ginkgo/v2" diff --git a/test/integration/suite_test.go b/test/policy_integration/suite_test.go similarity index 90% rename from test/integration/suite_test.go rename to test/policy_integration/suite_test.go index 69b90ffc8..409aed80c 100644 --- a/test/integration/suite_test.go +++ b/test/policy_integration/suite_test.go @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package integration +package policy_integration import ( "context" @@ -43,12 +43,10 @@ import ( "github.com/Kuadrant/multicluster-gateway-controller/pkg/apis/v1alpha1" . "github.com/Kuadrant/multicluster-gateway-controller/pkg/controllers/dnshealthcheckprobe" . "github.com/Kuadrant/multicluster-gateway-controller/pkg/controllers/dnspolicy" - . "github.com/Kuadrant/multicluster-gateway-controller/pkg/controllers/gateway" . "github.com/Kuadrant/multicluster-gateway-controller/pkg/controllers/managedzone" . "github.com/Kuadrant/multicluster-gateway-controller/pkg/controllers/tlspolicy" "github.com/Kuadrant/multicluster-gateway-controller/pkg/dns" "github.com/Kuadrant/multicluster-gateway-controller/pkg/health" - "github.com/Kuadrant/multicluster-gateway-controller/pkg/placement" //+kubebuilder:scaffold:imports ) @@ -86,6 +84,7 @@ var _ = BeforeSuite(func() { filepath.Join("../../", "config", "crd", "bases"), filepath.Join("../../", "config", "gateway-api", "crd", "standard"), filepath.Join("../../", "config", "cert-manager", "crd", "latest"), + //needed for now TODO remove once no longer need managedcluster in dnspolicy filepath.Join("../../", "config", "ocm", "crd"), }, ErrorIfCRDPathMissing: true, @@ -141,8 +140,6 @@ var _ = BeforeSuite(func() { err = k8sManager.Add(healthServer) Expect(err).ToNot(HaveOccurred()) - plc := placement.NewOCMPlacer(k8sManager.GetClient()) - dnsPolicyBaseReconciler := reconcilers.NewBaseReconciler( k8sManager.GetClient(), k8sManager.GetScheme(), k8sManager.GetAPIReader(), logger.WithName("dnspolicy"), @@ -154,7 +151,7 @@ var _ = BeforeSuite(func() { BaseReconciler: dnsPolicyBaseReconciler, }, DNSProvider: providerFactory, - }).SetupWithManager(k8sManager) + }).SetupWithManager(k8sManager, true) Expect(err).ToNot(HaveOccurred()) tlsPolicyBaseReconciler := reconcilers.NewBaseReconciler( @@ -170,19 +167,6 @@ var _ = BeforeSuite(func() { }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) - err = (&GatewayClassReconciler{ - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), - }).SetupWithManager(k8sManager) - Expect(err).ToNot(HaveOccurred()) - - err = (&GatewayReconciler{ - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), - Placement: plc, - }).SetupWithManager(k8sManager, ctx) - Expect(err).ToNot(HaveOccurred()) - err = (&ManagedZoneReconciler{ Client: k8sManager.GetClient(), Scheme: k8sManager.GetScheme(), diff --git a/test/integration/tlspolicy_controller_test.go b/test/policy_integration/tlspolicy_controller_test.go similarity index 99% rename from test/integration/tlspolicy_controller_test.go rename to test/policy_integration/tlspolicy_controller_test.go index fdd07e41e..243a99c11 100644 --- a/test/integration/tlspolicy_controller_test.go +++ b/test/policy_integration/tlspolicy_controller_test.go @@ -1,6 +1,6 @@ //go:build integration -package integration +package policy_integration import ( "encoding/json" diff --git a/test/policy_integration/util.go b/test/policy_integration/util.go new file mode 100644 index 000000000..c83e863d8 --- /dev/null +++ b/test/policy_integration/util.go @@ -0,0 +1,52 @@ +//go:build integration + +package policy_integration + +import ( + "context" + "time" + + "github.com/google/uuid" + . "github.com/onsi/gomega" + + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func CreateNamespace(namespace *string) { + var generatedTestNamespace = "test-namespace-" + uuid.New().String() + + nsObject := &v1.Namespace{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Namespace"}, + ObjectMeta: metav1.ObjectMeta{Name: generatedTestNamespace}, + } + + err := testClient().Create(context.Background(), nsObject) + Expect(err).ToNot(HaveOccurred()) + + existingNamespace := &v1.Namespace{} + Eventually(func() error { + return testClient().Get(context.Background(), types.NamespacedName{Name: generatedTestNamespace}, existingNamespace) + }, time.Minute, 5*time.Second).ShouldNot(HaveOccurred()) + + *namespace = existingNamespace.Name +} + +func DeleteNamespace(namespace *string) { + desiredTestNamespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: *namespace}} + err := testClient().Delete(context.Background(), desiredTestNamespace, client.PropagationPolicy(metav1.DeletePropagationForeground)) + + Expect(err).ToNot(HaveOccurred()) + + existingNamespace := &v1.Namespace{} + Eventually(func() error { + err := testClient().Get(context.Background(), types.NamespacedName{Name: *namespace}, existingNamespace) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil + }, 3*time.Minute, 2*time.Second).Should(BeNil()) +}