From 592b63610aa0fb9740f9332fcea10af31a59a6e6 Mon Sep 17 00:00:00 2001 From: Nima Kaviani Date: Wed, 15 Nov 2023 00:10:39 -0800 Subject: [PATCH] install nginx out of band - remove embedded nginx installation - add a new reconciler to deploy nginx resources - use the same resources previously installed via argo - wait for it to successfully deploy before proceeding - fix minor issues --- .gitignore | 2 + api/v1alpha1/localbuild_types.go | 1 + pkg/apps/resources.go | 3 - pkg/controllers/localbuild/controller.go | 1 + pkg/controllers/localbuild/nginx.go | 134 ++++++++++++++++++ .../resources/nginx/k8s}/ingress-nginx.yaml | 0 .../idpbuilder.cnoe.io_localbuilds.yaml | 2 + pkg/k8s/client.go | 3 + 8 files changed, 143 insertions(+), 3 deletions(-) create mode 100644 pkg/controllers/localbuild/nginx.go rename pkg/{apps/srv/nginx-ingress => controllers/localbuild/resources/nginx/k8s}/ingress-nginx.yaml (100%) diff --git a/.gitignore b/.gitignore index 39186f18..d7f11e71 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ idpbuilder bin/* .DS_Store cover.out +__debug* +.vscode diff --git a/api/v1alpha1/localbuild_types.go b/api/v1alpha1/localbuild_types.go index 3ba5f3b8..4947ec83 100644 --- a/api/v1alpha1/localbuild_types.go +++ b/api/v1alpha1/localbuild_types.go @@ -36,6 +36,7 @@ type LocalbuildStatus struct { GitServerAvailable bool `json:"gitServerAvailable,omitempty"` ArgoAvailable bool `json:"argoAvailable,omitempty"` + NginxAvailable bool `json:"nginxAvailable,omitempty"` ArgoAppsCreated bool `json:"argoAppsCreated,omitempty"` } diff --git a/pkg/apps/resources.go b/pkg/apps/resources.go index da7de87b..4f97035c 100644 --- a/pkg/apps/resources.go +++ b/pkg/apps/resources.go @@ -26,9 +26,6 @@ var ( }, { Name: "crossplane", Path: "crossplane", - }, { - Name: "nginx-ingress", - Path: "nginx-ingress", }} ) diff --git a/pkg/controllers/localbuild/controller.go b/pkg/controllers/localbuild/controller.go index 089d332f..60851092 100644 --- a/pkg/controllers/localbuild/controller.go +++ b/pkg/controllers/localbuild/controller.go @@ -70,6 +70,7 @@ func (r *LocalbuildReconciler) Reconcile(ctx context.Context, req ctrl.Request) subReconcilers := []subReconciler{ r.ReconcileProjectNamespace, r.ReconcileArgo, + r.ReconcileNginx, r.ReconcileEmbeddedGitServer, r.ReconcileArgoApps, } diff --git a/pkg/controllers/localbuild/nginx.go b/pkg/controllers/localbuild/nginx.go new file mode 100644 index 00000000..98d91db9 --- /dev/null +++ b/pkg/controllers/localbuild/nginx.go @@ -0,0 +1,134 @@ +package localbuild + +import ( + "context" + "embed" + "errors" + "time" + + "github.com/cnoe-io/idpbuilder/api/v1alpha1" + "github.com/cnoe-io/idpbuilder/pkg/k8s" + "github.com/cnoe-io/idpbuilder/pkg/util" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + nginxNamespace string = "ingress-nginx" + nginxDeployment string = "ingress-nginx-controller" +) + +//go:embed resources/nginx/k8s/* +var installNginxFS embed.FS +var timeout = time.After(5 * time.Minute) + +func RawNginxInstallResources() ([][]byte, error) { + return util.ConvertFSToBytes(installNginxFS, "resources/nginx/k8s") +} + +func NginxInstallResources(scheme *runtime.Scheme) ([]client.Object, error) { + rawResources, err := RawNginxInstallResources() + if err != nil { + return nil, err + } + + return k8s.ConvertRawResourcesToObjects(scheme, rawResources) +} + +func newNginxNamespace() *corev1.Namespace { + return &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nginxNamespace, + }, + } +} + +func (r LocalbuildReconciler) ReconcileNginx(ctx context.Context, req ctrl.Request, resource *v1alpha1.Localbuild) (ctrl.Result, error) { + log := log.FromContext(ctx) + + nginxNSClient := client.NewNamespacedClient(r.Client, nginxNamespace) + installObjs, err := NginxInstallResources(r.Scheme) + if err != nil { + return ctrl.Result{}, err + } + + // Ensure namespace exists + nginxNewNS := newNginxNamespace() + if err = r.Client.Get(ctx, types.NamespacedName{Name: nginxNamespace}, nginxNewNS); err != nil { + // We got an error so try creating the NS + if err = r.Client.Create(ctx, nginxNewNS); err != nil { + return ctrl.Result{}, err + } + } + + log.Info("Installing Nginx resources") + for _, obj := range installObjs { + if obj.GetObjectKind().GroupVersionKind().Kind == "Deployment" { + switch obj.GetName() { + case nginxDeployment: + gotObj := appsv1.Deployment{} + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()}, &gotObj); err != nil { + if err = controllerutil.SetControllerReference(resource, obj, r.Scheme); err != nil { + log.Error(err, "Setting controller reference for Nginx deployment", "deployment", obj) + return ctrl.Result{}, err + } + } + } + } + + // Create object + if err = k8s.EnsureObject(ctx, nginxNSClient, obj, nginxNamespace); err != nil { + return ctrl.Result{}, err + } + } + + // Wait for Nginx to become available + ready := make(chan error) + go func([]client.Object) { + for { + for _, obj := range installObjs { + if obj.GetObjectKind().GroupVersionKind().Kind == "Deployment" { + switch obj.GetName() { + case nginxDeployment: + gotObj := appsv1.Deployment{} + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()}, &gotObj); err != nil { + ready <- err + return + } + + if gotObj.Status.AvailableReplicas >= 1 { + close(ready) + return + } + } + } + } + + time.Sleep(1 * time.Second) + } + }(installObjs) + + select { + case <-timeout: + err := errors.New("Timeout") + log.Error(err, "Didn't reconcile Nginx on time.") + return ctrl.Result{}, err + case err, errOccurred := <-ready: + if !errOccurred { + log.Info("Nginx is ready!") + resource.Status.NginxAvailable = true + } else { + log.Error(err, "failed to reconcile the Nginx resources") + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, nil +} diff --git a/pkg/apps/srv/nginx-ingress/ingress-nginx.yaml b/pkg/controllers/localbuild/resources/nginx/k8s/ingress-nginx.yaml similarity index 100% rename from pkg/apps/srv/nginx-ingress/ingress-nginx.yaml rename to pkg/controllers/localbuild/resources/nginx/k8s/ingress-nginx.yaml diff --git a/pkg/controllers/resources/idpbuilder.cnoe.io_localbuilds.yaml b/pkg/controllers/resources/idpbuilder.cnoe.io_localbuilds.yaml index cac27634..2202c0c4 100644 --- a/pkg/controllers/resources/idpbuilder.cnoe.io_localbuilds.yaml +++ b/pkg/controllers/resources/idpbuilder.cnoe.io_localbuilds.yaml @@ -62,6 +62,8 @@ spec: type: boolean gitServerAvailable: type: boolean + nginxAvailable: + type: boolean observedGeneration: description: ObservedGeneration is the 'Generation' of the Service that was last processed by the controller. diff --git a/pkg/k8s/client.go b/pkg/k8s/client.go index d5ded3ed..7ef9f863 100644 --- a/pkg/k8s/client.go +++ b/pkg/k8s/client.go @@ -36,5 +36,8 @@ func EnsureObject(ctx context.Context, kubeClient client.Client, obj client.Obje if err != nil { return err } + + // hacky way to restore the GVK for the object after create corrupts it. didnt dig. not sure why? + obj.GetObjectKind().SetGroupVersionKind(curObj.GroupVersionKind()) return nil }