From 633cbe88d8223e7b3360d93f02fefeff071b49b6 Mon Sep 17 00:00:00 2001 From: Iaroslav Ivchenkov Date: Mon, 23 Dec 2024 17:39:07 +0200 Subject: [PATCH] Added docs and local helm registry - Simple docker registry is deployed to the kind cluster as OCI Helm registry, exposed to the local machine with the NodePort on 30500 - Added docker daemon installation on Linux-based systems - Added notes about `make help` to the README.md - Added docs with detailed explanation for General setup steps - Fixed helm package bug with missing templates directory (added .gitkeep files) --- .github/workflows/release.yml | 27 ------- Makefile | 75 +++++++++--------- README.md | 40 ++++++---- .../1-general-setup-bootstrap-kind-cluster.md | 16 ++++ documentation/2-general-setup-deploy-2a.md | 13 ++++ .../3-general-setup-helmrepo-setup.md | 7 ++ setup/helmRepository.yaml | 78 ++++++++++++++++--- setup/kind-cluster.yaml | 8 ++ .../templates/.gitkeep | 0 .../templates/.gitkeep | 0 .../demo-kyverno-3.2.6/templates/.gitkeep | 0 11 files changed, 173 insertions(+), 91 deletions(-) delete mode 100644 .github/workflows/release.yml create mode 100644 documentation/1-general-setup-bootstrap-kind-cluster.md create mode 100644 documentation/2-general-setup-deploy-2a.md create mode 100644 documentation/3-general-setup-helmrepo-setup.md create mode 100644 setup/kind-cluster.yaml create mode 100644 templates/service/demo-ingress-nginx-4.11.0/templates/.gitkeep create mode 100644 templates/service/demo-ingress-nginx-4.11.3/templates/.gitkeep create mode 100644 templates/service/demo-kyverno-3.2.6/templates/.gitkeep diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index f836ef3..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Package & Publish Demo Charts - -on: - push: - tags: - - "*" - branches: - - main - -jobs: - release: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Login to GHCR - uses: docker/login-action@v3.3.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Push charts to GHCR - run: REGISTRY_REPO="oci://ghcr.io/${GITHUB_REPOSITORY,,}/charts" make helm-push \ No newline at end of file diff --git a/Makefile b/Makefile index 8325faa..7e6debe 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,6 @@ TESTING_NAMESPACE ?= hmc-system TARGET_NAMESPACE ?= blue KIND_CLUSTER_NAME ?= hmc-management-local - OPENSSL_DOCKER_IMAGE ?= alpine/openssl:3.3.2 TEMPLATES_DIR := templates @@ -18,9 +17,9 @@ $(CHARTS_PACKAGE_DIR): | $(LOCALBIN) rm -rf $(CHARTS_PACKAGE_DIR) mkdir -p $(CHARTS_PACKAGE_DIR) -REGISTRY_PORT ?= 5001 -REGISTRY_REPO ?= oci://127.0.0.1:$(REGISTRY_PORT)/charts -REGISTRY_IS_OCI = $(shell echo $(REGISTRY_REPO) | grep -q oci && echo true || echo false) +HELM_REGISTRY_INTERNAL_PORT ?= 5000 +HELM_REGISTRY_EXTERNAL_PORT ?= 30500 +REGISTRY_REPO ?= oci://127.0.0.1:$(HELM_REGISTRY_EXTERNAL_PORT)/helm-charts ##@ General @@ -68,6 +67,8 @@ HELM_VERSION ?= v3.15.1 KUBECTL ?= PATH=$(LOCALBIN):$(PATH) kubectl +DOCKER_VERSION ?= 27.4.1 + # installs binary locally $(LOCALBIN)/%: $(LOCALBIN) @curl -sLo $(LOCALBIN)/$(binary) $(url);\ @@ -79,7 +80,20 @@ $(LOCALBIN)/%: $(LOCALBIN) || (echo "Can't find the $(binary) in path, installing it locally" && make $(LOCALBIN)/$(binary)) .check-binary-docker: - @which docker $ > /dev/null || (echo "Please install docker before proceeding" && exit 1) + @if ! which docker $ > /dev/null; then \ + if [ "$(OS)" = "linux" ]; then \ + curl -sLO https://download.docker.com/linux/static/stable/$(shell uname -m)/docker-$(DOCKER_VERSION).tgz;\ + tar xzvf docker-$(DOCKER_VERSION).tgz; \ + sudo cp docker/* /usr/bin/ ; \ + echo "Starting docker daemon..." ; \ + sudo dockerd > /dev/null 2>&1 & sudo groupadd docker ; \ + sudo usermod -aG docker $(shell whoami) ; \ + newgrp docker ; \ + echo "Docker engine installed and started"; \ + else \ + echo "Please install docker before proceeding. If your work on machine with MacOS, check this installation guide: https://docs.docker.com/desktop/setup/install/mac-install/" && exit 1; \ + fi; \ + fi; %kind: binary = kind %kind: url = "https://kind.sigs.k8s.io/dl/v$(KIND_VERSION)/kind-$(OS)-$(ARCH)" @@ -101,12 +115,17 @@ $(LOCALBIN)/helm: | $(LOCALBIN) ##@ General Setup +KIND_CLUSTER_CONFIG_PATH ?= $(LOCALBIN)/kind-cluster.yaml +$(KIND_CLUSTER_CONFIG_PATH): $(LOCALBIN) + @cat setup/kind-cluster.yaml | envsubst > $(LOCALBIN)/kind-cluster.yaml + .PHONY: bootstrap-kind-cluster -bootstrap-kind-cluster: .check-binary-docker .check-binary-kind .check-binary-kubectl ## Provision local kind cluster +bootstrap-kind-cluster: .check-binary-docker .check-binary-kind .check-binary-kubectl $(KIND_CLUSTER_CONFIG_PATH) ## Provision local kind cluster +bootstrap-kind-cluster: ## Provision local kind cluster @if $(KIND) get clusters | grep -q $(KIND_CLUSTER_NAME); then\ echo "$(KIND_CLUSTER_NAME) kind cluster already installed";\ else\ - $(KIND) create cluster --name=$(KIND_CLUSTER_NAME);\ + $(KIND) create cluster --name=$(KIND_CLUSTER_NAME) --config=$(KIND_CLUSTER_CONFIG_PATH);\ fi @$(KUBECTL) config use-context kind-$(KIND_CLUSTER_NAME) @@ -114,6 +133,13 @@ bootstrap-kind-cluster: .check-binary-docker .check-binary-kind .check-binary-ku deploy-2a: .check-binary-helm ## Deploy 2A to the management cluster $(HELM) install hmc $(HMC_REPO) --version $(HMC_VERSION) -n $(HMC_NAMESPACE) --create-namespace +.PHONY: setup-helmrepo +setup-helmrepo: ## Deploy local helm repository and register it in 2A + @envsubst < setup/helmRepository.yaml | $(KUBECTL) apply -f - + +.PHONY: push-helm-charts +push-helm-charts: helm-push ## Push helm charts with custom Cluster and Service templates + ##@ Infra Setup # AWS @@ -140,11 +166,6 @@ setup-azure-creds: .check-variable-azure-sp-password .check-variable-azure-sp-ap ##@ TBD -.PHONY: setup-helmrepo -setup-helmrepo: - kubectl apply -f setup/helmRepository.yaml - - # install-template will install a given template # $1 - yaml file define install-template @@ -473,38 +494,16 @@ package-chart-%: lint-chart-% .PHONY: helm-push helm-push: helm-package - if [ ! $(REGISTRY_IS_OCI) ]; then \ - repo_flag="--repo"; \ - fi; \ - for chart in $(CHARTS_PACKAGE_DIR)/*.tgz; do \ + @for chart in $(CHARTS_PACKAGE_DIR)/*.tgz; do \ base=$$(basename $$chart .tgz); \ chart_version=$$(echo $$base | grep -o "v\{0,1\}[0-9]\+\.[0-9]\+\.[0-9].*"); \ chart_name="$${base%-"$$chart_version"}"; \ echo "Verifying if chart $$chart_name, version $$chart_version already exists in $(REGISTRY_REPO)"; \ - if $(REGISTRY_IS_OCI); then \ - echo $(HELM) pull $$repo_flag $(REGISTRY_REPO)/$$chart_name --version $$chart_version --destination /tmp; \ - $(HELM) pull $$repo_flag $(REGISTRY_REPO)/$$chart_name --version $$chart_version --destination /tmp; \ - chart_exists=$$($(HELM) pull $$repo_flag $(REGISTRY_REPO)/$$chart_name --version $$chart_version --destination /tmp 2>&1 | grep "not found" || true); \ - else \ - echo $(HELM) pull $$repo_flag $(REGISTRY_REPO) $$chart_name --version $$chart_version --destination /tmp; \ - $(HELM) pull $$repo_flag $(REGISTRY_REPO) $$chart_name --version $$chart_version --destination /tmp; \ - chart_exists=$$($(HELM) pull $$repo_flag $(REGISTRY_REPO) $$chart_name --version $$chart_version --destination /tmp 2>&1 | grep "not found" || true); \ - fi; \ + chart_exists=$$($(HELM) show chart $(REGISTRY_REPO)/$$chart_name --version $$chart_version 2>&1 | grep "failed to download" || true); \ if [ -z "$$chart_exists" ]; then \ echo "Chart $$chart_name version $$chart_version already exists in the repository."; \ else \ - if $(REGISTRY_IS_OCI); then \ - echo "Pushing $$chart to $(REGISTRY_REPO)"; \ - $(HELM) push "$$chart" $(REGISTRY_REPO); \ - else \ - if [ ! $$REGISTRY_USERNAME ] && [ ! $$REGISTRY_PASSWORD ]; then \ - echo "REGISTRY_USERNAME and REGISTRY_PASSWORD must be populated to push the chart to an HTTPS repository"; \ - exit 1; \ - else \ - $(HELM) repo add hmc $(REGISTRY_REPO); \ - echo "Pushing $$chart to $(REGISTRY_REPO)"; \ - $(HELM) cm-push "$$chart" $(REGISTRY_REPO) --username $$REGISTRY_USERNAME --password $$REGISTRY_PASSWORD; \ - fi; \ - fi; \ + echo "Pushing $$chart to $(REGISTRY_REPO)"; \ + $(HELM) push "$$chart" $(REGISTRY_REPO); \ fi; \ done diff --git a/README.md b/README.md index b785c6a..63752bc 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ All demos in here provide their own complete ClusterTemplates and ServiceTemplat The Setup part for Demos is assumed to be created once before an actual demo is given. -Please make sure that docker is installed on your machine! It's required to run local kind cluster. +To get the full list of commands run `make help`. ### General Setup @@ -20,7 +20,9 @@ Please make sure that docker is installed on your machine! It's required to run ```shell make bootstrap-kind-cluster ``` - You could give it another name by specifying the `KIND_CLUSTER_NAME` environment variable. + You could give it another name by specifying the `KIND_CLUSTER_NAME` environment variable. + + For detailed explanation, please refer to the [documentation](./documentation/1-general-setup-bootstrap-kind-cluster.md). 2. Install 2A into kind cluster: ```shell @@ -31,24 +33,32 @@ Please make sure that docker is installed on your machine! It's required to run 3. Monitor the installation of 2A: ```shell PATH=$PATH:./bin kubectl get management hmc -o go-template='{{range $key, $value := .status.components}}{{$key}}: {{if $value.success}}{{$value.success}}{{else}}{{$value.error}}{{end}}{{"\n"}}{{end}}' - - ``` - If the installation of 2a succeeded, the output should look as follows - ``` - capi: true - cluster-api-provider-aws: true - cluster-api-provider-azure: true - cluster-api-provider-vsphere: true - hmc: true - k0smotron: true - projectsveltos: true - ``` + ``` + If the installation of 2a succeeded, the output should look as follows + ``` + capi: true + cluster-api-provider-aws: true + cluster-api-provider-azure: true + cluster-api-provider-vsphere: true + hmc: true + k0smotron: true + projectsveltos: true + ``` + For detailed explanation, please refer to the [documentation](./documentation/2-general-setup-deploy-2a.md). 4. Install the Demo Helm Repo into 2A: ```shell make setup-helmrepo ``` - This step adds a [`HelmRepository` resource](https://fluxcd.io/flux/components/source/helmrepositories/) to the cluster that contains Helm charts for this demo. + This step deploys simple local OCI Helm registry and adds a [`HelmRepository` resource](https://fluxcd.io/flux/components/source/helmrepositories/) to the cluster that contains Helm charts for this demo. + + For detailed explanation, please refer to the [documentation](./documentation/3-general-setup-helmrepo-setup.md). + +5. Push Helm charts with custom Cluster and Service Templates + ``` + make push-helm-charts + ``` + ### Infra Setup diff --git a/documentation/1-general-setup-bootstrap-kind-cluster.md b/documentation/1-general-setup-bootstrap-kind-cluster.md new file mode 100644 index 0000000..3431982 --- /dev/null +++ b/documentation/1-general-setup-bootstrap-kind-cluster.md @@ -0,0 +1,16 @@ +# Bootstrap local management cluster + +To demonstrate the capabilities of the 2A platform, we need to create a [management kubernetes cluster](https://mirantis.github.io/project-2a-docs/glossary/#management-cluster) through which [managed clusters](https://mirantis.github.io/project-2a-docs/glossary/#managed-cluster) will be deployed and ["beach-headed" services](https://mirantis.github.io/project-2a-docs/glossary/#beach-head-services) will be installed on them. + +For these purposes, we use the [kind utility](https://kind.sigs.k8s.io/), which easily allows you to create a local kubernetes cluster. +Cluster nodes are deployed inside containers, it turns out to be isolated and access for traffic "from the outside" is closed, except for the kubernetes API (to allow kubectl access). Since we will also need a Helm registry for further steps and in order to have access to it from our working machine, we will immediately, when creating the cluster, establish a connection from port 30500 of our working machine to port 30500 of kind cluster nodes. This port will later be used for a Helm registry NodePort service. You can find the kind cluster configuration at [setup/kind-cluster.yaml](../setup/kind-cluster.yaml). + +By default, kind cluster has the `hmc-management-local` name and it can be changed with the `KIND_CLUSTER_NAME` environment variable. + +## Prerequisites + +The `make bootstrap-kind-cluster` command makes several checks before running the command that creates the cluster: + +1. If Docker Engine is not installed on your machine, the binary will be installed for Linux-based OS and the error with the documentation link printed for Darwin OS. By default, the `27.4.1` Docker Engine version will be installed for Linux OS and it can be changed with the `DOCKER_VERSION` environment variable. +2. If `kind` binary is not detected on your machine, the one will be downloaded, installed in the `/bin` directory and be used during the whole demo. By default, will be installed kind version `0.25.0` and it can be changed with the `KIND_VERSION` environment variable. +3. If `kubectl` binary is not detected on your machine, it will be downloaded, installed in the `/bin` directory and be used during the whole. The latest stable version of `kubectl` will be installed. diff --git a/documentation/2-general-setup-deploy-2a.md b/documentation/2-general-setup-deploy-2a.md new file mode 100644 index 0000000..2185218 --- /dev/null +++ b/documentation/2-general-setup-deploy-2a.md @@ -0,0 +1,13 @@ +# 2A Deploy + +2A is deployed using the Helm chart on top of the management kubernetes cluster. It installs required CRDs, hmc kubernetes operator and a couple of common utilities. However, when Helm chart is successfully deployed, 2A is not ready to be used at the moment. HMC operator starts the reconciliation process that installs [CAPI](https://cluster-api.sigs.k8s.io/), different cloud providers, default built-in cluster and service templates, etc. + +To get the information about 2A platform readiness you can run this command: + +```shell +PATH=$PATH:./bin kubectl get management hmc -o go-template='{{range $key, $value := .status.components}}{{$key}}: {{if $value.success}}{{$value.success}}{{else}}{{$value.error}}{{end}}{{"\n"}}{{end}}' +``` + +It checks the `Management` object, which is the HMC custom resource. This object contains requirements on what providers must be installed, the HMC release version, etc. HMC operator reconciles the platform state to satisfy requirements from this object and updates the status. + +You can find detailed information about 2A installation in the [official documentation](https://mirantis.github.io/project-2a-docs/quick-start/2a-installation/). \ No newline at end of file diff --git a/documentation/3-general-setup-helmrepo-setup.md b/documentation/3-general-setup-helmrepo-setup.md new file mode 100644 index 0000000..5291aeb --- /dev/null +++ b/documentation/3-general-setup-helmrepo-setup.md @@ -0,0 +1,7 @@ +# Deploy and setup Helm registry + +During the demo we want to show how [BYO](https://mirantis.github.io/project-2a-docs/template/byo-templates/) `ClusterTemplate` and `ServiceTemplate` objects can be created and used to provision managed clusters and to install services on top of them. As 2A uses Flux CD, we will package configurations to Helm charts, push them to the Helm registry and use this registry in 2A. + +As a simple OCI Helm registry will be used docker registry, deployed to the kind cluster as pod and exposed with the NodePort service outside of the cluster on the `30500` port. As a result, the Helm registry can be accessed from the local working machine via `oci://127.0.0.1:30500/helm-charts` and `oci://helm-registry:5000/helm-charts` inside the cluster. + +You can find the registry configuration at [./setup/helmRepository.yaml](../setup/helmRepository.yaml). \ No newline at end of file diff --git a/setup/helmRepository.yaml b/setup/helmRepository.yaml index 9231fb7..a399eb0 100644 --- a/setup/helmRepository.yaml +++ b/setup/helmRepository.yaml @@ -1,13 +1,69 @@ -apiVersion: source.toolkit.fluxcd.io/v1 -kind: HelmRepository +--- +apiVersion: apps/v1 +kind: Deployment metadata: - name: 2a-demos - namespace: hmc-system - labels: - hmc.mirantis.com/managed: "true" + name: helm-registry + namespace: $TESTING_NAMESPACE spec: - insecure: true - interval: 10m0s - provider: generic - type: oci - url: oci://ghcr.io/mirantis/2a-demos/charts + selector: + matchLabels: + run: helm-registry + template: + metadata: + labels: + run: helm-registry + spec: + containers: + - image: registry:2 + name: helm-registry + ports: + - containerPort: $HELM_REGISTRY_INTERNAL_PORT + volumes: + - name: registry-storage + persistentVolumeClaim: + claimName: helm-registry-storage + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: helm-registry-storage + namespace: $TESTING_NAMESPACE +spec: + accessModes: + - ReadWriteOnce + storageClassName: standard # kind local-path provisioner + resources: + requests: + storage: 50Mi + +--- +apiVersion: v1 +kind: Service +metadata: + name: helm-registry + namespace: $TESTING_NAMESPACE +spec: + type: NodePort + ports: + - port: $HELM_REGISTRY_INTERNAL_PORT + protocol: TCP + targetPort: $HELM_REGISTRY_INTERNAL_PORT + nodePort: $HELM_REGISTRY_EXTERNAL_PORT + selector: + run: helm-registry + +# --- +# apiVersion: source.toolkit.fluxcd.io/v1 +# kind: HelmRepository +# metadata: +# name: 2a-demos +# namespace: $TESTING_NAMESPACE +# labels: +# hmc.mirantis.com/managed: "true" +# spec: +# insecure: true +# interval: 10m0s +# provider: generic +# type: oci +# url: oci://helm-registry.$TESTING_NAMESPACE.svc.cluster.local:5000/helm-charts diff --git a/setup/kind-cluster.yaml b/setup/kind-cluster.yaml new file mode 100644 index 0000000..6a04edc --- /dev/null +++ b/setup/kind-cluster.yaml @@ -0,0 +1,8 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraPortMappings: + - containerPort: $HELM_REGISTRY_EXTERNAL_PORT + hostPort: $HELM_REGISTRY_EXTERNAL_PORT + protocol: TCP \ No newline at end of file diff --git a/templates/service/demo-ingress-nginx-4.11.0/templates/.gitkeep b/templates/service/demo-ingress-nginx-4.11.0/templates/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/templates/service/demo-ingress-nginx-4.11.3/templates/.gitkeep b/templates/service/demo-ingress-nginx-4.11.3/templates/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/templates/service/demo-kyverno-3.2.6/templates/.gitkeep b/templates/service/demo-kyverno-3.2.6/templates/.gitkeep new file mode 100644 index 0000000..e69de29