Skip to content

Commit

Permalink
VolSync deploy as helm charts from addon-controller - https://issues.…
Browse files Browse the repository at this point in the history 
…redhat.com/browse/ACM-16311 (#512)

* render helm chart, deploy in manifestwork

- render local chart embedded in controller
- still need to download remote charts from helm repo
- helm test loading local index + ability to load remote repo

Signed-off-by: Tesshu Flower <[email protected]>

* refactoring getmanifestvalues

Signed-off-by: Tesshu Flower <[email protected]>

* only load helm charts locally, use helm always

- still have annotation as override to use olm for openshift
mgd clusters, but default will be helm
- bundle helmcharts, cleanup

Signed-off-by: Tesshu Flower <[email protected]>

* unit test updates, helmupdates

Signed-off-by: Tesshu Flower <[email protected]>

* - addon status check updates

Signed-off-by: Tesshu Flower <[email protected]>

* update imgs in stable-0.12 helm charts

Signed-off-by: Tesshu Flower <[email protected]>

* exclude volsync embedded helmcharts from sonar

Signed-off-by: Tesshu Flower <[email protected]>

* Simply branching in subhealthcheck

Signed-off-by: Tesshu Flower <[email protected]>

* Upd to latest downstream volsync build v0.12.0-4

Signed-off-by: Tesshu Flower <[email protected]>

* Update rhtap dockerfile

Signed-off-by: Tesshu Flower <[email protected]>

---------

Signed-off-by: Tesshu Flower <[email protected]>
  • Loading branch information
@tesshuflower
tesshuflower authored Dec 18, 2024
1 parent 56431f8 commit 002f01c
Show file tree
Hide file tree
Showing 42 changed files with 13,779 additions and 683 deletions.
4 changes: 4 additions & 0 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,11 @@ RUN microdnf -y --refresh update && \

WORKDIR /
COPY --from=builder /workspace/controller .
# VolSync helm charts
COPY helmcharts/ helmcharts/
# uid/gid: nobody/nobody
USER 65534:65534

ENV EMBEDDED_CHARTS_DIR=/helmcharts

ENTRYPOINT ["/controller"]
4 changes: 4 additions & 0 deletions Dockerfile.rhtap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,11 @@ RUN microdnf -y --refresh update && \

WORKDIR /
COPY --from=builder /workspace/controller .
# VolSync helm charts
COPY helmcharts/ helmcharts/
# uid/gid: nobody/nobody
USER 65534:65534

ENV EMBEDDED_CHARTS_DIR=/helmcharts

ENTRYPOINT ["/controller"]
208 changes: 63 additions & 145 deletions controllers/addoncontroller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,26 +2,28 @@ package controllers

import (
"embed"
"fmt"
"strings"

"github.com/openshift/library-go/pkg/assets"
operatorsv1 "github.com/operator-framework/api/pkg/operators/v1"
operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
appsv1 "k8s.io/api/apps/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/klog/v2"

"open-cluster-management.io/addon-framework/pkg/addonfactory"
"open-cluster-management.io/addon-framework/pkg/agent"
addonframeworkutils "open-cluster-management.io/addon-framework/pkg/utils"
addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
addonv1alpha1client "open-cluster-management.io/api/client/addon/clientset/versioned"
clusterv1 "open-cluster-management.io/api/cluster/v1"
workapiv1 "open-cluster-management.io/api/work/v1"

//helmreleasev1 "open-cluster-management.io/multicloud-operators-subscription/pkg/apis/apps/helmrelease/v1"
//appsubscriptionv1 "open-cluster-management.io/multicloud-operators-subscription/pkg/apis/apps/v1"
policyv1beta1 "open-cluster-management.io/config-policy-controller/api/v1beta1"
)

//
Expand All @@ -42,12 +44,17 @@ const (
operatorName = "volsync-product"
globalOperatorInstallNamespace = "openshift-operators"

// Defaults for ACM-2.12
// Defaults for ACM-2.13 Operator deploy
DefaultCatalogSource = "redhat-operators"
DefaultCatalogSourceNamespace = "openshift-marketplace"
DefaultChannel = "stable-0.11" // No "acm-x.y" channel anymore - aligning ACM-2.12 with stable-0.11
DefaultChannel = "stable-0.12" // aligning ACM-2.13 with stable-0.12
DefaultStartingCSV = "" // By default no starting CSV - will use the latest in the channel
DefaultInstallPlanApproval = "Automatic"

// Defaults for ACM-2.13 helm-based deploy
DefaultHelmChartKey = DefaultChannel // named the same as our operator channel
DefaultHelmChartName = "volsync"
DefaultHelmInstallNamespace = "volsync-system"
)

const (
Expand All @@ -56,7 +63,9 @@ const (
// trigger the deployment of the volsync operator on that managed cluster
ManagedClusterInstallVolSyncLabel = "addons.open-cluster-management.io/volsync"
ManagedClusterInstallVolSyncLabelValue = "true"
)

const (
// Annotations on the ManagedClusterAddOn for overriding operator settings (in the operator Subscription)
AnnotationChannelOverride = "operator-subscription-channel"
AnnotationInstallPlanApprovalOverride = "operator-subscription-installPlanApproval"
Expand All @@ -65,38 +74,40 @@ const (
AnnotationStartingCSVOverride = "operator-subscription-startingCSV"
)

const (
AnnotationVolSyncAddonDeployTypeOverride = "volsync-addon-deploy-type"
//AnnotationVolSyncAddonDeployTypeOverrideHelmValue = "helm"
AnnotationVolSyncAddonDeployTypeOverrideOLMValue = "olm"

AnnotationHelmChartKey = "helm-chart-key"
)

func init() {
utilruntime.Must(scheme.AddToScheme(genericScheme))
utilruntime.Must(operatorsv1.AddToScheme(genericScheme))
utilruntime.Must(operatorsv1alpha1.AddToScheme(genericScheme))
utilruntime.Must(apiextensionsv1.AddToScheme(genericScheme))
utilruntime.Must(policyv1beta1.AddToScheme(genericScheme))
}

//go:embed manifests
var fs embed.FS
var embedFS embed.FS

// If operator is deployed to a single namespace, the Namespace, OperatorGroup (and role to create the operatorgroup)
// is required, along with the Subscription for the operator
// This particular operator is deploying into all namespaces, but into a specific target namespace
// (Requires the annotation operatorframework.io/suggested-namespace: "mynamespace" to be set on the operator CSV)
var manifestFilesAllNamespacesInstallIntoSuggestedNamespace = []string{
"manifests/operatorgroup-aggregate-clusterrole.yaml",
"manifests/operator-namespace.yaml",
"manifests/operator-group-allnamespaces.yaml",
"manifests/operator-subscription.yaml",
// If operator is deployed to a all namespaces and the operator wil be deployed into the global operators namespace
// (openshift-operators on OCP), the only thing needed is the Subscription for the operator
var manifestFilesOperatorDeploy = []string{
"manifests/operator/operator-subscription.yaml",
}

// Use these manifest files if deploying an operator into own namespace
//var manifestFilesOwnNamepace = []string{
// "manifests/operatorgroup-aggregate-clusterrole.yaml",
// "manifests/operator-namespace.yaml",
// "manifests/operator-group-ownnamespace.yaml",
// "manifests/operator-subscription.yaml",
//}
var manifestFilesHelmDeploy = []string{
"manifests/helm-chart/namespace.yaml",
}

// If operator is deployed to a all namespaces and the operator wil be deployed into the global operators namespace
// (openshift-operators on OCP), the only thing needed is the Subscription for the operator
var manifestFilesAllNamespaces = []string{
"manifests/operator-subscription.yaml",
var manifestFilesHelmDeployOpenShift = []string{
// Policy to remove the operator since we're going to deploy as a helm chart instead
"manifests/helm-chart/volsync-operatorpolicy-aggregate-clusterrole.yaml",
"manifests/helm-chart/volsync-operatorpolicy-remove-operator.yaml",
"manifests/helm-chart/namespace.yaml",
}

// Another agent with registration enabled.
Expand All @@ -109,21 +120,8 @@ var _ agent.AgentAddon = &volsyncAgent{}
func (h *volsyncAgent) Manifests(cluster *clusterv1.ManagedCluster,
addon *addonapiv1alpha1.ManagedClusterAddOn,
) ([]runtime.Object, error) {
if !clusterSupportsAddonInstall(cluster) {
klog.InfoS("Cluster is not OpenShift, not deploying addon", "addonName",
addonName, "cluster", cluster.GetName())
return []runtime.Object{}, nil
}

objects := []runtime.Object{}
for _, file := range getManifestFileList(addon) {
object, err := h.loadManifestFromFile(file, cluster, addon)
if err != nil {
return nil, err
}
objects = append(objects, object)
}
return objects, nil
mh := getManifestHelper(embedFS, h.addonClient, cluster, addon)
return mh.loadManifests()
}

func (h *volsyncAgent) GetAgentAddonOptions() agent.AgentAddonOptions {
Expand All @@ -138,7 +136,7 @@ func (h *volsyncAgent) GetAgentAddonOptions() agent.AgentAddonOptions {
Group: "operators.coreos.com",
Resource: "subscriptions",
Name: operatorName,
Namespace: getInstallNamespace(),
Namespace: "openshift-operators",
},
ProbeRules: []workapiv1.FeedbackRule{
{
Expand All @@ -152,8 +150,22 @@ func (h *volsyncAgent) GetAgentAddonOptions() agent.AgentAddonOptions {
},
},
},
{
ResourceIdentifier: workapiv1.ResourceIdentifier{
Group: appsv1.GroupName,
Resource: "deployments",
Name: "volsync",
Namespace: "*",
//Namespace: "volsync-system",
},
ProbeRules: []workapiv1.FeedbackRule{
{
Type: workapiv1.WellKnownStatusType,
},
},
},
},
HealthCheck: subHealthCheck,
HealthChecker: subHealthChecker,
},
},
SupportedConfigGVRs: []schema.GroupVersionResource{
Expand All @@ -162,106 +174,11 @@ func (h *volsyncAgent) GetAgentAddonOptions() agent.AgentAddonOptions {
}
}

func subHealthCheck(identifier workapiv1.ResourceIdentifier, result workapiv1.StatusFeedbackResult) error {
for _, feedbackValue := range result.Values {
if feedbackValue.Name == "installedCSV" {
klog.InfoS("Addon subscription", "installedCSV", feedbackValue.Value)
if feedbackValue.Value.Type != workapiv1.String || feedbackValue.Value.String == nil ||
!strings.HasPrefix(*feedbackValue.Value.String, operatorName) {

installedCSVErr := fmt.Errorf("addon subscription has unexpected installedCSV value")
klog.ErrorS(installedCSVErr, "Sub may not have installed CSV")
return installedCSVErr
}
}
}
klog.InfoS("health check successful")
return nil
}

func (h *volsyncAgent) loadManifestFromFile(file string, cluster *clusterv1.ManagedCluster,
addon *addonapiv1alpha1.ManagedClusterAddOn,
) (runtime.Object, error) {
manifestConfig := struct {
OperatorName string
InstallNamespace string
OperatorGroupSpec string
CatalogSource string
CatalogSourceNamespace string
InstallPlanApproval string
Channel string
StartingCSV string
}{
OperatorName: operatorName,
InstallNamespace: getInstallNamespace(),
CatalogSource: getCatalogSource(addon),
CatalogSourceNamespace: getCatalogSourceNamespace(addon),
InstallPlanApproval: getInstallPlanApproval(addon),
Channel: getChannel(addon),
StartingCSV: getStartingCSV(addon),
}

manifestConfigValues := addonfactory.StructToValues(manifestConfig)

// Get values from addonDeploymentConfig
deploymentConfigValues, err := addonfactory.GetAddOnDeploymentConfigValues(
addonframeworkutils.NewAddOnDeploymentConfigGetter(h.addonClient),
addonfactory.ToAddOnDeploymentConfigValues,
)(cluster, addon)
if err != nil {
return nil, err
}

// Merge manifestConfig and deploymentConfigValues
mergedValues := addonfactory.MergeValues(manifestConfigValues, deploymentConfigValues)

template, err := fs.ReadFile(file)
if err != nil {
return nil, err
}

raw := assets.MustCreateAssetFromTemplate(file, template, &mergedValues).Data
object, _, err := genericCodec.Decode(raw, nil, nil)
if err != nil {
klog.ErrorS(err, "Error decoding manifest file", "filename", file)
return nil, err
}
return object, nil
}

func getManifestFileList(_ *addonapiv1alpha1.ManagedClusterAddOn) []string {
installNamespace := getInstallNamespace()
if installNamespace == globalOperatorInstallNamespace {
// Do not need to create an operator group, namespace etc if installing into the global operator ns
return manifestFilesAllNamespaces
}
return manifestFilesAllNamespacesInstallIntoSuggestedNamespace
}

func getInstallNamespace() string {
// The only namespace supported is openshift-operators, so ignore whatever is in the spec
return globalOperatorInstallNamespace
}

func getCatalogSource(addon *addonapiv1alpha1.ManagedClusterAddOn) string {
return getAnnotationOverrideOrDefault(addon, AnnotationCatalogSourceOverride, DefaultCatalogSource)
}

func getCatalogSourceNamespace(addon *addonapiv1alpha1.ManagedClusterAddOn) string {
return getAnnotationOverrideOrDefault(addon, AnnotationCatalogSourceNamespaceOverride,
DefaultCatalogSourceNamespace)
}

func getInstallPlanApproval(addon *addonapiv1alpha1.ManagedClusterAddOn) string {
return getAnnotationOverrideOrDefault(addon, AnnotationInstallPlanApprovalOverride, DefaultInstallPlanApproval)
}

func getChannel(addon *addonapiv1alpha1.ManagedClusterAddOn) string {
return getAnnotationOverrideOrDefault(addon, AnnotationChannelOverride, DefaultChannel)
}

func getStartingCSV(addon *addonapiv1alpha1.ManagedClusterAddOn) string {
return getAnnotationOverrideOrDefault(addon, AnnotationStartingCSVOverride, DefaultStartingCSV)
func subHealthChecker(fieldResults []agent.FieldResult,
cluster *clusterv1.ManagedCluster, managedClusterAddOn *addonapiv1alpha1.ManagedClusterAddOn) error {
// ManifestHelper will run the health check
mh := getManifestHelper(embedFS, nil /* not needed for heatlh check */, cluster, managedClusterAddOn)
return mh.subHealthCheck(fieldResults)
}

func getAnnotationOverrideOrDefault(addon *addonapiv1alpha1.ManagedClusterAddOn,
Expand All @@ -275,10 +192,11 @@ func getAnnotationOverrideOrDefault(addon *addonapiv1alpha1.ManagedClusterAddOn,
return defaultValue
}

func clusterSupportsAddonInstall(cluster *clusterv1.ManagedCluster) bool {
func isOpenShift(cluster *clusterv1.ManagedCluster) bool {
vendor, ok := cluster.Labels["vendor"]
if !ok || !strings.EqualFold(vendor, "OpenShift") {
return false
}

return true
}
Loading

0 comments on commit 002f01c

Please sign in to comment.