Merge pull request #96 from chanchiwai-ray/add-grafana-agent-k8s

Update observability plugin to deploy grafana-agent-k8s as well.

sunbeam-python/sunbeam/plugins/observability/etc/deploy-grafana-agent-k8s/main.tf

@@ -0,0 +1,89 @@
+# Terraform manifest for deployment of Grafana Agent
+#
+# Copyright (c) 2023 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+terraform {
+  required_providers {
+    juju = {
+      source  = "juju/juju"
+      version = "= 0.10.1"
+    }
+  }
+}
+
+data "terraform_remote_state" "cos" {
+  backend = var.cos-state-backend
+  config  = var.cos-state-config
+}
+
+resource "juju_application" "grafana-agent-k8s" {
+  name  = "grafana-agent-k8s"
+  model = var.model
+
+  # note that we need to make sure the "base" matches the environment we are
+  # deploying.
+  charm {
+    name     = "grafana-agent-k8s"
+    base     = "[email protected]"
+    channel  = var.grafana-agent-k8s-channel
+    revision = var.grafana-agent-k8s-revision
+  }
+
+  units  = 1
+  config = var.grafana-agent-k8s-config
+}
+
+# juju integrate grafana-agent-k8s:send-remote-write cos.prometheus-receive-remote-write
+resource "juju_integration" "grafana-agent-k8s-to-cos-prometheus" {
+  model = var.model
+
+  application {
+    name     = juju_application.grafana-agent-k8s.name
+    endpoint = "send-remote-write"
+  }
+
+  application {
+    offer_url = data.terraform_remote_state.cos.outputs.prometheus-receive-remote-write-offer-url
+  }
+}
+
+# juju integrate grafana-agent-k8s:logging-consumer cos.loki-logging
+resource "juju_integration" "grafana-agent-k8s-to-cos-loki" {
+  model = var.model
+
+  application {
+    name     = juju_application.grafana-agent-k8s.name
+    endpoint = "logging-consumer"
+  }
+
+  application {
+    offer_url = data.terraform_remote_state.cos.outputs.loki-logging-offer-url
+  }
+}
+
+# juju integrate grafana-agent-k8s:grafana_dashboard cos.grafana-dashboards
+resource "juju_integration" "grafana-agent-k8s-to-cos-grafana" {
+  model = var.model
+
+  application {
+    name     = juju_application.grafana-agent-k8s.name
+    endpoint = "grafana-dashboards-provider"
+  }
+
+  application {
+    offer_url = data.terraform_remote_state.cos.outputs.grafana-dashboard-offer-url
+  }
+}

sunbeam-python/sunbeam/plugins/observability/etc/deploy-grafana-agent-k8s/variables.tf

@@ -0,0 +1,50 @@
+# Terraform manifest for deployment of Grafana Agent
+#
+# Copyright (c) 2023 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+variable "model" {
+  type        = string
+  default     = "openstack"
+  description = "Name of the model where the application is deployed"
+}
+
+variable "cos-state-backend" {
+  description = "Backend type used for cos state"
+  type        = string
+  default     = "http"
+}
+
+variable "cos-state-config" {
+  type = map(any)
+}
+
+variable "grafana-agent-k8s-channel" {
+  type        = string
+  default     = "latest/stable"
+  description = "Operator channel for grafana-agent-k8s deployment"
+}
+
+variable "grafana-agent-k8s-revision" {
+  type        = number
+  default     = null
+  description = "Operator channel revision for grafana-agent-k8s deployment"
+}
+
+variable "grafana-agent-k8s-config" {
+  type        = map(string)
+  default     = {}
+  description = "Operator config for grafana-agent-k8s deployment"
+}

sunbeam-python/sunbeam/plugins/observability/plugin.py

@@ -74,6 +74,7 @@
 OBSERVABILITY_MODEL = "observability"
 OBSERVABILITY_DEPLOY_TIMEOUT = 1200  # 20 minutes
 CONTROLLER_MODEL = CONTROLLER_MODEL.split("/")[-1]
+GRAFANA_AGENT_K8S_CHANNEL = "latest/stable"
 
 
 class FillObservabilityOffersStep(BaseStep):
@@ -240,7 +241,7 @@ def __init__(
         tfhelper_cos: TerraformHelper,
         jhelper: JujuHelper,
     ):
-        super().__init__("Deploy Grafana Agent", "Deploy Grafana Agent")
+        super().__init__("Deploy Grafana Agent", "Deploying Grafana Agent")
         self.tfhelper = tfhelper
         self.tfhelper_cos = tfhelper_cos
         self.jhelper = jhelper
@@ -298,6 +299,76 @@ def run(self, status: Optional[Status] = None) -> Result:
         return Result(ResultType.COMPLETED)
 
 
+class DeployGrafanaAgentK8sStep(BaseStep, JujuStepHelper):
+    """Deploy Grafana Agent k8s using Terraform"""
+
+    def __init__(
+        self,
+        plugin: "ObservabilityPlugin",
+        tfhelper: TerraformHelper,
+        tfhelper_cos: TerraformHelper,
+        jhelper: JujuHelper,
+    ):
+        super().__init__("Deploy Grafana Agent k8s", "Deploying Grafana Agent k8s")
+        self.tfhelper = tfhelper
+        self.tfhelper_cos = tfhelper_cos
+        self.jhelper = jhelper
+        self.model = OPENSTACK_MODEL
+        self.read_config = lambda: plugin.get_plugin_info().get(
+            "grafana-agent-k8s-config", {}
+        )
+        self.update_config = lambda c: plugin.update_plugin_info(
+            {"grafana-agent-k8s-config": c}
+        )
+
+    def run(self, status: Optional[Status] = None) -> Result:
+        """Execute configuration using terraform."""
+        app = "grafana-agent-k8s"
+        cos_backend = self.tfhelper_cos.backend
+        cos_backend_config = self.tfhelper_cos.backend_config()
+        try:
+            config = self.read_config()
+        except ConfigItemNotFoundException as e:
+            LOG.exception("Failed deploying %s: unable to read config", app)
+            return Result(ResultType.FAILED, str(e))
+
+        tfvars = {
+            "cos-state-backend": cos_backend,
+            "cos-state-config": cos_backend_config,
+            "grafana-agent-k8s-channel": GRAFANA_AGENT_K8S_CHANNEL,
+            "model": self.model,
+        }
+        config.update(tfvars)
+        self.update_config(config)
+        self.tfhelper.write_tfvars(tfvars)
+
+        self.update_status(status, "deploying application")
+        try:
+            self.tfhelper.apply()
+        except TerraformException as e:
+            LOG.exception("Error deploying %s", app)
+            return Result(ResultType.FAILED, str(e))
+
+        LOG.debug("Application monitored for readiness: %s", app)
+        try:
+            # Note that grafana agent k8s will be blocked first if there's not
+            # workload "requires" relations. We will add them later in the
+            # steps.
+            run_sync(
+                self.jhelper.wait_application_ready(
+                    app,
+                    self.model,
+                    accepted_status=["active", "blocked"],
+                    timeout=OBSERVABILITY_DEPLOY_TIMEOUT,
+                )
+            )
+        except (JujuWaitException, TimeoutException) as e:
+            LOG.debug("Failed to deploys %s", app, exc_info=True)
+            return Result(ResultType.FAILED, str(e))
+
+        return Result(ResultType.COMPLETED)
+
+
 class RemoveObservabilityStackStep(BaseStep, JujuStepHelper):
     """Remove Observability Stack using Terraform"""
 
@@ -423,6 +494,43 @@ def run(self, status: Optional[Status] = None) -> Result:
         return Result(ResultType.COMPLETED)
 
 
+class RemoveGrafanaAgentK8sStep(BaseStep, JujuStepHelper):
+    """Remove Grafana Agent k8s using Terraform"""
+
+    def __init__(
+        self,
+        jhelper: JujuHelper,
+        tfhelper: TerraformHelper,
+    ):
+        super().__init__("Remove Grafana Agent k8s", "Removing Grafana Agent k8s")
+        self.jhelper = jhelper
+        self.tfhelper = tfhelper
+        self.model = OPENSTACK_MODEL
+
+    def run(self, status: Optional[Status] = None) -> Result:
+        """Execute configuration using terraform."""
+        app = "grafana-agent-k8s"
+        try:
+            self.tfhelper.destroy()
+        except TerraformException as e:
+            LOG.exception("Error destroying %s", app)
+            return Result(ResultType.FAILED, str(e))
+
+        try:
+            run_sync(
+                self.jhelper.wait_application_gone(
+                    [app],
+                    self.model,
+                    timeout=OBSERVABILITY_DEPLOY_TIMEOUT,
+                )
+            )
+        except TimeoutException as e:
+            LOG.debug("Failed to destroy %s", app, exc_info=True)
+            return Result(ResultType.FAILED, str(e))
+
+        return Result(ResultType.COMPLETED)
+
+
 class PatchCosLoadBalancerStep(PatchLoadBalancerServicesStep):
     SERVICES = ["traefik"]
     MODEL = OBSERVABILITY_MODEL
@@ -437,6 +545,7 @@ def __init__(self, client: Client) -> None:
         self.snap = Snap()
         self.tfplan_cos = "deploy-cos"
         self.tfplan_grafana_agent = "deploy-grafana-agent"
+        self.tfplan_grafana_agent_k8s = "deploy-grafana-agent-k8s"
 
     def pre_enable(self):
         src = Path(__file__).parent / "etc" / self.tfplan_cos
@@ -449,6 +558,11 @@ def pre_enable(self):
         LOG.debug(f"Updating {dst} from {src}...")
         shutil.copytree(src, dst, dirs_exist_ok=True)
 
+        src = Path(__file__).parent / "etc" / self.tfplan_grafana_agent_k8s
+        dst = self.snap.paths.user_common / "etc" / self.tfplan_grafana_agent_k8s
+        LOG.debug(f"Updating {dst} from {src}...")
+        shutil.copytree(src, dst, dirs_exist_ok=True)
+
     def run_enable_plans(self):
         data_location = self.snap.paths.user_data
         tfhelper_cos = TerraformHelper(
@@ -463,6 +577,12 @@ def run_enable_plans(self):
             backend="http",
             data_location=data_location,
         )
+        tfhelper_grafana_agent_k8s = TerraformHelper(
+            path=self.snap.paths.user_common / "etc" / self.tfplan_grafana_agent_k8s,
+            plan="grafana-agent-k8s-plan",
+            backend="http",
+            data_location=data_location,
+        )
         openstack_plan = "deploy-" + OPENSTACK_TERRAFORM_PLAN
         tfhelper_openstack = TerraformHelper(
             path=self.snap.paths.user_common / "etc" / openstack_plan,
@@ -487,8 +607,16 @@ def run_enable_plans(self):
             DeployGrafanaAgentStep(self, tfhelper_grafana_agent, tfhelper_cos, jhelper),
         ]
 
+        grafana_agent_k8s_plan = [
+            TerraformInitStep(tfhelper_grafana_agent_k8s),
+            DeployGrafanaAgentK8sStep(
+                self, tfhelper_grafana_agent_k8s, tfhelper_cos, jhelper
+            ),
+        ]
+
         run_plan(cos_plan, console)
         run_plan(grafana_agent_plan, console)
+        run_plan(grafana_agent_k8s_plan, console)
 
         click.echo("Observability enabled.")
 
@@ -509,6 +637,12 @@ def run_disable_plans(self):
             backend="http",
             data_location=data_location,
         )
+        tfhelper_grafana_agent_k8s = TerraformHelper(
+            path=self.snap.paths.user_common / "etc" / self.tfplan_grafana_agent_k8s,
+            plan="grafana-agent-k8s-plan",
+            backend="http",
+            data_location=data_location,
+        )
         openstack_plan = "deploy-" + OPENSTACK_TERRAFORM_PLAN
         tfhelper_openstack = TerraformHelper(
             path=self.snap.paths.user_common / "etc" / openstack_plan,
@@ -532,6 +666,12 @@ def run_disable_plans(self):
             RemoveGrafanaAgentStep(self, tfhelper_grafana_agent, tfhelper_cos, jhelper),
         ]
 
+        grafana_agent_k8s_plan = [
+            TerraformInitStep(tfhelper_grafana_agent_k8s),
+            RemoveGrafanaAgentK8sStep(jhelper, tfhelper_grafana_agent_k8s),
+        ]
+
+        run_plan(grafana_agent_k8s_plan, console)
         run_plan(grafana_agent_plan, console)
         run_plan(cos_plan, console)
         click.echo("Observability disabled.")