Add os-exporter to telemetry plugin

sunbeam-python/sunbeam/commands/terraform.py

@@ -246,6 +246,41 @@ def destroy(self):
             LOG.warning(e.stderr)
             raise TerraformException(str(e))
 
+    def output(self) -> dict:
+        """terraform output"""
+        os_env = os.environ.copy()
+        timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
+        tf_log = str(self.path / f"terraform-output-{timestamp}.log")
+        os_env.update({"TF_LOG_PATH": tf_log})
+        os_env.setdefault("TF_LOG", "INFO")
+        if self.env:
+            os_env.update(self.env)
+        if self.data_location:
+            os_env.update(self.update_juju_provider_credentials())
+
+        try:
+            cmd = [self.terraform, "output", "-json", "-no-color"]
+            LOG.debug(f'Running command {" ".join(cmd)}')
+            process = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                check=True,
+                cwd=self.path,
+                env=os_env,
+            )
+            stdout = process.stdout
+            LOG.debug(f"Command finished. stdout={stdout}, stderr={process.stderr}")
+            tf_output = json.loads(stdout)
+            output = {}
+            for key, value in tf_output.items():
+                output[key] = value["value"]
+            return output
+        except subprocess.CalledProcessError as e:
+            LOG.error(f"terraform output failed: {e.output}")
+            LOG.warning(e.stderr)
+            raise TerraformException(str(e))
+
 
 class TerraformInitStep(BaseStep):
     """Initialize Terraform with required providers."""

sunbeam-python/sunbeam/plugins/interface/v1/openstack.py

@@ -58,6 +58,8 @@
 console = Console()
 
 APPLICATION_DEPLOY_TIMEOUT = 900  # 15 minutes
+OPENSTACK_TERRAFORM_VARS = "TerraformVarsOpenstack"
+OPENSTACK_TERRAFORM_PLAN = "openstack"
 
 
 class TerraformPlanLocation(Enum):
@@ -96,7 +98,7 @@ def __init__(self, name: str, tf_plan_location: TerraformPlanLocation) -> None:
         # Based on terraform plan location, tfplan will be either
         # openstack or plugin name
         if self.tf_plan_location == TerraformPlanLocation.SUNBEAM_TERRAFORM_REPO:
-            self.tfplan = "openstack"
+            self.tfplan = OPENSTACK_TERRAFORM_PLAN
         else:
             self.tfplan = self.name
 
@@ -194,7 +196,7 @@ def get_tfvar_config_key(self) -> str:
         TerraformVars-<plugin name>.
         """
         if self.tf_plan_location == TerraformPlanLocation.SUNBEAM_TERRAFORM_REPO:
-            return "TerraformVarsOpenstack"
+            return OPENSTACK_TERRAFORM_VARS
         else:
             return f"TerraformVars{self.app_name}"
 

sunbeam-python/sunbeam/plugins/observability/plugin.py

@@ -29,6 +29,7 @@
 from rich.status import Status
 from snaphelpers import Snap
 
+from sunbeam.clusterd.client import Client
 from sunbeam.clusterd.service import (
     ClusterServiceUnavailableException,
     ConfigItemNotFoundException,
@@ -39,7 +40,7 @@
     MICROK8S_CLOUD,
     MICROK8S_DEFAULT_STORAGECLASS,
 )
-from sunbeam.commands.openstack import PatchLoadBalancerServicesStep
+from sunbeam.commands.openstack import OPENSTACK_MODEL, PatchLoadBalancerServicesStep
 from sunbeam.commands.terraform import (
     TerraformException,
     TerraformHelper,
@@ -49,7 +50,9 @@
     BaseStep,
     Result,
     ResultType,
+    read_config,
     run_plan,
+    update_config,
     update_status_background,
 )
 from sunbeam.jobs.juju import (
@@ -59,7 +62,11 @@
     TimeoutException,
     run_sync,
 )
-from sunbeam.plugins.interface.v1.base import EnableDisablePlugin
+from sunbeam.plugins.interface.v1.base import EnableDisablePlugin, PluginRequirement
+from sunbeam.plugins.interface.v1.openstack import (
+    OPENSTACK_TERRAFORM_PLAN,
+    OPENSTACK_TERRAFORM_VARS,
+)
 
 LOG = logging.getLogger(__name__)
 console = Console()
@@ -69,6 +76,89 @@
 CONTROLLER_MODEL = CONTROLLER_MODEL.split("/")[-1]
 
 
+class FillObservabilityOffersStep(BaseStep):
+    """Update terraform plan to fill observability offers."""
+
+    def __init__(
+        self,
+        tfhelper: TerraformHelper,
+        tfhelper_cos: TerraformHelper,
+        jhelper: JujuHelper,
+    ) -> None:
+        super().__init__(
+            "Fill Observability Offers",
+            "Fill Observability Offers in Openstack",
+        )
+        self.tfhelper = tfhelper
+        self.tfhelper_cos = tfhelper_cos
+        self.jhelper = jhelper
+        self.model = OPENSTACK_MODEL
+        self.client = Client()
+
+    def run(self, status: Optional[Status] = None) -> Result:
+        """Apply terraform configuration"""
+        config_key = OPENSTACK_TERRAFORM_VARS
+
+        try:
+            tfvars = read_config(self.client, config_key)
+        except ConfigItemNotFoundException:
+            tfvars = {}
+        output_vars = self.tfhelper_cos.output()
+
+        for key, value in output_vars.items():
+            if key in (
+                "prometheus-metrics-offer-url",
+                "grafana-dashboard-offer-url",
+            ):
+                tfvars[key] = value
+        update_config(self.client, config_key, tfvars)
+        self.tfhelper.write_tfvars(tfvars)
+
+        try:
+            self.tfhelper.apply()
+        except TerraformException as e:
+            return Result(ResultType.FAILED, str(e))
+
+        return Result(ResultType.COMPLETED)
+
+
+class RemoveObservabilityIntegrationStep(BaseStep):
+    def __init__(
+        self,
+        tfhelper: TerraformHelper,
+        jhelper: JujuHelper,
+    ) -> None:
+        super().__init__(
+            "Remove Observability Integration",
+            "Remove Observability Integration in Openstack",
+        )
+        self.tfhelper = tfhelper
+        self.jhelper = jhelper
+        self.model = OPENSTACK_MODEL
+        self.client = Client()
+
+    def run(self, status: Optional[Status] = None) -> Result:
+        """Apply terraform configuration"""
+        config_key = OPENSTACK_TERRAFORM_VARS
+
+        try:
+            tfvars = read_config(self.client, config_key)
+        except ConfigItemNotFoundException:
+            tfvars = {}
+
+        tfvars.pop("prometheus-metrics-offer-url", None)
+        tfvars.pop("grafana-dashboard-offer-url", None)
+        update_config(self.client, config_key, tfvars)
+        self.tfhelper.write_tfvars(tfvars)
+
+        try:
+            self.tfhelper.apply()
+        except TerraformException as e:
+            return Result(ResultType.FAILED, str(e))
+
+        return Result(ResultType.COMPLETED)
+
+
 class DeployObservabilityStackStep(BaseStep, JujuStepHelper):
     """Deploy Observability Stack using Terraform"""
 
@@ -338,6 +428,7 @@ class PatchCosLoadBalancerStep(PatchLoadBalancerServicesStep):
 
 class ObservabilityPlugin(EnableDisablePlugin):
     version = Version("0.0.1")
+    requires = {PluginRequirement("telemetry", optional=True)}
 
     def __init__(self) -> None:
         super().__init__(name="observability")
@@ -370,13 +461,21 @@ def run_enable_plans(self):
             backend="http",
             data_location=data_location,
         )
+        openstack_plan = "deploy-" + OPENSTACK_TERRAFORM_PLAN
+        tfhelper_openstack = TerraformHelper(
+            path=self.snap.paths.user_common / "etc" / openstack_plan,
+            plan=OPENSTACK_TERRAFORM_PLAN + "-plan",
+            backend="http",
+            data_location=data_location,
+        )
 
         jhelper = JujuHelper(data_location)
 
         cos_plan = [
             TerraformInitStep(tfhelper_cos),
             DeployObservabilityStackStep(self, tfhelper_cos, jhelper),
             PatchCosLoadBalancerStep(),
+            FillObservabilityOffersStep(tfhelper_openstack, tfhelper_cos, jhelper),
         ]
 
         grafana_agent_plan = [
@@ -406,11 +505,19 @@ def run_disable_plans(self):
             backend="http",
             data_location=data_location,
         )
+        openstack_plan = "deploy-" + OPENSTACK_TERRAFORM_PLAN
+        tfhelper_openstack = TerraformHelper(
+            path=self.snap.paths.user_common / "etc" / openstack_plan,
+            plan=OPENSTACK_TERRAFORM_PLAN + "-plan",
+            backend="http",
+            data_location=data_location,
+        )
 
         jhelper = JujuHelper(data_location)
 
         cos_plan = [
             TerraformInitStep(tfhelper_cos),
+            RemoveObservabilityIntegrationStep(tfhelper_openstack, jhelper),
             RemoveObservabilityStackStep(self, tfhelper_cos, jhelper),
         ]
 

sunbeam-python/sunbeam/plugins/telemetry/plugin.py

@@ -22,7 +22,7 @@
 from sunbeam.commands.hypervisor import ReapplyHypervisorTerraformPlanStep
 from sunbeam.commands.terraform import TerraformHelper, TerraformInitStep
 from sunbeam.jobs.common import run_plan
-from sunbeam.jobs.juju import JujuHelper
+from sunbeam.jobs.juju import JujuHelper, ModelNotFoundException, run_sync
 from sunbeam.plugins.interface.v1.openstack import (
     DisableOpenStackApplicationStep,
     EnableOpenStackApplicationStep,
@@ -94,11 +94,28 @@ def run_disable_plans(self) -> None:
         run_plan(plan, console)
         click.echo(f"OpenStack {self.name} application disabled.")
 
+    def _get_observability_offer_endpoints(self) -> dict:
+        """Fetch observability offers."""
+        data_location = self.snap.paths.user_data
+        jhelper = JujuHelper(data_location)
+        try:
+            model = run_sync(jhelper.get_model("observability"))
+        except ModelNotFoundException:
+            return {}
+        offer_query = run_sync(model.list_offers())
+        offer_vars = {}
+        for offer in offer_query["results"]:
+            if offer.offer_name == "grafana-dashboards":
+                offer_vars["grafana-dashboard-offer-url"] = offer.offer_url
+            if offer.offer_name == "prometheus-metrics-endpoint":
+                offer_vars["prometheus-metrics-offer-url"] = offer.offer_url
+        return offer_vars
+
     def set_application_names(self) -> list:
         """Application names handled by the terraform plan."""
         database_topology = self.get_database_topology()
 
-        apps = ["aodh", "aodh-mysql-router"]
+        apps = ["aodh", "aodh-mysql-router", "openstack-exporter"]
         if database_topology == "multi":
             apps.append("aodh-mysql")
 
@@ -114,6 +131,7 @@ def set_tfvars_on_enable(self) -> dict:
         return {
             "telemetry-channel": "2023.2/edge",
             "enable-telemetry": True,
+            **self._get_observability_offer_endpoints(),
         }
 
     def set_tfvars_on_disable(self) -> dict: