diff --git a/lib/charms/mysql/v0/mysql.py b/lib/charms/mysql/v0/mysql.py index ebbd23640..f79bd17e4 100644 --- a/lib/charms/mysql/v0/mysql.py +++ b/lib/charms/mysql/v0/mysql.py @@ -74,7 +74,7 @@ def wait_until_mysql_connection(self) -> None: from typing import Any, Dict, Iterable, List, Optional, Tuple import ops -from ops.charm import ActionEvent, CharmBase +from ops.charm import ActionEvent, CharmBase, RelationBrokenEvent from tenacity import ( retry, retry_if_exception_type, @@ -88,6 +88,7 @@ def wait_until_mysql_connection(self) -> None: BACKUPS_USERNAME, CLUSTER_ADMIN_PASSWORD_KEY, CLUSTER_ADMIN_USERNAME, + COS_AGENT_RELATION_NAME, MONITORING_PASSWORD_KEY, MONITORING_USERNAME, PASSWORD_LENGTH, @@ -110,7 +111,7 @@ def wait_until_mysql_connection(self) -> None: # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 38 +LIBPATCH = 41 UNIT_TEARDOWN_LOCKNAME = "unit-teardown" UNIT_ADD_LOCKNAME = "unit-add" @@ -123,6 +124,15 @@ def wait_until_mysql_connection(self) -> None: class Error(Exception): """Base class for exceptions in this module.""" + def __init__(self, message: str = "") -> None: + """Initialize the Error class. + + Args: + message: Optional message to pass to the exception. + """ + super().__init__(message) + self.message = message + def __repr__(self): """String representation of the Error class.""" return "<{}.{} {}>".format(type(self).__module__, type(self).__name__, self.args) @@ -132,11 +142,6 @@ def name(self): """Return a string representation of the model plus class.""" return "<{}.{}>".format(type(self).__module__, type(self).__name__) - @property - def message(self): - """Return the message passed as an argument.""" - return self.args[0] - class MySQLConfigureMySQLUsersError(Error): """Exception raised when creating a user fails.""" @@ -227,6 +232,10 @@ class MySQLGetClusterPrimaryAddressError(Error): """Exception raised when there is an issue getting the primary instance.""" +class MySQLSetClusterPrimaryError(Error): + """Exception raised when there is an issue setting the primary instance.""" + + class MySQLGrantPrivilegesToUserError(Error): """Exception raised when there is an issue granting privileges to user.""" @@ -326,6 +335,14 @@ class MySQLRescanClusterError(Error): """Exception raised when there is an issue rescanning the cluster.""" +class MySQLSetVariableError(Error): + """Exception raised when there is an issue setting a variable.""" + + +class MySQLServerNotUpgradableError(Error): + """Exception raised when there is an issue checking for upgradeability.""" + + class MySQLSecretError(Error): """Exception raised when there is an issue setting/getting a secret.""" @@ -354,6 +371,10 @@ def __init__(self, *args): self.framework.observe(self.on.get_password_action, self._on_get_password) self.framework.observe(self.on.set_password_action, self._on_set_password) + # Set in some event handlers in order to avoid passing event down a chain + # of methods + self.current_event = None + def _on_get_password(self, event: ActionEvent) -> None: """Action used to retrieve the system user's password.""" username = event.params.get("username") or ROOT_USERNAME @@ -404,6 +425,9 @@ def _on_set_password(self, event: ActionEvent) -> None: self.set_secret("app", secret_key, new_password) + if username == MONITORING_USERNAME and self.has_cos_relation: + self._mysql.restart_mysql_exporter() + def _get_cluster_status(self, event: ActionEvent) -> None: """Action used to retrieve the cluster status.""" if status := self._mysql.get_cluster_status(): @@ -452,6 +476,11 @@ def unit_peer_data(self) -> Dict: return self.peers.data[self.unit] + @property + def unit_label(self): + """Return unit label.""" + return self.unit.name.replace("/", "-") + @property def _is_peer_data_set(self): return bool( @@ -463,6 +492,22 @@ def _is_peer_data_set(self): and self.get_secret("app", BACKUPS_PASSWORD_KEY) ) + @property + def has_cos_relation(self) -> bool: + """Returns a bool indicating whether a relation with COS is present.""" + cos_relations = self.model.relations.get(COS_AGENT_RELATION_NAME, []) + active_cos_relations = list( + filter( + lambda relation: not ( + isinstance(self.current_event, RelationBrokenEvent) + and self.current_event.relation.id == relation.id + ), + cos_relations, + ) + ) + + return len(active_cos_relations) > 0 + def _get_secret_from_juju(self, scope: str, key: str) -> Optional[str]: """Retrieve and return the secret from the juju secret storage.""" if scope == "unit": @@ -975,6 +1020,38 @@ def remove_router_from_cluster_metadata(self, router_id: str) -> None: logger.exception(f"Failed to remove router from metadata with ID {router_id}") raise MySQLRemoveRouterFromMetadataError(e.message) + def set_dynamic_variable( + self, + variable: str, + value: str, + persist: bool = False, + instance_address: Optional[str] = None, + ) -> None: + """Set a dynamic variable value for the instance. + + Args: + variable: The name of the variable to set + value: The value to set the variable to + persist: Whether to persist the variable value across restarts + instance_address: instance address to set the variable, default to current + + Raises: + MySQLSetVariableError + """ + if not instance_address: + instance_address = self.instance_address + logger.debug(f"Setting {variable} to {value} on {instance_address}") + set_var_command = [ + f"shell.connect('{self.server_config_user}:{self.server_config_password}@{instance_address}')", + f"session.run_sql(\"SET {'PERSIST' if persist else 'GLOBAL'} {variable}={value}\")", + ] + + try: + self._run_mysqlsh_script("\n".join(set_var_command)) + except MySQLClientError: + logger.exception(f"Failed to set variable {variable} to {value}") + raise MySQLSetVariableError + def configure_instance(self, create_cluster_admin: bool = True) -> None: """Configure the instance to be used in an InnoDB cluster. @@ -1276,7 +1353,7 @@ def is_instance_in_cluster(self, unit_label: str) -> bool: ) return False - def get_cluster_status(self) -> Optional[dict]: + def get_cluster_status(self, extended: Optional[bool] = False) -> Optional[dict]: """Get the cluster status. Executes script to retrieve cluster status. @@ -1286,10 +1363,11 @@ def get_cluster_status(self) -> Optional[dict]: Cluster status as a dictionary, or None if running the status script fails. """ + options = {"extended": extended} status_commands = ( f"shell.connect('{self.cluster_admin_user}:{self.cluster_admin_password}@{self.instance_address}')", f"cluster = dba.get_cluster('{self.cluster_name}')", - "print(cluster.status())", + f"print(cluster.status({options}))", ) try: @@ -1422,7 +1500,9 @@ def remove_instance(self, unit_label: str) -> None: f"shell.connect('{self.cluster_admin_user}:{self.cluster_admin_password}@{self.instance_address}')", f"cluster = dba.get_cluster('{self.cluster_name}')", "number_cluster_members = len(cluster.status()['defaultReplicaSet']['topology'])", - f"cluster.remove_instance('{self.cluster_admin_user}@{self.instance_address}', {json.dumps(remove_instance_options)}) if number_cluster_members > 1 else cluster.dissolve({json.dumps(dissolve_cluster_options)})", + f"cluster.remove_instance('{self.cluster_admin_user}@{self.instance_address}', " + f"{json.dumps(remove_instance_options)}) if number_cluster_members > 1 else" + f" cluster.dissolve({json.dumps(dissolve_cluster_options)})", ) self._run_mysqlsh_script("\n".join(remove_instance_commands)) except MySQLClientError as e: @@ -1579,6 +1659,37 @@ def get_cluster_primary_address( return matches.group(1) + def get_primary_label(self) -> Optional[str]: + """Get the label of the cluster's primary.""" + status = self.get_cluster_status() + if not status: + return None + for label, value in status["defaultreplicaset"]["topology"].items(): + if value["memberrole"] == "primary": + return label + + def set_cluster_primary(self, new_primary_address: str) -> None: + """Set the cluster primary. + + Args: + new_primary_address: Address of node to set as cluster's primary + + Raises: + MySQLSetClusterPrimaryError: If the cluster primary could not be set + """ + logger.debug(f"Setting cluster primary to {new_primary_address}") + + set_cluster_primary_commands = ( + f"shell.connect_to_primary('{self.server_config_user}:{self.server_config_password}@{self.instance_address}')", + f"cluster = dba.get_cluster('{self.cluster_name}')", + f"cluster.set_primary_instance('{new_primary_address}')", + ) + try: + self._run_mysqlsh_script("\n".join(set_cluster_primary_commands)) + except MySQLClientError as e: + logger.exception("Failed to set cluster primary") + raise MySQLSetClusterPrimaryError(e.message) + def get_cluster_members_addresses(self) -> Optional[Iterable[str]]: """Get the addresses of the cluster's members. @@ -1605,6 +1716,32 @@ def get_cluster_members_addresses(self) -> Optional[Iterable[str]]: return set(matches.group(1).split(",")) + def verify_server_upgradable(self, instance: Optional[str] = None) -> None: + """Wrapper for API check_for_server_upgrade. + + Raises: + MySQLServerUpgradableError: If the server is not upgradable + """ + check_command = [ + f"shell.connect_to_primary('{self.server_config_user}" + f":{self.server_config_password}@{instance or self.instance_address}')", + "try:", + " util.check_for_server_upgrade(options={'outputFormat': 'JSON'})", + "except ValueError:", # ValueError is raised for same version check + " print('SAME_VERSION')", + ] + + try: + output = self._run_mysqlsh_script("\n".join(check_command)) + if "SAME_VERSION" in output: + return + result = json.loads(output) + if result["errorCount"] == 0: + return + raise MySQLServerNotUpgradableError(result.get("summary")) + except MySQLClientError: + raise MySQLServerNotUpgradableError("Failed to check for server upgrade") + def get_mysql_version(self) -> Optional[str]: """Get the MySQL version. @@ -2295,6 +2432,11 @@ def start_mysqld(self) -> None: """Starts the mysqld process.""" raise NotImplementedError + @abstractmethod + def restart_mysql_exporter(self) -> None: + """Restart the mysqld exporter.""" + raise NotImplementedError + @abstractmethod def wait_until_mysql_connection(self) -> None: """Wait until a connection to MySQL has been obtained. diff --git a/src/charm.py b/src/charm.py index 2f04e9476..520795123 100755 --- a/src/charm.py +++ b/src/charm.py @@ -29,6 +29,7 @@ ) from charms.mysql.v0.tls import MySQLTLS from charms.prometheus_k8s.v0.prometheus_scrape import MetricsEndpointProvider +from ops import RelationBrokenEvent, RelationCreatedEvent from ops.charm import LeaderElectedEvent, RelationChangedEvent, UpdateStatusEvent from ops.main import main from ops.model import ( @@ -46,6 +47,7 @@ CLUSTER_ADMIN_PASSWORD_KEY, CLUSTER_ADMIN_USERNAME, CONTAINER_NAME, + COS_AGENT_RELATION_NAME, GR_MAX_MEMBERS, MONITORING_PASSWORD_KEY, MONITORING_USERNAME, @@ -96,6 +98,13 @@ def __init__(self, *args): self.framework.observe(self.on[PEER].relation_joined, self._on_peer_relation_joined) self.framework.observe(self.on[PEER].relation_changed, self._on_peer_relation_changed) + self.framework.observe( + self.on[COS_AGENT_RELATION_NAME].relation_created, self._reconcile_mysqld_exporter + ) + self.framework.observe( + self.on[COS_AGENT_RELATION_NAME].relation_broken, self._reconcile_mysqld_exporter + ) + self.k8s_helpers = KubernetesHelpers(self) self.mysql_relation = MySQLRelation(self) self.database_relation = MySQLProvider(self) @@ -134,43 +143,43 @@ def _mysql(self) -> MySQL: self.get_secret("app", BACKUPS_PASSWORD_KEY), self.unit.get_container(CONTAINER_NAME), self.k8s_helpers, + self, ) @property def _pebble_layer(self) -> Layer: """Return a layer for the mysqld pebble service.""" - return Layer( - { - "summary": "mysqld services layer", - "description": "pebble config layer for mysqld safe and exporter", - "services": { - MYSQLD_SAFE_SERVICE: { - "override": "replace", - "summary": "mysqld safe", - "command": MYSQLD_SAFE_SERVICE, - "startup": "enabled", - "user": MYSQL_SYSTEM_USER, - "group": MYSQL_SYSTEM_GROUP, - "kill-delay": "24h", - }, - MYSQLD_EXPORTER_SERVICE: { - "override": "replace", - "summary": "mysqld exporter", - "command": "/start-mysqld-exporter.sh", - "startup": "enabled", - "user": MYSQL_SYSTEM_USER, - "group": MYSQL_SYSTEM_GROUP, - "environment": { - "DATA_SOURCE_NAME": ( - f"{MONITORING_USERNAME}:" - f"{self.get_secret('app', MONITORING_PASSWORD_KEY)}" - f"@unix({MYSQLD_SOCK_FILE})/" - ), - }, + layer = { + "summary": "mysqld services layer", + "description": "pebble config layer for mysqld safe and exporter", + "services": { + MYSQLD_SAFE_SERVICE: { + "override": "replace", + "summary": "mysqld safe", + "command": MYSQLD_SAFE_SERVICE, + "startup": "enabled", + "user": MYSQL_SYSTEM_USER, + "group": MYSQL_SYSTEM_GROUP, + "kill-delay": "24h", + }, + MYSQLD_EXPORTER_SERVICE: { + "override": "replace", + "summary": "mysqld exporter", + "command": "/start-mysqld-exporter.sh", + "startup": "enabled" if self.has_cos_relation else "disabled", + "user": MYSQL_SYSTEM_USER, + "group": MYSQL_SYSTEM_GROUP, + "environment": { + "DATA_SOURCE_NAME": ( + f"{MONITORING_USERNAME}:" + f"{self.get_secret('app', MONITORING_PASSWORD_KEY)}" + f"@unix({MYSQLD_SOCK_FILE})/" + ), }, }, - } - ) + }, + } + return Layer(layer) @property def active_status_message(self) -> str: @@ -345,12 +354,30 @@ def _reconcile_pebble_layer(self, container: Container) -> None: container.add_layer(MYSQLD_SAFE_SERVICE, new_layer, combine=True) container.replan() self._mysql.wait_until_mysql_connection() + + if ( + not self.has_cos_relation + and container.get_services(MYSQLD_EXPORTER_SERVICE)[ + MYSQLD_EXPORTER_SERVICE + ].is_running() + ): + container.stop(MYSQLD_EXPORTER_SERVICE) + self._on_update_status(None) # ========================================================================= # Charm event handlers # ========================================================================= + def _reconcile_mysqld_exporter( + self, event: RelationCreatedEvent | RelationBrokenEvent + ) -> None: + """Handle a COS relation created or broken event.""" + self.current_event = event + + container = self.unit.get_container(CONTAINER_NAME) + self._reconcile_pebble_layer(container) + def _on_peer_relation_joined(self, _) -> None: """Handle the peer relation joined event.""" # set some initial unit data @@ -413,8 +440,15 @@ def _configure_instance(self, container) -> bool: self._mysql.configure_mysql_users() # Configure instance as a cluster node self._mysql.configure_instance() - # Restart exporter service after configuration - container.restart(MYSQLD_EXPORTER_SERVICE) + + if self.has_cos_relation: + if container.get_services(MYSQLD_EXPORTER_SERVICE)[ + MYSQLD_EXPORTER_SERVICE + ].is_running(): + # Restart exporter service after configuration + container.restart(MYSQLD_EXPORTER_SERVICE) + else: + container.start(MYSQLD_EXPORTER_SERVICE) except ( MySQLConfigureInstanceError, MySQLConfigureMySQLUsersError, diff --git a/src/constants.py b/src/constants.py index 6503d92ef..36e9ed766 100644 --- a/src/constants.py +++ b/src/constants.py @@ -46,3 +46,5 @@ MYSQLD_EXPORTER_SERVICE = "mysqld_exporter" GR_MAX_MEMBERS = 9 SECRET_ID_KEY = "secret-id" +# TODO: should be changed when adopting cos-agent +COS_AGENT_RELATION_NAME = "metrics-endpoint" diff --git a/src/mysql_k8s_helpers.py b/src/mysql_k8s_helpers.py index 4a575f9c7..174c06304 100644 --- a/src/mysql_k8s_helpers.py +++ b/src/mysql_k8s_helpers.py @@ -17,6 +17,7 @@ MySQLStartMySQLDError, MySQLStopMySQLDError, ) +from ops.charm import CharmBase from ops.model import Container from ops.pebble import ChangeError, ExecError from tenacity import ( @@ -137,6 +138,7 @@ def __init__( backups_password: str, container: Container, k8s_helper: KubernetesHelpers, + charm: CharmBase, ): """Initialize the MySQL class. @@ -155,6 +157,7 @@ def __init__( backups_password: password for the backups user container: workload container object k8s_helper: KubernetesHelpers object + charm: charm object """ super().__init__( instance_address=instance_address, @@ -172,6 +175,7 @@ def __init__( ) self.container = container self.k8s_helper = k8s_helper + self.charm = charm def fix_data_dir(self, container: Container) -> None: """Ensure the data directory for mysql is writable for the "mysql" user. @@ -591,6 +595,10 @@ def start_mysqld(self) -> None: logger.exception(error_message) raise MySQLStartMySQLDError(error_message) + def restart_mysql_exporter(self) -> None: + """Restarts the mysqld exporter service in pebble.""" + self.charm._reconcile_pebble_layer(self.container) + def stop_group_replication(self) -> None: """Stop Group replication if enabled on the instance.""" stop_gr_command = ( diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 35d5ccdfe..fa11d23aa 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -422,3 +422,21 @@ async def retrieve_database_variable_value( ) return output[0] + + +async def start_mysqld_exporter(ops_test: OpsTest, unit: Unit) -> None: + """Start mysqld exporter pebble service on the provided unit. + + Args: + ops_test: The ops test framework + unit: The unit to start mysqld exporter on + """ + await ops_test.juju( + "ssh", + "--container", + CONTAINER_NAME, + unit.name, + "pebble", + "start", + "mysqld_exporter", + ) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 1322ab768..a606c9c3c 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -25,6 +25,7 @@ retrieve_database_variable_value, rotate_credentials, scale_application, + start_mysqld_exporter, ) logger = logging.getLogger(__name__) @@ -297,6 +298,8 @@ async def test_exporter_endpoints(ops_test: OpsTest) -> None: http = urllib3.PoolManager() for unit in application.units: + await start_mysqld_exporter(ops_test, unit) + unit_address = await get_unit_address(ops_test, unit.name) mysql_exporter_url = f"http://{unit_address}:9104/metrics" diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 1938fbcdb..bb5510055 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -28,8 +28,7 @@ def setUp(self) -> None: self.charm = self.harness.charm self.maxDiff = None - @property - def layer_dict(self): + def layer_dict(self, with_mysqld_exporter: bool = False): return { "summary": "mysqld services layer", "description": "pebble config layer for mysqld safe and exporter", @@ -47,7 +46,7 @@ def layer_dict(self): "override": "replace", "summary": "mysqld exporter", "command": "/start-mysqld-exporter.sh", - "startup": "enabled", + "startup": "enabled" if with_mysqld_exporter else "disabled", "user": "mysql", "group": "mysql", "environment": { @@ -67,7 +66,7 @@ def tearDown(self) -> None: def test_mysqld_layer(self): # Test layer property # Comparing output dicts - self.assertEqual(self.charm._pebble_layer.to_dict(), self.layer_dict) + self.assertEqual(self.charm._pebble_layer.to_dict(), self.layer_dict()) def test_on_leader_elected(self): # Test leader election setting of @@ -134,7 +133,13 @@ def test_mysql_pebble_ready( # After configuration run, plan should be populated plan = self.harness.get_container_pebble_plan("mysql") - self.assertEqual(plan.to_dict()["services"], self.layer_dict["services"]) + self.assertEqual(plan.to_dict()["services"], self.layer_dict()["services"]) + + self.harness.add_relation("metrics-endpoint", "test-cos-app") + plan = self.harness.get_container_pebble_plan("mysql") + self.assertEqual( + plan.to_dict()["services"], self.layer_dict(with_mysqld_exporter=True)["services"] + ) @patch("charm.MySQLOperatorCharm._mysql", new_callable=PropertyMock) def test_mysql_pebble_ready_non_leader(self, _mysql_mock): diff --git a/tests/unit/test_mysql_k8s_helpers.py b/tests/unit/test_mysql_k8s_helpers.py index ba5d07318..3fb0a9ce3 100644 --- a/tests/unit/test_mysql_k8s_helpers.py +++ b/tests/unit/test_mysql_k8s_helpers.py @@ -59,6 +59,7 @@ def setUp(self): "backupspassword", None, None, + None, ) @patch("ops.pebble.ExecProcess")