Skip to content

Commit

Permalink
fix(logcollection): save schema info output
Browse files Browse the repository at this point in the history
This commit brings 2 fixes:
1. use 'remote_file' context manager to save schema output
it may improve command performance
2. use silence decorator for every run of schema info command. It's
to prevent stop info collection on failure
  • Loading branch information
juliayakovlev committed Dec 11, 2024
1 parent ca9d4de commit 2f70b31
Show file tree
Hide file tree
Showing 3 changed files with 36 additions and 15 deletions.
12 changes: 9 additions & 3 deletions sdcm/cluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -311,9 +311,15 @@ def __init__(self, name, parent_cluster, ssh_login_info=None, base_logdir=None,

def save_cqlsh_output_in_file(self, cmd: str, log_file: str):
self.log.info("Save command '%s' output in the file. Node %s", cmd, self.name)
result_tables = self.run_cqlsh(cmd, split=True)
for line in result_tables:
self.remoter.run(f"echo '{line}' >> {log_file}")
file_path = self.remoter.sudo(f"touch {log_file};realpath {log_file}",
verbose=True, ignore_status=True).stdout.strip()
if not (result_tables := self.run_cqlsh(cmd).stdout):
return

self.log.debug("Schema file path: %s", file_path)
with remote_file(remoter=self.remoter, remote_path=file_path, sudo=True) as fobj:
fobj.truncate(0) # first clear the file
fobj.write(result_tables.strip())

def _is_node_ready_run_scylla_commands(self) -> bool:
"""
Expand Down
13 changes: 8 additions & 5 deletions sdcm/logcollector.py
Original file line number Diff line number Diff line change
Expand Up @@ -626,7 +626,8 @@ def collect_logs_per_node(node):
try:
log_entity.collect(node, local_node_dir, remote_node_dir, local_search_path=local_search_path)
except Exception as details: # pylint: disable=unused-variable, broad-except # noqa: BLE001
LOGGER.error("Error occured during collecting on host: %s\n%s", node.name, details)
LOGGER.error("Error occured during collecting of %s on host: %s\n%s",
log_entity.name, node.name, details)

LOGGER.debug("Nodes list %s", [node.name for node in self.nodes])

Expand Down Expand Up @@ -712,6 +713,8 @@ class ScyllaLogCollector(LogCollector):
"-u scylla-housekeeping-daily.service -o short-precise", search_locally=True),
FileLog(name='system_*',
search_locally=True),
FileLog(name='schema.log',
search_locally=True),
FileLog(name='kallsyms_*',
search_locally=True),
FileLog(name='lsof_*',
Expand Down Expand Up @@ -749,10 +752,10 @@ class ScyllaLogCollector(LogCollector):
command='cat /var/log/cloud-init-output.log'),
CommandLog(name='cloud-init.log',
command='cat /var/log/cloud-init.log'),
CommandLog(name='schema.log',
command='cat schema.log'),
CommandLog(name='system_schema_tables.log',
command='cat system_schema_tables.log'),
# CommandLog(name='schema.log',
# command='cat schema.log'),
# CommandLog(name='system_schema_tables.log',
# command='cat system_schema_tables.log'),
]

cmd = "test -f /etc/scylla/ssl_conf/{0} && cat /etc/scylla/ssl_conf/{0}"
Expand Down
26 changes: 19 additions & 7 deletions sdcm/tester.py
Original file line number Diff line number Diff line change
Expand Up @@ -2961,16 +2961,28 @@ def clean_resources(self):

self.destroy_credentials()

@silence(name='Save node schema')
def save_nodes_schema(self):
if self.db_cluster is None:
self.log.info("No nodes found in the Scylla cluster")
def _save(node, silence_name, cmd, log_file):
with silence(name=silence_name, raise_error_event=False):
node.save_cqlsh_output_in_file(cmd=cmd, log_file=log_file)

cluster = self.db_cluster or self.k8s_cluster
if cluster is None:
self.log.error("No nodes found in the Scylla cluster. Save schema is impossible")
return

self.log.info("Save nodes user schema in the files")
for node in self.db_cluster.nodes:
node.save_cqlsh_output_in_file(cmd="desc schema", log_file="schema.log")
node.save_cqlsh_output_in_file(cmd="select JSON * from system_schema.tables",
log_file="system_schema_tables.log")
if not (live_nodes := [node for node in cluster.nodes if node._is_node_ready_run_scylla_commands()]):
self.log.error("Nodes are ready for running Scylla commands are not found in cluster. Save schema is impossible")
return

node = live_nodes[0]
save_params = [[node, f"Save node '{node.name}' schema", "desc schema", "schema.log"],
[node, f"Save node '{node.name}' system_schema.tables",
"select JSON * from system_schema.tables", "system_schema_tables.log"]
]
parallel_obj = ParallelObject(objects=save_params, timeout=20 * 60, num_workers=2)
parallel_obj.run(_save, unpack_objects=True)

def tearDown(self):
self.teardown_started = True
Expand Down

0 comments on commit 2f70b31

Please sign in to comment.