Extract dump and syslog file into folder with testbed_idx in posttest (…

…sonic-net#11993)

Approach
What is the motivation for this PR?
In posttest, dump files are packaged into .tar.gz archives, which include syslog data. In Elastictest, we aim to make it easier for users to read both dump and syslog files. Therefore, we need to decompress and distinguish between the dump and syslog files.

How did you do it?
Add a param testbed_idx mark different testbed
Extract dump and syslog file into folder with testbed_idx


co-authorized by: [email protected]

tests/conftest.py

@@ -182,6 +182,8 @@ def pytest_addoption(parser):
     #   collect logs option    #
     ############################
     parser.addoption("--collect_db_data", action="store_true", default=False, help="Collect db info if test failed")
+    parser.addoption("--log_dir", action="store", default=None,
+                     help="Log dir name, can be used to put specific logs in a separate directory")
 
     ############################
     #   macsec options         #

tests/test_posttest.py

@@ -1,6 +1,10 @@
 import pytest
 import logging
 import time
+import os
+import glob
+import tarfile
+import gzip
 from tests.common.helpers.assertions import pytest_require
 
 logger = logging.getLogger(__name__)
@@ -13,11 +17,51 @@
     pytest.mark.skip_check_dut_health
 ]
 
+LOG_SAVE_PATH = 'logs/'
+
+
+def extract_tar_gz_file(tar_file_path, extract_path):
+    logger.info("Extracting {} to {}".format(tar_file_path, extract_path))
+    with tarfile.open(tar_file_path, 'r') as tar:
+        members = tar.getmembers()
+        for member in members:
+            try:
+                logger.info("Extract {}".format(member.name))
+                tar.extract(member, path=extract_path)
+            except Exception as e:
+                logger.info("Error extracting {}: {}".format(member.name, e))
+
+
+def extract_gz_file(gz_file_path):
+    extracted_file_path = os.path.splitext(gz_file_path)[0]
+    logger.info("Extracting {} to {}".format(gz_file_path, extracted_file_path))
+    with gzip.open(gz_file_path, 'rb') as f_in:
+        with open(extracted_file_path, 'wb') as f_out:
+            f_out.write(f_in.read())
+    os.remove(gz_file_path)
+
+
+def extract_dump_and_syslog(tar_file_path, dump_path, dir_name):
+    logger.info("Extracting tar.gz dump file {}".format(tar_file_path))
+    extract_tar_gz_file(tar_file_path, dump_path)
+    # rename dump folder
+    os.rename(tar_file_path.split('.tar.gz')[0], os.path.join(dump_path, dir_name))
+    # renmae .tar.gz dump file
+    os.rename(tar_file_path, os.path.join(dump_path, dir_name + '.tar.gz'))
+
+    syslog_path = dump_path + dir_name + '/log/'
+    syslog_gz_files = glob.glob(os.path.join(syslog_path, 'syslog*.gz'))
+    logger.info("Extracting syslog gz files: {}".format(syslog_gz_files))
+    if len(syslog_gz_files) > 0:
+        for syslog_gz in syslog_gz_files:
+            extract_gz_file(syslog_gz)
+
 
 def test_collect_techsupport(request, duthosts, enum_dut_hostname):
     since = request.config.getoption("--posttest_show_tech_since")
     if since == '':
         since = 'yesterday'
+    log_dir = request.config.getoption("--log_dir")
     duthost = duthosts[enum_dut_hostname]
     """
     A util for collecting techsupport after tests.
@@ -29,11 +73,18 @@ def test_collect_techsupport(request, duthosts, enum_dut_hostname):
     # Because Jenkins is configured to save artifacts from tests/logs,
     # and this util is mainly designed for running on Jenkins,
     # save path is fixed to logs for now.
-    TECHSUPPORT_SAVE_PATH = 'logs/'
     out = duthost.command("show techsupport --since {}".format(since), module_ignore_errors=True)
     if out['rc'] == 0:
         tar_file = out['stdout_lines'][-1]
-        duthost.fetch(src=tar_file, dest=TECHSUPPORT_SAVE_PATH, flat=True)
+        tar_file_name = tar_file.split('/')[-1]
+        dump_path = LOG_SAVE_PATH + 'dump/'
+        duthost.fetch(src=tar_file, dest=dump_path, flat=True)
+
+        if not log_dir:
+            log_dir = tar_file.split('/')[-1].split('.tar.gz')[0]
+        tar_file_path = os.path.join(dump_path, tar_file_name)
+        logger.info("tar_file_path: {}, dump_path: {}".format(tar_file_path, dump_path))
+        extract_dump_and_syslog(tar_file_path, dump_path, log_dir)
 
     assert True