Support share image for gnmi and telemetry (sonic-net#10348)

What is the motivation for this PR?
We will share docker image to support gnmi and telemetry.
So we need to update minigraph, gnmi and telemetry test to support this change.

How did you do it?
For telemetry test, check telemetry container at first, if there's no telemetry container, use gnmi container.
For gnmi test, check gnmi container at first, if there's no gnmi container, use telemetry container.
For minigraph test, check feature status to choose service.

How did you verify/test it?
Run end to end test.

tests/common/helpers/gnmi_utils.py

@@ -0,0 +1,53 @@
+from functools import lru_cache
+import pytest
+
+
+@lru_cache(maxsize=None)
+class GNMIEnvironment(object):
+    TELEMETRY_MODE = 0
+    GNMI_MODE = 1
+
+    def __init__(self, duthost, mode):
+        if mode == self.TELEMETRY_MODE:
+            ret = self.generate_telemetry_config(duthost)
+            if ret:
+                return
+            ret = self.generate_gnmi_config(duthost)
+            if ret:
+                return
+        elif mode == self.GNMI_MODE:
+            ret = self.generate_gnmi_config(duthost)
+            if ret:
+                return
+            ret = self.generate_telemetry_config(duthost)
+            if ret:
+                return
+        pytest.fail("Can't generate GNMI/TELEMETRY configuration, mode %d" % mode)
+
+    def generate_gnmi_config(self, duthost):
+        cmd = "docker images | grep -w sonic-gnmi"
+        if duthost.shell(cmd, module_ignore_errors=True)['rc'] == 0:
+            cmd = "docker ps | grep -w gnmi"
+            if duthost.shell(cmd, module_ignore_errors=True)['rc'] == 0:
+                self.gnmi_config_table = "GNMI"
+                self.gnmi_container = "gnmi"
+                self.gnmi_program = "gnmi-native"
+                self.gnmi_port = 50052
+                return True
+            else:
+                pytest.fail("GNMI is not running")
+        return False
+
+    def generate_telemetry_config(self, duthost):
+        cmd = "docker images | grep -w sonic-telemety"
+        if duthost.shell(cmd, module_ignore_errors=True)['rc'] == 0:
+            cmd = "docker ps | grep -w telemety"
+            if duthost.shell(cmd, module_ignore_errors=True)['rc'] == 0:
+                self.gnmi_config_table = "TELEMETRY"
+                self.gnmi_container = "telemetry"
+                self.gnmi_program = "telemetry"
+                self.gnmi_port = 50051
+                return True
+            else:
+                pytest.fail("Telemetry is not running")
+        return False

tests/dash/gnmi_utils.py

@@ -3,6 +3,7 @@
 import time
 import uuid
 from functools import lru_cache
+import pytest
 
 import proto_utils
 
@@ -12,19 +13,7 @@
 @lru_cache(maxsize=None)
 class GNMIEnvironment(object):
     def __init__(self, duthost):
-        self.duthost = duthost
         self.work_dir = "/tmp/" + str(uuid.uuid4()) + "/"
-        self.use_gnmi_container = duthost.shell("docker ps | grep -w gnmi", module_ignore_errors=True)['rc'] == 0
-        if self.use_gnmi_container:
-            self.gnmi_config_table = "GNMI"
-            self.gnmi_container = "gnmi"
-            self.gnmi_program = "gnmi-native"
-        else:
-            self.gnmi_config_table = "TELEMETRY"
-            self.gnmi_container = "telemetry"
-            self.gnmi_program = "telemetry"
-        self.gnmi_port = int(duthost.shell(
-            "sonic-db-cli CONFIG_DB hget '%s' 'port'" % (self.gnmi_config_table + '|gnmi'))['stdout'])
         self.gnmi_cert_path = "/etc/sonic/telemetry/"
         self.gnmi_ca_cert = "gnmiCA.pem"
         self.gnmi_ca_key = "gnmiCA.key"
@@ -34,6 +23,29 @@ def __init__(self, duthost):
         self.gnmi_client_key = "gnmiclient.key"
         self.gnmi_server_start_wait_time = 30
         self.enable_zmq = duthost.shell("netstat -na | grep -w 8100", module_ignore_errors=True)['rc'] == 0
+        cmd = "docker images | grep -w sonic-gnmi"
+        if duthost.shell(cmd, module_ignore_errors=True)['rc'] == 0:
+            cmd = "docker ps | grep -w gnmi"
+            if duthost.shell(cmd, module_ignore_errors=True)['rc'] == 0:
+                self.gnmi_config_table = "GNMI"
+                self.gnmi_container = "gnmi"
+                self.gnmi_program = "gnmi-native"
+                self.gnmi_port = 50052
+                return
+            else:
+                pytest.fail("GNMI is not running")
+        cmd = "docker images | grep -w sonic-telemetry"
+        if duthost.shell(cmd, module_ignore_errors=True)['rc'] == 0:
+            cmd = "docker ps | grep -w telemetry"
+            if duthost.shell(cmd, module_ignore_errors=True)['rc'] == 0:
+                self.gnmi_config_table = "TELEMETRY"
+                self.gnmi_container = "telemetry"
+                self.gnmi_program = "telemetry"
+                self.gnmi_port = 50051
+                return
+            else:
+                pytest.fail("Telemetry is not running")
+        pytest.fail("Can't find telemetry and gnmi image")
 
 
 def create_ext_conf(ip, filename):

tests/gnmi/helper.py

@@ -1,6 +1,7 @@
 import time
 import re
-from tests.common.utilities import get_image_type
+from tests.common.helpers.gnmi_utils import GNMIEnvironment
+
 
 GNMI_CONTAINER_NAME = ''
 GNMI_PROGRAM_NAME = ''
@@ -10,35 +11,8 @@
 
 
 def gnmi_container(duthost):
-    global GNMI_CONTAINER_NAME
-    if len(GNMI_CONTAINER_NAME) == 0:
-        if get_image_type(duthost) == "public":
-            GNMI_CONTAINER_NAME = "telemetry"
-        else:
-            GNMI_CONTAINER_NAME = "gnmi"
-    return GNMI_CONTAINER_NAME
-
-
-def gnmi_program(duthost):
-    global GNMI_PROGRAM_NAME
-    if len(GNMI_PROGRAM_NAME) == 0:
-        if get_image_type(duthost) == "public":
-            GNMI_PROGRAM_NAME = "telemetry"
-        else:
-            GNMI_PROGRAM_NAME = "gnmi-native"
-    return GNMI_PROGRAM_NAME
-
-
-def gnmi_port(duthost):
-    global GNMI_PORT
-    if GNMI_PORT == 0:
-        if get_image_type(duthost) == "public":
-            GNMI_CONFIG_KEY = 'TELEMETRY|gnmi'
-        else:
-            GNMI_CONFIG_KEY = 'GNMI|gnmi'
-        port = duthost.shell("sonic-db-cli CONFIG_DB hget '%s' 'port'" % GNMI_CONFIG_KEY)['stdout']
-        GNMI_PORT = int(port)
-    return GNMI_PORT
+    env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE)
+    return env.gnmi_container
 
 
 def create_ext_conf(ip, filename):
@@ -55,35 +29,36 @@ def create_ext_conf(ip, filename):
 
 
 def apply_cert_config(duthost):
-    port = gnmi_port(duthost)
-    assert int(port) > 0, "Invalid GNMI port"
-    dut_command = "docker exec %s supervisorctl stop %s" % (gnmi_container(duthost), gnmi_program(duthost))
+    env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE)
+    dut_command = "docker exec %s supervisorctl stop %s" % (env.gnmi_container, env.gnmi_program)
     duthost.shell(dut_command)
-    dut_command = "docker exec %s pkill telemetry" % (gnmi_container(duthost))
+    dut_command = "docker exec %s pkill telemetry" % (env.gnmi_container)
     duthost.shell(dut_command, module_ignore_errors=True)
-    dut_command = "docker exec %s bash -c " % gnmi_container(duthost)
-    dut_command += "\"/usr/bin/nohup /usr/sbin/telemetry -logtostderr --port %s " % port
+    dut_command = "docker exec %s bash -c " % env.gnmi_container
+    dut_command += "\"/usr/bin/nohup /usr/sbin/telemetry -logtostderr --port %s " % env.gnmi_port
     dut_command += "--server_crt /etc/sonic/telemetry/gnmiserver.crt --server_key /etc/sonic/telemetry/gnmiserver.key "
     dut_command += "--ca_crt /etc/sonic/telemetry/gnmiCA.pem -gnmi_native_write=true -v=10 >/root/gnmi.log 2>&1 &\""
     duthost.shell(dut_command)
     time.sleep(GNMI_SERVER_START_WAIT_TIME)
 
 
 def recover_cert_config(duthost):
-    dut_command = "docker exec %s supervisorctl status %s" % (gnmi_container(duthost), gnmi_program(duthost))
+    env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE)
+    dut_command = "docker exec %s supervisorctl status %s" % (env.gnmi_container, env.gnmi_program)
     output = duthost.command(dut_command, module_ignore_errors=True)['stdout'].strip()
     if 'RUNNING' in output:
         return
-    dut_command = "docker exec %s pkill telemetry" % (gnmi_container(duthost))
+    dut_command = "docker exec %s pkill telemetry" % (env.gnmi_container)
     duthost.shell(dut_command, module_ignore_errors=True)
-    dut_command = "docker exec %s supervisorctl start %s" % (gnmi_container(duthost), gnmi_program(duthost))
+    dut_command = "docker exec %s supervisorctl start %s" % (env.gnmi_container, env.gnmi_program)
     duthost.shell(dut_command)
     time.sleep(GNMI_SERVER_START_WAIT_TIME)
 
 
 def gnmi_capabilities(duthost, localhost):
+    env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE)
     ip = duthost.mgmt_ip
-    port = gnmi_port(duthost)
+    port = env.gnmi_port
     cmd = "gnmi/gnmi_cli -client_types=gnmi -a %s:%s " % (ip, port)
     cmd += "-logtostderr -client_crt ./gnmiclient.crt -client_key ./gnmiclient.key -ca_crt ./gnmiCA.pem -capabilities"
     output = localhost.shell(cmd, module_ignore_errors=True)
@@ -94,8 +69,9 @@ def gnmi_capabilities(duthost, localhost):
 
 
 def gnmi_set(duthost, localhost, delete_list, update_list, replace_list):
+    env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE)
     ip = duthost.mgmt_ip
-    port = gnmi_port(duthost)
+    port = env.gnmi_port
     cmd = "gnmi/gnmi_set -target_addr %s:%s " % (ip, port)
     cmd += "-alsologtostderr -cert ./gnmiclient.crt -key ./gnmiclient.key -ca ./gnmiCA.pem -time_out 60s"
     for delete in delete_list:
@@ -112,8 +88,9 @@ def gnmi_set(duthost, localhost, delete_list, update_list, replace_list):
 
 
 def gnmi_get(duthost, localhost, path_list):
+    env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE)
     ip = duthost.mgmt_ip
-    port = gnmi_port(duthost)
+    port = env.gnmi_port
     cmd = "gnmi/gnmi_get -target_addr %s:%s " % (ip, port)
     cmd += "-alsologtostderr -cert ./gnmiclient.crt -key ./gnmiclient.key -ca ./gnmiCA.pem"
     for path in path_list:
@@ -131,8 +108,9 @@ def gnmi_get(duthost, localhost, path_list):
 
 
 def gnoi_reboot(duthost, localhost, method, delay, message):
+    env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE)
     ip = duthost.mgmt_ip
-    port = gnmi_port(duthost)
+    port = env.gnmi_port
     cmd = "gnmi/gnoi_client -target %s:%s " % (ip, port)
     cmd += "-logtostderr -cert ./gnmiclient.crt -key ./gnmiclient.key -ca ./gnmiCA.pem -rpc Reboot "
     cmd += '-jsonin "{\\\"method\\\":%d, \\\"delay\\\":%d, \\\"message\\\":\\\"%s\\\"}"' % (method, delay, message)

tests/minigraph/test_masked_services.py

@@ -41,19 +41,36 @@ def change_service_state(duthost, service, enable):
 @pytest.mark.disable_loganalyzer
 def test_masked_services(duthosts, rand_one_dut_hostname):
     """
-    @summary: This test case will mask telemetry service, then test load_minigraph and check its success
+    @summary: This test case will mask telemetry/gnmi service, then test load_minigraph and check its success
     """
     duthost = duthosts[rand_one_dut_hostname]
-    test_service = "telemetry"
-    logging.info("Bringing down telemetry service")
-    service_status = duthost.critical_process_status("telemetry")
+    cmd = "docker images | grep -w sonic-telemetry"
+    if duthost.shell(cmd, module_ignore_errors=True)['rc'] == 0:
+        cmd = "docker ps | grep -w telemetry"
+        if duthost.shell(cmd, module_ignore_errors=True)['rc'] == 0:
+            test_service = "telemetry"
+        else:
+            pytest.fail("Telemetry is not running")
+    else:
+        cmd = "docker images | grep -w sonic-gnmi"
+        if duthost.shell(cmd, module_ignore_errors=True)['rc'] == 0:
+            cmd = "docker ps | grep -w gnmi"
+            if duthost.shell(cmd, module_ignore_errors=True)['rc'] == 0:
+                test_service = "gnmi"
+            else:
+                pytest.fail("GNMI is not running")
+        else:
+            pytest.fail("Can't find telemetry and gnmi image")
+    logging.info("Bringing down {} service".format(test_service))
+    service_status = duthost.critical_process_status(test_service)
 
     if not service_status:
-        pytest.fail("Make sure telemetry is up and running before calling this test")
+        pytest.fail("Make sure {} is up and running before calling this test".format(test_service))
 
     change_service_state(duthost, test_service, False)
     logging.info("Wait until service is masked and inactive")
-    pytest_assert(not wait_until(30, 10, 0, duthost.is_service_fully_started, "telemetry"), "TELEMETRY still running")
+    pytest_assert(not wait_until(30, 10, 0, duthost.is_service_fully_started, test_service),
+                  "{} still running".format(test_service))
 
     logging.info("Check service status")
     assert not is_service_loaded(duthost, test_service)
@@ -66,7 +83,8 @@ def test_masked_services(duthosts, rand_one_dut_hostname):
         logging.info("Bring back service if not up")
         change_service_state(duthost, test_service, True)
         logging.info("Wait until service is unmasked and active")
-        pytest_assert(wait_until(100, 10, 0, duthost.is_service_fully_started, "telemetry"), "TELEMETRY not started")
+        pytest_assert(wait_until(100, 10, 0, duthost.is_service_fully_started, test_service),
+                      "{} not started".format(test_service))
 
         if load_minigraph_error_code:
             pytest.fail("Test failed as load_minigraph was not successful")

tests/telemetry/conftest.py

@@ -4,11 +4,11 @@
 from tests.common.helpers.assertions import pytest_assert as py_assert
 from tests.common.errors import RunAnsibleModuleFail
 from tests.common.utilities import wait_until, wait_tcp_connection
+from tests.common.helpers.gnmi_utils import GNMIEnvironment
 from telemetry_utils import get_list_stdout, setup_telemetry_forpyclient, restore_telemetry_forpyclient
 
-logger = logging.getLogger(__name__)
 
-TELEMETRY_PORT = 50051
+logger = logging.getLogger(__name__)
 
 
 @pytest.fixture(scope="module")
@@ -36,11 +36,46 @@ def verify_telemetry_dockerimage(duthosts, enum_rand_one_per_hwsku_hostname):
     """
     docker_out_list = []
     duthost = duthosts[enum_rand_one_per_hwsku_hostname]
-    docker_out = duthost.shell('docker images docker-sonic-telemetry', module_ignore_errors=False)['stdout_lines']
+    docker_out = duthost.shell('docker images docker-sonic-gnmi', module_ignore_errors=False)['stdout_lines']
     docker_out_list = get_list_stdout(docker_out)
-    matching = [s for s in docker_out_list if b"docker-sonic-telemetry" in s]
+    matching = [s for s in docker_out_list if b"docker-sonic-gnmi" in s]
     if not (len(matching) > 0):
-        pytest.skip("docker-sonic-telemetry is not part of the image")
+        pytest.skip("docker-sonic-gnmi is not part of the image")
+
+
+def check_gnmi_config(duthost):
+    cmd = 'sonic-db-cli CONFIG_DB HGET "GNMI|gnmi" port'
+    port = duthost.shell(cmd, module_ignore_errors=False)['stdout']
+    return port != ""
+
+
+def create_gnmi_config(duthost):
+    cmd = "sonic-db-cli CONFIG_DB hset 'GNMI|gnmi' port 50052"
+    duthost.shell(cmd, module_ignore_errors=True)
+    cmd = "sonic-db-cli CONFIG_DB hset 'GNMI|gnmi' client_auth true"
+    duthost.shell(cmd, module_ignore_errors=True)
+    cmd = "sonic-db-cli CONFIG_DB hset 'GNMI|certs' "\
+          "ca_crt /etc/sonic/telemetry/dsmsroot.cer"
+    duthost.shell(cmd, module_ignore_errors=True)
+    cmd = "sonic-db-cli CONFIG_DB hset 'GNMI|certs' "\
+          "server_crt /etc/sonic/telemetry/streamingtelemetryserver.cer"
+    duthost.shell(cmd, module_ignore_errors=True)
+    cmd = "sonic-db-cli CONFIG_DB hset 'GNMI|certs' "\
+          "server_key /etc/sonic/telemetry/streamingtelemetryserver.key"
+    duthost.shell(cmd, module_ignore_errors=True)
+
+
+def delete_gnmi_config(duthost):
+    cmd = "sonic-db-cli CONFIG_DB hdel 'GNMI|gnmi' port"
+    duthost.shell(cmd, module_ignore_errors=True)
+    cmd = "sonic-db-cli CONFIG_DB hdel 'GNMI|gnmi' client_auth"
+    duthost.shell(cmd, module_ignore_errors=True)
+    cmd = "sonic-db-cli CONFIG_DB hdel 'GNMI|certs' ca_crt"
+    duthost.shell(cmd, module_ignore_errors=True)
+    cmd = "sonic-db-cli CONFIG_DB hdel 'GNMI|certs' server_crt"
+    duthost.shell(cmd, module_ignore_errors=True)
+    cmd = "sonic-db-cli CONFIG_DB hdel 'GNMI|certs' server_key"
+    duthost.shell(cmd, module_ignore_errors=True)
 
 
 @pytest.fixture(scope="module")
@@ -50,23 +85,28 @@ def setup_streaming_telemetry(duthosts, enum_rand_one_per_hwsku_hostname, localh
     """
     try:
         duthost = duthosts[enum_rand_one_per_hwsku_hostname]
+        has_gnmi_config = check_gnmi_config(duthost)
+        if not has_gnmi_config:
+            create_gnmi_config(duthost)
+        env = GNMIEnvironment(duthost, GNMIEnvironment.TELEMETRY_MODE)
         default_client_auth = setup_telemetry_forpyclient(duthost)
 
         if default_client_auth == "true":
-            duthost.shell('sonic-db-cli CONFIG_DB HSET "TELEMETRY|gnmi" "client_auth" "false"',
+            duthost.shell('sonic-db-cli CONFIG_DB HSET "%s|gnmi" "client_auth" "false"' % (env.gnmi_config_table),
                           module_ignore_errors=False)
-            duthost.shell("systemctl reset-failed telemetry")
-            duthost.service(name="telemetry", state="restarted")
+            duthost.shell("systemctl reset-failed %s" % (env.gnmi_container))
+            duthost.service(name=env.gnmi_container, state="restarted")
         else:
             logger.info('client auth is false. No need to restart telemetry')
 
         # Wait until telemetry was restarted
-        py_assert(wait_until(100, 10, 0, duthost.is_service_fully_started, "telemetry"), "TELEMETRY not started.")
+        py_assert(wait_until(100, 10, 0, duthost.is_service_fully_started, env.gnmi_container),
+                  "%s not started." % (env.gnmi_container))
         logger.info("telemetry process restarted. Now run pyclient on ptfdocker")
 
         # Wait until the TCP port was opened
         dut_ip = duthost.mgmt_ip
-        wait_tcp_connection(localhost, dut_ip, TELEMETRY_PORT, timeout_s=60)
+        wait_tcp_connection(localhost, dut_ip, env.gnmi_port, timeout_s=60)
 
         # pyclient should be available on ptfhost. If it was not available, then fail pytest.
         file_exists = ptfhost.stat(path=gnxi_path + "gnmi_cli_py/py_gnmicli.py")
@@ -78,3 +118,5 @@ def setup_streaming_telemetry(duthosts, enum_rand_one_per_hwsku_hostname, localh
 
     yield
     restore_telemetry_forpyclient(duthost, default_client_auth)
+    if not has_gnmi_config:
+        delete_gnmi_config(duthost)