Restructured testbed folder. Enhanced runDentCi

CI_Automation/DentCiArgParse.py

@@ -3,6 +3,7 @@
 import argparse
 import Utilities
 import globalSettings
+from re import search
 
 timestamp = Utilities.getTimestamp(includeMillisecond=True).replace(':', '-')
 
@@ -47,6 +48,7 @@ def parse(self):
         else:
             self.ciVars.testId = timestamp
 
+        self.ciVars.cmdLine:                  str = ' '.join(sys.argv)
         self.ciVars.timestamp:                str = timestamp
         self.ciVars.testSessionFolder:        str = f'{globalSettings.dentTestResultsFolder}/{self.ciVars.testId}'
         self.ciVars.testSessionLogsFolder:    str = f'{self.ciVars.testSessionFolder}/CI_Logs'
@@ -58,14 +60,17 @@ def parse(self):
         self.ciVars.reportFile:               str = f'{self.ciVars.testSessionFolder}/ciTestReport'
 
         if args.testSuites is None:
-            sys.exit(1, '-testSuites parameter is required with test suites to use for testing')
+            sys.exit('-testSuites parameter is required with test suites to use for testing')
         else:
             # Verify for user defined testSuites existence
             for eachTestSuite in args.testSuites:
                 testSuite = eachTestSuite.replace('.yml', '')
-                testSuiteFile = f'{self.ciVars.testSuiteFolder}/{testSuite}.yml'
-                if os.path.exists(testSuiteFile) is False:
-                    Utilities.sysExit(self.ciVars, f'No such test suite name found: {eachTestSuite}')
+                regexMatch = search('.*((hw|vm)/.*)', testSuite)
+                if regexMatch:
+                    testSuite = regexMatch.group(1)
+                    testSuiteFile = f'{self.ciVars.testSuiteFolder}/{testSuite}.yml'
+                else:
+                    testSuiteFile = f'{self.ciVars.testSuiteFolder}/{testSuite}.yml'
 
                 self.ciVars.testSuites.append(testSuiteFile)
 

CI_Automation/runDentCi.py

@@ -1,6 +1,8 @@
 #!/usr/bin/python3
 
 """
+version = v10
+
 Dent CI Automation Framework
 
 Automating the following stages:
@@ -75,6 +77,8 @@ class CiVars:
     # Stage 3
     runTest:                   bool = True  # requires cloneTestRepo for testbed configs
 
+    cmdLine:                  str = ''
+
     # The test ID timestamp (without the appended testName)
     timestamp:                str = ''
     # The follows are set in DentCiArgParse.py
@@ -155,6 +159,9 @@ class CiVars:
 class DentCI:
     def __init__(self, ciVars):
         self.ciVars = ciVars
+        self.stage1Result = None
+        self.stage2Result = None
+        self.stage3Result = None
 
         self.ciVars.testIdTestingBranch = f'{self.ciVars.testBranchFolder}/{self.ciVars.testId}'
 
@@ -176,6 +183,7 @@ def __init__(self, ciVars):
         self.removeDockerStaleImages()
 
         self.testMgmtData = {'pid': self.ciVars.pid,
+                             'cmdLine': self.ciVars.cmdLine,
                              'testId': self.ciVars.testId,
                              'testName': self.ciVars.testName,
                              'startTime': datetime.now().strftime('%m-%d-%Y %H:%M:%S:%f'),
@@ -229,6 +237,8 @@ def __init__(self, ciVars):
         self.ciVars.sessionLog.info(f'Testing repo: {self.ciVars.repo}')
         self.ciVars.sessionLog.info(f'Install branchName: {self.ciVars.branchName}')
 
+        self.verifyTestSuites()
+
         # Wait at this point until the CI system is enabled
         self.isCiSystemEnabled()
 
@@ -270,6 +280,13 @@ def __init__(self, ciVars):
         self.createDockerImageTag()
         self.lockTestbeds()
 
+    def verifyTestSuites(self):
+        # Verify for user defined testSuite
+        for eachTestSuiteFile in self.ciVars.testSuites:
+            if os.path.exists(eachTestSuiteFile) is False:
+                self.ciVars.sessionLog.error(f'No such test suite: {eachTestSuiteFile}')
+                sys.exit(1)
+
     def createDockerImageTag(self):
         """
         Create a docker image tag for each test
@@ -359,6 +376,12 @@ def isTestIdTestBranchExists(self, stage=None):
         """
         Verify if the testing branch successfully got cloned
         """
+        # Show the cloned test branch in the testIdTestingBranch directory for visibility and confirmation
+        self.ciVars.sessionLog.info(f'isTestIdTestBranchExists: Looking for testID: {self.ciVars.testId}')
+        output = Utilities.runLinuxCmd('ls -l', cwd=self.ciVars.testBranchFolder)
+        for line in output:
+            self.ciVars.sessionLog.info(line.strip())
+
         if os.path.exists(self.ciVars.testIdTestingBranch) is False:
             errorMsg = f'Stage={stage}: Is cloneTestBranch == True?  The testID test branch does not exists: {self.ciVars.testIdTestingBranch}.'
             self.ciVars.sessionLog.error(errorMsg)
@@ -415,7 +438,7 @@ def parseForTestbeds(self, configFullPath):
         Get all testbeds from test suites "config" parameter
         """
         if os.path.exists(configFullPath) is False:
-            print(f'Test Suite config does not exists: {configFullPath}')
+            self.ciVars.sessionLog.error(f'Test Suite config does not exists: {configFullPath}')
             Utilities.runDentCiTearDown(ciVars, f'Test Suite file has an incorrect "config" path: {configFullPath}')
 
         testbedContents = Utilities.readJson(configFullPath)
@@ -514,6 +537,7 @@ def cloneTestRepo(self):
         and name it the testId
         """
         if self.ciVars.cloneTestRepo is False:
+            self.stage1Result = 'failed'
             return
 
         self.isGitCloneSafeToRun()
@@ -530,9 +554,10 @@ def cloneTestRepo(self):
             self.testbedMgmtObj.unlockTestbeds()
             Utilities.closeTestMgmtStatus(overallSummaryFile=self.ciVars.overallSummaryFile,
                                           status='aborted', result='failed', threadLock=self.ciVars.lock)
-            Utilities.runDentCiTearDown(self.ciVars, 'Clone Dent test repo failed')
+            Utilities.runDentCiTearDown(self.ciVars, 'Cloning Dent test repo failed')
+            self.stage1Result = 'failed'
 
-        if result:
+        if self.stage1Result != 'failed' and result:
             # Parse out all testbeds from test suite files and from the cloned repo
             for testSuite in self.ciVars.testSuites:
                 contents = Utilities.readYaml(testSuite)
@@ -556,30 +581,35 @@ def cloneTestRepo(self):
                                           status='aborted', result='failed', threadLock=self.ciVars.lock)
             # This will unlocktestbed, remove testId test branch, create jenkinsCI result path
             Utilities.runDentCiTearDown(self.ciVars, 'Clone Dent test repo failed on clone verification')
+            self.stage1Result = 'failed'
 
         self.verifyIxNetworkVMFunctionality()
+        self.stage1Result = 'passed'
 
     def downloadBuilds(self):
         """
         STAGE 1: Download Dent build image to tftp server /srv/tftp
         """
         if self.ciVars.downloadNewBuilds is False:
+            self.stage1Result = 'failed'
             return
 
         downloadBuildsResults = downloadBuilds(self.ciVars)
         if downloadBuildsResults is False:
+            self.stage1Result = 'failed'
             self.testbedMgmtObj.unlockTestbeds()
             Utilities.runDentCiTearDown(self.ciVars, 'Download builds failed')
+        else:
+            if self.stage1Result != 'failed':
+                self.stage1Result = 'passed'
 
-    def installDentOS(self):
+    def installDentOS(self, stage='installDentOS'):
         """
         STAGE 2: Install build on Dent
         """
-        if self.ciVars.installDentOS is False:
+        if self.stage1Result == 'failed' or self.ciVars.installDentOS is False:
             return
 
-        stage = 'installDentOS'
-
         # Requires pulling the branch for testbed configs
         self.isTestIdTestBranchExists(stage=stage)
         updateDentResult = updateDent(self.ciVars)
@@ -589,24 +619,31 @@ def installDentOS(self):
         Utilities.runLinuxCmd(f'rm -rf {self.ciVars.downloadToServerFolder}')
 
         if updateDentResult is False:
+            self.stage2Result = 'failed'
             Utilities.closeTestMgmtStatus(overallSummaryFile=self.ciVars.overallSummaryFile,
                                           status='aborted', result='failed', threadLock=self.ciVars.lock)
             Utilities.runDentCiTearDown(self.ciVars, 'failed')
-
-        self.ciVars.sessionLog.info(f'{stage}: passed')
+        else:
+            if self.stage2Result != 'failed':
+                self.stage2Result = 'passed'
+                self.ciVars.sessionLog.info(f'{stage}: passed')
 
     def deployIxNetwork(self, forceBringUp=False):
         """
         STAGE 2: Deploy IxNetwork
         """
-        if self.ciVars.deployIxNetwork is False and forceBringUp is False:
+        if self.stage1Result != 'failed' and self.ciVars.deployIxNetwork is False and forceBringUp is False:
             return
 
         deployIxNetworkResult = deployIxNetworkInit(ciVars=self.ciVars)
         if deployIxNetworkResult is False:
+            self.stage2Result = 'failed'
             Utilities.closeTestMgmtStatus(overallSummaryFile=self.ciVars.overallSummaryFile,
                                           status='aborted', result='failed', threadLock=self.ciVars.lock)
             Utilities.runDentCiTearDown(self.ciVars, 'failed')
+        else:
+            if self.stage2Result != 'failed':
+                self.stage2Result = 'passed'
 
         # Enable back the CI system for other test to run
         if os.path.exists(f'{globalSettings.dentCiMgmtPath}/{globalSettings.disableSystemFilename}'):
@@ -618,26 +655,28 @@ def deployDentTestContainers(self):
         """
         STAGE 2: Deploy test containers
         """
-        if self.ciVars.deployDentTestContainers is False:
+        if self.stage2Result != 'failed' or self.ciVars.deployDentTestContainers is False:
             return
 
         dentContainerObj = DeployTestContainers(self.ciVars.testContainersLogFile, self.ciVars)
         result = dentContainerObj.removeAndBuild()
 
         if result is False:
+            self.stage2Result = 'failed'
             Utilities.closeTestMgmtStatus(overallSummaryFile=self.ciVars.overallSummaryFile,
                                           status='aborted', result='failed', threadLock=self.ciVars.lock)
             Utilities.runDentCiTearDown(self.ciVars, 'failed')
+        else:
+            if self.stage2Result != 'failed':
+                self.stage2Result = 'passed'
 
-    def runTest(self):
+    def runTest(self, stage='runTest'):
         """
         STAGE 3: Run test
         """
-        if self.ciVars.runTest is False:
+        if self.stage2Result != 'failed' or self.ciVars.runTest is False:
             return
 
-        stage = 'runTest'
-
         # Verify if test branch exists
         self.isTestIdTestBranchExists(stage=stage)
 
@@ -684,23 +723,29 @@ def runTest(self):
         Utilities.runThreads(ciVars, threads)
 
     # Stage 2
-    threads = []
-    if ciVars.installDentOS:
-        ci.installDentOS()
-        # threads.append(Thread(target=ci.installDentOS, name='installDentOS'))
+    if ci.stage1Result == 'passed':
+        threads = []
+        if ciVars.installDentOS:
+            ci.installDentOS()
+            # threads.append(Thread(target=ci.installDentOS, name='installDentOS'))
 
-    if ciVars.deployIxNetwork:
-        threads.append(Thread(target=ci.deployIxNetwork, name='deployIxNetwork'))
+        if ciVars.deployIxNetwork:
+            threads.append(Thread(target=ci.deployIxNetwork, name='deployIxNetwork'))
 
-    if ciVars.deployDentTestContainers:
-        threads.append(Thread(target=ci.deployDentTestContainers, name='deployDentTestContainers'))
+        if ciVars.deployDentTestContainers:
+            threads.append(Thread(target=ci.deployDentTestContainers, name='deployDentTestContainers'))
 
-    if threads:
-        Utilities.runThreads(ciVars, threads)
+        if threads:
+            Utilities.runThreads(ciVars, threads)
+    else:
+        raise Exception('Stage 1 failed. Aborting test.')
 
     # Stage 3
-    if ciVars.runTest:
-        ci.runTest()
+    if ci.stage2Result == 'passed':
+        if ciVars.runTest:
+            ci.runTest()
+    else:
+        raise Exception('Stage 2 failed. Aborting test.')
 
     # If in dev mode, it might not call runDentCiTearDown.
     # So call it to unlocktestbed

DentOS_Framework/DentOsTestbed/configuration/testbed_config/hw/sit/basic_agg1/agg1/DHCP

@@ -0,0 +1,6 @@
+#
+# ISC dhcpd configuration
+#
+
+
+option domain-name "dentlab-agg1";

DentOS_Framework/DentOsTestbed/configuration/testbed_config/hw/sit/basic_agg1/agg1/HOSTNAME

@@ -0,0 +1 @@
+dentlab-agg1

DentOS_Framework/DentOsTestbed/configuration/testbed_config/hw/sit/basic_agg1/agg1/HOSTS

@@ -0,0 +1,5 @@
+127.0.0.1       localhost
+::1             localhost ip6-localhost ip6-loopback
+ff02::1         ip6-allnodes
+ff02::2         ip6-allrouters
+127.0.1.1       dentlab-agg1

DentOS_Framework/DentOsTestbed/configuration/testbed_config/hw/sit/basic_agg1/agg1/NETWORK_INTERFACES

@@ -0,0 +1,16 @@
+#
+# ONL format of dent interfaces
+#
+# This file describes the network interfaces available on your system
+# and how to activate them. For more information, see interfaces(5).
+
+source /etc/network/interfaces.d/*.intf
+
+# The loopback network interface
+auto lo
+iface lo inet loopback
+    address 20.20.0.1/32
+
+# The management interface
+auto ma1
+iface ma1 inet dhcp

DentOS_Framework/DentOsTestbed/configuration/testbed_config/hw/sit/basic_agg1/agg1/NTP

@@ -0,0 +1,59 @@
+# /etc/ntp.conf, configuration for ntpd; see ntp.conf(5) for help
+
+
+driftfile /var/lib/ntp/ntp.drift
+
+
+# Enable this if you want statistics to be logged.
+#statsdir /var/log/ntpstats/
+
+statistics loopstats peerstats clockstats
+filegen loopstats file loopstats type day enable
+filegen peerstats file peerstats type day enable
+filegen clockstats file clockstats type day enable
+
+
+# You do need to talk to an NTP server or two (or three).
+#server ntp.your-provider.example
+
+
+# pool.ntp.org maps to about 1000 low-stratum NTP servers.  Your server will
+# pick a different set every time it starts up.  Please consider joining the
+# pool: <http://www.pool.ntp.org/join.html>
+#
+
+# Access control configuration; see /usr/share/doc/ntp-doc/html/accopt.html for
+# details.  The web page <http://support.ntp.org/bin/view/Support/AccessRestrictions>
+# might also be helpful.
+#
+# Note that "restrict" applies to both servers and clients, so a configuration
+# that might be intended to block requests from certain clients could also end
+# up blocking replies from your own upstream servers.
+
+# By default, exchange time with everybody, but don't allow configuration.
+restrict -4 default kod notrap nomodify nopeer noquery
+restrict -6 default kod notrap nomodify nopeer noquery
+
+# Local users may interrogate the ntp server more closely.
+restrict 127.0.0.1
+restrict ::1
+
+# Clients from this (example!) subnet have unlimited access, but only if
+# cryptographically authenticated.
+#restrict 192.168.123.0 mask 255.255.255.0 notrust
+
+
+# If you want to provide time to your local subnet, change the next line.
+# (Again, the address is an example only.)
+#broadcast 192.168.123.255
+
+# If you want to listen to time broadcasts on your local subnet, de-comment the
+# next lines.  Please do this only if you trust everybody on the network!
+#disable auth
+#broadcastclient
+
+# Specify interfaces, don't listen on switch ports
+#interface listen ma1
+interface listen lo
+interface listen lo:0
+interface listen dummy0

DentOS_Framework/DentOsTestbed/configuration/testbed_config/hw/sit/basic_agg1/agg1/QUAGGA_CONFIG

@@ -0,0 +1,11 @@
+username cumulus nopassword
+!
+service integrated-vtysh-config
+!
+log timestamp precision 6
+!
+no zebra nexthop kernel enable
+!
+line vty
+!
+end