diff --git a/CI_Automation/CloneTestingRepo.py b/CI_Automation/CloneTestingRepo.py index fb8ec7e44..07fcf919b 100644 --- a/CI_Automation/CloneTestingRepo.py +++ b/CI_Automation/CloneTestingRepo.py @@ -19,6 +19,8 @@ def clone(self): Utilities.updateStage(self.ciVars.overallSummaryFile, stage='cloneTestRepo', status='running', result='None', threadLock=self.lock) + self.log.info(f'clone: testIdTestingBranch: {self.testIdTestingBranch}') + # Create the TestBranch folder to hold all testId testing repos if os.path.exists(self.testBranchFolder) is False: Utilities.runLinuxCmd(f'mkdir -p {self.testBranchFolder}') @@ -41,7 +43,7 @@ def clone(self): Utilities.runLinuxCmd(f'chmod 770 {self.testIdTestingBranch}') if self.ciVars.localTestBranch: - Utilities.runLinuxCmd(f'cp -r {self.ciVars.localTestBranch} {self.testIdTestingBranch}', logObj=self.log) + Utilities.runLinuxCmd(f'cp -r {self.ciVars.localTestBranch}/* {self.testIdTestingBranch}', logObj=self.log) else: if self.branchName: cmd = f'git clone --branch {self.branchName} {self.repo} {self.testIdTestingBranch}' @@ -62,7 +64,7 @@ def clone(self): self.log.info('Verify if cloned repo test branch has files in it') output = Utilities.runLinuxCmd('ls', cwd=self.testIdTestingBranch, logObj=self.log) if len(output) > 0: - self.log.info('Verified cloned files!') + self.log.info('Verified cloned files') verified = True Utilities.updateStage(self.ciVars.overallSummaryFile, stage='cloneTestRepo', status='completed', result='passed', threadLock=self.lock) diff --git a/CI_Automation/DentCiArgParse.py b/CI_Automation/DentCiArgParse.py index 28dd59a41..94e23bbda 100644 --- a/CI_Automation/DentCiArgParse.py +++ b/CI_Automation/DentCiArgParse.py @@ -23,7 +23,8 @@ def __init__(self, ciVars: object) -> None: def parse(self): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('-testName', nargs='+', default=None, help='Give a test name to identify your test ID.') - parser.add_argument('-builds', nargs='+', default=None, help='Both ARM and AMD full path URLs for the builds') + parser.add_argument('-builds', nargs='+', default=None, help='Get from Dent website. Both ARM and AMD full path URLs for the builds') + parser.add_argument('-localBuilds', nargs='+', default=None, help='Local path to builds. Both ARM and AMD full path URLs for the builds') parser.add_argument('-testSuites', nargs='+', default=None, help='The test suite to run') parser.add_argument('-repo', nargs='+', default=None, help='The repo to clone for testing') parser.add_argument('-localBranch', nargs='+', default=None, help='Test with a local branch that is already cloned. Provide the path.') @@ -73,6 +74,11 @@ def parse(self): else: self.ciVars.builds = args.builds + if args.localBuilds is None: + self.ciVars.localBuilds = [] + else: + self.ciVars.localBuilds = args.localBuilds + if args.repo is None: # Default pulling the main branch self.ciVars.repo = self.ciVars.gitCloneDefaultRepo @@ -92,8 +98,12 @@ def parse(self): localBranch = args.localBranch[0] self.ciVars.localTestBranch = localBranch + self.ciVars.repo = localBranch else: - Utilities.sysExit(self.ciVars, f'No such local test branch: {args.localBranch[0]}') + print(f'No such local test branch: {args.localBranch[0]}') + sys.exit(f'No such local test branch: {args.localBranch[0]}') + else: + self.ciVars.localTestBranch = None if args.tftp is False and args.http is False: # Default to use http @@ -116,7 +126,6 @@ def parse(self): if args.disableDownloadNewBuilds: self.ciVars.downloadNewBuilds = False - self.ciVars.builds = [] else: if args.builds: self.ciVars.builds = args.builds diff --git a/CI_Automation/DeployDent.py b/CI_Automation/DeployDent.py index 9f0f4104d..f3f887786 100644 --- a/CI_Automation/DeployDent.py +++ b/CI_Automation/DeployDent.py @@ -94,9 +94,12 @@ def __init__(self, ciVars: object, dentDeviceLogObj: object = None, testbedObj: # http://10.36.118.11/DentBuildReleases/09-27-2023-00-15-49-343919_jenkinsCI_devTest self.serverPrefixPath = ciVars.serverPrefixPath # http path to download new build - self.srcBuildList = ciVars.builds + if ciVars.builds: + self.srcBuildList = ciVars.builds + if ciVars.localBuilds: + self.srcBuildList = ciVars.localBuilds self.spawnId = None - # This variable will get set by self.parseForRightImageToInstail() + # This variable will get set by self.parseForArmOrAmdImageToInstall() self.installBuild = None self.result = None self.checkIfItIsRebooting = False @@ -128,7 +131,7 @@ def parseForArmOrAmdImageToInstall(self): build = buildUrl.split('/')[-1] if self.cpuArchitecture.upper() in build: self.installBuild = build - self.log.info(f'DentOS {self.name} has CPU architecture: {self.cpuArchitecture}. Set image to: {build}') + self.log.info(f'parseForArmOrAmdImageToInstall: DentOS {self.name} has CPU architecture: {self.cpuArchitecture}. Set image to: {build}') def createMacAddressFolderInTftp(self): """ @@ -144,7 +147,7 @@ def createMacAddressFolderInTftp(self): if os.path.exists(self.dentSwitchMacAddressTftpFolder) is False: Utilities.runLinuxCmd(f'mkdir -p {self.dentSwitchMacAddressTftpFolder}', logObj=self.log) else: - self.log.info(f'A Mac Address folder already exists: {self.dentSwitchMacAddressTftpFolder}') + self.log.info(f'createMacAddressFolderInTftp: A Mac Address folder already exists: {self.dentSwitchMacAddressTftpFolder}') def createOnieDefaultImageInTftp(self): """ @@ -167,13 +170,13 @@ def createOnieDefaultImageInTftp(self): defaultDentFilename = f'{self.dentSwitchMacAddressTftpFolder}/onie-installer-arm64-accton_as4224_52p-r0' Utilities.runLinuxCmd(f'cp {self.tftpServerFolder}/{self.installBuild} {defaultDentFilename}', logObj=self.log) else: - self.log.error(f'Build not found in tftp server for Dent {self.name}: {self.tftpServerFolder}/{self.installBuild}') + self.log.error(f'createOnieDefaultImageInTftp: Build not found in tftp server for Dent {self.name}: {self.tftpServerFolder}/{self.installBuild}') return False except Exception as errMsg: self.log.error(f'createOnieDefaultImageInTftp: Error on Dent {self.name}: {traceback.format.exc(None, errMsg)}') return False - def updateDent(self, spawnTelnet=True, timeout=5) -> bool: + def updateDent(self, spawnTelnet=True, timeout=10) -> bool: """ Parameters: spawnTelent: : If 'Type the hot key to suspend the connection' is snagged, it will call itself with spawnTelnet=False. @@ -184,7 +187,7 @@ def updateDent(self, spawnTelnet=True, timeout=5) -> bool: """ try: if spawnTelnet: - self.log.info(f'updateDent {self.name}: Telnet to serial console: {self.serialConsoleIp} {self.serialConsolePort}') + self.log.info(f'updateDent: {self.name}: Telnet to serial console: {self.serialConsoleIp} {self.serialConsolePort}') self.spawnId = pexpect.spawn(f'telnet {self.serialConsoleIp} {self.serialConsolePort}') # 1 = turn on buffering self.spawnId.maxsize = 1 @@ -192,43 +195,48 @@ def updateDent(self, spawnTelnet=True, timeout=5) -> bool: # '.*localhost login.*' # 'root@localhost:~# *' - expectIndex = self.send('\n', expect=['Type the hot key to suspend the connection', - '.*ONIE:/.*', - '.*login:.*', - 'Marvell>>', - 'root@.*', - '.*This connection is in use.*' - ], timeout=timeout) + while True: + expectIndex = self.send('\n', expect=['Type the hot key to suspend the connection', + '.*ONIE:/.*', + '.*login:.*', + 'Marvell>>', + 'root@.*', + '.*This connection is in use.*' + ], timeout=timeout) + if expectIndex in [0, 1, 2, 3, 4, 5, 6]: + break + else: + time.sleep(1) if expectIndex == 0: self.updateDent(spawnTelnet=False) if expectIndex == 1: - self.log.info(f'Dent:{self.name}: got ONIE prompt') + self.log.info(f'UpdteDent: {self.name}: got ONIE prompt') if self.install() is False: self.updateDentResult('failed') else: - self.updateDentResult('passd') - self.log.info(f'Dent:{self.name}: End of update from ONIE. Final result: {self.result}') + self.updateDentResult('passed') + self.log.info(f'UpdateDent: {self.name}: End of update from ONIE. Final result: {self.result}') if expectIndex == 2: - self.log.info('Got Linux shell login prompt') + self.log.info(f'UpdateDent: {self.name}: Got Linux shell login prompt') if self.login(): if self.goToOnieFromLinuxShell() is False: self.updateDentResult('failed') else: - self.log.info(f'Dent device: {self.name}: Back to updateDent(). Calling install() ...') + self.log.info(f'UpdateDent: {self.name}: Back to updateDent(). Calling install() ...') if self.install() is False: self.updateDentResult('failed') else: self.updateDentResult('passed') - self.log.info(f'End of update from Linux shell login prompt. Final result: {self.result}') + self.log.info(f'UpdateDent: {self.name}: End of update from Linux shell login prompt. Final result: {self.result}') else: self.updateDentResult('failed') if expectIndex == 3: - self.log.info('Got Marvell prompt') + self.log.info(f'UpdateDent: {self.name}: Got Marvell prompt') if self.goToOniePromptFromMarvellPrompt() is False: self.updateDentResult('failed') else: @@ -237,10 +245,10 @@ def updateDent(self, spawnTelnet=True, timeout=5) -> bool: else: self.updateDentResult('passed') - self.log.info(f'End of update from Marvell. Final result: {self.result}') + self.log.info(f'UpdateDent: {self.name}: End of update from Marvell. Final result: {self.result}') if expectIndex == 4: - self.log.info('Linux shell prompt') + self.log.info(f'UpdateDent: {self.name}: Linux shell prompt') if self.goToOnieFromLinuxShell() is False: self.updateDentResult('failed') else: @@ -249,15 +257,15 @@ def updateDent(self, spawnTelnet=True, timeout=5) -> bool: else: self.updateDentResult('passed') - self.log.info(f'End of update from Linux shell prompt. Final result: {self.result}') + self.log.info(f'UpdateDent: {self.name}: End of update from Linux shell prompt. Final result: {self.result}') if expectIndex == 5: - self.log.failed(f'Somebody is using this serial console: {self.serialConsoleIp} {self.serialConsolePort}') + self.log.failed(f'UpdateDent: {self.name}: Somebody is using this serial console: {self.serialConsoleIp} {self.serialConsolePort}') self.updateDentResult('failed') # pexpect.TIMEOUT if expectIndex == 6: - self.log.failed(f'Failed to telnet into device: {self.serialConsoleIp} {self.serialConsolePort}. Somebody could be using the serial console!') + self.log.failed(f'UpdateDent timed out: {self.name}: Telnet serial console: {self.serialConsoleIp} {self.serialConsolePort}. Somebody could be using the serial console!') # Dent could be in the middle of rebooting if self.checkIfItIsRebooting is False: self.checkIfItIsRebooting = True @@ -266,11 +274,11 @@ def updateDent(self, spawnTelnet=True, timeout=5) -> bool: self.updateDentResult('failed') except pexpect.ExceptionPexpect: - self.log.failed(f'updateDent:{self.name}: {self.serialConsoleIp} {self.serialConsolePort} Expect timeout exceeded.') + self.log.failed(f'updateDent error 1: {self.name}: Telnet {self.serialConsoleIp} {self.serialConsolePort} timed out.') self.updateDentResult('failed') except Exception as errMsg: - self.log.failed(f'updateDenterror: {self.name}: {traceback.format_exc(None, errMsg)}') + self.log.failed(f'updateDent error 2: {self.name}: {traceback.format_exc(None, errMsg)}') self.updateDentResult('failed') def send(self, cmd: str, expect: list = [], timeout: int = 10) -> None: @@ -306,12 +314,18 @@ def send(self, cmd: str, expect: list = [], timeout: int = 10) -> None: def getCurrentShell(self, timeout=5): # '.*localhost login.*', # '.*root@localhost:.*', - expectIndex = self.send('\n', expect=['ONIE:.*', - '.*Marvell>>.*', - '.*login:.*', - '.*root@.*', - '.*This connection is in use.*' - ], timeout=timeout) + while True: + expectIndex = self.send('\n', expect=['ONIE:.*', + '.*Marvell>>.*', + '.*login:.*', + '.*root@.*', + '.*This connection is in use.*' + ], timeout=timeout) + if expectIndex in [0, 1, 2, 3, 4, 5, 6]: + break + else: + time.sleep(1) + if expectIndex == 0: return 'onie' elif expectIndex == 1: @@ -328,7 +342,7 @@ def getCurrentShell(self, timeout=5): def login(self) -> bool: expectIndex = self.send(cmd=self.linuxShellLogin, expect=['Password.*']) if expectIndex != 0: - self.log.failed(f'Dent device: {self.name}: Failed to log into Linux shell with username: {self.linuxShellLogin}') + self.log.failed(f'login: Dent device: {self.name}: Failed to log into Linux shell with username: {self.linuxShellLogin}') return False else: self.log.info('Got password prompt') @@ -345,7 +359,7 @@ def login(self) -> bool: return True if expectIndex == 1: - self.log.failed(f'log into the Linux shell failed for Dent: {self.name}') + self.log.failed(f'login: log into the Linux shell failed for Dent: {self.name}') return False def goToOniePrompt(self): @@ -354,25 +368,40 @@ def goToOniePrompt(self): onie-discovery-stop """ # 'root@localhost:~# *' - expectResult = self.send(cmd='\n', expect=['ONIE:.*', 'Marvell>>', 'root@.*'], timeout=3) + while True: + expectResult = self.send(cmd='\n', expect=['ONIE:.*', 'Marvell>>', 'root@.*'], timeout=3) + if expectResult in [0, 1, 2, 3]: + break + else: + time.sleep(1) + if expectResult == 0: - self.log.info('Got ONIE prompt. Done') + self.log.info(f'goToOniePrompt: {self.name}: Got ONIE prompt. Done') return True # Got Marvell>> prompt if expectResult == 1: - self.log.info('Got Marvell prompt') - self.log.info('Sending: run onie_bootcmd. Waiting up to 80 seconds ...') + self.log.info('goToOniePrompt: {self.name}: Got Marvell prompt') + self.log.info('goToOniePrompt: {self.name}: Sending: run onie_bootcmd. Waiting up to 80 seconds ...') return self.goToOniePromptFromMarvellPrompt() # Got into the Linux shell if expectResult == 2: expectIndex = self.send(cmd='reboot', expect=['Hit any key to stop autoboot'], timeout=60) if expectIndex == 0: - expectIndex = self.send(cmd='\n', expect=['Marvell>>'], timeout=120) + while True: + expectIndex = self.send(cmd='\n', expect=['Marvell>>'], timeout=120) + if expectIndex in [0, 1]: + break + else: + time.sleep(1) + if expectIndex == 0: return self.goToOniePromptFromMarvellPrompt() + if expectResult == 3: + self.log.failed(f'goToOniePrompt: {self.name}: timedout expecting prompts: ONIE:, Marvell>>, root@localhst') + return False def goToOniePromptFromMarvellPrompt(self): @@ -380,7 +409,13 @@ def goToOniePromptFromMarvellPrompt(self): self.spawnId.expect(r'.+') timeout = 120 - expectIndex = self.send('\n', expect=['Marvell>>'], timeout=5) + while True: + expectIndex = self.send('\n', expect=['Marvell>>'], timeout=5) + if expectIndex in [0, 1]: + break + else: + time.sleep(1) + if expectIndex == 2: self.log.failed('goToOniePromptFromMarvellPrompt: Dent:{self.name}: Failed to get the Marvell>> prompt. Cannot continue.') return False @@ -392,10 +427,10 @@ def goToOniePromptFromMarvellPrompt(self): self.log.info(f'goToOniePromptFromMarvellPrompt: Dent:{self.name} -> Sending onie-discovery-stop ...') expectIndex1 = self.send(cmd='onie-discovery-stop', expect=['.*ONIE:.*'], timeout=20) if expectIndex1 == 0: - self.log.info('goToOniePromptFromMarvellPrompt: Got ONIE prompt. Auto-discovery stopped.') + self.log.info('goToOniePromptFromMarvellPrompt: {self.name}: Got ONIE prompt. Auto-discovery stopped.') return True if expectIndex1 == 1: - self.log.failed('goToOniePromptFromMarvellPrompt: Did not get ONIE prompt after sending onei-discovery-stop') + self.log.failed('goToOniePromptFromMarvellPrompt: {self.name}: Did not get ONIE prompt after sending onei-discovery-stop') return False return False @@ -407,24 +442,30 @@ def goToOnieFromLinuxShell(self): If "123" was entered, it goes into the Marvell>> mode immediately. Otherwise, it goes into mode. """ - expectIndex = self.send(cmd='reboot', expect=['Hit any key to stop autoboot', + while True: + expectIndex = self.send(cmd='\n', expect=['Hit any key to stop autoboot', '.*Type 123 to STOP autoboot.*'], timeout=80) + if expectIndex in [0, 1, 2]: + break + else: + time.sleep(1) + if expectIndex == 0: return self.goToOniePromptFromMarvellPrompt() if expectIndex == 1: - self.log.info(f'Dent device: {self.name}: Got Type 123 on serial console. Sening 123 to go into Marvell>> ...') + self.log.info(f'goToOnieFromLinuxShell: Dent device: {self.name}: Got Type 123 on serial console. Sening 123 to go into Marvell>> ...') if self.spawnId.before: self.spawnId.expect(r'.+') self.spawnId.sendline('123\r\n') expectIndex1 = self.spawnId.expect(['.*Marvell>>.*', pexpect.TIMEOUT], timeout=5) - self.log.debug(f'spawnId.after: {self.spawnId.after}') + self.log.debug(f'goToOnieFromLinuxShell: {self.name}: spawnId.after: {self.spawnId.after}') # self.spawnId.after => got back after: b'Marvell>> \r\nMarvell>> \r\nMarvell>> ' if expectIndex1 == 0 or 'Marvell' in self.spawnId.after.decode('utf'): - self.log.info(f'Dent device: {self.name}: Got Marvell>> prompt.') + self.log.info(f'goToOnieFromLinuxShell: {self.name}: Got Marvell>> prompt.') result = self.goToOniePromptFromMarvellPrompt() return result @@ -432,7 +473,7 @@ def goToOnieFromLinuxShell(self): if expectIndex1 == 1: self.log.failed(f'goToOnieFromLinuxShell: Waiting for Marvell prompt timedout on Dent: {self.name}') currentShell = self.getCurrentShell() - self.log.debug(f'Dent device: {self.name}: Current mode is {currentShell}') + self.log.debug(f'goToOnieFromLinuxShell: {self.name}: Current mode is {currentShell}') if currentShell == 'marvell': result = self.goToOniePromptFromMarvellPrompt() return result @@ -461,8 +502,7 @@ def install(self, retry=False) -> bool: installServerPath = f'{self.serverPrefixPath}/{self.installBuild}' """ - onie-nos-install http://10.36.118.11/dentInstallations/DENTOS-HEAD_ONL- - OS10_2023-08-11.1533-be121f3_ARM64_INSTALLED_INSTALLER + onie-nos-install http://10.36.118.11/dentBuildReleases/DENTOS-HEAD_ONL-OS10_2023-12-13.1632-67e7fe4_ARM64_INSTALLED_INSTALLER discover: installer mode detected. Stopping: discover... done. Info: Attempting http://10.36.118.11/dentInstallations/DENTOS-HEAD_ONL-OS10_2023-08-11.1533-be121f3_ARM64_INSTALLED_INSTALLER ... @@ -476,18 +516,30 @@ def install(self, retry=False) -> bool: self.spawnId.timeout = timeout cmd = f'onie-nos-install {installServerPath}' - self.log.info(f'sending cmd on Dent: {self.name}: {cmd} ...') + self.log.info(f'install: sending cmd on Dent: {self.name}: {cmd} ...') # Must use sendline() instead of send() - self.log.info(f'Dent: {self.name}: Waiting for installation to complete and reboot to Linux mode ...') + self.log.info(f'install: Dent: {self.name}: Waiting for installation to complete and reboot to Linux mode ...') self.spawnId.sendline(cmd) + self.spawnId.expect(['.*'], timeout=3) # '.*localhost login.*' - expectIndex = self.spawnId.expect(['.*login:.*'], timeout=timeout) + while True: + expectIndex = self.send(cmd='\n', expect=['.*404 Not Found', + '.*login:.*'], timeout=80) + + if expectIndex in [0, 1, 2]: + break + else: + time.sleep(1) if expectIndex == 0: - self.log.info(f'Installation is complete for Dent device: {self.name}') - self.log.info(f'Logging into Dent Linux shell: {self.name} ...') + self.log.failed(f'install: {self.name}: wget: server returned error: HTTP/1.1 404 Not Found') + return False + + if expectIndex == 1: + self.log.info(f'install: Installation is complete for Dent device: {self.name}') + self.log.info(f'install: Logging into Dent Linux shell: {self.name} ...') if self.login(): result = self.verifyCurrentBuild() @@ -497,21 +549,21 @@ def install(self, retry=False) -> bool: # pexpect.TIMEOUT if expectIndex == 1: - self.log.failed(f'Timed out waiting for Linux login after onie installation for Dent device: {self.name}') - self.log.debug(f'Verifying which shell mode the Dent switch is in: {self.name}') + self.log.failed(f'install: Timed out waiting for Linux login after onie installation for Dent device: {self.name}') + self.log.debug(f'install: Verifying which shell mode the Dent switch is in: {self.name}') currentShell = self.getCurrentShell() - self.log.debug(f'The current shell for Dent device: {self.name} == {currentShell}') + self.log.debug(f'install: The current shell for Dent device: {self.name} == {currentShell}') if currentShell == 'onie': if retry is False: - self.log.warning(f'Dent device: {self.name}: Failed to install build. Still in ONIE mode. Attempt to install one more time ...') + self.log.warning(f'install: Dent device: {self.name}: Failed to install build. Still in ONIE mode. Attempt to install one more time ...') self.install(retry=True) if retry: - self.log.failed('Retried to install build failed again. Still in ONIE mode. Expecting the installation to complete in Linux mode') + self.log.failed('install: Retried to install build failed again. Still in ONIE mode. Expecting the installation to complete in Linux mode') return False if currentShell == 'linuxLogin': - self.log.debug('Since it is in the Linux login prompt, the installation is complete') + self.log.debug('install: Since it is in the Linux login prompt, the installation is complete') if self.login(): result = self.verifyCurrentBuild() return result @@ -521,7 +573,7 @@ def install(self, retry=False) -> bool: return False def verifyCurrentBuild(self): - self.log.info(f'verifyCurrentBuild for Dent device: {self.name} ...') + self.log.info(f'verifyCurrentBuild: Dent device: {self.name} ...') if self.spawnId.before: self.spawnId.expect(r'.+') @@ -546,10 +598,10 @@ def verifyCurrentBuild(self): hashTag = hashTag.replace("'", '') if bool(search(f'.*{hashTag}.*', self.installBuild)): - self.log.info(f'Verify Build: Passed: Dent device: {self.name}: {hashTag}') + self.log.info(f'verifyCurrentBuild: Passed: Dent device: {self.name}: {hashTag}') return True else: - self.log.failed(f'Verifying Build: Failed: Dent device {self.name}. The installed build is {self.installBuild}. Expecting build with hash tag: {hashTag}') + self.log.failed(f'verifyCurrentBuild: Failed: Dent device {self.name}. The installed build is {self.installBuild}. Expecting build with hash tag: {hashTag}') return False # Search for: /etc/onl/loader/versions.json:3: "VERSION_STRING": "DENT OS DENTOS-HEAD, 2023-08-11.15:33-be121f3", @@ -558,7 +610,7 @@ def verifyCurrentBuild(self): self.log.failed(f'verifyCurrentBuild: grep had no output for Dent device: {self.name}') if expectIndex == 1: - self.log.failed(f'Attempted to verify the installed image on Dent {self.name}, but failed to get grep output') + self.log.failed(f'verifyCurrentBuild Attempted to verify the installed image on Dent {self.name}, but failed to get grep output') return False def getCPUArchitecture(self) -> str: @@ -833,6 +885,13 @@ def updateDent(ciVars: object) -> bool: Utilities.updateStage(ciVars.overallSummaryFile, stage='installDentOS', status='running', threadLock=ciVars.lock) + if len(ciVars.testbeds) == 0: + ciVars.sessionLog.error('updateDent: ciVars.testbeds has no testbeds!') + finalResult = False + Utilities.updateStage(ciVars.overallSummaryFile, stage='installDentOS', status='stopped', + result='failed', threadLock=ciVars.lock) + return finalResult + for testbed in ciVars.testbeds: if testbed['os'] != 'dentos': continue diff --git a/CI_Automation/DeployIxNetwork.py b/CI_Automation/DeployIxNetwork.py index bfc8b07b1..cf08d4e72 100644 --- a/CI_Automation/DeployIxNetwork.py +++ b/CI_Automation/DeployIxNetwork.py @@ -32,6 +32,9 @@ echo 'start the vm' virsh autostart IxNetwork-930 virsh start IxNetwork-930 + +Debug: + virsh console IxNetwork-9.30 """ import os import time @@ -81,11 +84,11 @@ def __init__(self, ixNetworkVMFolder: str, logObj: object) -> None: def isIxNetworkVMExists(self) -> bool: vmExists = False - # Attempt to verify up to 5 times. Sometimes, this command + # Attempt to verify up to 5 times. Sometimes this command # is either delayed or doesn't show anything for counter in range(0, 5): output = Utilities.runLinuxCmd('virsh list', logObj=self.log) - self.log.info(f'isNetworkVMExists: output {counter}/5: {output}') + self.log.info(f'isNetworkVMExists: Verifying {counter}/5x: {output}') if output: for line in output: diff --git a/CI_Automation/DeployTestContainers.py b/CI_Automation/DeployTestContainers.py index 60cac1035..aa967a7cd 100644 --- a/CI_Automation/DeployTestContainers.py +++ b/CI_Automation/DeployTestContainers.py @@ -43,7 +43,7 @@ def removeAndBuild(self) -> str: if self.isDockerImagesExists(): Utilities.updateStage(self.overallSummaryFile, stage=stage, status='completed', result='passed', threadLock=self.lock) - self.sessionLog.info(f'dentContainerTag: {self.dentContainerTag}') + self.sessionLog.info(f'dentContainerTag: {self.ciVars.dockerImageTag}') return True else: errorMsg = 'Dent test docker images not found' diff --git a/CI_Automation/DownloadBuilds.py b/CI_Automation/DownloadBuilds.py index f02d18a64..7caea84ac 100644 --- a/CI_Automation/DownloadBuilds.py +++ b/CI_Automation/DownloadBuilds.py @@ -23,8 +23,8 @@ def __init__(self, ciVars: object): self.log = ciVars.sessionLog # Every test has its down testId folder to store the builds download + # /DentBuildReleases/06-11-2024-19-34-49-119160_devMode Utilities.runLinuxCmd(f'mkdir -p {self.downloadToServerFolder}', logObj=self.log) - ''' def getPullRequestList(self): """ @@ -217,6 +217,24 @@ def downloadBuilds(self, scrapedBuildList: bool = False): else: return False + def copyLocalBuildsToHttpServerTestIdFolder(self): + print('\ncopyLocalBuildsToHttpServerTestIdFolder') + for srcBuild in self.ciVars.localBuilds: + buildName = srcBuild.split('/')[-1] + downloadToDestPath = f'{self.downloadToServerFolder}/{buildName}' + + doOnce = True + if doOnce: + # buildName: DENTOS-HEAD_ONL-OS10_2023-09-20.1438-a00d7f6_ARM64_INSTALLED_INSTALLER + matchReg = search(r'DENTOS.*_([0-9]+-[0-9]+-[0-9]+)\.([0-9]+)-.*', buildName) + if matchReg: + data = Utilities.readJson(self.ciVars.overallSummaryFile) + data.update({'buildDate': matchReg.group(1), 'buildNumber': matchReg.group(2)}) + Utilities.writeToJson(jsonFile=self.ciVars.overallSummaryFile, data=data, mode='w', threadLock=self.ciVars.lock) + doOnce = False + + Utilities.runLinuxCmd(f'cp {srcBuild} {downloadToDestPath}', logObj=self.log) + def downloadBuilds(ciVars: object) -> bool: """ @@ -228,6 +246,12 @@ def downloadBuilds(ciVars: object) -> bool: downloadBuildsObj = DownloadBuilds(ciVars) srcBuildList = ciVars.builds + if ciVars.localBuilds: + downloadBuildsObj.copyLocalBuildsToHttpServerTestIdFolder() + Utilities.updateStage(ciVars.overallSummaryFile, stage='downloadNewBuilds', + status='completed', result='passed', threadLock=ciVars.lock) + return True + if ciVars.builds == []: # User did not provide builds. Scrape for the latest builds from the main branch in github srcBuildList = downloadBuildsObj.scrapeForLatestBuild(globalSettings.downloadBuildUrlPrefixPath) diff --git a/CI_Automation/TestMgmt.py b/CI_Automation/TestMgmt.py index fae849995..b6c008a25 100644 --- a/CI_Automation/TestMgmt.py +++ b/CI_Automation/TestMgmt.py @@ -57,8 +57,9 @@ def __init__(self, ciVars: object = None): # Testbed reservation self.testbedMgmtFolder = globalSettings.testbedMgmtFolder self.testbedWaitingFolder = globalSettings.testbedMgmtWaitingFolder - # Test session folder: /home/dent/testing + # Test session folder: /home/dent/DentCiMgmt/TestResults/06-14-2024-16-18-45-420609 self.testSessionFolder = ciVars.testSessionFolder + # testIdTestingBranch: /path/DentCiMgmt/TestBranches/06-14-2024-16-18-45-420609_devMode self.testIdTestingBranch = ciVars.testIdTestingBranch # The Linux host logged in user running the test self.user = ciVars.user @@ -71,9 +72,10 @@ def __init__(self, ciVars: object = None): self.overallSummaryFile = ciVars.overallSummaryFile self.reportFile = ciVars.reportFile self.abortTestOnError = ciVars.abortTestOnError - self.overallTestResult = 'Passed' + self.overallTestResult = 'None' self.testIdTestbeds = [] self.setTestcases() + self.isThereAnyResultInReport = None self.log.info(f'RunTest: testSuites={self.testSuites}') self.log.info(f'RunTest: logDir={self.testSessionFolder}') @@ -166,11 +168,13 @@ def report(self) -> str: But within each suite group has many testcases with results. """ testResult = None + + # /path/DentCiMgmt/tools/test_utils.py result = Utilities.runLinuxCmd(f'{sys.executable} {globalSettings.dentToolsPath}/test_utils.py csv -d {self.testSessionFolder}', logObj=self.log) + if result: report = '' - testResult = 'passed' # ['Name,Group,Subgroup,Status,Message', 'test_clean_config,basic_triggers,test_clean_config,pass,""'] for index, line in enumerate(result): if not line: @@ -185,6 +189,7 @@ def report(self) -> str: if len(line) != 5: continue + self.isThereAnyResultInReport = True status = line[3] # pass | failure if status != 'pass': @@ -193,6 +198,13 @@ def report(self) -> str: report += f'{line[0]:25} {line[1]:20} {line[2]:20} {line[3]:20} {line[4]:30}\n' + if self.isThereAnyResultInReport and testResult is None: + testResult = 'passed' + if self.isThereAnyResultInReport and testResult == 'failed': + testResult = 'failed' + if self.isThereAnyResultInReport is None and testResult is None: + testResult = 'none' + self.log.info(f'{report}', noTimestamp=True) Utilities.writeToFile(report, filename=self.reportFile, mode='w', printToStdout=False) @@ -269,23 +281,34 @@ def runTestSuite(self) -> None: testDeltaTime = str((stopTime - startTime)) if aborted: status = 'aborted' - self.overallTestResult = 'unknown' + self.overallTestResult = 'Aborted' else: status = 'completed' - testResult = self.report() + if status == 'completed': + if self.isThereAnyResultInReport and self.overallTestResult not in ['Aborted', 'Failed']: + self.overallTestResult = 'Passed' + + if self.isThereAnyResultInReport and self.overallTestResult in ['Aborted', 'Failed']: + self.overallTestResult = 'Failed' + + if self.isThereAnyResultInReport is None: + self.overallTestResult = 'Failed' + errorMsg = 'There were no test case results hmtl files found in the test branch folder. No result.' + + self.report() Utilities.updateTestMgmtData(self.overallSummaryFile, {'status': status, 'stopTime': stopTime.strftime('%m-%d-%Y %H:%M:%S:%f'), 'testDuration': testDeltaTime, 'error': errorMsg, 'aborted': aborted, - 'result': testResult + 'result': self.overallTestResult }, threadLock=self.lock) Utilities.updateStage(self.overallSummaryFile, stage='runTest', status=status, - result=testResult, error=None, threadLock=self.lock) + result=self.overallTestResult, error=None, threadLock=self.lock) if aborted and self.abortTestOnError: self.killTest(status='AbortTestOnError') @@ -389,7 +412,7 @@ def runSuiteGroup(self, suiteGroup: dict) -> None: Utilities.updateStage(self.overallSummaryFile, stage='runTest', status='error', result='None', error=errorMsg, threadLock=self.lock) self.log.error(errorMsg) - self.overallTestResult = 'unknown' + self.overallTestResult = 'Aborted' print(f'runSuiteGroup error: {errorMsg}') finally: diff --git a/CI_Automation/TestSuites/basic_agg1_testbed.yml b/CI_Automation/TestSuites/basic_agg1_testbed.yml new file mode 100644 index 000000000..a2c585aa0 --- /dev/null +++ b/CI_Automation/TestSuites/basic_agg1_testbed.yml @@ -0,0 +1,6 @@ +suiteGroups: + runInSeries: + - name: suite_group_clean_config_agg1 + config: ./DentOsTestbed/configuration/testbed_config/basic_agg1/testbed.json + suiteGroups: + - suite_group_clean_config diff --git a/CI_Automation/TestSuites/basic_infra1_testbed.yml b/CI_Automation/TestSuites/basic_infra1_testbed.yml new file mode 100644 index 000000000..c5defe648 --- /dev/null +++ b/CI_Automation/TestSuites/basic_infra1_testbed.yml @@ -0,0 +1,6 @@ +suiteGroups: + runInSeries: + - name: suite_group_clean_config_infra1 + config: ./DentOsTestbed/configuration/testbed_config/basic_infra1/testbed.json + suiteGroups: + - suite_group_clean_config diff --git a/CI_Automation/TestSuites/cleanConfig_sit.yml b/CI_Automation/TestSuites/cleanConfig_sit.yml new file mode 100755 index 000000000..7f966caa7 --- /dev/null +++ b/CI_Automation/TestSuites/cleanConfig_sit.yml @@ -0,0 +1,11 @@ +suiteGroups: + runInSeries: + - name: suite_group_clean_config + config: ./DentOsTestbed/configuration/testbed_config/sit_vm/testbed.json + suiteGroups: + - suite_group_clean_config + + #- name: suite_group_functional + # config: ./DentOsTestbed/configuration/testbed_config/sit/testbed.json + # suiteGroups: + # - suite_group_functional diff --git a/CI_Automation/TestSuites/hw/sit/basic_agg1_testbed.yml b/CI_Automation/TestSuites/hw/sit/basic_agg1_testbed.yml new file mode 100644 index 000000000..92497436b --- /dev/null +++ b/CI_Automation/TestSuites/hw/sit/basic_agg1_testbed.yml @@ -0,0 +1,6 @@ +suiteGroups: + runInSeries: + - name: suite_group_clean_config_agg1 + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/basic_agg1/testbed.json + suiteGroups: + - suite_group_clean_config diff --git a/CI_Automation/TestSuites/hw/sit/basic_infra1_testbed.yml b/CI_Automation/TestSuites/hw/sit/basic_infra1_testbed.yml new file mode 100644 index 000000000..7eb410724 --- /dev/null +++ b/CI_Automation/TestSuites/hw/sit/basic_infra1_testbed.yml @@ -0,0 +1,6 @@ +suiteGroups: + runInSeries: + - name: suite_group_clean_config_infra1 + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/basic_infra1/testbed.json + suiteGroups: + - suite_group_clean_config diff --git a/CI_Automation/TestSuites/hw/sit/cleanConfig.yml b/CI_Automation/TestSuites/hw/sit/cleanConfig.yml new file mode 100644 index 000000000..eddf2e8fe --- /dev/null +++ b/CI_Automation/TestSuites/hw/sit/cleanConfig.yml @@ -0,0 +1,6 @@ +suiteGroups: + runInSeries: + - name: suite_group_clean_config + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_clean_config diff --git a/CI_Automation/TestSuites/hw/sit/fullRegression.yml b/CI_Automation/TestSuites/hw/sit/fullRegression.yml new file mode 100644 index 000000000..4a208e02c --- /dev/null +++ b/CI_Automation/TestSuites/hw/sit/fullRegression.yml @@ -0,0 +1,105 @@ +suiteGroups: + runInSeries: + - name: suite_group_functional + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/basic_infra1/testbed.json + suiteGroups: + - suite_group_functional + + - name: clean_config_for_sit + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_clean_config + + - name: suite_group_test + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_test + + - name: suite_group_l3_tests + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_l3_tests + + - name: suite_group_basic_trigger_tests + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_basic_trigger_tests + + - name: suite_group_traffic_tests + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_traffic_tests + + - name: suite_group_tc_tests + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_tc_tests + + - name: suite_group_bgp_tests + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_bgp_tests + + - name: suite_group_system_wide_testing + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_system_wide_testing + + - name: suite_group_system_health + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_system_health + + - name: suite_group_system_health + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_system_health + + - name: suite_group_store_bringup + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_store_bringup + + - name: suite_group_alpha_lab_testing + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_alpha_lab_testing + + - name: suite_group_dentv2_testing + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_dentv2_testing + + - name: suite_group_connection + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_connection + + - name: suite_group_platform + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_platform + + - name: suite_group_stress_tests + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_stress_tests + +# - name: dentos_testbed_runtests +# config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json +# suiteGroups: +# - suite_group_test +# - suite_group_l3_tests +# - suite_group_basic_trigger_tests +# - suite_group_traffic_tests +# - suite_group_tc_tests +# - suite_group_bgp_tests +# - suite_group_system_wide_testing +# - suite_group_system_health +# - suite_group_store_bringup +# - suite_group_alpha_lab_testing +# - suite_group_dentv2_testing +# - suite_group_connection +# - suite_group_platform +# - suite_group_stress_tests +# - suite_group_functional diff --git a/CI_Automation/TestSuites/hw/sit/functional.yml b/CI_Automation/TestSuites/hw/sit/functional.yml new file mode 100644 index 000000000..8cd3dfe98 --- /dev/null +++ b/CI_Automation/TestSuites/hw/sit/functional.yml @@ -0,0 +1,11 @@ +suiteGroups: + runInSeries: + #- name: suite_group_clean_config + # config: ./DentOsTestbed/configuration/testbed_config/hw/sit/basic_infra1/testbed.json + # suiteGroups: + # - suite_group_clean_config + + - name: suite_group_functional + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/basic_infra1/testbed.json + suiteGroups: + - suite_group_functional diff --git a/CI_Automation/TestSuites/hw/sit/l3tests.yml b/CI_Automation/TestSuites/hw/sit/l3tests.yml new file mode 100644 index 000000000..1d5459118 --- /dev/null +++ b/CI_Automation/TestSuites/hw/sit/l3tests.yml @@ -0,0 +1,11 @@ +suiteGroups: + runInSeries: + - name: clean_config_for_sit + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_clean_config + + - name: suite_group_l3_tests + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_l3_tests diff --git a/CI_Automation/TestSuites/hw/sit/seriesAndParallelCleanConfig.yml b/CI_Automation/TestSuites/hw/sit/seriesAndParallelCleanConfig.yml new file mode 100644 index 000000000..5880d30ed --- /dev/null +++ b/CI_Automation/TestSuites/hw/sit/seriesAndParallelCleanConfig.yml @@ -0,0 +1,29 @@ +suiteGroups: + runInParallel: + - name: clean_config_infra1 + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/basic_infra1/testbed.json + suiteGroups: + - suite_group_clean_config + + - name: clean_config_infra2 + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/basic_infra2/testbed.json + suiteGroups: + - suite_group_clean_config + + #- name: suite_group_test + # config: ./DentOsTestbed/configuration/testbed_config/hw/sit/basic_infra1/testbed.json + # suiteGroups: + # - suite_group_test + # additionalParams: + # - -k "ipv4" --discovery-path ./DentOsTestbedLib/src/dent_os_testbed/discovery/modules/ + + runInSeries: + - name: clean_config_agg1 + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/basic_agg1/testbed.json + suiteGroups: + - suite_group_clean_config + + - name: clean_config_agg2 + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/basic_agg2/testbed.json + suiteGroups: + - suite_group_clean_config diff --git a/CI_Automation/TestSuites/hw/sit/sitTest.yml b/CI_Automation/TestSuites/hw/sit/sitTest.yml new file mode 100644 index 000000000..a433188dd --- /dev/null +++ b/CI_Automation/TestSuites/hw/sit/sitTest.yml @@ -0,0 +1,100 @@ +suiteGroups: + runInSeries: + - name: clean_config_for_sit + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_clean_config + + - name: suite_group_test + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_test + + - name: suite_group_l3_tests + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_l3_tests + + - name: suite_group_basic_trigger_tests + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_basic_trigger_tests + + - name: suite_group_traffic_tests + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_traffic_tests + + - name: suite_group_tc_tests + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_tc_tests + + - name: suite_group_bgp_tests + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_bgp_tests + + - name: suite_group_system_wide_testing + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_system_wide_testing + + - name: suite_group_system_health + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_system_health + + - name: suite_group_system_health + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_system_health + + - name: suite_group_store_bringup + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_store_bringup + + - name: suite_group_alpha_lab_testing + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_alpha_lab_testing + + - name: suite_group_dentv2_testing + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_dentv2_testing + + - name: suite_group_connection + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_connection + + - name: suite_group_platform + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_platform + + - name: suite_group_stress_tests + config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json + suiteGroups: + - suite_group_stress_tests + +# - name: dentos_testbed_runtests +# config: ./DentOsTestbed/configuration/testbed_config/hw/sit/testbed.json +# suiteGroups: +# - suite_group_test +# - suite_group_l3_tests +# - suite_group_basic_trigger_tests +# - suite_group_traffic_tests +# - suite_group_tc_tests +# - suite_group_bgp_tests +# - suite_group_system_wide_testing +# - suite_group_system_health +# - suite_group_store_bringup +# - suite_group_alpha_lab_testing +# - suite_group_dentv2_testing +# - suite_group_connection +# - suite_group_platform +# - suite_group_stress_tests +# - suite_group_functional diff --git a/CI_Automation/TestSuites/l3tests_sit.yml b/CI_Automation/TestSuites/l3tests_sit.yml new file mode 100755 index 000000000..b3ebed792 --- /dev/null +++ b/CI_Automation/TestSuites/l3tests_sit.yml @@ -0,0 +1,11 @@ +suiteGroups: + runInSeries: + - name: clean_config_for_sit + config: ./DentOsTestbed/configuration/testbed_config/sit_vm/testbed.json + suiteGroups: + - suite_group_clean_config + + - name: suite_group_l3_tests + config: ./DentOsTestbed/configuration/testbed_config/sit_vm/testbed.json + suiteGroups: + - suite_group_l3_tests diff --git a/CI_Automation/TestSuites/vm/sit/cleanConfig.yml b/CI_Automation/TestSuites/vm/sit/cleanConfig.yml new file mode 100755 index 000000000..b1ba9155a --- /dev/null +++ b/CI_Automation/TestSuites/vm/sit/cleanConfig.yml @@ -0,0 +1,11 @@ +suiteGroups: + runInSeries: + - name: suite_group_clean_config + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_clean_config + + #- name: suite_group_functional + # config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + # suiteGroups: + # - suite_group_functional diff --git a/CI_Automation/TestSuites/vm/sit/fullRegression.yml b/CI_Automation/TestSuites/vm/sit/fullRegression.yml new file mode 100755 index 000000000..8b1efdecc --- /dev/null +++ b/CI_Automation/TestSuites/vm/sit/fullRegression.yml @@ -0,0 +1,105 @@ +suiteGroups: + runInSeries: + - name: suite_group_functional + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_functional + + - name: clean_config_for_sit + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_clean_config + + - name: suite_group_test + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_test + + - name: suite_group_l3_tests + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_l3_tests + + - name: suite_group_basic_trigger_tests + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_basic_trigger_tests + + - name: suite_group_traffic_tests + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_traffic_tests + + - name: suite_group_tc_tests + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_tc_tests + + - name: suite_group_bgp_tests + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_bgp_tests + + - name: suite_group_system_wide_testing + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_system_wide_testing + + - name: suite_group_system_health + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_system_health + + - name: suite_group_system_health + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_system_health + + - name: suite_group_store_bringup + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_store_bringup + + - name: suite_group_alpha_lab_testing + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_alpha_lab_testing + + - name: suite_group_dentv2_testing + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_dentv2_testing + + - name: suite_group_connection + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_connection + + - name: suite_group_platform + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_platform + + - name: suite_group_stress_tests + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_stress_tests + +# - name: dentos_testbed_runtests +# config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json +# suiteGroups: +# - suite_group_test +# - suite_group_l3_tests +# - suite_group_basic_trigger_tests +# - suite_group_traffic_tests +# - suite_group_tc_tests +# - suite_group_bgp_tests +# - suite_group_system_wide_testing +# - suite_group_system_health +# - suite_group_store_bringup +# - suite_group_alpha_lab_testing +# - suite_group_dentv2_testing +# - suite_group_connection +# - suite_group_platform +# - suite_group_stress_tests +# - suite_group_functional diff --git a/CI_Automation/TestSuites/vm/sit/functional.yml b/CI_Automation/TestSuites/vm/sit/functional.yml new file mode 100755 index 000000000..f0b89e080 --- /dev/null +++ b/CI_Automation/TestSuites/vm/sit/functional.yml @@ -0,0 +1,11 @@ +suiteGroups: + runInSeries: + #- name: suite_group_clean_config + # config: ./DentOsTestbed/configuration/testbed_config/vm/sit//testbed.json + # suiteGroups: + # - suite_group_clean_config + + - name: suite_group_functional + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_functional diff --git a/CI_Automation/TestSuites/vm/sit/l3tests.yml b/CI_Automation/TestSuites/vm/sit/l3tests.yml new file mode 100755 index 000000000..5c9c23ea2 --- /dev/null +++ b/CI_Automation/TestSuites/vm/sit/l3tests.yml @@ -0,0 +1,11 @@ +suiteGroups: + runInSeries: + - name: clean_config_for_sit + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_clean_config + + - name: suite_group_l3_tests + config: ./DentOsTestbed/configuration/testbed_config/vm/sit/testbed.json + suiteGroups: + - suite_group_l3_tests diff --git a/CI_Automation/Utilities.py b/CI_Automation/Utilities.py index bbfa186c1..ae323348d 100644 --- a/CI_Automation/Utilities.py +++ b/CI_Automation/Utilities.py @@ -11,6 +11,7 @@ from shutil import rmtree from glob import glob from re import search, findall +import socket def getTimestamp(includeMillisecond: bool = True) -> str: @@ -222,16 +223,16 @@ def generatorRandom(size=6, chars=string.ascii_uppercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) -def sysExit(ciVars: object, errorMsg: str = ''): +def runDentCiTearDown(ciVars: object, errorMsg: str = ''): # In case the testID test branch did not get removed if ciVars.deleteTestBranchAfterTest and os.path.exists(ciVars.testIdTestingBranch): try: - ciVars.sessionLog.info(f'sysExit: Removing testId test branch: {ciVars.testIdTestingBranch}') + ciVars.sessionLog.info(f'runDentCiTearDown: Removing testId test branch: {ciVars.testIdTestingBranch}') rmtree(ciVars.testIdTestingBranch) except Exeption: pass - ciVars.sessionLog.info(f'sysExit: removeDockerImage: {ciVars.dockerImageTag}') + ciVars.sessionLog.info(f'runDentCiTearDown: removeDockerImage: {ciVars.dockerImageTag}') removeDockerImage(ciVars.dockerImageTag, ciVars.sessionLog) if ciVars.ciObj: @@ -251,9 +252,6 @@ def sysExit(ciVars: object, errorMsg: str = ''): else: ciVars.exitCode = 0 - # This will stop the test from running. It won't exit the main script. - sys.exit() - def closeTestMgmtStatus(overallSummaryFile: str = None, status: str = None, result: str = None, updateProperties: dict = None, @@ -426,6 +424,7 @@ def removeTestingRepo(path): Remove the testing branch """ if os.path.exists(path): + print(f'\nremoveTestingRepo: {path}') rmtree(path) @@ -552,12 +551,14 @@ def getTestbeds(ciVars): # /testbed_config/basic_infra1/testbed.json config = test['config'].split('configuration')[-1] - regexMatch = search('testbed_config/(.+)/testbed.json', config) + regexMatch = search('testbed_config/(.+)', config) if regexMatch is False: ciVars.sessionLog.error(f'getTestbeds: Expecting config pattern /testbed_config//testbed.json, but got pattern: {config}') - sysExit(ciVars, errorMsg=True) + runDentCiTearDown(ciVars, errorMsg=True) - testbedName = regexMatch.group(1) + # lockTestbed touches a testbed file that begins with sit/ (slash), but touch + # errors out because it see's slashes as paths. Need to replace slashes with dashes + testbedName = regexMatch.group(1).replace('/', '-') # Get a list of all the testId testbeds so the framework knows which testbed to unlock testbedNameWithTestId = f'{testbedName.split(".json")[0]}_{ciVars.testId}' @@ -657,6 +658,26 @@ def removeDockerImage(dockerTag: str, logObj: object): return False +def isReachable(ipOrName, port, timeout=3): + """ + If your server does not support ICMP (firewall might block it), + it most probably still offers a service on a TCP port. + In this case, you can perform a TCP ping (platform independently and + without installing additional python modules) like this. + """ + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(timeout) + + try: + s.connect((ipOrName, int(port))) + s.shutdown(socket.SHUT_RDWR) + return True + except Exception: + return False + finally: + s.close() + + class CreateLogObj: """ An object class to log various types of logs: diff --git a/CI_Automation/ciUtils.py b/CI_Automation/ciUtils.py index 3f5419199..075b488c9 100755 --- a/CI_Automation/ciUtils.py +++ b/CI_Automation/ciUtils.py @@ -11,6 +11,7 @@ - Support manual testbed locking """ +from tabulate import tabulate from typing import Union import argparse import os @@ -40,11 +41,24 @@ args: object = parser.parse_args() +def createTestResultFolder(): + if os.path.exists(globalSettings.dentTestResultsFolder) is False: + Utilities.makeFolder(globalSettings.dentTestResultsFolder) + + +def removeFolder(testIdFullPath, testIdName): + try: + rmtree(testIdFullPath) + except Exception as errMsg: + print(f'Failed to remove testId: {testIdName}. Error: {errMsg}') + + class ShowTests: """ Show all the test IDs """ def show(self): + createTestResultFolder() Utilities.runLinuxCmd(f'ls {globalSettings.dentTestResultsFolder}', showStdout=True) @@ -92,6 +106,9 @@ def show(self, testId: str = None) -> None: buildNumber = testIdDetails['buildNumber'] testbeds = testIdDetails['testbeds'] dockerImageTag = testIdDetails['dockerImageTag'] + startTime = testIdDetails['startTime'] + stopTime = testIdDetails['stopTime'] + testDuration = testIdDetails['testDuration'] testSuites = [] for testSuite in testIdDetails['testSuites']: regexMatch = search('.*/TestSuites/(.+)', testSuite) @@ -103,6 +120,8 @@ def show(self, testId: str = None) -> None: print(f'\tBuild Release: {buildDate}-{buildNumber}') print(f'\tTestSuites: {", ".join(testSuites)}') print(f'\tDocker Image Tag: {dockerImageTag}') + print(f'\tStartTime: {startTime} StopTime: {stopTime}') + print(f'\tTestDuration: {testDuration}') testbedlist = '' for index, testbed in enumerate(testbeds): # basic_infra1_10-06-2023-17-50-02-010517_fullRegression @@ -111,23 +130,35 @@ def show(self, testId: str = None) -> None: testbedlist = testbedlist.replace(' ', ', ')[:-2] print(f'\tTestbeds: {testbedlist}') - print() - print(f' {"Stage":10} {"Stage Name":27} {"Status":12} {"Result":10} Errors') - print(f' {"-"*69}') + stageRow = ['Stage', 'Stage Name', 'Status', 'Result', 'Errors'] + stageTable = [stageRow] + print() + stageData = [] for stage in testIdDetails['stageOrder']: - stageNumber = stage[0] + # stage = ['stage 1', 'cloneTestRepo'] + stageNumber = stage[0].split(' ')[1] stageName = stage[1] status = testIdDetails['stages'][stageName]['status'] result = testIdDetails['stages'][stageName]['result'] error = testIdDetails['stages'][stageName]['error'] - print(f' {stageNumber:10} {stageName:27} {str(status):12} {str(result):10} {str(error)}') + if len(error) > 0: + allErrors = '' + for eachError in error: + allErrors += f'{eachError}\n' + else: + allErrors = [] + + stageData = [stageNumber, stageName, status, result, allErrors] + stageTable.append(stageData) + print(tabulate(stageTable, headers='firstrow', tablefmt='fancy_grid', numalign='center')) print() - print(f' {"Suite Group":30} {"Testbed":30} {"Test Conduct":15} {"Status":20}') - print(f' {"-"*88}') testcases = testIdDetails['testcases'] + testRow = ['Suite Group', 'Testbed', 'Test Conduct', 'Status'] + testTable = [testRow] + testData = [] for index, testcase in enumerate(testcases): testbed = str(testIdDetails['testcases'][testcase]['testbed']) @@ -139,13 +170,10 @@ def show(self, testId: str = None) -> None: if status == 'Aborted': result = 'None' - if index == 0: - print(f' {testcase:30} {testbed:30} {testConduct:15} {status:20}') - else: - if status == 'Running': - print(f' {testcase:30} {testbed:30} {testConduct:15} \033[32;5m{status:20}\033[0m') - else: - print(f' {testcase:30} {testbed:30} {testConduct:15} {status:20}') + testData = [testcase, testbed, testConduct, status] + testTable.append(testData) + + print(tabulate(testTable, headers='firstrow', tablefmt='fancy_grid')) class ShowTestbeds: @@ -279,23 +307,20 @@ def removeTestId(self, testIds: list = []) -> None: ciSummaryFile = f'{testIdFullPath}/ciOverallSummary.json' if os.path.exists(ciSummaryFile): - testIdData = Utilities.readJson(ciSummaryFile) - testIdPid = str(testIdData['pid']) - - if testIdPid not in currentlyRunningPids: - print(f'Removing: {testIdName}') - - try: - rmtree(testIdFullPath) - - except Exception as errMsg: - print(f'Failed to remove testId: {testIdName}. Error: {errMsg}') + try: + testIdData = Utilities.readJson(ciSummaryFile) + testIdPid = str(testIdData['pid']) + if testIdPid not in currentlyRunningPids: + print(f'Removing: {testIdName}') + removeFolder(testIdFullPath) + except Exception: + removeFolder(testIdFullPath, testIdName) else: # The testId has no ciOverallSummary file. Just remove it. print(f'Removing stale testId result that does not have a ciOverallSummary.json file: {testIdName}') try: print(f'Removing: {testIdFullPath}') - rmtree(testIdFullPath) + rmtree(testIdFullPath, testIdName) except Exception as errMsg: print(f'Failed to remove testId: {testIdName}. Error: {errMsg}') diff --git a/CI_Automation/runDentCi.py b/CI_Automation/runDentCi.py index 2685faef5..2fef238f5 100755 --- a/CI_Automation/runDentCi.py +++ b/CI_Automation/runDentCi.py @@ -15,18 +15,20 @@ For Jenkins CI, must include -testName that begins with jenkinsCI_