From 137ee923c626afdce5f21174b7ae199dc8005aaf Mon Sep 17 00:00:00 2001 From: mkavulich Date: Thu, 3 Nov 2016 15:32:15 -0600 Subject: [PATCH] Improvements for run_from_github.py: - Allow command-line specification for url and branch - Strip whitespace around arguments - Return fatal for whitespace within url or branch name - Return fatal if URL appears to be invalid (e.g. linking directly to branch URL) - Add usage function with exit code argument --- run_from_github.py | 116 ++++++++++++++++++++++++++++++++------------- 1 file changed, 82 insertions(+), 34 deletions(-) diff --git a/run_from_github.py b/run_from_github.py index 012bef0..8831d0c 100755 --- a/run_from_github.py +++ b/run_from_github.py @@ -6,64 +6,112 @@ # Usage: ./run_from_github.py # When prompted, enter a repository URL. # When prompted, enter a branch name. +# Alternatively, you can also enter the "--url" and/or "--branch" options via command line arguments # # Author: Michael Kavulich, Jr. (September 2016) +# Update history: +# November 2016: Added command-line arguments for url and branch # import os +import re +import sys import shutil import tarfile -url = raw_input('Enter github URL (leave blank for https://github.com/wrf-model/WRF): ') -branch = raw_input('Enter branch name (leave blank for master): ') -if not url: - url = "https://github.com/wrf-model/WRF" +def usage(exit_code=0): #If no exit code is specified, this indicates successful execution, so return exit code "0" per convention -#In the future we hope to run tests from multiple forks, so we should disambiguate the tar files by adding the fork name -urlsplit = url.split("/") -if "http" in urlsplit[0]: - fork = urlsplit[3] -else: # Assume if it is not http then user is using ssh - urlsplitagain = urlsplit[0].split(":") - fork = urlsplitagain[1] + print("usage: " + __file__ + " --url=https://github.com/wrf-model/WRF --branch=branch_name") + print("") + print(" " + __file__ + " url : The main URL of the Github repository you wish to test (e.g. https://github.com/username/WRF for your fork)") + print(" " + __file__ + " fail : The name of the branch you wish to test") + sys.exit(exit_code) -if not branch: - branch = "master" -print "Repository URL is %s." % url -if "wrf-model" not in fork: - print "Fork is %s." % fork -print "Branch name is %s." % branch +def main(): -os.chdir("tarballs") + if len(sys.argv) > 1: + for arg in sys.argv[1:]: + if "--url=" in arg: + url=arg.split("=",1)[1] + elif "--branch=" in arg: + branch=arg.split("=",1)[1] + elif ("--help" in arg) or ("-h" in arg): + usage() + else: + print("Unrecognized option: " + arg) + usage(1) -if os.path.isdir("WRFV3"): - shutil.rmtree("WRFV3") + url = raw_input('Enter github URL (leave blank for https://github.com/wrf-model/WRF): ') + branch = raw_input('Enter branch name (leave blank for master): ') -os.system("git clone " + url + " WRFV3") -os.chdir("WRFV3") -os.system("git checkout " + branch) -os.chdir("../") + url = url.strip() + branch = branch.strip() + spaces = re.compile(r'\s') + if (re.search(spaces, url) is not None): + print("FATAL:\nInvalid URL: contains whitespace character(s)") + sys.exit(2) + elif (re.search(spaces, branch) is not None): + print("FATAL:\nInvalid branch name: contains whitespace character(s)") + sys.exit(3) + if "/tree/" in url: + print("The URL you typed appears to be a direct link to a branch rather than the top-level URL of a repository. \nYour URL should probably be in the format 'https://github.com/username/WRF/'.") + sys.exit(4) + if not url: + url = "https://github.com/wrf-model/WRF" -out = tarfile.open("github_" + fork + "_" + branch + ".tar", mode='w') -try: + #In the future we hope to run tests from multiple forks, so we should disambiguate the tar files by adding the fork name + urlsplit = url.split("/") + if "http" in urlsplit[0]: + fork = urlsplit[3] + else: # Assume if it is not http then user is using ssh + urlsplitagain = urlsplit[0].split(":") + fork = urlsplitagain[1] + + if not branch: + branch = "master" + + print "Repository URL is %s." % url + if "wrf-model" not in fork: + print "Fork is %s." % fork + print "Branch name is %s." % branch + + os.chdir("tarballs") + + if os.path.isdir("WRFV3"): + shutil.rmtree("WRFV3") + + os.system("git clone " + url + " WRFV3") + os.chdir("WRFV3") + os.system("git checkout " + branch) + os.chdir("../") + + + + out = tarfile.open("github_" + fork + "_" + branch + ".tar", mode='w') + try: out.add('WRFV3') # Adding "WRFV3" directory to tar file -finally: + finally: out.close() # Close tar file -os.chdir("../") + os.chdir("../") + + user = os.environ.get("USER") + + del(os.environ["MP_PE_AFFINITY"]) + if not os.path.isdir("/glade/scratch/" + user + "/TMPDIR_FOR_PGI_COMPILE"): + os.makedirs("/glade/scratch/" + user + "/TMPDIR_FOR_PGI_COMPILE") + os.environ["TMPDIR"] = "/glade/scratch/" + user + "/TMPDIR_FOR_PGI_COMPILE" + -user = os.environ.get("USER") + os.system('bsub < bsub_this_to_run_all_compilers.csh') -del(os.environ["MP_PE_AFFINITY"]) -if not os.path.isdir("/glade/scratch/" + user + "/TMPDIR_FOR_PGI_COMPILE"): - os.makedirs("/glade/scratch/" + user + "/TMPDIR_FOR_PGI_COMPILE") -os.environ["TMPDIR"] = "/glade/scratch/" + user + "/TMPDIR_FOR_PGI_COMPILE" +if __name__ == "__main__": + main() -os.system('bsub < bsub_this_to_run_all_compilers.csh')