diff --git a/Releaser.py b/Releaser.py index 38e6bd0..45c168f 100644 --- a/Releaser.py +++ b/Releaser.py @@ -16,6 +16,8 @@ from site_utils import * from version_utils import * +defaultPackageTypeToModule = True + def makeDirsWritable( dirPathTop ): userId = os.geteuid() for dirPath, dirs, files in os.walk(dirPathTop): @@ -458,24 +460,6 @@ def InstallPackage( self, installTop=None, force=False, rmFailed=False ): print("InstallPackage Error: Need valid installTop to determine installDir!") return status - # Is Package a module? - if os.path.split( self._packagePath )[0] == 'modules' \ - or self._repo.GetUrl().find('modules') >= 0: - # Package is a module - if installTop is not None: - if not installTop.endswith( '/' + self._packageName ): - if not installTop.endswith( '/modules' ): - installTop += '/modules' - if not os.path.isdir( installTop ): - print("Invalid top %s" % installTop) - installTop = None - if installTop is None: - epics_modules_top = determine_epics_modules_top() - if not epics_modules_top: - print("InstallPackage Error: Unable to determine EPICS modules installTop!") - return status - installTop = epics_modules_top - # Is Package an extension? if not installTop: if self._packagePath.find('extensions') >= 0 \ @@ -485,8 +469,9 @@ def InstallPackage( self, installTop=None, force=False, rmFailed=False ): # Is Package an IOC? if installTop is None \ and ( os.path.split( self._packagePath )[0].startswith('ioc') \ - or self._repo.GetUrl().find('ioc') >= 0 ): + or self._repo.GetUrl().find('ioc-') >= 0 ): # Package is an IOC + # TODO: If github repo uses ioc-foo-bar, we need to use installDir ioc/foo/bar topVariants = [ epics_site_top ] for topVariant in defEpicsTopVariants: topVariants.append( os.path.join( epics_site_top, topVariant ) ) @@ -496,6 +481,26 @@ def InstallPackage( self, installTop=None, force=False, rmFailed=False ): installTop = os.path.join( topVariant, self._packagePath ) if not os.path.isdir( installTop ): os.makedirs( installTop, 0o775 ) + break + + # Is Package a module? + if os.path.split( self._packagePath )[0] == 'modules' \ + or defaultPackageTypeToModule \ + or self._repo.GetUrl().find('modules') >= 0: + # Package is a module + if installTop is not None: + if not installTop.endswith( '/' + self._packageName ): + if not installTop.endswith( '/modules' ): + installTop += '/modules' + if not os.path.isdir( installTop ): + print("Invalid top %s" % installTop) + installTop = None + if installTop is None: + epics_modules_top = determine_epics_modules_top() + if not epics_modules_top: + print("InstallPackage Error: Unable to determine EPICS modules installTop!") + return status + installTop = epics_modules_top if not installTop: print("InstallPackage Error: Unable to determine installTop!") diff --git a/gitRepo.py b/gitRepo.py index 911d222..02060c7 100644 --- a/gitRepo.py +++ b/gitRepo.py @@ -57,12 +57,14 @@ def CheckoutRelease( self, buildDir, verbose=True, quiet=False, dryRun=False, de if len(gitOutput) == 1: curSha = gitOutput[0] - # Get the tag SHA + # Get the commit for the tag tagSha = None - cmdList = [ "git", "rev-parse", self._tag ] + cmdList = [ "git", "show-ref", "-d", self._tag ] gitOutput = subprocess.check_output( cmdList, universal_newlines=True ).splitlines() - if len(gitOutput) == 1: - tagSha = gitOutput[0] + if len(gitOutput) == 2: + tag_commit_info = gitOutput[1].split() + if len(tag_commit_info) == 2: + tagSha = tag_commit_info[0] # If they match, it's already checked out! if curSha == tagSha: @@ -111,8 +113,8 @@ def CheckoutRelease( self, buildDir, verbose=True, quiet=False, dryRun=False, de # Refresh the tags # TODO: May fail if git repo is read-only if verbose: - print("CheckoutRelease running: git fetch origin refs/tags/%s" % self._tag) - cmdList = [ "git", "fetch", "origin", "refs/tags/" + self._tag ] + print("CheckoutRelease running: git fetch %s refs/tags/%s" % (self._url, self._tag)) + cmdList = [ "git", "fetch", self._url, "refs/tags/" + self._tag ] subprocess.check_call( cmdList, stdout=outputPipe, stderr=outputPipe ) tagSha = gitGetTagSha( self._tag ) @@ -165,7 +167,7 @@ def RemoveTag( self, package=None, tag=None, verbose=True, dryRun=False ): if verbose: print("\nRemoving %s release tag %s ..." % ( package, tag )) subprocess.check_call( [ "git", "tag", "-d", tag ] ) - subprocess.check_call( [ 'git', 'push', '--delete', 'origin', tag ] ) + subprocess.check_call( [ 'git', 'push', '--delete', self._url, tag ] ) print("Successfully removed %s release tag %s." % ( package, tag )) def PushBranch( self, branchName=None, verbose=True, dryRun=False ): @@ -181,7 +183,7 @@ def PushBranch( self, branchName=None, verbose=True, dryRun=False ): return if verbose: print("Pushing branch %s ..." % ( branchName )) - subprocess.check_call( [ 'git', 'push', 'origin', branchName ] ) + subprocess.check_call( [ 'git', 'push', self._url, branchName ] ) def PushTag( self, release, verbose=True, dryRun=False ): if dryRun: @@ -190,7 +192,7 @@ def PushTag( self, release, verbose=True, dryRun=False ): if verbose: print("Pushing tag %s ..." % ( release )) - subprocess.check_call( [ 'git', 'push', 'origin', release ] ) + subprocess.check_call( [ 'git', 'push', self._url, release ] ) def TagRelease( self, packagePath=None, release=None, branch=None, message="", verbose=True, dryRun=False ): if release is None: @@ -204,6 +206,6 @@ def TagRelease( self, packagePath=None, release=None, branch=None, message="", v comment = "Release %s/%s: %s" % ( packagePath, release, message ) cmdList = [ "git", "tag", release, 'HEAD', "-m", comment ] subprocess.check_call( cmdList ) - subprocess.check_call( [ 'git', 'push', '-u', 'origin' ] ) - subprocess.check_call( [ 'git', 'push', 'origin', release ] ) + subprocess.check_call( [ 'git', 'push', '-u', self._url, ] ) + subprocess.check_call( [ 'git', 'push', self._url, release ] ) diff --git a/git_utils.py b/git_utils.py index 4b7910d..1e3d6ee 100644 --- a/git_utils.py +++ b/git_utils.py @@ -52,6 +52,8 @@ def parseGitModulesTxt(): def determineGitRoot( ): '''Get the root folder for GIT repos at SLAC''' gitRoot = DEF_AFS_GIT_REPOS + if not os.path.exists( gitRoot ): + gitRoot = DEF_GITHUB_REPOS # The GIT_REPO_ROOT variable is mainly used when testing eco and is not something that we really expect from the environment. if "GIT_REPO_ROOT" in os.environ: gitRoot = os.environ["GIT_REPO_ROOT"] @@ -389,6 +391,8 @@ def gitGetWorkingBranch( debug = False, verbose = False ): if debug: print(e) pass + if verbose: + print( "gitGetWorkingBranch: url=%s, branch=%s, tag=%s" % ( repo_url, repo_branch if repo_branch else 'None', repo_tag if repo_tag else 'None' ) ) return ( repo_url, repo_branch, repo_tag ) def determinePathToGitRepo( packagePath, verbose = False ): @@ -411,6 +415,19 @@ def determinePathToGitRepo( packagePath, verbose = False ): # Check under the root of the git repo area for a bare repo w/ the right name gitRoot = determineGitRoot() gitPackageDir = packageName + ".git" + if verbose: + print( "determinePathToGitRepo: gitRoot is %s" % gitRoot if gitRoot else None ) + + # Check github DEF_GITHUB_REPOS, just add the package name + defRepoPath = DEF_GITHUB_REPOS + '/' + gitPackageDir + if 'github.com' in gitRoot: + # This results in an extra github API access, but is needed to test if the package is hosted on github + repoTags = gitGetRemoteTags( defRepoPath, verbose=verbose ) + if len(repoTags) > 0: + if verbose: + print( "determinePathToGitRepo: github %s repo is %s" % (packagePath, defRepoPath) ) + return defRepoPath + gitPackagePath = packagePath + ".git" if not os.path.isdir( DEF_CVS_ROOT ) and gitRoot: # Must be offsite, assume gitRoot and an EPICS module path @@ -453,12 +470,7 @@ def gitFindPackageRelease( packageSpec, tag, debug = False, verbose = False ): print("gitFindPackageRelease: packageName=%s, packagePath=%s" % ( packageName, packagePath )) # See if the package was listed in $TOOLS/eco_modulelist/modulelist.txt - if packageName in git_package2Location: - url_path = determinePathToGitRepo( packageName, verbose=verbose ) - (repo_sha, repo_tag) = gitGetRemoteTag( url_path, tag, verbose=verbose ) - if repo_sha: - repo_url = url_path - else: + if not packageName in git_package2Location and not 'github.com' in DEF_GIT_REPO_PATH: for url_root in [ DEF_GIT_MODULES_PATH, DEF_GIT_EXTENSIONS_PATH, DEF_GIT_EPICS_PATH, DEF_GIT_REPO_PATH ]: if repo_url is not None: break @@ -471,9 +483,26 @@ def gitFindPackageRelease( packageSpec, tag, debug = False, verbose = False ): if packageName == packagePath: break + if not repo_url: + # Try our github repos + url_path = DEF_GITHUB_REPOS + '/' + packageName + ".git" + (repo_sha, repo_tag) = gitGetRemoteTag( url_path, tag, verbose=verbose ) + if repo_sha: + repo_url = url_path + + if not repo_url: + # Try full set of alternatives + url_path = determinePathToGitRepo( packageName, verbose=verbose ) + if url_path: + (repo_sha, repo_tag) = gitGetRemoteTag( url_path, tag, verbose=verbose ) + if repo_sha: + repo_url = url_path + if verbose: if repo_url: print("gitFindPackageRelease found %s/%s: url=%s, tag=%s" % ( packagePath, tag, repo_url, repo_tag )) + elif url_path is None: + print("gitFindPackageRelease Error: Cannot determine URL for repo %s/%s" % (packagePath, tag)) else: print("gitFindPackageRelease Error: Cannot find %s/%s" % (packagePath, tag)) return (repo_url, repo_tag) diff --git a/repo_defaults.py b/repo_defaults.py index d3aad0a..6d7d808 100644 --- a/repo_defaults.py +++ b/repo_defaults.py @@ -28,6 +28,9 @@ DEF_AFS_GIT_REPOS = "/afs/slac.stanford.edu/g/cd/swe/git/repos" DEF_AFS_GIT_REPOS2 = "/afs/slac/g/cd/swe/git/repos" +# Github Organitions +DEF_GITHUB_REPOS = "git@github.com:slac-epics" + DEF_GIT_EXT_TOP_TAG = "slac-trunk" DEF_GIT_RELEASE_DEPTH = 10