From f08a6e799b953ff23f577cefc17f69153412f95f Mon Sep 17 00:00:00 2001 From: Bruce Hill Date: Fri, 28 Feb 2025 05:00:57 -0800 Subject: [PATCH 1/6] Added DEF_GITHUB_REPOS to repo_defaults.py --- repo_defaults.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/repo_defaults.py b/repo_defaults.py index d3aad0a..6d7d808 100644 --- a/repo_defaults.py +++ b/repo_defaults.py @@ -28,6 +28,9 @@ DEF_AFS_GIT_REPOS = "/afs/slac.stanford.edu/g/cd/swe/git/repos" DEF_AFS_GIT_REPOS2 = "/afs/slac/g/cd/swe/git/repos" +# Github Organitions +DEF_GITHUB_REPOS = "git@github.com:slac-epics" + DEF_GIT_EXT_TOP_TAG = "slac-trunk" DEF_GIT_RELEASE_DEPTH = 10 From fa819b49bd448975cef4b83a51667e41d772c8b8 Mon Sep 17 00:00:00 2001 From: Bruce Hill Date: Fri, 28 Feb 2025 05:04:54 -0800 Subject: [PATCH 2/6] Updated git_utils to also ck for pkg under DEF_GITHUB_REPOS --- git_utils.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/git_utils.py b/git_utils.py index 4b7910d..432386c 100644 --- a/git_utils.py +++ b/git_utils.py @@ -52,6 +52,8 @@ def parseGitModulesTxt(): def determineGitRoot( ): '''Get the root folder for GIT repos at SLAC''' gitRoot = DEF_AFS_GIT_REPOS + if not os.path.exists( gitRoot ): + gitRoot = DEF_GITHUB_REPOS # The GIT_REPO_ROOT variable is mainly used when testing eco and is not something that we really expect from the environment. if "GIT_REPO_ROOT" in os.environ: gitRoot = os.environ["GIT_REPO_ROOT"] @@ -389,6 +391,8 @@ def gitGetWorkingBranch( debug = False, verbose = False ): if debug: print(e) pass + if verbose: + print( "gitGetWorkingBranch: url=%s, branch=%s, tag=%s" % ( repo_url, repo_branch if repo_branch else 'None', repo_tag if repo_tag else 'None' ) ) return ( repo_url, repo_branch, repo_tag ) def determinePathToGitRepo( packagePath, verbose = False ): @@ -411,6 +415,11 @@ def determinePathToGitRepo( packagePath, verbose = False ): # Check under the root of the git repo area for a bare repo w/ the right name gitRoot = determineGitRoot() gitPackageDir = packageName + ".git" + + # If we're using github.com for our root, just add the package name + if 'github.com' in gitRoot: + return gitRoot + '/' + gitPackageDir + gitPackagePath = packagePath + ".git" if not os.path.isdir( DEF_CVS_ROOT ) and gitRoot: # Must be offsite, assume gitRoot and an EPICS module path @@ -453,12 +462,7 @@ def gitFindPackageRelease( packageSpec, tag, debug = False, verbose = False ): print("gitFindPackageRelease: packageName=%s, packagePath=%s" % ( packageName, packagePath )) # See if the package was listed in $TOOLS/eco_modulelist/modulelist.txt - if packageName in git_package2Location: - url_path = determinePathToGitRepo( packageName, verbose=verbose ) - (repo_sha, repo_tag) = gitGetRemoteTag( url_path, tag, verbose=verbose ) - if repo_sha: - repo_url = url_path - else: + if not packageName in git_package2Location: for url_root in [ DEF_GIT_MODULES_PATH, DEF_GIT_EXTENSIONS_PATH, DEF_GIT_EPICS_PATH, DEF_GIT_REPO_PATH ]: if repo_url is not None: break @@ -471,6 +475,12 @@ def gitFindPackageRelease( packageSpec, tag, debug = False, verbose = False ): if packageName == packagePath: break + if not repo_url: + url_path = determinePathToGitRepo( packageName, verbose=verbose ) + (repo_sha, repo_tag) = gitGetRemoteTag( url_path, tag, verbose=verbose ) + if repo_sha: + repo_url = url_path + if verbose: if repo_url: print("gitFindPackageRelease found %s/%s: url=%s, tag=%s" % ( packagePath, tag, repo_url, repo_tag )) From 99400804a74f28e98ea376f74276850956c31d40 Mon Sep 17 00:00:00 2001 From: Bruce Hill Date: Fri, 28 Feb 2025 05:09:55 -0800 Subject: [PATCH 3/6] Modified Releaser to default to package type module if not an IOC or extension. Hack to get around not having substring 'module' in the url. --- Releaser.py | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/Releaser.py b/Releaser.py index 38e6bd0..45c168f 100644 --- a/Releaser.py +++ b/Releaser.py @@ -16,6 +16,8 @@ from site_utils import * from version_utils import * +defaultPackageTypeToModule = True + def makeDirsWritable( dirPathTop ): userId = os.geteuid() for dirPath, dirs, files in os.walk(dirPathTop): @@ -458,24 +460,6 @@ def InstallPackage( self, installTop=None, force=False, rmFailed=False ): print("InstallPackage Error: Need valid installTop to determine installDir!") return status - # Is Package a module? - if os.path.split( self._packagePath )[0] == 'modules' \ - or self._repo.GetUrl().find('modules') >= 0: - # Package is a module - if installTop is not None: - if not installTop.endswith( '/' + self._packageName ): - if not installTop.endswith( '/modules' ): - installTop += '/modules' - if not os.path.isdir( installTop ): - print("Invalid top %s" % installTop) - installTop = None - if installTop is None: - epics_modules_top = determine_epics_modules_top() - if not epics_modules_top: - print("InstallPackage Error: Unable to determine EPICS modules installTop!") - return status - installTop = epics_modules_top - # Is Package an extension? if not installTop: if self._packagePath.find('extensions') >= 0 \ @@ -485,8 +469,9 @@ def InstallPackage( self, installTop=None, force=False, rmFailed=False ): # Is Package an IOC? if installTop is None \ and ( os.path.split( self._packagePath )[0].startswith('ioc') \ - or self._repo.GetUrl().find('ioc') >= 0 ): + or self._repo.GetUrl().find('ioc-') >= 0 ): # Package is an IOC + # TODO: If github repo uses ioc-foo-bar, we need to use installDir ioc/foo/bar topVariants = [ epics_site_top ] for topVariant in defEpicsTopVariants: topVariants.append( os.path.join( epics_site_top, topVariant ) ) @@ -496,6 +481,26 @@ def InstallPackage( self, installTop=None, force=False, rmFailed=False ): installTop = os.path.join( topVariant, self._packagePath ) if not os.path.isdir( installTop ): os.makedirs( installTop, 0o775 ) + break + + # Is Package a module? + if os.path.split( self._packagePath )[0] == 'modules' \ + or defaultPackageTypeToModule \ + or self._repo.GetUrl().find('modules') >= 0: + # Package is a module + if installTop is not None: + if not installTop.endswith( '/' + self._packageName ): + if not installTop.endswith( '/modules' ): + installTop += '/modules' + if not os.path.isdir( installTop ): + print("Invalid top %s" % installTop) + installTop = None + if installTop is None: + epics_modules_top = determine_epics_modules_top() + if not epics_modules_top: + print("InstallPackage Error: Unable to determine EPICS modules installTop!") + return status + installTop = epics_modules_top if not installTop: print("InstallPackage Error: Unable to determine installTop!") From a7613da8d5311bf8f34f66d6064b9d2d90ec8d2f Mon Sep 17 00:00:00 2001 From: Bruce Hill Date: Fri, 28 Feb 2025 05:13:42 -0800 Subject: [PATCH 4/6] Fixed errors in gitRepo.CheckoutRelease. Check for tag already checked out was incorrect leading to unneeded refetch. Also fixed places where code assumed the current URL was the same as remote origin's url. That fails for repos originally cloned from AFS repos that are now hosted on github. --- gitRepo.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/gitRepo.py b/gitRepo.py index 911d222..02060c7 100644 --- a/gitRepo.py +++ b/gitRepo.py @@ -57,12 +57,14 @@ def CheckoutRelease( self, buildDir, verbose=True, quiet=False, dryRun=False, de if len(gitOutput) == 1: curSha = gitOutput[0] - # Get the tag SHA + # Get the commit for the tag tagSha = None - cmdList = [ "git", "rev-parse", self._tag ] + cmdList = [ "git", "show-ref", "-d", self._tag ] gitOutput = subprocess.check_output( cmdList, universal_newlines=True ).splitlines() - if len(gitOutput) == 1: - tagSha = gitOutput[0] + if len(gitOutput) == 2: + tag_commit_info = gitOutput[1].split() + if len(tag_commit_info) == 2: + tagSha = tag_commit_info[0] # If they match, it's already checked out! if curSha == tagSha: @@ -111,8 +113,8 @@ def CheckoutRelease( self, buildDir, verbose=True, quiet=False, dryRun=False, de # Refresh the tags # TODO: May fail if git repo is read-only if verbose: - print("CheckoutRelease running: git fetch origin refs/tags/%s" % self._tag) - cmdList = [ "git", "fetch", "origin", "refs/tags/" + self._tag ] + print("CheckoutRelease running: git fetch %s refs/tags/%s" % (self._url, self._tag)) + cmdList = [ "git", "fetch", self._url, "refs/tags/" + self._tag ] subprocess.check_call( cmdList, stdout=outputPipe, stderr=outputPipe ) tagSha = gitGetTagSha( self._tag ) @@ -165,7 +167,7 @@ def RemoveTag( self, package=None, tag=None, verbose=True, dryRun=False ): if verbose: print("\nRemoving %s release tag %s ..." % ( package, tag )) subprocess.check_call( [ "git", "tag", "-d", tag ] ) - subprocess.check_call( [ 'git', 'push', '--delete', 'origin', tag ] ) + subprocess.check_call( [ 'git', 'push', '--delete', self._url, tag ] ) print("Successfully removed %s release tag %s." % ( package, tag )) def PushBranch( self, branchName=None, verbose=True, dryRun=False ): @@ -181,7 +183,7 @@ def PushBranch( self, branchName=None, verbose=True, dryRun=False ): return if verbose: print("Pushing branch %s ..." % ( branchName )) - subprocess.check_call( [ 'git', 'push', 'origin', branchName ] ) + subprocess.check_call( [ 'git', 'push', self._url, branchName ] ) def PushTag( self, release, verbose=True, dryRun=False ): if dryRun: @@ -190,7 +192,7 @@ def PushTag( self, release, verbose=True, dryRun=False ): if verbose: print("Pushing tag %s ..." % ( release )) - subprocess.check_call( [ 'git', 'push', 'origin', release ] ) + subprocess.check_call( [ 'git', 'push', self._url, release ] ) def TagRelease( self, packagePath=None, release=None, branch=None, message="", verbose=True, dryRun=False ): if release is None: @@ -204,6 +206,6 @@ def TagRelease( self, packagePath=None, release=None, branch=None, message="", v comment = "Release %s/%s: %s" % ( packagePath, release, message ) cmdList = [ "git", "tag", release, 'HEAD', "-m", comment ] subprocess.check_call( cmdList ) - subprocess.check_call( [ 'git', 'push', '-u', 'origin' ] ) - subprocess.check_call( [ 'git', 'push', 'origin', release ] ) + subprocess.check_call( [ 'git', 'push', '-u', self._url, ] ) + subprocess.check_call( [ 'git', 'push', self._url, release ] ) From ee9ff0a5dcf8b8763848fbe815b17413ebbd585e Mon Sep 17 00:00:00 2001 From: Bruce Hill Date: Mon, 3 Mar 2025 01:55:22 -0800 Subject: [PATCH 5/6] Reduced github api accesses to avoid limits. Was getting many github api errors as code was generating tons of queries for github paths that don't exist. --- git_utils.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/git_utils.py b/git_utils.py index 432386c..c9a14ec 100644 --- a/git_utils.py +++ b/git_utils.py @@ -462,7 +462,7 @@ def gitFindPackageRelease( packageSpec, tag, debug = False, verbose = False ): print("gitFindPackageRelease: packageName=%s, packagePath=%s" % ( packageName, packagePath )) # See if the package was listed in $TOOLS/eco_modulelist/modulelist.txt - if not packageName in git_package2Location: + if not packageName in git_package2Location and not 'github.com' in DEF_GIT_REPO_PATH: for url_root in [ DEF_GIT_MODULES_PATH, DEF_GIT_EXTENSIONS_PATH, DEF_GIT_EPICS_PATH, DEF_GIT_REPO_PATH ]: if repo_url is not None: break @@ -476,11 +476,20 @@ def gitFindPackageRelease( packageSpec, tag, debug = False, verbose = False ): break if not repo_url: - url_path = determinePathToGitRepo( packageName, verbose=verbose ) + # Try our github repos + url_path = DEF_GITHUB_REPOS + '/' + packageName + ".git" (repo_sha, repo_tag) = gitGetRemoteTag( url_path, tag, verbose=verbose ) if repo_sha: repo_url = url_path + if not repo_url: + # Try full set of alternatives + url_path = determinePathToGitRepo( packageName, verbose=verbose ) + if url_path: + (repo_sha, repo_tag) = gitGetRemoteTag( url_path, tag, verbose=verbose ) + if repo_sha: + repo_url = url_path + if verbose: if repo_url: print("gitFindPackageRelease found %s/%s: url=%s, tag=%s" % ( packagePath, tag, repo_url, repo_tag )) From 7fdca1f6d9242c026a73d0d4430ab4291c53e477 Mon Sep 17 00:00:00 2001 From: Bruce Hill Date: Mon, 3 Mar 2025 01:57:36 -0800 Subject: [PATCH 6/6] Diagnostics, particulary for determinePathToGitRepo. Can be quite helpful when trying to debug epics-build when AFS may or may not be available, and most but not all EPICS repos have been moved to github.com:slac-epics --- git_utils.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/git_utils.py b/git_utils.py index c9a14ec..1e3d6ee 100644 --- a/git_utils.py +++ b/git_utils.py @@ -415,10 +415,18 @@ def determinePathToGitRepo( packagePath, verbose = False ): # Check under the root of the git repo area for a bare repo w/ the right name gitRoot = determineGitRoot() gitPackageDir = packageName + ".git" + if verbose: + print( "determinePathToGitRepo: gitRoot is %s" % gitRoot if gitRoot else None ) - # If we're using github.com for our root, just add the package name + # Check github DEF_GITHUB_REPOS, just add the package name + defRepoPath = DEF_GITHUB_REPOS + '/' + gitPackageDir if 'github.com' in gitRoot: - return gitRoot + '/' + gitPackageDir + # This results in an extra github API access, but is needed to test if the package is hosted on github + repoTags = gitGetRemoteTags( defRepoPath, verbose=verbose ) + if len(repoTags) > 0: + if verbose: + print( "determinePathToGitRepo: github %s repo is %s" % (packagePath, defRepoPath) ) + return defRepoPath gitPackagePath = packagePath + ".git" if not os.path.isdir( DEF_CVS_ROOT ) and gitRoot: @@ -462,7 +470,7 @@ def gitFindPackageRelease( packageSpec, tag, debug = False, verbose = False ): print("gitFindPackageRelease: packageName=%s, packagePath=%s" % ( packageName, packagePath )) # See if the package was listed in $TOOLS/eco_modulelist/modulelist.txt - if not packageName in git_package2Location and not 'github.com' in DEF_GIT_REPO_PATH: + if not packageName in git_package2Location and not 'github.com' in DEF_GIT_REPO_PATH: for url_root in [ DEF_GIT_MODULES_PATH, DEF_GIT_EXTENSIONS_PATH, DEF_GIT_EPICS_PATH, DEF_GIT_REPO_PATH ]: if repo_url is not None: break @@ -493,6 +501,8 @@ def gitFindPackageRelease( packageSpec, tag, debug = False, verbose = False ): if verbose: if repo_url: print("gitFindPackageRelease found %s/%s: url=%s, tag=%s" % ( packagePath, tag, repo_url, repo_tag )) + elif url_path is None: + print("gitFindPackageRelease Error: Cannot determine URL for repo %s/%s" % (packagePath, tag)) else: print("gitFindPackageRelease Error: Cannot find %s/%s" % (packagePath, tag)) return (repo_url, repo_tag)