Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Epics build on rocky9 #12

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Epics build on rocky9 #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f08a6e7
Added DEF_GITHUB_REPOS to repo_defaults.py
bhill-slac Feb 28, 2025
fa819b4
Updated git_utils to also ck for pkg under DEF_GITHUB_REPOS
bhill-slac Feb 28, 2025
9940080
Modified Releaser to default to package type module if not an IOC or …
bhill-slac Feb 28, 2025
a7613da
Fixed errors in gitRepo.CheckoutRelease.
bhill-slac Feb 28, 2025
ee9ff0a
Reduced github api accesses to avoid limits.
bhill-slac Mar 3, 2025
7fdca1f
Diagnostics, particulary for determinePathToGitRepo.
bhill-slac Mar 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 24 additions & 19 deletions Releaser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
from site_utils import *
from version_utils import *

defaultPackageTypeToModule = True

def makeDirsWritable( dirPathTop ):
userId = os.geteuid()
for dirPath, dirs, files in os.walk(dirPathTop):
Expand Down Expand Up @@ -458,24 +460,6 @@ def InstallPackage( self, installTop=None, force=False, rmFailed=False ):
print("InstallPackage Error: Need valid installTop to determine installDir!")
return status

# Is Package a module?
if os.path.split( self._packagePath )[0] == 'modules' \
or self._repo.GetUrl().find('modules') >= 0:
# Package is a module
if installTop is not None:
if not installTop.endswith( '/' + self._packageName ):
if not installTop.endswith( '/modules' ):
installTop += '/modules'
if not os.path.isdir( installTop ):
print("Invalid top %s" % installTop)
installTop = None
if installTop is None:
epics_modules_top = determine_epics_modules_top()
if not epics_modules_top:
print("InstallPackage Error: Unable to determine EPICS modules installTop!")
return status
installTop = epics_modules_top

# Is Package an extension?
if not installTop:
if self._packagePath.find('extensions') >= 0 \
Expand All @@ -485,8 +469,9 @@ def InstallPackage( self, installTop=None, force=False, rmFailed=False ):
# Is Package an IOC?
if installTop is None \
and ( os.path.split( self._packagePath )[0].startswith('ioc') \
or self._repo.GetUrl().find('ioc') >= 0 ):
or self._repo.GetUrl().find('ioc-') >= 0 ):
# Package is an IOC
# TODO: If github repo uses ioc-foo-bar, we need to use installDir ioc/foo/bar
topVariants = [ epics_site_top ]
for topVariant in defEpicsTopVariants:
topVariants.append( os.path.join( epics_site_top, topVariant ) )
Expand All @@ -496,6 +481,26 @@ def InstallPackage( self, installTop=None, force=False, rmFailed=False ):
installTop = os.path.join( topVariant, self._packagePath )
if not os.path.isdir( installTop ):
os.makedirs( installTop, 0o775 )
break

# Is Package a module?
if os.path.split( self._packagePath )[0] == 'modules' \
or defaultPackageTypeToModule \
or self._repo.GetUrl().find('modules') >= 0:
# Package is a module
if installTop is not None:
if not installTop.endswith( '/' + self._packageName ):
if not installTop.endswith( '/modules' ):
installTop += '/modules'
if not os.path.isdir( installTop ):
print("Invalid top %s" % installTop)
installTop = None
if installTop is None:
epics_modules_top = determine_epics_modules_top()
if not epics_modules_top:
print("InstallPackage Error: Unable to determine EPICS modules installTop!")
return status
installTop = epics_modules_top

if not installTop:
print("InstallPackage Error: Unable to determine installTop!")
Expand Down
24 changes: 13 additions & 11 deletions gitRepo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,14 @@ def CheckoutRelease( self, buildDir, verbose=True, quiet=False, dryRun=False, de
if len(gitOutput) == 1:
curSha = gitOutput[0]

# Get the tag SHA
# Get the commit for the tag
tagSha = None
cmdList = [ "git", "rev-parse", self._tag ]
cmdList = [ "git", "show-ref", "-d", self._tag ]
gitOutput = subprocess.check_output( cmdList, universal_newlines=True ).splitlines()
if len(gitOutput) == 1:
tagSha = gitOutput[0]
if len(gitOutput) == 2:
tag_commit_info = gitOutput[1].split()
if len(tag_commit_info) == 2:
tagSha = tag_commit_info[0]

# If they match, it's already checked out!
if curSha == tagSha:
Expand Down Expand Up @@ -111,8 +113,8 @@ def CheckoutRelease( self, buildDir, verbose=True, quiet=False, dryRun=False, de
# Refresh the tags
# TODO: May fail if git repo is read-only
if verbose:
print("CheckoutRelease running: git fetch origin refs/tags/%s" % self._tag)
cmdList = [ "git", "fetch", "origin", "refs/tags/" + self._tag ]
print("CheckoutRelease running: git fetch %s refs/tags/%s" % (self._url, self._tag))
cmdList = [ "git", "fetch", self._url, "refs/tags/" + self._tag ]
subprocess.check_call( cmdList, stdout=outputPipe, stderr=outputPipe )

tagSha = gitGetTagSha( self._tag )
Expand Down Expand Up @@ -165,7 +167,7 @@ def RemoveTag( self, package=None, tag=None, verbose=True, dryRun=False ):
if verbose:
print("\nRemoving %s release tag %s ..." % ( package, tag ))
subprocess.check_call( [ "git", "tag", "-d", tag ] )
subprocess.check_call( [ 'git', 'push', '--delete', 'origin', tag ] )
subprocess.check_call( [ 'git', 'push', '--delete', self._url, tag ] )
print("Successfully removed %s release tag %s." % ( package, tag ))

def PushBranch( self, branchName=None, verbose=True, dryRun=False ):
Expand All @@ -181,7 +183,7 @@ def PushBranch( self, branchName=None, verbose=True, dryRun=False ):
return
if verbose:
print("Pushing branch %s ..." % ( branchName ))
subprocess.check_call( [ 'git', 'push', 'origin', branchName ] )
subprocess.check_call( [ 'git', 'push', self._url, branchName ] )

def PushTag( self, release, verbose=True, dryRun=False ):
if dryRun:
Expand All @@ -190,7 +192,7 @@ def PushTag( self, release, verbose=True, dryRun=False ):

if verbose:
print("Pushing tag %s ..." % ( release ))
subprocess.check_call( [ 'git', 'push', 'origin', release ] )
subprocess.check_call( [ 'git', 'push', self._url, release ] )

def TagRelease( self, packagePath=None, release=None, branch=None, message="", verbose=True, dryRun=False ):
if release is None:
Expand All @@ -204,6 +206,6 @@ def TagRelease( self, packagePath=None, release=None, branch=None, message="", v
comment = "Release %s/%s: %s" % ( packagePath, release, message )
cmdList = [ "git", "tag", release, 'HEAD', "-m", comment ]
subprocess.check_call( cmdList )
subprocess.check_call( [ 'git', 'push', '-u', 'origin' ] )
subprocess.check_call( [ 'git', 'push', 'origin', release ] )
subprocess.check_call( [ 'git', 'push', '-u', self._url, ] )
subprocess.check_call( [ 'git', 'push', self._url, release ] )

41 changes: 35 additions & 6 deletions git_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ def parseGitModulesTxt():
def determineGitRoot( ):
'''Get the root folder for GIT repos at SLAC'''
gitRoot = DEF_AFS_GIT_REPOS
if not os.path.exists( gitRoot ):
gitRoot = DEF_GITHUB_REPOS
# The GIT_REPO_ROOT variable is mainly used when testing eco and is not something that we really expect from the environment.
if "GIT_REPO_ROOT" in os.environ:
gitRoot = os.environ["GIT_REPO_ROOT"]
Expand Down Expand Up @@ -389,6 +391,8 @@ def gitGetWorkingBranch( debug = False, verbose = False ):
if debug:
print(e)
pass
if verbose:
print( "gitGetWorkingBranch: url=%s, branch=%s, tag=%s" % ( repo_url, repo_branch if repo_branch else 'None', repo_tag if repo_tag else 'None' ) )
return ( repo_url, repo_branch, repo_tag )

def determinePathToGitRepo( packagePath, verbose = False ):
Expand All @@ -411,6 +415,19 @@ def determinePathToGitRepo( packagePath, verbose = False ):
# Check under the root of the git repo area for a bare repo w/ the right name
gitRoot = determineGitRoot()
gitPackageDir = packageName + ".git"
if verbose:
print( "determinePathToGitRepo: gitRoot is %s" % gitRoot if gitRoot else None )

# Check github DEF_GITHUB_REPOS, just add the package name
defRepoPath = DEF_GITHUB_REPOS + '/' + gitPackageDir
if 'github.com' in gitRoot:
# This results in an extra github API access, but is needed to test if the package is hosted on github
repoTags = gitGetRemoteTags( defRepoPath, verbose=verbose )
if len(repoTags) > 0:
if verbose:
print( "determinePathToGitRepo: github %s repo is %s" % (packagePath, defRepoPath) )
return defRepoPath
Comment on lines +422 to +429
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another option here might be to check if $GIT_SITE_TOP/path/to/repo.git/hooks/pre-receive exits with an error not. The modules & packages that we migrated to GitHub have a pre-receive hook that prints an error message and exits return code 1. If the pre-receive hook exits with 0, then you can do the extra github.com check.


gitPackagePath = packagePath + ".git"
if not os.path.isdir( DEF_CVS_ROOT ) and gitRoot:
# Must be offsite, assume gitRoot and an EPICS module path
Expand Down Expand Up @@ -453,12 +470,7 @@ def gitFindPackageRelease( packageSpec, tag, debug = False, verbose = False ):
print("gitFindPackageRelease: packageName=%s, packagePath=%s" % ( packageName, packagePath ))

# See if the package was listed in $TOOLS/eco_modulelist/modulelist.txt
if packageName in git_package2Location:
url_path = determinePathToGitRepo( packageName, verbose=verbose )
(repo_sha, repo_tag) = gitGetRemoteTag( url_path, tag, verbose=verbose )
if repo_sha:
repo_url = url_path
else:
if not packageName in git_package2Location and not 'github.com' in DEF_GIT_REPO_PATH:
for url_root in [ DEF_GIT_MODULES_PATH, DEF_GIT_EXTENSIONS_PATH, DEF_GIT_EPICS_PATH, DEF_GIT_REPO_PATH ]:
if repo_url is not None:
break
Expand All @@ -471,9 +483,26 @@ def gitFindPackageRelease( packageSpec, tag, debug = False, verbose = False ):
if packageName == packagePath:
break

if not repo_url:
# Try our github repos
url_path = DEF_GITHUB_REPOS + '/' + packageName + ".git"
(repo_sha, repo_tag) = gitGetRemoteTag( url_path, tag, verbose=verbose )
if repo_sha:
repo_url = url_path

if not repo_url:
# Try full set of alternatives
url_path = determinePathToGitRepo( packageName, verbose=verbose )
if url_path:
(repo_sha, repo_tag) = gitGetRemoteTag( url_path, tag, verbose=verbose )
if repo_sha:
repo_url = url_path

if verbose:
if repo_url:
print("gitFindPackageRelease found %s/%s: url=%s, tag=%s" % ( packagePath, tag, repo_url, repo_tag ))
elif url_path is None:
print("gitFindPackageRelease Error: Cannot determine URL for repo %s/%s" % (packagePath, tag))
else:
print("gitFindPackageRelease Error: Cannot find %s/%s" % (packagePath, tag))
return (repo_url, repo_tag)
Expand Down
3 changes: 3 additions & 0 deletions repo_defaults.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@
DEF_AFS_GIT_REPOS = "/afs/slac.stanford.edu/g/cd/swe/git/repos"
DEF_AFS_GIT_REPOS2 = "/afs/slac/g/cd/swe/git/repos"

# Github Organitions
DEF_GITHUB_REPOS = "[email protected]:slac-epics"

DEF_GIT_EXT_TOP_TAG = "slac-trunk"
DEF_GIT_RELEASE_DEPTH = 10

Expand Down