diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index d7774bacd3bb..b644d854f885 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -17,7 +17,6 @@ import shlex import shutil import stat -import string import subprocess import time import weakref @@ -252,7 +251,6 @@ def __init__( val_cb=lambda x, y: str(y), ) self.conf = copy.deepcopy(per_remote_defaults) - # Remove the 'salt://' from the beginning of any globally-defined # per-saltenv mountpoints for saltenv, saltenv_conf in self.global_saltenv.items(): @@ -458,46 +456,28 @@ def __init__( failhard(self.role) hash_type = getattr(hashlib, self.opts.get("hash_type", "md5")) - # Generate full id. - # Full id helps decrease the chances of collections in the gitfs cache. - try: - target = str(self.get_checkout_target()) - except AttributeError: - target = "" - self._full_id = "-".join( - [ - getattr(self, "name", ""), - self.id, - getattr(self, "env", ""), - getattr(self, "_root", ""), - self.role, - getattr(self, "base", ""), - getattr(self, "branch", ""), - target, - ] - ) # We loaded this data from yaml configuration files, so, its safe # to use UTF-8 - base64_hash = str( - base64.b64encode(hash_type(self._full_id.encode("utf-8")).digest()), + self._cache_basehash = str( + base64.b64encode(hash_type(self.id.encode("utf-8")).digest()), encoding="ascii", # base64 only outputs ascii ).replace( "/", "_" ) # replace "/" with "_" to not cause trouble with file system - - # limit name length to 19, so we don't eat up all the path length for windows - # this is due to pygit2 limitations - # replace any unknown char with "_" to not cause trouble with file system - name_chars = string.ascii_letters + string.digits + "-" - cache_name = "".join( - c if c in name_chars else "_" for c in getattr(self, "name", "")[:19] + self._cache_hash = salt.utils.path.join(cache_root, self._cache_basehash) + try: + self._cache_basename = self.get_checkout_target() + except AttributeError: + self._cache_basename = "_" + self._cache_full_basename = salt.utils.path.join( + self._cache_basehash, self._cache_basename ) - - self.cachedir_basename = f"{cache_name}-{base64_hash}" - self.cachedir = salt.utils.path.join(cache_root, self.cachedir_basename) - self.linkdir = salt.utils.path.join(cache_root, "links", self.cachedir_basename) - if not os.path.isdir(self.cachedir): - os.makedirs(self.cachedir) + self._cachedir = salt.utils.path.join(self._cache_hash, self._cache_basename) + self._linkdir = salt.utils.path.join( + cache_root, "links", self._cache_full_basename + ) + if not os.path.isdir(self._cachedir): + os.makedirs(self._cachedir) try: self.new = self.init_remote() @@ -510,11 +490,23 @@ def __init__( log.critical(msg, exc_info=True) failhard(self.role) - def full_id(self): - return self._full_id + def get_cache_basehash(self): + return self._cache_basehash + + def get_cache_hash(self): + return self._cache_hash + + def get_cache_basename(self): + return self._cache_basename + + def get_cache_full_basename(self): + return self._cache_full_basename - def get_cachedir_basename(self): - return self.cachedir_basename + def get_cachedir(self): + return self._cachedir + + def get_linkdir(self): + return self._linkdir def _get_envs_from_ref_paths(self, refs): """ @@ -643,7 +635,7 @@ def check_root(self): # No need to pass an environment to self.root() here since per-saltenv # configuration is a gitfs-only feature and check_root() is not used # for gitfs. - root_dir = salt.utils.path.join(self.cachedir, self.root()).rstrip(os.sep) + root_dir = salt.utils.path.join(self._cachedir, self.root()).rstrip(os.sep) if os.path.isdir(root_dir): return root_dir log.error( @@ -1068,7 +1060,7 @@ def init_remote(self): """ raise NotImplementedError() - def checkout(self): + def checkout(self, fetch_on_fail=True): """ This function must be overridden in a sub-class """ @@ -1274,7 +1266,7 @@ def __init__( role, ) - def checkout(self): + def checkout(self, fetch_on_fail=True): """ Checkout the configured branch/tag. We catch an "Exception" class here instead of a specific exception class because the exceptions raised by @@ -1344,6 +1336,15 @@ def checkout(self): except Exception: # pylint: disable=broad-except continue return self.check_root() + if fetch_on_fail: + log.debug( + "Failed to checkout %s from %s remote '%s': fetch and try again", + tgt_ref, + self.role, + self.id, + ) + self.fetch() + return self.checkout(fetch_on_fail=False) log.error( "Failed to checkout %s from %s remote '%s': remote ref does not exist", tgt_ref, @@ -1359,16 +1360,16 @@ def init_remote(self): initialized by this function. """ new = False - if not os.listdir(self.cachedir): + if not os.listdir(self._cachedir): # Repo cachedir is empty, initialize a new repo there - self.repo = git.Repo.init(self.cachedir) + self.repo = git.Repo.init(self._cachedir) new = True else: # Repo cachedir exists, try to attach try: - self.repo = git.Repo(self.cachedir) + self.repo = git.Repo(self._cachedir) except git.exc.InvalidGitRepositoryError: - log.error(_INVALID_REPO, self.cachedir, self.url, self.role) + log.error(_INVALID_REPO, self._cachedir, self.url, self.role) return new self.gitdir = salt.utils.path.join(self.repo.working_dir, ".git") @@ -1602,7 +1603,7 @@ def peel(self, obj): except AttributeError: return obj.get_object() - def checkout(self): + def checkout(self, fetch_on_fail=True): """ Checkout the configured branch/tag """ @@ -1795,6 +1796,15 @@ def _perform_checkout(checkout_ref, branch=True): exc_info=True, ) return None + if fetch_on_fail: + log.debug( + "Failed to checkout %s from %s remote '%s': fetch and try again", + tgt_ref, + self.role, + self.id, + ) + self.fetch() + return self.checkout(fetch_on_fail=False) log.error( "Failed to checkout %s from %s remote '%s': remote ref does not exist", tgt_ref, @@ -1836,16 +1846,16 @@ def init_remote(self): home = os.path.expanduser("~") pygit2.settings.search_path[pygit2.GIT_CONFIG_LEVEL_GLOBAL] = home new = False - if not os.listdir(self.cachedir): + if not os.listdir(self._cachedir): # Repo cachedir is empty, initialize a new repo there - self.repo = pygit2.init_repository(self.cachedir) + self.repo = pygit2.init_repository(self._cachedir) new = True else: # Repo cachedir exists, try to attach try: - self.repo = pygit2.Repository(self.cachedir) + self.repo = pygit2.Repository(self._cachedir) except KeyError: - log.error(_INVALID_REPO, self.cachedir, self.url, self.role) + log.error(_INVALID_REPO, self._cachedir, self.url, self.role) return new self.gitdir = salt.utils.path.join(self.repo.workdir, ".git") @@ -2487,7 +2497,7 @@ def init_remotes( # Don't allow collisions in cachedir naming cachedir_map = {} for repo in self.remotes: - cachedir_map.setdefault(repo.cachedir, []).append(repo.id) + cachedir_map.setdefault(repo.get_cachedir(), []).append(repo.id) collisions = [x for x in cachedir_map if len(cachedir_map[x]) > 1] if collisions: @@ -2512,10 +2522,11 @@ def clear_old_remotes(self): cachedir_ls = os.listdir(self.cache_root) except OSError: cachedir_ls = [] + log.critical(cachedir_ls) # Remove actively-used remotes from list for repo in self.remotes: try: - cachedir_ls.remove(repo.cachedir_basename) + cachedir_ls.remove(repo.get_cache_basehash()) except ValueError: pass to_remove = [] @@ -2858,7 +2869,7 @@ def write_remote_map(self): for repo in self.remotes: fp_.write( salt.utils.stringutils.to_str( - "{} = {}\n".format(repo.cachedir_basename, repo.id) + "{} = {}\n".format(repo.get_cache_basehash(), repo.id) ) ) except OSError: @@ -3303,8 +3314,10 @@ def link_mountpoint(self, repo): Ensure that the mountpoint is present in the correct location and points at the correct path """ - lcachelink = salt.utils.path.join(repo.linkdir, repo._mountpoint) - lcachedest = salt.utils.path.join(repo.cachedir, repo.root()).rstrip(os.sep) + lcachelink = salt.utils.path.join(repo.get_linkdir(), repo._mountpoint) + lcachedest = salt.utils.path.join(repo.get_cachedir(), repo.root()).rstrip( + os.sep + ) wipe_linkdir = False create_link = False try: