From f543a555f5dae3b57e590b7f994679ea8d48538b Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 10 Feb 2025 16:18:56 -0700 Subject: [PATCH 1/3] Fix tech-debt for Git Pillar data with os.walk --- salt/utils/gitfs.py | 69 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index 5bcef99d9ac..f76757ad213 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -2798,8 +2798,77 @@ def fetch_remotes(self, remotes=None): name = getattr(repo, "name", None) if not remotes or (repo.id, name) in remotes or name in remotes: try: + ## DGM # Find and place fetch_request file for all the other branches for this repo + ## DGM repo_work_hash = os.path.split(repo.get_salt_working_dir())[0] + ## DGM for branch in os.listdir(repo_work_hash): + ## DGM # Don't place fetch request in current branch being updated + ## DGM if branch == repo.get_cache_basename(): + ## DGM continue + ## DGM branch_salt_dir = salt.utils.path.join(repo_work_hash, branch) + ## DGM fetch_path = salt.utils.path.join( + ## DGM branch_salt_dir, "fetch_request" + ## DGM ) + ## DGM if os.path.isdir(branch_salt_dir): + ## DGM try: + ## DGM with salt.utils.files.fopen(fetch_path, "w"): + ## DGM pass + ## DGM except OSError as exc: # pylint: disable=broad-except + ## DGM log.error( + ## DGM "Failed to make fetch request: %s %s", + ## DGM fetch_path, + ## DGM exc, + ## DGM exc_info=True, + ## DGM ) + ## DGM else: + ## DGM log.error("Failed to make fetch request: %s", fetch_path) + # Find and place fetch_request file for all the other branches for this repo repo_work_hash = os.path.split(repo.get_salt_working_dir())[0] + print( + f"DGM class GitBase fetch_remotes, repo_work_hash '{repo_work_hash}', salt working dir '{repo.get_salt_working_dir()}'", + flush=True, + ) + + branches = [ + os.path.relpath(path, repo_work_hash) + for (path, subdirs, files) in os.walk(repo_work_hash) + if not subdirs + ] + print( + f"DGM class GitBase fetch_remotes, branches '{branches}' from repo_work_hash '{repo_work_hash}'", + flush=True, + ) + + ## DGM for branch in os.listdir(repo_work_hash): + for branch in os.listdir(branches): + # Don't place fetch request in current branch being updated + print( + f"DGM class GitBase fetch_remotes, for loop branch, branch '{branch}', repo get_cache_basename '{repo.get_cache_basename()}'", + flush=True, + ) + if branch == repo.get_cache_basename(): + continue + branch_salt_dir = salt.utils.path.join(repo_work_hash, branch) + fetch_path = salt.utils.path.join( + branch_salt_dir, "fetch_request" + ) + print( + f"DGM class GitBase fetch_remotes, for loop branch, branch_salt_dir '{branch_salt_dir}', fetch_path '{fetch_path}'", + flush=True, + ) + if os.path.isdir(branch_salt_dir): + try: + with salt.utils.files.fopen(fetch_path, "w"): + pass + except OSError as exc: # pylint: disable=broad-except + log.error( + "Failed to make fetch request: %s %s", + fetch_path, + exc, + exc_info=True, + ) + else: + log.error("Failed to make fetch request: %s", fetch_path) for branch in os.listdir(repo_work_hash): # Don't place fetch request in current branch being updated if branch == repo.get_cache_basename(): From 0acc1b2828fed1923000686a4cdcdac493f4ab0e Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 11 Feb 2025 13:58:45 -0700 Subject: [PATCH 2/3] Drop replacing slash in git branch name --- salt/utils/gitfs.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index f76757ad213..22972635076 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -491,9 +491,10 @@ def _val_cb(x, y): self._cache_basename = "_" if self.id.startswith("__env__"): try: - self._cache_basename = self.get_checkout_target().replace( - "/", "-" - ) # replace '/' with '-' to not cause trouble with file-system + ## DGM self._cache_basename = self.get_checkout_target().replace( + ## DGM "/", "-" + ## DGM ) # replace '/' with '-' to not cause trouble with file-system + self._cache_basename = self.get_checkout_target() except AttributeError: log.critical( From 9810689fc59bfc8de8530ee915d2fd2695363b0f Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 14 Feb 2025 13:37:51 -0700 Subject: [PATCH 3/3] Allow slash '/' in git branch names and traverse git branches using os.walk --- changelog/67733.fixed.md | 1 + salt/utils/gitfs.py | 47 +--------------------------------------- 2 files changed, 2 insertions(+), 46 deletions(-) create mode 100644 changelog/67733.fixed.md diff --git a/changelog/67733.fixed.md b/changelog/67733.fixed.md new file mode 100644 index 00000000000..19d49f3f55d --- /dev/null +++ b/changelog/67733.fixed.md @@ -0,0 +1 @@ +Use os.walk to traverse git branches, and no longer replace slash '/' in git branch names diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index 22972635076..c9310b03d04 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -491,9 +491,6 @@ def _val_cb(x, y): self._cache_basename = "_" if self.id.startswith("__env__"): try: - ## DGM self._cache_basename = self.get_checkout_target().replace( - ## DGM "/", "-" - ## DGM ) # replace '/' with '-' to not cause trouble with file-system self._cache_basename = self.get_checkout_target() except AttributeError: @@ -2799,64 +2796,22 @@ def fetch_remotes(self, remotes=None): name = getattr(repo, "name", None) if not remotes or (repo.id, name) in remotes or name in remotes: try: - ## DGM # Find and place fetch_request file for all the other branches for this repo - ## DGM repo_work_hash = os.path.split(repo.get_salt_working_dir())[0] - ## DGM for branch in os.listdir(repo_work_hash): - ## DGM # Don't place fetch request in current branch being updated - ## DGM if branch == repo.get_cache_basename(): - ## DGM continue - ## DGM branch_salt_dir = salt.utils.path.join(repo_work_hash, branch) - ## DGM fetch_path = salt.utils.path.join( - ## DGM branch_salt_dir, "fetch_request" - ## DGM ) - ## DGM if os.path.isdir(branch_salt_dir): - ## DGM try: - ## DGM with salt.utils.files.fopen(fetch_path, "w"): - ## DGM pass - ## DGM except OSError as exc: # pylint: disable=broad-except - ## DGM log.error( - ## DGM "Failed to make fetch request: %s %s", - ## DGM fetch_path, - ## DGM exc, - ## DGM exc_info=True, - ## DGM ) - ## DGM else: - ## DGM log.error("Failed to make fetch request: %s", fetch_path) - # Find and place fetch_request file for all the other branches for this repo repo_work_hash = os.path.split(repo.get_salt_working_dir())[0] - print( - f"DGM class GitBase fetch_remotes, repo_work_hash '{repo_work_hash}', salt working dir '{repo.get_salt_working_dir()}'", - flush=True, - ) - branches = [ os.path.relpath(path, repo_work_hash) for (path, subdirs, files) in os.walk(repo_work_hash) if not subdirs ] - print( - f"DGM class GitBase fetch_remotes, branches '{branches}' from repo_work_hash '{repo_work_hash}'", - flush=True, - ) - ## DGM for branch in os.listdir(repo_work_hash): - for branch in os.listdir(branches): + for branch in branches: # Don't place fetch request in current branch being updated - print( - f"DGM class GitBase fetch_remotes, for loop branch, branch '{branch}', repo get_cache_basename '{repo.get_cache_basename()}'", - flush=True, - ) if branch == repo.get_cache_basename(): continue branch_salt_dir = salt.utils.path.join(repo_work_hash, branch) fetch_path = salt.utils.path.join( branch_salt_dir, "fetch_request" ) - print( - f"DGM class GitBase fetch_remotes, for loop branch, branch_salt_dir '{branch_salt_dir}', fetch_path '{fetch_path}'", - flush=True, - ) if os.path.isdir(branch_salt_dir): try: with salt.utils.files.fopen(fetch_path, "w"):