From 3238bd4401bdec28805bbdfa382a860336e8a8b2 Mon Sep 17 00:00:00 2001 From: George Pickering <9803299+bigpick@users.noreply.github.com> Date: Mon, 6 May 2024 10:20:17 -0400 Subject: [PATCH 1/5] fix: File enumeration fails on older git binary Signed-off-by: George Pickering <9803299+bigpick@users.noreply.github.com> --- detect_secrets/core/baseline.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/detect_secrets/core/baseline.py b/detect_secrets/core/baseline.py index 41692f266..f56ee9fe1 100644 --- a/detect_secrets/core/baseline.py +++ b/detect_secrets/core/baseline.py @@ -361,22 +361,38 @@ def _get_git_tracked_files(rootdir='.'): :returns: filepaths to files which git currently tracks (locally) """ output = [] + git_dir_opts = [] try: with open(os.devnull, 'w') as fnull: - git_files = subprocess.check_output( + # git <1.8.5 https://github.com/git/git/commit/44e1e4d67d5148c245db362cc48c3cc6c2ec82ca + # doesn't support -C + git_major, git_minor, git_patch, _ = subprocess.check_output( [ 'git', - '-C', rootdir, - 'ls-files', + '--version', ], stderr=fnull, + ).decode('utf-8').split()[-1].split(".") + + if int(git_major) == 1 and int(git_minor) <= 8 and int(git_patch) < 5: + git_dir_opts = ["--git-dir", os.path.join(rootdir, ".git")] + else: + git_dir_opts = ["-C", rootdir] + + git_files = subprocess.check_output( + ['git' ] + git_dir_opts + ['ls-files'], + stderr=fnull, ) for filename in git_files.decode('utf-8').split(): relative_path = util.get_relative_path_if_in_cwd(rootdir, filename) if relative_path: output.append(relative_path) - except subprocess.CalledProcessError: + + except subprocess.CalledProcessError as err: + log.error('detect-secrets: Encountered error trying to list git tracked '+ + f'files for dir "{rootdir}": {str(err)}') pass + return output From 7023b5a332da8db6ce9c0613a5ea40e8d51b8654 Mon Sep 17 00:00:00 2001 From: George Pickering <9803299+bigpick@users.noreply.github.com> Date: Mon, 6 May 2024 10:24:45 -0400 Subject: [PATCH 2/5] fix: ignore all fields after patch Signed-off-by: George Pickering <9803299+bigpick@users.noreply.github.com> --- detect_secrets/core/baseline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detect_secrets/core/baseline.py b/detect_secrets/core/baseline.py index f56ee9fe1..5a2a39dcd 100644 --- a/detect_secrets/core/baseline.py +++ b/detect_secrets/core/baseline.py @@ -366,7 +366,7 @@ def _get_git_tracked_files(rootdir='.'): with open(os.devnull, 'w') as fnull: # git <1.8.5 https://github.com/git/git/commit/44e1e4d67d5148c245db362cc48c3cc6c2ec82ca # doesn't support -C - git_major, git_minor, git_patch, _ = subprocess.check_output( + git_major, git_minor, git_patch, *_ = subprocess.check_output( [ 'git', '--version', From a34594efb4858417f00009f19f2704f1ca0f2a22 Mon Sep 17 00:00:00 2001 From: George Pickering <9803299+bigpick@users.noreply.github.com> Date: Mon, 6 May 2024 12:14:55 -0400 Subject: [PATCH 3/5] fix: dont rely on -C at all Signed-off-by: George Pickering <9803299+bigpick@users.noreply.github.com> --- detect_secrets/core/baseline.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/detect_secrets/core/baseline.py b/detect_secrets/core/baseline.py index 5a2a39dcd..3e6c0b8fb 100644 --- a/detect_secrets/core/baseline.py +++ b/detect_secrets/core/baseline.py @@ -345,7 +345,6 @@ def format_baseline_for_output(baseline): separators=(',', ': '), ) - def _get_git_tracked_files(rootdir='.'): """Parsing .gitignore rules is hard. @@ -361,28 +360,18 @@ def _get_git_tracked_files(rootdir='.'): :returns: filepaths to files which git currently tracks (locally) """ output = [] - git_dir_opts = [] + + # git <1.8.5 https://github.com/git/git/commit/44e1e4d67d5148c245db362cc48c3cc6c2ec82ca + # doesn't support -C and we can achieve the same using cwd arg to subproc + cmd = ["git", "ls-files"] + if not os.path.exists(rootdir) or not os.path.isdir(rootdir): + log.debug(f"Skipping {rootdir} bc dir doesn't exist or isn't a directory") + return [] + try: with open(os.devnull, 'w') as fnull: - # git <1.8.5 https://github.com/git/git/commit/44e1e4d67d5148c245db362cc48c3cc6c2ec82ca - # doesn't support -C - git_major, git_minor, git_patch, *_ = subprocess.check_output( - [ - 'git', - '--version', - ], - stderr=fnull, - ).decode('utf-8').split()[-1].split(".") - - if int(git_major) == 1 and int(git_minor) <= 8 and int(git_patch) < 5: - git_dir_opts = ["--git-dir", os.path.join(rootdir, ".git")] - else: - git_dir_opts = ["-C", rootdir] + git_files = subprocess.check_output(cmd, cwd=rootdir, stderr=fnull) - git_files = subprocess.check_output( - ['git' ] + git_dir_opts + ['ls-files'], - stderr=fnull, - ) for filename in git_files.decode('utf-8').split(): relative_path = util.get_relative_path_if_in_cwd(rootdir, filename) if relative_path: From 1a4848a2205883fed042e0ce7ebf463fffd9e380 Mon Sep 17 00:00:00 2001 From: George Pickering <9803299+bigpick@users.noreply.github.com> Date: Mon, 6 May 2024 12:25:29 -0400 Subject: [PATCH 4/5] fix: dont rely on -C at all (2) Signed-off-by: George Pickering <9803299+bigpick@users.noreply.github.com> --- tests/core/baseline_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/baseline_test.py b/tests/core/baseline_test.py index efd7c7ea2..bb778e0fb 100644 --- a/tests/core/baseline_test.py +++ b/tests/core/baseline_test.py @@ -72,7 +72,7 @@ def test_error_when_getting_git_tracked_files(self, path): 'detect_secrets.core.baseline.subprocess.check_output', ( SubprocessMock( - expected_input='git -C ./test_data/files ls-files', + expected_input='git ls-files', should_throw_exception=True, mocked_output='', ), From f1692a2b66c5131c037ff51bdbbadfb9e382a1f9 Mon Sep 17 00:00:00 2001 From: George Pickering <9803299+bigpick@users.noreply.github.com> Date: Mon, 6 May 2024 12:50:58 -0400 Subject: [PATCH 5/5] chore: double quotes to singles Signed-off-by: George Pickering <9803299+bigpick@users.noreply.github.com> --- detect_secrets/core/baseline.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/detect_secrets/core/baseline.py b/detect_secrets/core/baseline.py index 3e6c0b8fb..f32ce48ca 100644 --- a/detect_secrets/core/baseline.py +++ b/detect_secrets/core/baseline.py @@ -345,6 +345,7 @@ def format_baseline_for_output(baseline): separators=(',', ': '), ) + def _get_git_tracked_files(rootdir='.'): """Parsing .gitignore rules is hard. @@ -363,9 +364,9 @@ def _get_git_tracked_files(rootdir='.'): # git <1.8.5 https://github.com/git/git/commit/44e1e4d67d5148c245db362cc48c3cc6c2ec82ca # doesn't support -C and we can achieve the same using cwd arg to subproc - cmd = ["git", "ls-files"] + cmd = ['git', 'ls-files'] if not os.path.exists(rootdir) or not os.path.isdir(rootdir): - log.debug(f"Skipping {rootdir} bc dir doesn't exist or isn't a directory") + log.debug(f'Skipping {rootdir} bc dir doesn\'t exist or isn\'t a directory') return [] try: @@ -378,8 +379,10 @@ def _get_git_tracked_files(rootdir='.'): output.append(relative_path) except subprocess.CalledProcessError as err: - log.error('detect-secrets: Encountered error trying to list git tracked '+ - f'files for dir "{rootdir}": {str(err)}') + log.error( + 'detect-secrets: Encountered error trying to list git tracked ' + + f'files for dir {rootdir}: {str(err)}', + ) pass return output