From 51696d50ddcdffd15057d94b33a921ac6208b5d1 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Mon, 12 Feb 2024 17:22:09 +0000 Subject: [PATCH 1/9] Fixed offset in test pattern --- examples/config/patterns.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/config/patterns.yml b/examples/config/patterns.yml index 866794e..b14d007 100644 --- a/examples/config/patterns.yml +++ b/examples/config/patterns.yml @@ -38,5 +38,5 @@ patterns: # optional, defaults to -1 (the end of the file) end_offset: 48 - name: django_key.txt - start_offset: 49 + start_offset: 60 end_offset: 97 From 441c1320b1c7ac7eca36a54c1b0708a1a5c0ea9f Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Mon, 12 Feb 2024 17:22:40 +0000 Subject: [PATCH 2/9] Added error output when there is more than one match on the test data --- secretscanning/test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/secretscanning/test.py b/secretscanning/test.py index ec8b665..d062fb2 100755 --- a/secretscanning/test.py +++ b/secretscanning/test.py @@ -547,6 +547,9 @@ def test_patterns( pattern_results = RESULTS.get(pattern.name, []) if len(pattern_results) > 1: + LOG.error("❌ matched more than once on test data on '%s' in '%s'", pattern.type, rel_dirpath) + for res in pattern_results: + LOG.error("%s%s%s", res['groups']['start'], Fore.RED + res['groups']['pattern'] + Style.RESET_ALL, res['groups']['end']) ok_test = False else: # did we match what we expected? @@ -592,9 +595,11 @@ def test_patterns( result_data.get("start_offset", -1), result_data.get("end_offset", -1), ) + LOG.debug("❌ Matched unexpected test data for: %s", pattern.type) ok_test = False if not ok_test: + LOG.debug("❌ Test OK flag set to False for %s:", pattern.type) ret = False else: @@ -731,12 +736,14 @@ def test_patterns( err, ) + LOG.debug("❌ Matched an unexpected result for %s", pattern.type) ok = False if ok and not quiet: LOG.info("✅ '%s' in '%s'", pattern.type, rel_dirpath) if not ok: + LOG.debug("❌ ok flag set to False for %s", pattern.type) ret = False else: @@ -1237,6 +1244,7 @@ def main() -> None: ) and not args.continue_on_fail ): + LOG.debug("Testing patterns returned False") exit(1) db, patterns = build_hyperscan_patterns( From 74036a910f07e7ae42dcb4f305814791a16cde89 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Mon, 12 Feb 2024 17:37:37 +0000 Subject: [PATCH 3/9] Rename test pattern --- examples/config/patterns.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/config/patterns.yml b/examples/config/patterns.yml index b14d007..7f166ce 100644 --- a/examples/config/patterns.yml +++ b/examples/config/patterns.yml @@ -3,7 +3,7 @@ name: Configuration Secrets patterns: - name: Django Secret Key - type: django_secret_key + type: test_django_secret_key_test regex: version: 0.1 # required From 4632e5fbff3e96a8d0c1ac7fbec8b511bf73a5f2 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Mon, 12 Feb 2024 17:40:26 +0000 Subject: [PATCH 4/9] Fixed type in script --- examples/update_custom_patterns_readme.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/update_custom_patterns_readme.sh b/examples/update_custom_patterns_readme.sh index 755b1d7..d2b1410 100755 --- a/examples/update_custom_patterns_readme.sh +++ b/examples/update_custom_patterns_readme.sh @@ -1,11 +1,13 @@ #!/bin/bash -GITHUB_REPOSITORY="{$GITHUB_REPOSITORY:-advanced-security/secret-scanning-custom-patterns}" +GITHUB_REPOSITORY="${GITHUB_REPOSITORY:-advanced-security/secret-scanning-custom-patterns}" CUSTOM_PATTERNS_PATH="${CUSTOM_PATTERNS_PATH:-$HOME/secret-scanning-custom-patterns}" +cd .. pipenv run markdown --github-repository "${GITHUB_REPOSITORY}" -p "${CUSTOM_PATTERNS_PATH}" cd "${CUSTOM_PATTERNS_PATH}" || exit 1 find . -type f -name 'README.md' -exec git add {} \; git commit -S -m "Updated README.md" git push + From f5f553fb32b13c962d9c94ce9632ba4e110ffc9f Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 14 Feb 2024 10:59:14 +0000 Subject: [PATCH 5/9] Made updating README script work when invoked from outside the 'examples' directory --- examples/update_custom_patterns_readme.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/update_custom_patterns_readme.sh b/examples/update_custom_patterns_readme.sh index d2b1410..a14f869 100755 --- a/examples/update_custom_patterns_readme.sh +++ b/examples/update_custom_patterns_readme.sh @@ -1,9 +1,11 @@ #!/bin/bash +SCRIPT_PATH="$(dirname -- "${BASH_SOURCE[0]}")" + GITHUB_REPOSITORY="${GITHUB_REPOSITORY:-advanced-security/secret-scanning-custom-patterns}" CUSTOM_PATTERNS_PATH="${CUSTOM_PATTERNS_PATH:-$HOME/secret-scanning-custom-patterns}" -cd .. +cd "${SCRIPT_PATH}"/.. || exit 1 pipenv run markdown --github-repository "${GITHUB_REPOSITORY}" -p "${CUSTOM_PATTERNS_PATH}" cd "${CUSTOM_PATTERNS_PATH}" || exit 1 From be9249677fb8794d86827ee1416929c832ead1fd Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:10:20 +0000 Subject: [PATCH 6/9] Updated README.md --- README.md | 128 +++--------------------------------------------------- 1 file changed, 6 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index 8ba3aea..4908e14 100644 --- a/README.md +++ b/README.md @@ -1,130 +1,14 @@ -# Secret Scanning Tools +# custom-pattern-secrets -> ℹ️ This is an _unofficial_ tool created by Field Security Services, and is not officially supported by GitHub. +Custom Secret Scanning Patterns repository. -This is a testing suite for GitHub Secret Scanning Custom Patterns. +## Patterns -It can be used in combination with GitHub Actions to test custom patterns before they are deployed. -An example repository that uses this Action is [advanced-security/secret-scanning-custom-patterns](https://github.com/advanced-security/secret-scanning-custom-patterns). -A sample custom patterns config file compatible with this tool suite is provided in [`examples/config/patterns.yml`](examples/config/patterns.yml). +### [Configuration Secrets](./examples/config) -## Usage in Actions -```yaml -- name: Secret Scanning Test Suite - uses: advanced-security/secret-scanning-tools@v1 -``` -### Advanced Configuration - -```yaml -- name: Secret Scanning Test Suite - uses: advanced-security/secret-scanning-tools@v1 - with: - # Modes to run - # > 'validate' (default), 'all', 'snapshot', 'markdown' - mode: 'validate' -``` - -### Using GitHub App Token - -```yaml -- name: Get Token - id: get_workflow_token - uses: peter-murray/workflow-application-token-action@v1 - with: - application_id: ${{ secrets.ADVANCED_SECURITY_APP_ID }} - application_private_key: ${{ secrets.ADVANCED_SECURITY_APP_KEY }} - -- name: Secret Scanning Test Suite - uses: advanced-security/secret-scanning-tools@v1 - with: - token: ${{ steps.get_workflow_token.outputs.token }} -``` - -### Using in Enterprise Server or with self-hosted Actions runners - -The Action will set up Python on GitHub Hosted runners or on runners with a configured tool cache. - -If you are using a self-hosted runner, or Enterprise Server, and you have not set up the tool cache, you will need to set up Python yourself (version 3.9+) - -You can do that in the workflow, or by installing it permanently into your runner's image and including it in the path of the Actions runner. - -## Defining expected results for online testing - -Create a snapshot of the results of a pattern by running the action with `snapshot`. - -This is checked in the `validate` mode, to check for changes in the results of the pattern. Check that these changes are expected, and if so, update the snapshot, or fix the pattern, as necessary. - -> ℹ️ the _earliest commit_ that a secret has been found at is reported by secret scanning, so it is not possible to cleanly define a current state of expected secrets in the repository, and test for those expected results. Instead, we use this overall snapshot approach. -> -> In contrast, [offline testing, below](#offline-testing-of-secret-scanning-custom-patterns) can be used to test for expected results, since it can be run on a single commit of a repository. - -## Using locally with pipenv - -Install the requirements using `pipenv install`, then run `pipenv run ` to run the commands: `markdown`, `validate`, `snapshot`. - -See [this sample script](./examples/update_custom_patterns_readme.sh) for how to update the `README.md` files for your custom patterns. - -## Offline testing of Secret Scanning custom patterns - -We have a test Python script, `secretscanning/test.py` that uses Intel's `hyperscan` to test custom GitHub Advanced Security Secret Scanning patterns. - -This is useful for thorough testing of patterns before they are deployed, whereas the rest of the test suite is primarily designed to be run in GitHub Actions for testing in CI. - -### Local test script usage - -Change directory to `secretscanning`. - -First run `make requirements` to install required dependencies. - -``` bash -./test.py -``` - -By default it searches the directory above the `testing` directory for `pattern.yml` files, and tests those patterns on the same directory that file was found in. - -or - -``` bash -./test.py --tests -``` - -For full usage use `./test.py --help` - -### Local test script requirements - -This only works on Intel-compatible platforms, since `hyperscan` is an Intel application and written to use Intel-specific instructions. - -The packages can be installed using `make requirements` on Ubuntu-compatible platforms. On other platforms, install the following dependencies: - -* Python 3.9+ -* Python packages (listed in `requirements.txt`) - * `hyperscan` - * `python-pcre` (requires `libpcre3`) - * `pygit2` - * `GitPython` - -### Development notes - -Please run `make lint` after any changes - -## License - -This project is licensed under the terms of the MIT open source license. Please refer to the [LICENSE](LICENSE) for the full terms. - -## Maintainers - -See [CODEOWNERS](CODEOWNERS) for the list of maintainers. - -## Support - -> ℹ️ This is an _unofficial_ tool created by Field Security Services, and is not officially supported by GitHub. - -See the [SUPPORT](SUPPORT.md) file. - -## Background - -See the [CHANGELOG](CHANGELOG.md), [CONTRIBUTING](CONTRIBUTING.md), [SECURITY](SECURITY.md), [SUPPORT](SUPPORT.md), [CODE OF CONDUCT](CODE_OF_CONDUCT.md) and [PRIVACY](PRIVACY.md) files for more information. +- Django Secret Key + \ No newline at end of file From 44439b91bbed2491994210b7e56d0258f3794a9a Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 26 Mar 2024 18:10:09 +0000 Subject: [PATCH 7/9] Better debug --- secretscanning/test.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/secretscanning/test.py b/secretscanning/test.py index d062fb2..7726ed8 100755 --- a/secretscanning/test.py +++ b/secretscanning/test.py @@ -91,6 +91,7 @@ def __init__( if self.test_data["end_offset"] == -1: self.test_data["end_offset"] = len(str(self.test_data["data"])) self.test_data["name"] = None + self.test_data["data"] = self.test_data["data"].strip() def regex_string(self) -> bytes: """Concatenate and UTF-8 encode.""" @@ -551,18 +552,24 @@ def test_patterns( for res in pattern_results: LOG.error("%s%s%s", res['groups']['start'], Fore.RED + res['groups']['pattern'] + Style.RESET_ALL, res['groups']['end']) ok_test = False + elif len(pattern_results) == 0: + LOG.error("❌ no matches on test data on '%s' in '%s'", pattern.type, rel_dirpath) + ok_test = False else: - # did we match what we expected? - if len(pattern_results) == 0 or not path_offsets_match( - pattern.test_data, pattern_results[0].get("file", {}) - ): + result = pattern_results[0].get("file", {}) + if not path_offsets_match(pattern.test_data, result): LOG.error( - "❌ did not match test data for '%s': '%s':%d-%d ", + "❌ did not match test data for '%s': '%s':%d-%d", pattern.type, pattern.test_data["data"], pattern.test_data["start_offset"], pattern.test_data["end_offset"], ) + LOG.error( + "❌ matched: %d-%d", + result["start_offset"], + result["end_offset"], + ) ok_test = False # did we match anything unexpected? @@ -623,8 +630,12 @@ def test_patterns( for filename in [f for f in filenames if f not in FILENAME_EXCLUDES]: path = (Path(dirpath) / filename).relative_to(tests_path) with (Path(tests_path) / path).resolve().open("rb") as f: + LOG.debug("Scanning file %s/%s", tests_path, path) + content = f.read() + LOG.debug(content) + # sideffect: writes to global RESULTS scan( db, @@ -645,6 +656,7 @@ def test_patterns( if pattern.expected: for expected in pattern.expected: pattern_results = RESULTS.get(pattern.name, []) + LOG.debug("Pattern results: %s", pattern_results) if not any( [ path_offsets_match(expected, result.get("file", {})) From b9faaf005d0dfafe9b3153fa607a9970b46cd11d Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 26 Mar 2024 18:12:29 +0000 Subject: [PATCH 8/9] Restored correct README --- README.md | 128 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 122 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4908e14..8ba3aea 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,130 @@ -# custom-pattern-secrets +# Secret Scanning Tools -Custom Secret Scanning Patterns repository. +> ℹ️ This is an _unofficial_ tool created by Field Security Services, and is not officially supported by GitHub. -## Patterns +This is a testing suite for GitHub Secret Scanning Custom Patterns. +It can be used in combination with GitHub Actions to test custom patterns before they are deployed. +An example repository that uses this Action is [advanced-security/secret-scanning-custom-patterns](https://github.com/advanced-security/secret-scanning-custom-patterns). -### [Configuration Secrets](./examples/config) +A sample custom patterns config file compatible with this tool suite is provided in [`examples/config/patterns.yml`](examples/config/patterns.yml). +## Usage in Actions +```yaml +- name: Secret Scanning Test Suite + uses: advanced-security/secret-scanning-tools@v1 +``` -- Django Secret Key - \ No newline at end of file +### Advanced Configuration + +```yaml +- name: Secret Scanning Test Suite + uses: advanced-security/secret-scanning-tools@v1 + with: + # Modes to run + # > 'validate' (default), 'all', 'snapshot', 'markdown' + mode: 'validate' +``` + +### Using GitHub App Token + +```yaml +- name: Get Token + id: get_workflow_token + uses: peter-murray/workflow-application-token-action@v1 + with: + application_id: ${{ secrets.ADVANCED_SECURITY_APP_ID }} + application_private_key: ${{ secrets.ADVANCED_SECURITY_APP_KEY }} + +- name: Secret Scanning Test Suite + uses: advanced-security/secret-scanning-tools@v1 + with: + token: ${{ steps.get_workflow_token.outputs.token }} +``` + +### Using in Enterprise Server or with self-hosted Actions runners + +The Action will set up Python on GitHub Hosted runners or on runners with a configured tool cache. + +If you are using a self-hosted runner, or Enterprise Server, and you have not set up the tool cache, you will need to set up Python yourself (version 3.9+) + +You can do that in the workflow, or by installing it permanently into your runner's image and including it in the path of the Actions runner. + +## Defining expected results for online testing + +Create a snapshot of the results of a pattern by running the action with `snapshot`. + +This is checked in the `validate` mode, to check for changes in the results of the pattern. Check that these changes are expected, and if so, update the snapshot, or fix the pattern, as necessary. + +> ℹ️ the _earliest commit_ that a secret has been found at is reported by secret scanning, so it is not possible to cleanly define a current state of expected secrets in the repository, and test for those expected results. Instead, we use this overall snapshot approach. +> +> In contrast, [offline testing, below](#offline-testing-of-secret-scanning-custom-patterns) can be used to test for expected results, since it can be run on a single commit of a repository. + +## Using locally with pipenv + +Install the requirements using `pipenv install`, then run `pipenv run ` to run the commands: `markdown`, `validate`, `snapshot`. + +See [this sample script](./examples/update_custom_patterns_readme.sh) for how to update the `README.md` files for your custom patterns. + +## Offline testing of Secret Scanning custom patterns + +We have a test Python script, `secretscanning/test.py` that uses Intel's `hyperscan` to test custom GitHub Advanced Security Secret Scanning patterns. + +This is useful for thorough testing of patterns before they are deployed, whereas the rest of the test suite is primarily designed to be run in GitHub Actions for testing in CI. + +### Local test script usage + +Change directory to `secretscanning`. + +First run `make requirements` to install required dependencies. + +``` bash +./test.py +``` + +By default it searches the directory above the `testing` directory for `pattern.yml` files, and tests those patterns on the same directory that file was found in. + +or + +``` bash +./test.py --tests +``` + +For full usage use `./test.py --help` + +### Local test script requirements + +This only works on Intel-compatible platforms, since `hyperscan` is an Intel application and written to use Intel-specific instructions. + +The packages can be installed using `make requirements` on Ubuntu-compatible platforms. On other platforms, install the following dependencies: + +* Python 3.9+ +* Python packages (listed in `requirements.txt`) + * `hyperscan` + * `python-pcre` (requires `libpcre3`) + * `pygit2` + * `GitPython` + +### Development notes + +Please run `make lint` after any changes + +## License + +This project is licensed under the terms of the MIT open source license. Please refer to the [LICENSE](LICENSE) for the full terms. + +## Maintainers + +See [CODEOWNERS](CODEOWNERS) for the list of maintainers. + +## Support + +> ℹ️ This is an _unofficial_ tool created by Field Security Services, and is not officially supported by GitHub. + +See the [SUPPORT](SUPPORT.md) file. + +## Background + +See the [CHANGELOG](CHANGELOG.md), [CONTRIBUTING](CONTRIBUTING.md), [SECURITY](SECURITY.md), [SUPPORT](SUPPORT.md), [CODE OF CONDUCT](CODE_OF_CONDUCT.md) and [PRIVACY](PRIVACY.md) files for more information. From 975d8e40af6381b40f3d7dcb7c428906627688ea Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 10 Jul 2024 18:03:05 +0100 Subject: [PATCH 9/9] Improve combining script --- secretscanning/combine.py | 65 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/secretscanning/combine.py b/secretscanning/combine.py index e762b6f..5154994 100755 --- a/secretscanning/combine.py +++ b/secretscanning/combine.py @@ -5,6 +5,7 @@ config files into one for eaasy upload using the Field browser extension """ +import fnmatch import yaml import json import logging @@ -12,7 +13,7 @@ import sys import argparse from pathlib import Path -from typing import Any +from typing import Any, List, Dict LOG = logging.getLogger(__name__) @@ -25,6 +26,28 @@ def add_args(parser: argparse.ArgumentParser) -> None: parser.add_argument( "input_dir", help="Directory with custom pattern config files in YAML format" ) + parser.add_argument( + "--exclude-type", type=str, nargs="+", help="Exclude patterns with a 'type' with these globs" + ) + parser.add_argument( + "--exclude-name", type=str, nargs="+", help="Exclude patterns with a 'name' with these globs" + ) + parser.add_argument( + "--include-type", type=str, nargs="+", help="Include patterns with a 'name' with these globs" + ) + parser.add_argument( + "--include-name", type=str, nargs="+", help="Include patterns with a 'name' with these globs" + ) + + +def glob_match(field: str, exclude: List[str]) -> bool: + """Check if field matches any of the exclude globs, using globbing library.""" + if exclude is None or not exclude: + return False + for pattern in exclude: + if fnmatch.fnmatch(field, pattern): + return True + return False def main() -> None: @@ -38,7 +61,9 @@ def main() -> None: if args.debug: LOG.setLevel(logging.DEBUG) - patterns = [] + LOG.debug(args.include_name) + + patterns: List[Dict[str, Any]] = [] # find patterns.yml in directory by walking it for root, dirs, filenames in os.walk(args.input_dir): @@ -51,7 +76,41 @@ def main() -> None: data = yaml.safe_load(f) if "patterns" in data: - patterns.extend(data["patterns"]) + for pattern in data["patterns"]: + include = True + if args.include_name is not None or args.include_type is not None: + include = False + if "name" in pattern and args.include_name is not None: + name = pattern.get("name", None) + if glob_match(name, args.include_name): + include = True + else: + LOG.debug("Excluding pattern named: %s", name) + if "type" in pattern and args.include_type is not None: + type_ = pattern.get("type", None) + if glob_match(type_, args.include_type): + include = True + else: + LOG.debug("Excluding pattern 'type': %s", type_) + if "type" in pattern and args.exclude_type is not None: + type_ = pattern.get("type", None) + if not glob_match(type_, args.exclude_type): + pass + else: + if include: + include = False + LOG.debug("Excluding pattern 'type': %s", type_) + if "name" in pattern and args.exclude_name is not None: + name = pattern.get("name", None) + if not glob_match(name, args.exclude_name): + pass + else: + if include: + include = False + LOG.debug("Excluding pattern 'name': %s", name) + if include: + patterns.append(pattern) + print(yaml.dump({"name": "Collection of custom patterns", "patterns": patterns}))