diff --git a/.travis.yml b/.travis.yml index 533351e6c..0a2e8661f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,23 +18,17 @@ deploy: repo: IBM/detect-secrets matrix: include: - # - env: TOXENV=py35 - # python: 3.5 - - env: TOXENV=py36 - python: 3.6.15 # We're targeting a specific patch version; if set to 3.6, the GitHub Travis build will use Python 3.6.7 and will fail due to a cryptography installation error - env: TOXENV=py37 python: 3.7.13 dist: xenial # Required for Python >= 3.7 (travis-ci/travis-ci#9069), the GitHub Travis build will use Python 3.7.1 by default if you provide 3.7 without a patch version and the build will fail with AttributeError: 'str' object has no attribute 'name' - env: TOXENV=py38 python: 3.8 dist: xenial # Required for Python >= 3.7 (travis-ci/travis-ci#9069) - # - env: TOXENV=pypy - # python: pypy before_install: - echo -e "machine github.com\n login $GH_ACCESS_TOKEN" >> ~/.netrc # Login to GitHub - echo -e "machine github.ibm.com\n login $GHE_ACCESS_TOKEN" >> ~/.netrc # Login to GitHub Enterprise install: - - pip install "certifi>=2022.12.07" tox pipenv + - pip install "certifi>=2022.12.07" "setuptools>=65.5.1" tox pipenv script: make setup-trivy && make trivy-scan-python-vulnerabilities && make test cache: directories: diff --git a/detect_secrets/__init__.py b/detect_secrets/__init__.py index cd7314c61..c5d2df46f 100644 --- a/detect_secrets/__init__.py +++ b/detect_secrets/__init__.py @@ -1 +1 @@ -VERSION = '0.13.1+ibm.55.dss' +VERSION = '0.13.1+ibm.56.dss' diff --git a/detect_secrets/plugins/github_enterprise.py b/detect_secrets/plugins/github_enterprise.py index be30a8778..54fd569c0 100644 --- a/detect_secrets/plugins/github_enterprise.py +++ b/detect_secrets/plugins/github_enterprise.py @@ -8,7 +8,10 @@ class GheDetector(RegexBasedDetector): - """ Scans for GitHub Enterprise credentials """ + """ Scans for GitHub Enterprise credentials generated both before (see forty_hex var) and + after (see new_ghe_token var) this update: + https://docs.github.com/en/enterprise-server@3.2/authentication/keeping-your-account-and-data-secure/about-authentication-to-github#githubs-token-formats + """ secret_type = 'GitHub Enterprise Credentials' denylist = None @@ -33,6 +36,10 @@ def __init__(self, ghe_instance=DEFAULT_GHE_INSTANCE, *args, **kwargs): api_endpoint = r'(?:{ghe_instance}|api.{ghe_instance})'\ .format(ghe_instance=self.ghe_instance) forty_hex = r'(?:(?<=\W)|(?<=^))([0-9a-f]{40})(?:(?=\W)|(?=$))' + # References: + # https://docs.github.com/en/enterprise-server@3.2/authentication/keeping-your-account-and-data-secure/about-authentication-to-github#githubs-token-formats + # ref. https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/ + new_ghe_token = (r'(ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{36}') b64_encoded_token = r'(?:(?<=\W)|(?<=^))([A-Za-z0-9+/]{55}=)(?:(?=\W)|(?=$))' opt_username = r'(?:[a-zA-Z0-9-]+:|)' self.denylist = [ @@ -65,6 +72,7 @@ def __init__(self, ghe_instance=DEFAULT_GHE_INSTANCE, *args, **kwargs): b64_encoded_token=b64_encoded_token, ), flags=re.IGNORECASE, ), + re.compile(new_ghe_token), ] def verify(self, token, *args, **kwargs): diff --git a/requirements-dev.txt b/requirements-dev.txt index 7b1ffd00a..b85306572 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,3 @@ -# on python 3.6.0 (xenial) pip>=21.1 urllib3>=1.26.4 coverage>=6.0b1 diff --git a/tests/plugins/gh_enterprise_test.py b/tests/plugins/gh_enterprise_test.py index 0b681e3ce..69c9c12d2 100644 --- a/tests/plugins/gh_enterprise_test.py +++ b/tests/plugins/gh_enterprise_test.py @@ -4,8 +4,9 @@ from detect_secrets.core.constants import VerifiedResult from detect_secrets.plugins.github_enterprise import GheDetector - -GHE_TOKEN = 'abcdef0123456789abcdef0123456789abcdef01' +GHE_TOKEN_OLD = 'abcdef0123456789abcdef0123456789abcdef01' +TOKEN_STRING = 'wWPw5k4aXcaT4fNP0UcnZwJUVFk6LO0pINUx' +GHE_TOKEN_NEW = 'ghp_' + TOKEN_STRING GHE_TOKEN_BYTES = b'abcdef0123456789abcdef0123456789abcdef01' @@ -14,6 +15,7 @@ class TestGheDetector(object): @pytest.mark.parametrize( 'payload, should_flag', [ + # Old GitHub Enterprise token format ('github-key 2764d47e6bf540911b7da8fe55caa9451e783549', True), ('github_pwd :53d49d5081266d939bac57a3d86c517ded974b19', True), ('gh-api-key=2764d47e6bf540911b7da8fe55caa9451e783549 ', True), @@ -63,6 +65,13 @@ class TestGheDetector(object): 'YWJjZWRmYWJlZmQzMzMzMTQ1OTA4YWJjZGRmY2JkZGUxMTQ1Njc4OQo=', False, ), ('Authorization: token %s', False), + # New GitHub token format + (GHE_TOKEN_NEW, True), + ('gho_' + TOKEN_STRING, True), + ('ghu_' + TOKEN_STRING, True), + ('ghs_' + TOKEN_STRING, True), + ('ghr_' + TOKEN_STRING, True), + ('new_ghe_token: abcdef0123456789abcdef0123456789abcdef01', False), # missing prefix ], ) def test_analyze_line(self, payload, should_flag): @@ -75,17 +84,26 @@ def test_analyze_line(self, payload, should_flag): 'payload, should_flag', [ ( - 'https://username:abcdef0123456789abcdef0123456789abcdef01@' - 'github.somecompany.com', True, + 'https://username:' + GHE_TOKEN_OLD + '@github.somecompany.com', True, ), ( - 'https://username:abcdef0123456789abcdef0123456789abcdef01@' - 'api.github.somecompany.com', True, + 'https://username:' + GHE_TOKEN_OLD + '@api.github.somecompany.com', True, ), - ('git+https://abcdef0123456789abcdef0123456789abcdef01@github.somecompany.com', True), + ('git+https://' + GHE_TOKEN_OLD + '@github.somecompany.com', True), ( - 'https://x-oauth-basic:abcdef0123456789abcdef0123456789abcdef01' - '@github.somecompany.com/org/repo.git', True, + 'https://x-oauth-basic:' + GHE_TOKEN_OLD + '@github.somecompany.com/org/repo.git', + True, + ), + ( + 'https://username:' + GHE_TOKEN_NEW + '@github.somecompany.com', True, + ), + ( + 'https://username:' + GHE_TOKEN_NEW + '@api.github.somecompany.com', True, + ), + ('git+https://' + GHE_TOKEN_NEW + '@github.somecompany.com', True), + ( + 'https://x-oauth-basic:' + GHE_TOKEN_NEW + '@github.somecompany.com/org/repo.git', + True, ), ], ) @@ -101,21 +119,21 @@ def test_verify_invalid_secret(self): responses.GET, 'https://github.ibm.com/api/v3', status=401, ) - assert GheDetector().verify(GHE_TOKEN) == VerifiedResult.VERIFIED_FALSE + assert GheDetector().verify(GHE_TOKEN_OLD) == VerifiedResult.VERIFIED_FALSE @responses.activate def test_verify_valid_secret(self): responses.add( responses.GET, 'https://github.ibm.com/api/v3', status=200, ) - assert GheDetector().verify(GHE_TOKEN) == VerifiedResult.VERIFIED_TRUE + assert GheDetector().verify(GHE_TOKEN_OLD) == VerifiedResult.VERIFIED_TRUE @responses.activate def test_verify_status_not_200_or_401(self): responses.add( responses.GET, 'https://github.ibm.com/api/v3', status=500, ) - assert GheDetector().verify(GHE_TOKEN) == VerifiedResult.UNVERIFIED + assert GheDetector().verify(GHE_TOKEN_OLD) == VerifiedResult.UNVERIFIED @responses.activate def test_verify_invalid_secret_bytes(self): @@ -141,4 +159,30 @@ def test_verify_status_not_200_or_401_bytes(self): @responses.activate def test_verify_unverified_secret(self): - assert GheDetector().verify(GHE_TOKEN) == VerifiedResult.UNVERIFIED + assert GheDetector().verify(GHE_TOKEN_OLD) == VerifiedResult.UNVERIFIED + + @responses.activate + def test_verify_invalid_secret_new(self): + responses.add( + responses.GET, 'https://github.ibm.com/api/v3', status=401, + ) + + assert GheDetector().verify(GHE_TOKEN_NEW) == VerifiedResult.VERIFIED_FALSE + + @responses.activate + def test_verify_status_not_200_or_401_new(self): + responses.add( + responses.GET, 'https://github.ibm.com/api/v3', status=500, + ) + assert GheDetector().verify(GHE_TOKEN_NEW) == VerifiedResult.UNVERIFIED + + @responses.activate + def test_verify_valid_secret_new(self): + responses.add( + responses.GET, 'https://github.ibm.com/api/v3', status=200, + ) + assert GheDetector().verify(GHE_TOKEN_NEW) == VerifiedResult.VERIFIED_TRUE + + @responses.activate + def test_verify_unverified_secret_new(self): + assert GheDetector().verify(GHE_TOKEN_NEW) == VerifiedResult.UNVERIFIED