Merge pull request #111 from IBM/test_new_ghe_token

Update GHE plugin token regex
IBM · Dec 29, 2022 · 63375ed · 63375ed
2 parents d732e5e + bd2c5db
commit 63375ed
Show file tree

Hide file tree

Showing 5 changed files with 68 additions and 23 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -18,23 +18,17 @@ deploy:
           repo: IBM/detect-secrets
 matrix:
     include:
-        # - env: TOXENV=py35
-        #   python: 3.5
-        - env: TOXENV=py36
-          python: 3.6.15 # We're targeting a specific patch version; if set to 3.6, the GitHub Travis build will use Python 3.6.7 and will fail due to a cryptography installation error
         - env: TOXENV=py37
           python: 3.7.13
           dist: xenial # Required for Python >= 3.7 (travis-ci/travis-ci#9069), the GitHub Travis build will use Python 3.7.1 by default if you provide 3.7 without a patch version and the build will fail with AttributeError: 'str' object has no attribute 'name'
         - env: TOXENV=py38
           python: 3.8
           dist: xenial # Required for Python >= 3.7 (travis-ci/travis-ci#9069)
-        # - env: TOXENV=pypy
-        #   python: pypy
 before_install:
     - echo -e "machine github.com\n  login $GH_ACCESS_TOKEN" >> ~/.netrc # Login to GitHub
     - echo -e "machine github.ibm.com\n  login $GHE_ACCESS_TOKEN" >> ~/.netrc # Login to GitHub Enterprise
 install:
-    - pip install "certifi>=2022.12.07" tox pipenv
+    - pip install "certifi>=2022.12.07" "setuptools>=65.5.1" tox pipenv
 script: make setup-trivy && make trivy-scan-python-vulnerabilities && make test
 cache:
     directories:

diff --git a/detect_secrets/__init__.py b/detect_secrets/__init__.py
@@ -1 +1 @@
-VERSION = '0.13.1+ibm.55.dss'
+VERSION = '0.13.1+ibm.56.dss'
diff --git a/detect_secrets/plugins/github_enterprise.py b/detect_secrets/plugins/github_enterprise.py
@@ -8,7 +8,10 @@
 
 
 class GheDetector(RegexBasedDetector):
-    """ Scans for GitHub Enterprise credentials """
+    """ Scans for GitHub Enterprise credentials generated both before (see forty_hex var) and
+    after (see new_ghe_token var) this update:
+    https://docs.github.com/en/[email protected]/authentication/keeping-your-account-and-data-secure/about-authentication-to-github#githubs-token-formats
+    """
 
     secret_type = 'GitHub Enterprise Credentials'
     denylist = None
@@ -33,6 +36,10 @@ def __init__(self, ghe_instance=DEFAULT_GHE_INSTANCE, *args, **kwargs):
         api_endpoint = r'(?:{ghe_instance}|api.{ghe_instance})'\
             .format(ghe_instance=self.ghe_instance)
         forty_hex = r'(?:(?<=\W)|(?<=^))([0-9a-f]{40})(?:(?=\W)|(?=$))'
+        # References:
+        # https://docs.github.com/en/[email protected]/authentication/keeping-your-account-and-data-secure/about-authentication-to-github#githubs-token-formats
+        # ref. https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/
+        new_ghe_token = (r'(ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{36}')
         b64_encoded_token = r'(?:(?<=\W)|(?<=^))([A-Za-z0-9+/]{55}=)(?:(?=\W)|(?=$))'
         opt_username = r'(?:[a-zA-Z0-9-]+:|)'
         self.denylist = [
@@ -65,6 +72,7 @@ def __init__(self, ghe_instance=DEFAULT_GHE_INSTANCE, *args, **kwargs):
                     b64_encoded_token=b64_encoded_token,
                 ), flags=re.IGNORECASE,
             ),
+            re.compile(new_ghe_token),
         ]
 
     def verify(self, token, *args, **kwargs):

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -1,4 +1,3 @@
-# on python 3.6.0 (xenial)
 pip>=21.1
 urllib3>=1.26.4
 coverage>=6.0b1

diff --git a/tests/plugins/gh_enterprise_test.py b/tests/plugins/gh_enterprise_test.py
@@ -4,8 +4,9 @@
 from detect_secrets.core.constants import VerifiedResult
 from detect_secrets.plugins.github_enterprise import GheDetector
 
-
-GHE_TOKEN = 'abcdef0123456789abcdef0123456789abcdef01'
+GHE_TOKEN_OLD = 'abcdef0123456789abcdef0123456789abcdef01'
+TOKEN_STRING = 'wWPw5k4aXcaT4fNP0UcnZwJUVFk6LO0pINUx'
+GHE_TOKEN_NEW = 'ghp_' + TOKEN_STRING
 GHE_TOKEN_BYTES = b'abcdef0123456789abcdef0123456789abcdef01'
 
 
@@ -14,6 +15,7 @@ class TestGheDetector(object):
     @pytest.mark.parametrize(
         'payload, should_flag',
         [
+            # Old GitHub Enterprise token format
             ('github-key 2764d47e6bf540911b7da8fe55caa9451e783549', True),
             ('github_pwd :53d49d5081266d939bac57a3d86c517ded974b19', True),
             ('gh-api-key=2764d47e6bf540911b7da8fe55caa9451e783549 ', True),
@@ -63,6 +65,13 @@ class TestGheDetector(object):
                 'YWJjZWRmYWJlZmQzMzMzMTQ1OTA4YWJjZGRmY2JkZGUxMTQ1Njc4OQo=', False,
             ),
             ('Authorization: token %s', False),
+            # New GitHub token format
+            (GHE_TOKEN_NEW, True),
+            ('gho_' + TOKEN_STRING, True),
+            ('ghu_' + TOKEN_STRING, True),
+            ('ghs_' + TOKEN_STRING, True),
+            ('ghr_' + TOKEN_STRING, True),
+            ('new_ghe_token: abcdef0123456789abcdef0123456789abcdef01', False),  # missing prefix
         ],
     )
     def test_analyze_line(self, payload, should_flag):
@@ -75,17 +84,26 @@ def test_analyze_line(self, payload, should_flag):
         'payload, should_flag',
         [
             (
-                'https://username:abcdef0123456789abcdef0123456789abcdef01@'
-                'github.somecompany.com', True,
+                'https://username:' + GHE_TOKEN_OLD + '@github.somecompany.com', True,
             ),
             (
-                'https://username:abcdef0123456789abcdef0123456789abcdef01@'
-                'api.github.somecompany.com', True,
+                'https://username:' + GHE_TOKEN_OLD + '@api.github.somecompany.com', True,
             ),
-            ('git+https://abcdef0123456789abcdef0123456789abcdef01@github.somecompany.com', True),
+            ('git+https://' + GHE_TOKEN_OLD + '@github.somecompany.com', True),
             (
-                'https://x-oauth-basic:abcdef0123456789abcdef0123456789abcdef01'
-                '@github.somecompany.com/org/repo.git', True,
+                'https://x-oauth-basic:' + GHE_TOKEN_OLD + '@github.somecompany.com/org/repo.git',
+                True,
+            ),
+            (
+                'https://username:' + GHE_TOKEN_NEW + '@github.somecompany.com', True,
+            ),
+            (
+                'https://username:' + GHE_TOKEN_NEW + '@api.github.somecompany.com', True,
+            ),
+            ('git+https://' + GHE_TOKEN_NEW + '@github.somecompany.com', True),
+            (
+                'https://x-oauth-basic:' + GHE_TOKEN_NEW + '@github.somecompany.com/org/repo.git',
+                True,
             ),
         ],
     )
@@ -101,21 +119,21 @@ def test_verify_invalid_secret(self):
             responses.GET, 'https://github.ibm.com/api/v3', status=401,
         )
 
-        assert GheDetector().verify(GHE_TOKEN) == VerifiedResult.VERIFIED_FALSE
+        assert GheDetector().verify(GHE_TOKEN_OLD) == VerifiedResult.VERIFIED_FALSE
 
     @responses.activate
     def test_verify_valid_secret(self):
         responses.add(
             responses.GET, 'https://github.ibm.com/api/v3', status=200,
         )
-        assert GheDetector().verify(GHE_TOKEN) == VerifiedResult.VERIFIED_TRUE
+        assert GheDetector().verify(GHE_TOKEN_OLD) == VerifiedResult.VERIFIED_TRUE
 
     @responses.activate
     def test_verify_status_not_200_or_401(self):
         responses.add(
             responses.GET, 'https://github.ibm.com/api/v3', status=500,
         )
-        assert GheDetector().verify(GHE_TOKEN) == VerifiedResult.UNVERIFIED
+        assert GheDetector().verify(GHE_TOKEN_OLD) == VerifiedResult.UNVERIFIED
 
     @responses.activate
     def test_verify_invalid_secret_bytes(self):
@@ -141,4 +159,30 @@ def test_verify_status_not_200_or_401_bytes(self):
 
     @responses.activate
     def test_verify_unverified_secret(self):
-        assert GheDetector().verify(GHE_TOKEN) == VerifiedResult.UNVERIFIED
+        assert GheDetector().verify(GHE_TOKEN_OLD) == VerifiedResult.UNVERIFIED
+
+    @responses.activate
+    def test_verify_invalid_secret_new(self):
+        responses.add(
+            responses.GET, 'https://github.ibm.com/api/v3', status=401,
+        )
+
+        assert GheDetector().verify(GHE_TOKEN_NEW) == VerifiedResult.VERIFIED_FALSE
+
+    @responses.activate
+    def test_verify_status_not_200_or_401_new(self):
+        responses.add(
+            responses.GET, 'https://github.ibm.com/api/v3', status=500,
+        )
+        assert GheDetector().verify(GHE_TOKEN_NEW) == VerifiedResult.UNVERIFIED
+
+    @responses.activate
+    def test_verify_valid_secret_new(self):
+        responses.add(
+            responses.GET, 'https://github.ibm.com/api/v3', status=200,
+        )
+        assert GheDetector().verify(GHE_TOKEN_NEW) == VerifiedResult.VERIFIED_TRUE
+
+    @responses.activate
+    def test_verify_unverified_secret_new(self):
+        assert GheDetector().verify(GHE_TOKEN_NEW) == VerifiedResult.UNVERIFIED
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		VERSION = '0.13.1+ibm.55.dss'
		VERSION = '0.13.1+ibm.56.dss'