Merge pull request #782 from pafmaf/plugin/gitlab_tokens

introducing GitLab Plugin analogous to GitHubTokenDetector

README.md

@@ -98,6 +98,7 @@ BasicAuthDetector
 CloudantDetector
 DiscordBotTokenDetector
 GitHubTokenDetector
+GitLabTokenDetector
 Base64HighEntropyString
 HexHighEntropyString
 IbmCloudIamDetector

detect_secrets/plugins/gitlab_token.py

@@ -0,0 +1,59 @@
+"""
+This plugin searches for GitLab tokens
+"""
+import re
+
+from detect_secrets.plugins.base import RegexBasedDetector
+
+
+class GitLabTokenDetector(RegexBasedDetector):
+    """Scans for GitLab tokens."""
+
+    secret_type = 'GitLab Token'
+
+    denylist = [
+        # ref:
+        #  - https://docs.gitlab.com/ee/security/token_overview.html#gitlab-tokens
+        #  - https://gitlab.com/groups/gitlab-org/-/epics/8923
+        #  - https://github.com/gitlabhq/gitlabhq/blob/master/gems
+        #        /gitlab-secret_detection/lib/gitleaks.toml#L6-L76
+
+        # `gl..-` prefix and a token of length >20
+        #    characters are typically alphanumeric, underscore, dash
+        # Most tokens are generated either with:
+        #  - `Devise.friendly_token`, a string with a default length of 20, or
+        #  - `SecureRandom.hex`, default data size of 16 bytes, encoded in different ways.
+        # String length may vary depending on the type of token, and probably
+        # even GL-settings in the future, so we expect between 20 and 50 chars.
+
+        # Personal Access Token - glpat
+        # Deploy Token - gldt
+        # Feed Token - glft
+        # OAuth Access Token - glsoat
+        # Runner Token - glrt
+        re.compile(
+            r'(glpat|gldt|glft|glsoat|glrt)-'
+            r'[A-Za-z0-9_\-]{20,50}(?!\w)',
+        ),
+
+        # Runner Registration Token
+        re.compile(r'GR1348941[A-Za-z0-9_\-]{20,50}(?!\w)'),
+
+        # CI/CD Token - `glcbt` or `glcbt-XY_` where XY is a 2-char hex 'partition_id'
+        re.compile(r'glcbt-([0-9a-fA-F]{2}_)?[A-Za-z0-9_\-]{20,50}(?!\w)'),
+
+        # Incoming Mail Token - generated by SecureRandom.hex, default length 16 bytes
+        # resulting token length is 26 when Base-36 encoded
+        re.compile(r'glimt-[A-Za-z0-9_\-]{25}(?!\w)'),
+
+        # Trigger Token - generated by `SecureRandom.hex(20)`
+        re.compile(r'glptt-[A-Za-z0-9_\-]{40}(?!\w)'),
+
+        # Agent Token - generated by `Devise.friendly_token(50)`
+        # tokens have a minimum length of 50 chars, up to 1024 chars
+        re.compile(r'glagent-[A-Za-z0-9_\-]{50,1024}(?!\w)'),
+
+        # GitLab OAuth Application Secret - generated by `SecureRandom.hex(32)`
+        # -> becomes 64 base64-encoded characters
+        re.compile(r'gloas-[A-Za-z0-9_\-]{64}(?!\w)'),
+    ]

tests/plugins/gitlab_token_test.py

@@ -0,0 +1,138 @@
+import pytest
+
+from detect_secrets.plugins.gitlab_token import GitLabTokenDetector
+
+
+class TestGitLabTokenDetector:
+    @pytest.mark.parametrize(
+        'payload, should_flag',
+        [
+            (
+                # valid PAT prefix and token length
+                'glpat-hellOworld380_testin',
+                True,
+            ),
+            (
+                # spaces are not part of the token
+                'glpat-hellOWorld380 testin',
+                False,
+            ),
+            (
+                # invalid separator (underscore VS dash)
+                'glpat_hellOworld380_testin',
+                False,
+            ),
+            (
+                # valid different prefix and token length
+                'gldt-HwllOuhfw-wu0rlD_yep',
+                True,
+            ),
+            (
+                # token < 20 chars should be too short
+                'gldt-seems_too000Sshorty',
+                False,
+            ),
+            (
+                # invalid prefix, but valid token length
+                'foo-hello-world80_testin',
+                False,
+            ),
+            (
+                # token length may vary depending on the impl., but <= 50 chars should be fine
+                'glsoat-PREfix_helloworld380_testin_pretty_long_token_long',
+                True,
+            ),
+            (
+                # token > 50 chars is too long
+                'glsoat-PREfix_helloworld380_testin_pretty_long_token_long_',
+                False,
+            ),
+            (
+                # GitLab is not GitHub
+                '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',
+                False,
+            ),
+        ],
+    )
+    def test_base_token_format(self, payload, should_flag):
+        logic = GitLabTokenDetector()
+        output = logic.analyze_line(filename='mock_filename', line=payload)
+        assert len(output) == int(should_flag)
+
+    @pytest.mark.parametrize(
+        'payload, should_flag',
+        [
+            ('GR1348941PREfix_helloworld380', True),
+            ('GR1348941PREfix_helloworld380_testin_pretty_long_token_long', True),
+            ('GR1348941PREfix_helloworld380_testin_pretty_long_token_long_', False),  # too long
+            ('GR1348941helloWord0', False),  # too short
+        ],
+    )
+    def test_runner_registration_token(self, payload, should_flag):
+        logic = GitLabTokenDetector()
+        output = logic.analyze_line(filename='mock_filename', line=payload)
+        assert len(output) == int(should_flag)
+
+    @pytest.mark.parametrize(
+        'payload, should_flag',
+        [
+            ('glcbt-helloworld380_testin', True),
+        ],
+    )
+    def test_cicd_token(self, payload, should_flag):
+        logic = GitLabTokenDetector()
+        output = logic.analyze_line(filename='mock_filename', line=payload)
+        assert len(output) == int(should_flag)
+
+    @pytest.mark.parametrize(
+        'payload, should_flag',
+        [
+            ('glimt-my-tokens_are-correctAB38', True),
+            ('glimt-my-tokens_are-correctAB', False),  # too short
+            ('glimt-my-tokens_are-correctAB38_280', False),  # too long
+        ],
+    )
+    def test_incoming_mail_token(self, payload, should_flag):
+        logic = GitLabTokenDetector()
+        output = logic.analyze_line(filename='mock_filename', line=payload)
+        assert len(output) == int(should_flag)
+
+    @pytest.mark.parametrize(
+        'payload, should_flag',
+        [
+            ('glptt-Need5_T00-be-exactly-40-chars--ELse_fail', True),
+            ('glptt-Need5_T00-be-exactly-40-chars--ELse_failing', False),  # too long
+            ('glptt-hellOworld380_testin', False),  # too short
+        ],
+    )
+    def test_trigger_token(self, payload, should_flag):
+        logic = GitLabTokenDetector()
+        output = logic.analyze_line(filename='mock_filename', line=payload)
+        assert len(output) == int(should_flag)
+
+    @pytest.mark.parametrize(
+        'payload, should_flag',
+        [
+            ('glagent-Need5_T00-bee-longer-than-50_chars-or-else-failING', True),
+            ('glagent-Need5_T00-bee-longer-than-50_chars-or-else-failING-still_OK', True),
+            (('glagent-' + 'X' * 1025), False),  # 2 long
+            ('glagent-hellOworld380_testin', False),  # len 20 is too short
+        ],
+    )
+    def test_agent_token(self, payload, should_flag):
+        logic = GitLabTokenDetector()
+        output = logic.analyze_line(filename='mock_filename', line=payload)
+        assert len(output) == int(should_flag)
+
+    @pytest.mark.parametrize(
+        'payload, should_flag',
+        [
+            ('gloas-checking_Length-Is-_exactly_64--checking_Length-Is-_exactly_64--', True),
+            ('gloas-checking_Length-Is-checking_Length-Is-', False),  # too short
+            ('gloas-checking_Length-Is-_exactly_64--Xchecking_Length-Is-_longer_longer', False),
+        ],
+    )
+    def test_oauth_application_secret(self, payload, should_flag):
+        logic = GitLabTokenDetector()
+        output = logic.analyze_line(filename='mock_filename', line=payload)
+        assert len(output) == int(should_flag)