Detection Updates

.github/workflows/ci.yml

@@ -8,14 +8,16 @@ on:
     branches: [ master ]
   pull_request:
   workflow_dispatch:
+  schedule:
+    - cron: '0 0 1 * *'
 
 jobs:
   main:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python: ['3.6', '3.7', '3.8', '3.9']
+        python: ['3.9', '3.10', '3.11', '3.12']
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2

.github/workflows/pypi.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python: ['3.6', '3.7', '3.8', '3.9']
+        python: ['3.9', '3.10', '3.11', '3.12']
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2

.gitignore

@@ -11,4 +11,19 @@
 .*ignore
 !.gitignore
 .python-version
-.vscode
+
+# Ignore editor specific configs
+/.idea
+/.vscode
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+.generators
+.rakeTasks
+
+# System Files
+.DS_Store
+Thumbs.db

.secrets.baseline

@@ -1,5 +1,5 @@
 {
-  "version": "1.4.0",
+  "version": "1.5.0",
   "plugins_used": [
     {
       "name": "ArtifactoryDetector"

CHANGELOG.md

@@ -25,9 +25,64 @@ If you love `detect-secrets`, please star our project on GitHub to show your sup
 [@xxxx]: https://github.com/xxxx
 -->
 
-<!--
-### Unreleased
--->
+### v1.5.0
+##### May 6th, 2024
+
+We apologise for the extreme delay in publishing a new release for our beloved `detect-secrets`. We at Yelp appreciate your continued support and your contributions to this valuable project!
+
+#### :newspaper: News
+- We're adding support for Python 3.10, 3.11 and 3.12 and we dropped support for Python 3.6 and 3.7! We hope this won't be too disruptive for you all. Be aware that in a next release, we'll remove support for Python 3.8 too, as it'll reach EOL in October 2024.
+
+#### :mega: Release Highlights
+- Added support for OS-agnostic baseline files ([#586])
+
+#### :tada: New Features
+- Added a detector for IP addresses ([#692])
+- Added a detector for GitLab tokens ([#782])
+- Added a detector for Telegram tokens ([#808])
+- Added a detector for Pypi and TestPypi tokens ([#819])
+- Added a detector for OpenAI tokens ([#823])
+
+#### :sparkles: Usability
+- Added filenames in errors thrown when a plugin file specified in the `.secrets.baseline` is not found. ([#719])
+- Changed the wording of the audit prompt ([#738])
+
+#### :telescope: Accuracy
+- Improved DiscordBotTokenDetector to reduce false negatives ([#628])
+- Improved KeywordDetector to reduce false positive for Golang ([#675])
+- Improved AWSKeyDetector by adding more access key formats ([#796])
+
+#### :bug: Bugfixes
+- Fixed `NotImplementedError` in StatisticsAggregator ([#678])
+- Fixed bug in YAMLTransformer related to parsing YAML files with achors and tags ([#679])
+- Fixed `IndexError` in `is_prefixed_with_dollar_sign` caused by passing empty strings ([#712])
+
+#### :snake: Miscellaneous
+- Dropped support for Python 3.6 ([#672])
+- Dropped support for Python 3.7 ([#724])
+- Added support for Python 3.10 ([#724])
+- Added support for Python 3.11 ([#730])
+- Added support for Python 3.12 ([#810])
+- Multiple dependency updates
+
+[#586]: https://github.com/Yelp/detect-secrets/pull/586
+[#628]: https://github.com/Yelp/detect-secrets/pull/628
+[#672]: https://github.com/Yelp/detect-secrets/pull/672
+[#675]: https://github.com/Yelp/detect-secrets/pull/675
+[#678]: https://github.com/Yelp/detect-secrets/pull/678
+[#679]: https://github.com/Yelp/detect-secrets/pull/679
+[#692]: https://github.com/Yelp/detect-secrets/pull/692
+[#712]: https://github.com/Yelp/detect-secrets/pull/712
+[#719]: https://github.com/Yelp/detect-secrets/pull/719
+[#724]: https://github.com/Yelp/detect-secrets/pull/724
+[#730]: https://github.com/Yelp/detect-secrets/pull/730
+[#738]: https://github.com/Yelp/detect-secrets/pull/738
+[#782]: https://github.com/Yelp/detect-secrets/pull/782
+[#796]: https://github.com/Yelp/detect-secrets/pull/796
+[#808]: https://github.com/Yelp/detect-secrets/pull/808
+[#810]: https://github.com/Yelp/detect-secrets/pull/810
+[#819]: https://github.com/Yelp/detect-secrets/pull/819
+[#823]: https://github.com/Yelp/detect-secrets/pull/823
 
 ### v1.4.0
 ##### October 4th, 2022

CONTRIBUTING.md

@@ -70,10 +70,10 @@ sys     0m2.486s
 
 ### Running the Entire Test Suite
 
-You can run the test suite in the interpreter of your choice (in this example, `py36`) by doing:
+You can run the test suite in the interpreter of your choice (in this example, `py37`) by doing:
 
 ```bash
-tox -e py36
+tox -e py37
 ```
 
 This will also run the code through our series of coverage tests, `mypy` rules and other linting

README.md

@@ -98,20 +98,25 @@ BasicAuthDetector
 CloudantDetector
 DiscordBotTokenDetector
 GitHubTokenDetector
+GitLabTokenDetector
 Base64HighEntropyString
 HexHighEntropyString
 IbmCloudIamDetector
 IbmCosHmacDetector
+IPPublicDetector
 JwtTokenDetector
 KeywordDetector
 MailchimpDetector
 NpmDetector
+OpenAIDetector
 PrivateKeyDetector
+PypiTokenDetector
 SendGridDetector
 SlackDetector
 SoftlayerDetector
 SquareOAuthDetector
 StripeDetector
+TelegramBotTokenDetector
 TwilioKeyDetector
 ```
 
@@ -392,7 +397,7 @@ We recommend setting this up as a pre-commit hook. One way to do this is by usin
 # .pre-commit-config.yaml
 repos:
 -   repo: https://github.com/Yelp/detect-secrets
-    rev: v1.4.0
+    rev: v1.5.0
     hooks:
     -   id: detect-secrets
         args: ['--baseline', '.secrets.baseline']

detect_secrets/__version__.py

@@ -1 +1 @@
-VERSION = '1.4.0'
+VERSION = '1.5.0'

detect_secrets/audit/analytics.py

@@ -58,7 +58,14 @@ def _get_plugin_counter(self, secret_type: str) -> 'StatisticsCounter':
         return cast(StatisticsCounter, self.data[secret_type]['stats'])
 
     def __str__(self) -> str:
-        raise NotImplementedError
+        output = ''
+
+        for secret_type, framework in self.data.items():
+            output += f'Plugin: {get_mapping_from_secret_type_to_class()[secret_type].__name__}\n'
+            for value in framework.values():
+                output += f'Statistics: {value}\n\n'
+
+        return output
 
     def json(self) -> Dict[str, Any]:
         output = {}
@@ -77,19 +84,36 @@ def __init__(self) -> None:
         self.incorrect: int = 0
         self.unknown: int = 0
 
-    def __repr__(self) -> str:
+    def __str__(self) -> str:
         return (
-            f'{self.__class__.__name__}(correct={self.correct}, '
-            'incorrect={self.incorrect}, unknown={self.unknown},)'
+            f'True Positives: {self.correct}, False Positives: {self.incorrect}, '
+            f'Unknown: {self.unknown}, Precision: {self.calculate_precision()}, '
+            f'Recall: {self.calculate_recall()}'
         )
 
     def json(self) -> Dict[str, Any]:
+        return {
+            'raw': {
+                'true-positives': self.correct,
+                'false-positives': self.incorrect,
+                'unknown': self.unknown,
+            },
+            'score': {
+                'precision': self.calculate_precision(),
+                'recall': self.calculate_recall(),
+            },
+        }
+
+    def calculate_precision(self) -> float:
         precision = (
             round(float(self.correct) / (self.correct + self.incorrect), 4)
             if (self.correct and self.incorrect)
             else 0.0
         )
 
+        return precision
+
+    def calculate_recall(self) -> float:
         # NOTE(2020-11-08|domanchi): This isn't the formal definition of `recall`, however,
         # this is the definition that we're going to attribute to it.
         #
@@ -124,14 +148,4 @@ def json(self) -> Dict[str, Any]:
             else 0.0
         )
 
-        return {
-            'raw': {
-                'true-positives': self.correct,
-                'false-positives': self.incorrect,
-                'unknown': self.unknown,
-            },
-            'score': {
-                'precision': precision,
-                'recall': recall,
-            },
-        }
+        return recall

detect_secrets/audit/audit.py

@@ -78,7 +78,7 @@ def _classify_secrets(iterator: BidirectionalIterator) -> bool:
         if decision == io.InputOptions.BACK:
             iterator.step_back_on_next_iteration()
 
-        # The question asked is: "Should this be committed to the repository?"
+        # The question asked is: "Should this string be committed to the repository?"
         elif decision == io.InputOptions.NO:
             secret.is_secret = True
             has_changes = True

detect_secrets/audit/io.py

@@ -130,7 +130,7 @@ def __init__(self, allow_labelling: bool, allow_backstep: bool) -> None:
 
     def __str__(self) -> str:
         if 'Y' in self.valid_input:
-            output = 'Is this a secret that should be committed to this repository?'
+            output = 'Should this string be committed to the repository?'
         else:
             output = 'What would you like to do?'
 

detect_secrets/core/plugins/initialize.py

@@ -31,7 +31,17 @@ def from_plugin_classname(classname: str) -> Plugin:
     """
     :raises: TypeError
     """
-    for plugin_type in get_mapping_from_secret_type_to_class().values():
+    try:
+        plugin_types = get_mapping_from_secret_type_to_class().values()
+    except FileNotFoundError as e:
+        log.error(f'Error: Failed to load `{classname}` plugin: {e}')
+        log.error(
+            'This error can occur when using a baseline that references a '
+            'custom plugin with a path that does not exist.',
+        )
+        raise
+
+    for plugin_type in plugin_types:
         if plugin_type.__name__ == classname:
             break
     else:

detect_secrets/core/potential_secret.py

@@ -6,6 +6,7 @@
 
 from ..util.color import AnsiColor
 from ..util.color import colorize
+from ..util.path import convert_local_os_path
 
 
 class PotentialSecret:
@@ -75,7 +76,7 @@ def load_secret_from_dict(cls, data: Dict[str, Union[str, int, bool]]) -> 'Poten
         """Custom JSON decoder"""
         kwargs: Dict[str, Any] = {
             'type': str(data['type']),
-            'filename': str(data['filename']),
+            'filename': convert_local_os_path(str(data['filename'])),
             'secret': 'will be replaced',
         }
 

detect_secrets/core/scan.py

@@ -138,8 +138,12 @@ def scan_line(line: str) -> Generator[PotentialSecret, None, None]:
 
 
 def scan_file(filename: str) -> Generator[PotentialSecret, None, None]:
-    if not get_plugins():   # pragma: no cover
-        log.error('No plugins to scan with!')
+    try:
+        if not get_plugins():   # pragma: no cover
+            log.error('No plugins to scan with!')
+            return
+    except FileNotFoundError:
+        log.error('Unable to load plugins!')
         return
 
     if _is_filtered_out(required_filter_parameters=['filename'], filename=filename):
@@ -387,13 +391,19 @@ def _is_filtered_out(required_filter_parameters: Iterable[str], **kwargs: Any) -
         try:
             if call_function_with_arguments(filter_fn, **kwargs):
                 if 'secret' in kwargs:
-                    debug_msg = f'Skipping "{kwargs["secret"]}" due to `{filter_fn.path}`.'
+                    debug_msg = f'Skipping "{0}" due to `{1}`.'.format(
+                        kwargs['secret'],
+                        filter_fn.path,
+                    )
                 elif list(kwargs.keys()) == ['filename']:
                     # We want to make sure this is only run if we're skipping files (as compared
                     # to other filters that may include `filename` as a parameter).
-                    debug_msg = f'Skipping "{kwargs["filename"]}" due to `{filter_fn.path}`'
+                    debug_msg = 'Skipping "{0}" due to `{1}`'.format(
+                        kwargs['filename'],
+                        filter_fn.path,
+                    )
                 else:
-                    debug_msg = f'Skipping secret due to `{filter_fn.path}`.'
+                    debug_msg = 'Skipping secret due to `{0}`.'.format(filter_fn.path)
 
                 log.info(debug_msg)
                 return True

detect_secrets/core/secrets_collection.py

@@ -10,6 +10,7 @@
 from typing import Tuple
 
 from . import scan
+from ..util.path import convert_local_os_path
 from .potential_secret import PotentialSecret
 from detect_secrets.settings import configure_settings_from_baseline
 from detect_secrets.settings import get_settings
@@ -40,7 +41,7 @@ def load_from_baseline(cls, baseline: Dict[str, Any]) -> 'SecretsCollection':
         for filename in baseline['results']:
             for item in baseline['results'][filename]:
                 secret = PotentialSecret.load_secret_from_dict({'filename': filename, **item})
-                output[filename].add(secret)
+                output[convert_local_os_path(filename)].add(secret)
 
         return output
 
@@ -72,8 +73,8 @@ def scan_files(self, *filenames: str, num_processors: Optional[int] = None) -> N
                     self[os.path.relpath(secret.filename, self.root)].add(secret)
 
     def scan_file(self, filename: str) -> None:
-        for secret in scan.scan_file(os.path.join(self.root, filename)):
-            self[filename].add(secret)
+        for secret in scan.scan_file(os.path.join(self.root, convert_local_os_path(filename))):
+            self[convert_local_os_path(filename)].add(secret)
 
     def scan_diff(self, diff: str) -> None:
         """

detect_secrets/filters/heuristic.py

@@ -164,7 +164,7 @@ def is_prefixed_with_dollar_sign(secret: str) -> bool:
     # false negatives than `is_templated_secret` (e.g. secrets that actually start with a $).
     # This is best used with files that actually use this as a means of referencing variables.
     # TODO: More intelligent filetype handling?
-    return secret[0] == '$'
+    return bool(secret) and secret[0] == '$'
 
 
 def is_indirect_reference(line: str) -> bool:

detect_secrets/plugins/aws.py

@@ -25,14 +25,14 @@ class AWSKeyDetector(RegexBasedDetector):
     secret_keyword = r'(?:key|pwd|pw|password|pass|token)'
 
     denylist = (
-        re.compile(r'AKIA[0-9A-Z]{16}'),
+        re.compile(r'(?:A3T[A-Z0-9]|ABIA|ACCA|AKIA|ASIA)[0-9A-Z]{16}'),
 
         # This examines the variable name to identify AWS secret tokens.
-        # The order is important since we want to prefer finding `AKIA`-based
+        # The order is important since we want to prefer finding access
         # keys (since they can be verified), rather than the secret tokens.
 
         re.compile(
-            r'aws.{{0,20}}?{secret_keyword}.{{0,20}}?[\'\"]([0-9a-zA-Z/+]{{40}})[\'\"]'.format(
+            r'aws.{{0,20}}?{secret_keyword}.{{0,20}}?[\'\"]?([0-9a-zA-Z/+]{{40}})[\'\"]?'.format(
                 secret_keyword=secret_keyword,
             ),
             flags=re.IGNORECASE,