fix \r in xlsx

credsweeper/deep_scanner/xlsx_scanner.py

@@ -25,11 +25,10 @@ def data_scan(
         candidates = []
         try:
             book = pd.read_excel(io.BytesIO(data_provider.data), sheet_name=None, header=None)
-            sheet_lines = []
             for sheet_name, sheet_data in book.items():
-                text = sheet_data.fillna('').astype(str)
-                for i in text.values:
-                    sheet_lines.append('\t'.join(i))
+                # replace open xml carriage returns _x000D_ before line feed only
+                df = sheet_data.replace(to_replace="_x000D_\n", value='\n', regex=True).fillna('').astype(str)
+                sheet_lines = ['\t'.join(x) for x in df.values]
                 string_data_provider = StringContentProvider(lines=sheet_lines,
                                                              file_path=data_provider.file_path,
                                                              file_type=data_provider.file_type,

credsweeper/rules/config.yaml

@@ -54,7 +54,7 @@
   confidence: moderate
   type: pattern
   values:
-    - (^|\s|(?P<variable>(?i:\bip[\s/]{1,80}id[\s/]{1,80}pw[\s/:]{0,80}))|(?P<url>://))(?P<ip>[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2})((\s*\()?|(?(variable)[\s,/]{1,80}|(?(url)[,]|[,/])))\s*\w[\w.-]{3,80}[\s,/]{1,80}(?P<value>(?(url)(?-i:(?P<a>[A-Z])|(?P<b>[a-z])|(?P<c>[0-9_+=~!@#$%^&*;?-])){7,31}(?(a)(?(b)(?(c)(\S|$)|(?!x)x)|(?!x)x)|(?!x)x)|(?-i:(?P<e>[A-Z])|(?P<f>[a-z])|(?P<g>[0-9/_+=~!@#$%^&*;?-])){7,31}(?(e)(?(f)(?(g)(\S|$)|(?!x)x)|(?!x)x)|(?!x)x)))(?:\s|[^/]|$)
+    - (^|\s|(?P<variable>(?i:\bip[\s/]{1,80}id[\s/]{1,80}pw[\s/:]{0,80}))|(?P<url>://))(?P<ip>(?<![0-9.])[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}(?![0-9.]))((\s*[(])?|(?(variable)[\s,/]{1,80}|(?(url)[,]|[,/])))\s*\w[\w.-]{3,80}[\s,/]{1,80}(?P<value>(?(url)(?-i:(?P<a>[A-Z])|(?P<b>[a-z])|(?P<c>[0-9_+=~!@#$%^&*;?-])){7,31}(?(a)(?(b)(?(c)(\S|$)|(?!x)x)|(?!x)x)|(?!x)x)|(?-i:(?P<e>[A-Z])|(?P<f>[a-z])|(?P<g>[0-9/_+=~!@#$%^&*;?-])){7,31}(?(e)(?(f)(?(g)(\S|$)|(?!x)x)|(?!x)x)|(?!x)x)))(?:\s|[^/]|$)
   filter_type:
     - ValueAllowlistCheck
     - ValuePatternCheck
@@ -1001,7 +1001,7 @@
   confidence: strong
   type: pattern
   values:
-    - (?<![0-9A-Za-z_-])(?P<value>BBDC-[NMO][ADgjQTwz][0-9A-Za-z_-]{42})(?![0-9A-Za-z_-])
+    - (?<![0-9A-Za-z_-])(?P<value>BBDC-[MNO][ADQTgjwz][AEIMQUYcgk][012345wxyz][0-9A-Za-z_-]{40})(?![0-9A-Za-z_-])
   filter_type:
     - ValueAtlassianTokenCheck
   min_line_len: 49
@@ -1042,7 +1042,7 @@
   confidence: strong
   type: pattern
   values:
-    - (?<![0-9A-Za-z_-])(?P<value>[NMO][ADgjQTwz][0-9A-Za-z_-]{42})(?![0-9A-Za-z_-])
+    - (?<![0-9A-Za-z_-])(?P<value>[MNO][ADQTgjwz][AEIMQUYcgk][012345wxyz][0-9A-Za-z_-]{40})(?![0-9A-Za-z_-])
   filter_type:
     - ValueAtlassianTokenCheck
     - ValueBase64PartCheck
@@ -1374,7 +1374,7 @@
   confidence: strong
   type: pattern
   values:
-    - (?<![0-9A-Za-z_-])(?P<value>[NMO][ADgjQTwz][0-9A-Za-z_-]{22,26}\.[0-9A-Za-z_-]{6}\.[0-9A-Za-z_-]{30,40})(?![0-9A-Za-z_-])
+    - (?<![0-9A-Za-z_-])(?P<value>[MNO][ADQTgjwz][AEIMQUYcgk][012345wxyz][0-9A-Za-z_-]{20,24}\.[0-9A-Za-z_-]{6}\.[0-9A-Za-z_-]{30,40})(?![0-9A-Za-z_-])
   min_line_len: 62
   filter_type:
     - ValueDiscordBotCheck

tests/__init__.py

@@ -17,10 +17,10 @@
 SAMPLES_POST_CRED_COUNT = SAMPLES_CRED_COUNT - ML_FILTERED
 
 # with option --doc
-SAMPLES_IN_DOC = 451
+SAMPLES_IN_DOC = 461
 
 # archived credentials that are not found without --depth
-SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 29
+SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 33
 SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 54
 SAMPLES_IN_DEEP_3 = SAMPLES_IN_DEEP_2 + 1