Sentry Auth Token patterns (#693)

* Sentry Organization Auth Token * Duplicate rules name check * Sentry User Auth Token * assertRaisesRegex * ssws && ntlm authentication schemes in keyword * sample added * test added
Samsung · Mar 7, 2025 · 57363b3 · 57363b3
1 parent 0361bde
commit 57363b3
Show file tree

Hide file tree

Showing 11 changed files with 376 additions and 7 deletions.
diff --git a/credsweeper/common/keyword_pattern.py b/credsweeper/common/keyword_pattern.py
@@ -26,7 +26,7 @@ class KeywordPattern:
     string_prefix = r"(((b|r|br|rb|u|f|rf|fr|l|@)(?=(\\*[`'\"])))?"
     left_quote = r"(?P<value_leftquote>((?P<esq>\\{1,8})?([`'\"]|&(quot|apos);)){1,4}))?"
     # Authentication scheme ( oauth | basic | bearer | apikey ) precedes to credential
-    auth_keywords = r"(\s?(oauth|bot|basic|bearer|apikey|accesskey)\s)?"
+    auth_keywords = r"(\s?(oauth|bot|basic|bearer|apikey|accesskey|ssws|ntlm)\s)?"
     value = r"(?P<value>" \
             r"(?(value_leftquote)" \
             r"(" \

diff --git a/credsweeper/rules/config.yaml b/credsweeper/rules/config.yaml
@@ -1407,6 +1407,37 @@
     - code
     - doc
 
+- name: Sentry Organization Auth Token
+  severity: high
+  confidence: strong
+  type: pattern
+  values:
+    - (?:(?<![0-9A-Za-z_-])|\\[0abfnrtv]|(%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu]([0-9A-Fa-f]{4}){1,2}|\x1B\[[0-9;]{0,80}m)(?P<value>sntrys_eyJ[0-9A-Za-z_-]{80,8000}=*([0-9A-Za-z_-]{32,256})?)(?![0-9A-Za-z_-])
+  min_line_len: 37
+  filter_type:
+    - ValuePatternCheck(5)
+    - ValueEntropyBase64Check
+  required_substrings:
+    - sntrys_eyJ
+  target:
+    - code
+    - doc
+
+- name: Sentry User Auth Token
+  severity: high
+  confidence: strong
+  type: pattern
+  values:
+    - (?:(?<![0-9A-Za-z_-])|\\[0abfnrtv]|(%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu]([0-9A-Fa-f]{4}){1,2}|\x1B\[[0-9;]{0,80}m)(?P<value>sntryu_[0-9a-f]{64})(?![0-9A-Za-z_-])
+  min_line_len: 37
+  filter_type:
+    - ValuePatternCheck(5)
+  required_substrings:
+    - sntryu_
+  target:
+    - code
+    - doc
+
 - name: Discord Bot Token
   severity: high
   confidence: strong

diff --git a/credsweeper/scanner/scanner.py b/credsweeper/scanner/scanner.py
@@ -69,6 +69,7 @@ def _set_rules_scanners(self, rule_path: Union[None, str, Path]) -> None:
             rule_path = APP_PATH / "rules" / "config.yaml"
         rule_templates = Util.yaml_load(rule_path)
         if rule_templates and isinstance(rule_templates, list):
+            rule_names = set()
             for rule_template in rule_templates:
                 try:
                     rule = Rule(self.config, rule_template)
@@ -77,6 +78,10 @@ def _set_rules_scanners(self, rule_path: Union[None, str, Path]) -> None:
                     raise exc
                 if not self._is_available(rule):
                     continue
+                if rule.rule_name in rule_names:
+                    raise RuntimeError(f"Duplicated rule name {rule.rule_name}")
+                else:
+                    rule_names.add(rule.rule_name)
                 if 0 < rule.min_line_len:
                     if rule.rule_type == RuleType.KEYWORD:
                         self.min_keyword_len = min(self.min_keyword_len, rule.min_line_len)
@@ -141,7 +146,7 @@ def scan(self, provider: ContentProvider) -> List[Candidate]:
             # "cache" - YAPF and pycharm formatters ...
             matched_keyword = \
                 target_line_stripped_len >= self.min_keyword_len and (  #
-                    '=' in target_line_stripped or ':' in target_line_stripped)  #
+                        '=' in target_line_stripped or ':' in target_line_stripped)  #
             matched_pem_key = \
                 target_line_stripped_len >= self.min_pem_key_len \
                 and PEM_BEGIN_PATTERN in target_line_stripped and "PRIVATE" in target_line_stripped

diff --git a/tests/__init__.py b/tests/__init__.py
@@ -1,13 +1,13 @@
 from pathlib import Path
 
 # total number of files in test samples
-SAMPLES_FILES_COUNT = 148
+SAMPLES_FILES_COUNT = 149
 
 # the lowest value of ML threshold is used to display possible lowest values
 NEGLIGIBLE_ML_THRESHOLD = 0.0001
 
 # credentials count after scan with negligible ML threshold
-SAMPLES_CRED_COUNT = 472
+SAMPLES_CRED_COUNT = 475
 SAMPLES_CRED_LINE_COUNT = SAMPLES_CRED_COUNT + 19
 
 # Number of filtered credentials with ML
@@ -17,7 +17,7 @@
 SAMPLES_POST_CRED_COUNT = SAMPLES_CRED_COUNT - ML_FILTERED
 
 # with option --doc
-SAMPLES_IN_DOC = 654
+SAMPLES_IN_DOC = 656
 
 # archived credentials that are not found without --depth
 SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 89

diff --git a/tests/data/depth_3.json b/tests/data/depth_3.json
@@ -342,6 +342,31 @@
             }
         ]
     },
+    {
+        "rule": "Auth",
+        "severity": "medium",
+        "confidence": "moderate",
+        "ml_probability": 1.0,
+        "line_data_list": [
+            {
+                "line": "Authorization: NTLM TlRMTUAAABABoITVNIAAZI1AQBuOh4cSAQ8B1A=",
+                "line_num": 4,
+                "path": "./tests/samples/auth.hs",
+                "info": "FILE|RAW",
+                "value": "TlRMTUAAABABoITVNIAAZI1AQBuOh4cSAQ8B1A=",
+                "value_start": 20,
+                "value_end": 59,
+                "variable": "Authorization",
+                "variable_start": 0,
+                "variable_end": 13,
+                "entropy_validation": {
+                    "iterator": "BASE64STDPAD_CHARS",
+                    "entropy": 4.002348372264613,
+                    "valid": false
+                }
+            }
+        ]
+    },
     {
         "rule": "Auth",
         "severity": "medium",
@@ -10056,6 +10081,56 @@
             }
         ]
     },
+    {
+        "rule": "Sentry Organization Auth Token",
+        "severity": "high",
+        "confidence": "strong",
+        "ml_probability": null,
+        "line_data_list": [
+            {
+                "line": "sntrys_eyJpYXQiOjE3NDEyNjQzNTYuMDAwMCwidXJsIjoiaHR0cHM6Ly9zZW50cnkuaW8iLCJyZWdpb25fdXJsIjoiaHR0cHM6Ly91YS5zZW50cnkuaW8iLCJvcmciOiIifQ==v8D-whr2cUQK91Civi4yNoLRjC3MDZH5I2aMcs_j5GDv",
+                "line_num": 1,
+                "path": "./tests/samples/sentry",
+                "info": "FILE|RAW",
+                "value": "sntrys_eyJpYXQiOjE3NDEyNjQzNTYuMDAwMCwidXJsIjoiaHR0cHM6Ly9zZW50cnkuaW8iLCJyZWdpb25fdXJsIjoiaHR0cHM6Ly91YS5zZW50cnkuaW8iLCJvcmciOiIifQ==v8D-whr2cUQK91Civi4yNoLRjC3MDZH5I2aMcs_j5GDv",
+                "value_start": 0,
+                "value_end": 179,
+                "variable": null,
+                "variable_start": -2,
+                "variable_end": -2,
+                "entropy_validation": {
+                    "iterator": "BASE64STDPAD_CHARS",
+                    "entropy": 5.424476232986917,
+                    "valid": true
+                }
+            }
+        ]
+    },
+    {
+        "rule": "Sentry User Auth Token",
+        "severity": "high",
+        "confidence": "strong",
+        "ml_probability": null,
+        "line_data_list": [
+            {
+                "line": "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",
+                "line_num": 2,
+                "path": "./tests/samples/sentry",
+                "info": "FILE|RAW",
+                "value": "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",
+                "value_start": 0,
+                "value_end": 71,
+                "variable": null,
+                "variable_start": -2,
+                "variable_end": -2,
+                "entropy_validation": {
+                    "iterator": "BASE36_CHARS",
+                    "entropy": 4.0572205343720595,
+                    "valid": true
+                }
+            }
+        ]
+    },
     {
         "rule": "Shopify Token",
         "severity": "high",

diff --git a/tests/data/doc.json b/tests/data/doc.json
@@ -16413,6 +16413,56 @@
             }
         ]
     },
+    {
+        "rule": "Sentry Organization Auth Token",
+        "severity": "high",
+        "confidence": "strong",
+        "ml_probability": null,
+        "line_data_list": [
+            {
+                "line": "sntrys_eyJpYXQiOjE3NDEyNjQzNTYuMDAwMCwidXJsIjoiaHR0cHM6Ly9zZW50cnkuaW8iLCJyZWdpb25fdXJsIjoiaHR0cHM6Ly91YS5zZW50cnkuaW8iLCJvcmciOiIifQ==v8D-whr2cUQK91Civi4yNoLRjC3MDZH5I2aMcs_j5GDv",
+                "line_num": 1,
+                "path": "./tests/samples/sentry",
+                "info": "FILE|RAW",
+                "value": "sntrys_eyJpYXQiOjE3NDEyNjQzNTYuMDAwMCwidXJsIjoiaHR0cHM6Ly9zZW50cnkuaW8iLCJyZWdpb25fdXJsIjoiaHR0cHM6Ly91YS5zZW50cnkuaW8iLCJvcmciOiIifQ==v8D-whr2cUQK91Civi4yNoLRjC3MDZH5I2aMcs_j5GDv",
+                "value_start": 0,
+                "value_end": 179,
+                "variable": null,
+                "variable_start": -2,
+                "variable_end": -2,
+                "entropy_validation": {
+                    "iterator": "BASE64STDPAD_CHARS",
+                    "entropy": 5.424476232986917,
+                    "valid": true
+                }
+            }
+        ]
+    },
+    {
+        "rule": "Sentry User Auth Token",
+        "severity": "high",
+        "confidence": "strong",
+        "ml_probability": null,
+        "line_data_list": [
+            {
+                "line": "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",
+                "line_num": 2,
+                "path": "./tests/samples/sentry",
+                "info": "FILE|RAW",
+                "value": "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",
+                "value_start": 0,
+                "value_end": 71,
+                "variable": null,
+                "variable_start": -2,
+                "variable_end": -2,
+                "entropy_validation": {
+                    "iterator": "BASE36_CHARS",
+                    "entropy": 4.0572205343720595,
+                    "valid": true
+                }
+            }
+        ]
+    },
     {
         "rule": "Shopify Token",
         "severity": "high",

diff --git a/tests/data/ml_threshold.json b/tests/data/ml_threshold.json
@@ -249,6 +249,31 @@
             }
         ]
     },
+    {
+        "rule": "Auth",
+        "severity": "medium",
+        "confidence": "moderate",
+        "ml_probability": 1.0,
+        "line_data_list": [
+            {
+                "line": "Authorization: NTLM TlRMTUAAABABoITVNIAAZI1AQBuOh4cSAQ8B1A=",
+                "line_num": 4,
+                "path": "./tests/samples/auth.hs",
+                "info": "",
+                "value": "TlRMTUAAABABoITVNIAAZI1AQBuOh4cSAQ8B1A=",
+                "value_start": 20,
+                "value_end": 59,
+                "variable": "Authorization",
+                "variable_start": 0,
+                "variable_end": 13,
+                "entropy_validation": {
+                    "iterator": "BASE64STDPAD_CHARS",
+                    "entropy": 4.002348372264613,
+                    "valid": false
+                }
+            }
+        ]
+    },
     {
         "rule": "Auth",
         "severity": "medium",
@@ -10197,6 +10222,56 @@
             }
         ]
     },
+    {
+        "rule": "Sentry Organization Auth Token",
+        "severity": "high",
+        "confidence": "strong",
+        "ml_probability": null,
+        "line_data_list": [
+            {
+                "line": "sntrys_eyJpYXQiOjE3NDEyNjQzNTYuMDAwMCwidXJsIjoiaHR0cHM6Ly9zZW50cnkuaW8iLCJyZWdpb25fdXJsIjoiaHR0cHM6Ly91YS5zZW50cnkuaW8iLCJvcmciOiIifQ==v8D-whr2cUQK91Civi4yNoLRjC3MDZH5I2aMcs_j5GDv",
+                "line_num": 1,
+                "path": "./tests/samples/sentry",
+                "info": "",
+                "value": "sntrys_eyJpYXQiOjE3NDEyNjQzNTYuMDAwMCwidXJsIjoiaHR0cHM6Ly9zZW50cnkuaW8iLCJyZWdpb25fdXJsIjoiaHR0cHM6Ly91YS5zZW50cnkuaW8iLCJvcmciOiIifQ==v8D-whr2cUQK91Civi4yNoLRjC3MDZH5I2aMcs_j5GDv",
+                "value_start": 0,
+                "value_end": 179,
+                "variable": null,
+                "variable_start": -2,
+                "variable_end": -2,
+                "entropy_validation": {
+                    "iterator": "BASE64STDPAD_CHARS",
+                    "entropy": 5.424476232986917,
+                    "valid": true
+                }
+            }
+        ]
+    },
+    {
+        "rule": "Sentry User Auth Token",
+        "severity": "high",
+        "confidence": "strong",
+        "ml_probability": null,
+        "line_data_list": [
+            {
+                "line": "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",
+                "line_num": 2,
+                "path": "./tests/samples/sentry",
+                "info": "",
+                "value": "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",
+                "value_start": 0,
+                "value_end": 71,
+                "variable": null,
+                "variable_start": -2,
+                "variable_end": -2,
+                "entropy_validation": {
+                    "iterator": "BASE36_CHARS",
+                    "entropy": 4.0572205343720595,
+                    "valid": true
+                }
+            }
+        ]
+    },
     {
         "rule": "Shopify Token",
         "severity": "high",