Github Fine-granted Token pattern (#245)

* New rule for Github token

* Update tests/test_app.py

* Update tests/test_app.py

credsweeper/rules/config.yaml

@@ -424,7 +424,7 @@
   severity: high
   type: pattern
   values:
-    - (?P<value>[0-9]{10}:AA[\w\\-_-]{33})
+    - (?P<value>[0-9]{8,10}:[0-9A-Za-z_-]{35})
   filter_type: GeneralPattern
   use_ml: false
   validations: []
@@ -469,6 +469,19 @@
     - ghp_
   min_line_len: 40
 
+- name: Github Fine-granted Token
+  severity: high
+  type: pattern
+  values:
+    - (^|[^0-9A-Za-z])(?P<value>github_pat_[0-9A-Za-z_]{80,255})
+  filter_type: GeneralPattern
+  use_ml: false
+  validations:
+    - GithubTokenValidation
+  required_substrings:
+    - github_pat_
+  min_line_len: 90
+
 - name: Firebase Domain
   severity: info
   type: pattern

setup.py

@@ -28,6 +28,7 @@
     packages=setuptools.find_packages(include=("credsweeper*", )),
     package_data={
         "credsweeper": [
+            "py.typed",  #
             "common/keyword_checklist.txt",  #
             "ml_model/ml_model.onnx",  #
             "ml_model/model_config.json",  #

tests/__init__.py

@@ -1,14 +1,14 @@
 from pathlib import Path
 
 # total number of files in test samples, included .gitignore
-SAMPLES_FILES_COUNT: int = 55
+SAMPLES_FILES_COUNT: int = 64
 
 # credentials count after scan
-SAMPLES_CRED_COUNT: int = 51
-SAMPLES_CRED_LINE_COUNT: int = 54
+SAMPLES_CRED_COUNT: int = 62
+SAMPLES_CRED_LINE_COUNT: int = 65
 
 # credentials count after post-processing
-SAMPLES_POST_CRED_COUNT: int = 20
+SAMPLES_POST_CRED_COUNT: int = 29
 
 # archived credentials that not found without --depth
 SAMPLES_IN_DEEP_1 = 6

tests/samples/aws_s3_bucket

@@ -0,0 +1 @@
+{"url" : "https://circle-production-action-output.s3.amazonaws.com/283746ds4?X-Amz-Algorithm=AWS4-HMAC-SHA256"}

tests/samples/firebase_domain

@@ -0,0 +1,2 @@
+test-app-domain-42.firebaseapp.com
+test2.io.firebaseio.com

tests/samples/github_personal_token

@@ -0,0 +1,4 @@
+# Fine-granted token
+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
+# Classic
+ghp_3Wf1ikDwUMP1I9Vz3z2qZ1Ge2uCLN230RKuq

tests/samples/github_token

@@ -0,0 +1 @@
+ghr_Ku7ikDwqZ1Ge2u3Wf1UM3z2CLN230RP1I9Vz

tests/samples/instagram_access_token

@@ -0,0 +1 @@
+IGQVJLoremIpsumDolorSitAmetConsecteturAdipiscingElitSedDoEiusmodTemporIncididuntUtLaboreEtDoloreMagnaAliqua

tests/samples/json_web_token

@@ -0,0 +1 @@
+$payload = 'eyJgsIZgSmlP.eyJcaaF9xCe7shE0ENPiBlEJOpS'

tests/samples/pypi_token

@@ -0,0 +1 @@
+pypi-AgEIcHlvdS5bcmcCJGDmMTllXWJjLTRhODAtNDBhYi01Y2Y4LThjNjcxZTg3MsRjOAACKlszLCJlPDYzYmUxOC0wNjQ3LTQ4AzQtODFhYy1jZGRlNTk2MGYxNDYiXQAABiDDbVAEnaUMn4zZHlFr5NWy7JEYfvoay--cVmKO5lr3Xp

tests/samples/telegram_api_key

@@ -0,0 +1 @@
+508627689:AAEuLPKs-EhrjrYGnz60bnYNZqakf6HJxc0

tests/samples/url_cred

@@ -0,0 +1 @@
+mysql://root:root@localhost:3306/main

tests/test_app.py

@@ -580,3 +580,16 @@ def test_denylist_line_n(self) -> None:
             with open(json_filename, "r") as json_file:
                 report = json.load(json_file)
                 assert len(report) == 1
+
+    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+    def test_rules_p(self) -> None:
+        _stdout, _stderr = self._m_credsweeper(["--log", "silence", "--ml_threshold", "0", "--path", str(SAMPLES_DIR)])
+        assert len(_stderr) == 0
+        output = _stdout.decode(errors='replace')
+        rules = Util.yaml_load(PROJECT_DIR / "credsweeper" / "rules" / "config.yaml")
+        for rule in rules:
+            rule_name = rule["name"]
+            if rule_name in ["Nonce", "Salt", "Certificate"]:
+                continue
+            self.assertIn(f"rule: {rule_name}", output)