From 434a6c1540b0dd6f43600494b3939e9ff7594e04 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Fri, 13 Jan 2023 16:12:40 +0200 Subject: [PATCH] Github Fine-granted Token pattern (#245) * New rule for Github token * Update tests/test_app.py * Update tests/test_app.py --- credsweeper/py.typed | 0 credsweeper/rules/config.yaml | 15 ++++++++++++++- setup.py | 1 + tests/__init__.py | 8 ++++---- tests/samples/aws_s3_bucket | 1 + tests/samples/firebase_domain | 2 ++ tests/samples/github_personal_token | 4 ++++ tests/samples/github_token | 1 + tests/samples/instagram_access_token | 1 + tests/samples/json_web_token | 1 + tests/samples/pypi_token | 1 + tests/samples/telegram_api_key | 1 + tests/samples/url_cred | 1 + tests/test_app.py | 13 +++++++++++++ 14 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 credsweeper/py.typed create mode 100644 tests/samples/aws_s3_bucket create mode 100644 tests/samples/firebase_domain create mode 100644 tests/samples/github_personal_token create mode 100644 tests/samples/github_token create mode 100644 tests/samples/instagram_access_token create mode 100644 tests/samples/json_web_token create mode 100644 tests/samples/pypi_token create mode 100644 tests/samples/telegram_api_key create mode 100644 tests/samples/url_cred diff --git a/credsweeper/py.typed b/credsweeper/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/credsweeper/rules/config.yaml b/credsweeper/rules/config.yaml index 51a5d1e89..13cf7d288 100644 --- a/credsweeper/rules/config.yaml +++ b/credsweeper/rules/config.yaml @@ -424,7 +424,7 @@ severity: high type: pattern values: - - (?P[0-9]{10}:AA[\w\\-_-]{33}) + - (?P[0-9]{8,10}:[0-9A-Za-z_-]{35}) filter_type: GeneralPattern use_ml: false validations: [] @@ -469,6 +469,19 @@ - ghp_ min_line_len: 40 +- name: Github Fine-granted Token + severity: high + type: pattern + values: + - (^|[^0-9A-Za-z])(?Pgithub_pat_[0-9A-Za-z_]{80,255}) + filter_type: GeneralPattern + use_ml: false + validations: + - GithubTokenValidation + required_substrings: + - github_pat_ + min_line_len: 90 + - name: Firebase Domain severity: info type: pattern diff --git a/setup.py b/setup.py index d55aebfeb..a49f76e47 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ packages=setuptools.find_packages(include=("credsweeper*", )), package_data={ "credsweeper": [ + "py.typed", # "common/keyword_checklist.txt", # "ml_model/ml_model.onnx", # "ml_model/model_config.json", # diff --git a/tests/__init__.py b/tests/__init__.py index a36563ba0..3a9fdbe93 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,14 +1,14 @@ from pathlib import Path # total number of files in test samples, included .gitignore -SAMPLES_FILES_COUNT: int = 55 +SAMPLES_FILES_COUNT: int = 64 # credentials count after scan -SAMPLES_CRED_COUNT: int = 51 -SAMPLES_CRED_LINE_COUNT: int = 54 +SAMPLES_CRED_COUNT: int = 62 +SAMPLES_CRED_LINE_COUNT: int = 65 # credentials count after post-processing -SAMPLES_POST_CRED_COUNT: int = 20 +SAMPLES_POST_CRED_COUNT: int = 29 # archived credentials that not found without --depth SAMPLES_IN_DEEP_1 = 6 diff --git a/tests/samples/aws_s3_bucket b/tests/samples/aws_s3_bucket new file mode 100644 index 000000000..a30e779da --- /dev/null +++ b/tests/samples/aws_s3_bucket @@ -0,0 +1 @@ +{"url" : "https://circle-production-action-output.s3.amazonaws.com/283746ds4?X-Amz-Algorithm=AWS4-HMAC-SHA256"} diff --git a/tests/samples/firebase_domain b/tests/samples/firebase_domain new file mode 100644 index 000000000..5f8454d78 --- /dev/null +++ b/tests/samples/firebase_domain @@ -0,0 +1,2 @@ +test-app-domain-42.firebaseapp.com +test2.io.firebaseio.com \ No newline at end of file diff --git a/tests/samples/github_personal_token b/tests/samples/github_personal_token new file mode 100644 index 000000000..5bf9905f4 --- /dev/null +++ b/tests/samples/github_personal_token @@ -0,0 +1,4 @@ +# Fine-granted token +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 +# Classic +ghp_3Wf1ikDwUMP1I9Vz3z2qZ1Ge2uCLN230RKuq diff --git a/tests/samples/github_token b/tests/samples/github_token new file mode 100644 index 000000000..315dac312 --- /dev/null +++ b/tests/samples/github_token @@ -0,0 +1 @@ +ghr_Ku7ikDwqZ1Ge2u3Wf1UM3z2CLN230RP1I9Vz diff --git a/tests/samples/instagram_access_token b/tests/samples/instagram_access_token new file mode 100644 index 000000000..6f64ebab9 --- /dev/null +++ b/tests/samples/instagram_access_token @@ -0,0 +1 @@ +IGQVJLoremIpsumDolorSitAmetConsecteturAdipiscingElitSedDoEiusmodTemporIncididuntUtLaboreEtDoloreMagnaAliqua \ No newline at end of file diff --git a/tests/samples/json_web_token b/tests/samples/json_web_token new file mode 100644 index 000000000..a2e9960b0 --- /dev/null +++ b/tests/samples/json_web_token @@ -0,0 +1 @@ +$payload = 'eyJgsIZgSmlP.eyJcaaF9xCe7shE0ENPiBlEJOpS' \ No newline at end of file diff --git a/tests/samples/pypi_token b/tests/samples/pypi_token new file mode 100644 index 000000000..fe0d459df --- /dev/null +++ b/tests/samples/pypi_token @@ -0,0 +1 @@ +pypi-AgEIcHlvdS5bcmcCJGDmMTllXWJjLTRhODAtNDBhYi01Y2Y4LThjNjcxZTg3MsRjOAACKlszLCJlPDYzYmUxOC0wNjQ3LTQ4AzQtODFhYy1jZGRlNTk2MGYxNDYiXQAABiDDbVAEnaUMn4zZHlFr5NWy7JEYfvoay--cVmKO5lr3Xp diff --git a/tests/samples/telegram_api_key b/tests/samples/telegram_api_key new file mode 100644 index 000000000..5a4a32bc0 --- /dev/null +++ b/tests/samples/telegram_api_key @@ -0,0 +1 @@ +508627689:AAEuLPKs-EhrjrYGnz60bnYNZqakf6HJxc0 diff --git a/tests/samples/url_cred b/tests/samples/url_cred new file mode 100644 index 000000000..ad06346c4 --- /dev/null +++ b/tests/samples/url_cred @@ -0,0 +1 @@ +mysql://root:root@localhost:3306/main \ No newline at end of file diff --git a/tests/test_app.py b/tests/test_app.py index d0d3a2254..950d36731 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -580,3 +580,16 @@ def test_denylist_line_n(self) -> None: with open(json_filename, "r") as json_file: report = json.load(json_file) assert len(report) == 1 + + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + + def test_rules_p(self) -> None: + _stdout, _stderr = self._m_credsweeper(["--log", "silence", "--ml_threshold", "0", "--path", str(SAMPLES_DIR)]) + assert len(_stderr) == 0 + output = _stdout.decode(errors='replace') + rules = Util.yaml_load(PROJECT_DIR / "credsweeper" / "rules" / "config.yaml") + for rule in rules: + rule_name = rule["name"] + if rule_name in ["Nonce", "Salt", "Certificate"]: + continue + self.assertIn(f"rule: {rule_name}", output)