diff --git a/.gitignore b/.gitignore index 1f1012a6..e007d829 100644 --- a/.gitignore +++ b/.gitignore @@ -170,3 +170,6 @@ examples/coze-js-node/src/config/config.ts !examples/coze-js-web/src/lib/wavtools/dist packages/chat-sdk/.env.development + +opensource_git_commit.log +sensitive_info_result.txt diff --git a/common/autoinstallers/sensitive-check/package.json b/common/autoinstallers/sensitive-check/package.json new file mode 100644 index 00000000..7fb0a999 --- /dev/null +++ b/common/autoinstallers/sensitive-check/package.json @@ -0,0 +1,13 @@ +{ + "name": "sensitive-check", + "version": "1.0.0", + "private": true, + "author": "chenjiawei.inizio@bytedance.com", + "resolutions": {}, + "devDependencies": { + "@types/node": "^22.13.13", + "commander": "^11.0.0", + "sucrase": "^3.32.0", + "typescript": "^5.8.2" + } +} diff --git a/common/autoinstallers/sensitive-check/pnpm-lock.yaml b/common/autoinstallers/sensitive-check/pnpm-lock.yaml new file mode 100644 index 00000000..9ef517d3 --- /dev/null +++ b/common/autoinstallers/sensitive-check/pnpm-lock.yaml @@ -0,0 +1,420 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + devDependencies: + '@types/node': + specifier: ^22.13.13 + version: 22.13.13 + commander: + specifier: ^11.0.0 + version: 11.1.0 + sucrase: + specifier: ^3.32.0 + version: 3.35.0 + typescript: + specifier: ^5.8.2 + version: 5.8.2 + +packages: + + '@isaacs/cliui@8.0.2': + resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} + engines: {node: '>=12'} + + '@jridgewell/gen-mapping@0.3.8': + resolution: {integrity: sha512-imAbBGkb+ebQyxKgzv5Hu2nmROxoDOXHh80evxdoXNOrvAnVx7zimzc1Oo5h9RlfV4vPXaE2iM5pOFbvOCClWA==} + engines: {node: '>=6.0.0'} + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + + '@jridgewell/set-array@1.2.1': + resolution: {integrity: sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==} + engines: {node: '>=6.0.0'} + + '@jridgewell/sourcemap-codec@1.5.0': + resolution: {integrity: sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==} + + '@jridgewell/trace-mapping@0.3.25': + resolution: {integrity: sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==} + + '@pkgjs/parseargs@0.11.0': + resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} + engines: {node: '>=14'} + + '@types/node@22.13.13': + resolution: {integrity: sha512-ClsL5nMwKaBRwPcCvH8E7+nU4GxHVx1axNvMZTFHMEfNI7oahimt26P5zjVCRrjiIWj6YFXfE1v3dEp94wLcGQ==} + + ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + + ansi-regex@6.1.0: + resolution: {integrity: sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==} + engines: {node: '>=12'} + + ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + + ansi-styles@6.2.1: + resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==} + engines: {node: '>=12'} + + any-promise@1.3.0: + resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==} + + balanced-match@1.0.2: + resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + + brace-expansion@2.0.1: + resolution: {integrity: sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==} + + color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + + color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + + commander@11.1.0: + resolution: {integrity: sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==} + engines: {node: '>=16'} + + commander@4.1.1: + resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} + engines: {node: '>= 6'} + + cross-spawn@7.0.6: + resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} + engines: {node: '>= 8'} + + eastasianwidth@0.2.0: + resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + + emoji-regex@9.2.2: + resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + + foreground-child@3.3.1: + resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} + engines: {node: '>=14'} + + glob@10.4.5: + resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} + hasBin: true + + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + + isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + + jackspeak@3.4.3: + resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + + lines-and-columns@1.2.4: + resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} + + lru-cache@10.4.3: + resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + + minimatch@9.0.5: + resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==} + engines: {node: '>=16 || 14 >=14.17'} + + minipass@7.1.2: + resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} + engines: {node: '>=16 || 14 >=14.17'} + + mz@2.7.0: + resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} + + object-assign@4.1.1: + resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} + engines: {node: '>=0.10.0'} + + package-json-from-dist@1.0.1: + resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} + + path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + + path-scurry@1.11.1: + resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} + engines: {node: '>=16 || 14 >=14.18'} + + pirates@4.0.6: + resolution: {integrity: sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==} + engines: {node: '>= 6'} + + shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + + shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + + string-width@5.1.2: + resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} + engines: {node: '>=12'} + + strip-ansi@6.0.1: + resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} + engines: {node: '>=8'} + + strip-ansi@7.1.0: + resolution: {integrity: sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==} + engines: {node: '>=12'} + + sucrase@3.35.0: + resolution: {integrity: sha512-8EbVDiu9iN/nESwxeSxDKe0dunta1GOlHufmSSXxMD2z2/tMZpDMpvXQGsc+ajGo8y2uYUmixaSRUc/QPoQ0GA==} + engines: {node: '>=16 || 14 >=14.17'} + hasBin: true + + thenify-all@1.6.0: + resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==} + engines: {node: '>=0.8'} + + thenify@3.3.1: + resolution: {integrity: sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==} + + ts-interface-checker@0.1.13: + resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} + + typescript@5.8.2: + resolution: {integrity: sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@6.20.0: + resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==} + + which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + + wrap-ansi@7.0.0: + resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} + engines: {node: '>=10'} + + wrap-ansi@8.1.0: + resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} + engines: {node: '>=12'} + +snapshots: + + '@isaacs/cliui@8.0.2': + dependencies: + string-width: 5.1.2 + string-width-cjs: string-width@4.2.3 + strip-ansi: 7.1.0 + strip-ansi-cjs: strip-ansi@6.0.1 + wrap-ansi: 8.1.0 + wrap-ansi-cjs: wrap-ansi@7.0.0 + + '@jridgewell/gen-mapping@0.3.8': + dependencies: + '@jridgewell/set-array': 1.2.1 + '@jridgewell/sourcemap-codec': 1.5.0 + '@jridgewell/trace-mapping': 0.3.25 + + '@jridgewell/resolve-uri@3.1.2': {} + + '@jridgewell/set-array@1.2.1': {} + + '@jridgewell/sourcemap-codec@1.5.0': {} + + '@jridgewell/trace-mapping@0.3.25': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.0 + + '@pkgjs/parseargs@0.11.0': + optional: true + + '@types/node@22.13.13': + dependencies: + undici-types: 6.20.0 + + ansi-regex@5.0.1: {} + + ansi-regex@6.1.0: {} + + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + ansi-styles@6.2.1: {} + + any-promise@1.3.0: {} + + balanced-match@1.0.2: {} + + brace-expansion@2.0.1: + dependencies: + balanced-match: 1.0.2 + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + commander@11.1.0: {} + + commander@4.1.1: {} + + cross-spawn@7.0.6: + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + + eastasianwidth@0.2.0: {} + + emoji-regex@8.0.0: {} + + emoji-regex@9.2.2: {} + + foreground-child@3.3.1: + dependencies: + cross-spawn: 7.0.6 + signal-exit: 4.1.0 + + glob@10.4.5: + dependencies: + foreground-child: 3.3.1 + jackspeak: 3.4.3 + minimatch: 9.0.5 + minipass: 7.1.2 + package-json-from-dist: 1.0.1 + path-scurry: 1.11.1 + + is-fullwidth-code-point@3.0.0: {} + + isexe@2.0.0: {} + + jackspeak@3.4.3: + dependencies: + '@isaacs/cliui': 8.0.2 + optionalDependencies: + '@pkgjs/parseargs': 0.11.0 + + lines-and-columns@1.2.4: {} + + lru-cache@10.4.3: {} + + minimatch@9.0.5: + dependencies: + brace-expansion: 2.0.1 + + minipass@7.1.2: {} + + mz@2.7.0: + dependencies: + any-promise: 1.3.0 + object-assign: 4.1.1 + thenify-all: 1.6.0 + + object-assign@4.1.1: {} + + package-json-from-dist@1.0.1: {} + + path-key@3.1.1: {} + + path-scurry@1.11.1: + dependencies: + lru-cache: 10.4.3 + minipass: 7.1.2 + + pirates@4.0.6: {} + + shebang-command@2.0.0: + dependencies: + shebang-regex: 3.0.0 + + shebang-regex@3.0.0: {} + + signal-exit@4.1.0: {} + + string-width@4.2.3: + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + + string-width@5.1.2: + dependencies: + eastasianwidth: 0.2.0 + emoji-regex: 9.2.2 + strip-ansi: 7.1.0 + + strip-ansi@6.0.1: + dependencies: + ansi-regex: 5.0.1 + + strip-ansi@7.1.0: + dependencies: + ansi-regex: 6.1.0 + + sucrase@3.35.0: + dependencies: + '@jridgewell/gen-mapping': 0.3.8 + commander: 4.1.1 + glob: 10.4.5 + lines-and-columns: 1.2.4 + mz: 2.7.0 + pirates: 4.0.6 + ts-interface-checker: 0.1.13 + + thenify-all@1.6.0: + dependencies: + thenify: 3.3.1 + + thenify@3.3.1: + dependencies: + any-promise: 1.3.0 + + ts-interface-checker@0.1.13: {} + + typescript@5.8.2: {} + + undici-types@6.20.0: {} + + which@2.0.2: + dependencies: + isexe: 2.0.0 + + wrap-ansi@7.0.0: + dependencies: + ansi-styles: 4.3.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + + wrap-ansi@8.1.0: + dependencies: + ansi-styles: 6.2.1 + string-width: 5.1.2 + strip-ansi: 7.1.0 diff --git a/common/autoinstallers/sensitive-check/tool.ts b/common/autoinstallers/sensitive-check/tool.ts new file mode 100644 index 00000000..3d22f165 --- /dev/null +++ b/common/autoinstallers/sensitive-check/tool.ts @@ -0,0 +1,180 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { exec } from 'child_process'; + +const checkPath = path.resolve('./'); + +// 获取 commit log +function opensourceGetCommitLog(repoFolderPath: string): Promise { + return new Promise((resolve, reject) => { + process.chdir(repoFolderPath); + exec('git log -p > ./opensource_git_commit.log', (error, stdout) => { + if (error) { + reject(error); + return; + } + resolve(); + }); + }); +} + +// 删除 commit log +function opensourceRmCommitLog(repoFolderPath: string): void { + const file = 'opensource_git_commit.log'; + const filePath = path.join(repoFolderPath, file); + fs.unlinkSync(filePath); +} + +function shouldIgnoreKeywords(ignoreListKeywords: string[], line: string): boolean { + return ignoreListKeywords.some(ignoreKey => { + const pattern = new RegExp(ignoreKey, 'i'); + return pattern.test(line); + }); +} + +interface SensitiveCheckOptions { + keywordsList: string[]; + ignoreListKeywords: string[]; + repoFolderPath: string; + aigcKeywordsGroup1: string[]; + aigcKeywordsGroup2: string[]; +} + +async function checkSensitiveInformation({ + keywordsList, + ignoreListKeywords, + repoFolderPath, + aigcKeywordsGroup1, + aigcKeywordsGroup2 +}: SensitiveCheckOptions): Promise { + const ignoredExtensions = ['.tgz', '.zip', '.tar', '.rar', '.gif', '.jpg', '.png', + '.jpeg', '.svg', '.tiff', '.raw', '.ico', '.webp', '.tga']; + const ignoredPaths = ['/.git/', '/node_modules/', '/temp/', '/sensitive-check/']; + + function walkSync(dir: string) { + // 检查是否应该忽略该文件 + if (ignoredPaths.some(p => dir.includes(p))) { + return; + } + const files = fs.readdirSync(dir); + files.forEach(file => { + const filePath = path.join(dir, file); + const stat = fs.statSync(filePath); + + if (stat.isDirectory()) { + walkSync(filePath); + } else { + // 检查是否应该忽略该文件 + if (ignoredExtensions.some(ext => file.endsWith(ext))) { + return; + } + try { + let cnt = 0; + const content = fs.readFileSync(filePath, 'latin1'); + const lines = content.split('\n'); + + lines.forEach((line, index) => { + cnt = index + 1; + if (!shouldIgnoreKeywords(ignoreListKeywords, line.trim())) { + keywordsList.forEach(pattern => { + const regex = new RegExp(pattern, 'i'); + const result = line.trim().match(regex); + + if (pattern === '(tokenizer|transformer|token_id|tokenid|attention_head).{0,20}') { + if (file.endsWith('.json') && result) { + logSensitiveInfo(filePath, cnt, result[0]); + } + } else if (result) { + logSensitiveInfo(filePath, cnt, result[0]); + } + }); + } + }); + + // 检查 AIGC 关键词 + if (file.endsWith('.json')) { + if (aigcKeywordsGroup1.every(keyword => content.includes(keyword))) { + logAigcSensitiveInfo(filePath, aigcKeywordsGroup1); + } else if (aigcKeywordsGroup2.every(keyword => content.includes(keyword))) { + logAigcSensitiveInfo(filePath, aigcKeywordsGroup2); + } + } + } catch (error) { + console.error(`Error processing file ${filePath}:`, error); + } + } + }); + } + + walkSync(repoFolderPath); +} + +function logSensitiveInfo(filePath: string, line: number, sensitiveContent: string): void { + const message = `File "${filePath}, line, ${line}," have some sensitive information: ${sensitiveContent}`; + console.log('\x1b[36m%s\x1b[0m', message); + fs.appendFileSync('./sensitive_info_result.txt', message + '\n', 'utf8'); +} + +function logAigcSensitiveInfo(filePath: string, keywords: string[]): void { + const message = `File "${filePath} have some aigc sensitive information: ${JSON.stringify(keywords)}`; + console.log('\x1b[36m%s\x1b[0m', message); + fs.appendFileSync('./sensitive_info_result.txt', message + '\n', 'utf8'); +} + +function detectionResult(): void { + if (!fs.existsSync('./sensitive_info_result.txt')) { + console.log('=======Detection passed, no sensitive information found======='); + } +} + +async function main(path: string): Promise { + const repoFolderPath = path; + + await opensourceGetCommitLog(repoFolderPath); + + const keywordsList = [ + String.raw`npm\s{1,20}install.{1,30}`, + String.raw`AKLT\w{43,44}`, + String.raw`AKAP\w{43,44}`, + String.raw`(tokenizer|transformer|token_id|tokenid|attention_head).{0,20}`, + String.raw`(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}`, + String.raw`(LTAI)[a-z0-9]{20}`, + String.raw`AKTP\w{43,44}`, + String.raw`([^*<\s|:>]{0,7})(app_id|appid)([^]()!<>;/@&,]{0,10}[(=:]\s{0,6}[\"']{0,1}[0-9]{6,32}[\"']{0,1})`, + String.raw`.{0,15}\.?byted.org.{0,20}`, + String.raw`.{0,15}\.?bytedance.net.{0,20}`, + String.raw`.{0,20}.bytedance\.feishu\.cn.{0,50}`, + String.raw`.{0,20}.bytedance\.larkoffice\.com.{0,50}`, + String.raw`(10\.\d{1,3}\.\d{1,3}\.\d{1,3})`, + String.raw`([^*<\s|:>]{0,4})(testak|testsk|ak|sk|key|token|auth|pass|cookie|session|password|app_id|appid|secret_key|access_key|secretkey|accesskey|credential|secret|access)(\s{0,10}[(=:]\s{0,6}[\"']{0,1}(?=[a-zA-Z]*[0-9])(?=[0-9]*[a-zA-Z])[a-zA-Z0-9]{16,32}[\"']{0,1})` + ]; + + const aigcKeywordsGroup1 = ["token", "temp", "role"]; + const aigcKeywordsGroup2 = ["layer", "token", "head"]; + + const ignoreListKeywords = [ + String.raw`[^*<>]{0,6}token[^]()!<>;/@&,]{0,10}[=:].{0,1}null,`, + String.raw`.{0,5}user.{0,10}[=:].{0,1}null`, + String.raw`.{0,5}pass.{0,10}[=:].{0,1}null`, + String.raw`passport[=:]`, + String.raw`[^*<>]{0,6}key[^]()!<>;/]{0,10}[=:].{0,1}string.{0,10}`, + String.raw`.{0,5}user.{0,10}[=:].{0,1}string`, + String.raw`.{0,5}pass.{0,10}[=:].{0,1}string`, + String.raw`.{0,5}app_id[^]()!<>;/@&,]{0,10}[=:].{0,10}\+`, + String.raw`.{0,5}appid[^]()!<>;/@&,]{0,10}[=:].{0,10}\+` + ]; + + await checkSensitiveInformation({ + keywordsList, + ignoreListKeywords, + repoFolderPath, + aigcKeywordsGroup1, + aigcKeywordsGroup2 + }); + + detectionResult(); + opensourceRmCommitLog(repoFolderPath); +} + +// 执行主函数 +main(checkPath).catch(console.error); diff --git a/common/config/rush/command-line.json b/common/config/rush/command-line.json index 04ccfa9f..91ef0f2d 100644 --- a/common/config/rush/command-line.json +++ b/common/config/rush/command-line.json @@ -231,6 +231,14 @@ "safeForSimultaneousRushProcesses": true, "autoinstallerName": "rush-lint-staged", "shellCommand": "lint-staged --config common/autoinstallers/rush-lint-staged/.lintstagedrc.js --shell '/bin/bash'" + }, + { + "name": "sensitive-check", + "commandKind": "global", + "summary": "⭐️️ Use to run sensitive check", + "safeForSimultaneousRushProcesses": true, + "autoinstallerName": "sensitive-check", + "shellCommand": "sucrase-node common/autoinstallers/sensitive-check/tool.ts" } ],