diff --git a/lib/logger/core.js b/lib/logger/core.js index b3c1a79c5..ba3fc50fc 100644 --- a/lib/logger/core.js +++ b/lib/logger/core.js @@ -5,6 +5,7 @@ var moment = require('moment'); var Util = require('../util'); var Errors = require('../errors'); +var SecretDetector = new (require('../secret_detector.js'))(); var LOG_LEVEL_ERROR = { @@ -308,6 +309,9 @@ exports.createLogger = function (options, logMessage) '[%s]: %s', moment().format('h:mm:ss.ms A'), message); } + // mask secrets + message = SecretDetector.maskSecrets(message).maskedtxt; + // log the message logMessage(targetLevelObject.tag, message, bufferMaxLength); } diff --git a/lib/secret_detector.js b/lib/secret_detector.js new file mode 100644 index 000000000..8759cd851 --- /dev/null +++ b/lib/secret_detector.js @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2021 Snowflake Computing Inc. All rights reserved. + */ + +/** + * The secret detector detects sensitive information. + * It masks secrets that might be leaked from two potential avenues + * 1. Out of Band Telemetry + * 2. Logging + * +* @param {Object} customPatterns contains 'regex' and 'mask' for customized masking + * @param {Object} mock + * + * @returns {Object} + * @constructor + */ +function secret_detector(customPatterns, mock) +{ + var CUSTOM_PATTERNS_REGEX = []; + var CUSTOM_PATTERNS_MASK = []; + var CUSTOM_PATTERNS_LENGTH; + + if (customPatterns) + { + // Check that the customPatterns object contains the keys 'regex' and 'mask + if (!customPatterns.regex) + { + throw new Error("The customPatterns object must contain the 'regex' key"); + } + if (!customPatterns.mask) + { + throw new Error("The customPatterns object must contain the 'mask' key"); + } + // Also check that their lengths are equal + if (customPatterns.regex.length !== customPatterns.mask.length) + { + throw new Error("The customPatterns object must have equal length for both 'regex' and 'mask'"); + } + + CUSTOM_PATTERNS_LENGTH = customPatterns.regex.length; + + // Push the regex and mask elements onto their respective arrays + for (var index = 0; index < CUSTOM_PATTERNS_LENGTH; index++) + { + CUSTOM_PATTERNS_REGEX.push(new RegExp(`${customPatterns.regex[index]}`, 'gi')); + CUSTOM_PATTERNS_MASK.push(String.raw`${customPatterns.mask[index]}`); + } + } + + function maskCustomPattern(text) + { + var result; + for (var index = 0; index < CUSTOM_PATTERNS_LENGTH; index++) + { + result = text.replace(CUSTOM_PATTERNS_REGEX[index], CUSTOM_PATTERNS_MASK[index]); + // If the text is replaced, return the result + if (text !== result) + { + return result; + } + } + // If text is unchanged, return the original + return text; + } + + const AWS_KEY_PATTERN = new RegExp(String.raw`(aws_key_id|aws_secret_key|access_key_id|secret_access_key)\s*=\s*'([^']+)'`, + 'gi'); + const AWS_TOKEN_PATTERN = new RegExp(String.raw`(accessToken|tempToken|keySecret)\s*:\s*"([a-z0-9/+]{32,}={0,2})"`, + 'gi'); + const SAS_TOKEN_PATTERN = new RegExp(String.raw`(sig|signature|AWSAccessKeyId|password|passcode)=(\?P[a-z0-9%/+]{16,})`, + 'gi'); + const PRIVATE_KEY_PATTERN = new RegExp(String.raw`-----BEGIN PRIVATE KEY-----\\n([a-z0-9/+=\\n]{32,})\\n-----END PRIVATE KEY-----`, + 'gim'); + const PRIVATE_KEY_DATA_PATTERN = new RegExp(String.raw`"privateKeyData": "([a-z0-9/+=\\n]{10,})"`, + 'gim'); + const CONNECTION_TOKEN_PATTERN = new RegExp(String.raw`(token|assertion content)([\'\"\s:=]+)([a-z0-9=/_\-\+]{8,})`, + 'gi'); + const PASSWORD_PATTERN = new RegExp( + String.raw`(password|pwd)([\'\"\s:=]+)([a-z0-9!\"#\$%&\\\'\(\)\*\+\,-\./:;<=>\?\@\[\]\^_` + + '`' + + String.raw`\{\|\}~]{8,})`, + 'gi'); + + function maskAwsKeys(text) + { + return text.replace(AWS_KEY_PATTERN, String.raw`$1$2****`); + } + + function maskAwsToken(text) + { + return text.replace(AWS_TOKEN_PATTERN, String.raw`$1":"XXXX"`); + } + + function maskSasToken(text) + { + return text.replace(SAS_TOKEN_PATTERN, String.raw`$1=****`); + } + + function maskPrivateKey(text) + { + return text.replace(PRIVATE_KEY_PATTERN, String.raw`-----BEGIN PRIVATE KEY-----\\\\nXXXX\\\\n-----END PRIVATE KEY-----`); + } + + function maskPrivateKeyData(text) + { + return text.replace(PRIVATE_KEY_DATA_PATTERN, String.raw`"privateKeyData": "XXXX"`); + } + + function maskConnectionToken(text) + { + return text.replace(CONNECTION_TOKEN_PATTERN, String.raw`$1$2****`); + } + + function maskPassword(text) + { + return text.replace(PASSWORD_PATTERN, String.raw`$1$2****`); + } + + /** + * Masks any secrets. + * + * @param {String} text may contain a secret. + * + * @returns {Object} the masked string. + */ + this.maskSecrets = function (text) + { + var result; + if (!text) + { + result = + { + masked: false, + maskedtxt: text, + errstr: null + }; + return result; + } + + var masked = false; + var errstr = null; + try + { + if (mock) + { + mock.execute(); + } + + maskedtxt = + maskConnectionToken( + maskPassword( + maskPrivateKeyData( + maskPrivateKey( + maskAwsToken( + maskSasToken( + maskAwsKeys(text) + ) + ) + ) + ) + ) + ) + if (CUSTOM_PATTERNS_LENGTH > 0) + { + maskedtxt = maskCustomPattern(maskedtxt); + } + if (maskedtxt != text) + { + masked = true; + } + } + catch (err) + { + masked = true; + maskedtxt = err.toString(); + errstr = err.toString(); + } + + result = + { + masked: masked, + maskedtxt: maskedtxt, + errstr: errstr + }; + return result; + } +} + +module.exports = secret_detector; diff --git a/test/unit/secret_detector_test.js b/test/unit/secret_detector_test.js new file mode 100644 index 000000000..fc264d10f --- /dev/null +++ b/test/unit/secret_detector_test.js @@ -0,0 +1,324 @@ +/* + * Copyright (c) 2021 Snowflake Computing Inc. All rights reserved. + */ + +var assert = require('assert'); +var SnowflakeSecretDetector = require('./../../lib/secret_detector'); + + +describe('Secret Detector', function () +{ + var SecretDetector; + + const errstr = new Error('Test exception'); + var mock = + { + execute: function () + { + throw errstr; + } + } + + this.beforeEach(function () + { + SecretDetector = new SnowflakeSecretDetector(); + }); + + it('basic masking - null', async function () + { + var txt = null; + var result = SecretDetector.maskSecrets(txt); + assert.strictEqual(result.masked, false); + assert.strictEqual(result.maskedtxt, null); + assert.strictEqual(result.errstr, null); + }); + + it('basic masking - empty', async function () + { + var txt = ''; + var result = SecretDetector.maskSecrets(txt); + assert.strictEqual(result.masked, false); + assert.strictEqual(result.maskedtxt, txt); + assert.strictEqual(result.errstr, null); + }); + + it('basic masking - no masking', async function () + { + var txt = 'This string is innocuous'; + var result = SecretDetector.maskSecrets(txt); + assert.strictEqual(result.masked, false); + assert.strictEqual(result.maskedtxt, txt); + assert.strictEqual(result.errstr, null); + }); + + it('exception - masking', async function () + { + SecretDetector = new SnowflakeSecretDetector(null, mock); + var result = SecretDetector.maskSecrets('test'); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, errstr.toString()); + assert.strictEqual(result.errstr, errstr.toString()); + }); + + it('test - mask token', async function () + { + var longToken = '_Y1ZNETTn5/qfUWj3Jedby7gipDzQs=U' + + 'KyJH9DS=nFzzWnfZKGV+C7GopWCGD4Lj' + + 'OLLFZKOE26LXHDt3pTi4iI1qwKuSpf/F' + + 'mClCMBSissVsU3Ei590FP0lPQQhcSGcD' + + 'u69ZL_1X6e9h5z62t/iY7ZkII28n2qU=' + + 'nrBJUgPRCIbtJQkVJXIuOHjX4G5yUEKj' + + 'ZBAx4w6=_lqtt67bIA=o7D=oUSjfywsR' + + 'FoloNIkBPXCwFTv+1RVUHgVA2g8A9Lw5' + + 'XdJYuI8vhg=f0bKSq7AhQ2Bh'; + + var tokenWithPrefix = 'Token =' + longToken; + var result = SecretDetector.maskSecrets(tokenWithPrefix); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'Token =****'); + assert.strictEqual(result.errstr, null); + + var idTokenWithPrefix = 'idToken : ' + longToken + result = SecretDetector.maskSecrets(idTokenWithPrefix); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'idToken : ****'); + assert.strictEqual(result.errstr, null); + + var sessionTokenWithPrefix = 'sessionToken : ' + longToken + result = SecretDetector.maskSecrets(sessionTokenWithPrefix); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'sessionToken : ****'); + assert.strictEqual(result.errstr, null); + + var masterTokenWithPrefix = 'masterToken : ' + longToken + result = SecretDetector.maskSecrets(masterTokenWithPrefix); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'masterToken : ****'); + assert.strictEqual(result.errstr, null); + + var assertionWithPrefix = 'assertion content : ' + longToken + result = SecretDetector.maskSecrets(assertionWithPrefix); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'assertion content : ****'); + assert.strictEqual(result.errstr, null); + }); + + + it('test - false positive', async function () + { + var falsePositiveToken = "2020-04-30 23:06:04,069 - MainThread auth.py:397" + + " - write_temporary_credential() - DEBUG - no ID " + + "token is given when try to store temporary credential"; + + var result = SecretDetector.maskSecrets(falsePositiveToken); + assert.strictEqual(result.masked, false); + assert.strictEqual(result.maskedtxt, falsePositiveToken); + assert.strictEqual(result.errstr, null); + }); + + it('test - password', async function () + { + var randomPassword = 'Fh[+2J~AcqeqW%?'; + + var randomPasswordWithPrefix = 'password:' + randomPassword; + var result = SecretDetector.maskSecrets(randomPasswordWithPrefix); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'password:****'); + assert.strictEqual(result.errstr, null); + + var randomPasswordCaps = 'PASSWORD:' + randomPassword; + result = SecretDetector.maskSecrets(randomPasswordCaps); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'PASSWORD:****'); + assert.strictEqual(result.errstr, null); + + var randomPasswordMixedCase = 'PassWorD:' + randomPassword; + result = SecretDetector.maskSecrets(randomPasswordMixedCase); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'PassWorD:****'); + assert.strictEqual(result.errstr, null); + + var randomPasswordEqualSign = 'password =' + randomPassword; + result = SecretDetector.maskSecrets(randomPasswordEqualSign); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'password =****'); + assert.strictEqual(result.errstr, null); + + randomPasswordWithPrefix = 'pwd:' + randomPassword; + result = SecretDetector.maskSecrets(randomPasswordWithPrefix); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, 'pwd:****'); + assert.strictEqual(result.errstr, null); + }); + + + it('test - token password', async function () + { + var longToken = '_Y1ZNETTn5/qfUWj3Jedby7gipDzQs=U' + + 'KyJH9DS=nFzzWnfZKGV+C7GopWCGD4Lj' + + 'OLLFZKOE26LXHDt3pTi4iI1qwKuSpf/F' + + 'mClCMBSissVsU3Ei590FP0lPQQhcSGcD' + + 'u69ZL_1X6e9h5z62t/iY7ZkII28n2qU=' + + 'nrBJUgPRCIbtJQkVJXIuOHjX4G5yUEKj' + + 'ZBAx4w6=_lqtt67bIA=o7D=oUSjfywsR' + + 'FoloNIkBPXCwFTv+1RVUHgVA2g8A9Lw5' + + 'XdJYuI8vhg=f0bKSq7AhQ2Bh'; + + var longToken2 = 'ktL57KJemuq4-M+Q0pdRjCIMcf1mzcr' + + 'MwKteDS5DRE/Pb+5MzvWjDH7LFPV5b_' + + '/tX/yoLG3b4TuC6Q5qNzsARPPn_zs/j' + + 'BbDOEg1-IfPpdsbwX6ETeEnhxkHIL4H' + + 'sP-V'; + + var randomPwd = 'Fh[+2J~AcqeqW%?'; + var randomPwd2 = randomPwd + 'vdkav13'; + + var testStringWithPrefix = "token=" + longToken + + " random giberish " + + "password:" + randomPwd; + var result = SecretDetector.maskSecrets(testStringWithPrefix); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, + 'token=****' + + " random giberish " + + "password:****" + ); + assert.strictEqual(result.errstr, null); + + var testStringWithPrefixReversed = "password:" + randomPwd + + " random giberish " + + "token=" + longToken; + result = SecretDetector.maskSecrets(testStringWithPrefixReversed); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, + "password:****"+ + " random giberish " + + 'token=****' + ); + assert.strictEqual(result.errstr, null); + + var testStringWithPrefixMultiToken = "token=" + longToken + + " random giberish " + + "password:" + randomPwd + + " random giberish " + + "idToken:" + longToken2; + result = SecretDetector.maskSecrets(testStringWithPrefixMultiToken); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, + 'token=****' + + " random giberish " + + "password:****" + + " random giberish " + + "idToken:****" + ); + assert.strictEqual(result.errstr, null); + + var testStringWithPrefixMultiPass = "password=" + randomPwd + + " random giberish " + + "password=" + randomPwd2 + + " random giberish " + + "password=" + randomPwd; + result = SecretDetector.maskSecrets(testStringWithPrefixMultiPass); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, + "password=" + "****" + + " random giberish " + + "password=" + "****" + + " random giberish " + + "password=" + "****" + ); + assert.strictEqual(result.errstr, null); + }); + + it('custom pattern - success', async function () + { + var customPatterns = { + regex: [ + String.raw`(testCustomPattern\s*:\s*"([a-z]{8,})")`, + String.raw`(testCustomPattern\s*:\s*"([0-9]{8,})")` + ], + mask: [ + 'maskCustomPattern1', + 'maskCustomPattern2' + ] + } + + SecretDetector = new SnowflakeSecretDetector(customPatterns); + + var txt = 'testCustomPattern: "abcdefghijklmnop"'; + var result = SecretDetector.maskSecrets(txt); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, customPatterns.mask[0]); + assert.strictEqual(result.errstr, null); + + txt = 'testCustomPattern: "01123456978"'; + result = SecretDetector.maskSecrets(txt); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, customPatterns.mask[1]); + assert.strictEqual(result.errstr, null); + + txt = 'password=asdfasdfasdfasdfasdf ' + + 'testCustomPattern: "abcdefghijklmnop"'; + var result = SecretDetector.maskSecrets(txt); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, + "password=**** " + + customPatterns.mask[0]); + assert.strictEqual(result.errstr, null); + + txt = 'password=asdfasdfasdfasdfasdf ' + + 'testCustomPattern: "01123456978"'; + result = SecretDetector.maskSecrets(txt); + assert.strictEqual(result.masked, true); + assert.strictEqual(result.maskedtxt, + "password=**** " + + customPatterns.mask[1]); + assert.strictEqual(result.errstr, null); + }); + + it('custom pattern - regex error', async function () + { + var customPatterns = { + mask: ['maskCustomPattern1', 'maskCustomPattern2'] + } + try + { + SecretDetector = new SnowflakeSecretDetector(customPatterns); + } + catch (err) + { + assert.strictEqual(err.toString(), "Error: The customPatterns object must contain the 'regex' key"); + } + }); + + it('custom pattern - mask error', async function () + { + var customPatterns = { + regex: ['regexCustomPattern1', 'regexCustomPattern2'] + } + try + { + SecretDetector = new SnowflakeSecretDetector(customPatterns); + } + catch (err) + { + assert.strictEqual(err.toString(), "Error: The customPatterns object must contain the 'mask' key"); + } + }); + + it('custom pattern - unequal length error', async function () + { + var customPatterns = { + regex: ['regexCustomPattern1', 'regexCustomPattern2'], + mask: ['maskCustomPattern1'] + } + try + { + SecretDetector = new SnowflakeSecretDetector(customPatterns); + } + catch (err) + { + assert.strictEqual(err.toString(), "Error: The customPatterns object must have equal length for both 'regex' and 'mask'"); + } + }); +});