|
| 1 | +"use strict"; |
| 2 | +var __importDefault = (this && this.__importDefault) || function (mod) { |
| 3 | + return (mod && mod.__esModule) ? mod : { "default": mod }; |
| 4 | +}; |
| 5 | +Object.defineProperty(exports, "__esModule", { value: true }); |
| 6 | +exports.extract = void 0; |
| 7 | +var fs_1 = __importDefault(require("fs")); |
| 8 | +var path_1 = __importDefault(require("path")); |
| 9 | +var html_entities_1 = require("html-entities"); |
| 10 | +var util_1 = require("./util"); |
| 11 | +var extractorPath = path_1.default.join(__dirname, 'extractors'); |
| 12 | +var typeExtractors = {}; |
| 13 | +var regexExtractors = []; |
| 14 | +var failedExtractorTypes = {}; |
| 15 | +var STRIP_ONLY_SINGLE_LINEBREAKS = /(^|[^\n])\n(?!\n)/g; |
| 16 | +var WHITELIST_PRESERVE_LINEBREAKS = /[^A-Za-z\x80-\xFF\x24\u20AC\xA3\xA5 0-9 \u2015\u2116\u2018\u2019\u201C|\u201D\u2026 \uFF0C \u2013 \u2014 \u00C0-\u1FFF \u2C00-\uD7FF \uFB50–\uFDFF \uFE70–\uFEFF \uFF01-\uFFE6 .,?""!@#$%^&*()-_=+;:<>/\\|}{[\]`~'-\w\n\r]*/g, WHITELIST_STRIP_LINEBREAKS = /[^A-Za-z\x80-\xFF\x24\u20AC\xA3\xA5 0-9 \u2015\u2116\u2018\u2019\u201C|\u201D\u2026 \uFF0C \u2013 \u2014 \u00C0-\u1FFF \u2C00-\uD7FF \uFB50–\uFDFF \uFE70–\uFEFF \uFF01-\uFFE6 .,?""!@#$%^&*()-_=+;:<>/\\|}{[\]`~'-\w]*/g; |
| 17 | +var totalExtractors = 0, satisfiedExtractors = 0, hasInitialized = false; |
| 18 | +function registerExtractor(extractor) { |
| 19 | + if (extractor.types) { |
| 20 | + extractor.types.forEach(function (type) { |
| 21 | + if (typeof type === 'string') { |
| 22 | + type = type.toLowerCase(); |
| 23 | + typeExtractors[type] = extractor.extract; |
| 24 | + } |
| 25 | + else if (type instanceof RegExp) { |
| 26 | + regexExtractors.push({ reg: type, extractor: extractor.extract }); |
| 27 | + } |
| 28 | + }); |
| 29 | + } |
| 30 | +} |
| 31 | +function registerFailedExtractor(extractor, failedMessage) { |
| 32 | + if (extractor.types) { |
| 33 | + extractor.types.forEach(function (type) { |
| 34 | + failedExtractorTypes[type.toLowerCase()] = failedMessage; |
| 35 | + }); |
| 36 | + } |
| 37 | +} |
| 38 | +function testExtractor(extractor, options) { |
| 39 | + extractor.test(options, function (passedTest, failedMessage) { |
| 40 | + satisfiedExtractors++; |
| 41 | + if (passedTest) { |
| 42 | + registerExtractor(extractor); |
| 43 | + } |
| 44 | + else { |
| 45 | + registerFailedExtractor(extractor, failedMessage); |
| 46 | + } |
| 47 | + }); |
| 48 | +} |
| 49 | +// global, all file type, content cleansing |
| 50 | +function cleanseText(options, cb) { |
| 51 | + return function (error, text) { |
| 52 | + if (!error) { |
| 53 | + // clean up text |
| 54 | + text = (0, util_1.replaceBadCharacters)(text); |
| 55 | + if (options.preserveLineBreaks || options.preserveOnlyMultipleLineBreaks) { |
| 56 | + if (options.preserveOnlyMultipleLineBreaks) { |
| 57 | + text = text.replace(STRIP_ONLY_SINGLE_LINEBREAKS, '$1 ').trim(); |
| 58 | + } |
| 59 | + text = text.replace(WHITELIST_PRESERVE_LINEBREAKS, ' '); |
| 60 | + } |
| 61 | + else { |
| 62 | + text = text.replace(WHITELIST_STRIP_LINEBREAKS, ' '); |
| 63 | + } |
| 64 | + // multiple spaces, tabs, vertical tabs, non-breaking space] |
| 65 | + text = text.replace(/ (?! )/g, '').replace(/[ \t\v\u00A0]{2,}/g, ' '); |
| 66 | + text = (0, html_entities_1.decode)(text); |
| 67 | + } |
| 68 | + cb(error, text); |
| 69 | + }; |
| 70 | +} |
| 71 | +function initializeExtractors(options) { |
| 72 | + hasInitialized = true; |
| 73 | + // discover available extractors |
| 74 | + var extractors = fs_1.default.readdirSync(extractorPath).map(function (item) { |
| 75 | + var fullExtractorPath = path_1.default.join(extractorPath, item); |
| 76 | + // get the extractor |
| 77 | + // eslint-disable-next-line global-require,import/no-dynamic-require |
| 78 | + return require(fullExtractorPath); |
| 79 | + }); |
| 80 | + // perform any binary tests to ensure extractor is possible |
| 81 | + // given execution environment |
| 82 | + extractors.forEach(function (extractor) { |
| 83 | + if (extractor.test) { |
| 84 | + testExtractor(extractor, options); |
| 85 | + } |
| 86 | + else { |
| 87 | + satisfiedExtractors++; |
| 88 | + registerExtractor(extractor); |
| 89 | + } |
| 90 | + }); |
| 91 | + // need to keep track of how many extractors we have in total |
| 92 | + totalExtractors = extractors.length; |
| 93 | +} |
| 94 | +function findExtractor(type) { |
| 95 | + var iLen = regexExtractors.length; |
| 96 | + var extractor, regexExtractor; |
| 97 | + type = type.toLowerCase(); |
| 98 | + if (typeExtractors[type]) { |
| 99 | + extractor = typeExtractors[type]; |
| 100 | + } |
| 101 | + else { |
| 102 | + for (var i = 0; i < iLen; i++) { |
| 103 | + regexExtractor = regexExtractors[i]; |
| 104 | + if (type.match(regexExtractor.reg)) { |
| 105 | + extractor = regexExtractor.extractor; |
| 106 | + } |
| 107 | + } |
| 108 | + } |
| 109 | + return extractor; |
| 110 | +} |
| 111 | +function extract(type, filePath, options, cb) { |
| 112 | + var error, msg, theExtractor; |
| 113 | + if (!hasInitialized) { |
| 114 | + initializeExtractors(options); |
| 115 | + } |
| 116 | + // registration of extractors complete? |
| 117 | + if (totalExtractors === satisfiedExtractors) { |
| 118 | + theExtractor = findExtractor(type); |
| 119 | + if (theExtractor) { |
| 120 | + cb = cleanseText(options, cb); |
| 121 | + theExtractor(filePath, options, cb); |
| 122 | + } |
| 123 | + else { |
| 124 | + // cannot extract this file type |
| 125 | + msg = "Error for type: [[ ".concat(type, " ]], file: [[ ").concat(filePath, " ]]"); |
| 126 | + // update error message if type is supported but just not configured/installed properly |
| 127 | + if (failedExtractorTypes[type]) { |
| 128 | + msg += "".concat(', extractor for type exists, but failed to initialize. Message:').concat(failedExtractorTypes[type]); |
| 129 | + } |
| 130 | + error = new Error(msg); |
| 131 | + cb(error, null); |
| 132 | + } |
| 133 | + } |
| 134 | + else { |
| 135 | + // async registration has not wrapped up |
| 136 | + // try again later |
| 137 | + setTimeout(function () { |
| 138 | + extract(type, filePath, options, cb); |
| 139 | + }, 100); |
| 140 | + } |
| 141 | +} |
| 142 | +exports.extract = extract; |
0 commit comments