diff --git a/queries/cdmq/add-run.js b/queries/cdmq/add-run.js index 45074696..072d484d 100755 --- a/queries/cdmq/add-run.js +++ b/queries/cdmq/add-run.js @@ -1,15 +1,17 @@ //# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=javascript -var cdm = require('./cdm'); -var fs = require('fs'); -var xz = require('xz'); -var readline = require('readline'); -var path = require('path'); -var program = require('commander'); -var instances = []; // opensearch instances +const { XzReadableStream } = require('xz-decompress'); +const { Readable } = require('stream'); +const cdm = require('./cdm'); +const fs = require('fs'); +const xz = require('xz'); +const readline = require('readline'); +const path = require('path'); +const program = require('commander'); +const instances = []; // opensearch instances function save_host(host) { - var host_info = { host: host, header: { 'Content-Type': 'application/json' } }; + const host_info = { host: host, header: { 'Content-Type': 'application/json' } }; instances.push(host_info); } @@ -37,10 +39,25 @@ function save_ver(ver) { } } +// Read an xz file and decompress to string +async function decompressXzFile(filename) { + // Create a readable stream from file + const fileStream = Readable.toWeb(fs.createReadStream(filename)); + + // Decompress using XzReadableStream + const decompressedResponse = new Response(new XzReadableStream(fileStream)); + + // Get the decompressed content as string + const text = await decompressedResponse.text(); + return text; +} + async function readNdjsonXzToString(filePath) { return new Promise((resolve, reject) => { try { - const fileStream = fs.createReadStream(filePath); + const fileStream = fs.createReadStream(filePath, { + highWaterMark: 5 * 1024 * 1024 + }); const decompressStream = new xz.Decompressor(); let decompressedString = ''; @@ -52,37 +69,43 @@ async function readNdjsonXzToString(filePath) { }); decompressStream.on('end', () => { + fileStream.destroy(); resolve(decompressedString); }); decompressStream.on('error', (error) => { + fileStream.destroy(); reject(error); }); fileStream.on('error', (error) => { + fileStream.destroy(); reject(error); }); } catch (error) { reject(error); - } + } /*finally { + // Always cleanup, regardless of success or failure + if (fileStream && !fileStream.destroyed) { + fileStream.destroy(); + } + }*/ }); } async function processDir(instance, dir, mode) { - var jsonArr = []; - var files = fs.readdirSync(dir); - - for (var i = 0; i < files.length; i++) { - var regExp = /\.ndjson\.xz$/; - var matches = regExp.exec(files[i]); - // skip non ndjson.xz files - if (!matches) { - continue; - } + const jsonArr = []; + const info = { runIds: {} }; + const allFiles = fs.readdirSync(dir); + const regExp = /\.ndjson\.xz$/; + const xzFiles = allFiles.filter((item) => regExp.test(item)); + + for (var i = 0; i < xzFiles.length; i++) { + const regExp = /\.ndjson\.xz$/; + const filePath = path.join(program.dir, xzFiles[i]); try { - const filePath = path.join(program.dir, files[i]); - const decompressedData = await readNdjsonXzToString(filePath); - var lines = decompressedData.split('\n'); + const decompressedData = await decompressXzFile(filePath); + const lines = decompressedData.split('\n'); for (var j = 0; j < lines.length; j++) { // TODO: validate JSON syntax and possible validate document schema? if (lines[j] != '') { @@ -92,51 +115,100 @@ async function processDir(instance, dir, mode) { } catch (error) { console.error('Error processing NDJSON.XZ file:', error); } - } + cdm.memUsage(); - if (mode == 'index') { - var responses = await esJsonArrRequest(instance, '', '/_bulk', jsonArr); - } else if (mode == 'getinfo') { - var info = { runIds: [], indices: {} }; - console.log('going to process ' + jsonArr.length + ' lines'); - for (var k = 1; k < jsonArr.length; k += 2) { - try { - var action = JSON.parse(jsonArr[k - 1]); - var doc = JSON.parse(jsonArr[k]); - } catch (jsonError) { - console.log('Could not porse: [' + jsonArr[k] + ']'); - continue; - } - runId = doc['run']['run-uuid']; - if (!info['runIds'].includes(runId)) { - info['runIds'].push(runId); - } - // { "index": { "_index": "cdmv8dev-metric_data" } } - if (Object.keys(action).includes('index') && Object.keys(action['index']).includes('_index')) { - const indexName = action['index']['_index']; - const regExp = /^cdm-*(v7dev|v8dev|v9dev)-([^@]+)(@\d\d\d\d\.\d\d)*$/; - var matches = regExp.exec(indexName); - if (matches) { - cdmVer = matches[1]; - if (!Object.keys(info['indices']).includes(cdmVer)) { - info['indices'][cdmVer] = []; + // After so much data is accumulated or at the last file, index what we have then clear the array. + if (jsonArr.length > 4000000 || i == xzFiles.length) { + console.log( + mode + + ': After reading ' + + i + + ' of ' + + (xzFiles.length + 1) + + ' files, going to process ' + + jsonArr.length + + ' lines' + ); + + if (mode == 'index') { + // The final argument for esJsonArrRequest, 'yearDotMonth', is not + // required here because the jsonArr includes the exact index name + // in the 'action' part of the array + const responses = await esJsonArrRequest(instance, '', '/_bulk', jsonArr); + } else if (mode == 'getinfo') { + //const info = { runIds: [], indices: {} }; + for (var k = 1; k < jsonArr.length; k += 2) { + try { + var action = JSON.parse(jsonArr[k - 1]); + var doc = JSON.parse(jsonArr[k]); + } catch (jsonError) { + console.log('Could not porse: [' + jsonArr[k] + ']'); + continue; } - if (!info['indices'][cdmVer].includes(indexName)) { - debuglog('going to add indexname to info[indices][' + cdmVer + ']: ' + indexName); - info['indices'][cdmVer].push(indexName); + let runId = doc['run']['run-uuid']; + if (!Object.keys(info['runIds']).includes(runId)) { + info['runIds'][runId] = { indices: {}, yearDotMonth: '' }; + } + // example action: { "index": { "_index": "cdmv8dev-metric_data" } } + // exmaple doc: + // {"cdm":{"ver":"v9dev"}, + // "run":{"run-uuid":"c0e04edb-ddbc-4081-8bbc-9b9e84e6538d"}} + if (Object.keys(action).includes('index') && Object.keys(action['index']).includes('_index')) { + const indexName = action['index']['_index']; + const regExp = /^cdm-*(v7dev|v8dev|v9dev)-([^@]+)(@\d\d\d\d\.\d\d)*$/; + const matches = regExp.exec(indexName); + //console.log("doc:\n" + JSON.stringify(doc, null, 2)); + if (matches) { + const cdmVer = matches[1]; + let yearDotMonth = ''; + let runId = ''; + if (cdmVer == 'v7dev') { + runId = doc['run']['id']; + } else { + // v8dev and newer use run-uuid + runId = doc['run']['run-uuid']; + yearDotMonth = matches[3]; + } + if (!Object.keys(info['runIds'][runId]).includes('indices')) { + info['runIds'][runId]['indices'] = {}; + } + if (!Object.keys(info['runIds'][runId]['indices']).includes(cdmVer)) { + info['runIds'][runId]['indices'][cdmVer] = []; + } + if (!info['runIds'][runId]['indices'][cdmVer].includes(indexName)) { + debuglog( + 'going to add indexname to info[runIds][' + runId + '][indices][' + cdmVer + ']: ' + indexName + ); + info['runIds'][runId]['indices'][cdmVer].push(indexName); + } + if (yearDotMonth != '') { + if (info['runIds'][runId]['yearDotMonth'] == '') { + info['runIds'][runId]['yearDotMonth'] = yearDotMonth; + } else { + if (info['runIds'][runId]['yearDotMonth'] != yearDotMonth) { + console.log('ERROR: found more than one year and month for same runId [' + runId + ']'); + process.exit(1); + } + } + } + } else { + console.log('ERROR: the index name [' + indexName + '] was not recognized'); + process.exit(1); + } + } else { + console.log('the ndjson action [' + action + '] was not valid'); + process.exit(1); } - } else { - console.log('ERROR: the index name [' + indexName + '] was not recognized'); - process.exit(1); } } else { - console.log('the ndjson action [' + action + '] was not valid'); - process.exit(1); + console.log('mode [' + mode + '] is not supported'); } - } + // Must make array empty here for next pass + jsonArr.length = 0; + } // if (jsonArr.length > 100000 || i == xzFiles.length) + } + if (mode == 'getinfo') { return info; - } else { - console.log('mode [' + mode + '] is not supported'); } } @@ -155,45 +227,65 @@ async function main() { } getInstancesInfo(instances); + // Always use the last instance (last --host) provided by the user - var instance = instances[instances.length - 1]; + const instance = instances[instances.length - 1]; if (program.dir) { - var allDocTypes = ['run', 'iteration', 'sample', 'period', 'param', 'tag', 'metric_desc', 'metric_data']; - var info = await processDir(instance, program.dir, 'getinfo'); + const allDocTypes = ['run', 'iteration', 'sample', 'period', 'param', 'tag', 'metric_desc', 'metric_data']; + const info = await processDir(instance, program.dir, 'getinfo'); // The cdmVer going forward must be set based on the data found in the ndjson files. // We cannot rely on automatic detection of cdmVer based on what is present in the instance // because the instance may contain only cdmVer that is different from the *new* data we are adding. // // Also, the cdm version is embedded in this new data that will be indexed. It is not possible // (without significant effort) to index data already in one cdm version to another [directly]. - if (Object.keys(info['indices']).length == 1) { - const cdmVer = Object.keys(info['indices'])[0]; - if (!cdm.supportedCdmVersions.includes(cdmVer)) { - console.log( - 'ERROR: the CDM version found in the documents to be indexed [' + - cdmver + - '] is not included in the list of supported CDM versions [' + - cdm.supportedCdmVersions + - ']' - ); + runIds = Object.keys(info['runIds']); + for (var runIdx = 0; runIdx < runIds.length; runIdx++) { + const runId = runIds[runIdx]; + var cdmVer; + if (Object.keys(info['runIds'][runId]['indices']).length == 1) { + cdmVer = Object.keys(info['runIds'][runId]['indices'])[0]; + if (!cdm.supportedCdmVersions.includes(cdmVer)) { + console.log( + 'ERROR: the CDM version found in the documents to be indexed [' + + cdmver + + '] is not included in the list of supported CDM versions [' + + cdm.supportedCdmVersions + + ']' + ); + process.exit(1); + } + instance['ver'] = cdmVer; + //if (!Object.keys(instance['indices']).includes(cdmVer)) { + //instance['indices'][cdmVer] = []; + //} + } else { + console.log('ERROR: there was not exactly one CDM version found in the data to be indexed:\n'); + console.log(Object.keys(info['indices'])); + console.log('info\n' + JSON.stringify(info['indices'], null, 2)); process.exit(1); } - instance['cdmVer'] = cdmVer; - if (!Object.keys(instance['indices']).includes(cdmVer)) { - instance['indices'][cdmVer] = []; + // For cdmv9 and newer, any time a document is to be indexed, it is imperitive + // that a check for the existence of the index is done, and if not found, create + // the index with the *correct* mappings and settings. If this is not done, an index + // may be auto-created with the *incorrect* mappings and settings, and documents can + // be indexed, but not properly, and subsequent queries will *NOT* work. + debuglog(JSON.stringify(info['runIds'][runId]['indices'][cdmVer], null, 2)); + for (var i = 0; i < info['runIds'][runId]['indices'][cdmVer].length; i++) { + debuglog('checking for index ' + info['runIds'][runId]['indices'][cdmVer][i]); + cdm.checkCreateIndex(instance, info['runIds'][runId]['indices'][cdmVer][i]); } - } else { - console.log('ERROR: there was not exactly one CDM version found in the data to be indexed:\n'); - console.log(Object.keys(info['indices'])); - console.log('info\n' + JSON.stringify(info['indices'], null, 2)); - process.exit(1); - } - for (i = 0; i < info['runIds'].length; i++) { - console.log('Deleting any existing documents for runId ' + info['runIds'][i]); - var runId = info['runIds'][i]; - var q = { query: { bool: { filter: [{ term: { 'run.run-uuid': runId } }] } } }; - cdm.deleteDocs(instance, allDocTypes, q); - var numDocTypes = await cdm.waitForDeletedDocs(instances[instances.length - 1], runId, allDocTypes); + // Before indexing any documents, we must check for any existing ones. Having duplicate + // documents is really bad + console.log('Deleting any existing documents for runId ' + runId); + const q = { query: { bool: { filter: [{ term: { 'run.run-uuid': runId } }] } } }; + cdm.deleteDocs(instance, allDocTypes, q, info['runIds'][runId]['yearDotMonth']); + const numDocTypes = await cdm.waitForDeletedDocs( + instances[instances.length - 1], + runId, + allDocTypes, + info['runIds'][runId]['yearDotMonth'] + ); if (numDocTypes > 0) { console.log('Warning: could not delete all documents for ' + docTypes + ' with ' + numAttempts); console.log( @@ -201,12 +293,12 @@ async function main() { ); process.exit(1); } + const begin = Date.now() / 1000; + console.log('Indexing documents'); + await processDir(instance, program.dir, 'index'); + const end = Date.now() / 1000; + console.log('Time (seconds) to submit all documents for indexing: ' + (end - begin)); } - for (i = 0; i < info['indices'].length; i++) { - cdm.checkCreateIndex(instance, info['indices'][i]); - } - console.log('Indexing documents'); - await processDir(instance, program.dir, 'index'); } else { console.log('You must provide a --dir '); process.exit(1); diff --git a/queries/cdmq/cdm.js b/queries/cdmq/cdm.js index 4b0d92fa..573df30c 100644 --- a/queries/cdmq/cdm.js +++ b/queries/cdmq/cdm.js @@ -329,6 +329,26 @@ indexDefs['v8dev']['metric_data']['mappings']['properties']['metric_data'] = { }; indexDefs['v9dev']['metric_data'] = JSON.parse(JSON.stringify(indexDefs['v8dev']['metric_data'])); +function memUsage() { + const memUsage = process.memoryUsage(); + debuglog({ + rss: `${Math.round(memUsage.rss / 1024 / 1024)} MB`, // Resident Set Size + heapTotal: `${Math.round(memUsage.heapTotal / 1024 / 1024)} MB`, // Total heap size + heapUsed: `${Math.round(memUsage.heapUsed / 1024 / 1024)} MB`, // Used heap size + external: `${Math.round(memUsage.external / 1024 / 1024)} MB` // External memory + }); +} +exports.memUsage = memUsage; + +function numMBytes(a, str) { + var totalBytes = 0; + a.forEach((element) => { + totalBytes += JSON.stringify(element).length; + }); + var mb = totalBytes / 1024 / 1024; + return mb; +} + function getCdmVer(instance) { return instance['ver']; } @@ -340,6 +360,18 @@ debuglog = function (str) { }; exports.debuglog = debuglog; +getCdmVerFromIndex = function (index) { + var regExp = /^cdm-*v([\d+])dev-(.+)/; + var matches = regExp.exec(index); + if (matches) { + cdmVer = 'v' + matches[1] + 'dev'; + } else { + console.log('ERROR: index [' + index + '] is not recognized'); + process.exit(1); + } + return cdmVer; +}; + // cdmv9 adds indices dynamically based on year and month, so // we need a way to check for existence of the index and create // if not found. Since this moves toward a auto-create index @@ -347,26 +379,36 @@ exports.debuglog = debuglog; // This will eliminate the need for other projects (like Crucible) // to maintain the indices. checkCreateIndex = function (instance, index) { - const cdmVer = getCdmVer(instance); + const cdmVer = getCdmVerFromIndex(index); + //if (!Object.keys(instance).includes('indices')) { + //instance['indices'] = {}; + //} - debuglog('instance:\n' + JSON.stringify(instance, null, 2)); - debuglog('index: ' + index); + //debuglog('instance:\n' + JSON.stringify(instance, null, 2)); + //debuglog('index: ' + index); if (Object.keys(instance['indices']).includes(cdmVer)) { - debuglog('found cdmver: ' + cdmVer); + //debuglog('found cdmver: ' + cdmVer); if (instance['indices'][cdmVer].includes(index)) { - debuglog('found index: ' + index); + //debuglog('found index: ' + index); return; } } - const docType = getDocType(index, cdmVer); - debuglog('got docType: [' + docType + ']'); + var regExp = /\*$/; + var matches = regExp.exec(index); + if (matches) { + //debuglog('Not going to create index because it includes a wildcard: [' + index + ']'); + return; + } + + const docType = getDocType(index); + debuglog('checkCreateIndex(): got docType: [' + docType + ']'); var url = 'http://' + instance['host'] + '/' + index; var resp = request('PUT', url, { headers: instance['header'], body: JSON.stringify(indexDefs[cdmVer][docType]) }); var data = JSON.parse(resp.getBody()); debuglog('response:::\n' + JSON.stringify(data, null, 2)); if (!Object.keys(instance['indices']).includes(cdmVer)) { - push(instance['indices'], cdmver); + instance['indices'][cdmver] = []; } instance['indices'][cdmVer].push(index); //TODO: query opensearch to verify index is present @@ -374,8 +416,10 @@ checkCreateIndex = function (instance, index) { }; exports.checkCreateIndex = checkCreateIndex; -function getDocType(index, cdmVer) { - debuglog('cdmver: [' + cdmVer + ']'); +function getDocType(index) { + const cdmVer = getCdmVerFromIndex(index); + + //debuglog('cdmver: [' + cdmVer + ']'); if (cdmVer == 'v7dev' || cdmVer == 'v8dev') { var regExp = /^cdmv[7|8]dev-(.+)/; var matches = regExp.exec(index); @@ -394,7 +438,7 @@ function getDocType(index, cdmVer) { } if (cdmVer == 'v9dev') { - var regExp = /^cdm-v9dev-([^@]+)@\d\d\d\d\.\d\d/; + var regExp = /^cdm-v9dev-([^@]+)(@\d\d\d\d\.\d\d|\*)/; var matches = regExp.exec(index); if (matches) { docType = matches[1]; @@ -417,7 +461,7 @@ function getDocType(index, cdmVer) { function getIndexBaseName(instance) { // CDM version support is effectively determined here cdmVer = getCdmVer(instance); - debuglog('cdmver: [' + cdmVer + ']'); + //debuglog('cdmver: [' + cdmVer + ']'); if (cdmVer == 'v7dev' || cdmVer == 'v8dev') { return 'cdm' + cdmVer + '-'; } else if (cdmVer == 'v9dev') { @@ -431,16 +475,17 @@ function getIndexBaseName(instance) { } } -function getIndexName(docType, instance) { +function getIndexName(docType, instance, yearDotMonth) { const baseName = getIndexBaseName(instance); cdmVer = getCdmVer(instance); if (cdmVer == 'v7dev' || cdmVer == 'v8dev') { name = docType; } else { - //TODO: return correct year.month(s) - name = docType + '@2025.02'; + name = docType + yearDotMonth; } fullName = baseName + name; + //debuglog('getIndexName() fullName: [' + fullName + ']'); + // necessary? checkCreateIndex(instance, fullName); return fullName; } @@ -502,7 +547,8 @@ intersectAllArrays = function (a2D) { exports.intersectAllArrays = intersectAllArrays; async function fetchBatchedData(instance, reqs, batchSize = 16) { - debuglog('fetchBatchedData() begin'); + //debuglog('fetchBatchedData() begin'); + //debuglog('fetchBatchedData() reqs.length: ' + reqs.length); const responses = []; const batches = []; @@ -511,21 +557,21 @@ async function fetchBatchedData(instance, reqs, batchSize = 16) { } for (const batch of batches) { - debuglog('fetchBatchedData() processing batch'); + //debuglog('fetchBatchedData() processing batch'); const promises = batch.map(async (req) => { try { // thenRequest will abolutely *not* work unless this header is converted to string and back const headerStr = JSON.stringify(instance['header']); const hdrs = JSON.parse(headerStr); - debuglog('fetchBatchedData() calling thenRequest()'); + //debuglog('fetchBatchedData() calling thenRequest()'); const response = await thenRequest('POST', req.url, { body: req.body, headers: hdrs }); - debuglog('fetchBatchedData() returned from thenRequest()'); + //debuglog('fetchBatchedData() returned from thenRequest()'); if (response.statusCode >= 200 && response.statusCode < 300) { try { - debuglog('fetchBatchedData() about to return with JSON.parse'); + //debuglog('fetchBatchedData() about to return with JSON.parse'); return JSON.parse(response.getBody('utf8')); // Attempt JSON parsing } catch (jsonError) { - debuglog('fetchBatchedData() about to return with JSON(no-parse)'); + //debuglog('fetchBatchedData() about to return with JSON(no-parse)'); return response.getBody('utf8'); // return text if JSON parsing fails } } else { @@ -538,7 +584,7 @@ async function fetchBatchedData(instance, reqs, batchSize = 16) { }); const batchResults = await Promise.all(promises); - debuglog('fetchBatchedData() batchResults.length:\n' + batchResults.length); + //debuglog('fetchBatchedData() batchResults.length:' + batchResults.length); for (const batchResult of batchResults) { const keys = Object.keys(batchResult); // items present if this is document creation @@ -550,21 +596,32 @@ async function fetchBatchedData(instance, reqs, batchSize = 16) { responses.push(...batchResult.responses); } } + //debuglog('fetchBatchedData() responses.length so far:' + responses.length); + //debuglog('fetchBatchedData() responses[-1]:\n' + JSON.stringify(responses[responses.length - 1], null, 2)); } } - debuglog('fetchBatchedData() responses.length:\n' + responses.length); + //debuglog('fetchBatchedData() responses.length:' + responses.length); return responses; } -esJsonArrRequest = async function (instance, index, action, jsonArr) { - debuglog('esJsonArrRequest() begin'); +esJsonArrRequest = async function (instance, docType, action, jsonArr, yearDotMonth) { + //debuglog('esJsonArrRequest begin: yearDotMonth: ' + yearDotMonth); + debuglog('esJsonArrRequest jsonArr.length: ' + jsonArr.length); + memUsage(); + debuglog('jsonArr MB at beginning of esJsonArrRequest: ' + numMBytes(jsonArr)); + + //if (jsonArr.length > 500) { + //process.exit(1); + //} + + var thisJson = ''; var url = ''; - if (index == '') { + if (docType == '') { // Expect to have the Index and action in the jsonArr itself url = 'http://' + instance['host'] + '/_bulk'; } else { - url = 'http://' + instance['host'] + '/' + getIndexName(index, instance) + action; + url = 'http://' + instance['host'] + '/' + getIndexName(docType, instance, yearDotMonth) + action; } var max = 16384; var idx = 0; @@ -576,45 +633,65 @@ esJsonArrRequest = async function (instance, index, action, jsonArr) { var allResponses = []; // Process queries in chunks no larger than 'max' chars while (idx < jsonArr.length) { - // Add the first request (2 lines) even if it exceeds our limit (we don't really have a choice) - // The limit we have is likely much lower than what can be handled, but if this becomes a - // problem, we'll have to look at either an alternate way to submit a huge request, or we will - // have to break up the request into mulitple requests with fewer metric_id's, then sum the - // responses. - q_count++; - ndjson += jsonArr[idx] + '\n' + jsonArr[idx + 1] + '\n'; - idx += 2; - // Add more requests if are any left and the max will not be exceeded - if (idx + 2 < jsonArr.length && ndjson.length + jsonArr[idx].length + jsonArr[idx + 1].length < max) { + while (reqs.length < 64 && idx < jsonArr.length) { + // Limit size in order to not exhaust heap + // Add the first request (2 lines) even if it exceeds our limit (we don't really have a choice) + // The limit we have is likely much lower than what can be handled, but if this becomes a + // problem, we'll have to look at either an alternate way to submit a huge request, or we will + // have to break up the request into mulitple requests with fewer metric_id's, then sum the + // responses. q_count++; ndjson += jsonArr[idx] + '\n' + jsonArr[idx + 1] + '\n'; idx += 2; - } else { + // Add more requests if are any left and the max will not be exceeded + if (idx + 2 < jsonArr.length && ndjson.length + jsonArr[idx].length + jsonArr[idx + 1].length < max) { + q_count++; + ndjson += jsonArr[idx] + '\n' + jsonArr[idx + 1] + '\n'; + // Remove data from jsonArr to conserve memory (should be cleaned up by GC) + delete jsonArr[idx]; + delete jsonArr[idx + 1]; + idx += 2; + } else { + req_count++; + q_count = 0; + const req = { url: url, body: ndjson }; + reqs.push(req); + ndjson = ''; + } + } //while + // Max was not exceeded but there are some requests that have not been submitted + if (ndjson != '') { req_count++; q_count = 0; const req = { url: url, body: ndjson }; reqs.push(req); - ndjson = ''; - } - } //while - // Max was not exceeded but there are some requests that have not been submitted - if (ndjson != '') { - req_count++; - q_count = 0; - const req = { url: url, body: ndjson }; - reqs.push(req); - } - debuglog('esJsonArrRequest(): There are ' + reqs.length + ' requests:\n' + JSON.stringify(reqs, null, 2)); - const responses = await fetchBatchedData(instance, reqs); - debuglog('esJsonArrRequest(): responses.length:\n' + responses.length); + } + + debuglog('esJsonArrRequest reqs.length:\n' + reqs.length); + var responses = await fetchBatchedData(instance, reqs); + reqs = []; + + debuglog('esJsonArrRequest jsonArr MB: ' + numMBytes(jsonArr)); + debuglog('esJsonArrRequest responses MB: ' + numMBytes(responses)); + memUsage(); + + allResponses.push(...responses); + responses = []; + + debuglog('esJsonArrRequest responses MB after clearing: ' + numMBytes(responses)); + memUsage(); + } + + debuglog('iesJsonArrRequest allResponses ' + numMBytes(allResponses)); + debuglog('esJsonArrRequest end'); - return responses; + return allResponses; }; exports.esJsonArrRequest = esJsonArrRequest; -function esRequest(instance, idx, action, q) { +function esRequest(instance, docType, action, q, yearDotMonth) { // This is the only http request remainig that still uses a sync-request - var url = 'http://' + instance['host'] + '/' + getIndexName(idx, instance) + action; + var url = 'http://' + instance['host'] + '/' + getIndexName(docType, instance, yearDotMonth) + action; debuglog('esRequest() url: ' + url); // The var q can be an object or a string. If you are submitting NDJSON // for a _msearch, it must be a [multi-line] string. @@ -635,7 +712,7 @@ function esRequest(instance, idx, action, q) { // of http requests. // Note: termKeys is a 1D array, while values is a 2D array. // termKeys[x] uses list of values from values[x] -mSearch = async function (instance, index, termKeys, values, source, aggs, size, sort) { +mSearch = async function (instance, index, yearDotMonth, termKeys, values, source, aggs, size, sort) { if (typeof termKeys !== typeof []) return; if (typeof values !== typeof []) return; var jsonArr = []; @@ -664,8 +741,9 @@ mSearch = async function (instance, index, termKeys, values, source, aggs, size, jsonArr.push(JSON.stringify(req)); } debuglog('mSearch(): calling esJsonArrRequest()'); - var responses = await esJsonArrRequest(instance, index, '/_msearch', jsonArr); - debuglog('mSearch(): returned from calling esJsonArrRequest(), responses:\n' + JSON.stringify(responses, null, 2)); + var responses = await esJsonArrRequest(instance, index, '/_msearch', jsonArr, yearDotMonth); + //debuglog('mSearch(): returned from calling esJsonArrRequest(), responses:\n' + JSON.stringify(responses, null, 2)); + memUsage(); // Unpack response and organize in array of arrays var retData = []; @@ -747,10 +825,11 @@ exports.mSearch = mSearch; // but these functions just wrap the value in 2D array for msearch (and a key in a 1D array). Effectively // all query functions use msearch, even if there is a single query. -mgetPrimaryMetric = async function (instance, iterations) { +mgetPrimaryMetric = async function (instance, iterations, yearDotMonth) { var metrics = await mSearch( instance, 'iteration', + yearDotMonth, ['iteration.iteration-uuid'], [iterations], 'iteration.primary-metric' @@ -765,16 +844,17 @@ mgetPrimaryMetric = async function (instance, iterations) { }; exports.mgetPrimaryMetric = mgetPrimaryMetric; -getPrimaryMetric = async function (instance, iteration) { - var primaryMetrics = await mgetPrimaryMetric(instance, [iteration]); +getPrimaryMetric = async function (instance, iteration, yearDotMonth) { + var primaryMetrics = await mgetPrimaryMetric(instance, [iteration], yearDotMonth); return primaryMetrics[0][0]; }; exports.getPrimaryMetric = getPrimaryMetric; -mgetPrimaryPeriodName = async function (instance, iterations) { +mgetPrimaryPeriodName = async function (instance, iterations, yearDotMonth) { var data = await mSearch( instance, 'iteration', + yearDotMonth, ['iteration.iteration-uuid'], [iterations], 'iteration.primary-period' @@ -789,21 +869,29 @@ mgetPrimaryPeriodName = async function (instance, iterations) { }; exports.mgetPrimaryPeriodName = mgetPrimaryPeriodName; -getPrimaryPeriodName = async function (instance, iteration) { - var primaryPeriodNames = await mgetPrimaryPeriodName(instance, [iteration]); +getPrimaryPeriodName = async function (instance, iteration, yearDotMonth) { + var primaryPeriodNames = await mgetPrimaryPeriodName(instance, [iteration], yearDotMonth); return primaryPeriodNames[0][0]; }; exports.getPrimaryPeriodName = getPrimaryPeriodName; -mgetSamples = async function (instance, iters) { - return await mSearch(instance, 'sample', ['iteration.iteration-uuid'], [iters], 'sample.sample-uuid', null, 1000, [ - { 'sample.num': { order: 'asc', numeric_type: 'long' } } - ]); +mgetSamples = async function (instance, iters, yearDotMonth) { + return await mSearch( + instance, + 'sample', + yearDotMonth, + ['iteration.iteration-uuid'], + [iters], + 'sample.sample-uuid', + null, + 1000, + [{ 'sample.num': { order: 'asc', numeric_type: 'long' } }] + ); }; exports.mgetSamples = mgetSamples; -getSamples = async function (instance, iter) { - var samples = await mgetSamples(instance, [iter]); +getSamples = async function (instance, iter, yearDotMonth) { + var samples = await mgetSamples(instance, [iter], yearDotMonth); return samples[0]; }; exports.getSamples = getSamples; @@ -812,25 +900,25 @@ exports.getSamples = getSamples; // find all the metadata names shared among all // found metric docs. These names are what can be // used for "breakouts". -mgetMetricNames = async function (instance, runIds, sources, types) { +mgetMetricNames = async function (instance, runIds, sources, types, yearDotMonth) { return await mSearch( instance, 'metric_desc', + yearDotMonth, ['run.run-uuid', 'metric_desc.source', 'metric_desc.type'], [runIds, sources, types], '', - { source: { terms: { field: 'metric_desc.names-list', size: bigQuerySize } } }, - 0 + { source: { terms: { field: 'metric_desc.names-list', size: bigQuerySize } } } ); }; exports.mgetMetricNames = mgetMetricNames; -getMetricNames = async function (instance, runId, source, type) { - var metricNames = await mgetMetricNames(instance, [runId], [source], [type]); +getMetricNames = async function (instance, runId, source, type, yearDotMonth) { + var metricNames = await mgetMetricNames(instance, [runId], [source], [type], yearDotMonth); return metricNames[0]; }; -mgetSampleNums = async function (instance, Ids) { +mgetSampleNums = async function (instance, Ids, yearDotMonth) { var sampleIds = []; var idx = 0; for (var i = 0; i < Ids.length; i++) { @@ -840,7 +928,16 @@ mgetSampleNums = async function (instance, Ids) { } } - var data = await mSearch(instance, 'sample', ['sample.sample-uuid'], [sampleIds], 'sample.num', null, 1); + var data = await mSearch( + instance, + 'sample', + yearDotMonth, + ['sample.sample-uuid'], + [sampleIds], + 'sample.num', + null, + 1 + ); var sampleNums = []; // Will be 2D array of [iter][sampIds]; idx = 0; for (var i = 0; i < Ids.length; i++) { @@ -856,13 +953,13 @@ mgetSampleNums = async function (instance, Ids) { }; exports.mgetSampleNums = mgetSampleNums; -getSampleNum = async function (instance, sampId) { - var sampleNum = await mgetSampleNums(instance, [sampId]); +getSampleNum = async function (instance, sampId, yearDotMonth) { + var sampleNum = await mgetSampleNums(instance, [sampId], yearDotMonth); return sampleNums[0][0]; }; exports.getSampleNum = getSampleNum; -mgetSampleStatuses = async function (instance, Ids) { +mgetSampleStatuses = async function (instance, Ids, yearDotMonth) { var sampleIds = []; var idx = 0; for (var i = 0; i < Ids.length; i++) { @@ -872,7 +969,16 @@ mgetSampleStatuses = async function (instance, Ids) { } } - var data = await mSearch(instance, 'sample', ['sample.sample-uuid'], [sampleIds], 'sample.status', null, 1); + var data = await mSearch( + instance, + 'sample', + yearDotMonth, + ['sample.sample-uuid'], + [sampleIds], + 'sample.status', + null, + 1 + ); var sampleStatus = []; // Will be 2D array of [iter][sampIds]; idx = 0; for (var i = 0; i < Ids.length; i++) { @@ -894,7 +1000,7 @@ getSampleStatus = async function (instance, sampId) { }; exports.getSampleStatus = getSampleStatus; -mgetPrimaryPeriodId = async function (instance, sampIds, periNames) { +mgetPrimaryPeriodId = async function (instance, sampIds, periNames, yearDotMonth) { // needs 2D array iterSampleIds: [iter][samp] and 1D array iterPrimaryPeriodNames [iter] // returns 2D array [iter][samp] if (periNames.length == 1) { @@ -915,6 +1021,7 @@ mgetPrimaryPeriodId = async function (instance, sampIds, periNames) { var data = await mSearch( instance, 'period', + yearDotMonth, ['sample.sample-uuid', 'period.name'], [sampleIds, perSamplePeriNames], 'period.period-uuid', @@ -945,7 +1052,7 @@ getPrimaryPeriodId = async function (instance, sampId, periName) { }; exports.getPrimaryPeriodId = getPrimaryPeriodId; -mgetPeriodRange = async function (instance, periodIds) { +mgetPeriodRange = async function (instance, periodIds, yearDotMonth) { // needs 2D array periodIds: [iter][peri] // returns 2D array [iter][samp] of { "begin": x, "end": y } @@ -958,7 +1065,7 @@ mgetPeriodRange = async function (instance, periodIds) { idx++; } } - var data = await mSearch(instance, 'period', ['period.period-uuid'], [Ids], 'period', null, 1); + var data = await mSearch(instance, 'period', yearDotMonth, ['period.period-uuid'], [Ids], 'period', null, 1); // mSearch returns a 2D array, in other words, a list of values (inner array) for each query (outer array) // In this case, the queries are 1 per sampleId/periodName (for all iterations ordered), and the list of values // happens to be exactly 1 value, the primaryPeriodId. @@ -981,16 +1088,17 @@ mgetPeriodRange = async function (instance, periodIds) { }; exports.mgetPeriodRange = mgetPeriodRange; -getPeriodRange = async function (instance, periId) { - var periodRanges = await mgetPeriodRange(instance, [[periId]]); +getPeriodRange = async function (instance, periId, yearDotMonth) { + var periodRanges = await mgetPeriodRange(instance, [[periId]], yearDotMonth); return periodRanges[0][0]; }; exports.getPeriodRange = getPeriodRange; -mgetMetricDescs = async function (instance, runIds) { +mgetMetricDescs = async function (instance, runIds, yearDotMonth) { return await mSearch( instance, 'metric_desc', + yearDotMonth, ['run.run-uuid'], [runIds], 'metric_desc.metric_desc-uuid', @@ -1005,8 +1113,17 @@ getMetricDescs = async function (instance, runId) { }; exports.getMetricDescs = getMetricDescs; -mgetMetricDataDocs = async function (instance, metricIds) { - return await mSearch(instance, 'metric_data', ['metric_desc.metric_desc-uuid'], [metricIds], '', null, bigQuerySize); +mgetMetricDataDocs = async function (instance, metricIds, yearDotMonth) { + return await mSearch( + instance, + 'metric_data', + yearDotMonth, + ['metric_desc.metric_desc-uuid'], + [metricIds], + '', + null, + bigQuerySize + ); }; getMetricDataDocs = async function (instance, metricId) { @@ -1015,10 +1132,11 @@ getMetricDataDocs = async function (instance, metricId) { }; exports.getMetricDataDocs = getMetricDataDocs; -mgetMetricTypes = async function (instance, runIds, metricSources) { +mgetMetricTypes = async function (instance, runIds, metricSources, yearDotMonth) { return await mSearch( instance, 'metric_desc', + yearDotMonth, ['run.run-uuid', 'metric_desc.source'], [runIds, metricSources], null, @@ -1034,44 +1152,61 @@ getMetricTypes = async function (instance, runId, metricSource) { }; exports.getMetricTypes = getMetricTypes; -mgetIterations = async function (instance, runIds) { - return await mSearch(instance, 'iteration', ['run.run-uuid'], [runIds], 'iteration.iteration-uuid', null, 1000, [ - { 'iteration.num': { order: 'asc', numeric_type: 'long' } } - ]); +mgetIterations = async function (instance, runIds, yearDotMonth) { + return await mSearch( + instance, + 'iteration', + yearDotMonth, + ['run.run-uuid'], + [runIds], + 'iteration.iteration-uuid', + null, + 1000, + [{ 'iteration.num': { order: 'asc', numeric_type: 'long' } }] + ); }; -getIterations = async function (instance, runId) { - var iterations = await mgetIterations(instance, [runId]); +getIterations = async function (instance, runId, yearDotMonth) { + var iterations = await mgetIterations(instance, [runId], yearDotMonth); return iterations[0]; }; exports.getIterations = getIterations; -mgetTags = async function (instance, runIds) { - return await mSearch(instance, 'tag', ['run.run-uuid'], [runIds], 'tag', null, 1000); +mgetTags = async function (instance, runIds, yearDotMonth) { + return await mSearch(instance, 'tag', yearDotMonth, ['run.run-uuid'], [runIds], 'tag', null, 1000); }; -getTags = async function (instance, runId) { - var tags = await mgetTags(instance, [runId]); +getTags = async function (instance, runId, yearDotMonth) { + var tags = await mgetTags(instance, [runId], yearDotMonth); return tags[0]; }; exports.getTags = getTags; -mgetRunFromIter = async function (instance, iterIds) { - return await mSearch(instance, 'iteration', ['iteration.iteration-uuid'], [iterIds], 'run.run-uuid', null, 1000); +mgetRunFromIter = async function (instance, iterIds, yearDotMonth) { + return await mSearch( + instance, + 'iteration', + yearDotMonth, + ['iteration.iteration-uuid'], + [iterIds], + 'run.run-uuid', + null, + 1000 + ); }; -getRunFromIter = async function (instance, iterId) { - var runsFromIter = await mgetRunFromIter(instance, [iterId]); +getRunFromIter = async function (instance, iterId, yearDotMonth) { + var runsFromIter = await mgetRunFromIter(instance, [iterId], yearDotMonth); return runsFromIter[0][0]; }; exports.getRunFromIter = getRunFromIter; -mgetRunFromPeriod = async function (instance, periIds) { - return await mSearch(instance, 'period', ['period.period-uuid'], [periIds], 'run.run-uuid', null, 1); +mgetRunFromPeriod = async function (instance, periIds, yearDotMonth) { + return await mSearch(instance, 'period', yearDotMonth, ['period.period-uuid'], [periIds], 'run.run-uuid', null, 1); }; -getRunFromPeriod = async function (instance, periId) { - var runFromPeriods = await mgetRunFromPeriod(instance, [periId]); +getRunFromPeriod = async function (instance, periId, yearDotMonth) { + var runFromPeriods = await mgetRunFromPeriod(instance, [periId], yearDotMonth); return runFromPeriods[0][0]; }; exports.getRunFromPeriod = getRunFromPeriod; @@ -1093,11 +1228,11 @@ getInstancesInfo = function (instances) { } if (typeof resp != 'undefined') { var indices = JSON.parse(resp.getBody()); - debuglog('indices:\n' + JSON.stringify(indices, null, 2)); instances[inst_idx]['indices'] = {}; for (const index of indices) { var name = index['index']; if (/^cdm/.exec(name)) { + debuglog('index:\n' + JSON.stringify(index, null, 2)); const match = name.match(/^cdm[-]{0,1}(v[\d+]dev)/); const cdmver = match[1]; if (!Object.keys(instances[inst_idx]['indices']).includes(cdmver)) { @@ -1109,7 +1244,7 @@ getInstancesInfo = function (instances) { } if (Object.keys(instances[inst_idx]).includes('ver')) { // User has already requested a specific cdm version - return; + continue; } if (Object.keys(instances[inst_idx]['indices']).length != 0) { // If mulitple versions of indices exist, default to the latest version @@ -1119,12 +1254,10 @@ getInstancesInfo = function (instances) { // default to the newer cdm version. var cdmvers = Object.keys(instances[inst_idx]['indices']).sort(); instances[inst_idx]['ver'] = cdmvers[cdmvers.length - 1]; - return; + continue; } // There are no cdm indices at all, so we have to pick a default cdm version. - // Currently this is v8dev until this code is well tested for v8dev, then - // it will move to 9dev - instances[inst_idx]['ver'] = 'v8dev'; + instances[inst_idx]['ver'] = 'v9dev'; } }; @@ -1156,6 +1289,24 @@ invalidInstance = function (instance) { return false; }; +findYearDotMonthFromRun = async function (instance, runId) { + // Versions older than v9dev don't have a year and month + if (instance['ver'] == 'v7dev' || instance['ver'] == 'v8dev') { + return ''; + } + + var q = { query: { bool: { filter: [{ term: { 'run.run-uuid': runId } }] } } }; + var resp = esRequest(instance, 'run', '/_search', q, '@*'); + var data = JSON.parse(resp.getBody()); + var index = data['hits']['hits'][0]['_index']; + var regExp = /(@\d\d\d\d\.\d\d)$/; + var matches = regExp.exec(index); + if (matches) { + return matches[1]; + } +}; +exports.findYearDotMonthFromRun = findYearDotMonthFromRun; + findInstanceFromRun = async function (instances, runId) { var foundInstance; for (const instance of instances) { @@ -1165,7 +1316,7 @@ findInstanceFromRun = async function (instances, runId) { } // Use any function which searches by run id and always returns something if the run id is present debuglog('findInstanceFromRun(): about to call getIterations()'); - var result = await getIterations(instance, runId); + var result = await getIterations(instance, runId, '@*'); debuglog('findInstanceFromRun(): returned from calling getIterations()'); if (typeof result != 'undefined') { debuglog('found valid instance: ' + JSON.stringify(instance, null, 2)); @@ -1176,6 +1327,7 @@ findInstanceFromRun = async function (instances, runId) { debuglog('findInstanceFromRun(): about to return'); return foundInstance; }; +exports.findInstanceFromRun = findInstanceFromRun; findInstanceFromPeriod = async function (instances, periId) { var foundInstance; @@ -1184,7 +1336,7 @@ findInstanceFromPeriod = async function (instances, periId) { continue; } //var result = await mgetRunFromPeriod(instance, [periId])[0][0]; - var result = await getRunFromPeriod(instance, [periId]); + var result = await getRunFromPeriod(instance, [periId], '@*'); if (typeof result != 'undefined') { foundInstance = instance; break; @@ -1192,53 +1344,63 @@ findInstanceFromPeriod = async function (instances, periId) { } return foundInstance; }; +exports.findInstanceFromPeriod = findInstanceFromPeriod; -mgetParams = async function (instance, iterIds) { - return await mSearch(instance, 'param', ['iteration.iteration-uuid'], [iterIds], 'param', null, 1000); +mgetParams = async function (instance, iterIds, yearDotMonth) { + return await mSearch(instance, 'param', yearDotMonth, ['iteration.iteration-uuid'], [iterIds], 'param', null, 1000); }; exports.mgetParams = mgetParams; -getParams = async function (instance, iterId) { - return await mgetParams(instance, [iterId])[0]; +getParams = async function (instance, iterId, yearDotMonth) { + return await mgetParams(instance, [iterId], yearDotMonth)[0]; }; exports.getParams = getParams; -mgetIterationDoc = async function (instance, iterIds) { - return await mSearch(instance, 'iteration', ['iteration.iteration-uuid'], [iterIds], '', null, 1000); +mgetIterationDoc = async function (instance, iterIds, yearDotMonth) { + return await mSearch(instance, 'iteration', yearDotMonth, ['iteration.iteration-uuid'], [iterIds], '', null, 1000); }; -getIterationDoc = async function (instance, iterId) { - var iterationDocs = await mgetIterationDoc(instance, [iterId]); +getIterationDoc = async function (instance, iterId, yearDotMonth) { + var iterationDocs = await mgetIterationDoc(instance, [iterId], yearDotMonth); return iterationDocs[0][0]; }; exports.getIterationDoc = getIterationDoc; -mgetBenchmarkNameFromIter = async function (instance, Ids) { - return await mSearch(instance, 'iteration', ['iteration.iteration-uuid'], [Ids], 'run.benchmark', null, 1); +mgetBenchmarkNameFromIter = async function (instance, Ids, yearDotMonth) { + return await mSearch( + instance, + 'iteration', + yearDotMonth, + ['iteration.iteration-uuid'], + [Ids], + 'run.benchmark', + null, + 1 + ); }; -getBenchmarkNameFromIter = async function (instance, Id) { - var benchmarkNameforIters = await mgetBenchmarkNameFromIter(instance, [Id]); +getBenchmarkNameFromIter = async function (instance, Id, yearDotMonth) { + var benchmarkNameforIters = await mgetBenchmarkNameFromIter(instance, [Id], yearDotMonth); return benchmarkNameFromIters[0][0]; }; exports.getBenchmarkNameFromIter = getBenchmarkNameFromIter; -mgetBenchmarkName = async function (instance, runIds) { - return await mSearch(instance, 'run', ['run.run-uuid'], [runIds], 'run.benchmark', null, 1); +mgetBenchmarkName = async function (instance, runIds, yearDotMonth) { + return await mSearch(instance, 'run', yearDotMonth, ['run.run-uuid'], [runIds], 'run.benchmark', null, 1); }; -getBenchmarkName = async function (instance, runId) { - var benchmarkNames = await mgetBenchmarkName(instance, [runId]); +getBenchmarkName = async function (instance, runId, yearDotMonth) { + var benchmarkNames = await mgetBenchmarkName(instance, [runId], yearDotMonth); return benchmarkNames[0][0]; }; exports.getBenchmarkName = getBenchmarkName; -mgetRunData = async function (instance, runIds) { - return await mSearch(instance, 'run', ['run.run-uuid'], [runIds], '', null, 1000); +mgetRunData = async function (instance, runIds, yearDotMonth) { + return await mSearch(instance, 'run', yearDotMonth, ['run.run-uuid'], [runIds], '', null, 1000); }; -getRunData = async function (instance, runId) { - var runData = await mgetRunData(instance, [runId]); +getRunData = async function (instance, runId, yearDotMonth) { + var runData = await mgetRunData(instance, [runId], yearDotMonth); return runData[0]; }; exports.getRunData = getRunData; @@ -1364,10 +1526,10 @@ getIterMetrics = async function (instance, iterId) { return await mgetIterMetrics(instance, [iterId]); }; -deleteDocs = function (instance, docTypes, q) { +deleteDocs = function (instance, docTypes, q, yearDotMonth) { docTypes.forEach((docType) => { debuglog('deleteDocs() q: ' + JSON.stringify(q, null, 2)); - var resp = esRequest(instance, docType, '/_delete_by_query?wait_for_completion=false', q); + var resp = esRequest(instance, docType, '/_delete_by_query?wait_for_completion=false', q, yearDotMonth); var responses = JSON.parse(resp.getBody()); debuglog(JSON.stringify(responses, null, 2)); }); @@ -2122,7 +2284,7 @@ getIters = async function ( }; exports.getIters = getIters; -exports.getMetricSources = function (instance, runId) { +exports.getMetricSources = function (instance, runId, yearDotMonth) { var q = { query: { bool: { filter: [{ term: { 'run.run-uuid': runId } }] } }, aggs: { @@ -2130,7 +2292,7 @@ exports.getMetricSources = function (instance, runId) { }, size: 0 }; - var resp = esRequest(instance, 'metric_desc', '/_search', q); + var resp = esRequest(instance, 'metric_desc', '/_search', q, yearDotMonth); var data = JSON.parse(resp.getBody()); if (Array.isArray(data.aggregations.source.buckets)) { var sources = []; @@ -2141,9 +2303,9 @@ exports.getMetricSources = function (instance, runId) { } }; -getDocCount = function (instance, runId, docType) { +getDocCount = function (instance, runId, docType, yearDotMonth) { var q = { query: { bool: { filter: [{ term: { 'run.run-uuid': runId } }] } } }; - var resp = esRequest(instance, docType, '/_count', q); + var resp = esRequest(instance, docType, '/_count', q, yearDotMonth); var data = JSON.parse(resp.getBody()); return data.count; }; @@ -2247,7 +2409,7 @@ getMetricGroupTermsByLabel = function (metricGroupTerms) { return metricGroupTermsByLabel; }; -mgetMetricIdsFromTerms = async function (instance, termsSets) { +mgetMetricIdsFromTerms = async function (instance, termsSets, yearDotMonth) { // termsSets is an array of: // { 'period': x, 'run': y, 'termsByLabel': {} } // termsByLabel is a dict/hash of: @@ -2281,7 +2443,7 @@ mgetMetricIdsFromTerms = async function (instance, termsSets) { totalReqs++; }); } - var responses = await esJsonArrRequest(instance, 'metric_desc', '/_msearch', jsonArr); + var responses = await esJsonArrRequest(instance, 'metric_desc', '/_msearch', jsonArr, yearDotMonth); if (totalReqs != responses.length) { console.log( 'mgetMetricIdsFromTerms(): ERROR, number of _msearch responses (' + @@ -2343,7 +2505,7 @@ exports.mgetMetricIdsFromTerms = mgetMetricIdsFromTerms; // wants to "break-out" the metric (by some metadatam like cpu-id, devtype, etc). // Find the number of groups needed based on the --breakout options, then find out // what metric IDs belong in each group. -getMetricGroupsFromBreakouts = async function (instance, sets) { +getMetricGroupsFromBreakouts = async function (instance, sets, yearDotMonth) { var metricGroupIdsByLabel = []; var indexjson = '{}\n'; var index = JSON.parse(indexjson); @@ -2382,7 +2544,7 @@ getMetricGroupsFromBreakouts = async function (instance, sets) { jsonArr.push(JSON.stringify(index)); jsonArr.push(JSON.stringify(q)); }); - var responses = await esJsonArrRequest(instance, 'metric_desc', '/_msearch', jsonArr); + var responses = await esJsonArrRequest(instance, 'metric_desc', '/_msearch', jsonArr, yearDotMonth); var metricGroupIdsByLabelSets = []; var metricGroupTermsSets = []; @@ -2401,7 +2563,7 @@ getMetricGroupsFromBreakouts = async function (instance, sets) { }; termsSets.push(thisLabelSet); } - metricGroupIdsByLabelSets = await mgetMetricIdsFromTerms(instance, termsSets); + metricGroupIdsByLabelSets = await mgetMetricIdsFromTerms(instance, termsSets, yearDotMonth); return metricGroupIdsByLabelSets; }; exports.getMetricGroupsFromBreakouts = getMetricGroupsFromBreakouts; @@ -2419,6 +2581,358 @@ getMetricGroupsFromBreakout = async function (instance, runId, periId, source, t }; exports.getMetricGroupsFromBreakout = getMetricGroupsFromBreakout; +sendMetricReq = async function ( + jsonArr, + jsonArrTracker, + jsonArrIdx, + responses, + valueSets, + set, + label, + lastPass, + instance, + begin, + end, + resolution, + metricIds, + yearDotMonth +) { + debuglog('sendMetricReq begin'); + debuglog( + 'sendMetricReq, jsonArr MB [' + + numMBytes(jsonArr) + + '] responses MB [' + + numMBytes(responses) + + '] jsonArrIdx: [' + + jsonArrIdx + + ']' + ); + + // When jsonArr goes over this, submit the requests we have so far, so we can + // get responses and delete these reqs from jsonArr and process and delete the + // matching responses. + var chunkMBytes = 32; + + // Create a query for each data-point in a line graph. Resolution = number of data-points + // These vars are used for defining the requests and are altered in each loop cycle below (while) + // These are not used for processing responses, as response processing is triggered on demand, + // and these vars won't have the correct info. + // + // The resolution determines how many times we compute a value, each value for a + // different "slice" in the original begin-to-end time domain. + var duration = Math.floor((end - begin) / resolution); + var thisBegin = begin; + var thisEnd = begin + duration; + + // To have the correct info for processing responses, it is stoted in this array, where the array index*2 + // corresponds to the index in the jsonArr request. + //var info = { 'label': label, 'set': set, 'begin': thisBegin, 'end': thisEnd, 'numMetricIds': metricIds.length }; + + while (true) { + // Calculating a single value representing an average for thisBegin - thisEnd + // relies on an [weighted average] aggregation, plus a few other queries. An + // alternative method would involve querying all documents for the orignal + // begin - end time range, then [locally] computing a weighted average per + // thisBegin - thisEnd slice. Each method has pros/cons depending on the + // resolution and the total number of metric_data documents. + // + // This first request is for the weighted average, but does not include the + // documents which are partially outside the time range we need. + indexjson = '{"index": "' + getIndexName('metric_data', instance, yearDotMonth) + '" }\n'; + reqjson = '{'; + reqjson += ' "size": 0,'; + reqjson += ' "query": {'; + reqjson += ' "bool": {'; + reqjson += ' "filter": ['; + reqjson += ' {"range": {"metric_data.end": { "lte": "' + thisEnd + '"}}},'; + reqjson += ' {"range": {"metric_data.begin": { "gte": "' + thisBegin + '"}}},'; + reqjson += ' {"terms": {"metric_desc.metric_desc-uuid": ' + JSON.stringify(metricIds) + '}}'; + reqjson += ' ]'; + reqjson += ' }'; + reqjson += ' },'; + reqjson += ' "aggs": {'; + reqjson += ' "metric_avg": {'; + reqjson += ' "weighted_avg": {'; + reqjson += ' "value": {'; + reqjson += ' "field": "metric_data.value"'; + reqjson += ' },'; + reqjson += ' "weight": {'; + reqjson += ' "field": "metric_data.duration"'; + reqjson += ' }'; + reqjson += ' }'; + reqjson += ' }'; + reqjson += ' }'; + reqjson += '}'; + var index = JSON.parse(indexjson); + var req = JSON.parse(reqjson); + jsonArr.push(JSON.stringify(index)); + jsonArr.push(JSON.stringify(req)); + jsonArrTracker.push({ label: label, set: set, begin: thisBegin, end: thisEnd, numMetricIds: metricIds.length }); + // This second request is for the total weight of the previous weighted average request. + // We need this because we are going to recompute the weighted average by adding + // a few more documents that are partially outside the time domain. + indexjson = '{"index": "' + getIndexName('metric_data', instance, yearDotMonth) + '" }\n'; + reqjson = '{'; + reqjson += ' "size": 0,'; + reqjson += ' "query": {'; + reqjson += ' "bool": {'; + reqjson += ' "filter": ['; + reqjson += ' {"range": {"metric_data.end": { "lte": "' + thisEnd + '"}}},'; + reqjson += ' {"range": {"metric_data.begin": { "gte": "' + thisBegin + '"}}},'; + reqjson += ' {"terms": {"metric_desc.metric_desc-uuid": ' + JSON.stringify(metricIds) + '}}'; + reqjson += ' ]'; + reqjson += ' }'; + reqjson += ' },'; + reqjson += ' "aggs": {'; + reqjson += ' "total_weight": {'; + reqjson += ' "sum": {"field": "metric_data.duration"}'; + reqjson += ' }'; + reqjson += ' }'; + reqjson += '}\n'; + index = JSON.parse(indexjson); + req = JSON.parse(reqjson); + jsonArr.push(JSON.stringify(index)); + jsonArr.push(JSON.stringify(req)); + jsonArrTracker.push({}); + // This third request is for documents that had its begin during or before the time range, but + // its end was after the time range. + // + // Due to some limitations in how many documents can be returned from a query, + // (in spite of using size:) + // the number of metricIds inlcuded in the search is limited to 10,000. If there + // are more than 10,000 metric IDs to query for, use more queries. + const chunkSize = 10000; + for (let i = 0; i < metricIds.length; i += chunkSize) { + indexjson = '{"index": "' + getIndexName('metric_data', instance, yearDotMonth) + '" }\n'; + reqjson = '{'; + reqjson += ' "size": ' + bigQuerySize + ','; + reqjson += ' "_source": ["metric_data.begin", "metric_data.end", "metric_data.value"],'; + reqjson += ' "query": {'; + reqjson += ' "bool": {'; + reqjson += ' "filter": ['; + reqjson += ' {"range": {"metric_data.end": { "gt": "' + thisEnd + '"}}},'; + reqjson += ' {"range": {"metric_data.begin": { "lte": "' + thisEnd + '"}}},'; + reqjson += + ' {"terms": {"metric_desc.metric_desc-uuid": ' + + JSON.stringify(metricIds.slice(i, i + chunkSize)) + + '}}\n'; + reqjson += ' ]'; + reqjson += ' }'; + reqjson += ' }'; + reqjson += '}'; + index = JSON.parse(indexjson); + req = JSON.parse(reqjson); + jsonArr.push(JSON.stringify(index)); + jsonArr.push(JSON.stringify(req)); + jsonArrTracker.push({}); + // This fourth request is for documents that had its begin before the time range, but + // its end was during or after the time range + var indexjson = '{"index": "' + getIndexName('metric_data', instance, yearDotMonth) + '" }\n'; + var reqjson = ''; + reqjson += '{'; + reqjson += ' "size": ' + bigQuerySize + ','; + reqjson += ' "_source": ["metric_data.begin", "metric_data.end", "metric_data.value"],'; + reqjson += ' "query": {'; + reqjson += ' "bool": {'; + reqjson += ' "filter": ['; + reqjson += ' {"range": {"metric_data.end": { "gte": ' + thisBegin + '}}},'; + reqjson += ' {"range": {"metric_data.begin": { "lt": ' + thisBegin + '}}},'; + reqjson += + ' {"terms": {"metric_desc.metric_desc-uuid": ' + + JSON.stringify(metricIds.slice(i, i + chunkSize)) + + '}}\n'; + reqjson += ' ]'; + reqjson += ' }'; + reqjson += ' }'; + reqjson += '}\n'; + index = JSON.parse(indexjson); + req = JSON.parse(reqjson); + jsonArr.push(JSON.stringify(index)); + jsonArr.push(JSON.stringify(req)); + jsonArrTracker.push({}); + } + + debuglog('jsonArrTracker.length: ' + jsonArrTracker.length); + debuglog('jsonArrTracker right before begin and end are updated: ' + JSON.stringify(jsonArrTracker, null, 2)); + + // Cycle through every "slice" of the time domain, adding the requests for the entire time domain + thisBegin = thisEnd + 1; + thisEnd += duration + 1; + if (thisEnd > end) { + thisEnd = end; + } + + // We can't let the jsonArr or the responses arrays to grow too big, so + // we have to work them down as we submit requests. + // Two things can trigger submitting the request: + // 1) The jsonArr has grown too large + // 2) The jsonArr may does not exceed the size threshold, but + // This is the final call to sendMetricReq() and on the final data-point, + // so this is the last opportunity to submit the request. + if (numMBytes(jsonArr) > chunkMBytes || (thisBegin > thisEnd && lastPass)) { + debuglog('sendMetricReq jsonArr size MB: ' + numMBytes(jsonArr)); + debuglog('sendMetricReq responses size MB: ' + numMBytes(responses)); + const theseResponses = await esJsonArrRequest(instance, 'metric_data', '/_msearch', jsonArr, yearDotMonth); + responses.push(...theseResponses); + jsonArr.length = 0; // No longer needed since we have the response; Deleting to save memory. + + // Now that there are some responses available, we process those so we can also delete the data + // in the reeponses array. The elements in the responses array can get very big relative + // to the data calculated and stored in the valueSets. + // Note: the number of elements on responses should be exaclty half the number of + // elements in jsonArr, because each request in the jsonArr uses one entry for the index + // and another entry for the query, which generates a single entry in the responses array. + debuglog('sendMetricReq jsonArrTracker.length:' + jsonArrTracker.length); + debuglog('sendMetricReq jsonArrTracker:' + JSON.stringify(jsonArrTracker, null, 2)); + debuglog('jsonArrIdx:' + jsonArrIdx); + while (jsonArrIdx < responses.length * 2) { + const setIdx = jsonArrTracker[jsonArrIdx / 2]['set']; + const label = jsonArrTracker[jsonArrIdx / 2]['label']; + debuglog('sendMetricReq setIdx: [' + setIdx + '] label: [' + label + ']'); + if (typeof valueSets[setIdx] == 'undefined') { + valueSets[setIdx] = {}; + } + if (typeof valueSets[setIdx][label] == 'undefined') { + valueSets[setIdx][label] = []; + } + jsonArrIdx = calcAvg( + jsonArrTracker[jsonArrIdx / 2]['begin'], + jsonArrTracker[jsonArrIdx / 2]['end'], + responses, + jsonArrIdx, + jsonArrTracker, + jsonArrTracker[jsonArrIdx / 2]['numMetricIds'], + valueSets[setIdx][label] + ); + } + } + + if (thisBegin > thisEnd) { + // why not thisBegin > end ? + break; + } + } + + debuglog('sendMetricReq end, jsonArr MB [' + numMBytes(jsonArr) + '] responses MB [' + numMBytes(responses) + ']'); + debuglog('sendMetricReq end, lastPass: ' + lastPass); +}; + +calcAvg = function (thisBegin, thisEnd, responses, jsonArrIdx, jsonArrTracker, numMetricIds, values) { + debuglog('calcAvg start'); + debuglog( + 'calcAvg jsonArrIdx: [' + + jsonArrIdx + + '] thisBegin: [' + + thisBegin + + '] thisEnd: [' + + thisEnd + + '] numMetricIds: [' + + numMetricIds + + ']' + ); + + var timeWindowDuration = thisEnd - thisBegin + 1; + var totalWeightTimesMetrics = timeWindowDuration * numMetricIds; + var aggAvg; + var aggWeight; + var aggAvgTimesWeight; + var newWeight; + debuglog('calcAvg responses[' + jsonArrIdx / 2 + ']:' + JSON.stringify(responses[jsonArrIdx / 2], null, 2)); + aggAvg = responses[jsonArrIdx / 2].aggregations.metric_avg.value; + if (typeof aggAvg != 'undefined') { + // We have the weighted average for documents that don't overlap the time range, + // but we need to combine that with the documents that are partially outside + // the time range. We need to know the total weight from the documents we + // just finished in order to add the new documents and recompute the new weighted + // average. + aggWeight = responses[jsonArrIdx / 2 + 1].aggregations.total_weight.value; + aggAvgTimesWeight = aggAvg * aggWeight; + } else { + // It is possible that the aggregation returned no results because all of the documents + // were partially outside the time domain. This can happen when + // 1) A metric does not change during the entire test, and therefore only 1 document + // is created with a huge duration with begin before the time range and after after the + // time range. + // 2) The time domain we have is really small because the resolution we are using is + // very big. + // + // In eithr case, we have to set the average and total_weight to 0, and then the + // recompuation of the weighted average [with the last two requests in this set, finding + // all of th docs that are partially in the time domain] will work. + aggAvg = 0; + aggWeight = 0; + aggAvgTimesWeight = 0; + } + + // Process the remaining responses in the 'set'. These are typically 2 or more documents. + // Since these docs have a time range partially outside the time range we want, + // we have to get a new, reduced duration and use that to agment our weighted average. + var sumValueTimesWeight = 0; + var sumWeight = 0; + // It is possible to have the same document returned from these remaining queries. + // This can happen when the document's begin is before $this_begin *and* the document's end + // if after $this_end. + // You must not process the document twice. Perform a consolidation by organizing by the + // returned document's '_id' + var partialDocs = {}; + var k; + delete responses[jsonArrIdx / 2]; + delete responses[jsonArrIdx / 2 + 1]; + delete jsonArrTracker[jsonArrIdx / 2]; + delete jsonArrTracker[jsonArrIdx / 2 + 1]; + jsonArrIdx += 4; //advance to the non-aggreation responses + // There can be 1 to many multiples of 2 of these types of responses here. + // We know these type of responses have ended when the next response does + // have an aggregation in it. + while (jsonArrIdx / 2 < responses.length && !Object.keys(responses[jsonArrIdx / 2]).includes('aggregations')) { + if (responses[jsonArrIdx / 2].hits.total.value !== responses[jsonArrIdx / 2].hits.hits.length) { + console.log( + 'WARNING! getMetricDataFromIdsSets() responses[' + + (jsonArrIdx / 2 + k) + + '].hits.total.value (' + + responses[jsonArrIdx / 2].hits.total.value + + ') and responses[' + + jsonArrIdx / 2 + + '].hits.hits.length (' + + responses[jsonArrIdx / 2].hits.hits.length + + ') are not equal, which means the retured data is probably incomplete' + ); + } + responses[jsonArrIdx / 2].hits.hits.forEach((element) => { + partialDocs[element._id] = {}; + Object.keys(element._source.metric_data).forEach((key) => { + partialDocs[element._id][key] = element._source.metric_data[key]; + }); + }); + delete responses[jsonArrIdx / 2]; + delete jsonArrTracker[jsonArrIdx / 2]; + jsonArrIdx += 2; + } + // Now we can process the partialDocs + Object.keys(partialDocs).forEach((id) => { + //var docDuration = partialDocs[id].duration; + var docDuration = partialDocs[id].end - partialDocs[id].begin; + if (partialDocs[id].begin < thisBegin) { + docDuration -= thisBegin - partialDocs[id].begin; + } + if (partialDocs[id].end > thisEnd) { + docDuration -= partialDocs[id].end - thisEnd; + } + var valueTimesWeight = partialDocs[id].value * docDuration; + sumValueTimesWeight += valueTimesWeight; + sumWeight += docDuration; + }); + var result = (aggAvgTimesWeight + sumValueTimesWeight) / totalWeightTimesMetrics; + result *= numMetricIds; + var dataSample = {}; + dataSample.begin = thisBegin; + dataSample.end = thisEnd; + dataSample.value = result; + values.push(dataSample); + + return jsonArrIdx; +}; + // From a set of metric_desc ID's, return 1 or more values depending on resolution. // For each metric ID, there should be exactly 1 metric_desc doc and at least 1 metric_data docs. // A metric_data doc has a 'value', a 'begin' timestamp, and and 'end' timestamp (also a @@ -2430,287 +2944,74 @@ exports.getMetricGroupsFromBreakout = getMetricGroupsFromBreakout; // function is called with begin=5 and end=1005, and there are 2 metric_data documents [having the same // metric_id in metricIds], and their respective (begin,end) are (0,500) and (501,2000), // then there are enough metric_data documents to compute the results. -getMetricDataFromIdsSets = async function (instance, sets, metricGroupIdsByLabelSets) { - var jsonArr = []; - for (var idx = 0; idx < metricGroupIdsByLabelSets.length; idx++) { - Object.keys(metricGroupIdsByLabelSets[idx]) - .sort() - .forEach(function (label) { - var metricIds = metricGroupIdsByLabelSets[idx][label]; - if (typeof sets[idx].begin == 'undefined') { - console.log('ERROR: sets.[' + idx + '].begin is not defined:\n' + JSON.stringify(sets[idx]), null, 2); - process.exit(1); - } - var begin = Number(sets[idx].begin); - if (isNaN(begin)) { - console.log('ERROR: begin is not defined'); - process.exit(1); - } - if (typeof sets[idx].end == 'undefined') { - console.log('ERROR: sets.[' + idx + '].end is not defined'); - process.exit(1); - } - var end = Number(sets[idx].end); - var resolution = Number(sets[idx].resolution); - var duration = Math.floor((end - begin) / resolution); - var thisBegin = begin; - var thisEnd = begin + duration; - // The resolution determines how many times we compute a value, each value for a - // different "slice" in the original begin-to-end time domain. - while (true) { - // Calculating a single value representing an average for thisBegin - thisEnd - // relies on an [weighted average] aggregation, plus a few other queries. An - // alternative method would involve querying all documents for the orignal - // begin - end time range, then [locally] computing a weighted average per - // thisBegin - thisEnd slice. Each method has pros/cons depending on the - // resolution and the total number of metric_data documents. - // - // This first request is for the weighted average, but does not include the - // documents which are partially outside the time range we need. - indexjson = '{"index": "' + getIndexName('metric_data', instance) + '" }\n'; - reqjson = '{'; - reqjson += ' "size": 0,'; - reqjson += ' "query": {'; - reqjson += ' "bool": {'; - reqjson += ' "filter": ['; - reqjson += ' {"range": {"metric_data.end": { "lte": "' + thisEnd + '"}}},'; - reqjson += ' {"range": {"metric_data.begin": { "gte": "' + thisBegin + '"}}},'; - reqjson += ' {"terms": {"metric_desc.metric_desc-uuid": ' + JSON.stringify(metricIds) + '}}'; - reqjson += ' ]'; - reqjson += ' }'; - reqjson += ' },'; - reqjson += ' "aggs": {'; - reqjson += ' "metric_avg": {'; - reqjson += ' "weighted_avg": {'; - reqjson += ' "value": {'; - reqjson += ' "field": "metric_data.value"'; - reqjson += ' },'; - reqjson += ' "weight": {'; - reqjson += ' "field": "metric_data.duration"'; - reqjson += ' }'; - reqjson += ' }'; - reqjson += ' }'; - reqjson += ' }'; - reqjson += '}'; - var index = JSON.parse(indexjson); - var req = JSON.parse(reqjson); - jsonArr.push(JSON.stringify(index)); - jsonArr.push(JSON.stringify(req)); - // This second request is for the total weight of the previous weighted average request. - // We need this because we are going to recompute the weighted average by adding - // a few more documents that are partially outside the time domain. - indexjson = '{"index": "' + getIndexName('metric_data', instance) + '" }\n'; - reqjson = '{'; - reqjson += ' "size": 0,'; - reqjson += ' "query": {'; - reqjson += ' "bool": {'; - reqjson += ' "filter": ['; - reqjson += ' {"range": {"metric_data.end": { "lte": "' + thisEnd + '"}}},'; - reqjson += ' {"range": {"metric_data.begin": { "gte": "' + thisBegin + '"}}},'; - reqjson += ' {"terms": {"metric_desc.metric_desc-uuid": ' + JSON.stringify(metricIds) + '}}'; - reqjson += ' ]'; - reqjson += ' }'; - reqjson += ' },'; - reqjson += ' "aggs": {'; - reqjson += ' "total_weight": {'; - reqjson += ' "sum": {"field": "metric_data.duration"}'; - reqjson += ' }'; - reqjson += ' }'; - reqjson += '}\n'; - index = JSON.parse(indexjson); - req = JSON.parse(reqjson); - jsonArr.push(JSON.stringify(index)); - jsonArr.push(JSON.stringify(req)); - // This third request is for documents that had its begin during or before the time range, but - // its end was after the time range. - indexjson = '{"index": "' + getIndexName('metric_data', instance) + '" }\n'; - reqjson = '{'; - reqjson += ' "size": ' + bigQuerySize + ','; - reqjson += ' "query": {'; - reqjson += ' "bool": {'; - reqjson += ' "filter": ['; - reqjson += ' {"range": {"metric_data.end": { "gt": "' + thisEnd + '"}}},'; - reqjson += ' {"range": {"metric_data.begin": { "lte": "' + thisEnd + '"}}},'; - reqjson += ' {"terms": {"metric_desc.metric_desc-uuid": ' + JSON.stringify(metricIds) + '}}\n'; - reqjson += ' ]'; - reqjson += ' }'; - reqjson += ' }'; - reqjson += '}'; - index = JSON.parse(indexjson); - req = JSON.parse(reqjson); - jsonArr.push(JSON.stringify(index)); - jsonArr.push(JSON.stringify(req)); - // This fourth request is for documents that had its begin before the time range, but - // its end was during or after the time range - var indexjson = '{"index": "' + getIndexName('metric_data', instance) + '" }\n'; - var reqjson = ''; - reqjson += '{'; - reqjson += ' "size": ' + bigQuerySize + ','; - reqjson += ' "query": {'; - reqjson += ' "bool": {'; - reqjson += ' "filter": ['; - reqjson += ' {"range": {"metric_data.end": { "gte": ' + thisBegin + '}}},'; - reqjson += ' {"range": {"metric_data.begin": { "lt": ' + thisBegin + '}}},'; - reqjson += ' {"terms": {"metric_desc.metric_desc-uuid": ' + JSON.stringify(metricIds) + '}}\n'; - reqjson += ' ]'; - reqjson += ' }'; - reqjson += ' }'; - reqjson += '}\n'; - index = JSON.parse(indexjson); - req = JSON.parse(reqjson); - jsonArr.push(JSON.stringify(index)); - jsonArr.push(JSON.stringify(req)); - - // Cycle through every "slice" of the time domain, adding the requests for the entire time domain - thisBegin = thisEnd + 1; - thisEnd += duration + 1; - if (thisEnd > end) { - thisEnd = end; - } - if (thisBegin > thisEnd) { - break; - } - } - }); - } - - var responses = await esJsonArrRequest(instance, 'metric_data', '/_msearch', jsonArr); - var elements = responses.length; - +getMetricDataFromIdsSets = async function (instance, sets, metricGroupIdsByLabelSets, yearDotMonth) { + var jsonArr = []; // What is used to submit metric query requests in bulk + var jsonArrTracker = []; // Detailed iInfo (set, label, begin, end) about each element in jsonArr + var jsonArrIdx = 0; // Index of next element in jsonArr that needs its response processed + var responses = []; // Ordered responses for jsonArr var valueSets = []; + var reqSize = 0; var count = 0; for (var idx = 0; idx < metricGroupIdsByLabelSets.length; idx++) { - thisSetElements = elements / metricGroupIdsByLabelSets.length; - var valuesByLabel = {}; - Object.keys(metricGroupIdsByLabelSets[idx]) - .sort() - .forEach(function (label) { - valuesByLabel[label] = []; - thisLabelElements = metricGroupIdsByLabelSets[idx][label].length; - var metricIds = metricGroupIdsByLabelSets[idx][label]; - var values = []; - var begin = Number(sets[idx].begin); - var end = Number(sets[idx].end); - var resolution = Number(sets[idx].resolution); - var duration = Math.floor((end - begin) / resolution); - var thisBegin = begin; - var thisEnd = begin + duration; - var subCount = 0; - //var elements = data.responses.length / metricGroupIdsByLabelSets.length; - var numMetricIds = metricIds.length; - while (true) { - var timeWindowDuration = thisEnd - thisBegin + 1; - var totalWeightTimesMetrics = timeWindowDuration * numMetricIds; - subCount++; - var aggAvg; - var aggWeight; - var aggAvgTimesWeight; - var newWeight; - aggAvg = responses[count].aggregations.metric_avg.value; //$$resp_ref{'responses'}[$count]{'aggregations'}{'metric_avg'}{'value'}; - if (typeof aggAvg != 'undefined') { - // We have the weighted average for documents that don't overlap the time range, - // but we need to combine that with the documents that are partially outside - // the time range. We need to know the total weight from the documents we - // just finished in order to add the new documents and recompute the new weighted - // average. - aggWeight = responses[count + 1].aggregations.total_weight.value; - aggAvgTimesWeight = aggAvg * aggWeight; - } else { - // It is possible that the aggregation returned no results because all of the documents - // were partially outside the time domain. This can happen when - // 1) A metric does not change during the entire test, and therefore only 1 document - // is created with a huge duration with begin before the time range and after after the - // time range. - // 2) The time domain we have is really small because the resolution we are using is - // very big. - // - // In eithr case, we have to set the average and total_weight to 0, and then the - // recompuation of the weighted average [with the last two requests in this set, finding - // all of th docs that are partially in the time domain] will work. - aggAvg = 0; - aggWeight = 0; - aggAvgTimesWeight = 0; - } - - // Process last 2 of the 4 responses in the 'set' - // Since these docs have a time range partially outside the time range we want, - // we have to get a new, reduced duration and use that to agment our weighted average. - var sumValueTimesWeight = 0; - var sumWeight = 0; - // It is possible to have the same document returned from the last two queries in this set of 4. - // This can happen when the document's begin is before $this_begin *and* the document's end - // if after $this_end. - // You must not process the document twice. Perform a consolidation by organizing by the - // returned document's '_id' - var partialDocs = {}; - var k; - for (k = 2; k < 4; k++) { - //for my $j (@{ $$resp_ref{'responses'}[$count + $k]{'hits'}{'hits'} }) { - if (responses[count + k].hits.total.value !== responses[count + k].hits.hits.length) { - console.log( - 'WARNING! getMetricDataFromIdsSets() responses[' + - (count + k) + - '].hits.total.value (' + - responses[count + k].hits.total.value + - ') and responses[' + - (count + k) + - '].hits.hits.length (' + - responses[count + k].hits.hits.length + - ') are not equal, which means the retured data is probably incomplete' - ); - } - responses[count + k].hits.hits.forEach((element) => { - //for my $key (keys %{ $$j{'_source'}{'metric_data'} }) { - partialDocs[element._id] = {}; - Object.keys(element._source.metric_data).forEach((key) => { - //partial_docs[{$$j{'_id'}}{$key} = $$j{'_source'}{'metric_data'}{$key}; - partialDocs[element._id][key] = element._source.metric_data[key]; - }); - }); - } - // Now we can process the partialDocs - Object.keys(partialDocs).forEach((id) => { - var docDuration = partialDocs[id].duration; - if (partialDocs[id].begin < thisBegin) { - docDuration -= thisBegin - partialDocs[id].begin; - } - if (partialDocs[id].end > thisEnd) { - docDuration -= partialDocs[id].end - thisEnd; - } - var valueTimesWeight = partialDocs[id].value * docDuration; - sumValueTimesWeight += valueTimesWeight; - sumWeight += docDuration; - }); - var result = (aggAvgTimesWeight + sumValueTimesWeight) / totalWeightTimesMetrics; - result *= numMetricIds; - //result = Number.parseFloat(result).toPrecision(4); - var dataSample = {}; - dataSample.begin = thisBegin; - dataSample.end = thisEnd; - dataSample.value = result; - values.push(dataSample); - - count += 4; // Bumps count to the next set of responses - - // Cycle through every "slice" of the time domain, adding the requests for the entire time domain - thisBegin = thisEnd + 1; - thisEnd += duration + 1; - if (thisEnd > end) { - thisEnd = end; - } - if (thisBegin > thisEnd) { - break; - } - } - valuesByLabel[label] = values; - }); - valueSets[idx] = valuesByLabel; + const sortedKeys = Object.keys(metricGroupIdsByLabelSets[idx]).sort(); + for (var k = 0; k < sortedKeys.length; k++) { + const label = sortedKeys[k]; + debuglog('label: [' + label + ']'); + var metricIds = metricGroupIdsByLabelSets[idx][label]; + if (typeof sets[idx].begin == 'undefined') { + console.log('ERROR: sets.[' + idx + '].begin is not defined:\n' + JSON.stringify(sets[idx]), null, 2); + process.exit(1); + } + var begin = Number(sets[idx].begin); + if (isNaN(begin)) { + console.log('ERROR: begin is not defined'); + process.exit(1); + } + if (typeof sets[idx].end == 'undefined') { + console.log('ERROR: sets.[' + idx + '].end is not defined'); + process.exit(1); + } + var end = Number(sets[idx].end); + var resolution = Number(sets[idx].resolution); + var duration = Math.floor((end - begin) / resolution); + + const lastPass = idx + 1 >= metricGroupIdsByLabelSets.length && k + 1 >= sortedKeys.length; + await sendMetricReq( + jsonArr, + jsonArrTracker, + jsonArrIdx, + responses, + valueSets, + idx, + label, + lastPass, + instance, + begin, + end, + resolution, + metricIds, + yearDotMonth + ); + } } return valueSets; }; + exports.getMetricDataFromIdsSets = getMetricDataFromIdsSets; -getMetricData = async function (instance, runId, periId, source, type, begin, end, resolution, breakout, filter) { +getMetricData = async function ( + instance, + runId, + periId, + source, + type, + begin, + end, + resolution, + breakout, + filter, + yearDotMonth +) { var sets = []; var thisSet = { run: runId, @@ -2724,7 +3025,7 @@ getMetricData = async function (instance, runId, periId, source, type, begin, en filter: filter }; sets.push(thisSet); - var dataSets = await getMetricDataSets(instance, sets); + var dataSets = await getMetricDataSets(instance, sets, yearDotMonth); return dataSets[0]; }; exports.getMetricData = getMetricData; @@ -2741,7 +3042,7 @@ exports.getMetricData = getMetricData; // from this [benchmark-iteration-sample-]period (from doc which contains the periId) // *if* the metric is from a benchmark. If you want to query for corresponding // tool data, use the same begin and end as the benchmark-iteration-sample-period. -getMetricDataSets = async function (instance, sets) { +getMetricDataSets = async function (instance, sets, yearDotMonth) { for (var i = 0; i < sets.length; i++) { // If a begin and end are not defined, get it from the period.begin & period.end. // If a begin and/or end are not defined, and the period is not defined, error out. @@ -2749,7 +3050,8 @@ getMetricDataSets = async function (instance, sets) { // If a run and period are not defined, error out. if (typeof sets[i].run == 'undefined') { if (typeof sets[i].period != 'undefined') { - sets[i].run = await getRunFromPeriod(instance, sets[i].period); + console.log('getRunFromPeriod'); + sets[i].run = await getRunFromPeriod(instance, sets[i].period, yearDotMonth); } else { console.log('ERROR: run and period was not defined'); } @@ -2757,7 +3059,7 @@ getMetricDataSets = async function (instance, sets) { var periodRange; if (typeof sets[i].begin == 'undefined') { if (typeof sets[i].period != 'undefined') { - periodRange = await getPeriodRange(instance, sets[i].period); + periodRange = await getPeriodRange(instance, sets[i].period, yearDotMonth); sets[i]['begin'] = periodRange['begin']; } else { console.log('ERROR: begin is not defined and a period was not defined'); @@ -2766,7 +3068,7 @@ getMetricDataSets = async function (instance, sets) { if (typeof sets[i].end == 'undefined') { if (typeof sets[i].period != 'undefined') { if (typeof periodRange == 'undefined') { - periodRange = await getPeriodRange(instance, sets[i].period); + periodRange = await getPeriodRange(instance, sets[i].period, yearDotMonth); } sets[i].end = periodRange.end; } else { @@ -2791,8 +3093,8 @@ getMetricDataSets = async function (instance, sets) { delete sets[i].period; } - var metricGroupIdsByLabelSets = await getMetricGroupsFromBreakouts(instance, sets); - var dataSets = await getMetricDataFromIdsSets(instance, sets, metricGroupIdsByLabelSets); + var metricGroupIdsByLabelSets = await getMetricGroupsFromBreakouts(instance, sets, yearDotMonth); + var dataSets = await getMetricDataFromIdsSets(instance, sets, metricGroupIdsByLabelSets, yearDotMonth); if (dataSets.length != sets.length) { console.log( @@ -2814,7 +3116,7 @@ getMetricDataSets = async function (instance, sets) { sources[i] = sets[i].source; types[i] = sets[i].type; } - var setBreakouts = await mgetMetricNames(instance, runIds, sources, types); + var setBreakouts = await mgetMetricNames(instance, runIds, sources, types, yearDotMonth); for (var i = 0; i < sets.length; i++) { // Rearrange the actual data into 'values' section @@ -2868,7 +3170,7 @@ getMetricDataSets = async function (instance, sets) { }; exports.getMetricDataSets = getMetricDataSets; -async function waitForDeletedDocs(instance, runId, docTypes) { +async function waitForDeletedDocs(instance, runId, docTypes, yearDotMonth) { var numAttempts = 1; var maxAttempts = 30; var remainingDocTypes = docTypes; @@ -2884,7 +3186,7 @@ async function waitForDeletedDocs(instance, runId, docTypes) { totalDocCount = 0; docWaitStr = ''; for (let i = 0; i < docTypes.length; i++) { - var thisNumDocs = getDocCount(instance, runId, docTypes[i]); + var thisNumDocs = getDocCount(instance, runId, docTypes[i], yearDotMonth); if (thisNumDocs > 0) { //console.log(' ' + docTypes[i] + ': doc count: ' + thisNumDocs); docWaitStr += ' ' + docTypes[i] + ': doc count: ' + thisNumDocs + '\n'; diff --git a/queries/cdmq/delete-run.js b/queries/cdmq/delete-run.js index bf5d5cd5..adcc6d98 100644 --- a/queries/cdmq/delete-run.js +++ b/queries/cdmq/delete-run.js @@ -59,7 +59,10 @@ async function main() { q = { query: { bool: { filter: [{ term: { 'run.run-uuid': program.run } }] } } }; // When deleting, you must use exactly one instance, so we use the last provided. // We don't want to search for an instance with this run, because we don't want - // to delete just any copy of this run. + // to delete just any copy of this run. + // + // In order to delete, we must first find out which year and date the run comes from + const yearDotMonth = cdm.findYearDotMonthFromRun(instances[instances.length - 1], program.run); cdm.deleteDocs(instances[instances.length - 1], allDocTypes, q); var numDocTypes = await cdm.waitForDeletedDocs(instances[instances.length - 1], program.run, allDocTypes); if (numDocTypes > 0) { diff --git a/queries/cdmq/get-instances-info.js b/queries/cdmq/get-instances-info.js index f367186a..b5c832fd 100755 --- a/queries/cdmq/get-instances-info.js +++ b/queries/cdmq/get-instances-info.js @@ -45,7 +45,7 @@ async function main() { save_host('localhost:9200'); } getInstancesInfo(instances); - console.log('instances:\n' + JSON.stringify(instances, null, 2)); + console.log(JSON.stringify(instances, null, 2)); } main(); diff --git a/queries/cdmq/get-metric-data.js b/queries/cdmq/get-metric-data.js index a5b57e22..b2f7c50c 100644 --- a/queries/cdmq/get-metric-data.js +++ b/queries/cdmq/get-metric-data.js @@ -110,12 +110,21 @@ async function main() { } getInstancesInfo(instances); - var instance = await findInstanceFromPeriod(instances, program.period); + var yearDotMonth; + var instance; + if (program.run != null) { + instance = await findInstanceFromRun(instances, program.run); + } else if (program.period != null) { + instance = await findInstanceFromPeriod(instances, program.period); + // We don't yet know the yearDotMonth, so use wildcard to query all period indices + program.run = await getRunFromPeriod(instance, program.period, '@*'); + } else { + process.exit(1); + } if (typeof instance == 'undefined') { - console.log('None of the instances provided were valid, so going to abort'); process.exit(1); } - + var yearDotMonth = await findYearDotMonthFromRun(instance, program.run); metric_data = await cdm.getMetricData( instance, program.run, @@ -126,7 +135,8 @@ async function main() { program.end, program.resolution, program.breakout, - program.filter + program.filter, + yearDotMonth ); if (Object.keys(metric_data.values).length == 0) { diff --git a/queries/cdmq/get-result-summary.js b/queries/cdmq/get-result-summary.js index 0377123e..6836de8a 100644 --- a/queries/cdmq/get-result-summary.js +++ b/queries/cdmq/get-result-summary.js @@ -97,7 +97,7 @@ async function main() { continue; } cdm.debuglog('main(): calling cdm.mSearch()'); - var instanceRunIds = await cdm.mSearch(instance, 'run', termKeys, values, 'run.run-uuid', null, 1000); + var instanceRunIds = await cdm.mSearch(instance, 'run', '@*', termKeys, values, 'run.run-uuid', null, 1000); cdm.debuglog('main(): returned from cdm.mSearch()'); cdm.debuglog('instanceRunIds:\n' + JSON.stringify(instanceRunIds, null, 2)); if (typeof instanceRunIds[0] != 'undefined') { @@ -121,11 +121,11 @@ async function main() { var thisRun = {}; const runId = runIds[runIdx]; var instance = await findInstanceFromRun(instances, runId); - cdm.debuglog('instance: ' + JSON.stringify(instance, null, 2)); + var yearDotMonth = await findYearDotMonthFromRun(instance, runId); logOutput('\nrun-id: ' + runId, program.outputFormat); thisRun['run-id'] = runId; thisRun['iterations'] = []; - var tags = await cdm.getTags(instance, runId); + var tags = await cdm.getTags(instance, runId, yearDotMonth); tags.sort((a, b) => (a.name < b.name ? -1 : 1)); thisRun['tags'] = tags; var tagList = ' tags: '; @@ -134,31 +134,36 @@ async function main() { tagList += tag.name + '=' + tag.val + ' '; }); logOutput(tagList, program.outputFormat); - var benchName = await cdm.getBenchmarkName(instance, runId); + var benchName = await cdm.getBenchmarkName(instance, runId, yearDotMonth); var benchmarks = list(benchName); logOutput(' benchmark: ' + benchName, program.outputFormat); - var benchIterations = await cdm.getIterations(instance, runId); + var benchIterations = await cdm.getIterations(instance, runId, yearDotMonth); if (benchIterations.length == 0) { cdm.debuglog('There were no iterations found, exiting'); process.exit(1); } - var iterParams = await cdm.mgetParams(instance, benchIterations); + var iterParams = await cdm.mgetParams(instance, benchIterations, yearDotMonth); //returns 1D array [iter] - var iterPrimaryPeriodNames = await cdm.mgetPrimaryPeriodName(instance, benchIterations); + var iterPrimaryPeriodNames = await cdm.mgetPrimaryPeriodName(instance, benchIterations, yearDotMonth); //input: 1D array //output: 2D array [iter][samp] - var iterSampleIds = await cdm.mgetSamples(instance, benchIterations); + var iterSampleIds = await cdm.mgetSamples(instance, benchIterations, yearDotMonth); //input: 2D array iterSampleIds: [iter][samp] //output: 2D array [iter][samp] - var iterSampleStatuses = await cdm.mgetSampleStatuses(instance, iterSampleIds); + var iterSampleStatuses = await cdm.mgetSampleStatuses(instance, iterSampleIds, yearDotMonth); //needs 2D array iterSampleIds: [iter][samp] and 1D array iterPrimaryPeriodNames [iter] //returns 2D array [iter][samp] - var iterPrimaryPeriodIds = await cdm.mgetPrimaryPeriodId(instance, iterSampleIds, iterPrimaryPeriodNames); - var iterPrimaryPeriodRanges = await cdm.mgetPeriodRange(instance, iterPrimaryPeriodIds); + var iterPrimaryPeriodIds = await cdm.mgetPrimaryPeriodId( + instance, + iterSampleIds, + iterPrimaryPeriodNames, + yearDotMonth + ); + var iterPrimaryPeriodRanges = await cdm.mgetPeriodRange(instance, iterPrimaryPeriodIds, yearDotMonth); // Find the params which are the same in every iteration - var iterPrimaryMetrics = await cdm.mgetPrimaryMetric(instance, benchIterations); + var iterPrimaryMetrics = await cdm.mgetPrimaryMetric(instance, benchIterations, yearDotMonth); var primaryMetrics = list(iterPrimaryMetrics[0]); // For now only dump params when 1 primary metric is used if (primaryMetrics.length == 1) { @@ -192,13 +197,13 @@ async function main() { } logOutput(' metrics:', program.outputFormat); - var metricSources = await cdm.getMetricSources(instance, runId); + var metricSources = await cdm.getMetricSources(instance, runId, yearDotMonth); var theseRunIds = []; thisRun['metrics'] = []; for (var i = 0; i < metricSources.length; i++) { theseRunIds[i] = runId; } - var metricTypes = await cdm.mgetMetricTypes(instance, theseRunIds, metricSources); + var metricTypes = await cdm.mgetMetricTypes(instance, theseRunIds, metricSources, yearDotMonth); for (var i = 0; i < metricSources.length; i++) { var thisMetricSourceName = metricSources[i]; @@ -234,7 +239,7 @@ async function main() { source = sourceType[0]; type = sourceType[1]; } else { - console.log('sourceType array is an unexpected length, ' + sourceType.length); + console.log('ERROR: sourceType array is an unexpected length, ' + sourceType.length); process.exit(1); } var set = { @@ -250,7 +255,7 @@ async function main() { sets.push(set); if (sets.length == batchedQuerySize) { // Submit a chunk of the query and save the result - metricDataSetsChunks[chunkNum] = await cdm.getMetricDataSets(instance, sets); + metricDataSetsChunks[chunkNum] = await cdm.getMetricDataSets(instance, sets, yearDotMonth); chunkNum++; sets = []; } @@ -259,7 +264,7 @@ async function main() { } if (sets.length > 0) { // Submit a chunk of the query and save the result - metricDataSetsChunks[chunkNum] = await cdm.getMetricDataSets(instance, sets); + metricDataSetsChunks[chunkNum] = await cdm.getMetricDataSets(instance, sets, yearDotMonth); chunkNum++; sets = []; } diff --git a/queries/cdmq/node_modules/.package-lock.json b/queries/cdmq/node_modules/.package-lock.json index 4cd6c063..f310cdc7 100644 --- a/queries/cdmq/node_modules/.package-lock.json +++ b/queries/cdmq/node_modules/.package-lock.json @@ -292,6 +292,14 @@ "dependencies": { "node-addon-api": "^1.6.1" } + }, + "node_modules/xz-decompress": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/xz-decompress/-/xz-decompress-0.2.2.tgz", + "integrity": "sha512-DSOnX+ZLVTrsW+CtjZPwjrMWvuRkzCcEpwLsY2faZyVgLH/ZHpTg3h3+KyN16mGuduMgO+/pc9rSEG735oGN0g==", + "engines": { + "node": ">=16" + } } } } diff --git a/queries/cdmq/node_modules/argparse/CHANGELOG.md b/queries/cdmq/node_modules/argparse/CHANGELOG.md new file mode 100644 index 00000000..dc39ed69 --- /dev/null +++ b/queries/cdmq/node_modules/argparse/CHANGELOG.md @@ -0,0 +1,216 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + + +## [2.0.1] - 2020-08-29 +### Fixed +- Fix issue with `process.argv` when used with interpreters (`coffee`, `ts-node`, etc.), #150. + + +## [2.0.0] - 2020-08-14 +### Changed +- Full rewrite. Now port from python 3.9.0 & more precise following. + See [doc](./doc) for difference and migration info. +- node.js 10+ required +- Removed most of local docs in favour of original ones. + + +## [1.0.10] - 2018-02-15 +### Fixed +- Use .concat instead of + for arrays, #122. + + +## [1.0.9] - 2016-09-29 +### Changed +- Rerelease after 1.0.8 - deps cleanup. + + +## [1.0.8] - 2016-09-29 +### Changed +- Maintenance (deps bump, fix node 6.5+ tests, coverage report). + + +## [1.0.7] - 2016-03-17 +### Changed +- Teach `addArgument` to accept string arg names. #97, @tomxtobin. + + +## [1.0.6] - 2016-02-06 +### Changed +- Maintenance: moved to eslint & updated CS. + + +## [1.0.5] - 2016-02-05 +### Changed +- Removed lodash dependency to significantly reduce install size. + Thanks to @mourner. + + +## [1.0.4] - 2016-01-17 +### Changed +- Maintenance: lodash update to 4.0.0. + + +## [1.0.3] - 2015-10-27 +### Fixed +- Fix parse `=` in args: `--examplepath="C:\myfolder\env=x64"`. #84, @CatWithApple. + + +## [1.0.2] - 2015-03-22 +### Changed +- Relaxed lodash version dependency. + + +## [1.0.1] - 2015-02-20 +### Changed +- Changed dependencies to be compatible with ancient nodejs. + + +## [1.0.0] - 2015-02-19 +### Changed +- Maintenance release. +- Replaced `underscore` with `lodash`. +- Bumped version to 1.0.0 to better reflect semver meaning. +- HISTORY.md -> CHANGELOG.md + + +## [0.1.16] - 2013-12-01 +### Changed +- Maintenance release. Updated dependencies and docs. + + +## [0.1.15] - 2013-05-13 +### Fixed +- Fixed #55, @trebor89 + + +## [0.1.14] - 2013-05-12 +### Fixed +- Fixed #62, @maxtaco + + +## [0.1.13] - 2013-04-08 +### Changed +- Added `.npmignore` to reduce package size + + +## [0.1.12] - 2013-02-10 +### Fixed +- Fixed conflictHandler (#46), @hpaulj + + +## [0.1.11] - 2013-02-07 +### Added +- Added 70+ tests (ported from python), @hpaulj +- Added conflictHandler, @applepicke +- Added fromfilePrefixChar, @hpaulj + +### Fixed +- Multiple bugfixes, @hpaulj + + +## [0.1.10] - 2012-12-30 +### Added +- Added [mutual exclusion](http://docs.python.org/dev/library/argparse.html#mutual-exclusion) + support, thanks to @hpaulj + +### Fixed +- Fixed options check for `storeConst` & `appendConst` actions, thanks to @hpaulj + + +## [0.1.9] - 2012-12-27 +### Fixed +- Fixed option dest interferens with other options (issue #23), thanks to @hpaulj +- Fixed default value behavior with `*` positionals, thanks to @hpaulj +- Improve `getDefault()` behavior, thanks to @hpaulj +- Improve negative argument parsing, thanks to @hpaulj + + +## [0.1.8] - 2012-12-01 +### Fixed +- Fixed parser parents (issue #19), thanks to @hpaulj +- Fixed negative argument parse (issue #20), thanks to @hpaulj + + +## [0.1.7] - 2012-10-14 +### Fixed +- Fixed 'choices' argument parse (issue #16) +- Fixed stderr output (issue #15) + + +## [0.1.6] - 2012-09-09 +### Fixed +- Fixed check for conflict of options (thanks to @tomxtobin) + + +## [0.1.5] - 2012-09-03 +### Fixed +- Fix parser #setDefaults method (thanks to @tomxtobin) + + +## [0.1.4] - 2012-07-30 +### Fixed +- Fixed pseudo-argument support (thanks to @CGamesPlay) +- Fixed addHelp default (should be true), if not set (thanks to @benblank) + + +## [0.1.3] - 2012-06-27 +### Fixed +- Fixed formatter api name: Formatter -> HelpFormatter + + +## [0.1.2] - 2012-05-29 +### Fixed +- Removed excess whitespace in help +- Fixed error reporting, when parcer with subcommands + called with empty arguments + +### Added +- Added basic tests + + +## [0.1.1] - 2012-05-23 +### Fixed +- Fixed line wrapping in help formatter +- Added better error reporting on invalid arguments + + +## [0.1.0] - 2012-05-16 +### Added +- First release. + + +[2.0.1]: https://github.com/nodeca/argparse/compare/2.0.0...2.0.1 +[2.0.0]: https://github.com/nodeca/argparse/compare/1.0.10...2.0.0 +[1.0.10]: https://github.com/nodeca/argparse/compare/1.0.9...1.0.10 +[1.0.9]: https://github.com/nodeca/argparse/compare/1.0.8...1.0.9 +[1.0.8]: https://github.com/nodeca/argparse/compare/1.0.7...1.0.8 +[1.0.7]: https://github.com/nodeca/argparse/compare/1.0.6...1.0.7 +[1.0.6]: https://github.com/nodeca/argparse/compare/1.0.5...1.0.6 +[1.0.5]: https://github.com/nodeca/argparse/compare/1.0.4...1.0.5 +[1.0.4]: https://github.com/nodeca/argparse/compare/1.0.3...1.0.4 +[1.0.3]: https://github.com/nodeca/argparse/compare/1.0.2...1.0.3 +[1.0.2]: https://github.com/nodeca/argparse/compare/1.0.1...1.0.2 +[1.0.1]: https://github.com/nodeca/argparse/compare/1.0.0...1.0.1 +[1.0.0]: https://github.com/nodeca/argparse/compare/0.1.16...1.0.0 +[0.1.16]: https://github.com/nodeca/argparse/compare/0.1.15...0.1.16 +[0.1.15]: https://github.com/nodeca/argparse/compare/0.1.14...0.1.15 +[0.1.14]: https://github.com/nodeca/argparse/compare/0.1.13...0.1.14 +[0.1.13]: https://github.com/nodeca/argparse/compare/0.1.12...0.1.13 +[0.1.12]: https://github.com/nodeca/argparse/compare/0.1.11...0.1.12 +[0.1.11]: https://github.com/nodeca/argparse/compare/0.1.10...0.1.11 +[0.1.10]: https://github.com/nodeca/argparse/compare/0.1.9...0.1.10 +[0.1.9]: https://github.com/nodeca/argparse/compare/0.1.8...0.1.9 +[0.1.8]: https://github.com/nodeca/argparse/compare/0.1.7...0.1.8 +[0.1.7]: https://github.com/nodeca/argparse/compare/0.1.6...0.1.7 +[0.1.6]: https://github.com/nodeca/argparse/compare/0.1.5...0.1.6 +[0.1.5]: https://github.com/nodeca/argparse/compare/0.1.4...0.1.5 +[0.1.4]: https://github.com/nodeca/argparse/compare/0.1.3...0.1.4 +[0.1.3]: https://github.com/nodeca/argparse/compare/0.1.2...0.1.3 +[0.1.2]: https://github.com/nodeca/argparse/compare/0.1.1...0.1.2 +[0.1.1]: https://github.com/nodeca/argparse/compare/0.1.0...0.1.1 +[0.1.0]: https://github.com/nodeca/argparse/releases/tag/0.1.0 diff --git a/queries/cdmq/node_modules/argparse/LICENSE b/queries/cdmq/node_modules/argparse/LICENSE new file mode 100644 index 00000000..66a3ac80 --- /dev/null +++ b/queries/cdmq/node_modules/argparse/LICENSE @@ -0,0 +1,254 @@ +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations, which became +Zope Corporation. In 2001, the Python Software Foundation (PSF, see +https://www.python.org/psf/) was formed, a non-profit organization +created specifically to own Python-related Intellectual Property. +Zope Corporation was a sponsoring member of the PSF. + +All Python releases are Open Source (see http://www.opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Python Software Foundation; +All Rights Reserved" are retained in Python alone or in any derivative version +prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the Internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the Internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/queries/cdmq/node_modules/argparse/README.md b/queries/cdmq/node_modules/argparse/README.md new file mode 100644 index 00000000..550b5c9b --- /dev/null +++ b/queries/cdmq/node_modules/argparse/README.md @@ -0,0 +1,84 @@ +argparse +======== + +[![Build Status](https://secure.travis-ci.org/nodeca/argparse.svg?branch=master)](http://travis-ci.org/nodeca/argparse) +[![NPM version](https://img.shields.io/npm/v/argparse.svg)](https://www.npmjs.org/package/argparse) + +CLI arguments parser for node.js, with [sub-commands](https://docs.python.org/3.9/library/argparse.html#sub-commands) support. Port of python's [argparse](http://docs.python.org/dev/library/argparse.html) (version [3.9.0](https://github.com/python/cpython/blob/v3.9.0rc1/Lib/argparse.py)). + +**Difference with original.** + +- JS has no keyword arguments support. + - Pass options instead: `new ArgumentParser({ description: 'example', add_help: true })`. +- JS has no python's types `int`, `float`, ... + - Use string-typed names: `.add_argument('-b', { type: 'int', help: 'help' })`. +- `%r` format specifier uses `require('util').inspect()`. + +More details in [doc](./doc). + + +Example +------- + +`test.js` file: + +```javascript +#!/usr/bin/env node +'use strict'; + +const { ArgumentParser } = require('argparse'); +const { version } = require('./package.json'); + +const parser = new ArgumentParser({ + description: 'Argparse example' +}); + +parser.add_argument('-v', '--version', { action: 'version', version }); +parser.add_argument('-f', '--foo', { help: 'foo bar' }); +parser.add_argument('-b', '--bar', { help: 'bar foo' }); +parser.add_argument('--baz', { help: 'baz bar' }); + +console.dir(parser.parse_args()); +``` + +Display help: + +``` +$ ./test.js -h +usage: test.js [-h] [-v] [-f FOO] [-b BAR] [--baz BAZ] + +Argparse example + +optional arguments: + -h, --help show this help message and exit + -v, --version show program's version number and exit + -f FOO, --foo FOO foo bar + -b BAR, --bar BAR bar foo + --baz BAZ baz bar +``` + +Parse arguments: + +``` +$ ./test.js -f=3 --bar=4 --baz 5 +{ foo: '3', bar: '4', baz: '5' } +``` + + +API docs +-------- + +Since this is a port with minimal divergence, there's no separate documentation. +Use original one instead, with notes about difference. + +1. [Original doc](https://docs.python.org/3.9/library/argparse.html). +2. [Original tutorial](https://docs.python.org/3.9/howto/argparse.html). +3. [Difference with python](./doc). + + +argparse for enterprise +----------------------- + +Available as part of the Tidelift Subscription + +The maintainers of argparse and thousands of other packages are working with Tidelift to deliver commercial support and maintenance for the open source dependencies you use to build your applications. Save time, reduce risk, and improve code health, while paying the maintainers of the exact dependencies you use. [Learn more.](https://tidelift.com/subscription/pkg/npm-argparse?utm_source=npm-argparse&utm_medium=referral&utm_campaign=enterprise&utm_term=repo) diff --git a/queries/cdmq/node_modules/argparse/argparse.js b/queries/cdmq/node_modules/argparse/argparse.js new file mode 100644 index 00000000..2b8c8c63 --- /dev/null +++ b/queries/cdmq/node_modules/argparse/argparse.js @@ -0,0 +1,3707 @@ +// Port of python's argparse module, version 3.9.0: +// https://github.com/python/cpython/blob/v3.9.0rc1/Lib/argparse.py + +'use strict' + +// Copyright (C) 2010-2020 Python Software Foundation. +// Copyright (C) 2020 argparse.js authors + +/* + * Command-line parsing library + * + * This module is an optparse-inspired command-line parsing library that: + * + * - handles both optional and positional arguments + * - produces highly informative usage messages + * - supports parsers that dispatch to sub-parsers + * + * The following is a simple usage example that sums integers from the + * command-line and writes the result to a file:: + * + * parser = argparse.ArgumentParser( + * description='sum the integers at the command line') + * parser.add_argument( + * 'integers', metavar='int', nargs='+', type=int, + * help='an integer to be summed') + * parser.add_argument( + * '--log', default=sys.stdout, type=argparse.FileType('w'), + * help='the file where the sum should be written') + * args = parser.parse_args() + * args.log.write('%s' % sum(args.integers)) + * args.log.close() + * + * The module contains the following public classes: + * + * - ArgumentParser -- The main entry point for command-line parsing. As the + * example above shows, the add_argument() method is used to populate + * the parser with actions for optional and positional arguments. Then + * the parse_args() method is invoked to convert the args at the + * command-line into an object with attributes. + * + * - ArgumentError -- The exception raised by ArgumentParser objects when + * there are errors with the parser's actions. Errors raised while + * parsing the command-line are caught by ArgumentParser and emitted + * as command-line messages. + * + * - FileType -- A factory for defining types of files to be created. As the + * example above shows, instances of FileType are typically passed as + * the type= argument of add_argument() calls. + * + * - Action -- The base class for parser actions. Typically actions are + * selected by passing strings like 'store_true' or 'append_const' to + * the action= argument of add_argument(). However, for greater + * customization of ArgumentParser actions, subclasses of Action may + * be defined and passed as the action= argument. + * + * - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter, + * ArgumentDefaultsHelpFormatter -- Formatter classes which + * may be passed as the formatter_class= argument to the + * ArgumentParser constructor. HelpFormatter is the default, + * RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser + * not to change the formatting for help text, and + * ArgumentDefaultsHelpFormatter adds information about argument defaults + * to the help. + * + * All other classes in this module are considered implementation details. + * (Also note that HelpFormatter and RawDescriptionHelpFormatter are only + * considered public as object names -- the API of the formatter objects is + * still considered an implementation detail.) + */ + +const SUPPRESS = '==SUPPRESS==' + +const OPTIONAL = '?' +const ZERO_OR_MORE = '*' +const ONE_OR_MORE = '+' +const PARSER = 'A...' +const REMAINDER = '...' +const _UNRECOGNIZED_ARGS_ATTR = '_unrecognized_args' + + +// ================================== +// Utility functions used for porting +// ================================== +const assert = require('assert') +const util = require('util') +const fs = require('fs') +const sub = require('./lib/sub') +const path = require('path') +const repr = util.inspect + +function get_argv() { + // omit first argument (which is assumed to be interpreter - `node`, `coffee`, `ts-node`, etc.) + return process.argv.slice(1) +} + +function get_terminal_size() { + return { + columns: +process.env.COLUMNS || process.stdout.columns || 80 + } +} + +function hasattr(object, name) { + return Object.prototype.hasOwnProperty.call(object, name) +} + +function getattr(object, name, value) { + return hasattr(object, name) ? object[name] : value +} + +function setattr(object, name, value) { + object[name] = value +} + +function setdefault(object, name, value) { + if (!hasattr(object, name)) object[name] = value + return object[name] +} + +function delattr(object, name) { + delete object[name] +} + +function range(from, to, step=1) { + // range(10) is equivalent to range(0, 10) + if (arguments.length === 1) [ to, from ] = [ from, 0 ] + if (typeof from !== 'number' || typeof to !== 'number' || typeof step !== 'number') { + throw new TypeError('argument cannot be interpreted as an integer') + } + if (step === 0) throw new TypeError('range() arg 3 must not be zero') + + let result = [] + if (step > 0) { + for (let i = from; i < to; i += step) result.push(i) + } else { + for (let i = from; i > to; i += step) result.push(i) + } + return result +} + +function splitlines(str, keepends = false) { + let result + if (!keepends) { + result = str.split(/\r\n|[\n\r\v\f\x1c\x1d\x1e\x85\u2028\u2029]/) + } else { + result = [] + let parts = str.split(/(\r\n|[\n\r\v\f\x1c\x1d\x1e\x85\u2028\u2029])/) + for (let i = 0; i < parts.length; i += 2) { + result.push(parts[i] + (i + 1 < parts.length ? parts[i + 1] : '')) + } + } + if (!result[result.length - 1]) result.pop() + return result +} + +function _string_lstrip(string, prefix_chars) { + let idx = 0 + while (idx < string.length && prefix_chars.includes(string[idx])) idx++ + return idx ? string.slice(idx) : string +} + +function _string_split(string, sep, maxsplit) { + let result = string.split(sep) + if (result.length > maxsplit) { + result = result.slice(0, maxsplit).concat([ result.slice(maxsplit).join(sep) ]) + } + return result +} + +function _array_equal(array1, array2) { + if (array1.length !== array2.length) return false + for (let i = 0; i < array1.length; i++) { + if (array1[i] !== array2[i]) return false + } + return true +} + +function _array_remove(array, item) { + let idx = array.indexOf(item) + if (idx === -1) throw new TypeError(sub('%r not in list', item)) + array.splice(idx, 1) +} + +// normalize choices to array; +// this isn't required in python because `in` and `map` operators work with anything, +// but in js dealing with multiple types here is too clunky +function _choices_to_array(choices) { + if (choices === undefined) { + return [] + } else if (Array.isArray(choices)) { + return choices + } else if (choices !== null && typeof choices[Symbol.iterator] === 'function') { + return Array.from(choices) + } else if (typeof choices === 'object' && choices !== null) { + return Object.keys(choices) + } else { + throw new Error(sub('invalid choices value: %r', choices)) + } +} + +// decorator that allows a class to be called without new +function _callable(cls) { + let result = { // object is needed for inferred class name + [cls.name]: function (...args) { + let this_class = new.target === result || !new.target + return Reflect.construct(cls, args, this_class ? cls : new.target) + } + } + result[cls.name].prototype = cls.prototype + // fix default tag for toString, e.g. [object Action] instead of [object Object] + cls.prototype[Symbol.toStringTag] = cls.name + return result[cls.name] +} + +function _alias(object, from, to) { + try { + let name = object.constructor.name + Object.defineProperty(object, from, { + value: util.deprecate(object[to], sub('%s.%s() is renamed to %s.%s()', + name, from, name, to)), + enumerable: false + }) + } catch {} +} + +// decorator that allows snake_case class methods to be called with camelCase and vice versa +function _camelcase_alias(_class) { + for (let name of Object.getOwnPropertyNames(_class.prototype)) { + let camelcase = name.replace(/\w_[a-z]/g, s => s[0] + s[2].toUpperCase()) + if (camelcase !== name) _alias(_class.prototype, camelcase, name) + } + return _class +} + +function _to_legacy_name(key) { + key = key.replace(/\w_[a-z]/g, s => s[0] + s[2].toUpperCase()) + if (key === 'default') key = 'defaultValue' + if (key === 'const') key = 'constant' + return key +} + +function _to_new_name(key) { + if (key === 'defaultValue') key = 'default' + if (key === 'constant') key = 'const' + key = key.replace(/[A-Z]/g, c => '_' + c.toLowerCase()) + return key +} + +// parse options +let no_default = Symbol('no_default_value') +function _parse_opts(args, descriptor) { + function get_name() { + let stack = new Error().stack.split('\n') + .map(x => x.match(/^ at (.*) \(.*\)$/)) + .filter(Boolean) + .map(m => m[1]) + .map(fn => fn.match(/[^ .]*$/)[0]) + + if (stack.length && stack[0] === get_name.name) stack.shift() + if (stack.length && stack[0] === _parse_opts.name) stack.shift() + return stack.length ? stack[0] : '' + } + + args = Array.from(args) + let kwargs = {} + let result = [] + let last_opt = args.length && args[args.length - 1] + + if (typeof last_opt === 'object' && last_opt !== null && !Array.isArray(last_opt) && + (!last_opt.constructor || last_opt.constructor.name === 'Object')) { + kwargs = Object.assign({}, args.pop()) + } + + // LEGACY (v1 compatibility): camelcase + let renames = [] + for (let key of Object.keys(descriptor)) { + let old_name = _to_legacy_name(key) + if (old_name !== key && (old_name in kwargs)) { + if (key in kwargs) { + // default and defaultValue specified at the same time, happens often in old tests + //throw new TypeError(sub('%s() got multiple values for argument %r', get_name(), key)) + } else { + kwargs[key] = kwargs[old_name] + } + renames.push([ old_name, key ]) + delete kwargs[old_name] + } + } + if (renames.length) { + let name = get_name() + deprecate('camelcase_' + name, sub('%s(): following options are renamed: %s', + name, renames.map(([ a, b ]) => sub('%r -> %r', a, b)))) + } + // end + + let missing_positionals = [] + let positional_count = args.length + + for (let [ key, def ] of Object.entries(descriptor)) { + if (key[0] === '*') { + if (key.length > 0 && key[1] === '*') { + // LEGACY (v1 compatibility): camelcase + let renames = [] + for (let key of Object.keys(kwargs)) { + let new_name = _to_new_name(key) + if (new_name !== key && (key in kwargs)) { + if (new_name in kwargs) { + // default and defaultValue specified at the same time, happens often in old tests + //throw new TypeError(sub('%s() got multiple values for argument %r', get_name(), new_name)) + } else { + kwargs[new_name] = kwargs[key] + } + renames.push([ key, new_name ]) + delete kwargs[key] + } + } + if (renames.length) { + let name = get_name() + deprecate('camelcase_' + name, sub('%s(): following options are renamed: %s', + name, renames.map(([ a, b ]) => sub('%r -> %r', a, b)))) + } + // end + result.push(kwargs) + kwargs = {} + } else { + result.push(args) + args = [] + } + } else if (key in kwargs && args.length > 0) { + throw new TypeError(sub('%s() got multiple values for argument %r', get_name(), key)) + } else if (key in kwargs) { + result.push(kwargs[key]) + delete kwargs[key] + } else if (args.length > 0) { + result.push(args.shift()) + } else if (def !== no_default) { + result.push(def) + } else { + missing_positionals.push(key) + } + } + + if (Object.keys(kwargs).length) { + throw new TypeError(sub('%s() got an unexpected keyword argument %r', + get_name(), Object.keys(kwargs)[0])) + } + + if (args.length) { + let from = Object.entries(descriptor).filter(([ k, v ]) => k[0] !== '*' && v !== no_default).length + let to = Object.entries(descriptor).filter(([ k ]) => k[0] !== '*').length + throw new TypeError(sub('%s() takes %s positional argument%s but %s %s given', + get_name(), + from === to ? sub('from %s to %s', from, to) : to, + from === to && to === 1 ? '' : 's', + positional_count, + positional_count === 1 ? 'was' : 'were')) + } + + if (missing_positionals.length) { + let strs = missing_positionals.map(repr) + if (strs.length > 1) strs[strs.length - 1] = 'and ' + strs[strs.length - 1] + let str_joined = strs.join(strs.length === 2 ? '' : ', ') + throw new TypeError(sub('%s() missing %i required positional argument%s: %s', + get_name(), strs.length, strs.length === 1 ? '' : 's', str_joined)) + } + + return result +} + +let _deprecations = {} +function deprecate(id, string) { + _deprecations[id] = _deprecations[id] || util.deprecate(() => {}, string) + _deprecations[id]() +} + + +// ============================= +// Utility functions and classes +// ============================= +function _AttributeHolder(cls = Object) { + /* + * Abstract base class that provides __repr__. + * + * The __repr__ method returns a string in the format:: + * ClassName(attr=name, attr=name, ...) + * The attributes are determined either by a class-level attribute, + * '_kwarg_names', or by inspecting the instance __dict__. + */ + + return class _AttributeHolder extends cls { + [util.inspect.custom]() { + let type_name = this.constructor.name + let arg_strings = [] + let star_args = {} + for (let arg of this._get_args()) { + arg_strings.push(repr(arg)) + } + for (let [ name, value ] of this._get_kwargs()) { + if (/^[a-z_][a-z0-9_$]*$/i.test(name)) { + arg_strings.push(sub('%s=%r', name, value)) + } else { + star_args[name] = value + } + } + if (Object.keys(star_args).length) { + arg_strings.push(sub('**%s', repr(star_args))) + } + return sub('%s(%s)', type_name, arg_strings.join(', ')) + } + + toString() { + return this[util.inspect.custom]() + } + + _get_kwargs() { + return Object.entries(this) + } + + _get_args() { + return [] + } + } +} + + +function _copy_items(items) { + if (items === undefined) { + return [] + } + return items.slice(0) +} + + +// =============== +// Formatting Help +// =============== +const HelpFormatter = _camelcase_alias(_callable(class HelpFormatter { + /* + * Formatter for generating usage messages and argument help strings. + * + * Only the name of this class is considered a public API. All the methods + * provided by the class are considered an implementation detail. + */ + + constructor() { + let [ + prog, + indent_increment, + max_help_position, + width + ] = _parse_opts(arguments, { + prog: no_default, + indent_increment: 2, + max_help_position: 24, + width: undefined + }) + + // default setting for width + if (width === undefined) { + width = get_terminal_size().columns + width -= 2 + } + + this._prog = prog + this._indent_increment = indent_increment + this._max_help_position = Math.min(max_help_position, + Math.max(width - 20, indent_increment * 2)) + this._width = width + + this._current_indent = 0 + this._level = 0 + this._action_max_length = 0 + + this._root_section = this._Section(this, undefined) + this._current_section = this._root_section + + this._whitespace_matcher = /[ \t\n\r\f\v]+/g // equivalent to python /\s+/ with ASCII flag + this._long_break_matcher = /\n\n\n+/g + } + + // =============================== + // Section and indentation methods + // =============================== + _indent() { + this._current_indent += this._indent_increment + this._level += 1 + } + + _dedent() { + this._current_indent -= this._indent_increment + assert(this._current_indent >= 0, 'Indent decreased below 0.') + this._level -= 1 + } + + _add_item(func, args) { + this._current_section.items.push([ func, args ]) + } + + // ======================== + // Message building methods + // ======================== + start_section(heading) { + this._indent() + let section = this._Section(this, this._current_section, heading) + this._add_item(section.format_help.bind(section), []) + this._current_section = section + } + + end_section() { + this._current_section = this._current_section.parent + this._dedent() + } + + add_text(text) { + if (text !== SUPPRESS && text !== undefined) { + this._add_item(this._format_text.bind(this), [text]) + } + } + + add_usage(usage, actions, groups, prefix = undefined) { + if (usage !== SUPPRESS) { + let args = [ usage, actions, groups, prefix ] + this._add_item(this._format_usage.bind(this), args) + } + } + + add_argument(action) { + if (action.help !== SUPPRESS) { + + // find all invocations + let invocations = [this._format_action_invocation(action)] + for (let subaction of this._iter_indented_subactions(action)) { + invocations.push(this._format_action_invocation(subaction)) + } + + // update the maximum item length + let invocation_length = Math.max(...invocations.map(invocation => invocation.length)) + let action_length = invocation_length + this._current_indent + this._action_max_length = Math.max(this._action_max_length, + action_length) + + // add the item to the list + this._add_item(this._format_action.bind(this), [action]) + } + } + + add_arguments(actions) { + for (let action of actions) { + this.add_argument(action) + } + } + + // ======================= + // Help-formatting methods + // ======================= + format_help() { + let help = this._root_section.format_help() + if (help) { + help = help.replace(this._long_break_matcher, '\n\n') + help = help.replace(/^\n+|\n+$/g, '') + '\n' + } + return help + } + + _join_parts(part_strings) { + return part_strings.filter(part => part && part !== SUPPRESS).join('') + } + + _format_usage(usage, actions, groups, prefix) { + if (prefix === undefined) { + prefix = 'usage: ' + } + + // if usage is specified, use that + if (usage !== undefined) { + usage = sub(usage, { prog: this._prog }) + + // if no optionals or positionals are available, usage is just prog + } else if (usage === undefined && !actions.length) { + usage = sub('%(prog)s', { prog: this._prog }) + + // if optionals and positionals are available, calculate usage + } else if (usage === undefined) { + let prog = sub('%(prog)s', { prog: this._prog }) + + // split optionals from positionals + let optionals = [] + let positionals = [] + for (let action of actions) { + if (action.option_strings.length) { + optionals.push(action) + } else { + positionals.push(action) + } + } + + // build full usage string + let action_usage = this._format_actions_usage([].concat(optionals).concat(positionals), groups) + usage = [ prog, action_usage ].map(String).join(' ') + + // wrap the usage parts if it's too long + let text_width = this._width - this._current_indent + if (prefix.length + usage.length > text_width) { + + // break usage into wrappable parts + let part_regexp = /\(.*?\)+(?=\s|$)|\[.*?\]+(?=\s|$)|\S+/g + let opt_usage = this._format_actions_usage(optionals, groups) + let pos_usage = this._format_actions_usage(positionals, groups) + let opt_parts = opt_usage.match(part_regexp) || [] + let pos_parts = pos_usage.match(part_regexp) || [] + assert(opt_parts.join(' ') === opt_usage) + assert(pos_parts.join(' ') === pos_usage) + + // helper for wrapping lines + let get_lines = (parts, indent, prefix = undefined) => { + let lines = [] + let line = [] + let line_len + if (prefix !== undefined) { + line_len = prefix.length - 1 + } else { + line_len = indent.length - 1 + } + for (let part of parts) { + if (line_len + 1 + part.length > text_width && line) { + lines.push(indent + line.join(' ')) + line = [] + line_len = indent.length - 1 + } + line.push(part) + line_len += part.length + 1 + } + if (line.length) { + lines.push(indent + line.join(' ')) + } + if (prefix !== undefined) { + lines[0] = lines[0].slice(indent.length) + } + return lines + } + + let lines + + // if prog is short, follow it with optionals or positionals + if (prefix.length + prog.length <= 0.75 * text_width) { + let indent = ' '.repeat(prefix.length + prog.length + 1) + if (opt_parts.length) { + lines = get_lines([prog].concat(opt_parts), indent, prefix) + lines = lines.concat(get_lines(pos_parts, indent)) + } else if (pos_parts.length) { + lines = get_lines([prog].concat(pos_parts), indent, prefix) + } else { + lines = [prog] + } + + // if prog is long, put it on its own line + } else { + let indent = ' '.repeat(prefix.length) + let parts = [].concat(opt_parts).concat(pos_parts) + lines = get_lines(parts, indent) + if (lines.length > 1) { + lines = [] + lines = lines.concat(get_lines(opt_parts, indent)) + lines = lines.concat(get_lines(pos_parts, indent)) + } + lines = [prog].concat(lines) + } + + // join lines into usage + usage = lines.join('\n') + } + } + + // prefix with 'usage:' + return sub('%s%s\n\n', prefix, usage) + } + + _format_actions_usage(actions, groups) { + // find group indices and identify actions in groups + let group_actions = new Set() + let inserts = {} + for (let group of groups) { + let start = actions.indexOf(group._group_actions[0]) + if (start === -1) { + continue + } else { + let end = start + group._group_actions.length + if (_array_equal(actions.slice(start, end), group._group_actions)) { + for (let action of group._group_actions) { + group_actions.add(action) + } + if (!group.required) { + if (start in inserts) { + inserts[start] += ' [' + } else { + inserts[start] = '[' + } + if (end in inserts) { + inserts[end] += ']' + } else { + inserts[end] = ']' + } + } else { + if (start in inserts) { + inserts[start] += ' (' + } else { + inserts[start] = '(' + } + if (end in inserts) { + inserts[end] += ')' + } else { + inserts[end] = ')' + } + } + for (let i of range(start + 1, end)) { + inserts[i] = '|' + } + } + } + } + + // collect all actions format strings + let parts = [] + for (let [ i, action ] of Object.entries(actions)) { + + // suppressed arguments are marked with None + // remove | separators for suppressed arguments + if (action.help === SUPPRESS) { + parts.push(undefined) + if (inserts[+i] === '|') { + delete inserts[+i] + } else if (inserts[+i + 1] === '|') { + delete inserts[+i + 1] + } + + // produce all arg strings + } else if (!action.option_strings.length) { + let default_value = this._get_default_metavar_for_positional(action) + let part = this._format_args(action, default_value) + + // if it's in a group, strip the outer [] + if (group_actions.has(action)) { + if (part[0] === '[' && part[part.length - 1] === ']') { + part = part.slice(1, -1) + } + } + + // add the action string to the list + parts.push(part) + + // produce the first way to invoke the option in brackets + } else { + let option_string = action.option_strings[0] + let part + + // if the Optional doesn't take a value, format is: + // -s or --long + if (action.nargs === 0) { + part = action.format_usage() + + // if the Optional takes a value, format is: + // -s ARGS or --long ARGS + } else { + let default_value = this._get_default_metavar_for_optional(action) + let args_string = this._format_args(action, default_value) + part = sub('%s %s', option_string, args_string) + } + + // make it look optional if it's not required or in a group + if (!action.required && !group_actions.has(action)) { + part = sub('[%s]', part) + } + + // add the action string to the list + parts.push(part) + } + } + + // insert things at the necessary indices + for (let i of Object.keys(inserts).map(Number).sort((a, b) => b - a)) { + parts.splice(+i, 0, inserts[+i]) + } + + // join all the action items with spaces + let text = parts.filter(Boolean).join(' ') + + // clean up separators for mutually exclusive groups + text = text.replace(/([\[(]) /g, '$1') + text = text.replace(/ ([\])])/g, '$1') + text = text.replace(/[\[(] *[\])]/g, '') + text = text.replace(/\(([^|]*)\)/g, '$1', text) + text = text.trim() + + // return the text + return text + } + + _format_text(text) { + if (text.includes('%(prog)')) { + text = sub(text, { prog: this._prog }) + } + let text_width = Math.max(this._width - this._current_indent, 11) + let indent = ' '.repeat(this._current_indent) + return this._fill_text(text, text_width, indent) + '\n\n' + } + + _format_action(action) { + // determine the required width and the entry label + let help_position = Math.min(this._action_max_length + 2, + this._max_help_position) + let help_width = Math.max(this._width - help_position, 11) + let action_width = help_position - this._current_indent - 2 + let action_header = this._format_action_invocation(action) + let indent_first + + // no help; start on same line and add a final newline + if (!action.help) { + let tup = [ this._current_indent, '', action_header ] + action_header = sub('%*s%s\n', ...tup) + + // short action name; start on the same line and pad two spaces + } else if (action_header.length <= action_width) { + let tup = [ this._current_indent, '', action_width, action_header ] + action_header = sub('%*s%-*s ', ...tup) + indent_first = 0 + + // long action name; start on the next line + } else { + let tup = [ this._current_indent, '', action_header ] + action_header = sub('%*s%s\n', ...tup) + indent_first = help_position + } + + // collect the pieces of the action help + let parts = [action_header] + + // if there was help for the action, add lines of help text + if (action.help) { + let help_text = this._expand_help(action) + let help_lines = this._split_lines(help_text, help_width) + parts.push(sub('%*s%s\n', indent_first, '', help_lines[0])) + for (let line of help_lines.slice(1)) { + parts.push(sub('%*s%s\n', help_position, '', line)) + } + + // or add a newline if the description doesn't end with one + } else if (!action_header.endsWith('\n')) { + parts.push('\n') + } + + // if there are any sub-actions, add their help as well + for (let subaction of this._iter_indented_subactions(action)) { + parts.push(this._format_action(subaction)) + } + + // return a single string + return this._join_parts(parts) + } + + _format_action_invocation(action) { + if (!action.option_strings.length) { + let default_value = this._get_default_metavar_for_positional(action) + let metavar = this._metavar_formatter(action, default_value)(1)[0] + return metavar + + } else { + let parts = [] + + // if the Optional doesn't take a value, format is: + // -s, --long + if (action.nargs === 0) { + parts = parts.concat(action.option_strings) + + // if the Optional takes a value, format is: + // -s ARGS, --long ARGS + } else { + let default_value = this._get_default_metavar_for_optional(action) + let args_string = this._format_args(action, default_value) + for (let option_string of action.option_strings) { + parts.push(sub('%s %s', option_string, args_string)) + } + } + + return parts.join(', ') + } + } + + _metavar_formatter(action, default_metavar) { + let result + if (action.metavar !== undefined) { + result = action.metavar + } else if (action.choices !== undefined) { + let choice_strs = _choices_to_array(action.choices).map(String) + result = sub('{%s}', choice_strs.join(',')) + } else { + result = default_metavar + } + + function format(tuple_size) { + if (Array.isArray(result)) { + return result + } else { + return Array(tuple_size).fill(result) + } + } + return format + } + + _format_args(action, default_metavar) { + let get_metavar = this._metavar_formatter(action, default_metavar) + let result + if (action.nargs === undefined) { + result = sub('%s', ...get_metavar(1)) + } else if (action.nargs === OPTIONAL) { + result = sub('[%s]', ...get_metavar(1)) + } else if (action.nargs === ZERO_OR_MORE) { + let metavar = get_metavar(1) + if (metavar.length === 2) { + result = sub('[%s [%s ...]]', ...metavar) + } else { + result = sub('[%s ...]', ...metavar) + } + } else if (action.nargs === ONE_OR_MORE) { + result = sub('%s [%s ...]', ...get_metavar(2)) + } else if (action.nargs === REMAINDER) { + result = '...' + } else if (action.nargs === PARSER) { + result = sub('%s ...', ...get_metavar(1)) + } else if (action.nargs === SUPPRESS) { + result = '' + } else { + let formats + try { + formats = range(action.nargs).map(() => '%s') + } catch (err) { + throw new TypeError('invalid nargs value') + } + result = sub(formats.join(' '), ...get_metavar(action.nargs)) + } + return result + } + + _expand_help(action) { + let params = Object.assign({ prog: this._prog }, action) + for (let name of Object.keys(params)) { + if (params[name] === SUPPRESS) { + delete params[name] + } + } + for (let name of Object.keys(params)) { + if (params[name] && params[name].name) { + params[name] = params[name].name + } + } + if (params.choices !== undefined) { + let choices_str = _choices_to_array(params.choices).map(String).join(', ') + params.choices = choices_str + } + // LEGACY (v1 compatibility): camelcase + for (let key of Object.keys(params)) { + let old_name = _to_legacy_name(key) + if (old_name !== key) { + params[old_name] = params[key] + } + } + // end + return sub(this._get_help_string(action), params) + } + + * _iter_indented_subactions(action) { + if (typeof action._get_subactions === 'function') { + this._indent() + yield* action._get_subactions() + this._dedent() + } + } + + _split_lines(text, width) { + text = text.replace(this._whitespace_matcher, ' ').trim() + // The textwrap module is used only for formatting help. + // Delay its import for speeding up the common usage of argparse. + let textwrap = require('./lib/textwrap') + return textwrap.wrap(text, { width }) + } + + _fill_text(text, width, indent) { + text = text.replace(this._whitespace_matcher, ' ').trim() + let textwrap = require('./lib/textwrap') + return textwrap.fill(text, { width, + initial_indent: indent, + subsequent_indent: indent }) + } + + _get_help_string(action) { + return action.help + } + + _get_default_metavar_for_optional(action) { + return action.dest.toUpperCase() + } + + _get_default_metavar_for_positional(action) { + return action.dest + } +})) + +HelpFormatter.prototype._Section = _callable(class _Section { + + constructor(formatter, parent, heading = undefined) { + this.formatter = formatter + this.parent = parent + this.heading = heading + this.items = [] + } + + format_help() { + // format the indented section + if (this.parent !== undefined) { + this.formatter._indent() + } + let item_help = this.formatter._join_parts(this.items.map(([ func, args ]) => func.apply(null, args))) + if (this.parent !== undefined) { + this.formatter._dedent() + } + + // return nothing if the section was empty + if (!item_help) { + return '' + } + + // add the heading if the section was non-empty + let heading + if (this.heading !== SUPPRESS && this.heading !== undefined) { + let current_indent = this.formatter._current_indent + heading = sub('%*s%s:\n', current_indent, '', this.heading) + } else { + heading = '' + } + + // join the section-initial newline, the heading and the help + return this.formatter._join_parts(['\n', heading, item_help, '\n']) + } +}) + + +const RawDescriptionHelpFormatter = _camelcase_alias(_callable(class RawDescriptionHelpFormatter extends HelpFormatter { + /* + * Help message formatter which retains any formatting in descriptions. + * + * Only the name of this class is considered a public API. All the methods + * provided by the class are considered an implementation detail. + */ + + _fill_text(text, width, indent) { + return splitlines(text, true).map(line => indent + line).join('') + } +})) + + +const RawTextHelpFormatter = _camelcase_alias(_callable(class RawTextHelpFormatter extends RawDescriptionHelpFormatter { + /* + * Help message formatter which retains formatting of all help text. + * + * Only the name of this class is considered a public API. All the methods + * provided by the class are considered an implementation detail. + */ + + _split_lines(text/*, width*/) { + return splitlines(text) + } +})) + + +const ArgumentDefaultsHelpFormatter = _camelcase_alias(_callable(class ArgumentDefaultsHelpFormatter extends HelpFormatter { + /* + * Help message formatter which adds default values to argument help. + * + * Only the name of this class is considered a public API. All the methods + * provided by the class are considered an implementation detail. + */ + + _get_help_string(action) { + let help = action.help + // LEGACY (v1 compatibility): additional check for defaultValue needed + if (!action.help.includes('%(default)') && !action.help.includes('%(defaultValue)')) { + if (action.default !== SUPPRESS) { + let defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] + if (action.option_strings.length || defaulting_nargs.includes(action.nargs)) { + help += ' (default: %(default)s)' + } + } + } + return help + } +})) + + +const MetavarTypeHelpFormatter = _camelcase_alias(_callable(class MetavarTypeHelpFormatter extends HelpFormatter { + /* + * Help message formatter which uses the argument 'type' as the default + * metavar value (instead of the argument 'dest') + * + * Only the name of this class is considered a public API. All the methods + * provided by the class are considered an implementation detail. + */ + + _get_default_metavar_for_optional(action) { + return typeof action.type === 'function' ? action.type.name : action.type + } + + _get_default_metavar_for_positional(action) { + return typeof action.type === 'function' ? action.type.name : action.type + } +})) + + +// ===================== +// Options and Arguments +// ===================== +function _get_action_name(argument) { + if (argument === undefined) { + return undefined + } else if (argument.option_strings.length) { + return argument.option_strings.join('/') + } else if (![ undefined, SUPPRESS ].includes(argument.metavar)) { + return argument.metavar + } else if (![ undefined, SUPPRESS ].includes(argument.dest)) { + return argument.dest + } else { + return undefined + } +} + + +const ArgumentError = _callable(class ArgumentError extends Error { + /* + * An error from creating or using an argument (optional or positional). + * + * The string value of this exception is the message, augmented with + * information about the argument that caused it. + */ + + constructor(argument, message) { + super() + this.name = 'ArgumentError' + this._argument_name = _get_action_name(argument) + this._message = message + this.message = this.str() + } + + str() { + let format + if (this._argument_name === undefined) { + format = '%(message)s' + } else { + format = 'argument %(argument_name)s: %(message)s' + } + return sub(format, { message: this._message, + argument_name: this._argument_name }) + } +}) + + +const ArgumentTypeError = _callable(class ArgumentTypeError extends Error { + /* + * An error from trying to convert a command line string to a type. + */ + + constructor(message) { + super(message) + this.name = 'ArgumentTypeError' + } +}) + + +// ============== +// Action classes +// ============== +const Action = _camelcase_alias(_callable(class Action extends _AttributeHolder(Function) { + /* + * Information about how to convert command line strings to Python objects. + * + * Action objects are used by an ArgumentParser to represent the information + * needed to parse a single argument from one or more strings from the + * command line. The keyword arguments to the Action constructor are also + * all attributes of Action instances. + * + * Keyword Arguments: + * + * - option_strings -- A list of command-line option strings which + * should be associated with this action. + * + * - dest -- The name of the attribute to hold the created object(s) + * + * - nargs -- The number of command-line arguments that should be + * consumed. By default, one argument will be consumed and a single + * value will be produced. Other values include: + * - N (an integer) consumes N arguments (and produces a list) + * - '?' consumes zero or one arguments + * - '*' consumes zero or more arguments (and produces a list) + * - '+' consumes one or more arguments (and produces a list) + * Note that the difference between the default and nargs=1 is that + * with the default, a single value will be produced, while with + * nargs=1, a list containing a single value will be produced. + * + * - const -- The value to be produced if the option is specified and the + * option uses an action that takes no values. + * + * - default -- The value to be produced if the option is not specified. + * + * - type -- A callable that accepts a single string argument, and + * returns the converted value. The standard Python types str, int, + * float, and complex are useful examples of such callables. If None, + * str is used. + * + * - choices -- A container of values that should be allowed. If not None, + * after a command-line argument has been converted to the appropriate + * type, an exception will be raised if it is not a member of this + * collection. + * + * - required -- True if the action must always be specified at the + * command line. This is only meaningful for optional command-line + * arguments. + * + * - help -- The help string describing the argument. + * + * - metavar -- The name to be used for the option's argument with the + * help string. If None, the 'dest' value will be used as the name. + */ + + constructor() { + let [ + option_strings, + dest, + nargs, + const_value, + default_value, + type, + choices, + required, + help, + metavar + ] = _parse_opts(arguments, { + option_strings: no_default, + dest: no_default, + nargs: undefined, + const: undefined, + default: undefined, + type: undefined, + choices: undefined, + required: false, + help: undefined, + metavar: undefined + }) + + // when this class is called as a function, redirect it to .call() method of itself + super('return arguments.callee.call.apply(arguments.callee, arguments)') + + this.option_strings = option_strings + this.dest = dest + this.nargs = nargs + this.const = const_value + this.default = default_value + this.type = type + this.choices = choices + this.required = required + this.help = help + this.metavar = metavar + } + + _get_kwargs() { + let names = [ + 'option_strings', + 'dest', + 'nargs', + 'const', + 'default', + 'type', + 'choices', + 'help', + 'metavar' + ] + return names.map(name => [ name, getattr(this, name) ]) + } + + format_usage() { + return this.option_strings[0] + } + + call(/*parser, namespace, values, option_string = undefined*/) { + throw new Error('.call() not defined') + } +})) + + +const BooleanOptionalAction = _camelcase_alias(_callable(class BooleanOptionalAction extends Action { + + constructor() { + let [ + option_strings, + dest, + default_value, + type, + choices, + required, + help, + metavar + ] = _parse_opts(arguments, { + option_strings: no_default, + dest: no_default, + default: undefined, + type: undefined, + choices: undefined, + required: false, + help: undefined, + metavar: undefined + }) + + let _option_strings = [] + for (let option_string of option_strings) { + _option_strings.push(option_string) + + if (option_string.startsWith('--')) { + option_string = '--no-' + option_string.slice(2) + _option_strings.push(option_string) + } + } + + if (help !== undefined && default_value !== undefined) { + help += ` (default: ${default_value})` + } + + super({ + option_strings: _option_strings, + dest, + nargs: 0, + default: default_value, + type, + choices, + required, + help, + metavar + }) + } + + call(parser, namespace, values, option_string = undefined) { + if (this.option_strings.includes(option_string)) { + setattr(namespace, this.dest, !option_string.startsWith('--no-')) + } + } + + format_usage() { + return this.option_strings.join(' | ') + } +})) + + +const _StoreAction = _callable(class _StoreAction extends Action { + + constructor() { + let [ + option_strings, + dest, + nargs, + const_value, + default_value, + type, + choices, + required, + help, + metavar + ] = _parse_opts(arguments, { + option_strings: no_default, + dest: no_default, + nargs: undefined, + const: undefined, + default: undefined, + type: undefined, + choices: undefined, + required: false, + help: undefined, + metavar: undefined + }) + + if (nargs === 0) { + throw new TypeError('nargs for store actions must be != 0; if you ' + + 'have nothing to store, actions such as store ' + + 'true or store const may be more appropriate') + } + if (const_value !== undefined && nargs !== OPTIONAL) { + throw new TypeError(sub('nargs must be %r to supply const', OPTIONAL)) + } + super({ + option_strings, + dest, + nargs, + const: const_value, + default: default_value, + type, + choices, + required, + help, + metavar + }) + } + + call(parser, namespace, values/*, option_string = undefined*/) { + setattr(namespace, this.dest, values) + } +}) + + +const _StoreConstAction = _callable(class _StoreConstAction extends Action { + + constructor() { + let [ + option_strings, + dest, + const_value, + default_value, + required, + help + //, metavar + ] = _parse_opts(arguments, { + option_strings: no_default, + dest: no_default, + const: no_default, + default: undefined, + required: false, + help: undefined, + metavar: undefined + }) + + super({ + option_strings, + dest, + nargs: 0, + const: const_value, + default: default_value, + required, + help + }) + } + + call(parser, namespace/*, values, option_string = undefined*/) { + setattr(namespace, this.dest, this.const) + } +}) + + +const _StoreTrueAction = _callable(class _StoreTrueAction extends _StoreConstAction { + + constructor() { + let [ + option_strings, + dest, + default_value, + required, + help + ] = _parse_opts(arguments, { + option_strings: no_default, + dest: no_default, + default: false, + required: false, + help: undefined + }) + + super({ + option_strings, + dest, + const: true, + default: default_value, + required, + help + }) + } +}) + + +const _StoreFalseAction = _callable(class _StoreFalseAction extends _StoreConstAction { + + constructor() { + let [ + option_strings, + dest, + default_value, + required, + help + ] = _parse_opts(arguments, { + option_strings: no_default, + dest: no_default, + default: true, + required: false, + help: undefined + }) + + super({ + option_strings, + dest, + const: false, + default: default_value, + required, + help + }) + } +}) + + +const _AppendAction = _callable(class _AppendAction extends Action { + + constructor() { + let [ + option_strings, + dest, + nargs, + const_value, + default_value, + type, + choices, + required, + help, + metavar + ] = _parse_opts(arguments, { + option_strings: no_default, + dest: no_default, + nargs: undefined, + const: undefined, + default: undefined, + type: undefined, + choices: undefined, + required: false, + help: undefined, + metavar: undefined + }) + + if (nargs === 0) { + throw new TypeError('nargs for append actions must be != 0; if arg ' + + 'strings are not supplying the value to append, ' + + 'the append const action may be more appropriate') + } + if (const_value !== undefined && nargs !== OPTIONAL) { + throw new TypeError(sub('nargs must be %r to supply const', OPTIONAL)) + } + super({ + option_strings, + dest, + nargs, + const: const_value, + default: default_value, + type, + choices, + required, + help, + metavar + }) + } + + call(parser, namespace, values/*, option_string = undefined*/) { + let items = getattr(namespace, this.dest, undefined) + items = _copy_items(items) + items.push(values) + setattr(namespace, this.dest, items) + } +}) + + +const _AppendConstAction = _callable(class _AppendConstAction extends Action { + + constructor() { + let [ + option_strings, + dest, + const_value, + default_value, + required, + help, + metavar + ] = _parse_opts(arguments, { + option_strings: no_default, + dest: no_default, + const: no_default, + default: undefined, + required: false, + help: undefined, + metavar: undefined + }) + + super({ + option_strings, + dest, + nargs: 0, + const: const_value, + default: default_value, + required, + help, + metavar + }) + } + + call(parser, namespace/*, values, option_string = undefined*/) { + let items = getattr(namespace, this.dest, undefined) + items = _copy_items(items) + items.push(this.const) + setattr(namespace, this.dest, items) + } +}) + + +const _CountAction = _callable(class _CountAction extends Action { + + constructor() { + let [ + option_strings, + dest, + default_value, + required, + help + ] = _parse_opts(arguments, { + option_strings: no_default, + dest: no_default, + default: undefined, + required: false, + help: undefined + }) + + super({ + option_strings, + dest, + nargs: 0, + default: default_value, + required, + help + }) + } + + call(parser, namespace/*, values, option_string = undefined*/) { + let count = getattr(namespace, this.dest, undefined) + if (count === undefined) { + count = 0 + } + setattr(namespace, this.dest, count + 1) + } +}) + + +const _HelpAction = _callable(class _HelpAction extends Action { + + constructor() { + let [ + option_strings, + dest, + default_value, + help + ] = _parse_opts(arguments, { + option_strings: no_default, + dest: SUPPRESS, + default: SUPPRESS, + help: undefined + }) + + super({ + option_strings, + dest, + default: default_value, + nargs: 0, + help + }) + } + + call(parser/*, namespace, values, option_string = undefined*/) { + parser.print_help() + parser.exit() + } +}) + + +const _VersionAction = _callable(class _VersionAction extends Action { + + constructor() { + let [ + option_strings, + version, + dest, + default_value, + help + ] = _parse_opts(arguments, { + option_strings: no_default, + version: undefined, + dest: SUPPRESS, + default: SUPPRESS, + help: "show program's version number and exit" + }) + + super({ + option_strings, + dest, + default: default_value, + nargs: 0, + help + }) + this.version = version + } + + call(parser/*, namespace, values, option_string = undefined*/) { + let version = this.version + if (version === undefined) { + version = parser.version + } + let formatter = parser._get_formatter() + formatter.add_text(version) + parser._print_message(formatter.format_help(), process.stdout) + parser.exit() + } +}) + + +const _SubParsersAction = _camelcase_alias(_callable(class _SubParsersAction extends Action { + + constructor() { + let [ + option_strings, + prog, + parser_class, + dest, + required, + help, + metavar + ] = _parse_opts(arguments, { + option_strings: no_default, + prog: no_default, + parser_class: no_default, + dest: SUPPRESS, + required: false, + help: undefined, + metavar: undefined + }) + + let name_parser_map = {} + + super({ + option_strings, + dest, + nargs: PARSER, + choices: name_parser_map, + required, + help, + metavar + }) + + this._prog_prefix = prog + this._parser_class = parser_class + this._name_parser_map = name_parser_map + this._choices_actions = [] + } + + add_parser() { + let [ + name, + kwargs + ] = _parse_opts(arguments, { + name: no_default, + '**kwargs': no_default + }) + + // set prog from the existing prefix + if (kwargs.prog === undefined) { + kwargs.prog = sub('%s %s', this._prog_prefix, name) + } + + let aliases = getattr(kwargs, 'aliases', []) + delete kwargs.aliases + + // create a pseudo-action to hold the choice help + if ('help' in kwargs) { + let help = kwargs.help + delete kwargs.help + let choice_action = this._ChoicesPseudoAction(name, aliases, help) + this._choices_actions.push(choice_action) + } + + // create the parser and add it to the map + let parser = new this._parser_class(kwargs) + this._name_parser_map[name] = parser + + // make parser available under aliases also + for (let alias of aliases) { + this._name_parser_map[alias] = parser + } + + return parser + } + + _get_subactions() { + return this._choices_actions + } + + call(parser, namespace, values/*, option_string = undefined*/) { + let parser_name = values[0] + let arg_strings = values.slice(1) + + // set the parser name if requested + if (this.dest !== SUPPRESS) { + setattr(namespace, this.dest, parser_name) + } + + // select the parser + if (hasattr(this._name_parser_map, parser_name)) { + parser = this._name_parser_map[parser_name] + } else { + let args = {parser_name, + choices: this._name_parser_map.join(', ')} + let msg = sub('unknown parser %(parser_name)r (choices: %(choices)s)', args) + throw new ArgumentError(this, msg) + } + + // parse all the remaining options into the namespace + // store any unrecognized options on the object, so that the top + // level parser can decide what to do with them + + // In case this subparser defines new defaults, we parse them + // in a new namespace object and then update the original + // namespace for the relevant parts. + let subnamespace + [ subnamespace, arg_strings ] = parser.parse_known_args(arg_strings, undefined) + for (let [ key, value ] of Object.entries(subnamespace)) { + setattr(namespace, key, value) + } + + if (arg_strings.length) { + setdefault(namespace, _UNRECOGNIZED_ARGS_ATTR, []) + getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).push(...arg_strings) + } + } +})) + + +_SubParsersAction.prototype._ChoicesPseudoAction = _callable(class _ChoicesPseudoAction extends Action { + constructor(name, aliases, help) { + let metavar = name, dest = name + if (aliases.length) { + metavar += sub(' (%s)', aliases.join(', ')) + } + super({ option_strings: [], dest, help, metavar }) + } +}) + + +const _ExtendAction = _callable(class _ExtendAction extends _AppendAction { + call(parser, namespace, values/*, option_string = undefined*/) { + let items = getattr(namespace, this.dest, undefined) + items = _copy_items(items) + items = items.concat(values) + setattr(namespace, this.dest, items) + } +}) + + +// ============== +// Type classes +// ============== +const FileType = _callable(class FileType extends Function { + /* + * Factory for creating file object types + * + * Instances of FileType are typically passed as type= arguments to the + * ArgumentParser add_argument() method. + * + * Keyword Arguments: + * - mode -- A string indicating how the file is to be opened. Accepts the + * same values as the builtin open() function. + * - bufsize -- The file's desired buffer size. Accepts the same values as + * the builtin open() function. + * - encoding -- The file's encoding. Accepts the same values as the + * builtin open() function. + * - errors -- A string indicating how encoding and decoding errors are to + * be handled. Accepts the same value as the builtin open() function. + */ + + constructor() { + let [ + flags, + encoding, + mode, + autoClose, + emitClose, + start, + end, + highWaterMark, + fs + ] = _parse_opts(arguments, { + flags: 'r', + encoding: undefined, + mode: undefined, // 0o666 + autoClose: undefined, // true + emitClose: undefined, // false + start: undefined, // 0 + end: undefined, // Infinity + highWaterMark: undefined, // 64 * 1024 + fs: undefined + }) + + // when this class is called as a function, redirect it to .call() method of itself + super('return arguments.callee.call.apply(arguments.callee, arguments)') + + Object.defineProperty(this, 'name', { + get() { + return sub('FileType(%r)', flags) + } + }) + this._flags = flags + this._options = {} + if (encoding !== undefined) this._options.encoding = encoding + if (mode !== undefined) this._options.mode = mode + if (autoClose !== undefined) this._options.autoClose = autoClose + if (emitClose !== undefined) this._options.emitClose = emitClose + if (start !== undefined) this._options.start = start + if (end !== undefined) this._options.end = end + if (highWaterMark !== undefined) this._options.highWaterMark = highWaterMark + if (fs !== undefined) this._options.fs = fs + } + + call(string) { + // the special argument "-" means sys.std{in,out} + if (string === '-') { + if (this._flags.includes('r')) { + return process.stdin + } else if (this._flags.includes('w')) { + return process.stdout + } else { + let msg = sub('argument "-" with mode %r', this._flags) + throw new TypeError(msg) + } + } + + // all other arguments are used as file names + let fd + try { + fd = fs.openSync(string, this._flags, this._options.mode) + } catch (e) { + let args = { filename: string, error: e.message } + let message = "can't open '%(filename)s': %(error)s" + throw new ArgumentTypeError(sub(message, args)) + } + + let options = Object.assign({ fd, flags: this._flags }, this._options) + if (this._flags.includes('r')) { + return fs.createReadStream(undefined, options) + } else if (this._flags.includes('w')) { + return fs.createWriteStream(undefined, options) + } else { + let msg = sub('argument "%s" with mode %r', string, this._flags) + throw new TypeError(msg) + } + } + + [util.inspect.custom]() { + let args = [ this._flags ] + let kwargs = Object.entries(this._options).map(([ k, v ]) => { + if (k === 'mode') v = { value: v, [util.inspect.custom]() { return '0o' + this.value.toString(8) } } + return [ k, v ] + }) + let args_str = [] + .concat(args.filter(arg => arg !== -1).map(repr)) + .concat(kwargs.filter(([/*kw*/, arg]) => arg !== undefined) + .map(([kw, arg]) => sub('%s=%r', kw, arg))) + .join(', ') + return sub('%s(%s)', this.constructor.name, args_str) + } + + toString() { + return this[util.inspect.custom]() + } +}) + +// =========================== +// Optional and Positional Parsing +// =========================== +const Namespace = _callable(class Namespace extends _AttributeHolder() { + /* + * Simple object for storing attributes. + * + * Implements equality by attribute names and values, and provides a simple + * string representation. + */ + + constructor(options = {}) { + super() + Object.assign(this, options) + } +}) + +// unset string tag to mimic plain object +Namespace.prototype[Symbol.toStringTag] = undefined + + +const _ActionsContainer = _camelcase_alias(_callable(class _ActionsContainer { + + constructor() { + let [ + description, + prefix_chars, + argument_default, + conflict_handler + ] = _parse_opts(arguments, { + description: no_default, + prefix_chars: no_default, + argument_default: no_default, + conflict_handler: no_default + }) + + this.description = description + this.argument_default = argument_default + this.prefix_chars = prefix_chars + this.conflict_handler = conflict_handler + + // set up registries + this._registries = {} + + // register actions + this.register('action', undefined, _StoreAction) + this.register('action', 'store', _StoreAction) + this.register('action', 'store_const', _StoreConstAction) + this.register('action', 'store_true', _StoreTrueAction) + this.register('action', 'store_false', _StoreFalseAction) + this.register('action', 'append', _AppendAction) + this.register('action', 'append_const', _AppendConstAction) + this.register('action', 'count', _CountAction) + this.register('action', 'help', _HelpAction) + this.register('action', 'version', _VersionAction) + this.register('action', 'parsers', _SubParsersAction) + this.register('action', 'extend', _ExtendAction) + // LEGACY (v1 compatibility): camelcase variants + ;[ 'storeConst', 'storeTrue', 'storeFalse', 'appendConst' ].forEach(old_name => { + let new_name = _to_new_name(old_name) + this.register('action', old_name, util.deprecate(this._registry_get('action', new_name), + sub('{action: "%s"} is renamed to {action: "%s"}', old_name, new_name))) + }) + // end + + // raise an exception if the conflict handler is invalid + this._get_handler() + + // action storage + this._actions = [] + this._option_string_actions = {} + + // groups + this._action_groups = [] + this._mutually_exclusive_groups = [] + + // defaults storage + this._defaults = {} + + // determines whether an "option" looks like a negative number + this._negative_number_matcher = /^-\d+$|^-\d*\.\d+$/ + + // whether or not there are any optionals that look like negative + // numbers -- uses a list so it can be shared and edited + this._has_negative_number_optionals = [] + } + + // ==================== + // Registration methods + // ==================== + register(registry_name, value, object) { + let registry = setdefault(this._registries, registry_name, {}) + registry[value] = object + } + + _registry_get(registry_name, value, default_value = undefined) { + return getattr(this._registries[registry_name], value, default_value) + } + + // ================================== + // Namespace default accessor methods + // ================================== + set_defaults(kwargs) { + Object.assign(this._defaults, kwargs) + + // if these defaults match any existing arguments, replace + // the previous default on the object with the new one + for (let action of this._actions) { + if (action.dest in kwargs) { + action.default = kwargs[action.dest] + } + } + } + + get_default(dest) { + for (let action of this._actions) { + if (action.dest === dest && action.default !== undefined) { + return action.default + } + } + return this._defaults[dest] + } + + + // ======================= + // Adding argument actions + // ======================= + add_argument() { + /* + * add_argument(dest, ..., name=value, ...) + * add_argument(option_string, option_string, ..., name=value, ...) + */ + let [ + args, + kwargs + ] = _parse_opts(arguments, { + '*args': no_default, + '**kwargs': no_default + }) + // LEGACY (v1 compatibility), old-style add_argument([ args ], { options }) + if (args.length === 1 && Array.isArray(args[0])) { + args = args[0] + deprecate('argument-array', + sub('use add_argument(%(args)s, {...}) instead of add_argument([ %(args)s ], { ... })', { + args: args.map(repr).join(', ') + })) + } + // end + + // if no positional args are supplied or only one is supplied and + // it doesn't look like an option string, parse a positional + // argument + let chars = this.prefix_chars + if (!args.length || args.length === 1 && !chars.includes(args[0][0])) { + if (args.length && 'dest' in kwargs) { + throw new TypeError('dest supplied twice for positional argument') + } + kwargs = this._get_positional_kwargs(...args, kwargs) + + // otherwise, we're adding an optional argument + } else { + kwargs = this._get_optional_kwargs(...args, kwargs) + } + + // if no default was supplied, use the parser-level default + if (!('default' in kwargs)) { + let dest = kwargs.dest + if (dest in this._defaults) { + kwargs.default = this._defaults[dest] + } else if (this.argument_default !== undefined) { + kwargs.default = this.argument_default + } + } + + // create the action object, and add it to the parser + let action_class = this._pop_action_class(kwargs) + if (typeof action_class !== 'function') { + throw new TypeError(sub('unknown action "%s"', action_class)) + } + // eslint-disable-next-line new-cap + let action = new action_class(kwargs) + + // raise an error if the action type is not callable + let type_func = this._registry_get('type', action.type, action.type) + if (typeof type_func !== 'function') { + throw new TypeError(sub('%r is not callable', type_func)) + } + + if (type_func === FileType) { + throw new TypeError(sub('%r is a FileType class object, instance of it' + + ' must be passed', type_func)) + } + + // raise an error if the metavar does not match the type + if ('_get_formatter' in this) { + try { + this._get_formatter()._format_args(action, undefined) + } catch (err) { + // check for 'invalid nargs value' is an artifact of TypeError and ValueError in js being the same + if (err instanceof TypeError && err.message !== 'invalid nargs value') { + throw new TypeError('length of metavar tuple does not match nargs') + } else { + throw err + } + } + } + + return this._add_action(action) + } + + add_argument_group() { + let group = _ArgumentGroup(this, ...arguments) + this._action_groups.push(group) + return group + } + + add_mutually_exclusive_group() { + // eslint-disable-next-line no-use-before-define + let group = _MutuallyExclusiveGroup(this, ...arguments) + this._mutually_exclusive_groups.push(group) + return group + } + + _add_action(action) { + // resolve any conflicts + this._check_conflict(action) + + // add to actions list + this._actions.push(action) + action.container = this + + // index the action by any option strings it has + for (let option_string of action.option_strings) { + this._option_string_actions[option_string] = action + } + + // set the flag if any option strings look like negative numbers + for (let option_string of action.option_strings) { + if (this._negative_number_matcher.test(option_string)) { + if (!this._has_negative_number_optionals.length) { + this._has_negative_number_optionals.push(true) + } + } + } + + // return the created action + return action + } + + _remove_action(action) { + _array_remove(this._actions, action) + } + + _add_container_actions(container) { + // collect groups by titles + let title_group_map = {} + for (let group of this._action_groups) { + if (group.title in title_group_map) { + let msg = 'cannot merge actions - two groups are named %r' + throw new TypeError(sub(msg, group.title)) + } + title_group_map[group.title] = group + } + + // map each action to its group + let group_map = new Map() + for (let group of container._action_groups) { + + // if a group with the title exists, use that, otherwise + // create a new group matching the container's group + if (!(group.title in title_group_map)) { + title_group_map[group.title] = this.add_argument_group({ + title: group.title, + description: group.description, + conflict_handler: group.conflict_handler + }) + } + + // map the actions to their new group + for (let action of group._group_actions) { + group_map.set(action, title_group_map[group.title]) + } + } + + // add container's mutually exclusive groups + // NOTE: if add_mutually_exclusive_group ever gains title= and + // description= then this code will need to be expanded as above + for (let group of container._mutually_exclusive_groups) { + let mutex_group = this.add_mutually_exclusive_group({ + required: group.required + }) + + // map the actions to their new mutex group + for (let action of group._group_actions) { + group_map.set(action, mutex_group) + } + } + + // add all actions to this container or their group + for (let action of container._actions) { + group_map.get(action)._add_action(action) + } + } + + _get_positional_kwargs() { + let [ + dest, + kwargs + ] = _parse_opts(arguments, { + dest: no_default, + '**kwargs': no_default + }) + + // make sure required is not specified + if ('required' in kwargs) { + let msg = "'required' is an invalid argument for positionals" + throw new TypeError(msg) + } + + // mark positional arguments as required if at least one is + // always required + if (![OPTIONAL, ZERO_OR_MORE].includes(kwargs.nargs)) { + kwargs.required = true + } + if (kwargs.nargs === ZERO_OR_MORE && !('default' in kwargs)) { + kwargs.required = true + } + + // return the keyword arguments with no option strings + return Object.assign(kwargs, { dest, option_strings: [] }) + } + + _get_optional_kwargs() { + let [ + args, + kwargs + ] = _parse_opts(arguments, { + '*args': no_default, + '**kwargs': no_default + }) + + // determine short and long option strings + let option_strings = [] + let long_option_strings = [] + let option_string + for (option_string of args) { + // error on strings that don't start with an appropriate prefix + if (!this.prefix_chars.includes(option_string[0])) { + let args = {option: option_string, + prefix_chars: this.prefix_chars} + let msg = 'invalid option string %(option)r: ' + + 'must start with a character %(prefix_chars)r' + throw new TypeError(sub(msg, args)) + } + + // strings starting with two prefix characters are long options + option_strings.push(option_string) + if (option_string.length > 1 && this.prefix_chars.includes(option_string[1])) { + long_option_strings.push(option_string) + } + } + + // infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x' + let dest = kwargs.dest + delete kwargs.dest + if (dest === undefined) { + let dest_option_string + if (long_option_strings.length) { + dest_option_string = long_option_strings[0] + } else { + dest_option_string = option_strings[0] + } + dest = _string_lstrip(dest_option_string, this.prefix_chars) + if (!dest) { + let msg = 'dest= is required for options like %r' + throw new TypeError(sub(msg, option_string)) + } + dest = dest.replace(/-/g, '_') + } + + // return the updated keyword arguments + return Object.assign(kwargs, { dest, option_strings }) + } + + _pop_action_class(kwargs, default_value = undefined) { + let action = getattr(kwargs, 'action', default_value) + delete kwargs.action + return this._registry_get('action', action, action) + } + + _get_handler() { + // determine function from conflict handler string + let handler_func_name = sub('_handle_conflict_%s', this.conflict_handler) + if (typeof this[handler_func_name] === 'function') { + return this[handler_func_name] + } else { + let msg = 'invalid conflict_resolution value: %r' + throw new TypeError(sub(msg, this.conflict_handler)) + } + } + + _check_conflict(action) { + + // find all options that conflict with this option + let confl_optionals = [] + for (let option_string of action.option_strings) { + if (hasattr(this._option_string_actions, option_string)) { + let confl_optional = this._option_string_actions[option_string] + confl_optionals.push([ option_string, confl_optional ]) + } + } + + // resolve any conflicts + if (confl_optionals.length) { + let conflict_handler = this._get_handler() + conflict_handler.call(this, action, confl_optionals) + } + } + + _handle_conflict_error(action, conflicting_actions) { + let message = conflicting_actions.length === 1 ? + 'conflicting option string: %s' : + 'conflicting option strings: %s' + let conflict_string = conflicting_actions.map(([ option_string/*, action*/ ]) => option_string).join(', ') + throw new ArgumentError(action, sub(message, conflict_string)) + } + + _handle_conflict_resolve(action, conflicting_actions) { + + // remove all conflicting options + for (let [ option_string, action ] of conflicting_actions) { + + // remove the conflicting option + _array_remove(action.option_strings, option_string) + delete this._option_string_actions[option_string] + + // if the option now has no option string, remove it from the + // container holding it + if (!action.option_strings.length) { + action.container._remove_action(action) + } + } + } +})) + + +const _ArgumentGroup = _callable(class _ArgumentGroup extends _ActionsContainer { + + constructor() { + let [ + container, + title, + description, + kwargs + ] = _parse_opts(arguments, { + container: no_default, + title: undefined, + description: undefined, + '**kwargs': no_default + }) + + // add any missing keyword arguments by checking the container + setdefault(kwargs, 'conflict_handler', container.conflict_handler) + setdefault(kwargs, 'prefix_chars', container.prefix_chars) + setdefault(kwargs, 'argument_default', container.argument_default) + super(Object.assign({ description }, kwargs)) + + // group attributes + this.title = title + this._group_actions = [] + + // share most attributes with the container + this._registries = container._registries + this._actions = container._actions + this._option_string_actions = container._option_string_actions + this._defaults = container._defaults + this._has_negative_number_optionals = + container._has_negative_number_optionals + this._mutually_exclusive_groups = container._mutually_exclusive_groups + } + + _add_action(action) { + action = super._add_action(action) + this._group_actions.push(action) + return action + } + + _remove_action(action) { + super._remove_action(action) + _array_remove(this._group_actions, action) + } +}) + + +const _MutuallyExclusiveGroup = _callable(class _MutuallyExclusiveGroup extends _ArgumentGroup { + + constructor() { + let [ + container, + required + ] = _parse_opts(arguments, { + container: no_default, + required: false + }) + + super(container) + this.required = required + this._container = container + } + + _add_action(action) { + if (action.required) { + let msg = 'mutually exclusive arguments must be optional' + throw new TypeError(msg) + } + action = this._container._add_action(action) + this._group_actions.push(action) + return action + } + + _remove_action(action) { + this._container._remove_action(action) + _array_remove(this._group_actions, action) + } +}) + + +const ArgumentParser = _camelcase_alias(_callable(class ArgumentParser extends _AttributeHolder(_ActionsContainer) { + /* + * Object for parsing command line strings into Python objects. + * + * Keyword Arguments: + * - prog -- The name of the program (default: sys.argv[0]) + * - usage -- A usage message (default: auto-generated from arguments) + * - description -- A description of what the program does + * - epilog -- Text following the argument descriptions + * - parents -- Parsers whose arguments should be copied into this one + * - formatter_class -- HelpFormatter class for printing help messages + * - prefix_chars -- Characters that prefix optional arguments + * - fromfile_prefix_chars -- Characters that prefix files containing + * additional arguments + * - argument_default -- The default value for all arguments + * - conflict_handler -- String indicating how to handle conflicts + * - add_help -- Add a -h/-help option + * - allow_abbrev -- Allow long options to be abbreviated unambiguously + * - exit_on_error -- Determines whether or not ArgumentParser exits with + * error info when an error occurs + */ + + constructor() { + let [ + prog, + usage, + description, + epilog, + parents, + formatter_class, + prefix_chars, + fromfile_prefix_chars, + argument_default, + conflict_handler, + add_help, + allow_abbrev, + exit_on_error, + debug, // LEGACY (v1 compatibility), debug mode + version // LEGACY (v1 compatibility), version + ] = _parse_opts(arguments, { + prog: undefined, + usage: undefined, + description: undefined, + epilog: undefined, + parents: [], + formatter_class: HelpFormatter, + prefix_chars: '-', + fromfile_prefix_chars: undefined, + argument_default: undefined, + conflict_handler: 'error', + add_help: true, + allow_abbrev: true, + exit_on_error: true, + debug: undefined, // LEGACY (v1 compatibility), debug mode + version: undefined // LEGACY (v1 compatibility), version + }) + + // LEGACY (v1 compatibility) + if (debug !== undefined) { + deprecate('debug', + 'The "debug" argument to ArgumentParser is deprecated. Please ' + + 'override ArgumentParser.exit function instead.' + ) + } + + if (version !== undefined) { + deprecate('version', + 'The "version" argument to ArgumentParser is deprecated. Please use ' + + "add_argument(..., { action: 'version', version: 'N', ... }) instead." + ) + } + // end + + super({ + description, + prefix_chars, + argument_default, + conflict_handler + }) + + // default setting for prog + if (prog === undefined) { + prog = path.basename(get_argv()[0] || '') + } + + this.prog = prog + this.usage = usage + this.epilog = epilog + this.formatter_class = formatter_class + this.fromfile_prefix_chars = fromfile_prefix_chars + this.add_help = add_help + this.allow_abbrev = allow_abbrev + this.exit_on_error = exit_on_error + // LEGACY (v1 compatibility), debug mode + this.debug = debug + // end + + this._positionals = this.add_argument_group('positional arguments') + this._optionals = this.add_argument_group('optional arguments') + this._subparsers = undefined + + // register types + function identity(string) { + return string + } + this.register('type', undefined, identity) + this.register('type', null, identity) + this.register('type', 'auto', identity) + this.register('type', 'int', function (x) { + let result = Number(x) + if (!Number.isInteger(result)) { + throw new TypeError(sub('could not convert string to int: %r', x)) + } + return result + }) + this.register('type', 'float', function (x) { + let result = Number(x) + if (isNaN(result)) { + throw new TypeError(sub('could not convert string to float: %r', x)) + } + return result + }) + this.register('type', 'str', String) + // LEGACY (v1 compatibility): custom types + this.register('type', 'string', + util.deprecate(String, 'use {type:"str"} or {type:String} instead of {type:"string"}')) + // end + + // add help argument if necessary + // (using explicit default to override global argument_default) + let default_prefix = prefix_chars.includes('-') ? '-' : prefix_chars[0] + if (this.add_help) { + this.add_argument( + default_prefix + 'h', + default_prefix.repeat(2) + 'help', + { + action: 'help', + default: SUPPRESS, + help: 'show this help message and exit' + } + ) + } + // LEGACY (v1 compatibility), version + if (version) { + this.add_argument( + default_prefix + 'v', + default_prefix.repeat(2) + 'version', + { + action: 'version', + default: SUPPRESS, + version: this.version, + help: "show program's version number and exit" + } + ) + } + // end + + // add parent arguments and defaults + for (let parent of parents) { + this._add_container_actions(parent) + Object.assign(this._defaults, parent._defaults) + } + } + + // ======================= + // Pretty __repr__ methods + // ======================= + _get_kwargs() { + let names = [ + 'prog', + 'usage', + 'description', + 'formatter_class', + 'conflict_handler', + 'add_help' + ] + return names.map(name => [ name, getattr(this, name) ]) + } + + // ================================== + // Optional/Positional adding methods + // ================================== + add_subparsers() { + let [ + kwargs + ] = _parse_opts(arguments, { + '**kwargs': no_default + }) + + if (this._subparsers !== undefined) { + this.error('cannot have multiple subparser arguments') + } + + // add the parser class to the arguments if it's not present + setdefault(kwargs, 'parser_class', this.constructor) + + if ('title' in kwargs || 'description' in kwargs) { + let title = getattr(kwargs, 'title', 'subcommands') + let description = getattr(kwargs, 'description', undefined) + delete kwargs.title + delete kwargs.description + this._subparsers = this.add_argument_group(title, description) + } else { + this._subparsers = this._positionals + } + + // prog defaults to the usage message of this parser, skipping + // optional arguments and with no "usage:" prefix + if (kwargs.prog === undefined) { + let formatter = this._get_formatter() + let positionals = this._get_positional_actions() + let groups = this._mutually_exclusive_groups + formatter.add_usage(this.usage, positionals, groups, '') + kwargs.prog = formatter.format_help().trim() + } + + // create the parsers action and add it to the positionals list + let parsers_class = this._pop_action_class(kwargs, 'parsers') + // eslint-disable-next-line new-cap + let action = new parsers_class(Object.assign({ option_strings: [] }, kwargs)) + this._subparsers._add_action(action) + + // return the created parsers action + return action + } + + _add_action(action) { + if (action.option_strings.length) { + this._optionals._add_action(action) + } else { + this._positionals._add_action(action) + } + return action + } + + _get_optional_actions() { + return this._actions.filter(action => action.option_strings.length) + } + + _get_positional_actions() { + return this._actions.filter(action => !action.option_strings.length) + } + + // ===================================== + // Command line argument parsing methods + // ===================================== + parse_args(args = undefined, namespace = undefined) { + let argv + [ args, argv ] = this.parse_known_args(args, namespace) + if (argv && argv.length > 0) { + let msg = 'unrecognized arguments: %s' + this.error(sub(msg, argv.join(' '))) + } + return args + } + + parse_known_args(args = undefined, namespace = undefined) { + if (args === undefined) { + args = get_argv().slice(1) + } + + // default Namespace built from parser defaults + if (namespace === undefined) { + namespace = new Namespace() + } + + // add any action defaults that aren't present + for (let action of this._actions) { + if (action.dest !== SUPPRESS) { + if (!hasattr(namespace, action.dest)) { + if (action.default !== SUPPRESS) { + setattr(namespace, action.dest, action.default) + } + } + } + } + + // add any parser defaults that aren't present + for (let dest of Object.keys(this._defaults)) { + if (!hasattr(namespace, dest)) { + setattr(namespace, dest, this._defaults[dest]) + } + } + + // parse the arguments and exit if there are any errors + if (this.exit_on_error) { + try { + [ namespace, args ] = this._parse_known_args(args, namespace) + } catch (err) { + if (err instanceof ArgumentError) { + this.error(err.message) + } else { + throw err + } + } + } else { + [ namespace, args ] = this._parse_known_args(args, namespace) + } + + if (hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR)) { + args = args.concat(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR)) + delattr(namespace, _UNRECOGNIZED_ARGS_ATTR) + } + + return [ namespace, args ] + } + + _parse_known_args(arg_strings, namespace) { + // replace arg strings that are file references + if (this.fromfile_prefix_chars !== undefined) { + arg_strings = this._read_args_from_files(arg_strings) + } + + // map all mutually exclusive arguments to the other arguments + // they can't occur with + let action_conflicts = new Map() + for (let mutex_group of this._mutually_exclusive_groups) { + let group_actions = mutex_group._group_actions + for (let [ i, mutex_action ] of Object.entries(mutex_group._group_actions)) { + let conflicts = action_conflicts.get(mutex_action) || [] + conflicts = conflicts.concat(group_actions.slice(0, +i)) + conflicts = conflicts.concat(group_actions.slice(+i + 1)) + action_conflicts.set(mutex_action, conflicts) + } + } + + // find all option indices, and determine the arg_string_pattern + // which has an 'O' if there is an option at an index, + // an 'A' if there is an argument, or a '-' if there is a '--' + let option_string_indices = {} + let arg_string_pattern_parts = [] + let arg_strings_iter = Object.entries(arg_strings)[Symbol.iterator]() + for (let [ i, arg_string ] of arg_strings_iter) { + + // all args after -- are non-options + if (arg_string === '--') { + arg_string_pattern_parts.push('-') + for ([ i, arg_string ] of arg_strings_iter) { + arg_string_pattern_parts.push('A') + } + + // otherwise, add the arg to the arg strings + // and note the index if it was an option + } else { + let option_tuple = this._parse_optional(arg_string) + let pattern + if (option_tuple === undefined) { + pattern = 'A' + } else { + option_string_indices[i] = option_tuple + pattern = 'O' + } + arg_string_pattern_parts.push(pattern) + } + } + + // join the pieces together to form the pattern + let arg_strings_pattern = arg_string_pattern_parts.join('') + + // converts arg strings to the appropriate and then takes the action + let seen_actions = new Set() + let seen_non_default_actions = new Set() + let extras + + let take_action = (action, argument_strings, option_string = undefined) => { + seen_actions.add(action) + let argument_values = this._get_values(action, argument_strings) + + // error if this argument is not allowed with other previously + // seen arguments, assuming that actions that use the default + // value don't really count as "present" + if (argument_values !== action.default) { + seen_non_default_actions.add(action) + for (let conflict_action of action_conflicts.get(action) || []) { + if (seen_non_default_actions.has(conflict_action)) { + let msg = 'not allowed with argument %s' + let action_name = _get_action_name(conflict_action) + throw new ArgumentError(action, sub(msg, action_name)) + } + } + } + + // take the action if we didn't receive a SUPPRESS value + // (e.g. from a default) + if (argument_values !== SUPPRESS) { + action(this, namespace, argument_values, option_string) + } + } + + // function to convert arg_strings into an optional action + let consume_optional = start_index => { + + // get the optional identified at this index + let option_tuple = option_string_indices[start_index] + let [ action, option_string, explicit_arg ] = option_tuple + + // identify additional optionals in the same arg string + // (e.g. -xyz is the same as -x -y -z if no args are required) + let action_tuples = [] + let stop + for (;;) { + + // if we found no optional action, skip it + if (action === undefined) { + extras.push(arg_strings[start_index]) + return start_index + 1 + } + + // if there is an explicit argument, try to match the + // optional's string arguments to only this + if (explicit_arg !== undefined) { + let arg_count = this._match_argument(action, 'A') + + // if the action is a single-dash option and takes no + // arguments, try to parse more single-dash options out + // of the tail of the option string + let chars = this.prefix_chars + if (arg_count === 0 && !chars.includes(option_string[1])) { + action_tuples.push([ action, [], option_string ]) + let char = option_string[0] + option_string = char + explicit_arg[0] + let new_explicit_arg = explicit_arg.slice(1) || undefined + let optionals_map = this._option_string_actions + if (hasattr(optionals_map, option_string)) { + action = optionals_map[option_string] + explicit_arg = new_explicit_arg + } else { + let msg = 'ignored explicit argument %r' + throw new ArgumentError(action, sub(msg, explicit_arg)) + } + + // if the action expect exactly one argument, we've + // successfully matched the option; exit the loop + } else if (arg_count === 1) { + stop = start_index + 1 + let args = [ explicit_arg ] + action_tuples.push([ action, args, option_string ]) + break + + // error if a double-dash option did not use the + // explicit argument + } else { + let msg = 'ignored explicit argument %r' + throw new ArgumentError(action, sub(msg, explicit_arg)) + } + + // if there is no explicit argument, try to match the + // optional's string arguments with the following strings + // if successful, exit the loop + } else { + let start = start_index + 1 + let selected_patterns = arg_strings_pattern.slice(start) + let arg_count = this._match_argument(action, selected_patterns) + stop = start + arg_count + let args = arg_strings.slice(start, stop) + action_tuples.push([ action, args, option_string ]) + break + } + } + + // add the Optional to the list and return the index at which + // the Optional's string args stopped + assert(action_tuples.length) + for (let [ action, args, option_string ] of action_tuples) { + take_action(action, args, option_string) + } + return stop + } + + // the list of Positionals left to be parsed; this is modified + // by consume_positionals() + let positionals = this._get_positional_actions() + + // function to convert arg_strings into positional actions + let consume_positionals = start_index => { + // match as many Positionals as possible + let selected_pattern = arg_strings_pattern.slice(start_index) + let arg_counts = this._match_arguments_partial(positionals, selected_pattern) + + // slice off the appropriate arg strings for each Positional + // and add the Positional and its args to the list + for (let i = 0; i < positionals.length && i < arg_counts.length; i++) { + let action = positionals[i] + let arg_count = arg_counts[i] + let args = arg_strings.slice(start_index, start_index + arg_count) + start_index += arg_count + take_action(action, args) + } + + // slice off the Positionals that we just parsed and return the + // index at which the Positionals' string args stopped + positionals = positionals.slice(arg_counts.length) + return start_index + } + + // consume Positionals and Optionals alternately, until we have + // passed the last option string + extras = [] + let start_index = 0 + let max_option_string_index = Math.max(-1, ...Object.keys(option_string_indices).map(Number)) + while (start_index <= max_option_string_index) { + + // consume any Positionals preceding the next option + let next_option_string_index = Math.min( + // eslint-disable-next-line no-loop-func + ...Object.keys(option_string_indices).map(Number).filter(index => index >= start_index) + ) + if (start_index !== next_option_string_index) { + let positionals_end_index = consume_positionals(start_index) + + // only try to parse the next optional if we didn't consume + // the option string during the positionals parsing + if (positionals_end_index > start_index) { + start_index = positionals_end_index + continue + } else { + start_index = positionals_end_index + } + } + + // if we consumed all the positionals we could and we're not + // at the index of an option string, there were extra arguments + if (!(start_index in option_string_indices)) { + let strings = arg_strings.slice(start_index, next_option_string_index) + extras = extras.concat(strings) + start_index = next_option_string_index + } + + // consume the next optional and any arguments for it + start_index = consume_optional(start_index) + } + + // consume any positionals following the last Optional + let stop_index = consume_positionals(start_index) + + // if we didn't consume all the argument strings, there were extras + extras = extras.concat(arg_strings.slice(stop_index)) + + // make sure all required actions were present and also convert + // action defaults which were not given as arguments + let required_actions = [] + for (let action of this._actions) { + if (!seen_actions.has(action)) { + if (action.required) { + required_actions.push(_get_action_name(action)) + } else { + // Convert action default now instead of doing it before + // parsing arguments to avoid calling convert functions + // twice (which may fail) if the argument was given, but + // only if it was defined already in the namespace + if (action.default !== undefined && + typeof action.default === 'string' && + hasattr(namespace, action.dest) && + action.default === getattr(namespace, action.dest)) { + setattr(namespace, action.dest, + this._get_value(action, action.default)) + } + } + } + } + + if (required_actions.length) { + this.error(sub('the following arguments are required: %s', + required_actions.join(', '))) + } + + // make sure all required groups had one option present + for (let group of this._mutually_exclusive_groups) { + if (group.required) { + let no_actions_used = true + for (let action of group._group_actions) { + if (seen_non_default_actions.has(action)) { + no_actions_used = false + break + } + } + + // if no actions were used, report the error + if (no_actions_used) { + let names = group._group_actions + .filter(action => action.help !== SUPPRESS) + .map(action => _get_action_name(action)) + let msg = 'one of the arguments %s is required' + this.error(sub(msg, names.join(' '))) + } + } + } + + // return the updated namespace and the extra arguments + return [ namespace, extras ] + } + + _read_args_from_files(arg_strings) { + // expand arguments referencing files + let new_arg_strings = [] + for (let arg_string of arg_strings) { + + // for regular arguments, just add them back into the list + if (!arg_string || !this.fromfile_prefix_chars.includes(arg_string[0])) { + new_arg_strings.push(arg_string) + + // replace arguments referencing files with the file content + } else { + try { + let args_file = fs.readFileSync(arg_string.slice(1), 'utf8') + let arg_strings = [] + for (let arg_line of splitlines(args_file)) { + for (let arg of this.convert_arg_line_to_args(arg_line)) { + arg_strings.push(arg) + } + } + arg_strings = this._read_args_from_files(arg_strings) + new_arg_strings = new_arg_strings.concat(arg_strings) + } catch (err) { + this.error(err.message) + } + } + } + + // return the modified argument list + return new_arg_strings + } + + convert_arg_line_to_args(arg_line) { + return [arg_line] + } + + _match_argument(action, arg_strings_pattern) { + // match the pattern for this action to the arg strings + let nargs_pattern = this._get_nargs_pattern(action) + let match = arg_strings_pattern.match(new RegExp('^' + nargs_pattern)) + + // raise an exception if we weren't able to find a match + if (match === null) { + let nargs_errors = { + undefined: 'expected one argument', + [OPTIONAL]: 'expected at most one argument', + [ONE_OR_MORE]: 'expected at least one argument' + } + let msg = nargs_errors[action.nargs] + if (msg === undefined) { + msg = sub(action.nargs === 1 ? 'expected %s argument' : 'expected %s arguments', action.nargs) + } + throw new ArgumentError(action, msg) + } + + // return the number of arguments matched + return match[1].length + } + + _match_arguments_partial(actions, arg_strings_pattern) { + // progressively shorten the actions list by slicing off the + // final actions until we find a match + let result = [] + for (let i of range(actions.length, 0, -1)) { + let actions_slice = actions.slice(0, i) + let pattern = actions_slice.map(action => this._get_nargs_pattern(action)).join('') + let match = arg_strings_pattern.match(new RegExp('^' + pattern)) + if (match !== null) { + result = result.concat(match.slice(1).map(string => string.length)) + break + } + } + + // return the list of arg string counts + return result + } + + _parse_optional(arg_string) { + // if it's an empty string, it was meant to be a positional + if (!arg_string) { + return undefined + } + + // if it doesn't start with a prefix, it was meant to be positional + if (!this.prefix_chars.includes(arg_string[0])) { + return undefined + } + + // if the option string is present in the parser, return the action + if (arg_string in this._option_string_actions) { + let action = this._option_string_actions[arg_string] + return [ action, arg_string, undefined ] + } + + // if it's just a single character, it was meant to be positional + if (arg_string.length === 1) { + return undefined + } + + // if the option string before the "=" is present, return the action + if (arg_string.includes('=')) { + let [ option_string, explicit_arg ] = _string_split(arg_string, '=', 1) + if (option_string in this._option_string_actions) { + let action = this._option_string_actions[option_string] + return [ action, option_string, explicit_arg ] + } + } + + // search through all possible prefixes of the option string + // and all actions in the parser for possible interpretations + let option_tuples = this._get_option_tuples(arg_string) + + // if multiple actions match, the option string was ambiguous + if (option_tuples.length > 1) { + let options = option_tuples.map(([ /*action*/, option_string/*, explicit_arg*/ ]) => option_string).join(', ') + let args = {option: arg_string, matches: options} + let msg = 'ambiguous option: %(option)s could match %(matches)s' + this.error(sub(msg, args)) + + // if exactly one action matched, this segmentation is good, + // so return the parsed action + } else if (option_tuples.length === 1) { + let [ option_tuple ] = option_tuples + return option_tuple + } + + // if it was not found as an option, but it looks like a negative + // number, it was meant to be positional + // unless there are negative-number-like options + if (this._negative_number_matcher.test(arg_string)) { + if (!this._has_negative_number_optionals.length) { + return undefined + } + } + + // if it contains a space, it was meant to be a positional + if (arg_string.includes(' ')) { + return undefined + } + + // it was meant to be an optional but there is no such option + // in this parser (though it might be a valid option in a subparser) + return [ undefined, arg_string, undefined ] + } + + _get_option_tuples(option_string) { + let result = [] + + // option strings starting with two prefix characters are only + // split at the '=' + let chars = this.prefix_chars + if (chars.includes(option_string[0]) && chars.includes(option_string[1])) { + if (this.allow_abbrev) { + let option_prefix, explicit_arg + if (option_string.includes('=')) { + [ option_prefix, explicit_arg ] = _string_split(option_string, '=', 1) + } else { + option_prefix = option_string + explicit_arg = undefined + } + for (let option_string of Object.keys(this._option_string_actions)) { + if (option_string.startsWith(option_prefix)) { + let action = this._option_string_actions[option_string] + let tup = [ action, option_string, explicit_arg ] + result.push(tup) + } + } + } + + // single character options can be concatenated with their arguments + // but multiple character options always have to have their argument + // separate + } else if (chars.includes(option_string[0]) && !chars.includes(option_string[1])) { + let option_prefix = option_string + let explicit_arg = undefined + let short_option_prefix = option_string.slice(0, 2) + let short_explicit_arg = option_string.slice(2) + + for (let option_string of Object.keys(this._option_string_actions)) { + if (option_string === short_option_prefix) { + let action = this._option_string_actions[option_string] + let tup = [ action, option_string, short_explicit_arg ] + result.push(tup) + } else if (option_string.startsWith(option_prefix)) { + let action = this._option_string_actions[option_string] + let tup = [ action, option_string, explicit_arg ] + result.push(tup) + } + } + + // shouldn't ever get here + } else { + this.error(sub('unexpected option string: %s', option_string)) + } + + // return the collected option tuples + return result + } + + _get_nargs_pattern(action) { + // in all examples below, we have to allow for '--' args + // which are represented as '-' in the pattern + let nargs = action.nargs + let nargs_pattern + + // the default (None) is assumed to be a single argument + if (nargs === undefined) { + nargs_pattern = '(-*A-*)' + + // allow zero or one arguments + } else if (nargs === OPTIONAL) { + nargs_pattern = '(-*A?-*)' + + // allow zero or more arguments + } else if (nargs === ZERO_OR_MORE) { + nargs_pattern = '(-*[A-]*)' + + // allow one or more arguments + } else if (nargs === ONE_OR_MORE) { + nargs_pattern = '(-*A[A-]*)' + + // allow any number of options or arguments + } else if (nargs === REMAINDER) { + nargs_pattern = '([-AO]*)' + + // allow one argument followed by any number of options or arguments + } else if (nargs === PARSER) { + nargs_pattern = '(-*A[-AO]*)' + + // suppress action, like nargs=0 + } else if (nargs === SUPPRESS) { + nargs_pattern = '(-*-*)' + + // all others should be integers + } else { + nargs_pattern = sub('(-*%s-*)', 'A'.repeat(nargs).split('').join('-*')) + } + + // if this is an optional action, -- is not allowed + if (action.option_strings.length) { + nargs_pattern = nargs_pattern.replace(/-\*/g, '') + nargs_pattern = nargs_pattern.replace(/-/g, '') + } + + // return the pattern + return nargs_pattern + } + + // ======================== + // Alt command line argument parsing, allowing free intermix + // ======================== + + parse_intermixed_args(args = undefined, namespace = undefined) { + let argv + [ args, argv ] = this.parse_known_intermixed_args(args, namespace) + if (argv.length) { + let msg = 'unrecognized arguments: %s' + this.error(sub(msg, argv.join(' '))) + } + return args + } + + parse_known_intermixed_args(args = undefined, namespace = undefined) { + // returns a namespace and list of extras + // + // positional can be freely intermixed with optionals. optionals are + // first parsed with all positional arguments deactivated. The 'extras' + // are then parsed. If the parser definition is incompatible with the + // intermixed assumptions (e.g. use of REMAINDER, subparsers) a + // TypeError is raised. + // + // positionals are 'deactivated' by setting nargs and default to + // SUPPRESS. This blocks the addition of that positional to the + // namespace + + let extras + let positionals = this._get_positional_actions() + let a = positionals.filter(action => [ PARSER, REMAINDER ].includes(action.nargs)) + if (a.length) { + throw new TypeError(sub('parse_intermixed_args: positional arg' + + ' with nargs=%s', a[0].nargs)) + } + + for (let group of this._mutually_exclusive_groups) { + for (let action of group._group_actions) { + if (positionals.includes(action)) { + throw new TypeError('parse_intermixed_args: positional in' + + ' mutuallyExclusiveGroup') + } + } + } + + let save_usage + try { + save_usage = this.usage + let remaining_args + try { + if (this.usage === undefined) { + // capture the full usage for use in error messages + this.usage = this.format_usage().slice(7) + } + for (let action of positionals) { + // deactivate positionals + action.save_nargs = action.nargs + // action.nargs = 0 + action.nargs = SUPPRESS + action.save_default = action.default + action.default = SUPPRESS + } + [ namespace, remaining_args ] = this.parse_known_args(args, + namespace) + for (let action of positionals) { + // remove the empty positional values from namespace + let attr = getattr(namespace, action.dest) + if (Array.isArray(attr) && attr.length === 0) { + // eslint-disable-next-line no-console + console.warn(sub('Do not expect %s in %s', action.dest, namespace)) + delattr(namespace, action.dest) + } + } + } finally { + // restore nargs and usage before exiting + for (let action of positionals) { + action.nargs = action.save_nargs + action.default = action.save_default + } + } + let optionals = this._get_optional_actions() + try { + // parse positionals. optionals aren't normally required, but + // they could be, so make sure they aren't. + for (let action of optionals) { + action.save_required = action.required + action.required = false + } + for (let group of this._mutually_exclusive_groups) { + group.save_required = group.required + group.required = false + } + [ namespace, extras ] = this.parse_known_args(remaining_args, + namespace) + } finally { + // restore parser values before exiting + for (let action of optionals) { + action.required = action.save_required + } + for (let group of this._mutually_exclusive_groups) { + group.required = group.save_required + } + } + } finally { + this.usage = save_usage + } + return [ namespace, extras ] + } + + // ======================== + // Value conversion methods + // ======================== + _get_values(action, arg_strings) { + // for everything but PARSER, REMAINDER args, strip out first '--' + if (![PARSER, REMAINDER].includes(action.nargs)) { + try { + _array_remove(arg_strings, '--') + } catch (err) {} + } + + let value + // optional argument produces a default when not present + if (!arg_strings.length && action.nargs === OPTIONAL) { + if (action.option_strings.length) { + value = action.const + } else { + value = action.default + } + if (typeof value === 'string') { + value = this._get_value(action, value) + this._check_value(action, value) + } + + // when nargs='*' on a positional, if there were no command-line + // args, use the default if it is anything other than None + } else if (!arg_strings.length && action.nargs === ZERO_OR_MORE && + !action.option_strings.length) { + if (action.default !== undefined) { + value = action.default + } else { + value = arg_strings + } + this._check_value(action, value) + + // single argument or optional argument produces a single value + } else if (arg_strings.length === 1 && [undefined, OPTIONAL].includes(action.nargs)) { + let arg_string = arg_strings[0] + value = this._get_value(action, arg_string) + this._check_value(action, value) + + // REMAINDER arguments convert all values, checking none + } else if (action.nargs === REMAINDER) { + value = arg_strings.map(v => this._get_value(action, v)) + + // PARSER arguments convert all values, but check only the first + } else if (action.nargs === PARSER) { + value = arg_strings.map(v => this._get_value(action, v)) + this._check_value(action, value[0]) + + // SUPPRESS argument does not put anything in the namespace + } else if (action.nargs === SUPPRESS) { + value = SUPPRESS + + // all other types of nargs produce a list + } else { + value = arg_strings.map(v => this._get_value(action, v)) + for (let v of value) { + this._check_value(action, v) + } + } + + // return the converted value + return value + } + + _get_value(action, arg_string) { + let type_func = this._registry_get('type', action.type, action.type) + if (typeof type_func !== 'function') { + let msg = '%r is not callable' + throw new ArgumentError(action, sub(msg, type_func)) + } + + // convert the value to the appropriate type + let result + try { + try { + result = type_func(arg_string) + } catch (err) { + // Dear TC39, why would you ever consider making es6 classes not callable? + // We had one universal interface, [[Call]], which worked for anything + // (with familiar this-instanceof guard for classes). Now we have two. + if (err instanceof TypeError && + /Class constructor .* cannot be invoked without 'new'/.test(err.message)) { + // eslint-disable-next-line new-cap + result = new type_func(arg_string) + } else { + throw err + } + } + + } catch (err) { + // ArgumentTypeErrors indicate errors + if (err instanceof ArgumentTypeError) { + //let name = getattr(action.type, 'name', repr(action.type)) + let msg = err.message + throw new ArgumentError(action, msg) + + // TypeErrors or ValueErrors also indicate errors + } else if (err instanceof TypeError) { + let name = getattr(action.type, 'name', repr(action.type)) + let args = {type: name, value: arg_string} + let msg = 'invalid %(type)s value: %(value)r' + throw new ArgumentError(action, sub(msg, args)) + } else { + throw err + } + } + + // return the converted value + return result + } + + _check_value(action, value) { + // converted value must be one of the choices (if specified) + if (action.choices !== undefined && !_choices_to_array(action.choices).includes(value)) { + let args = {value, + choices: _choices_to_array(action.choices).map(repr).join(', ')} + let msg = 'invalid choice: %(value)r (choose from %(choices)s)' + throw new ArgumentError(action, sub(msg, args)) + } + } + + // ======================= + // Help-formatting methods + // ======================= + format_usage() { + let formatter = this._get_formatter() + formatter.add_usage(this.usage, this._actions, + this._mutually_exclusive_groups) + return formatter.format_help() + } + + format_help() { + let formatter = this._get_formatter() + + // usage + formatter.add_usage(this.usage, this._actions, + this._mutually_exclusive_groups) + + // description + formatter.add_text(this.description) + + // positionals, optionals and user-defined groups + for (let action_group of this._action_groups) { + formatter.start_section(action_group.title) + formatter.add_text(action_group.description) + formatter.add_arguments(action_group._group_actions) + formatter.end_section() + } + + // epilog + formatter.add_text(this.epilog) + + // determine help from format above + return formatter.format_help() + } + + _get_formatter() { + // eslint-disable-next-line new-cap + return new this.formatter_class({ prog: this.prog }) + } + + // ===================== + // Help-printing methods + // ===================== + print_usage(file = undefined) { + if (file === undefined) file = process.stdout + this._print_message(this.format_usage(), file) + } + + print_help(file = undefined) { + if (file === undefined) file = process.stdout + this._print_message(this.format_help(), file) + } + + _print_message(message, file = undefined) { + if (message) { + if (file === undefined) file = process.stderr + file.write(message) + } + } + + // =============== + // Exiting methods + // =============== + exit(status = 0, message = undefined) { + if (message) { + this._print_message(message, process.stderr) + } + process.exit(status) + } + + error(message) { + /* + * error(message: string) + * + * Prints a usage message incorporating the message to stderr and + * exits. + * + * If you override this in a subclass, it should not return -- it + * should either exit or raise an exception. + */ + + // LEGACY (v1 compatibility), debug mode + if (this.debug === true) throw new Error(message) + // end + this.print_usage(process.stderr) + let args = {prog: this.prog, message: message} + this.exit(2, sub('%(prog)s: error: %(message)s\n', args)) + } +})) + + +module.exports = { + ArgumentParser, + ArgumentError, + ArgumentTypeError, + BooleanOptionalAction, + FileType, + HelpFormatter, + ArgumentDefaultsHelpFormatter, + RawDescriptionHelpFormatter, + RawTextHelpFormatter, + MetavarTypeHelpFormatter, + Namespace, + Action, + ONE_OR_MORE, + OPTIONAL, + PARSER, + REMAINDER, + SUPPRESS, + ZERO_OR_MORE +} + +// LEGACY (v1 compatibility), Const alias +Object.defineProperty(module.exports, 'Const', { + get() { + let result = {} + Object.entries({ ONE_OR_MORE, OPTIONAL, PARSER, REMAINDER, SUPPRESS, ZERO_OR_MORE }).forEach(([ n, v ]) => { + Object.defineProperty(result, n, { + get() { + deprecate(n, sub('use argparse.%s instead of argparse.Const.%s', n, n)) + return v + } + }) + }) + Object.entries({ _UNRECOGNIZED_ARGS_ATTR }).forEach(([ n, v ]) => { + Object.defineProperty(result, n, { + get() { + deprecate(n, sub('argparse.Const.%s is an internal symbol and will no longer be available', n)) + return v + } + }) + }) + return result + }, + enumerable: false +}) +// end diff --git a/queries/cdmq/node_modules/argparse/lib/sub.js b/queries/cdmq/node_modules/argparse/lib/sub.js new file mode 100644 index 00000000..e3eb3215 --- /dev/null +++ b/queries/cdmq/node_modules/argparse/lib/sub.js @@ -0,0 +1,67 @@ +// Limited implementation of python % string operator, supports only %s and %r for now +// (other formats are not used here, but may appear in custom templates) + +'use strict' + +const { inspect } = require('util') + + +module.exports = function sub(pattern, ...values) { + let regex = /%(?:(%)|(-)?(\*)?(?:\((\w+)\))?([A-Za-z]))/g + + let result = pattern.replace(regex, function (_, is_literal, is_left_align, is_padded, name, format) { + if (is_literal) return '%' + + let padded_count = 0 + if (is_padded) { + if (values.length === 0) throw new TypeError('not enough arguments for format string') + padded_count = values.shift() + if (!Number.isInteger(padded_count)) throw new TypeError('* wants int') + } + + let str + if (name !== undefined) { + let dict = values[0] + if (typeof dict !== 'object' || dict === null) throw new TypeError('format requires a mapping') + if (!(name in dict)) throw new TypeError(`no such key: '${name}'`) + str = dict[name] + } else { + if (values.length === 0) throw new TypeError('not enough arguments for format string') + str = values.shift() + } + + switch (format) { + case 's': + str = String(str) + break + case 'r': + str = inspect(str) + break + case 'd': + case 'i': + if (typeof str !== 'number') { + throw new TypeError(`%${format} format: a number is required, not ${typeof str}`) + } + str = String(str.toFixed(0)) + break + default: + throw new TypeError(`unsupported format character '${format}'`) + } + + if (padded_count > 0) { + return is_left_align ? str.padEnd(padded_count) : str.padStart(padded_count) + } else { + return str + } + }) + + if (values.length) { + if (values.length === 1 && typeof values[0] === 'object' && values[0] !== null) { + // mapping + } else { + throw new TypeError('not all arguments converted during string formatting') + } + } + + return result +} diff --git a/queries/cdmq/node_modules/argparse/lib/textwrap.js b/queries/cdmq/node_modules/argparse/lib/textwrap.js new file mode 100644 index 00000000..23d51cdb --- /dev/null +++ b/queries/cdmq/node_modules/argparse/lib/textwrap.js @@ -0,0 +1,440 @@ +// Partial port of python's argparse module, version 3.9.0 (only wrap and fill functions): +// https://github.com/python/cpython/blob/v3.9.0b4/Lib/textwrap.py + +'use strict' + +/* + * Text wrapping and filling. + */ + +// Copyright (C) 1999-2001 Gregory P. Ward. +// Copyright (C) 2002, 2003 Python Software Foundation. +// Copyright (C) 2020 argparse.js authors +// Originally written by Greg Ward + +// Hardcode the recognized whitespace characters to the US-ASCII +// whitespace characters. The main reason for doing this is that +// some Unicode spaces (like \u00a0) are non-breaking whitespaces. +// +// This less funky little regex just split on recognized spaces. E.g. +// "Hello there -- you goof-ball, use the -b option!" +// splits into +// Hello/ /there/ /--/ /you/ /goof-ball,/ /use/ /the/ /-b/ /option!/ +const wordsep_simple_re = /([\t\n\x0b\x0c\r ]+)/ + +class TextWrapper { + /* + * Object for wrapping/filling text. The public interface consists of + * the wrap() and fill() methods; the other methods are just there for + * subclasses to override in order to tweak the default behaviour. + * If you want to completely replace the main wrapping algorithm, + * you'll probably have to override _wrap_chunks(). + * + * Several instance attributes control various aspects of wrapping: + * width (default: 70) + * the maximum width of wrapped lines (unless break_long_words + * is false) + * initial_indent (default: "") + * string that will be prepended to the first line of wrapped + * output. Counts towards the line's width. + * subsequent_indent (default: "") + * string that will be prepended to all lines save the first + * of wrapped output; also counts towards each line's width. + * expand_tabs (default: true) + * Expand tabs in input text to spaces before further processing. + * Each tab will become 0 .. 'tabsize' spaces, depending on its position + * in its line. If false, each tab is treated as a single character. + * tabsize (default: 8) + * Expand tabs in input text to 0 .. 'tabsize' spaces, unless + * 'expand_tabs' is false. + * replace_whitespace (default: true) + * Replace all whitespace characters in the input text by spaces + * after tab expansion. Note that if expand_tabs is false and + * replace_whitespace is true, every tab will be converted to a + * single space! + * fix_sentence_endings (default: false) + * Ensure that sentence-ending punctuation is always followed + * by two spaces. Off by default because the algorithm is + * (unavoidably) imperfect. + * break_long_words (default: true) + * Break words longer than 'width'. If false, those words will not + * be broken, and some lines might be longer than 'width'. + * break_on_hyphens (default: true) + * Allow breaking hyphenated words. If true, wrapping will occur + * preferably on whitespaces and right after hyphens part of + * compound words. + * drop_whitespace (default: true) + * Drop leading and trailing whitespace from lines. + * max_lines (default: None) + * Truncate wrapped lines. + * placeholder (default: ' [...]') + * Append to the last line of truncated text. + */ + + constructor(options = {}) { + let { + width = 70, + initial_indent = '', + subsequent_indent = '', + expand_tabs = true, + replace_whitespace = true, + fix_sentence_endings = false, + break_long_words = true, + drop_whitespace = true, + break_on_hyphens = true, + tabsize = 8, + max_lines = undefined, + placeholder=' [...]' + } = options + + this.width = width + this.initial_indent = initial_indent + this.subsequent_indent = subsequent_indent + this.expand_tabs = expand_tabs + this.replace_whitespace = replace_whitespace + this.fix_sentence_endings = fix_sentence_endings + this.break_long_words = break_long_words + this.drop_whitespace = drop_whitespace + this.break_on_hyphens = break_on_hyphens + this.tabsize = tabsize + this.max_lines = max_lines + this.placeholder = placeholder + } + + + // -- Private methods ----------------------------------------------- + // (possibly useful for subclasses to override) + + _munge_whitespace(text) { + /* + * _munge_whitespace(text : string) -> string + * + * Munge whitespace in text: expand tabs and convert all other + * whitespace characters to spaces. Eg. " foo\\tbar\\n\\nbaz" + * becomes " foo bar baz". + */ + if (this.expand_tabs) { + text = text.replace(/\t/g, ' '.repeat(this.tabsize)) // not strictly correct in js + } + if (this.replace_whitespace) { + text = text.replace(/[\t\n\x0b\x0c\r]/g, ' ') + } + return text + } + + _split(text) { + /* + * _split(text : string) -> [string] + * + * Split the text to wrap into indivisible chunks. Chunks are + * not quite the same as words; see _wrap_chunks() for full + * details. As an example, the text + * Look, goof-ball -- use the -b option! + * breaks into the following chunks: + * 'Look,', ' ', 'goof-', 'ball', ' ', '--', ' ', + * 'use', ' ', 'the', ' ', '-b', ' ', 'option!' + * if break_on_hyphens is True, or in: + * 'Look,', ' ', 'goof-ball', ' ', '--', ' ', + * 'use', ' ', 'the', ' ', '-b', ' ', option!' + * otherwise. + */ + let chunks = text.split(wordsep_simple_re) + chunks = chunks.filter(Boolean) + return chunks + } + + _handle_long_word(reversed_chunks, cur_line, cur_len, width) { + /* + * _handle_long_word(chunks : [string], + * cur_line : [string], + * cur_len : int, width : int) + * + * Handle a chunk of text (most likely a word, not whitespace) that + * is too long to fit in any line. + */ + // Figure out when indent is larger than the specified width, and make + // sure at least one character is stripped off on every pass + let space_left + if (width < 1) { + space_left = 1 + } else { + space_left = width - cur_len + } + + // If we're allowed to break long words, then do so: put as much + // of the next chunk onto the current line as will fit. + if (this.break_long_words) { + cur_line.push(reversed_chunks[reversed_chunks.length - 1].slice(0, space_left)) + reversed_chunks[reversed_chunks.length - 1] = reversed_chunks[reversed_chunks.length - 1].slice(space_left) + + // Otherwise, we have to preserve the long word intact. Only add + // it to the current line if there's nothing already there -- + // that minimizes how much we violate the width constraint. + } else if (!cur_line) { + cur_line.push(...reversed_chunks.pop()) + } + + // If we're not allowed to break long words, and there's already + // text on the current line, do nothing. Next time through the + // main loop of _wrap_chunks(), we'll wind up here again, but + // cur_len will be zero, so the next line will be entirely + // devoted to the long word that we can't handle right now. + } + + _wrap_chunks(chunks) { + /* + * _wrap_chunks(chunks : [string]) -> [string] + * + * Wrap a sequence of text chunks and return a list of lines of + * length 'self.width' or less. (If 'break_long_words' is false, + * some lines may be longer than this.) Chunks correspond roughly + * to words and the whitespace between them: each chunk is + * indivisible (modulo 'break_long_words'), but a line break can + * come between any two chunks. Chunks should not have internal + * whitespace; ie. a chunk is either all whitespace or a "word". + * Whitespace chunks will be removed from the beginning and end of + * lines, but apart from that whitespace is preserved. + */ + let lines = [] + let indent + if (this.width <= 0) { + throw Error(`invalid width ${this.width} (must be > 0)`) + } + if (this.max_lines !== undefined) { + if (this.max_lines > 1) { + indent = this.subsequent_indent + } else { + indent = this.initial_indent + } + if (indent.length + this.placeholder.trimStart().length > this.width) { + throw Error('placeholder too large for max width') + } + } + + // Arrange in reverse order so items can be efficiently popped + // from a stack of chucks. + chunks = chunks.reverse() + + while (chunks.length > 0) { + + // Start the list of chunks that will make up the current line. + // cur_len is just the length of all the chunks in cur_line. + let cur_line = [] + let cur_len = 0 + + // Figure out which static string will prefix this line. + let indent + if (lines) { + indent = this.subsequent_indent + } else { + indent = this.initial_indent + } + + // Maximum width for this line. + let width = this.width - indent.length + + // First chunk on line is whitespace -- drop it, unless this + // is the very beginning of the text (ie. no lines started yet). + if (this.drop_whitespace && chunks[chunks.length - 1].trim() === '' && lines.length > 0) { + chunks.pop() + } + + while (chunks.length > 0) { + let l = chunks[chunks.length - 1].length + + // Can at least squeeze this chunk onto the current line. + if (cur_len + l <= width) { + cur_line.push(chunks.pop()) + cur_len += l + + // Nope, this line is full. + } else { + break + } + } + + // The current line is full, and the next chunk is too big to + // fit on *any* line (not just this one). + if (chunks.length && chunks[chunks.length - 1].length > width) { + this._handle_long_word(chunks, cur_line, cur_len, width) + cur_len = cur_line.map(l => l.length).reduce((a, b) => a + b, 0) + } + + // If the last chunk on this line is all whitespace, drop it. + if (this.drop_whitespace && cur_line.length > 0 && cur_line[cur_line.length - 1].trim() === '') { + cur_len -= cur_line[cur_line.length - 1].length + cur_line.pop() + } + + if (cur_line) { + if (this.max_lines === undefined || + lines.length + 1 < this.max_lines || + (chunks.length === 0 || + this.drop_whitespace && + chunks.length === 1 && + !chunks[0].trim()) && cur_len <= width) { + // Convert current line back to a string and store it in + // list of all lines (return value). + lines.push(indent + cur_line.join('')) + } else { + let had_break = false + while (cur_line) { + if (cur_line[cur_line.length - 1].trim() && + cur_len + this.placeholder.length <= width) { + cur_line.push(this.placeholder) + lines.push(indent + cur_line.join('')) + had_break = true + break + } + cur_len -= cur_line[-1].length + cur_line.pop() + } + if (!had_break) { + if (lines) { + let prev_line = lines[lines.length - 1].trimEnd() + if (prev_line.length + this.placeholder.length <= + this.width) { + lines[lines.length - 1] = prev_line + this.placeholder + break + } + } + lines.push(indent + this.placeholder.lstrip()) + } + break + } + } + } + + return lines + } + + _split_chunks(text) { + text = this._munge_whitespace(text) + return this._split(text) + } + + // -- Public interface ---------------------------------------------- + + wrap(text) { + /* + * wrap(text : string) -> [string] + * + * Reformat the single paragraph in 'text' so it fits in lines of + * no more than 'self.width' columns, and return a list of wrapped + * lines. Tabs in 'text' are expanded with string.expandtabs(), + * and all other whitespace characters (including newline) are + * converted to space. + */ + let chunks = this._split_chunks(text) + // not implemented in js + //if (this.fix_sentence_endings) { + // this._fix_sentence_endings(chunks) + //} + return this._wrap_chunks(chunks) + } + + fill(text) { + /* + * fill(text : string) -> string + * + * Reformat the single paragraph in 'text' to fit in lines of no + * more than 'self.width' columns, and return a new string + * containing the entire wrapped paragraph. + */ + return this.wrap(text).join('\n') + } +} + + +// -- Convenience interface --------------------------------------------- + +function wrap(text, options = {}) { + /* + * Wrap a single paragraph of text, returning a list of wrapped lines. + * + * Reformat the single paragraph in 'text' so it fits in lines of no + * more than 'width' columns, and return a list of wrapped lines. By + * default, tabs in 'text' are expanded with string.expandtabs(), and + * all other whitespace characters (including newline) are converted to + * space. See TextWrapper class for available keyword args to customize + * wrapping behaviour. + */ + let { width = 70, ...kwargs } = options + let w = new TextWrapper(Object.assign({ width }, kwargs)) + return w.wrap(text) +} + +function fill(text, options = {}) { + /* + * Fill a single paragraph of text, returning a new string. + * + * Reformat the single paragraph in 'text' to fit in lines of no more + * than 'width' columns, and return a new string containing the entire + * wrapped paragraph. As with wrap(), tabs are expanded and other + * whitespace characters converted to space. See TextWrapper class for + * available keyword args to customize wrapping behaviour. + */ + let { width = 70, ...kwargs } = options + let w = new TextWrapper(Object.assign({ width }, kwargs)) + return w.fill(text) +} + +// -- Loosely related functionality ------------------------------------- + +let _whitespace_only_re = /^[ \t]+$/mg +let _leading_whitespace_re = /(^[ \t]*)(?:[^ \t\n])/mg + +function dedent(text) { + /* + * Remove any common leading whitespace from every line in `text`. + * + * This can be used to make triple-quoted strings line up with the left + * edge of the display, while still presenting them in the source code + * in indented form. + * + * Note that tabs and spaces are both treated as whitespace, but they + * are not equal: the lines " hello" and "\\thello" are + * considered to have no common leading whitespace. + * + * Entirely blank lines are normalized to a newline character. + */ + // Look for the longest leading string of spaces and tabs common to + // all lines. + let margin = undefined + text = text.replace(_whitespace_only_re, '') + let indents = text.match(_leading_whitespace_re) || [] + for (let indent of indents) { + indent = indent.slice(0, -1) + + if (margin === undefined) { + margin = indent + + // Current line more deeply indented than previous winner: + // no change (previous winner is still on top). + } else if (indent.startsWith(margin)) { + // pass + + // Current line consistent with and no deeper than previous winner: + // it's the new winner. + } else if (margin.startsWith(indent)) { + margin = indent + + // Find the largest common whitespace between current line and previous + // winner. + } else { + for (let i = 0; i < margin.length && i < indent.length; i++) { + if (margin[i] !== indent[i]) { + margin = margin.slice(0, i) + break + } + } + } + } + + if (margin) { + text = text.replace(new RegExp('^' + margin, 'mg'), '') + } + return text +} + +module.exports = { wrap, fill, dedent } diff --git a/queries/cdmq/node_modules/argparse/package.json b/queries/cdmq/node_modules/argparse/package.json new file mode 100644 index 00000000..647d2aff --- /dev/null +++ b/queries/cdmq/node_modules/argparse/package.json @@ -0,0 +1,31 @@ +{ + "name": "argparse", + "description": "CLI arguments parser. Native port of python's argparse.", + "version": "2.0.1", + "keywords": [ + "cli", + "parser", + "argparse", + "option", + "args" + ], + "main": "argparse.js", + "files": [ + "argparse.js", + "lib/" + ], + "license": "Python-2.0", + "repository": "nodeca/argparse", + "scripts": { + "lint": "eslint .", + "test": "npm run lint && nyc mocha", + "coverage": "npm run test && nyc report --reporter html" + }, + "devDependencies": { + "@babel/eslint-parser": "^7.11.0", + "@babel/plugin-syntax-class-properties": "^7.10.4", + "eslint": "^7.5.0", + "mocha": "^8.0.1", + "nyc": "^15.1.0" + } +} diff --git a/queries/cdmq/node_modules/xz-decompress/README.md b/queries/cdmq/node_modules/xz-decompress/README.md new file mode 100644 index 00000000..4245b03a --- /dev/null +++ b/queries/cdmq/node_modules/xz-decompress/README.md @@ -0,0 +1,70 @@ +# XZ-Decompress - Streaming XZ decompression for the browser & Node [![Build Status](https://github.com/httptoolkit/xz-decompress/workflows/CI/badge.svg)](https://github.com/httptoolkit/xz-decompress/actions) + +> _Part of [HTTP Toolkit](https://httptoolkit.com): powerful tools for building, testing & debugging HTTP(S)_ + +This is an NPM package compatible with both Node.js & browsers that can decompress XZ streams. + +You can use this if you want your web server to return XZ-encoded content and have your JavaScript code see the uncompressed data (as an alternative to Gzip or Brotli), or if you want to decompress XZ files within Node.js without needing to mess with any native addons. + +_This is a fork of https://github.com/SteveSanderson/xzwasm, intended for use in [Frida-JS](https://github.com/httptoolkit/frida-js/) and [HTTP Toolkit](https://httptoolkit.com)._ + +## Installation + +``` +npm install --save xz-decompress +``` + +You can then import things from `xz-decompress` in your existing JavaScript/TypeScript files. Example: + +```js +import { XzReadableStream } from 'xz-decompress'; +``` + +## How to use + +Given an XZ-compressed stream, such as a `fetch` response body, you can get a decompressed response by wrapping it with `XzReadableStream`. Example: + +```js +const compressedResponse = await fetch('somefile.xz'); + +const decompressedResponse = new Response( + new XzReadableStream(compressedResponse.body) +); + +// We now have a regular Response object, so can use standard APIs to parse its body data, +// such as .text(), .json(), or .arrayBuffer(): +const text = await decompressedResponse.text(); +``` + +The API is designed to be as JavaScript-standard as possible, so `XzReadableStream` is a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream) instance, which in turn means you can feed it into a [`Response`](https://developer.mozilla.org/en-US/docs/Web/API/Response), and in turn get a blob, an ArrayBuffer, JSON data, or anything else that you browser can do with a `Response`. + +## What about `.tar.xz` files? + +Since the `.xz` format only represents one file, it's common for people to bundle up a collection of files as `.tar`, and then compress this to `.tar.xz`. + +XZ-Decompress doesn't have built-in support for `.tar`. However, you can use it to convert a `.tar.xz` stream to a stream representing the `.tar` file, and then pass this data to another library such as [js-untar](https://github.com/InvokIT/js-untar) or [tarballjs](https://github.com/ankitrohatgi/tarballjs) to get the bundled files. + +## Building code in this repo + +**Note:** This is only needed if you want to work on xz-decompress itself, not if you just want to use it. + + * Clone this repo + * Clone/update submodules + * `git submodule update --init --recursive` + * Ensure you have a working Clang toolchain that can build wasm + * For example, install https://github.com/WebAssembly/wasi-sdk + * `export wasisdkroot=/path/to/wask-sdk` + * (For testing only) Ensure you have `xz` and `brotli` available as commands on $PATH + * Run `make` + +### Building the NPM package contents + + * Have `node` installed + * `npm install` + * Run `make package` + +### Running scenario/perf tests + + * Have `node` installed + * `npm install -g http-server` + * `make run-sample` diff --git a/queries/cdmq/node_modules/xz-decompress/dist/package/xz-decompress.js b/queries/cdmq/node_modules/xz-decompress/dist/package/xz-decompress.js new file mode 100644 index 00000000..fcae0f8e --- /dev/null +++ b/queries/cdmq/node_modules/xz-decompress/dist/package/xz-decompress.js @@ -0,0 +1,220 @@ +/*! + * Based on xzwasm (c) Steve Sanderson. License: MIT - https://github.com/SteveSanderson/xzwasm + * Contains xz-embedded by Lasse Collin and Igor Pavlov. License: Public domain - https://tukaani.org/xz/embedded.html + * and walloc (c) 2020 Igalia, S.L. License: MIT - https://github.com/wingo/walloc + */ +(function webpackUniversalModuleDefinition(root, factory) { + if(typeof exports === 'object' && typeof module === 'object') + module.exports = factory(require("stream/web")); + else if(typeof define === 'function' && define.amd) + define(["stream/web"], factory); + else if(typeof exports === 'object') + exports["xz-decompress"] = factory(require("stream/web")); + else + root["xz-decompress"] = factory(root["stream/web"]); +})(this, (__WEBPACK_EXTERNAL_MODULE__2__) => { +return /******/ (() => { // webpackBootstrap +/******/ "use strict"; +/******/ var __webpack_modules__ = ([ +/* 0 */, +/* 1 */ +/***/ ((module) => { + +module.exports = "data:application/wasm;base64,AGFzbQEAAAABOApgAX8Bf2ABfwBgAABgA39/fwF/YAABf2ACf38AYAN/f34BfmACf38Bf2AEf39/fwF/YAN/f38AAyEgAAABAgMDAwMEAQUAAgMCBgcIBwUDAAMHAQcABwcBAwkFAwEAAgYIAX8BQfCgBAsHTgUGbWVtb3J5AgAOY3JlYXRlX2NvbnRleHQACA9kZXN0cm95X2NvbnRleHQACQxzdXBwbHlfaW5wdXQACg9nZXRfbmV4dF9vdXRwdXQACwqNYCDfAgEFf0EAIQECQCAAQQdqIgJBEEkNAEEBIQEgAkEDdiIDQQJGDQBBAiEBIAJBIEkNAEEDIQEgA0EERg0AQQQhASACQTBJDQBBBSEBIANBBkYNAEEGIQEgAkHIAEkNAEEHIQEgAkHYAEkNAEEIIQEgAkGIAUkNAEEJIQEgAkGIAkkNACAAEIGAgIAAIgBBCGpBACAAGw8LAkACQCABQQJ0QcCIgIAAaiIEKAIAIgANAEEAIQACQAJAQQAoAuSIgIAAIgJFDQBBACACKAIANgLkiICAAAwBC0EAEIGAgIAAIgJFDQILIAJBgIB8cSIAIAJBCHZB/wFxIgJyIAE6AAAgAkEIdCAAckGAAmohAEEAIQJBACABQQJ0QYCIgIAAaigCACIDayEFIAMhAQNAIAAgBWoiACACNgIAIAAhAiABIANqIgFBgQJJDQALIAQgADYCAAsgBCAAKAIANgIACyAAC+4HAQd/AkACQAJAAkACQEEALQC0iICAAEUNAEEAQQA6ALSIgIAAQQAoArCIgIAAIgFFDQFBsIiAgAAhAgNAAkACQCABQQhqIgMgASgCBCIEaiIFQQh2Qf8BcSIGDQAgASECDAELAkADQCAFQYCAfHEgBmotAABB/gFHDQFBsIiAgAAhBgNAIAYiBygCACIGIAVHDQALIAcgBSgCADYCACABIAQgBSgCBGpBCGoiBDYCBCAHIAIgAiAFRhshAiADIARqIgVBCHZB/wFxIgYNAAsLIAIoAgAhAgsgAigCACIBDQALC0EAKAKwiICAACIFRQ0AIABBhwJqQYB+cSEDQX8hAkGwiICAACEEQQAhAUGwiICAACEGA0AgBiEHAkAgBSIGKAIEIgUgAEkNACAFIAJPDQAgBSECIAchBCAGIQEgBUEIaiADRw0AIAchBCAFIQIgBiEBDAQLIAYoAgAiBQ0ACyABDQIMAQtBsIiAgAAhBAs/AEEQdCEBIABBiAJqIQdBACEDAkACQEEAKAK4iICAACICRQ0AQQAhBSABIQYMAQtBACABQfCghIAAQf//A2pBgIB8cSIGayICNgK4iICAACACIQULAkAgByAFTQ0AIAcgBWsiByACQQF2IgIgAiAHSRtB//8DaiIHQRB2QABBf0YNAkEAQQAoAriIgIAAIAdBgIB8cSIDajYCuIiAgAALIAZFDQEgBkH/AToAASAGQQAoArCIgIAANgKAAiAGQYQCaiADIAVqQYCAfHFB+H1qIgI2AgAgBkGAAmohAQsgAUGAgHxxIgYgAUEIdkH/AXFyQf8BOgAAIAQgASgCADYCAAJAIAIgAGtBgH5xIgUNACABDwsgASEDAkAgBiABQQhqIgQgAmoiByAFQX9zakGAgHxxRg0AIARB//8DcSEFAkAgAEH3/QNLDQAgBiAEQQh2Qf8BcWpB/gE6AAAgAUEAKAKwiICAADYCACABQYCABCAFayIFNgIEQQAgATYCsIiAgAAQg4CAgAAgBkGEggRqIAIgBWtB+H1qIgU2AgAgBkGBgARqQf8BOgAAIAZBgIIEaiEDIAUgAGtBgH5xIQUMAQsgAiAFaiAAIAVqQX9qQYCAfHFrQYCAfGohBSABIQMLIAMgAygCBCAFazYCBCAFQfgBaiEGIAcgBWtBCHZB/wFxIQUCQANAIAYiB0GAfmohBiAFIgQNAUEBIQUgB0H4AUcNAAsLAkAgB0H4AUYNACACIAFqIAZrQYCAfHEiBSAEakH+AToAACAFIARBCHRqIgVBACgCsIiAgAA2AgAgBSAGNgIEQQAgBTYCsIiAgAAQg4CAgAALIAMPC0EAC3wBAn8CQCAARQ0AAkAgAEGAgHxxIABBCHZB/wFxciIBLQAAIgJB/wFHDQAgAEF4aiIAQQAoArCIgIAANgIAQQAgADYCsIiAgAAgAUH+AToAAEEAQQE6ALSIgIAADwsgACACQQJ0QcCIgIAAaiICKAIANgIAIAIgADYCAAsLawECfwJAQQAoArCIgIAAIgAoAgRB/wFLDQAgAEGAgHxxIgEgAEEIdkH/AXEiAHJBCToAAEEAQQAoArCIgIAAKAIANgKwiICAACABIABBCHRyIgBBACgC5IiAgAA2AgBBACAANgLkiICAAAsLTgECfwJAIAAgAUYNACACRQ0AA0ACQCAALQAAIgMgAS0AACIERg0AQQFBfyADIARLGw8LIAFBAWohASAAQQFqIQAgAkF/aiICDQALC0EAC3gBAX8CQAJAIAAgAU8NACACRQ0BIAAhAwNAIAMgAS0AADoAACABQQFqIQEgA0EBaiEDIAJBf2oiAg0ADAILCyAAIAFNDQAgAkUNACABQX9qIQEgAEF/aiEDA0AgAyACaiABIAJqLQAAOgAAIAJBf2oiAg0ACwsgAAssAQF/AkAgAkUNACAAIQMDQCADIAE6AAAgA0EBaiEDIAJBf2oiAg0ACwsgAAuCAQEBfwJAAkAgAEEDcQ0AIAEgAnJBA3ENACACQQRJDQEgAkECdiECIAAhAwNAIAMgASgCADYCACABQQRqIQEgA0EEaiEDIAJBf2oiAg0ADAILCyACRQ0AIAAhAwNAIAMgAS0AADoAACABQQFqIQEgA0EBaiEDIAJBf2oiAg0ACwsgAAuIAQECfwJAQQAtAOiIgIAADQBBAEEBOgDoiICAABCMgICAABCOgICAAAtBoIAIEICAgIAAIgBBgIAENgIAQQJBgICAIBCXgICAACEBIABBFGpCgICAgICAwAA3AgAgAEEQaiAAQaCABGo2AgAgAEEIakIANwMAIAAgAEEgajYCBCAAIAE2AhwgAAsVACAAKAIcEJiAgIAAIAAQgoCAgAALFgAgAEEMaiABNgIAIABBCGpBADYCAAsbACAAKAIcIABBBGogAEEMaigCAEUQloCAgAALVAEDf0EAIQADQEEIIQEgACECA0BBACACQQFxa0GghuLtfnEgAkEBdnMhAiABQX9qIgENAAsgAEECdEHwiICAAGogAjYCACAAQQFqIgBBgAJHDQALC0oAIAJBf3MhAgJAIAFFDQADQCACQf8BcSAALQAAc0ECdEHwiICAAGooAgAgAkEIdnMhAiAAQQFqIQAgAUF/aiIBDQALCyACQX9zC10DAX4BfwF+QgAhAANAQQghASAAIQIDQEIAIAJCAYN9QsKenLzd8pW2SYMgAkIBiIUhAiABQX9qIgENAAsgAKdBA3RB8JCAgABqIAI3AwAgAEIBfCIAQoACUg0ACwtLACACQn+FIQICQCABRQ0AA0AgAkL/AYMgADEAAIWnQQN0QfCQgIAAaikDACACQgiIhSECIABBAWohACABQX9qIgENAAsLIAJCf4UL1RACDH8CfgJAAkAgACgCJEUNACAAKAIAIQIMAQtBACECIABBADoAKCAAQgA3AwAgAEIANwMYIABByABqQQBB5AAQhoCAgAAaIABBrAFqQQw2AgALIAAgASgCBCIDNgIQIABBsAFqIQQgAEHgAGohBSAAQcgAaiEGIABBtgFqIQcgAEGoAWohCCABKAIQIQkCQAJAAkACQANAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAIAIOCgECAAQFBgcICQoPCyABKAIAIQogACgCqAEhAiAAKAKsASELIAEoAgQhDCABKAIIIQ0MAgsgCCAAKAKoASIMakEIaiABKAIAIAEoAgQiAmogASgCCCACayICIAAoAqwBIAxrIgwgAiAMSRsiAhCHgICAABogASABKAIEIAJqNgIEQQAhDCAAQQAgACgCqAEgAmoiAiACIAAoAqwBIgtGGzYCqAEgAiALRw0RIABBATYCAAJAIARBqIiAgABBBhCEgICAAEUNAEEFIQwMEgsgB0ECQQAQjYCAgAAgACgAuAFHDRBBBiEMIActAAANESAAIAAtALcBIgI2AiAgAkEESw0RQQEgAnRBE3FFDRELIAEoAgQiDCABKAIIIg1GDQ4CQCABKAIAIgogDGotAAAiCw0AIAAgDDYCECABIAxBAWo2AgRBBiECDAwLQQAhAiAAQQA2AqgBIABBAjYCACAAIAtBAnRBBGoiCzYCrAEgACALNgJACyAIIAJqQQhqIAogDGogDSAMayIMIAsgAmsiAiAMIAJJGyICEIeAgIAAGiABIAIgASgCBGo2AgRBACEMIABBACAAKAKoASACaiICIAIgACgCrAEiC0YbNgKoASACIAtHDQ8gACACQXxqIgI2AqwBQQchDCAEIAJBABCNgICAACAAIAAoAqwBIgtqQbABaigAAEcNDyAAQQI2AqgBIAAtALEBIgJBP3ENDAJAAkAgAkHAAHFFDQAgACAEIAggCxCRgICAAEEBRw0RIAAgACkDCDcDMCAALQCxASECDAELIABCfzcDMAtCfyEOAkAgAkEYdEEYdUF/Sg0AIAAgBCAIIAAoAqwBEJGAgIAAQQFHDRAgACkDCCEOCyAAIA43AzggACgCrAEiDSAAKAKoASICa0ECSQ0PIAAgAkEBaiIKNgKoASAIIAJqQQhqLQAAQSFHDQwgACACQQJqIgs2AqgBIAggCmpBCGotAABBAUcNDCANIAtGDQ8gACACQQNqNgKoASAAKAKwCSAIIAtqQQhqLQAAEJyAgIAAIgwNDyAAKAKoASIMIAAoAqwBIgIgDCACSxshDQJAA0AgDSAMRg0BIAggDEEBaiICNgIAIAQgDGohCyACIQwgCy0AAA0ODAALCyAGQgA3AwAgAEEANgKoASAAQQM2AgAgBkEIakIANwMACyAAIAEoAgQ2AhAgACABKAIQNgIUIAAoArAJIAEQmYCAgAAhDCAAIAApA0ggASgCBCAAKAIQa618Ig43A0ggACAAKQNQIAEoAhAgACgCFCICayILrXwiDzcDUCAOIAApAzBWDQ0gDyAAKQM4Vg0NAkACQAJAAkAgACgCIEF/ag4EAAMDAQMLIAEoAgwgAmogCyAAKAIYEI2AgIAArSEODAELIAEoAgwgAmogCyAAKQMYEI+AgIAAIQ4LIAAgDjcDGAsgDEEBRw0OAkAgACkDMCIOQn9RDQAgDiAGKQMAUg0OCwJAIAApAzgiDkJ/UQ0AQQchDCAOIAApA1BSDQ8LIAAgACkDSCAANQJAfCAAKQNgfCIPNwNgQgQhDgJAAkACQCAAKAIgQX9qDgQBAgIAAgtCCCEOCyAFIA4gD3w3AwALIAAgACkDaCAAKQNQfDcDaCAAIAVBGCAAKAJwEI2AgIAANgJwIABBBDYCACAAIAApA1hCAXw3A1gLAkAgBikDACIOQgODUA0AIA5CAXwhDiABKAIEIQwgASgCCCELA0AgCyAMRg0NIAEgDEEBaiICNgIEIAEoAgAgDGotAAANDiAGIA43AwAgDkIDgyEPIA5CAXwhDiACIQwgD0IAUg0ACwsgAEEFNgIAC0EBIQIgACgCIEF/ag4EBgcHBQcLIAAgARCSgICAACIMQQFHDQsgAEEHNgIAC0EAIAAoAhBrIQggAEGAAWopAwAhDiABKAIEIQwCQANAIA4gCCAMaq18QgODUA0BAkAgDCABKAIIRw0AIAAgARCTgICAAAwLCyABIAxBAWoiAjYCBCABKAIAIAxqIQsgAiEMIAstAAANCwwACwsgACABEJOAgIAAQQchDCAFIABBkAFqQRgQhICAgAANCiAAQQg2AgALIAAgAUEgEJSAgIAAIgxBAUcNCSAAQQk2AgBBDCELIABBDDYCrAEMAQsgACgCrAEhCwsgAEGoAWogACgCqAEiDGpBCGogASgCACABKAIEIgJqIAEoAgggAmsiAiALIAxrIgwgAiAMSRsiAhCHgICAABogASABKAIEIAJqNgIEQQAhDCAAQQAgACgCqAEgAmoiAiACIAAoAqwBIgtGGzYCqAEgAiALRw0HIAAQlYCAgAAhDAwHC0EBIQIgACABQcAAEJSAgIAAIgxBAUcNBgwBC0EBIQIgACABQSAQlICAgAAiDEEBRw0FCyAAIAI2AgAMAAsLQQYhDAwCC0EAIQwMAQtBByEMCwJAAkAgACgCJA0AAkACQCAMDgIAAwELQQdBCCABKAIEIAEoAghGGyEMCyABIAk2AhAgASADNgIEIAwPCwJAIAwNACADIAEoAgRHDQAgCSABKAIQRw0AIAAtACghASAAQQE6ACggAUEDdA8LIABBADoAKAsgDAuaAQEDfwJAIAAoAgQiBA0AIABCADcDCAsgAigCACEFA0ACQCAFIANJDQBBAA8LIAEgBWotAAAhBiACIAVBAWoiBTYCACAAIAZB/wBxrSAErYYgACkDCIQ3AwgCQAJAIAZBgAFxDQACQCAGDQBBByEGIAQNAgsgAEEANgIEQQEPC0EHIQYgACAEQQdqIgQ2AgQgBEE/Rw0BCwsgBguhAgIDfwF+IABBkAFqIQIgAUEEaiEDA0ACQCAAIAEoAgAgAyABKAIIEJGAgIAAIgRBAUYNACAAQYABaiIDIAMpAwAgASgCBCAAKAIQIgNrIgKtfDcDACAAIAMgASgCAGogAiAAKAIYEI2AgIAArTcDGCAEDwsCQAJAAkACQAJAIAAoAngOAwACAQMLIAAgACkDCCIFNwOIAQJAIAUgACkDWFENAEEHDwsgAEEBNgJ4DAMLIAAgACkDmAEgACkDCHw3A5gBIAAgAkEYIAAoAqABEI2AgIAANgKgASAAQQE2AnggACAAKQOIAUJ/fCIFNwOIAQwCCyAAQQI2AnggACAAKQOQASAAKQMIfDcDkAELIAApA4gBIQULIAVCAFINAAtBAQtAAQJ/IABBgAFqIgIgAikDACABKAIEIAAoAhAiAmsiA618NwMAIAAgAiABKAIAaiADIAAoAhgQjYCAgACtNwMYC3wBBH8gASgCBCEDIAEoAgghBANAAkAgBCADRw0AQQAPCyABIANBAWoiBTYCBAJAIAEoAgAgA2otAAAgACkDGCAAKAIEIgOtiKdB/wFxRg0AQQcPCyAAIANBCGoiBjYCBCAFIQMgBiACSQ0ACyAAQQA2AgQgAEIANwMYQQELbwEBf0EHIQECQCAAQboBai8AAEHZtAFHDQAgAEG0AWpBBkEAEI2AgIAAIABBsAFqKAAARw0AIABBgAFqKQMAQgKIIAA1ALQBUg0AIABBuAFqLQAADQBBAUEHIAAoAiAgAEG5AWotAABGGyEBCyABC7QCAQR/AkACQCAAKAIkRQ0AIAAoAgAhAwwBC0EAIQMgAEEAOgAoIABCADcDACAAQgA3AxggAEHIAGpBAEHkABCGgICAABogAEGsAWpBDDYCAEEBIQILIABByABqIQQCQAJAA0ACQCADQQpHDQAgASgCBCIDIAEoAggiBUYNAiABKAIAIQYCQANAIAYgA2otAAANASABIANBAWoiAzYCBCAAIAAoAgRBAWpBA3E2AgQgBSADRg0EDAALCwJAIAAoAgRFDQBBBw8LIAAoAiRFDQAgAEEAOgAoIABCADcDACAAQgA3AxggBEEAQeQAEIaAgIAAGiAAQQw2AqwBCyAAIAEQkICAgAAiA0EBRw0CQQohAyAAQQo2AgAMAAsLAkAgAg0AQQAPC0EHQQEgACgCBBshAwsgAwt1AQF/AkBBuAkQgICAgAAiAkUNACACIAA2AiQgAiAAIAEQm4CAgAAiADYCsAkCQCAARQ0AIAJBADoAKCACQgA3AwAgAkIANwMYIAJByABqQQBB5AAQhoCAgAAaIAJBrAFqQQw2AgAgAg8LIAIQgoCAgAALQQALHgACQCAARQ0AIAAoArAJEJ2AgIAAIAAQgoCAgAALC4ARAQx/IABB6N0BaiECIABB1ABqIQMgAEEcaiIEQQhqIQUCQAJAA0AgACgCQCEGAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkAgASgCBCIHIAEoAggiCEkNACAGQQdGDQEMEgsgBg4JAQIDBAUGBwAJDwsgACgCTCEGDAcLQQEhCSABIAdBAWo2AgQgASgCACAHai0AACIHRQ0IAkACQCAHQd8BSw0AIAdBAUcNAQsgAEGAAjsBUAJAIAAoAjwNACAAIAEoAgwgASgCECIGajYCGCAAIAEoAhQgBms2AiwLIARCADcCACAFQgA3AgAMCwsgAC0AUEUNCgwOCyABIAdBAWo2AgQgASgCACAHai0AACEHIABBAjYCQCAAIAdBCHQgACgCSGo2AkgMDAsgASAHQQFqNgIEIAEoAgAgB2otAAAhByAAQQM2AkAgACAHIAAoAkhqQQFqNgJIDAsLIAEgB0EBajYCBCABKAIAIAdqLQAAIQcgAEEENgJAIAAgB0EIdDYCTAwKCyABIAdBAWo2AgQgASgCACAHai0AACEHIAAgACgCRDYCQCAAIAcgACgCTGpBAWo2AkwMCQsgASAHQQFqNgIEQQchCSABKAIAIAdqLQAAIgdB4AFLDQNBACEGAkACQCAHQS1PDQBBACEIDAELIAdBU2oiByAHQf8BcUEtbiIIQS1sayEHIAhBAWohCAsgAEF/IAh0QX9zNgJ0AkAgB0H/AXFBCUkNACAHQXdqIgcgB0H/AXFBCW4iBkEJbGshByAGQQFqIQYLIAAgBjYCcCAAIAdB/wFxIgc2AmwgBiAHakEESw0DIANCADcCACADQQhqQgA3AgAgA0EQakEANgIAIABBfyAGdEF/czYCcEH4ACEHA0AgACAHakGACDsBACAHQQJqIgdB5N0BRw0ACyAAQQY2AkAgAEEFNgIIIABC/////w83AgALIAAoAkwiCUEFSQ0IAkAgACgCCCIHRQ0AIAdBf2ohBiABKAIEIQcgASgCCCEKA0AgCiAHRg0LIAEgB0EBaiIINgIEIAEoAgAgB2otAAAhByAAIAY2AgggACAHIAAoAgRBCHRyNgIEIAghByAGQX9qIgZBf0cNAAsLIABBBzYCQCAAIAlBe2oiBjYCTAsgACAAKAIgIgcgASgCFCABKAIQayIIIAAoAkgiCiAIIApJGyIIaiAAKAIsIgogCiAHayAISxs2AiggASgCCCIJIAEoAgQiCGshBwJAAkACQCAAKALk3QEiCg0AIAYNAUEAIQYLIABB5N0BaiILIApqQQRqIAEoAgAgCGogByAGIAprIgZBKiAKayIIIAggBksbIgYgBiAHSxsiBxCHgICAABoCQAJAIAcgACgC5N0BIghqIgYgACgCTEcNACALIAhqIAdqQQRqQQBBPyAGaxCGgICAABogACgC5N0BIAdqIQYMAQsCQCAGQRRLDQAgACAGNgLk3QEgASABKAIEIAdqNgIEDAMLIAZBa2ohBgsgAEEANgIQIAAgAjYCDCAAIAY2AhRBByEJIAAQmoCAgABFDQMgACgCECIIIAAoAuTdASIKIAdqSw0DIAAgACgCTCAIayIGNgJMAkAgCiAITQ0AIAAgCiAIayIHNgLk3QEgAiALIAhqQQRqIAcQhYCAgAAaDAILIABBADYC5N0BIAEgASgCBCAIIApraiIINgIEIAEoAggiCSAIayEHCwJAIAdBFUkNACAAIAg2AhAgACABKAIANgIMIAAgCUFraiAIIAZqIAcgBkEVakkbNgIUQQchCSAAEJqAgIAARQ0DIAAoAkwiByAAKAIQIgggASgCBGsiBkkNAyABIAg2AgQgACAHIAZrIgY2AkwgASgCCCAIayIHQRRLDQELIAIgASgCACAIaiAGIAcgByAGSxsiBxCHgICAABogACAHNgLk3QEgASABKAIEIAdqNgIECyAAKAIgIgYgACgCHCIIayEHAkAgACgCPEUNAAJAIAYgACgCLEcNACAAQQA2AiALIAEoAgwgASgCEGogACgCGCAIaiAHEIeAgIAAGiAAKAIgIQYLIAAgBjYCHCABIAEoAhAgB2oiBjYCECAAIAAoAkggB2siBzYCSAJAIAcNAEEHIQkgACgCTA0CIAAoAmgNAiAAKAIEDQIgAEEANgJADAULQQAhCSAGIAEoAhRGDQEgASgCBCABKAIIRw0GIAAoAuTdASAAKAJMTw0GDAELIAAoAkwiCkUNAUEAIQkgCCAHTQ0AA0AgASgCFCIGIAEoAhAiC00NASAAIAogCiAAKAIsIAAoAiAiDGsiDSAIIAdrIgggBiALayIGIAggBkkbIgYgBiANSxsiBiAGIApLGyIGazYCTCAMIAAoAhhqIAEoAgAgB2ogBhCFgICAABogACAAKAIgIAZqIgc2AiACQCAAKAIkIAdPDQAgACAHNgIkCwJAIAAoAjxFDQACQCAHIAAoAixHDQAgAEEANgIgCyABKAIMIAEoAhBqIAEoAgAgASgCBGogBhCFgICAABogACgCICEHCyAAIAc2AhwgASABKAIQIAZqNgIQIAEgASgCBCAGaiIHNgIEIAAoAkwiCkUNAiABKAIIIgggB0sNAAsLIAkPCyAAQQA2AkAMAwsgB0EYdEEYdUF/Sg0BIABBATYCQCAAIAdBEHRBgID8AHE2AkgCQCAHQcABSQ0AIABBBTYCRCAAQQA6AFEMAwsgAC0AUQ0DIABBBjYCRCAHQaABSQ0CIANCADcCACADQRBqQQA2AgAgA0EIakIANwIAQfgAIQcDQCAAIAdqQYAIOwEAIAdBAmoiB0Hk3QFHDQALCyAAQQU2AgggAEL/////DzcCAAwBCyAHQQJLDQEgAEKDgICAgAE3AkAMAAsLQQcPC0EAC/8XARJ/IABBGGohAQJAIABBIGooAgAiAiAAQShqKAIAIgNPDQAgAEHoAGoiBCgCAEUNACABIAQgACgCVBCegICAABogACgCKCEDIAAoAiAhAgsCQCACIANPDQAgAEHcDWohBSAAQegAaiEGIABB4BVqIQcgAEHUAGohCANAIAAoAhAiCSAAKAIUSw0BIAAgACgCZCIKQQV0aiAAKAJ0IAJxIgtBAXRqIgxB+ABqIQ0CQAJAIAAoAgAiBEGAgIAISQ0AIAAoAgQhDgwBCyAAIARBCHQiBDYCACAAIAlBAWoiAzYCECAAIAAoAgRBCHQgACgCDCAJai0AAHIiDjYCBCADIQkLAkACQCAOIARBC3YgDS8BACIPbCIDTw0AIAAgAzYCACANIA9BgBAgD2tBBXZqOwEAIAJBf2ohBAJAIAINACAAKAIsIARqIQQLAkACQCAAKAIkIg8NAEEAIQQMAQsgACgCGCAEai0AACEECyAAKAJwIAJxIAAoAmwiDXQgBEEIIA1rdmohDAJAAkAgCkEGSw0AQQEhBANAIAAgDEGADGxqIARBAXQiBGpB5B1qIQ0CQAJAIANB////B00NACADIQoMAQsgACADQQh0Igo2AgAgACAJQQFqIgM2AhAgACAOQQh0IAAoAgwgCWotAAByIg42AgQgAyEJCwJAAkAgDiAKQQt2IA0vAQAiD2wiA0kNACAAIA4gA2siDjYCBCAAIAogA2siAzYCACANIA8gD0EFdms7AQAgBEEBciEEDAELIAAgAzYCACANIA9BgBAgD2tBBXZqOwEACyAEQYACSQ0ADAILCyACIAAoAlQiDUF/c2ohBAJAIAIgDUsNACAAKAIsIARqIQQLAkACQCAPDQBBACEQDAELIAAoAhggBGotAAAhEAtBASEEQYACIQ0DQCAAIAxBgAxsaiAQQQF0IhAgDXEiESANaiAEakEBdGpB5B1qIQ8CQAJAIANB////B00NACADIQsMAQsgACADQQh0Igs2AgAgACAJQQFqIgM2AhAgACAOQQh0IAAoAgwgCWotAAByIg42AgQgAyEJCwJAAkAgDiALQQt2IA8vAQAiCmwiA08iEg0AIAAgAzYCACAKQYAQIAprQQV2aiEKDAELIAAgDiADayIONgIEIAAgCyADayIDNgIAIAogCkEFdmshCkEAIQ0LIA8gCjsBACANIBFzIQ0gBEEBdCASciIEQYACSQ0ACwsgACACQQFqNgIgIAAoAhggAmogBDoAAAJAIAAoAiQgACgCICICTw0AIAAgAjYCJAtBACEDAkAgACgCZCIEQQRJDQACQCAEQQlLDQAgBEF9aiEDDAELIARBemohAwsgACADNgJkDAELIAAgDiADayIONgIEIAAgBCADayIDNgIAIA0gDyAPQQV2azsBACAAIApBAXRqIg9B+ANqIQQCQAJAIANB////B00NACAJIQoMAQsgACADQQh0IgM2AgAgACAJQQFqIgo2AhAgACAOQQh0IAAoAgwgCWotAAByIg42AgQLAkACQCAOIANBC3YgBC8BACINbCIJSQ0AIAAgDiAJayIONgIEIAAgAyAJayIDNgIAIAQgDSANQQV2azsBACAPQZAEaiENAkACQCADQf///wdNDQAgCiEQDAELIAAgA0EIdCIDNgIAIAAgCkEBaiIQNgIQIAAgDkEIdCAAKAIMIApqLQAAciIONgIECwJAAkAgDiADQQt2IA0vAQAiCWwiBE8NACANIAlBgBAgCWtBBXZqOwEAIAxB2ARqIQMCQCAEQf///wdLDQAgACAEQQh0IgQ2AgAgACAQQQFqNgIQIAAgDkEIdCAAKAIMIBBqLQAAciIONgIECwJAIA4gBEELdiADLwEAIg1sIglJDQAgACAOIAlrNgIEIAAgBCAJazYCACADIA0gDUEFdms7AQAMAgsgAyANQYAQIA1rQQV2ajsBACAAIAk2AgAgAEEBNgJoIABBCUELIAAoAmRBB0kbNgJkDAMLIAAgDiAEayIONgIEIA0gCSAJQQV2azsBACAPQagEaiENAkACQCADIARrIgNB////B00NACAQIQoMAQsgACADQQh0IgM2AgAgACAQQQFqIgo2AhAgACAOQQh0IAAoAgwgEGotAAByIg42AgQLAkACQCAOIANBC3YgDS8BACIEbCIJTw0AIAAgCTYCACANIARBgBAgBGtBBXZqOwEAIAAoAlghAwwBCyAAIA4gCWsiDjYCBCANIAQgBEEFdms7AQAgD0HABGohBAJAIAMgCWsiA0H///8HSw0AIAAgA0EIdCIDNgIAIAAgCkEBajYCECAAIA5BCHQgACgCDCAKai0AAHIiDjYCBAsCQAJAIA4gA0ELdiAELwEAIg1sIglPDQAgACAJNgIAIAQgDUGAECANa0EFdmo7AQAgACgCXCEDDAELIAAgDiAJazYCBCAAIAMgCWs2AgAgACgCYCEDIAAgACgCXDYCYCAEIA0gDUEFdms7AQALIAAgACgCWDYCXAsgACAAKAJUNgJYIAAgAzYCVAsgAEEIQQsgACgCZEEHSRs2AmQgACAHIAsQn4CAgAAMAQsgBCANQYAQIA1rQQV2ajsBACAAIAk2AgAgACAAKAJcNgJgIAAgACkCVDcCWCAAQQdBCiAAKAJkQQdJGzYCZCAAIAUgCxCfgICAACAAKAJoIgNBfmpBAyADQQZJGyEKIAAoAgAhA0EBIQ4DQCAAIApBB3RqIA5BAXQiDmpB2AdqIQ0CQAJAIANBgICACEkNACAAKAIEIQQMAQsgACADQQh0IgM2AgAgACAAKAIQIgRBAWo2AhAgACAAKAIEQQh0IAQgACgCDGotAAByIgQ2AgQLAkACQCAEIANBC3YgDS8BACIJbCIPSQ0AIAAgBCAPayIENgIEIAAgAyAPayIDNgIAIA0gCSAJQQV2azsBACAOQQFyIQ4MAQsgACAPNgIAIA0gCUGAECAJa0EFdmo7AQAgDyEDCyAOQcAASQ0ACwJAIA5BQGoiCUEDSw0AIAAgCTYCVAwBCyAAIA5BAXFBAnIiDTYCVCAJQQF2IQ8CQCAJQQ1LDQAgACANIA9Bf2oiDHQiCzYCVEEBIQ0gCCALQQF0akGEC2ohEEE/IA5rIRFBACEPA0AgECARIA1qQQF0aiEOAkACQCADQf///wdNDQAgAyEKDAELIAAgA0EIdCIKNgIAIAAgACgCECIDQQFqNgIQIAAgBEEIdCADIAAoAgxqLQAAciIENgIECwJAAkAgBCAKQQt2IA4vAQAiCWwiA0kNACAAIAQgA2siBDYCBCAAIAogA2siAzYCACAOIAkgCUEFdms7AQAgAEEBIA90IAtqIgs2AlQgDUEBdEEBciENDAELIAAgAzYCACAOIAlBgBAgCWtBBXZqOwEAIA1BAXQhDQsgDCAPQQFqIg9HDQAMAgsLIA9Be2ohDgNAAkAgA0H///8HSw0AIAAgA0EIdCIDNgIAIAAgACgCECIJQQFqNgIQIARBCHQgCSAAKAIMai0AAHIhBAsgACADQQF2IgM2AgAgACAEIANrIgRBH3UiCSANQQF0akEBaiINNgJUIAAgCSADcSAEaiIENgIEIA5Bf2oiDg0ACyAAIA1BBHQiCzYCVEEAIQ9BASEOA0AgACAOQQF0Ig5qQbwNaiENAkACQCADQf///wdNDQAgAyEKDAELIAAgA0EIdCIKNgIAIAAgACgCECIDQQFqNgIQIAAgBEEIdCADIAAoAgxqLQAAciIENgIECwJAAkAgBCAKQQt2IA0vAQAiCWwiA0kNACAAIAQgA2siBDYCBCAAIAogA2siAzYCACANIAkgCUEFdms7AQAgAEEBIA90IAtqIgs2AlQgDkEBciEODAELIAAgAzYCACANIAlBgBAgCWtBBXZqOwEACyAPQQFqIg9BBEcNAAsLAkAgASAGIAAoAlQQnoCAgAANAEEADwsgACgCICECCyACIAAoAihJDQALC0EBIQMCQCAAKAIAIgRB////B0sNACAAIARBCHQ2AgBBASEDIAAgACgCECIEQQFqNgIQIAAgACgCBEEIdCAEIAAoAgxqLQAAcjYCBAsgAwtwAQF/AkBBqN4BEICAgIAAIgJFDQAgAkE0aiABNgIAIAJBPGogADYCAAJAAkACQCAAQX9qDgIAAQILIAIgARCAgICAACIANgIYIAANASACEIKAgIAADAILIAJBADYCGCACQThqQQA2AgALIAIPC0EAC9IBAQJ/QQYhAgJAIAFBJ0sNACAAQTBqIAFBAXFBAnIgAUEBdkELanQiATYCAAJAAkAgAEE8aigCACIDRQ0AQQQhAiABIABBNGooAgBLDQIgAEEsaiABNgIAIANBAkcNACAAQThqIgMoAgAgAU8NACAAIAE2AjggACgCGBCCgICAACAAIAAoAjAQgICAgAAiATYCGCABDQBBAyECDAELQQAhAiAAQQA2AkAgAEHQAGpBAToAACAAQegAakEANgIAIABB5N0BaiEDCyADQQA2AgALIAILIwACQCAAQTxqKAIARQ0AIAAoAhgQgoCAgAALIAAQgoCAgAAL9QEBBH9BACEDAkAgACgCDCACTQ0AIAAoAhggAk0NACABIAEoAgAiBCAAKAIQIAAoAggiBWsiBiAEIAYgBEkbIgRrNgIAIAUgAkF/c2ohAQJAIAUgAksNACAAKAIUIAFqIQELIAAoAgAiAiABai0AACEGQQEhAyAAIAVBAWo2AgggAiAFaiAGOgAAAkAgBEF/aiICRQ0AA0AgACgCACIFQQAgAUEBaiIBIAEgACgCFEYbIgFqLQAAIQQgACAAKAIIIgZBAWo2AgggBSAGaiAEOgAAIAJBf2oiAg0ACwsgACgCDCAAKAIIIgFPDQAgACABNgIMCyADC8gEAQd/AkACQCAAKAIAIgNBgICACEkNACAAKAIEIQQMAQsgACADQQh0IgM2AgAgACAAKAIQIgVBAWo2AhAgACAAKAIEQQh0IAUgACgCDGotAAByIgQ2AgQLAkACQCAEIANBC3YgAS8BACIGbCIFTw0AIAEgBkGAECAGa0EFdmo7AQAgASACQQR0akEEaiEHQQghCEECIQkMAQsgACAEIAVrIgQ2AgQgASAGIAZBBXZrOwEAAkAgAyAFayIDQf///wdLDQAgACADQQh0IgM2AgAgACAAKAIQIgVBAWo2AhAgACAEQQh0IAUgACgCDGotAAByIgQ2AgQLAkAgBCADQQt2IAEvAQIiBmwiBU8NACABIAZBgBAgBmtBBXZqOwECIAEgAkEEdGpBhAJqIQdBCCEIQQohCQwBCyAAIAQgBWsiBDYCBCABIAYgBkEFdms7AQIgAUGEBGohByADIAVrIQVBgAIhCEESIQkLIABB6ABqIAk2AgBBASEBA0AgByABQQF0IgFqIQMCQAJAIAVB////B00NACAFIQIMAQsgACAFQQh0IgI2AgAgACAAKAIQIgVBAWo2AhAgACAEQQh0IAUgACgCDGotAAByIgQ2AgQLAkACQCAEIAJBC3YgAy8BACIGbCIFSQ0AIAAgBCAFayIENgIEIAAgAiAFayIFNgIAIAMgBiAGQQV2azsBACABQQFyIQEMAQsgACAFNgIAIAMgBkGAECAGa0EFdmo7AQALIAEgCEkNAAsgAEHoAGogASAIayAJajYCAAsLNQEAQYAICy4IAAAAEAAAABgAAAAgAAAAKAAAADAAAABAAAAAUAAAAIAAAAAAAQAA/Td6WFoA"; + +/***/ }), +/* 2 */ +/***/ ((module) => { + +module.exports = __WEBPACK_EXTERNAL_MODULE__2__; + +/***/ }) +/******/ ]); +/************************************************************************/ +/******/ // The module cache +/******/ var __webpack_module_cache__ = {}; +/******/ +/******/ // The require function +/******/ function __webpack_require__(moduleId) { +/******/ // Check if module is in cache +/******/ var cachedModule = __webpack_module_cache__[moduleId]; +/******/ if (cachedModule !== undefined) { +/******/ return cachedModule.exports; +/******/ } +/******/ // Create a new module (and put it into the cache) +/******/ var module = __webpack_module_cache__[moduleId] = { +/******/ // no module.id needed +/******/ // no module.loaded needed +/******/ exports: {} +/******/ }; +/******/ +/******/ // Execute the module function +/******/ __webpack_modules__[moduleId](module, module.exports, __webpack_require__); +/******/ +/******/ // Return the exports of the module +/******/ return module.exports; +/******/ } +/******/ +/************************************************************************/ +/******/ /* webpack/runtime/define property getters */ +/******/ (() => { +/******/ // define getter functions for harmony exports +/******/ __webpack_require__.d = (exports, definition) => { +/******/ for(var key in definition) { +/******/ if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) { +/******/ Object.defineProperty(exports, key, { enumerable: true, get: definition[key] }); +/******/ } +/******/ } +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/hasOwnProperty shorthand */ +/******/ (() => { +/******/ __webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop)) +/******/ })(); +/******/ +/******/ /* webpack/runtime/make namespace object */ +/******/ (() => { +/******/ // define __esModule on exports +/******/ __webpack_require__.r = (exports) => { +/******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) { +/******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' }); +/******/ } +/******/ Object.defineProperty(exports, '__esModule', { value: true }); +/******/ }; +/******/ })(); +/******/ +/************************************************************************/ +var __webpack_exports__ = {}; +// This entry need to be wrapped in an IIFE because it need to be isolated against other modules in the chunk. +(() => { +__webpack_require__.r(__webpack_exports__); +/* harmony export */ __webpack_require__.d(__webpack_exports__, { +/* harmony export */ XzReadableStream: () => (/* binding */ XzReadableStream) +/* harmony export */ }); +/* harmony import */ var _dist_native_xz_decompress_wasm__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(1); + + +const ReadableStream = globalThis.ReadableStream + // Node < 18 support web streams, but it's not available as a global, so we need to require it. + // This won't be reached in modern browsers, and bundlers will ignore due to 'browser' field in package.json: + || (__webpack_require__(2).ReadableStream); + +const XZ_OK = 0; +const XZ_STREAM_END = 1; + +class XzContext { + constructor(moduleInstance) { + this.exports = moduleInstance.exports; + this.memory = this.exports.memory; + this.ptr = this.exports.create_context(); + this._refresh(); + this.bufSize = this.mem32[0]; + this.inStart = this.mem32[1] - this.ptr; + this.inEnd = this.inStart + this.bufSize; + this.outStart = this.mem32[4] - this.ptr; + } + + supplyInput(sourceDataUint8Array) { + this._refresh(); + const inBuffer = this.mem8.subarray(this.inStart, this.inEnd); + inBuffer.set(sourceDataUint8Array, 0); + this.exports.supply_input(this.ptr, sourceDataUint8Array.byteLength); + this._refresh(); + } + + getNextOutput() { + const result = this.exports.get_next_output(this.ptr); + this._refresh(); + if (result !== XZ_OK && result !== XZ_STREAM_END) { + throw new Error(`get_next_output failed with error code ${result}`); + } + const outChunk = this.mem8.slice(this.outStart, this.outStart + /* outPos */ this.mem32[5]); + return { outChunk, finished: result === XZ_STREAM_END }; + } + + needsMoreInput() { + return /* inPos */ this.mem32[2] === /* inSize */ this.mem32[3]; + } + + outputBufferIsFull() { + return /* outPos */ this.mem32[5] === this.bufSize; + } + + resetOutputBuffer() { + this.outPos = this.mem32[5] = 0; + } + + dispose() { + this.exports.destroy_context(this.ptr); + this.exports = null; + } + + _refresh() { + if (this.memory.buffer !== this.mem8?.buffer) { + this.mem8 = new Uint8Array(this.memory.buffer, this.ptr); + this.mem32 = new Uint32Array(this.memory.buffer, this.ptr); + } + } +} + +class XzReadableStream extends ReadableStream { + static _moduleInstancePromise; + static _moduleInstance; + static async _getModuleInstance() { + const base64Wasm = _dist_native_xz_decompress_wasm__WEBPACK_IMPORTED_MODULE_0__.replace('data:application/wasm;base64,', ''); + const wasmBytes = Uint8Array.from(atob(base64Wasm), c => c.charCodeAt(0)).buffer; + const wasmOptions = {}; + const module = await WebAssembly.instantiate(wasmBytes, wasmOptions); + XzReadableStream._moduleInstance = module.instance; + } + + constructor(compressedStream) { + let xzContext; + let unconsumedInput = null; + const compressedReader = compressedStream.getReader(); + + super({ + async start(controller) { + if (!XzReadableStream._moduleInstance) { + await (XzReadableStream._moduleInstancePromise || (XzReadableStream._moduleInstancePromise = XzReadableStream._getModuleInstance())); + } + xzContext = new XzContext(XzReadableStream._moduleInstance); + }, + + async pull(controller) { + if (xzContext.needsMoreInput()) { + if (unconsumedInput === null || unconsumedInput.byteLength === 0) { + const { done, value } = await compressedReader.read(); + if (!done) { + unconsumedInput = value; + } + } + const nextInputLength = Math.min(xzContext.bufSize, unconsumedInput.byteLength); + xzContext.supplyInput(unconsumedInput.subarray(0, nextInputLength)); + unconsumedInput = unconsumedInput.subarray(nextInputLength); + } + + const nextOutputResult = xzContext.getNextOutput(); + controller.enqueue(nextOutputResult.outChunk); + xzContext.resetOutputBuffer(); + + if (nextOutputResult.finished) { + xzContext.dispose(); // Not sure if this always happens + controller.close(); + } + }, + cancel() { + xzContext.dispose(); // Not sure if this always happens + return compressedReader.cancel(); + } + }); + } +} + +})(); + +/******/ return __webpack_exports__; +/******/ })() +; +}); \ No newline at end of file diff --git a/queries/cdmq/node_modules/xz-decompress/dist/package/xz-decompress.min.js b/queries/cdmq/node_modules/xz-decompress/dist/package/xz-decompress.min.js new file mode 100644 index 00000000..547473d6 --- /dev/null +++ b/queries/cdmq/node_modules/xz-decompress/dist/package/xz-decompress.min.js @@ -0,0 +1,6 @@ +/*! + * Based on xzwasm (c) Steve Sanderson. License: MIT - https://github.com/SteveSanderson/xzwasm + * Contains xz-embedded by Lasse Collin and Igor Pavlov. License: Public domain - https://tukaani.org/xz/embedded.html + * and walloc (c) 2020 Igalia, S.L. License: MIT - https://github.com/wingo/walloc + */ +!function(A,I){"object"==typeof exports&&"object"==typeof module?module.exports=I(require("stream/web")):"function"==typeof define&&define.amd?define(["stream/web"],I):"object"==typeof exports?exports["xz-decompress"]=I(require("stream/web")):A["xz-decompress"]=I(A["stream/web"])}(this,(A=>(()=>{"use strict";var I=[,A=>{A.exports="data:application/wasm;base64,AGFzbQEAAAABOApgAX8Bf2ABfwBgAABgA39/fwF/YAABf2ACf38AYAN/f34BfmACf38Bf2AEf39/fwF/YAN/f38AAyEgAAABAgMDAwMEAQUAAgMCBgcIBwUDAAMHAQcABwcBAwkFAwEAAgYIAX8BQfCgBAsHTgUGbWVtb3J5AgAOY3JlYXRlX2NvbnRleHQACA9kZXN0cm95X2NvbnRleHQACQxzdXBwbHlfaW5wdXQACg9nZXRfbmV4dF9vdXRwdXQACwqNYCDfAgEFf0EAIQECQCAAQQdqIgJBEEkNAEEBIQEgAkEDdiIDQQJGDQBBAiEBIAJBIEkNAEEDIQEgA0EERg0AQQQhASACQTBJDQBBBSEBIANBBkYNAEEGIQEgAkHIAEkNAEEHIQEgAkHYAEkNAEEIIQEgAkGIAUkNAEEJIQEgAkGIAkkNACAAEIGAgIAAIgBBCGpBACAAGw8LAkACQCABQQJ0QcCIgIAAaiIEKAIAIgANAEEAIQACQAJAQQAoAuSIgIAAIgJFDQBBACACKAIANgLkiICAAAwBC0EAEIGAgIAAIgJFDQILIAJBgIB8cSIAIAJBCHZB/wFxIgJyIAE6AAAgAkEIdCAAckGAAmohAEEAIQJBACABQQJ0QYCIgIAAaigCACIDayEFIAMhAQNAIAAgBWoiACACNgIAIAAhAiABIANqIgFBgQJJDQALIAQgADYCAAsgBCAAKAIANgIACyAAC+4HAQd/AkACQAJAAkACQEEALQC0iICAAEUNAEEAQQA6ALSIgIAAQQAoArCIgIAAIgFFDQFBsIiAgAAhAgNAAkACQCABQQhqIgMgASgCBCIEaiIFQQh2Qf8BcSIGDQAgASECDAELAkADQCAFQYCAfHEgBmotAABB/gFHDQFBsIiAgAAhBgNAIAYiBygCACIGIAVHDQALIAcgBSgCADYCACABIAQgBSgCBGpBCGoiBDYCBCAHIAIgAiAFRhshAiADIARqIgVBCHZB/wFxIgYNAAsLIAIoAgAhAgsgAigCACIBDQALC0EAKAKwiICAACIFRQ0AIABBhwJqQYB+cSEDQX8hAkGwiICAACEEQQAhAUGwiICAACEGA0AgBiEHAkAgBSIGKAIEIgUgAEkNACAFIAJPDQAgBSECIAchBCAGIQEgBUEIaiADRw0AIAchBCAFIQIgBiEBDAQLIAYoAgAiBQ0ACyABDQIMAQtBsIiAgAAhBAs/AEEQdCEBIABBiAJqIQdBACEDAkACQEEAKAK4iICAACICRQ0AQQAhBSABIQYMAQtBACABQfCghIAAQf//A2pBgIB8cSIGayICNgK4iICAACACIQULAkAgByAFTQ0AIAcgBWsiByACQQF2IgIgAiAHSRtB//8DaiIHQRB2QABBf0YNAkEAQQAoAriIgIAAIAdBgIB8cSIDajYCuIiAgAALIAZFDQEgBkH/AToAASAGQQAoArCIgIAANgKAAiAGQYQCaiADIAVqQYCAfHFB+H1qIgI2AgAgBkGAAmohAQsgAUGAgHxxIgYgAUEIdkH/AXFyQf8BOgAAIAQgASgCADYCAAJAIAIgAGtBgH5xIgUNACABDwsgASEDAkAgBiABQQhqIgQgAmoiByAFQX9zakGAgHxxRg0AIARB//8DcSEFAkAgAEH3/QNLDQAgBiAEQQh2Qf8BcWpB/gE6AAAgAUEAKAKwiICAADYCACABQYCABCAFayIFNgIEQQAgATYCsIiAgAAQg4CAgAAgBkGEggRqIAIgBWtB+H1qIgU2AgAgBkGBgARqQf8BOgAAIAZBgIIEaiEDIAUgAGtBgH5xIQUMAQsgAiAFaiAAIAVqQX9qQYCAfHFrQYCAfGohBSABIQMLIAMgAygCBCAFazYCBCAFQfgBaiEGIAcgBWtBCHZB/wFxIQUCQANAIAYiB0GAfmohBiAFIgQNAUEBIQUgB0H4AUcNAAsLAkAgB0H4AUYNACACIAFqIAZrQYCAfHEiBSAEakH+AToAACAFIARBCHRqIgVBACgCsIiAgAA2AgAgBSAGNgIEQQAgBTYCsIiAgAAQg4CAgAALIAMPC0EAC3wBAn8CQCAARQ0AAkAgAEGAgHxxIABBCHZB/wFxciIBLQAAIgJB/wFHDQAgAEF4aiIAQQAoArCIgIAANgIAQQAgADYCsIiAgAAgAUH+AToAAEEAQQE6ALSIgIAADwsgACACQQJ0QcCIgIAAaiICKAIANgIAIAIgADYCAAsLawECfwJAQQAoArCIgIAAIgAoAgRB/wFLDQAgAEGAgHxxIgEgAEEIdkH/AXEiAHJBCToAAEEAQQAoArCIgIAAKAIANgKwiICAACABIABBCHRyIgBBACgC5IiAgAA2AgBBACAANgLkiICAAAsLTgECfwJAIAAgAUYNACACRQ0AA0ACQCAALQAAIgMgAS0AACIERg0AQQFBfyADIARLGw8LIAFBAWohASAAQQFqIQAgAkF/aiICDQALC0EAC3gBAX8CQAJAIAAgAU8NACACRQ0BIAAhAwNAIAMgAS0AADoAACABQQFqIQEgA0EBaiEDIAJBf2oiAg0ADAILCyAAIAFNDQAgAkUNACABQX9qIQEgAEF/aiEDA0AgAyACaiABIAJqLQAAOgAAIAJBf2oiAg0ACwsgAAssAQF/AkAgAkUNACAAIQMDQCADIAE6AAAgA0EBaiEDIAJBf2oiAg0ACwsgAAuCAQEBfwJAAkAgAEEDcQ0AIAEgAnJBA3ENACACQQRJDQEgAkECdiECIAAhAwNAIAMgASgCADYCACABQQRqIQEgA0EEaiEDIAJBf2oiAg0ADAILCyACRQ0AIAAhAwNAIAMgAS0AADoAACABQQFqIQEgA0EBaiEDIAJBf2oiAg0ACwsgAAuIAQECfwJAQQAtAOiIgIAADQBBAEEBOgDoiICAABCMgICAABCOgICAAAtBoIAIEICAgIAAIgBBgIAENgIAQQJBgICAIBCXgICAACEBIABBFGpCgICAgICAwAA3AgAgAEEQaiAAQaCABGo2AgAgAEEIakIANwMAIAAgAEEgajYCBCAAIAE2AhwgAAsVACAAKAIcEJiAgIAAIAAQgoCAgAALFgAgAEEMaiABNgIAIABBCGpBADYCAAsbACAAKAIcIABBBGogAEEMaigCAEUQloCAgAALVAEDf0EAIQADQEEIIQEgACECA0BBACACQQFxa0GghuLtfnEgAkEBdnMhAiABQX9qIgENAAsgAEECdEHwiICAAGogAjYCACAAQQFqIgBBgAJHDQALC0oAIAJBf3MhAgJAIAFFDQADQCACQf8BcSAALQAAc0ECdEHwiICAAGooAgAgAkEIdnMhAiAAQQFqIQAgAUF/aiIBDQALCyACQX9zC10DAX4BfwF+QgAhAANAQQghASAAIQIDQEIAIAJCAYN9QsKenLzd8pW2SYMgAkIBiIUhAiABQX9qIgENAAsgAKdBA3RB8JCAgABqIAI3AwAgAEIBfCIAQoACUg0ACwtLACACQn+FIQICQCABRQ0AA0AgAkL/AYMgADEAAIWnQQN0QfCQgIAAaikDACACQgiIhSECIABBAWohACABQX9qIgENAAsLIAJCf4UL1RACDH8CfgJAAkAgACgCJEUNACAAKAIAIQIMAQtBACECIABBADoAKCAAQgA3AwAgAEIANwMYIABByABqQQBB5AAQhoCAgAAaIABBrAFqQQw2AgALIAAgASgCBCIDNgIQIABBsAFqIQQgAEHgAGohBSAAQcgAaiEGIABBtgFqIQcgAEGoAWohCCABKAIQIQkCQAJAAkACQANAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAIAIOCgECAAQFBgcICQoPCyABKAIAIQogACgCqAEhAiAAKAKsASELIAEoAgQhDCABKAIIIQ0MAgsgCCAAKAKoASIMakEIaiABKAIAIAEoAgQiAmogASgCCCACayICIAAoAqwBIAxrIgwgAiAMSRsiAhCHgICAABogASABKAIEIAJqNgIEQQAhDCAAQQAgACgCqAEgAmoiAiACIAAoAqwBIgtGGzYCqAEgAiALRw0RIABBATYCAAJAIARBqIiAgABBBhCEgICAAEUNAEEFIQwMEgsgB0ECQQAQjYCAgAAgACgAuAFHDRBBBiEMIActAAANESAAIAAtALcBIgI2AiAgAkEESw0RQQEgAnRBE3FFDRELIAEoAgQiDCABKAIIIg1GDQ4CQCABKAIAIgogDGotAAAiCw0AIAAgDDYCECABIAxBAWo2AgRBBiECDAwLQQAhAiAAQQA2AqgBIABBAjYCACAAIAtBAnRBBGoiCzYCrAEgACALNgJACyAIIAJqQQhqIAogDGogDSAMayIMIAsgAmsiAiAMIAJJGyICEIeAgIAAGiABIAIgASgCBGo2AgRBACEMIABBACAAKAKoASACaiICIAIgACgCrAEiC0YbNgKoASACIAtHDQ8gACACQXxqIgI2AqwBQQchDCAEIAJBABCNgICAACAAIAAoAqwBIgtqQbABaigAAEcNDyAAQQI2AqgBIAAtALEBIgJBP3ENDAJAAkAgAkHAAHFFDQAgACAEIAggCxCRgICAAEEBRw0RIAAgACkDCDcDMCAALQCxASECDAELIABCfzcDMAtCfyEOAkAgAkEYdEEYdUF/Sg0AIAAgBCAIIAAoAqwBEJGAgIAAQQFHDRAgACkDCCEOCyAAIA43AzggACgCrAEiDSAAKAKoASICa0ECSQ0PIAAgAkEBaiIKNgKoASAIIAJqQQhqLQAAQSFHDQwgACACQQJqIgs2AqgBIAggCmpBCGotAABBAUcNDCANIAtGDQ8gACACQQNqNgKoASAAKAKwCSAIIAtqQQhqLQAAEJyAgIAAIgwNDyAAKAKoASIMIAAoAqwBIgIgDCACSxshDQJAA0AgDSAMRg0BIAggDEEBaiICNgIAIAQgDGohCyACIQwgCy0AAA0ODAALCyAGQgA3AwAgAEEANgKoASAAQQM2AgAgBkEIakIANwMACyAAIAEoAgQ2AhAgACABKAIQNgIUIAAoArAJIAEQmYCAgAAhDCAAIAApA0ggASgCBCAAKAIQa618Ig43A0ggACAAKQNQIAEoAhAgACgCFCICayILrXwiDzcDUCAOIAApAzBWDQ0gDyAAKQM4Vg0NAkACQAJAAkAgACgCIEF/ag4EAAMDAQMLIAEoAgwgAmogCyAAKAIYEI2AgIAArSEODAELIAEoAgwgAmogCyAAKQMYEI+AgIAAIQ4LIAAgDjcDGAsgDEEBRw0OAkAgACkDMCIOQn9RDQAgDiAGKQMAUg0OCwJAIAApAzgiDkJ/UQ0AQQchDCAOIAApA1BSDQ8LIAAgACkDSCAANQJAfCAAKQNgfCIPNwNgQgQhDgJAAkACQCAAKAIgQX9qDgQBAgIAAgtCCCEOCyAFIA4gD3w3AwALIAAgACkDaCAAKQNQfDcDaCAAIAVBGCAAKAJwEI2AgIAANgJwIABBBDYCACAAIAApA1hCAXw3A1gLAkAgBikDACIOQgODUA0AIA5CAXwhDiABKAIEIQwgASgCCCELA0AgCyAMRg0NIAEgDEEBaiICNgIEIAEoAgAgDGotAAANDiAGIA43AwAgDkIDgyEPIA5CAXwhDiACIQwgD0IAUg0ACwsgAEEFNgIAC0EBIQIgACgCIEF/ag4EBgcHBQcLIAAgARCSgICAACIMQQFHDQsgAEEHNgIAC0EAIAAoAhBrIQggAEGAAWopAwAhDiABKAIEIQwCQANAIA4gCCAMaq18QgODUA0BAkAgDCABKAIIRw0AIAAgARCTgICAAAwLCyABIAxBAWoiAjYCBCABKAIAIAxqIQsgAiEMIAstAAANCwwACwsgACABEJOAgIAAQQchDCAFIABBkAFqQRgQhICAgAANCiAAQQg2AgALIAAgAUEgEJSAgIAAIgxBAUcNCSAAQQk2AgBBDCELIABBDDYCrAEMAQsgACgCrAEhCwsgAEGoAWogACgCqAEiDGpBCGogASgCACABKAIEIgJqIAEoAgggAmsiAiALIAxrIgwgAiAMSRsiAhCHgICAABogASABKAIEIAJqNgIEQQAhDCAAQQAgACgCqAEgAmoiAiACIAAoAqwBIgtGGzYCqAEgAiALRw0HIAAQlYCAgAAhDAwHC0EBIQIgACABQcAAEJSAgIAAIgxBAUcNBgwBC0EBIQIgACABQSAQlICAgAAiDEEBRw0FCyAAIAI2AgAMAAsLQQYhDAwCC0EAIQwMAQtBByEMCwJAAkAgACgCJA0AAkACQCAMDgIAAwELQQdBCCABKAIEIAEoAghGGyEMCyABIAk2AhAgASADNgIEIAwPCwJAIAwNACADIAEoAgRHDQAgCSABKAIQRw0AIAAtACghASAAQQE6ACggAUEDdA8LIABBADoAKAsgDAuaAQEDfwJAIAAoAgQiBA0AIABCADcDCAsgAigCACEFA0ACQCAFIANJDQBBAA8LIAEgBWotAAAhBiACIAVBAWoiBTYCACAAIAZB/wBxrSAErYYgACkDCIQ3AwgCQAJAIAZBgAFxDQACQCAGDQBBByEGIAQNAgsgAEEANgIEQQEPC0EHIQYgACAEQQdqIgQ2AgQgBEE/Rw0BCwsgBguhAgIDfwF+IABBkAFqIQIgAUEEaiEDA0ACQCAAIAEoAgAgAyABKAIIEJGAgIAAIgRBAUYNACAAQYABaiIDIAMpAwAgASgCBCAAKAIQIgNrIgKtfDcDACAAIAMgASgCAGogAiAAKAIYEI2AgIAArTcDGCAEDwsCQAJAAkACQAJAIAAoAngOAwACAQMLIAAgACkDCCIFNwOIAQJAIAUgACkDWFENAEEHDwsgAEEBNgJ4DAMLIAAgACkDmAEgACkDCHw3A5gBIAAgAkEYIAAoAqABEI2AgIAANgKgASAAQQE2AnggACAAKQOIAUJ/fCIFNwOIAQwCCyAAQQI2AnggACAAKQOQASAAKQMIfDcDkAELIAApA4gBIQULIAVCAFINAAtBAQtAAQJ/IABBgAFqIgIgAikDACABKAIEIAAoAhAiAmsiA618NwMAIAAgAiABKAIAaiADIAAoAhgQjYCAgACtNwMYC3wBBH8gASgCBCEDIAEoAgghBANAAkAgBCADRw0AQQAPCyABIANBAWoiBTYCBAJAIAEoAgAgA2otAAAgACkDGCAAKAIEIgOtiKdB/wFxRg0AQQcPCyAAIANBCGoiBjYCBCAFIQMgBiACSQ0ACyAAQQA2AgQgAEIANwMYQQELbwEBf0EHIQECQCAAQboBai8AAEHZtAFHDQAgAEG0AWpBBkEAEI2AgIAAIABBsAFqKAAARw0AIABBgAFqKQMAQgKIIAA1ALQBUg0AIABBuAFqLQAADQBBAUEHIAAoAiAgAEG5AWotAABGGyEBCyABC7QCAQR/AkACQCAAKAIkRQ0AIAAoAgAhAwwBC0EAIQMgAEEAOgAoIABCADcDACAAQgA3AxggAEHIAGpBAEHkABCGgICAABogAEGsAWpBDDYCAEEBIQILIABByABqIQQCQAJAA0ACQCADQQpHDQAgASgCBCIDIAEoAggiBUYNAiABKAIAIQYCQANAIAYgA2otAAANASABIANBAWoiAzYCBCAAIAAoAgRBAWpBA3E2AgQgBSADRg0EDAALCwJAIAAoAgRFDQBBBw8LIAAoAiRFDQAgAEEAOgAoIABCADcDACAAQgA3AxggBEEAQeQAEIaAgIAAGiAAQQw2AqwBCyAAIAEQkICAgAAiA0EBRw0CQQohAyAAQQo2AgAMAAsLAkAgAg0AQQAPC0EHQQEgACgCBBshAwsgAwt1AQF/AkBBuAkQgICAgAAiAkUNACACIAA2AiQgAiAAIAEQm4CAgAAiADYCsAkCQCAARQ0AIAJBADoAKCACQgA3AwAgAkIANwMYIAJByABqQQBB5AAQhoCAgAAaIAJBrAFqQQw2AgAgAg8LIAIQgoCAgAALQQALHgACQCAARQ0AIAAoArAJEJ2AgIAAIAAQgoCAgAALC4ARAQx/IABB6N0BaiECIABB1ABqIQMgAEEcaiIEQQhqIQUCQAJAA0AgACgCQCEGAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkAgASgCBCIHIAEoAggiCEkNACAGQQdGDQEMEgsgBg4JAQIDBAUGBwAJDwsgACgCTCEGDAcLQQEhCSABIAdBAWo2AgQgASgCACAHai0AACIHRQ0IAkACQCAHQd8BSw0AIAdBAUcNAQsgAEGAAjsBUAJAIAAoAjwNACAAIAEoAgwgASgCECIGajYCGCAAIAEoAhQgBms2AiwLIARCADcCACAFQgA3AgAMCwsgAC0AUEUNCgwOCyABIAdBAWo2AgQgASgCACAHai0AACEHIABBAjYCQCAAIAdBCHQgACgCSGo2AkgMDAsgASAHQQFqNgIEIAEoAgAgB2otAAAhByAAQQM2AkAgACAHIAAoAkhqQQFqNgJIDAsLIAEgB0EBajYCBCABKAIAIAdqLQAAIQcgAEEENgJAIAAgB0EIdDYCTAwKCyABIAdBAWo2AgQgASgCACAHai0AACEHIAAgACgCRDYCQCAAIAcgACgCTGpBAWo2AkwMCQsgASAHQQFqNgIEQQchCSABKAIAIAdqLQAAIgdB4AFLDQNBACEGAkACQCAHQS1PDQBBACEIDAELIAdBU2oiByAHQf8BcUEtbiIIQS1sayEHIAhBAWohCAsgAEF/IAh0QX9zNgJ0AkAgB0H/AXFBCUkNACAHQXdqIgcgB0H/AXFBCW4iBkEJbGshByAGQQFqIQYLIAAgBjYCcCAAIAdB/wFxIgc2AmwgBiAHakEESw0DIANCADcCACADQQhqQgA3AgAgA0EQakEANgIAIABBfyAGdEF/czYCcEH4ACEHA0AgACAHakGACDsBACAHQQJqIgdB5N0BRw0ACyAAQQY2AkAgAEEFNgIIIABC/////w83AgALIAAoAkwiCUEFSQ0IAkAgACgCCCIHRQ0AIAdBf2ohBiABKAIEIQcgASgCCCEKA0AgCiAHRg0LIAEgB0EBaiIINgIEIAEoAgAgB2otAAAhByAAIAY2AgggACAHIAAoAgRBCHRyNgIEIAghByAGQX9qIgZBf0cNAAsLIABBBzYCQCAAIAlBe2oiBjYCTAsgACAAKAIgIgcgASgCFCABKAIQayIIIAAoAkgiCiAIIApJGyIIaiAAKAIsIgogCiAHayAISxs2AiggASgCCCIJIAEoAgQiCGshBwJAAkACQCAAKALk3QEiCg0AIAYNAUEAIQYLIABB5N0BaiILIApqQQRqIAEoAgAgCGogByAGIAprIgZBKiAKayIIIAggBksbIgYgBiAHSxsiBxCHgICAABoCQAJAIAcgACgC5N0BIghqIgYgACgCTEcNACALIAhqIAdqQQRqQQBBPyAGaxCGgICAABogACgC5N0BIAdqIQYMAQsCQCAGQRRLDQAgACAGNgLk3QEgASABKAIEIAdqNgIEDAMLIAZBa2ohBgsgAEEANgIQIAAgAjYCDCAAIAY2AhRBByEJIAAQmoCAgABFDQMgACgCECIIIAAoAuTdASIKIAdqSw0DIAAgACgCTCAIayIGNgJMAkAgCiAITQ0AIAAgCiAIayIHNgLk3QEgAiALIAhqQQRqIAcQhYCAgAAaDAILIABBADYC5N0BIAEgASgCBCAIIApraiIINgIEIAEoAggiCSAIayEHCwJAIAdBFUkNACAAIAg2AhAgACABKAIANgIMIAAgCUFraiAIIAZqIAcgBkEVakkbNgIUQQchCSAAEJqAgIAARQ0DIAAoAkwiByAAKAIQIgggASgCBGsiBkkNAyABIAg2AgQgACAHIAZrIgY2AkwgASgCCCAIayIHQRRLDQELIAIgASgCACAIaiAGIAcgByAGSxsiBxCHgICAABogACAHNgLk3QEgASABKAIEIAdqNgIECyAAKAIgIgYgACgCHCIIayEHAkAgACgCPEUNAAJAIAYgACgCLEcNACAAQQA2AiALIAEoAgwgASgCEGogACgCGCAIaiAHEIeAgIAAGiAAKAIgIQYLIAAgBjYCHCABIAEoAhAgB2oiBjYCECAAIAAoAkggB2siBzYCSAJAIAcNAEEHIQkgACgCTA0CIAAoAmgNAiAAKAIEDQIgAEEANgJADAULQQAhCSAGIAEoAhRGDQEgASgCBCABKAIIRw0GIAAoAuTdASAAKAJMTw0GDAELIAAoAkwiCkUNAUEAIQkgCCAHTQ0AA0AgASgCFCIGIAEoAhAiC00NASAAIAogCiAAKAIsIAAoAiAiDGsiDSAIIAdrIgggBiALayIGIAggBkkbIgYgBiANSxsiBiAGIApLGyIGazYCTCAMIAAoAhhqIAEoAgAgB2ogBhCFgICAABogACAAKAIgIAZqIgc2AiACQCAAKAIkIAdPDQAgACAHNgIkCwJAIAAoAjxFDQACQCAHIAAoAixHDQAgAEEANgIgCyABKAIMIAEoAhBqIAEoAgAgASgCBGogBhCFgICAABogACgCICEHCyAAIAc2AhwgASABKAIQIAZqNgIQIAEgASgCBCAGaiIHNgIEIAAoAkwiCkUNAiABKAIIIgggB0sNAAsLIAkPCyAAQQA2AkAMAwsgB0EYdEEYdUF/Sg0BIABBATYCQCAAIAdBEHRBgID8AHE2AkgCQCAHQcABSQ0AIABBBTYCRCAAQQA6AFEMAwsgAC0AUQ0DIABBBjYCRCAHQaABSQ0CIANCADcCACADQRBqQQA2AgAgA0EIakIANwIAQfgAIQcDQCAAIAdqQYAIOwEAIAdBAmoiB0Hk3QFHDQALCyAAQQU2AgggAEL/////DzcCAAwBCyAHQQJLDQEgAEKDgICAgAE3AkAMAAsLQQcPC0EAC/8XARJ/IABBGGohAQJAIABBIGooAgAiAiAAQShqKAIAIgNPDQAgAEHoAGoiBCgCAEUNACABIAQgACgCVBCegICAABogACgCKCEDIAAoAiAhAgsCQCACIANPDQAgAEHcDWohBSAAQegAaiEGIABB4BVqIQcgAEHUAGohCANAIAAoAhAiCSAAKAIUSw0BIAAgACgCZCIKQQV0aiAAKAJ0IAJxIgtBAXRqIgxB+ABqIQ0CQAJAIAAoAgAiBEGAgIAISQ0AIAAoAgQhDgwBCyAAIARBCHQiBDYCACAAIAlBAWoiAzYCECAAIAAoAgRBCHQgACgCDCAJai0AAHIiDjYCBCADIQkLAkACQCAOIARBC3YgDS8BACIPbCIDTw0AIAAgAzYCACANIA9BgBAgD2tBBXZqOwEAIAJBf2ohBAJAIAINACAAKAIsIARqIQQLAkACQCAAKAIkIg8NAEEAIQQMAQsgACgCGCAEai0AACEECyAAKAJwIAJxIAAoAmwiDXQgBEEIIA1rdmohDAJAAkAgCkEGSw0AQQEhBANAIAAgDEGADGxqIARBAXQiBGpB5B1qIQ0CQAJAIANB////B00NACADIQoMAQsgACADQQh0Igo2AgAgACAJQQFqIgM2AhAgACAOQQh0IAAoAgwgCWotAAByIg42AgQgAyEJCwJAAkAgDiAKQQt2IA0vAQAiD2wiA0kNACAAIA4gA2siDjYCBCAAIAogA2siAzYCACANIA8gD0EFdms7AQAgBEEBciEEDAELIAAgAzYCACANIA9BgBAgD2tBBXZqOwEACyAEQYACSQ0ADAILCyACIAAoAlQiDUF/c2ohBAJAIAIgDUsNACAAKAIsIARqIQQLAkACQCAPDQBBACEQDAELIAAoAhggBGotAAAhEAtBASEEQYACIQ0DQCAAIAxBgAxsaiAQQQF0IhAgDXEiESANaiAEakEBdGpB5B1qIQ8CQAJAIANB////B00NACADIQsMAQsgACADQQh0Igs2AgAgACAJQQFqIgM2AhAgACAOQQh0IAAoAgwgCWotAAByIg42AgQgAyEJCwJAAkAgDiALQQt2IA8vAQAiCmwiA08iEg0AIAAgAzYCACAKQYAQIAprQQV2aiEKDAELIAAgDiADayIONgIEIAAgCyADayIDNgIAIAogCkEFdmshCkEAIQ0LIA8gCjsBACANIBFzIQ0gBEEBdCASciIEQYACSQ0ACwsgACACQQFqNgIgIAAoAhggAmogBDoAAAJAIAAoAiQgACgCICICTw0AIAAgAjYCJAtBACEDAkAgACgCZCIEQQRJDQACQCAEQQlLDQAgBEF9aiEDDAELIARBemohAwsgACADNgJkDAELIAAgDiADayIONgIEIAAgBCADayIDNgIAIA0gDyAPQQV2azsBACAAIApBAXRqIg9B+ANqIQQCQAJAIANB////B00NACAJIQoMAQsgACADQQh0IgM2AgAgACAJQQFqIgo2AhAgACAOQQh0IAAoAgwgCWotAAByIg42AgQLAkACQCAOIANBC3YgBC8BACINbCIJSQ0AIAAgDiAJayIONgIEIAAgAyAJayIDNgIAIAQgDSANQQV2azsBACAPQZAEaiENAkACQCADQf///wdNDQAgCiEQDAELIAAgA0EIdCIDNgIAIAAgCkEBaiIQNgIQIAAgDkEIdCAAKAIMIApqLQAAciIONgIECwJAAkAgDiADQQt2IA0vAQAiCWwiBE8NACANIAlBgBAgCWtBBXZqOwEAIAxB2ARqIQMCQCAEQf///wdLDQAgACAEQQh0IgQ2AgAgACAQQQFqNgIQIAAgDkEIdCAAKAIMIBBqLQAAciIONgIECwJAIA4gBEELdiADLwEAIg1sIglJDQAgACAOIAlrNgIEIAAgBCAJazYCACADIA0gDUEFdms7AQAMAgsgAyANQYAQIA1rQQV2ajsBACAAIAk2AgAgAEEBNgJoIABBCUELIAAoAmRBB0kbNgJkDAMLIAAgDiAEayIONgIEIA0gCSAJQQV2azsBACAPQagEaiENAkACQCADIARrIgNB////B00NACAQIQoMAQsgACADQQh0IgM2AgAgACAQQQFqIgo2AhAgACAOQQh0IAAoAgwgEGotAAByIg42AgQLAkACQCAOIANBC3YgDS8BACIEbCIJTw0AIAAgCTYCACANIARBgBAgBGtBBXZqOwEAIAAoAlghAwwBCyAAIA4gCWsiDjYCBCANIAQgBEEFdms7AQAgD0HABGohBAJAIAMgCWsiA0H///8HSw0AIAAgA0EIdCIDNgIAIAAgCkEBajYCECAAIA5BCHQgACgCDCAKai0AAHIiDjYCBAsCQAJAIA4gA0ELdiAELwEAIg1sIglPDQAgACAJNgIAIAQgDUGAECANa0EFdmo7AQAgACgCXCEDDAELIAAgDiAJazYCBCAAIAMgCWs2AgAgACgCYCEDIAAgACgCXDYCYCAEIA0gDUEFdms7AQALIAAgACgCWDYCXAsgACAAKAJUNgJYIAAgAzYCVAsgAEEIQQsgACgCZEEHSRs2AmQgACAHIAsQn4CAgAAMAQsgBCANQYAQIA1rQQV2ajsBACAAIAk2AgAgACAAKAJcNgJgIAAgACkCVDcCWCAAQQdBCiAAKAJkQQdJGzYCZCAAIAUgCxCfgICAACAAKAJoIgNBfmpBAyADQQZJGyEKIAAoAgAhA0EBIQ4DQCAAIApBB3RqIA5BAXQiDmpB2AdqIQ0CQAJAIANBgICACEkNACAAKAIEIQQMAQsgACADQQh0IgM2AgAgACAAKAIQIgRBAWo2AhAgACAAKAIEQQh0IAQgACgCDGotAAByIgQ2AgQLAkACQCAEIANBC3YgDS8BACIJbCIPSQ0AIAAgBCAPayIENgIEIAAgAyAPayIDNgIAIA0gCSAJQQV2azsBACAOQQFyIQ4MAQsgACAPNgIAIA0gCUGAECAJa0EFdmo7AQAgDyEDCyAOQcAASQ0ACwJAIA5BQGoiCUEDSw0AIAAgCTYCVAwBCyAAIA5BAXFBAnIiDTYCVCAJQQF2IQ8CQCAJQQ1LDQAgACANIA9Bf2oiDHQiCzYCVEEBIQ0gCCALQQF0akGEC2ohEEE/IA5rIRFBACEPA0AgECARIA1qQQF0aiEOAkACQCADQf///wdNDQAgAyEKDAELIAAgA0EIdCIKNgIAIAAgACgCECIDQQFqNgIQIAAgBEEIdCADIAAoAgxqLQAAciIENgIECwJAAkAgBCAKQQt2IA4vAQAiCWwiA0kNACAAIAQgA2siBDYCBCAAIAogA2siAzYCACAOIAkgCUEFdms7AQAgAEEBIA90IAtqIgs2AlQgDUEBdEEBciENDAELIAAgAzYCACAOIAlBgBAgCWtBBXZqOwEAIA1BAXQhDQsgDCAPQQFqIg9HDQAMAgsLIA9Be2ohDgNAAkAgA0H///8HSw0AIAAgA0EIdCIDNgIAIAAgACgCECIJQQFqNgIQIARBCHQgCSAAKAIMai0AAHIhBAsgACADQQF2IgM2AgAgACAEIANrIgRBH3UiCSANQQF0akEBaiINNgJUIAAgCSADcSAEaiIENgIEIA5Bf2oiDg0ACyAAIA1BBHQiCzYCVEEAIQ9BASEOA0AgACAOQQF0Ig5qQbwNaiENAkACQCADQf///wdNDQAgAyEKDAELIAAgA0EIdCIKNgIAIAAgACgCECIDQQFqNgIQIAAgBEEIdCADIAAoAgxqLQAAciIENgIECwJAAkAgBCAKQQt2IA0vAQAiCWwiA0kNACAAIAQgA2siBDYCBCAAIAogA2siAzYCACANIAkgCUEFdms7AQAgAEEBIA90IAtqIgs2AlQgDkEBciEODAELIAAgAzYCACANIAlBgBAgCWtBBXZqOwEACyAPQQFqIg9BBEcNAAsLAkAgASAGIAAoAlQQnoCAgAANAEEADwsgACgCICECCyACIAAoAihJDQALC0EBIQMCQCAAKAIAIgRB////B0sNACAAIARBCHQ2AgBBASEDIAAgACgCECIEQQFqNgIQIAAgACgCBEEIdCAEIAAoAgxqLQAAcjYCBAsgAwtwAQF/AkBBqN4BEICAgIAAIgJFDQAgAkE0aiABNgIAIAJBPGogADYCAAJAAkACQCAAQX9qDgIAAQILIAIgARCAgICAACIANgIYIAANASACEIKAgIAADAILIAJBADYCGCACQThqQQA2AgALIAIPC0EAC9IBAQJ/QQYhAgJAIAFBJ0sNACAAQTBqIAFBAXFBAnIgAUEBdkELanQiATYCAAJAAkAgAEE8aigCACIDRQ0AQQQhAiABIABBNGooAgBLDQIgAEEsaiABNgIAIANBAkcNACAAQThqIgMoAgAgAU8NACAAIAE2AjggACgCGBCCgICAACAAIAAoAjAQgICAgAAiATYCGCABDQBBAyECDAELQQAhAiAAQQA2AkAgAEHQAGpBAToAACAAQegAakEANgIAIABB5N0BaiEDCyADQQA2AgALIAILIwACQCAAQTxqKAIARQ0AIAAoAhgQgoCAgAALIAAQgoCAgAAL9QEBBH9BACEDAkAgACgCDCACTQ0AIAAoAhggAk0NACABIAEoAgAiBCAAKAIQIAAoAggiBWsiBiAEIAYgBEkbIgRrNgIAIAUgAkF/c2ohAQJAIAUgAksNACAAKAIUIAFqIQELIAAoAgAiAiABai0AACEGQQEhAyAAIAVBAWo2AgggAiAFaiAGOgAAAkAgBEF/aiICRQ0AA0AgACgCACIFQQAgAUEBaiIBIAEgACgCFEYbIgFqLQAAIQQgACAAKAIIIgZBAWo2AgggBSAGaiAEOgAAIAJBf2oiAg0ACwsgACgCDCAAKAIIIgFPDQAgACABNgIMCyADC8gEAQd/AkACQCAAKAIAIgNBgICACEkNACAAKAIEIQQMAQsgACADQQh0IgM2AgAgACAAKAIQIgVBAWo2AhAgACAAKAIEQQh0IAUgACgCDGotAAByIgQ2AgQLAkACQCAEIANBC3YgAS8BACIGbCIFTw0AIAEgBkGAECAGa0EFdmo7AQAgASACQQR0akEEaiEHQQghCEECIQkMAQsgACAEIAVrIgQ2AgQgASAGIAZBBXZrOwEAAkAgAyAFayIDQf///wdLDQAgACADQQh0IgM2AgAgACAAKAIQIgVBAWo2AhAgACAEQQh0IAUgACgCDGotAAByIgQ2AgQLAkAgBCADQQt2IAEvAQIiBmwiBU8NACABIAZBgBAgBmtBBXZqOwECIAEgAkEEdGpBhAJqIQdBCCEIQQohCQwBCyAAIAQgBWsiBDYCBCABIAYgBkEFdms7AQIgAUGEBGohByADIAVrIQVBgAIhCEESIQkLIABB6ABqIAk2AgBBASEBA0AgByABQQF0IgFqIQMCQAJAIAVB////B00NACAFIQIMAQsgACAFQQh0IgI2AgAgACAAKAIQIgVBAWo2AhAgACAEQQh0IAUgACgCDGotAAByIgQ2AgQLAkACQCAEIAJBC3YgAy8BACIGbCIFSQ0AIAAgBCAFayIENgIEIAAgAiAFayIFNgIAIAMgBiAGQQV2azsBACABQQFyIQEMAQsgACAFNgIAIAMgBkGAECAGa0EFdmo7AQALIAEgCEkNAAsgAEHoAGogASAIayAJajYCAAsLNQEAQYAICy4IAAAAEAAAABgAAAAgAAAAKAAAADAAAABAAAAAUAAAAIAAAAAAAQAA/Td6WFoA"},I=>{I.exports=A}],g={};function C(A){var Q=g[A];if(void 0!==Q)return Q.exports;var B=g[A]={exports:{}};return I[A](B,B.exports,C),B.exports}C.d=(A,I)=>{for(var g in I)C.o(I,g)&&!C.o(A,g)&&Object.defineProperty(A,g,{enumerable:!0,get:I[g]})},C.o=(A,I)=>Object.prototype.hasOwnProperty.call(A,I),C.r=A=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(A,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(A,"__esModule",{value:!0})};var Q={};return(()=>{C.r(Q),C.d(Q,{XzReadableStream:()=>B});var A=C(1);const I=globalThis.ReadableStream||C(2).ReadableStream;class g{constructor(A){this.exports=A.exports,this.memory=this.exports.memory,this.ptr=this.exports.create_context(),this._refresh(),this.bufSize=this.mem32[0],this.inStart=this.mem32[1]-this.ptr,this.inEnd=this.inStart+this.bufSize,this.outStart=this.mem32[4]-this.ptr}supplyInput(A){this._refresh();this.mem8.subarray(this.inStart,this.inEnd).set(A,0),this.exports.supply_input(this.ptr,A.byteLength),this._refresh()}getNextOutput(){const A=this.exports.get_next_output(this.ptr);if(this._refresh(),0!==A&&1!==A)throw new Error(`get_next_output failed with error code ${A}`);return{outChunk:this.mem8.slice(this.outStart,this.outStart+this.mem32[5]),finished:1===A}}needsMoreInput(){return this.mem32[2]===this.mem32[3]}outputBufferIsFull(){return this.mem32[5]===this.bufSize}resetOutputBuffer(){this.outPos=this.mem32[5]=0}dispose(){this.exports.destroy_context(this.ptr),this.exports=null}_refresh(){this.memory.buffer!==this.mem8?.buffer&&(this.mem8=new Uint8Array(this.memory.buffer,this.ptr),this.mem32=new Uint32Array(this.memory.buffer,this.ptr))}}class B extends I{static _moduleInstancePromise;static _moduleInstance;static async _getModuleInstance(){const I=A.replace("data:application/wasm;base64,",""),g=Uint8Array.from(atob(I),(A=>A.charCodeAt(0))).buffer,C=await WebAssembly.instantiate(g,{});B._moduleInstance=C.instance}constructor(A){let I,C=null;const Q=A.getReader();super({async start(A){B._moduleInstance||await(B._moduleInstancePromise||(B._moduleInstancePromise=B._getModuleInstance())),I=new g(B._moduleInstance)},async pull(A){if(I.needsMoreInput()){if(null===C||0===C.byteLength){const{done:A,value:I}=await Q.read();A||(C=I)}const A=Math.min(I.bufSize,C.byteLength);I.supplyInput(C.subarray(0,A)),C=C.subarray(A)}const g=I.getNextOutput();A.enqueue(g.outChunk),I.resetOutputBuffer(),g.finished&&(I.dispose(),A.close())},cancel:()=>(I.dispose(),Q.cancel())})}}})(),Q})())); \ No newline at end of file diff --git a/queries/cdmq/node_modules/xz-decompress/package.json b/queries/cdmq/node_modules/xz-decompress/package.json new file mode 100644 index 00000000..1f30f745 --- /dev/null +++ b/queries/cdmq/node_modules/xz-decompress/package.json @@ -0,0 +1,54 @@ +{ + "name": "xz-decompress", + "version": "0.2.2", + "description": "XZ streaming decompression for the browser & Node without native code, via WebAssembly ", + "main": "dist/package/xz-decompress.js", + "types": "./types.d.ts", + "browser": { + "stream/web": false + }, + "scripts": { + "test": "mocha -r ts-node/register 'test/**/*.spec.ts'", + "webpack": "webpack" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/httptoolkit/xz-decompress.git" + }, + "keywords": [ + "xz", + "wasm" + ], + "author": "Tim Perry ", + "contributors": [ + { + "name": "Tim Perry", + "email": "tim@httptoolkit.com" + }, + { + "name": "Steven Sanderson" + } + ], + "license": "MIT", + "bugs": { + "url": "https://github.com/httptoolkit/xz-decompress/issues" + }, + "homepage": "https://github.com/httptoolkit/xz-decompress#readme", + "devDependencies": { + "@types/chai": "^4.3.5", + "@types/mocha": "^10.0.1", + "@types/node": "^20.3.0", + "chai": "^4.3.7", + "mocha": "^10.2.0", + "ts-node": "^10.9.1", + "webpack": "^5.85.1", + "webpack-cli": "^5.1.3" + }, + "engines": { + "node": ">=16" + }, + "files": [ + "dist/package/**", + "types.d.ts" + ] +} diff --git a/queries/cdmq/node_modules/xz-decompress/types.d.ts b/queries/cdmq/node_modules/xz-decompress/types.d.ts new file mode 100644 index 00000000..fdb7c3de --- /dev/null +++ b/queries/cdmq/node_modules/xz-decompress/types.d.ts @@ -0,0 +1,3 @@ +export class XzReadableStream extends ReadableStream { + constructor(compressedStream: ReadableStream); +} \ No newline at end of file diff --git a/queries/cdmq/package-lock.json b/queries/cdmq/package-lock.json index 593da6f9..e826dd50 100644 --- a/queries/cdmq/package-lock.json +++ b/queries/cdmq/package-lock.json @@ -14,7 +14,8 @@ "sprintf-js": "^1.1.2", "sync-request": "^6.1.0", "then-request": "^6.0.2", - "xz": "^2.0.2" + "xz": "^2.0.2", + "xz-decompress": "^0.2.2" }, "devDependencies": { "prettier": "3.0.3" @@ -308,6 +309,14 @@ "dependencies": { "node-addon-api": "^1.6.1" } + }, + "node_modules/xz-decompress": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/xz-decompress/-/xz-decompress-0.2.2.tgz", + "integrity": "sha512-DSOnX+ZLVTrsW+CtjZPwjrMWvuRkzCcEpwLsY2faZyVgLH/ZHpTg3h3+KyN16mGuduMgO+/pc9rSEG735oGN0g==", + "engines": { + "node": ">=16" + } } } } diff --git a/queries/cdmq/package.json b/queries/cdmq/package.json index 4cfb9038..77ea707f 100644 --- a/queries/cdmq/package.json +++ b/queries/cdmq/package.json @@ -14,7 +14,8 @@ "sprintf-js": "^1.1.2", "sync-request": "^6.1.0", "then-request": "^6.0.2", - "xz": "^2.0.2" + "xz": "^2.0.2", + "xz-decompress": "^0.2.2" }, "devDependencies": { "prettier": "3.0.3"