From c6f5ef6951c32cb923f1539db7bdb7de64e3fdc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cjeramysoucy=E2=80=9D?= Date: Wed, 27 Dec 2023 09:55:33 -0500 Subject: [PATCH 1/7] Fixes GH issue description, improves error reporting --- lib/github/batch.js | 4 +-- lib/github/index.js | 30 +++++++++++++------- lib/github/labels.js | 51 +++++++++++++++++++++------------- lib/snyk.js | 65 ++++++++++++++++++++++++++++++++------------ 4 files changed, 101 insertions(+), 49 deletions(-) diff --git a/lib/github/batch.js b/lib/github/batch.js index 9918356..492554b 100644 --- a/lib/github/batch.js +++ b/lib/github/batch.js @@ -35,8 +35,8 @@ module.exports = async (issues) => { const headerText = 'This issue has been created automatically by [snyk-github-issue-creator](https://github.com/elastic/snyk-github-issue-creator).\r\n\r\nSnyk project(s):' const projectText = projects .map( - ({ name, browseUrl, imageTag }) => - `\r\n * [\`${name}\`](${browseUrl}) (manifest version ${imageTag})` + ({ name, browseUrl }) => + `\r\n * [\`${name}\`](${browseUrl})` ) .join('') const sectionText = Object.keys(sevMap) diff --git a/lib/github/index.js b/lib/github/index.js index 4485f1c..095bc5d 100644 --- a/lib/github/index.js +++ b/lib/github/index.js @@ -21,20 +21,30 @@ module.exports = { }, // retrieve issue IDs already created in GitHub - async existingIssues () { - return await octokit.paginate( - `GET /search/issues?q=repo%3A${conf.ghOwner}/${conf.ghRepo}+is%3Aissue+label%3Asnyk`, - (response) => - response.data.map((existingIssue) => [ - existingIssue.title, - existingIssue.number - ]) - ) + async existingIssues() { + try { + return await octokit.paginate( + `GET /search/issues?q=repo%3A${conf.ghOwner}/${conf.ghRepo}+is%3Aissue+label%3Asnyk`, + (response) => + response.data.map((existingIssue) => [ + existingIssue.title, + existingIssue.number + ]) + ) + } + catch (err) { + throw new Error(`Failed to paginate octakit request for existing issues in repository ${conf.ghRepo}, error: ${err.message}`, { cause: err }) + } }, async createIssue (options) { if (conf.dryRun) return - return await this.client.issues.create(options) + try { + return await this.client.issues.create(options) + } + catch (err) { + throw new Error(`Failed to ${options.issue_number?'update':'create'} issue: ${options.issue_number?options.issue_number:options.title}, error: ${err.message}`, { cause: err }) + } }, async createIssues (issues, existingIssues) { diff --git a/lib/github/labels.js b/lib/github/labels.js index dae2aec..3d0bf1e 100644 --- a/lib/github/labels.js +++ b/lib/github/labels.js @@ -50,14 +50,22 @@ const getLabelAttributes = (name) => { const ensureLabelsAreCreated = async (octokit, client, ghOwner, ghRepo, issues) => { const labels = getLabels(issues) - const responseData = await octokit.paginate( - await client.issues.listLabelsForRepo({ - owner: ghOwner, - repo: ghRepo, - per_page: 100 - }), - (response) => response.data - ) + + let responseData + try { + responseData = await octokit.paginate( + await client.issues.listLabelsForRepo({ + owner: ghOwner, + repo: ghRepo, + per_page: 100 + }), + (response) => response.data + ) + } + catch (err) { + throw new Error(`Failed to paginate octakit request for labels in repository ${ghRepo}, error: ${err.message}`, { cause: err }) + } + const currentLabels = responseData.map((x) => x.name) const labelsToCreate = labels.filter((x) => !currentLabels.includes(x)) if (!labelsToCreate.length || conf.dryRun) { @@ -65,17 +73,22 @@ const ensureLabelsAreCreated = async (octokit, client, ghOwner, ghRepo, issues) } await Promise.all( - labelsToCreate.map((name) => - client.issues - .createLabel({ - owner: ghOwner, - repo: ghRepo, - ...getLabelAttributes(name) - }) - .then(() => { - console.log(`Created GitHub label: "${name}"`) - }) - ) + labelsToCreate.map((name) => { + try { + client.issues + .createLabel({ + owner: ghOwner, + repo: ghRepo, + ...getLabelAttributes(name) + }) + .then(() => { + console.log(`Created GitHub label: "${name}"`) + }) + } + catch (err) { + throw new Error(`Failed to create GitHub label '${name}', error: ${err.message}`, { cause: err }) + } + }) ) } diff --git a/lib/snyk.js b/lib/snyk.js index e31aa9a..26fcd58 100644 --- a/lib/snyk.js +++ b/lib/snyk.js @@ -3,7 +3,7 @@ const request = require('request-promise-native') const baseV1Url = 'https://snyk.io/api/v1' -const baseRestUrl = 'https://api.snyk.io' +const baseRestUrl = 'https://api.snyk.io/rest' module.exports = class Snyk { constructor ({ token, orgId, minimumSeverity }) { @@ -29,25 +29,49 @@ module.exports = class Snyk { ).orgs } + async queryOrgInfo(organizationId) { + try { + const response = await request({ + method: 'get', + url: `${baseRestUrl}/orgs/${organizationId}?version=2023-11-27`, + headers: this._headers, + json: true + }) + + if (response.data === undefined || response.data.attributes === undefined || response.data.attributes.name === undefined) { + throw new Error (`expected response to include data.attributes.name`) + } + + return response.data + } + catch (err) { + throw new Error(`Failed to query snyk organization with id ${organizationId}, error: ${err.message}`, { cause: err }) + } + } + async projects (orgId, selectedProjects = []) { const organizationId = orgId || this._orgId + const organization = await this.queryOrgInfo(organizationId) + const responseData = await paginateRestResponseData( - `${baseRestUrl}/rest/orgs/${organizationId}/projects?version=2023-11-27&meta.latest_issue_counts=true&limit=20`, + `${baseRestUrl}/orgs/${organizationId}/projects?version=2023-11-27&meta.latest_issue_counts=true&limit=20`, this._headers ) return responseData.map((project) => { const { critical, high, medium, low } = project.meta.latest_issue_counts const issueCountTotal = critical + high + medium + low + return { id: project.id, name: project.attributes.name, isMonitored: project.attributes.status === 'active', - issueCountTotal + issueCountTotal, + browseUrl: `https://app.snyk.io/org/${organization.attributes.name.toLowerCase()}/project/${project.id}`, } - }).filter(({ id, isMonitored, issueCountTotal }) => { + }).filter(({ id, isMonitored }) => { if (selectedProjects.includes(id)) { return true } @@ -98,19 +122,24 @@ function getSeverities (minimumSeverity) { return ['critical', 'high', 'medium', 'low'] } -async function paginateRestResponseData (url, headers, method = 'get') { - const reponseData = [] - do { - const response = await request({ - method, - url, - headers, - json: true - }) - reponseData.push(...response.data) - if (response.links.next) url = baseRestUrl + response.links.next - else url = undefined - } while (url) +async function paginateRestResponseData(url, headers, method = 'get') { + try { + const reponseData = [] + do { + const response = await request({ + method, + url, + headers, + json: true + }) + reponseData.push(...response.data) + if (response.links.next) url = baseRestUrl + response.links.next / trimStart('/rest') + else url = undefined + } while (url) - return reponseData + return reponseData + } + catch (err) { + throw new Error(`Failed to paginate request for ${method} ${url}, error: ${err.message}`, { cause: err }) + } } From af7fdc544766c1beeb212137135a19624a129d5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cjeramysoucy=E2=80=9D?= Date: Wed, 27 Dec 2023 10:28:31 -0500 Subject: [PATCH 2/7] Fixes style --- lib/github/index.js | 10 ++++------ lib/github/labels.js | 8 +++----- lib/snyk.js | 16 +++++++--------- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/lib/github/index.js b/lib/github/index.js index 095bc5d..5a39efd 100644 --- a/lib/github/index.js +++ b/lib/github/index.js @@ -21,7 +21,7 @@ module.exports = { }, // retrieve issue IDs already created in GitHub - async existingIssues() { + async existingIssues () { try { return await octokit.paginate( `GET /search/issues?q=repo%3A${conf.ghOwner}/${conf.ghRepo}+is%3Aissue+label%3Asnyk`, @@ -31,8 +31,7 @@ module.exports = { existingIssue.number ]) ) - } - catch (err) { + } catch (err) { throw new Error(`Failed to paginate octakit request for existing issues in repository ${conf.ghRepo}, error: ${err.message}`, { cause: err }) } }, @@ -41,9 +40,8 @@ module.exports = { if (conf.dryRun) return try { return await this.client.issues.create(options) - } - catch (err) { - throw new Error(`Failed to ${options.issue_number?'update':'create'} issue: ${options.issue_number?options.issue_number:options.title}, error: ${err.message}`, { cause: err }) + } catch (err) { + throw new Error(`Failed to ${options.issue_number ? 'update' : 'create'} issue: ${options.issue_number ? options.issue_number : options.title}, error: ${err.message}`, { cause: err }) } }, diff --git a/lib/github/labels.js b/lib/github/labels.js index 3d0bf1e..c73564d 100644 --- a/lib/github/labels.js +++ b/lib/github/labels.js @@ -61,8 +61,7 @@ const ensureLabelsAreCreated = async (octokit, client, ghOwner, ghRepo, issues) }), (response) => response.data ) - } - catch (err) { + } catch (err) { throw new Error(`Failed to paginate octakit request for labels in repository ${ghRepo}, error: ${err.message}`, { cause: err }) } @@ -75,7 +74,7 @@ const ensureLabelsAreCreated = async (octokit, client, ghOwner, ghRepo, issues) await Promise.all( labelsToCreate.map((name) => { try { - client.issues + return client.issues .createLabel({ owner: ghOwner, repo: ghRepo, @@ -84,8 +83,7 @@ const ensureLabelsAreCreated = async (octokit, client, ghOwner, ghRepo, issues) .then(() => { console.log(`Created GitHub label: "${name}"`) }) - } - catch (err) { + } catch (err) { throw new Error(`Failed to create GitHub label '${name}', error: ${err.message}`, { cause: err }) } }) diff --git a/lib/snyk.js b/lib/snyk.js index 26fcd58..991467c 100644 --- a/lib/snyk.js +++ b/lib/snyk.js @@ -29,7 +29,7 @@ module.exports = class Snyk { ).orgs } - async queryOrgInfo(organizationId) { + async queryOrgInfo (organizationId) { try { const response = await request({ method: 'get', @@ -39,12 +39,11 @@ module.exports = class Snyk { }) if (response.data === undefined || response.data.attributes === undefined || response.data.attributes.name === undefined) { - throw new Error (`expected response to include data.attributes.name`) + throw new Error('expected response to include data.attributes.name') } return response.data - } - catch (err) { + } catch (err) { throw new Error(`Failed to query snyk organization with id ${organizationId}, error: ${err.message}`, { cause: err }) } } @@ -69,7 +68,7 @@ module.exports = class Snyk { isMonitored: project.attributes.status === 'active', issueCountTotal, - browseUrl: `https://app.snyk.io/org/${organization.attributes.name.toLowerCase()}/project/${project.id}`, + browseUrl: `https://app.snyk.io/org/${organization.attributes.name.toLowerCase()}/project/${project.id}` } }).filter(({ id, isMonitored }) => { if (selectedProjects.includes(id)) { @@ -122,7 +121,7 @@ function getSeverities (minimumSeverity) { return ['critical', 'high', 'medium', 'low'] } -async function paginateRestResponseData(url, headers, method = 'get') { +async function paginateRestResponseData (url, headers, method = 'get') { try { const reponseData = [] do { @@ -133,13 +132,12 @@ async function paginateRestResponseData(url, headers, method = 'get') { json: true }) reponseData.push(...response.data) - if (response.links.next) url = baseRestUrl + response.links.next / trimStart('/rest') + if (response.links.next) url = baseRestUrl + response.links.next.trimStart('/rest') else url = undefined } while (url) return reponseData - } - catch (err) { + } catch (err) { throw new Error(`Failed to paginate request for ${method} ${url}, error: ${err.message}`, { cause: err }) } } From acd1063d8f33a75f5cf74b48c5c8789142ea6130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cjeramysoucy=E2=80=9D?= Date: Wed, 27 Dec 2023 11:33:55 -0500 Subject: [PATCH 3/7] Further improves error reporting --- lib/github/index.js | 5 +++-- lib/github/labels.js | 6 +++--- lib/snyk.js | 5 +++-- lib/utils.js | 7 ++++++- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/github/index.js b/lib/github/index.js index 5a39efd..a260a43 100644 --- a/lib/github/index.js +++ b/lib/github/index.js @@ -7,6 +7,7 @@ const chalk = require('chalk') const getBatchIssue = require('./batch') const { conf } = require('../config') const { getProjectName, getGraph } = require('./utils') +const { dressError } = require('../utils') const { getLabels, ensureLabelsAreCreated } = require('./labels') @@ -32,7 +33,7 @@ module.exports = { ]) ) } catch (err) { - throw new Error(`Failed to paginate octakit request for existing issues in repository ${conf.ghRepo}, error: ${err.message}`, { cause: err }) + throw new Error(dressError(err, `Failed to paginate octakit request for existing issues in repository ${conf.ghRepo}`)) } }, @@ -41,7 +42,7 @@ module.exports = { try { return await this.client.issues.create(options) } catch (err) { - throw new Error(`Failed to ${options.issue_number ? 'update' : 'create'} issue: ${options.issue_number ? options.issue_number : options.title}, error: ${err.message}`, { cause: err }) + throw new Error(dressError(err, `Failed to ${options.issue_number ? 'update' : 'create'} issue: ${options.issue_number ? options.issue_number : options.title}`)) } }, diff --git a/lib/github/labels.js b/lib/github/labels.js index c73564d..b4e6272 100644 --- a/lib/github/labels.js +++ b/lib/github/labels.js @@ -1,7 +1,7 @@ 'use strict' const { conf } = require('../config') -const { uniq } = require('../utils') +const { uniq, dressError } = require('../utils') const LABELS = { snyk: { @@ -62,7 +62,7 @@ const ensureLabelsAreCreated = async (octokit, client, ghOwner, ghRepo, issues) (response) => response.data ) } catch (err) { - throw new Error(`Failed to paginate octakit request for labels in repository ${ghRepo}, error: ${err.message}`, { cause: err }) + throw new Error(dressError(err, `Failed to paginate octakit request for labels in repository: ${ghRepo}`)) } const currentLabels = responseData.map((x) => x.name) @@ -84,7 +84,7 @@ const ensureLabelsAreCreated = async (octokit, client, ghOwner, ghRepo, issues) console.log(`Created GitHub label: "${name}"`) }) } catch (err) { - throw new Error(`Failed to create GitHub label '${name}', error: ${err.message}`, { cause: err }) + throw new Error(dressError(err, `Failed to create GitHub label '${name}' in repository: ${ghRepo}`)) } }) ) diff --git a/lib/snyk.js b/lib/snyk.js index 991467c..9f2adff 100644 --- a/lib/snyk.js +++ b/lib/snyk.js @@ -1,6 +1,7 @@ 'use strict' const request = require('request-promise-native') +const { dressError } = require('./utils') const baseV1Url = 'https://snyk.io/api/v1' const baseRestUrl = 'https://api.snyk.io/rest' @@ -44,7 +45,7 @@ module.exports = class Snyk { return response.data } catch (err) { - throw new Error(`Failed to query snyk organization with id ${organizationId}, error: ${err.message}`, { cause: err }) + throw new Error(dressError(err, `Failed to query snyk organization with id ${organizationId}`)) } } @@ -138,6 +139,6 @@ async function paginateRestResponseData (url, headers, method = 'get') { return reponseData } catch (err) { - throw new Error(`Failed to paginate request for ${method} ${url}, error: ${err.message}`, { cause: err }) + throw new Error(dressError(err, `Failed to paginate request for ${method} ${url}`)) } } diff --git a/lib/utils.js b/lib/utils.js index c245224..fdb2c10 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -73,6 +73,10 @@ const capitalize = (s) => { const uniq = (array) => [...new Set(array)] +const dressError = (err, msg) => { + return `${msg}, error: ${err.status ? err.status + ' - ' : ''}${err.message}, response: ${err.response}` +} + module.exports = { capitalize, compare: { @@ -82,5 +86,6 @@ module.exports = { versionArrays: compareVersionArrays, arrays: compareArrays }, - uniq + uniq, + dressError } From a3318cbf70af7283767119c43cf4f856a27cfb35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cjeramysoucy=E2=80=9D?= Date: Wed, 27 Dec 2023 13:17:59 -0500 Subject: [PATCH 4/7] Resolves issue with fetch API call in octokit paginate --- lib/github/index.js | 2 +- lib/github/labels.js | 11 ++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/lib/github/index.js b/lib/github/index.js index a260a43..99f760d 100644 --- a/lib/github/index.js +++ b/lib/github/index.js @@ -33,7 +33,7 @@ module.exports = { ]) ) } catch (err) { - throw new Error(dressError(err, `Failed to paginate octakit request for existing issues in repository ${conf.ghRepo}`)) + throw new Error(dressError(err, `Failed to paginate octokit request for existing issues in repository ${conf.ghRepo}`)) } }, diff --git a/lib/github/labels.js b/lib/github/labels.js index b4e6272..9491027 100644 --- a/lib/github/labels.js +++ b/lib/github/labels.js @@ -53,16 +53,9 @@ const ensureLabelsAreCreated = async (octokit, client, ghOwner, ghRepo, issues) let responseData try { - responseData = await octokit.paginate( - await client.issues.listLabelsForRepo({ - owner: ghOwner, - repo: ghRepo, - per_page: 100 - }), - (response) => response.data - ) + responseData = await octokit.paginate(`GET /repos/${conf.ghOwner}/${conf.ghRepo}/labels`) } catch (err) { - throw new Error(dressError(err, `Failed to paginate octakit request for labels in repository: ${ghRepo}`)) + throw new Error(dressError(err, `Failed to paginate octokit request for labels in repository: ${ghRepo}`)) } const currentLabels = responseData.map((x) => x.name) From c250e2868253bcbdf79829670bd3b0459c5b7f04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cjeramysoucy=E2=80=9D?= Date: Wed, 27 Dec 2023 14:01:15 -0500 Subject: [PATCH 5/7] Uses v1 single project API to retrieve browseUrl and imageTag. Removes previous queryOrganization funtion. --- lib/github/batch.js | 4 ++-- lib/snyk.js | 49 ++++++++++++++++++++++----------------------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/lib/github/batch.js b/lib/github/batch.js index 492554b..9918356 100644 --- a/lib/github/batch.js +++ b/lib/github/batch.js @@ -35,8 +35,8 @@ module.exports = async (issues) => { const headerText = 'This issue has been created automatically by [snyk-github-issue-creator](https://github.com/elastic/snyk-github-issue-creator).\r\n\r\nSnyk project(s):' const projectText = projects .map( - ({ name, browseUrl }) => - `\r\n * [\`${name}\`](${browseUrl})` + ({ name, browseUrl, imageTag }) => + `\r\n * [\`${name}\`](${browseUrl}) (manifest version ${imageTag})` ) .join('') const sectionText = Object.keys(sevMap) diff --git a/lib/snyk.js b/lib/snyk.js index 9f2adff..06173c1 100644 --- a/lib/snyk.js +++ b/lib/snyk.js @@ -30,48 +30,47 @@ module.exports = class Snyk { ).orgs } - async queryOrgInfo (organizationId) { + async queryProjectDetails (organizationId, projectId) { try { - const response = await request({ + return await request({ method: 'get', - url: `${baseRestUrl}/orgs/${organizationId}?version=2023-11-27`, + url: `${baseV1Url}/org/${organizationId}/project/${projectId}`, // project snapshot via v1 POST org/:orgId/project/:projectId/history headers: this._headers, json: true }) - - if (response.data === undefined || response.data.attributes === undefined || response.data.attributes.name === undefined) { - throw new Error('expected response to include data.attributes.name') - } - - return response.data } catch (err) { - throw new Error(dressError(err, `Failed to query snyk organization with id ${organizationId}`)) + throw new Error(dressError(err, `Failed to query snyk project details. Organization ID: ${organizationId}, Project ID: ${projectId}`)) } } async projects (orgId, selectedProjects = []) { const organizationId = orgId || this._orgId - const organization = await this.queryOrgInfo(organizationId) - const responseData = await paginateRestResponseData( `${baseRestUrl}/orgs/${organizationId}/projects?version=2023-11-27&meta.latest_issue_counts=true&limit=20`, this._headers ) - return responseData.map((project) => { - const { critical, high, medium, low } = project.meta.latest_issue_counts - const issueCountTotal = critical + high + medium + low - - return { - id: project.id, - name: project.attributes.name, - isMonitored: - project.attributes.status === 'active', - issueCountTotal, - browseUrl: `https://app.snyk.io/org/${organization.attributes.name.toLowerCase()}/project/${project.id}` - } - }).filter(({ id, isMonitored }) => { + const projects = await Promise.all( + responseData.map(async (project) => { + const { critical, high, medium, low } = project.meta.latest_issue_counts + const issueCountTotal = critical + high + medium + low + + const projectDetails = await this.queryProjectDetails(organizationId, project.id) + + return { + id: project.id, + name: project.attributes.name, + isMonitored: + project.attributes.status === 'active', + issueCountTotal, + browseUrl: projectDetails.browseUrl, + imageTag: projectDetails.imageTag + } + }) + ) + + return projects.filter(({ id, isMonitored }) => { if (selectedProjects.includes(id)) { return true } From 0b919f5cce7fcbd378cc8056272e2d2adbe04650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cjeramysoucy=E2=80=9D?= Date: Wed, 27 Dec 2023 14:27:51 -0500 Subject: [PATCH 6/7] Changes pagination size to 100 --- lib/github/labels.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/github/labels.js b/lib/github/labels.js index 9491027..77a61fa 100644 --- a/lib/github/labels.js +++ b/lib/github/labels.js @@ -53,7 +53,7 @@ const ensureLabelsAreCreated = async (octokit, client, ghOwner, ghRepo, issues) let responseData try { - responseData = await octokit.paginate(`GET /repos/${conf.ghOwner}/${conf.ghRepo}/labels`) + responseData = await octokit.paginate(`GET /repos/${conf.ghOwner}/${conf.ghRepo}/labels?per_page=100`) } catch (err) { throw new Error(dressError(err, `Failed to paginate octokit request for labels in repository: ${ghRepo}`)) } From 84910b95f87902a701cd5c0e07bb2ccd32dd8e31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cjeramysoucy=E2=80=9D?= Date: Wed, 27 Dec 2023 14:34:51 -0500 Subject: [PATCH 7/7] Removes unecessary comment --- lib/snyk.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/snyk.js b/lib/snyk.js index 06173c1..685f8f1 100644 --- a/lib/snyk.js +++ b/lib/snyk.js @@ -34,7 +34,7 @@ module.exports = class Snyk { try { return await request({ method: 'get', - url: `${baseV1Url}/org/${organizationId}/project/${projectId}`, // project snapshot via v1 POST org/:orgId/project/:projectId/history + url: `${baseV1Url}/org/${organizationId}/project/${projectId}`, headers: this._headers, json: true })