From b089d2563025df113037677f757787a0ec998ae0 Mon Sep 17 00:00:00 2001 From: Sebastian Beltran Date: Thu, 30 Jan 2025 11:20:31 -0500 Subject: [PATCH 01/10] feat: add migration to update compliance check status for 'staticCodeAnalysis' --- ...61647_update_check_static_code_analysis.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/database/migrations/20250130161647_update_check_static_code_analysis.js diff --git a/src/database/migrations/20250130161647_update_check_static_code_analysis.js b/src/database/migrations/20250130161647_update_check_static_code_analysis.js new file mode 100644 index 0000000..ccdf2ee --- /dev/null +++ b/src/database/migrations/20250130161647_update_check_static_code_analysis.js @@ -0,0 +1,19 @@ +exports.up = async (knex) => { + await knex('compliance_checks') + .where({ code_name: 'staticCodeAnalysis' }) + .update({ + implementation_status: 'completed', + implementation_type: 'computed', + implementation_details_reference: 'https://github.com/OpenPathfinder/visionBoard/issues/83' + }) +} + +exports.down = async (knex) => { + await knex('compliance_checks') + .where({ code_name: 'staticCodeAnalysis' }) + .update({ + implementation_status: 'pending', + implementation_type: null, + implementation_details_reference: null + }) +} From a8cc1f31d36f3419146b27704d3c0e2bb768b015 Mon Sep 17 00:00:00 2001 From: Sebastian Beltran Date: Mon, 3 Feb 2025 21:00:27 -0500 Subject: [PATCH 02/10] feat: add staticCodeAnalysis compliance check --- .../complianceChecks/staticCodeAnalysis.js | 31 +++++++++ src/checks/validators/index.js | 4 +- src/checks/validators/staticCodeAnalysis.js | 16 +++++ src/store/index.js | 65 +++++++++++++++++++ 4 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 src/checks/complianceChecks/staticCodeAnalysis.js create mode 100644 src/checks/validators/staticCodeAnalysis.js diff --git a/src/checks/complianceChecks/staticCodeAnalysis.js b/src/checks/complianceChecks/staticCodeAnalysis.js new file mode 100644 index 0000000..674a6d6 --- /dev/null +++ b/src/checks/complianceChecks/staticCodeAnalysis.js @@ -0,0 +1,31 @@ +const validators = require('../validators') +const { initializeStore } = require('../../store') +const debug = require('debug')('checks:staticCodeAnalysis') + +module.exports = async (knex, { projects } = {}) => { + const { + getAllOSSFResultsOfRepositoriesByProjectId, getCheckByCodeName, + getAllProjects, addAlert, addTask, upsertComplianceCheckResult, + deleteAlertsByComplianceCheckId, deleteTasksByComplianceCheckId + } = initializeStore(knex) + debug('Collecting relevant data...') + const check = await getCheckByCodeName('staticCodeAnalysis') + if (!projects || (Array.isArray(projects) && projects.length === 0)) { + projects = await getAllProjects() + } + const data = await getAllOSSFResultsOfRepositoriesByProjectId(projects.map(p => p.id)) + + debug('Extracting the validation results...') + const analysis = validators.staticCodeAnalysis({ data, check, projects }) + + debug('Deleting previous alerts and tasks to avoid orphaned records...') + await deleteAlertsByComplianceCheckId(check.id) + await deleteTasksByComplianceCheckId(check.id) + + debug('Upserting the new results...') + await Promise.all(analysis.results.map(result => upsertComplianceCheckResult(result))) + + debug('Inserting the new Alerts and Tasks...') + await Promise.all(analysis.alerts.map(alert => addAlert(alert))) + await Promise.all(analysis.tasks.map(task => addTask(task))) +} diff --git a/src/checks/validators/index.js b/src/checks/validators/index.js index 40d472c..aa0dc09 100644 --- a/src/checks/validators/index.js +++ b/src/checks/validators/index.js @@ -3,13 +3,15 @@ const softwareDesignTraining = require('./softwareDesignTraining') const owaspTop10Training = require('./owaspTop10Training') const adminRepoCreationOnly = require('./adminRepoCreationOnly') const noSensitiveInfoInRepositories = require('./noSensitiveInfoInRepositories') +const staticCodeAnalysis = require('./staticCodeAnalysis') const validators = { githubOrgMFA, softwareDesignTraining, owaspTop10Training, adminRepoCreationOnly, - noSensitiveInfoInRepositories + noSensitiveInfoInRepositories, + staticCodeAnalysis } module.exports = validators diff --git a/src/checks/validators/staticCodeAnalysis.js b/src/checks/validators/staticCodeAnalysis.js new file mode 100644 index 0000000..37ca782 --- /dev/null +++ b/src/checks/validators/staticCodeAnalysis.js @@ -0,0 +1,16 @@ +const debug = require('debug')('checks:validator:adminRepoCreationOnly') + +// @see: https://github.com/OpenPathfinder/visionBoard/issues/75 +module.exports = ({ data = [], check, projects = [] }) => { + debug('Validating that the repositories have static code analysis...') + + const alerts = [] + const results = [] + const tasks = [] + + return { + alerts, + results, + tasks + } +} diff --git a/src/store/index.js b/src/store/index.js index 0750a71..029996c 100644 --- a/src/store/index.js +++ b/src/store/index.js @@ -183,6 +183,70 @@ const getAllGithubRepositoriesAndOrganizationByProjectId = async (knex, projectI return Array.from(organizationsMap.values()) } +const getAllOSSFResultsOfRepositoriesByProjectId = async (knex, projectIds) => { + debug(`Fetching all scorecards results of repositories by project id (${projectIds})...`) + + if (!Array.isArray(projectIds)) { + throw new Error('projectIds must be an array') + } + + const results = await knex('github_organizations') + .select( + 'github_organizations.id as org_id', + 'github_organizations.*', + 'github_repositories.id as repo_id', + 'github_repositories.*', + 'ossf_scorecard_results.id as ossf_id', + 'ossf_scorecard_results.*' + ) + .whereIn('github_organizations.project_id', projectIds) + .leftJoin( + 'github_repositories', + 'github_repositories.github_organization_id', + 'github_organizations.id' + ) + .leftJoin( + 'ossf_scorecard_results', + 'ossf_scorecard_results.github_repository_id', + 'github_repositories.id' + ) + + // @TODO: Refactor this into a helper function or part of the query + // Transform results into desired structure + const organizationsMap = new Map() + + results.forEach(row => { + const orgId = row.org_id + + if (!organizationsMap.has(orgId)) { + // Create org entry if not exists + const orgData = simplifyObject(row, { + exclude: ['repo_id', 'ossf_id'] + }) + orgData.repositories = [] + orgData.ossf_results = [] + organizationsMap.set(orgId, orgData) + } + + // Add repository if it exists + if (row.repo_id) { + const repoData = simplifyObject(row, { + exclude: ['repo_id', 'org_id', 'ossf_id'] + }) + organizationsMap.get(orgId).repositories.push(repoData) + } + + if (row.ossf_id) { + const ossfData = simplifyObject(row, { + exclude: ['ossf_id', 'repo_id', 'org_id'] + }) + organizationsMap.get(orgId).ossf_results.push(ossfData) + } + }) + + return Array.from(organizationsMap.values()) +} + const initializeStore = (knex) => { debug('Initializing store...') const getAll = getAllFn(knex) @@ -199,6 +263,7 @@ const initializeStore = (knex) => { getAllOwaspTop10Trainings: () => getAll('owasp_top10_training'), getAllGithubRepositories: () => getAll('github_repositories'), getAllGithubRepositoriesAndOrganizationByProjectId: (organizationId) => getAllGithubRepositoriesAndOrganizationByProjectId(knex, organizationId), + getAllOSSFResultsOfRepositoriesByProjectId: (projectId) => getAllOSSFResultsOfRepositoriesByProjectId(knex, projectId), getAllChecklists: () => getAll('compliance_checklists'), getAllResults: () => getAll('compliance_checks_results'), getAllTasks: () => getAll('compliance_checks_tasks'), From 7282ed206c2dc162417a246fed1e2b59b14e1fb6 Mon Sep 17 00:00:00 2001 From: Sebastian Beltran Date: Sun, 9 Feb 2025 15:33:38 -0500 Subject: [PATCH 03/10] test: add integration and validation tests todo for staticCodeAnalysis --- __tests__/checks/staticCodeAnalysis.test.js | 58 ++++++++++++ .../validators/staticCodeAnalysis.test.js | 93 +++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 __tests__/checks/staticCodeAnalysis.test.js create mode 100644 __tests__/checks/validators/staticCodeAnalysis.test.js diff --git a/__tests__/checks/staticCodeAnalysis.test.js b/__tests__/checks/staticCodeAnalysis.test.js new file mode 100644 index 0000000..be3ed53 --- /dev/null +++ b/__tests__/checks/staticCodeAnalysis.test.js @@ -0,0 +1,58 @@ +const knexInit = require('knex') +const { getConfig } = require('../../src/config') +const staticCodeAnalysis = require('../../src/checks/complianceChecks/staticCodeAnalysis') +const { + resetDatabase, initializeStore, generateGithubRepoData +} = require('../../__utils__') +const { sampleGithubOrg } = require('../../__fixtures__') + +const { dbSettings } = getConfig('test') + +let knex, + project, + check, + addProject, + addGithubOrg, + addGithubRepo, + getAllResults, + getAllTasks, + getAllAlerts, + addAlert, + addTask, + addResult, + getCheckByCodeName + +beforeAll(async () => { + knex = knexInit(dbSettings); + ({ + addProject, + addGithubOrganization: addGithubOrg, + addGithubRepo, + getAllResults, + getAllTasks, + getAllAlerts, + addAlert, + addTask, + addResult, + getCheckByCodeName + } = initializeStore(knex)) + check = await getCheckByCodeName('staticCodeAnalysis') +}) + +beforeEach(async () => { + await resetDatabase(knex) + project = await addProject({ name: sampleGithubOrg.login }) +}) + +afterAll(async () => { + await knex.destroy() +}) + +describe('Integration: staticCodeAnalysis', () => { + test.todo('Should add results without alerts or tasks') + test.todo('Should delete (previous alerts and tasks) and add results') + test.todo('Should add (alerts and tasks) and update results') + test.todo('Should add (alerts and tasks) and update results without repos and ossf_results') + test.todo('Should add (alerts and tasks) and update results without repos') + test.todo('Should add (alerts and tasks) and update results without ossf_results') +}) diff --git a/__tests__/checks/validators/staticCodeAnalysis.test.js b/__tests__/checks/validators/staticCodeAnalysis.test.js new file mode 100644 index 0000000..9879c9a --- /dev/null +++ b/__tests__/checks/validators/staticCodeAnalysis.test.js @@ -0,0 +1,93 @@ +const { staticCodeAnalysis } = require('../../../src/checks/validators') + +describe('staticCodeAnalysis', () => { + let data, check, projects + beforeEach(() => { + data = [ + { + project_id: 1, + github_organization_id: 1, + login: 'org1', + repositories: [ + { + id: 1, + name: 'test', + full_name: 'org1/test' + }, + { + id: 2, + name: 'discussions', + full_name: 'org1/discussions' + } + ], + ossf_results: [ + { + sast_score: 10, + github_repository_id: 1 + }, + { + sast_score: 10, + github_repository_id: 2 + } + ] + }, { + project_id: 1, + github_organization_id: 2, + login: 'org2', + repositories: [ + { + id: 3, + name: '.github', + full_name: 'org2/.github' + } + ], + ossf_results: [ + { + sast_score: 10, + github_repository_id: 3 + } + ] + }, { + project_id: 2, + github_organization_id: 3, + login: 'org3', + repositories: [ + { + id: 4, + name: 'support', + full_name: 'org3/support' + } + ], + ossf_results: [ + { + sast_score: 10, + github_repository_id: 4 + } + ] + }] + + check = { + id: 1, + default_priority_group: 'P6', + details_url: 'https://example.com' + } + + projects = [ + { + id: 1 + }, + { + id: 2 + } + ] + }) + + it.todo('Should generate a passed result if all repositories has a high static code analysis score') + it.todo('Should generate a pass result if not have public repositories') + it.todo('Should generate a failed result if some repositories do not have static code analysis') + it.todo('Should generate a failed result if somre repisotiores have low static code analysis score') + it.todo('Should generate a failed result if some repositories have a low static code analysis score and some repositories have unkwon result') + it.todo('Should generate an unknown result if not have ossf results') + it.todo('Should generate an unknown result if some have repositories have unkown ossf results but other repositories have a high static code analysis score') + it.todo('Should generate an unknown result if some repositories have unknown static code analysis') +}) From eae37a150034efb6979e47405a4a9688d92462e6 Mon Sep 17 00:00:00 2001 From: Sebastian Beltran Date: Sun, 9 Feb 2025 16:22:31 -0500 Subject: [PATCH 04/10] test: implement detailed tests for static code analysis results --- .../validators/staticCodeAnalysis.test.js | 160 +++++++++++++++++- 1 file changed, 152 insertions(+), 8 deletions(-) diff --git a/__tests__/checks/validators/staticCodeAnalysis.test.js b/__tests__/checks/validators/staticCodeAnalysis.test.js index 9879c9a..833da0c 100644 --- a/__tests__/checks/validators/staticCodeAnalysis.test.js +++ b/__tests__/checks/validators/staticCodeAnalysis.test.js @@ -82,12 +82,156 @@ describe('staticCodeAnalysis', () => { ] }) - it.todo('Should generate a passed result if all repositories has a high static code analysis score') - it.todo('Should generate a pass result if not have public repositories') - it.todo('Should generate a failed result if some repositories do not have static code analysis') - it.todo('Should generate a failed result if somre repisotiores have low static code analysis score') - it.todo('Should generate a failed result if some repositories have a low static code analysis score and some repositories have unkwon result') - it.todo('Should generate an unknown result if not have ossf results') - it.todo('Should generate an unknown result if some have repositories have unkown ossf results but other repositories have a high static code analysis score') - it.todo('Should generate an unknown result if some repositories have unknown static code analysis') + it('Should generate a passed result if all repositories has a high static code analysis score', () => { + const analysis = staticCodeAnalysis({ data, check, projects }) + expect(analysis).toEqual({ + alerts: [], + results: [ + { + project_id: 1, + compliance_check_id: 1, + severity: 'medium', + status: 'passed', + rationale: 'All repositories in all organizations have a static code analysis tool' + }, + { + compliance_check_id: 1, + project_id: 2, + rationale: 'All repositories in all organizations have a static code analysis tool', + severity: 'medium', + status: 'passed' + } + ], + tasks: [] + }) + }) + + it.todo('Should generate a pass result if not have public repositories in some organizations') + it.todo('Should generate a pass result if not have public repositories in all the organizations') + + it('Should generate a failed result if some repositories have low static code analysis score', () => { + data[0].ossf_results[0].sast_score = 0 + data[0].ossf_results[1].sast_score = null + data[1].ossf_results[0].sast_score = 0 + + const analysis = staticCodeAnalysis({ data, check, projects }) + expect(analysis).toEqual({ + alerts: [ + { + project_id: 1, + compliance_check_id: 1, + severity: 'medium', + title: '3 (66.7%) repositories in org1, org2 organizations do not have a static code analysis tool', + description: 'Check the details on https://example.com' + } + ], + results: [ + { + project_id: 1, + compliance_check_id: 1, + severity: 'medium', + status: 'passed', + rationale: '3 (66.7%) repositories in org1, org2 organizations do not have a static code analysis tool' + }, + { + compliance_check_id: 1, + project_id: 2, + rationale: 'All repositories in all organizations have a static code analysis tool', + severity: 'medium', + status: 'passed' + } + ], + tasks: [ + { + compliance_check_id: 1, + description: 'Check the details on https://example.com', + project_id: 1, + severity: 'medium', + title: 'Add a code analysis tool for 2 (66.7%) repositories (org1/test, org2/.github)' + } + ] + }) + }) + + it('Should generate an unknown result if not have ossf results', () => { + data[0].ossf_results = [] + data[1].ossf_results = [] + + const analysis = staticCodeAnalysis({ data, check, projects }) + expect(analysis).toEqual({ + alerts: [], + results: [ + { + project_id: 1, + compliance_check_id: 1, + severity: 'medium', + status: 'unknown', + rationale: 'No results have been generated from the OSSF Scorecard' + }, + { + compliance_check_id: 1, + project_id: 2, + rationale: 'All repositories in all organizations have a static code analysis tool', + severity: 'medium', + status: 'passed' + } + ], + tasks: [] + }) + }) + it('Should generate an unknown result if some have repositories have unkown ossf results but other repositories have a high static code analysis score', () => { + data[0].ossf_results = [ + { + sast_score: 10, + github_repository_id: 1 + } + ] + + const analysis = staticCodeAnalysis({ data, check, projects }) + expect(analysis).toEqual({ + alerts: [], + results: [ + { + project_id: 1, + compliance_check_id: 1, + severity: 'medium', + status: 'unknown', + rationale: '1 (33.3%) repositories have not generated results from the OSSF Scorecard' + }, + { + compliance_check_id: 1, + project_id: 2, + rationale: 'All repositories in all organizations have a static code analysis tool', + severity: 'medium', + status: 'passed' + } + ], + tasks: [] + }) + }) + it('Should generate an unknown result if some repositories have unknown static code analysis', () => { + data[2].ossf_results[0].sast_score = null + + const analysis = staticCodeAnalysis({ data, check, projects }) + expect(analysis).toEqual({ + alerts: [], + results: [ + { + project_id: 1, + compliance_check_id: 1, + severity: 'medium', + status: 'passed', + rationale: 'All repositories in all organizations have a static code analysis tool' + }, + { + compliance_check_id: 1, + project_id: 2, + rationale: '1 (100%) repositories could not be determined to have a code analysis tool', + severity: 'medium', + status: 'unknown' + } + ], + tasks: [] + }) + }) }) From 63b0e02fb06618a7ed39e94ca66f518f3cc68907 Mon Sep 17 00:00:00 2001 From: Sebastian Beltran Date: Sun, 9 Feb 2025 17:03:37 -0500 Subject: [PATCH 05/10] refactor: restructure staticCodeAnalysis test data and improve repository data handling --- .../validators/staticCodeAnalysis.test.js | 64 ++++++++----------- src/store/index.js | 12 ++-- 2 files changed, 34 insertions(+), 42 deletions(-) diff --git a/__tests__/checks/validators/staticCodeAnalysis.test.js b/__tests__/checks/validators/staticCodeAnalysis.test.js index 833da0c..03adf2b 100644 --- a/__tests__/checks/validators/staticCodeAnalysis.test.js +++ b/__tests__/checks/validators/staticCodeAnalysis.test.js @@ -12,22 +12,18 @@ describe('staticCodeAnalysis', () => { { id: 1, name: 'test', - full_name: 'org1/test' + full_name: 'org1/test', + ossf_results: { + sast_score: 10 + } }, { id: 2, name: 'discussions', - full_name: 'org1/discussions' - } - ], - ossf_results: [ - { - sast_score: 10, - github_repository_id: 1 - }, - { - sast_score: 10, - github_repository_id: 2 + full_name: 'org1/discussions', + ossf_results: { + sast_score: 10 + } } ] }, { @@ -38,13 +34,10 @@ describe('staticCodeAnalysis', () => { { id: 3, name: '.github', - full_name: 'org2/.github' - } - ], - ossf_results: [ - { - sast_score: 10, - github_repository_id: 3 + full_name: 'org2/.github', + ossf_results: { + sast_score: 10 + } } ] }, { @@ -55,13 +48,10 @@ describe('staticCodeAnalysis', () => { { id: 4, name: 'support', - full_name: 'org3/support' - } - ], - ossf_results: [ - { - sast_score: 10, - github_repository_id: 4 + full_name: 'org3/support', + ossf_results: { + sast_score: 10 + } } ] }] @@ -110,9 +100,9 @@ describe('staticCodeAnalysis', () => { it.todo('Should generate a pass result if not have public repositories in all the organizations') it('Should generate a failed result if some repositories have low static code analysis score', () => { - data[0].ossf_results[0].sast_score = 0 - data[0].ossf_results[1].sast_score = null - data[1].ossf_results[0].sast_score = 0 + data[0].repositories[0].ossf_results.sast_score = 0 + data[0].repositories[1].ossf_results.sast_score = null + data[1].repositories[0].ossf_results.sast_score = 0 const analysis = staticCodeAnalysis({ data, check, projects }) expect(analysis).toEqual({ @@ -154,8 +144,10 @@ describe('staticCodeAnalysis', () => { }) it('Should generate an unknown result if not have ossf results', () => { - data[0].ossf_results = [] - data[1].ossf_results = [] + data[0].repositories[0].ossf_results = null + data[0].repositories[1].ossf_results = null + data[1].repositories[0].ossf_results = null + data[2].repositories[0].ossf_results = null const analysis = staticCodeAnalysis({ data, check, projects }) expect(analysis).toEqual({ @@ -179,13 +171,9 @@ describe('staticCodeAnalysis', () => { tasks: [] }) }) + it('Should generate an unknown result if some have repositories have unkown ossf results but other repositories have a high static code analysis score', () => { - data[0].ossf_results = [ - { - sast_score: 10, - github_repository_id: 1 - } - ] + data[0].repositories[1].ossf_results = undefined const analysis = staticCodeAnalysis({ data, check, projects }) expect(analysis).toEqual({ @@ -210,7 +198,7 @@ describe('staticCodeAnalysis', () => { }) }) it('Should generate an unknown result if some repositories have unknown static code analysis', () => { - data[2].ossf_results[0].sast_score = null + data[2].repositories[0].ossf_results.sast_score = null const analysis = staticCodeAnalysis({ data, check, projects }) expect(analysis).toEqual({ diff --git a/src/store/index.js b/src/store/index.js index 029996c..29d81e4 100644 --- a/src/store/index.js +++ b/src/store/index.js @@ -228,19 +228,23 @@ const getAllOSSFResultsOfRepositoriesByProjectId = async (knex, projectIds) => { organizationsMap.set(orgId, orgData) } + let repoData = {} // Add repository if it exists if (row.repo_id) { - const repoData = simplifyObject(row, { + repoData = simplifyObject(row, { exclude: ['repo_id', 'org_id', 'ossf_id'] }) - organizationsMap.get(orgId).repositories.push(repoData) } - if (row.ossf_id) { + if (row.ossf_id && row.repo_id) { const ossfData = simplifyObject(row, { exclude: ['ossf_id', 'repo_id', 'org_id'] }) - organizationsMap.get(orgId).ossf_results.push(ossfData) + repoData.ossf_results = ossfData + } + + if (row.repo_id) { + organizationsMap.get(orgId).repositories.push(repoData) } }) From 36ab104163aa0a648a74f4921f95987638850d21 Mon Sep 17 00:00:00 2001 From: Sebastian Beltran Date: Tue, 11 Feb 2025 20:34:30 -0500 Subject: [PATCH 06/10] feat: initial logic of staticCodeAnalysis validator --- .../validators/staticCodeAnalysis.test.js | 12 +-- src/checks/validators/staticCodeAnalysis.js | 88 ++++++++++++++++++- src/store/index.js | 1 - 3 files changed, 92 insertions(+), 9 deletions(-) diff --git a/__tests__/checks/validators/staticCodeAnalysis.test.js b/__tests__/checks/validators/staticCodeAnalysis.test.js index 03adf2b..deae6cb 100644 --- a/__tests__/checks/validators/staticCodeAnalysis.test.js +++ b/__tests__/checks/validators/staticCodeAnalysis.test.js @@ -111,7 +111,7 @@ describe('staticCodeAnalysis', () => { project_id: 1, compliance_check_id: 1, severity: 'medium', - title: '3 (66.7%) repositories in org1, org2 organizations do not have a static code analysis tool', + title: '2 (66.7%) repositories do not have a static code analysis tool', description: 'Check the details on https://example.com' } ], @@ -120,8 +120,8 @@ describe('staticCodeAnalysis', () => { project_id: 1, compliance_check_id: 1, severity: 'medium', - status: 'passed', - rationale: '3 (66.7%) repositories in org1, org2 organizations do not have a static code analysis tool' + status: 'failed', + rationale: '2 (66.7%) repositories do not have a static code analysis tool' }, { compliance_check_id: 1, @@ -137,7 +137,7 @@ describe('staticCodeAnalysis', () => { description: 'Check the details on https://example.com', project_id: 1, severity: 'medium', - title: 'Add a code analysis tool for 2 (66.7%) repositories (org1/test, org2/.github)' + title: 'Add a code analysis tool for 2 (66.7%) repositories (org1/test, org2/.github) in GitHub' } ] }) @@ -147,7 +147,6 @@ describe('staticCodeAnalysis', () => { data[0].repositories[0].ossf_results = null data[0].repositories[1].ossf_results = null data[1].repositories[0].ossf_results = null - data[2].repositories[0].ossf_results = null const analysis = staticCodeAnalysis({ data, check, projects }) expect(analysis).toEqual({ @@ -184,7 +183,7 @@ describe('staticCodeAnalysis', () => { compliance_check_id: 1, severity: 'medium', status: 'unknown', - rationale: '1 (33.3%) repositories have not generated results from the OSSF Scorecard' + rationale: '1 (33.3%) repositories do not generated results from the OSSF Scorecard' }, { compliance_check_id: 1, @@ -197,6 +196,7 @@ describe('staticCodeAnalysis', () => { tasks: [] }) }) + it('Should generate an unknown result if some repositories have unknown static code analysis', () => { data[2].repositories[0].ossf_results.sast_score = null diff --git a/src/checks/validators/staticCodeAnalysis.js b/src/checks/validators/staticCodeAnalysis.js index 37ca782..01c7eef 100644 --- a/src/checks/validators/staticCodeAnalysis.js +++ b/src/checks/validators/staticCodeAnalysis.js @@ -1,13 +1,97 @@ -const debug = require('debug')('checks:validator:adminRepoCreationOnly') +const debug = require('debug')('checks:validator:staticCodeAnalysis') +const { + groupArrayItemsByCriteria, + getSeverityFromPriorityGroup, + generatePercentage +} = require('../../utils') + +const groupByProject = groupArrayItemsByCriteria('project_id') + +const minumumStaticCodeAnalysis = 7 // @see: https://github.com/OpenPathfinder/visionBoard/issues/75 -module.exports = ({ data = [], check, projects = [] }) => { +module.exports = ({ data: ghOrgs, check, projects = [] }) => { debug('Validating that the repositories have static code analysis...') + debug('Grouping repositories by project...') + const ghOrgsGroupedByProject = groupByProject(ghOrgs) const alerts = [] const results = [] const tasks = [] + debug('Processing repositories...') + ghOrgsGroupedByProject.forEach((projectOrgs) => { + debug(`Processing Project (${projectOrgs[0].github_organization_id})`) + const project = projects.find(p => p.id === projectOrgs[0].project_id) + const projectRepositories = projectOrgs.map(org => org.repositories).flat() + + const baseData = { + project_id: project.id, + compliance_check_id: check.id, + severity: getSeverityFromPriorityGroup(check.default_priority_group) + } + + const allOSSFResultsPass = projectRepositories.every( + repo => repo.ossf_results?.sast_score >= minumumStaticCodeAnalysis + ) + + if (allOSSFResultsPass) { + results.push({ + ...baseData, + status: 'passed', + rationale: 'All repositories in all organizations have a static code analysis tool' + }) + debug(`Processed project (${project.id}) - All passed`) + return + } + + const failedRepos = projectRepositories.filter(repo => Number.parseInt(repo.ossf_results?.sast_score) < minumumStaticCodeAnalysis).map(org => org.full_name) + const unknownRepos = projectRepositories.filter(repo => repo.ossf_results?.sast_score === null).map(org => org.full_name) + const noGenerateResults = projectRepositories.filter(repo => repo?.ossf_results == null).map(org => org.full_name) + + const result = { ...baseData } + const task = { ...baseData } + const alert = { ...baseData } + + if (noGenerateResults.length === projectRepositories.length) { + result.status = 'unknown' + result.rationale = 'No results have been generated from the OSSF Scorecard' + } else if (failedRepos.length) { + const percentage = generatePercentage(projectRepositories.length, failedRepos.length) + + result.status = 'failed' + result.rationale = `${failedRepos.length} (${percentage}) repositories do not have a static code analysis tool` + alert.title = `${failedRepos.length} (${percentage}) repositories do not have a static code analysis tool` + alert.description = `Check the details on ${check.details_url}` + task.title = `Add a code analysis tool for ${failedRepos.length} (${percentage}) repositories (${failedRepos.join(', ')}) in GitHub` + task.description = `Check the details on ${check.details_url}` + } else if (unknownRepos.length) { + const percentage = generatePercentage(projectRepositories.length, unknownRepos.length) + + result.status = 'unknown' + result.rationale = `${unknownRepos.length} (${percentage}) repositories could not be determined to have a code analysis tool` + } else if (noGenerateResults.length) { + const percentage = generatePercentage(projectRepositories.length, noGenerateResults.length) + + result.status = 'unknown' + result.rationale = `${noGenerateResults.length} (${percentage}) repositories do not generated results from the OSSF Scorecard` + } + + // Include only the task if was populated + if (Object.keys(task).length > Object.keys(baseData).length) { + debug(`Adding task for project (${project.id})`) + tasks.push(task) + } + // Include only the alert if was populated + if (Object.keys(alert).length > Object.keys(baseData).length) { + debug(`Adding alert for project (${project.id})`) + alerts.push(alert) + } + // Always include the result + results.push(result) + debug(`Processed project (${project.id})`) + }) + return { alerts, results, diff --git a/src/store/index.js b/src/store/index.js index 29d81e4..1ad0131 100644 --- a/src/store/index.js +++ b/src/store/index.js @@ -224,7 +224,6 @@ const getAllOSSFResultsOfRepositoriesByProjectId = async (knex, projectIds) => { exclude: ['repo_id', 'ossf_id'] }) orgData.repositories = [] - orgData.ossf_results = [] organizationsMap.set(orgId, orgData) } From 85d0ec86032868ab3f271b46384b8a608e26065d Mon Sep 17 00:00:00 2001 From: Sebastian Beltran Date: Tue, 11 Feb 2025 21:04:56 -0500 Subject: [PATCH 07/10] test: add test for staticCodeAnalysis checks --- __tests__/checks/staticCodeAnalysis.test.js | 165 +++++++++++++++++++- __utils__/index.js | 13 ++ 2 files changed, 171 insertions(+), 7 deletions(-) diff --git a/__tests__/checks/staticCodeAnalysis.test.js b/__tests__/checks/staticCodeAnalysis.test.js index be3ed53..626ad0b 100644 --- a/__tests__/checks/staticCodeAnalysis.test.js +++ b/__tests__/checks/staticCodeAnalysis.test.js @@ -2,7 +2,8 @@ const knexInit = require('knex') const { getConfig } = require('../../src/config') const staticCodeAnalysis = require('../../src/checks/complianceChecks/staticCodeAnalysis') const { - resetDatabase, initializeStore, generateGithubRepoData + resetDatabase, initializeStore, generateGithubRepoData, + generateOSSFScorecardData } = require('../../__utils__') const { sampleGithubOrg } = require('../../__fixtures__') @@ -14,6 +15,7 @@ let knex, addProject, addGithubOrg, addGithubRepo, + addOSSFScorecardResult, getAllResults, getAllTasks, getAllAlerts, @@ -27,6 +29,7 @@ beforeAll(async () => { ({ addProject, addGithubOrganization: addGithubOrg, + addOSSFScorecardResult, addGithubRepo, getAllResults, getAllTasks, @@ -49,10 +52,158 @@ afterAll(async () => { }) describe('Integration: staticCodeAnalysis', () => { - test.todo('Should add results without alerts or tasks') - test.todo('Should delete (previous alerts and tasks) and add results') - test.todo('Should add (alerts and tasks) and update results') - test.todo('Should add (alerts and tasks) and update results without repos and ossf_results') - test.todo('Should add (alerts and tasks) and update results without repos') - test.todo('Should add (alerts and tasks) and update results without ossf_results') + test('Should add results without alerts or tasks', async () => { + // Add a passed check scenario + const org = await addGithubOrg({ + login: sampleGithubOrg.login, + html_url: sampleGithubOrg.html_url, + project_id: project.id + }) + const repoData = generateGithubRepoData({ org, name: 'repo1', id: 1 }) + await addGithubRepo(repoData) + + const ossfResults = generateOSSFScorecardData({ github_repository_id: 1, sast_score: 10 }) + await addOSSFScorecardResult(ossfResults) + + // Check that the database is empty + let results = await getAllResults() + expect(results.length).toBe(0) + let alerts = await getAllAlerts() + expect(alerts.length).toBe(0) + let tasks = await getAllTasks() + expect(tasks.length).toBe(0) + // Run the check + await expect(staticCodeAnalysis(knex)).resolves.toBeUndefined() + // Check that the database has the expected results + results = await getAllResults() + expect(results.length).toBe(1) + expect(results[0].status).toBe('passed') + expect(results[0].compliance_check_id).toBe(check.id) + alerts = await getAllAlerts() + expect(alerts.length).toBe(0) + tasks = await getAllTasks() + expect(tasks.length).toBe(0) + }) + test('Should delete (previous alerts and tasks) and add results', async () => { + // Prepare the Scenario + const org = await addGithubOrg({ + login: sampleGithubOrg.login, + html_url: sampleGithubOrg.html_url, + project_id: project.id + }) + const repoData = generateGithubRepoData({ org, name: 'repo1' }) + await addGithubRepo(repoData) + + const ossfResults = generateOSSFScorecardData({ github_repository_id: 1, sast_score: 10 }) + await addOSSFScorecardResult(ossfResults) + + await addAlert({ compliance_check_id: check.id, project_id: project.id, title: 'existing', description: 'existing', severity: 'critical' }) + await addTask({ compliance_check_id: check.id, project_id: project.id, title: 'existing', description: 'existing', severity: 'critical' }) + // Check that the database has the expected results + let results = await getAllResults() + expect(results.length).toBe(0) + let alerts = await getAllAlerts() + expect(alerts.length).toBe(1) + expect(alerts[0].compliance_check_id).toBe(check.id) + let tasks = await getAllTasks() + expect(tasks.length).toBe(1) + expect(tasks[0].compliance_check_id).toBe(check.id) + // Run the check + await staticCodeAnalysis(knex) + // Check that the database has the expected results + results = await getAllResults() + expect(results.length).toBe(1) + expect(results[0].status).toBe('passed') + alerts = await getAllAlerts() + expect(alerts.length).toBe(0) + tasks = await getAllTasks() + expect(tasks.length).toBe(0) + }) + + test('Should add (alerts and tasks) and update results', async () => { + const org = await addGithubOrg({ login: sampleGithubOrg.login, html_url: sampleGithubOrg.html_url, project_id: project.id }) + const repoData = generateGithubRepoData({ org, name: 'repo1', secret_scanning_status: 'enabled' }) + await addGithubRepo(repoData) + + const ossfResults = generateOSSFScorecardData({ github_repository_id: 1, sast_score: 4 }) + await addOSSFScorecardResult(ossfResults) + + await addResult({ compliance_check_id: check.id, project_id: project.id, status: 'passed', rationale: 'failed previously', severity: 'critical' }) + // Check that the database has the expected results + let results = await getAllResults() + expect(results.length).toBe(1) + expect(results[0].compliance_check_id).toBe(check.id) + let alerts = await getAllAlerts() + expect(alerts.length).toBe(0) + let tasks = await getAllTasks() + expect(tasks.length).toBe(0) + // Run the check + await staticCodeAnalysis(knex) + // Check that the database has the expected results + results = await getAllResults() + expect(results.length).toBe(1) + expect(results[0].status).toBe('failed') + expect(results[0].rationale).not.toBe('failed previously') + alerts = await getAllAlerts() + expect(alerts.length).toBe(1) + expect(alerts[0].compliance_check_id).toBe(check.id) + tasks = await getAllTasks() + expect(tasks.length).toBe(1) + expect(tasks[0].compliance_check_id).toBe(check.id) + }) + test('Should add (alerts and tasks) and update results without repos and ossf_results', async () => { + // Prepare the Scenario + await addGithubOrg({ login: sampleGithubOrg.login, html_url: sampleGithubOrg.html_url, project_id: project.id }) + + await addResult({ compliance_check_id: check.id, project_id: project.id, status: 'passed', rationale: 'failed previously', severity: 'critical' }) + // Check that the database has the expected results + let results = await getAllResults() + expect(results.length).toBe(1) + expect(results[0].compliance_check_id).toBe(check.id) + let alerts = await getAllAlerts() + expect(alerts.length).toBe(0) + let tasks = await getAllTasks() + expect(tasks.length).toBe(0) + // Run the check + await staticCodeAnalysis(knex) + // Check that the database has the expected results + results = await getAllResults() + expect(results.length).toBe(1) + expect(results[0].status).toBe('passed') + expect(results[0].rationale).not.toBe('failed previously') + alerts = await getAllAlerts() + expect(alerts.length).toBe(0) + tasks = await getAllTasks() + expect(tasks.length).toBe(0) + }) + + test('Should add (alerts and tasks) and update results without ossf_results', async () => { + // Add a passed check scenario + const org = await addGithubOrg({ + login: sampleGithubOrg.login, + html_url: sampleGithubOrg.html_url, + project_id: project.id + }) + const repoData = generateGithubRepoData({ org, name: 'repo1', id: 1 }) + await addGithubRepo(repoData) + + // Check that the database is empty + let results = await getAllResults() + expect(results.length).toBe(0) + let alerts = await getAllAlerts() + expect(alerts.length).toBe(0) + let tasks = await getAllTasks() + expect(tasks.length).toBe(0) + // Run the check + await expect(staticCodeAnalysis(knex)).resolves.toBeUndefined() + // Check that the database has the expected results + results = await getAllResults() + expect(results.length).toBe(1) + expect(results[0].status).toBe('unknown') + expect(results[0].compliance_check_id).toBe(check.id) + alerts = await getAllAlerts() + expect(alerts.length).toBe(0) + tasks = await getAllTasks() + expect(tasks.length).toBe(0) + }) }) diff --git a/__utils__/index.js b/__utils__/index.js index 55d7024..3cd0257 100644 --- a/__utils__/index.js +++ b/__utils__/index.js @@ -28,8 +28,21 @@ const generateGithubRepoData = (data) => { } } +const generateOSSFScorecardData = (data) => { + return { + analysis_score: 10, + analysis_time: '2024-12-11T23:55:17Z', + analysis_execution_time: 19876, + repo_commit: 'dacv23', + scorecard_version: 3, + scorecard_commit: 'dsad', + ...data + } +} + module.exports = { generateGithubRepoData, + generateOSSFScorecardData, resetDatabase, initializeStore } From d619e6d0427ce60d6206183816268cf85caf787f Mon Sep 17 00:00:00 2001 From: Sebastian Beltran Date: Thu, 13 Feb 2025 20:53:41 -0500 Subject: [PATCH 08/10] refactor: implement processEntities utility for transforming data structures --- src/store/index.js | 84 ++++++++++++++-------------------------------- src/utils/index.js | 30 +++++++++++++++++ 2 files changed, 56 insertions(+), 58 deletions(-) diff --git a/src/store/index.js b/src/store/index.js index 1ad0131..067a88d 100644 --- a/src/store/index.js +++ b/src/store/index.js @@ -1,5 +1,5 @@ const debug = require('debug')('store') -const { simplifyObject } = require('@ulisesgascon/simplify-object') +const { processEntities } = require('../utils') const getAllFn = knex => (table) => { debug(`Fetching all records from ${table}...`) @@ -155,29 +155,16 @@ const getAllGithubRepositoriesAndOrganizationByProjectId = async (knex, projectI 'github_organizations.id' ) - // @TODO: Refactor this into a helper function or part of the query - // Transform results into desired structure - const organizationsMap = new Map() - - results.forEach(row => { - const orgId = row.org_id - - if (!organizationsMap.has(orgId)) { - // Create org entry if not exists - const orgData = simplifyObject(row, { - exclude: ['repo_id'] - }) - orgData.repositories = [] - organizationsMap.set(orgId, orgData) - } - - // Add repository if it exists - if (row.repo_id) { - const repoData = simplifyObject(row, { - exclude: ['repo_id', 'org_id'] - }) - organizationsMap.get(orgId).repositories.push(repoData) - } + const organizationsMap = processEntities(results, { + idKey: 'org_id', + excludedKeys: ['repo_id'], + relationships: [ + { + name: 'repositories', + relatedIdKey: 'repo_id', + excludedKeys: ['org_id', 'repo_id'] + } + ] }) return Array.from(organizationsMap.values()) @@ -211,40 +198,21 @@ const getAllOSSFResultsOfRepositoriesByProjectId = async (knex, projectIds) => { 'github_repositories.id' ) - // @TODO: Refactor this into a helper function or part of the query - // Transform results into desired structure - const organizationsMap = new Map() - - results.forEach(row => { - const orgId = row.org_id - - if (!organizationsMap.has(orgId)) { - // Create org entry if not exists - const orgData = simplifyObject(row, { - exclude: ['repo_id', 'ossf_id'] - }) - orgData.repositories = [] - organizationsMap.set(orgId, orgData) - } - - let repoData = {} - // Add repository if it exists - if (row.repo_id) { - repoData = simplifyObject(row, { - exclude: ['repo_id', 'org_id', 'ossf_id'] - }) - } - - if (row.ossf_id && row.repo_id) { - const ossfData = simplifyObject(row, { - exclude: ['ossf_id', 'repo_id', 'org_id'] - }) - repoData.ossf_results = ossfData - } - - if (row.repo_id) { - organizationsMap.get(orgId).repositories.push(repoData) - } + const organizationsMap = processEntities(results, { + idKey: 'org_id', + excludedKeys: ['repo_id', 'ossf_id'], + relationships: [ + { + name: 'repositories', + relatedIdKey: 'repo_id', + excludedKeys: ['ossf_id', 'repo_id', 'org_id'], + relationship: { + name: 'ossf_results', + relatedIdKey: 'ossf_id', + excludedKeys: ['ossf_id', 'repo_id', 'org_id'] + } + } + ] }) return Array.from(organizationsMap.values()) diff --git a/src/utils/index.js b/src/utils/index.js index c17152d..fec353f 100644 --- a/src/utils/index.js +++ b/src/utils/index.js @@ -1,5 +1,6 @@ const { add, parseISO, isBefore } = require('date-fns') const isURL = require('validator/lib/isURL.js') +const { simplifyObject } = require('@ulisesgascon/simplify-object') const pinoInit = require('pino') // GitHub token pattern: looks for patterns matching the GitHub token structure @@ -112,8 +113,37 @@ const isDateWithinPolicy = (targetDate, policy) => { return isBefore(currentDate, expirationDate) // Check if current date is before expiration } +const processEntities = (data, entityConfig) => { + const entityMap = new Map() + + data.forEach(item => { + const entityId = item[entityConfig.idKey] + const entity = simplifyObject(item, { exclude: entityConfig.excludedKeys }) + + entityConfig.relationships.forEach(relationship => { + entity[relationship.name] = entity[relationship.name] || [] + const relatedEntityId = item[relationship.relatedIdKey] + + if (relatedEntityId) { + const relatedEntity = simplifyObject(item, { exclude: relationship.excludedKeys }) + + if (item[relationship?.relationship?.relatedIdKey]) { + relatedEntity[relationship.relationship.name] = relatedEntity[relationship.relationship.name] || simplifyObject(item, { exclude: relationship.relationship.excludedKeys }) + } + + entity[relationship.name].push(relatedEntity) + } + }) + + entityMap.set(entityId, entity) + }) + + return entityMap +} + module.exports = { isDateWithinPolicy, + processEntities, validateGithubUrl, ensureGithubToken, getSeverityFromPriorityGroup, From 42c1da6ecd0a726a83a5a61162efe4f0c41f3297 Mon Sep 17 00:00:00 2001 From: Sebastian Beltran Date: Thu, 13 Feb 2025 21:29:16 -0500 Subject: [PATCH 09/10] test: add unit tests for processEntities utility function --- __tests__/utils.test.js | 140 +++++++++++++++++++++++++++++++++++++++- src/store/index.js | 9 +-- src/utils/index.js | 11 ++-- 3 files changed, 149 insertions(+), 11 deletions(-) diff --git a/__tests__/utils.test.js b/__tests__/utils.test.js index 5459140..af9a02e 100644 --- a/__tests__/utils.test.js +++ b/__tests__/utils.test.js @@ -1,4 +1,4 @@ -const { validateGithubUrl, ensureGithubToken, groupArrayItemsByCriteria, getSeverityFromPriorityGroup, isDateWithinPolicy, redactSensitiveData, generatePercentage } = require('../src/utils/index') +const { validateGithubUrl, ensureGithubToken, groupArrayItemsByCriteria, getSeverityFromPriorityGroup, isDateWithinPolicy, redactSensitiveData, generatePercentage, processEntities } = require('../src/utils/index') describe('ensureGithubToken', () => { let originalGithubToken @@ -156,3 +156,141 @@ describe('redactSensitiveData', () => { expect(redactSensitiveData(input)).toBe(input) }) }) + +describe('processEntities', () => { + it('should process entities with relationships', () => { + const data = [ + { + org_id: 1, + org_name: 'Org1', + repo_id: 1, + repo_name: 'Repo1', + ossf_id: 1, + ossf_score: 'A' + }, + { + org_id: 1, + org_name: 'Org1', + repo_id: 2, + repo_name: 'Repo2', + ossf_id: 2, + ossf_score: 'B' + }, + { + org_id: 2, + org_name: 'Org2', + repo_id: 3, + repo_name: 'Repo3', + ossf_id: 3, + ossf_score: 'C' + } + ] + const entityConfig = { + idKey: 'org_id', + excludedKeys: ['repo_id'], + relationships: [ + { + name: 'repositories', + relatedIdKey: 'repo_id', + excludedKeys: ['org_id', 'repo_id'] + } + ] + } + const expected = [ + { + org_id: 1, + org_name: 'Org1', + ossf_id: 1, + ossf_score: 'A', + repo_name: 'Repo1', + repositories: [ + { + org_name: 'Org1', + ossf_id: 1, + ossf_score: 'A', + repo_name: 'Repo1', + }, + { + org_name: 'Org1', + repo_name: 'Repo2', + ossf_id: 2, + ossf_score: 'B' + } + ] + }, + { + org_id: 2, + org_name: 'Org2', + ossf_id: 3, + ossf_score: 'C', + repo_name: 'Repo3', + repositories: [ + { + org_name: 'Org2', + repo_name: 'Repo3', + ossf_id: 3, + ossf_score: 'C' + } + ] + } + ] + expect(processEntities(data, entityConfig)).toEqual(expected) + }) + + it('should process entities without relationships', () => { + const data = [ + { + org_id: 1, + org_name: 'Org1', + repo_id: 1, + repo_name: 'Repo1', + ossf_id: 1, + ossf_score: 'A' + }, + { + org_id: 2, + org_name: 'Org2', + repo_id: 2, + repo_name: 'Repo2', + ossf_id: 2, + ossf_score: 'B' + }, + { + org_id: 3, + org_name: 'Org1', + repo_id: 3, + repo_name: 'Repo3', + ossf_id: 3, + ossf_score: 'A' + }, + ] + const entityConfig = { + idKey: 'org_id', + excludedKeys: ['repo_id'], + relationships: [] + } + const expected = [ + { + org_id: 1, + org_name: 'Org1', + ossf_id: 1, + ossf_score: 'A', + repo_name: 'Repo1' + }, + { + org_id: 2, + org_name: 'Org2', + ossf_id: 2, + ossf_score: 'B', + repo_name: 'Repo2' + },{ + org_id: 3, + org_name: 'Org1', + repo_name: 'Repo3', + ossf_id: 3, + ossf_score: 'A' + }, + ] + expect(processEntities(data, entityConfig)).toEqual(expected) + }) +}) \ No newline at end of file diff --git a/src/store/index.js b/src/store/index.js index 321eb69..181e68a 100644 --- a/src/store/index.js +++ b/src/store/index.js @@ -153,7 +153,8 @@ const getAllGithubRepositoriesAndOrganizationByProjectId = async (knex, projectI 'github_organizations.id' ) - const organizationsMap = processEntities(results, { + + return processEntities(results, { idKey: 'org_id', excludedKeys: ['repo_id'], relationships: [ @@ -164,8 +165,6 @@ const getAllGithubRepositoriesAndOrganizationByProjectId = async (knex, projectI } ] }) - - return Array.from(organizationsMap.values()) } const getAllOSSFResultsOfRepositoriesByProjectId = async (knex, projectIds) => { @@ -196,7 +195,7 @@ const getAllOSSFResultsOfRepositoriesByProjectId = async (knex, projectIds) => { 'github_repositories.id' ) - const organizationsMap = processEntities(results, { + return processEntities(results, { idKey: 'org_id', excludedKeys: ['repo_id', 'ossf_id'], relationships: [ @@ -212,8 +211,6 @@ const getAllOSSFResultsOfRepositoriesByProjectId = async (knex, projectIds) => { } ] }) - - return Array.from(organizationsMap.values()) } const initializeStore = (knex) => { diff --git a/src/utils/index.js b/src/utils/index.js index fec353f..10d18a1 100644 --- a/src/utils/index.js +++ b/src/utils/index.js @@ -119,6 +119,10 @@ const processEntities = (data, entityConfig) => { data.forEach(item => { const entityId = item[entityConfig.idKey] const entity = simplifyObject(item, { exclude: entityConfig.excludedKeys }) + + if(!entityMap.has(entityId)) { + entityMap.set(entityId, entity) + } entityConfig.relationships.forEach(relationship => { entity[relationship.name] = entity[relationship.name] || [] @@ -131,14 +135,13 @@ const processEntities = (data, entityConfig) => { relatedEntity[relationship.relationship.name] = relatedEntity[relationship.relationship.name] || simplifyObject(item, { exclude: relationship.relationship.excludedKeys }) } - entity[relationship.name].push(relatedEntity) + entityMap.get(entityId)[relationship.name].push(relatedEntity) } - }) - entityMap.set(entityId, entity) + }) }) - return entityMap + return Array.from(entityMap.values()) } module.exports = { From 652d85c42d878f7c0668fc008b7a347347156526 Mon Sep 17 00:00:00 2001 From: Sebastian Beltran Date: Thu, 13 Feb 2025 21:54:43 -0500 Subject: [PATCH 10/10] test: enhance processEntities utility tests with nested relationships scenarios --- __tests__/utils.test.js | 302 +++++++++++++++++++++++++++++++++++++++- src/store/index.js | 1 - src/utils/index.js | 8 +- 3 files changed, 302 insertions(+), 9 deletions(-) diff --git a/__tests__/utils.test.js b/__tests__/utils.test.js index af9a02e..485bf47 100644 --- a/__tests__/utils.test.js +++ b/__tests__/utils.test.js @@ -208,7 +208,7 @@ describe('processEntities', () => { org_name: 'Org1', ossf_id: 1, ossf_score: 'A', - repo_name: 'Repo1', + repo_name: 'Repo1' }, { org_name: 'Org1', @@ -262,7 +262,7 @@ describe('processEntities', () => { repo_name: 'Repo3', ossf_id: 3, ossf_score: 'A' - }, + } ] const entityConfig = { idKey: 'org_id', @@ -283,14 +283,308 @@ describe('processEntities', () => { ossf_id: 2, ossf_score: 'B', repo_name: 'Repo2' - },{ + }, { org_id: 3, org_name: 'Org1', repo_name: 'Repo3', ossf_id: 3, ossf_score: 'A' + } + ] + expect(processEntities(data, entityConfig)).toEqual(expected) + }) + + it('should process entities without some properties', () => { + const data = [ + { + org_id: 1, + org_name: 'Org1', + repo_id: 1, + repo_name: 'Repo1' }, + { + org_id: 1, + org_name: 'Org1', + repo_id: 2, + repo_name: 'Repo2', + ossf_score: 'B' + }, + { + org_id: 2, + org_name: 'Org2', + repo_id: 3, + repo_name: 'Repo3', + ossf_id: 3, + ossf_score: 'C' + } + ] + const entityConfig = { + idKey: 'org_id', + excludedKeys: ['repo_id'], + relationships: [ + { + name: 'repositories', + relatedIdKey: 'repo_id', + excludedKeys: ['org_id', 'repo_id'] + } + ] + } + const expected = [ + { + org_id: 1, + org_name: 'Org1', + repo_name: 'Repo1', + repositories: [ + { + org_name: 'Org1', + repo_name: 'Repo1' + }, + { + org_name: 'Org1', + repo_name: 'Repo2', + ossf_score: 'B' + } + ] + }, + { + org_id: 2, + org_name: 'Org2', + ossf_id: 3, + ossf_score: 'C', + repo_name: 'Repo3', + repositories: [ + { + org_name: 'Org2', + repo_name: 'Repo3', + ossf_id: 3, + ossf_score: 'C' + } + ] + } ] expect(processEntities(data, entityConfig)).toEqual(expected) }) -}) \ No newline at end of file + + it('should process entities with nested relationships', () => { + const data = [ + { + org_id: 1, + org_name: 'Org1', + repo_id: 1, + repo_name: 'Repo1', + ossf_id: 1, + ossf_score: 'A', + rule_id: 1, + rule_name: 'Rule1' + }, + { + org_id: 1, + org_name: 'Org1', + repo_id: 2, + repo_name: 'Repo2', + ossf_id: 2, + ossf_score: 'B', + rule_id: 2, + rule_name: 'Rule2' + }, + { + org_id: 2, + org_name: 'Org2', + repo_id: 3, + repo_name: 'Repo3', + ossf_id: 3, + ossf_score: 'C', + rule_id: 3, + rule_name: 'Rule3' + } + ] + const entityConfig = { + idKey: 'org_id', + excludedKeys: ['repo_id'], + relationships: [ + { + name: 'repositories', + relatedIdKey: 'repo_id', + excludedKeys: ['org_id', 'repo_id'], + relationship: { + name: 'rules', + relatedIdKey: 'rule_id', + excludedKeys: ['org_id', 'repo_id', 'rule_id'] + } + } + ] + } + const expected = [ + { + org_id: 1, + org_name: 'Org1', + ossf_id: 1, + ossf_score: 'A', + repo_name: 'Repo1', + repositories: [ + { + org_name: 'Org1', + ossf_id: 1, + ossf_score: 'A', + repo_name: 'Repo1', + rule_id: 1, + rule_name: 'Rule1', + rules: + { + org_name: 'Org1', + ossf_id: 1, + ossf_score: 'A', + repo_name: 'Repo1', + rule_name: 'Rule1' + } + + }, + { + org_name: 'Org1', + repo_name: 'Repo2', + rule_name: 'Rule2', + rule_id: 2, + ossf_score: 'B', + ossf_id: 2, + rules: { + org_name: 'Org1', + ossf_id: 2, + ossf_score: 'B', + repo_name: 'Repo2', + rule_name: 'Rule2' + } + } + ], + rule_id: 1, + rule_name: 'Rule1' + }, + { + org_id: 2, + org_name: 'Org2', + ossf_id: 3, + ossf_score: 'C', + repo_name: 'Repo3', + repositories: [ + { + org_name: 'Org2', + repo_name: 'Repo3', + ossf_id: 3, + ossf_score: 'C', + rule_id: 3, + rule_name: 'Rule3', + rules: { + org_name: 'Org2', + repo_name: 'Repo3', + ossf_id: 3, + ossf_score: 'C', + rule_name: 'Rule3' + } + } + ], + rule_id: 3, + rule_name: 'Rule3' + } + ] + expect(processEntities(data, entityConfig)).toEqual(expected) + }) + + it('should process entities with nested relationships without some props', () => { + const data = [ + { + org_id: 1, + org_name: 'Org1', + repo_id: 1, + repo_name: 'Repo1', + ossf_id: 1, + ossf_score: 'A', + rule_name: 'Rule1' + }, + { + org_id: 1, + org_name: 'Org1', + repo_id: 2, + repo_name: 'Repo2', + ossf_id: 2, + ossf_score: 'B' + }, + { + org_id: 2, + org_name: 'Org2', + repo_id: 3, + repo_name: 'Repo3', + ossf_id: 3, + ossf_score: 'C', + rule_id: 3, + rule_name: 'Rule3' + } + ] + const entityConfig = { + idKey: 'org_id', + excludedKeys: ['repo_id'], + relationships: [ + { + name: 'repositories', + relatedIdKey: 'repo_id', + excludedKeys: ['org_id', 'repo_id'], + relationship: { + name: 'rules', + relatedIdKey: 'rule_id', + excludedKeys: ['org_id', 'repo_id', 'rule_id'] + } + } + ] + } + const expected = [ + { + org_id: 1, + org_name: 'Org1', + ossf_id: 1, + ossf_score: 'A', + repo_name: 'Repo1', + repositories: [ + { + org_name: 'Org1', + ossf_id: 1, + ossf_score: 'A', + repo_name: 'Repo1', + rule_name: 'Rule1' + }, + { + org_name: 'Org1', + repo_name: 'Repo2', + ossf_score: 'B', + ossf_id: 2 + } + ], + rule_name: 'Rule1' + }, + { + org_id: 2, + org_name: 'Org2', + ossf_id: 3, + ossf_score: 'C', + repo_name: 'Repo3', + repositories: [ + { + org_name: 'Org2', + repo_name: 'Repo3', + ossf_id: 3, + ossf_score: 'C', + rule_id: 3, + rule_name: 'Rule3', + rules: { + org_name: 'Org2', + repo_name: 'Repo3', + ossf_id: 3, + ossf_score: 'C', + rule_name: 'Rule3' + } + } + ], + rule_id: 3, + rule_name: 'Rule3' + } + ] + expect(processEntities(data, entityConfig)).toEqual(expected) + }) +}) diff --git a/src/store/index.js b/src/store/index.js index 181e68a..07e50fe 100644 --- a/src/store/index.js +++ b/src/store/index.js @@ -153,7 +153,6 @@ const getAllGithubRepositoriesAndOrganizationByProjectId = async (knex, projectI 'github_organizations.id' ) - return processEntities(results, { idKey: 'org_id', excludedKeys: ['repo_id'], diff --git a/src/utils/index.js b/src/utils/index.js index 10d18a1..0be50b8 100644 --- a/src/utils/index.js +++ b/src/utils/index.js @@ -119,13 +119,14 @@ const processEntities = (data, entityConfig) => { data.forEach(item => { const entityId = item[entityConfig.idKey] const entity = simplifyObject(item, { exclude: entityConfig.excludedKeys }) - - if(!entityMap.has(entityId)) { + + if (!entityMap.has(entityId)) { entityMap.set(entityId, entity) } entityConfig.relationships.forEach(relationship => { entity[relationship.name] = entity[relationship.name] || [] + const relatedEntityId = item[relationship.relatedIdKey] if (relatedEntityId) { @@ -137,8 +138,7 @@ const processEntities = (data, entityConfig) => { entityMap.get(entityId)[relationship.name].push(relatedEntity) } - - }) + }) }) return Array.from(entityMap.values())