diff --git a/jest/helmTool.test.ts b/jest/helmTool.test.ts index d6a78947..dcbf9ffd 100644 --- a/jest/helmTool.test.ts +++ b/jest/helmTool.test.ts @@ -68,4 +68,24 @@ describe('helmTool', () => { }) expect(cmd).toEqual('helm repo add --force-update test https://example.com --username test --password test') }) + + // test getIndexYaml + it('can get index.yaml', async () => { + const tool = new HelmTool(edition, store, tmpDir) + const repo = await helmRepoStore.create({ + data: { + name: 'test-getindex-repo', + active: true, + url: 'https://charts.bitnami.com/bitnami', + }, + }) + const index = await tool.getIndexYaml(repo) + expect(index).toBeDefined() + }, 1200000) + + // test updateDbHelmCharts + it('can run updateDbHelmCharts', async () => { + const tool = new HelmTool(edition, store, tmpDir) + await tool.updateDbHelmCharts() + }, 1200000) }) diff --git a/jest/store/helmchart.test.ts b/jest/store/helmchart.test.ts index 3a9b41a0..4b074eda 100644 --- a/jest/store/helmchart.test.ts +++ b/jest/store/helmchart.test.ts @@ -41,7 +41,9 @@ describe('HelmChartStore', () => { repository_id: repo.id, icon: 'https://example.com/icon.png', version: '1.0.0', + verified: false, keywords: ['test', 'chart'], + urls: ['https://example.com/chart.tgz'], }, }) expect(chart).toMatchObject({ @@ -60,7 +62,9 @@ describe('HelmChartStore', () => { repository_id: repo.id, icon: 'https://example.com/icon.png', version: '1.0.0', + verified: false, keywords: ['test', 'chart'], + urls: ['https://example.com/chart.tgz'], }, }) @@ -82,7 +86,9 @@ describe('HelmChartStore', () => { repository_id: repo.id, icon: 'https://example.com/icon.png', version: '1.0.0', + verified: false, keywords: ['test', 'chart'], + urls: ['https://example.com/chart.tgz'], }, }) const getChart = await store.get({ id: chart.id }) @@ -91,6 +97,67 @@ describe('HelmChartStore', () => { }) }) + it('should get an exact helm chart', async () => { + const chart = await store.create({ + data: { + active: true, + name: 'test-chart-getexact', + app_version: '1.0.0', + description: 'test chart description', + digest: 'test-digest', + repository_id: repo.id, + icon: 'https://example.com/icon.png', + version: '1.0.0', + verified: false, + keywords: ['test', 'chart'], + urls: ['https://example.com/chart.tgz'], + }, + }) + const getChart = await store.getExact({ + name: chart.name, + version: chart.version, + repository_id: chart.repository_id, + }) + expect(getChart).toMatchObject({ + ...chart, + }) + }) + + it('should get matching helm charts', async () => { + const chartOne = await store.create({ + data: { + active: true, + name: 'test-chart-getmatching', + app_version: '1.0.0', + description: 'test chart description one', + digest: 'test-digest', + repository_id: repo.id, + icon: 'https://example.com/icon.png', + version: '1.0.0', + verified: false, + keywords: ['test', 'chart'], + urls: ['https://example.com/chart-100.tgz'], + }, + }) + const chartTwo = await store.create({ + data: { + active: true, + name: 'test-chart-getmatching', + app_version: '2.0.0', + description: 'test chart description', + digest: 'test-digest', + repository_id: repo.id, + icon: 'https://example.com/icon.png', + version: '2.0.0', + verified: false, + keywords: ['test', 'chart'], + urls: ['https://example.com/chart-200.tgz'], + }, + }) + const matchingCharts = await store.getMatching({ name: chartOne.name }) + expect(matchingCharts.length).toBe(2) + }) + it('should list all helm charts', async () => { const chart = await store.create({ data: { @@ -102,11 +169,11 @@ describe('HelmChartStore', () => { repository_id: repo.id, icon: 'https://example.com/icon.png', version: '1.0.0', + verified: false, keywords: ['test', 'chart'], + urls: ['https://example.com/chart.tgz'], }, }) - await store.delete({ id: chart.id }) - const charts = await store.list() expect(charts.length).toBeGreaterThan(0) }) @@ -122,7 +189,9 @@ describe('HelmChartStore', () => { repository_id: repo.id, icon: 'https://example.com/icon.png', version: '1.0.0', + verified: false, keywords: ['test', 'chart'], + urls: ['https://example.com/chart.tgz'], }, }) const updatedChart = await store.update({ diff --git a/jest/store/helmrepository.test.ts b/jest/store/helmrepository.test.ts index d6ac30f2..6e70f80e 100644 --- a/jest/store/helmrepository.test.ts +++ b/jest/store/helmrepository.test.ts @@ -97,4 +97,17 @@ describe('HelmRepositoryStore', () => { active: 'false', }) }) + it('should get a helm chart by url', async () => { + const repo = await helmRepoStore.create({ + data: { + name: 'test-repo-getbyurl', + active: true, + url: 'https://charts.example.com/url', + }, + }) + const getRepo = await helmRepoStore.getByUrl({ url: repo.url }) + expect(getRepo).toMatchObject({ + ...repo, + }) + }) }) diff --git a/migrations/202303091537_helmchart.js b/migrations/202303091537_helmchart.js index 5b621ed0..17e93722 100644 --- a/migrations/202303091537_helmchart.js +++ b/migrations/202303091537_helmchart.js @@ -6,18 +6,20 @@ const up = (knex) => { table.boolean('active').defaultTo('true').notNullable() table.string('app_version').notNullable() table.specificType('created_at', 'timestamp default now()') - table.string('description').notNullable() + table.text('description').notNullable() table.string('digest').notNullable() - table.string('icon') + table.text('icon') table.specificType('id', 'serial primary key not null') table.string('keywords') table.string('kube_version') - table.string('name').unique().notNullable() + table.string('name').notNullable() table.specificType('refreshed_at', 'timestamp') table.specificType('repository_id', 'integer references helmrepository(id)') table.specificType('updated_at', 'timestamp default now()') + table.string('urls').notNullable() table.boolean('verified').defaultTo(false).notNullable() table.string('version').notNullable() + table.unique(['name', 'version', 'repository_id']) }), ]) } diff --git a/package.json b/package.json index 71abd6e5..cc522bd0 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "download-helm-charts": "if [ -r dist/src/download-helm-charts.js ]; then node dist/src/download-helm-charts.js; else node src/download-helm-charts.js; fi", "copy-helm-fixtures": "tar -xf ./test/fixtures/helmCharts.tar.gz -C .", "remove-helm-fixtures": "rm -rf ./helmCharts/*", - "preserve": "npm run download-helm-charts && npm run migrate", + "preserve": " npm run migrate && npm run download-helm-charts", "serve": "if [ -r dist/src/index.js ]; then node dist/src/index.js ; else node src/index.js; fi", "generate-swagger": "node src/generateSwagger.js", "pretest": "npm run copy-helm-fixtures", diff --git a/src/helmTool.ts b/src/helmTool.ts index f2256ea3..74695dd8 100644 --- a/src/helmTool.ts +++ b/src/helmTool.ts @@ -4,12 +4,43 @@ import * as util from 'util' import { ChartTable, Edition, HelmRepository } from './edition-type' import { getLogger } from './logging' import { Store } from './store' +import * as yaml from 'js-yaml' const logger = getLogger({ name: 'helmTool', }) const exec = util.promisify(childProcess.exec) +type HelmIndex = { + apiVersion: string + entries: { + [key: string]: { + apiVersion: string + appVersion: string + created: string + dependencies: { + name: string + repository: string + version: string + alias?: string + condition?: string + [key: string]: unknown + }[] + description: string + digest: string + name: string + type: string + urls: string[] + keywords: string[] + kubeVersion?: string + version: string + icon?: string + [key: string]: unknown + }[] + } + generated: string +} + export class HelmTool { chartTable: ChartTable destinationDir: string @@ -25,7 +56,7 @@ export class HelmTool { logger.info({ destinationDir }, 'HelmTool created') } - async add() { + async addRepositories() { // runs helm add for all the repos in the editions object // iterate through the repos array adding each one const runHelmAdd = async (repo: HelmRepository) => { @@ -40,6 +71,7 @@ export class HelmTool { } for (const repo of this.helmRepos) { await runHelmAdd(repo) + await this.getIndexYaml(repo) } } @@ -66,6 +98,22 @@ export class HelmTool { return parseVal[0].version as string } + // Given a helm repository, download the index.yaml file and return it as an object + async getIndexYaml(repo: HelmRepository) { + const repoIndex = `${repo.url}/index.yaml` + logger.info({ action: 'downloading index.yaml', repoIndex }) + const response = await fetch(repoIndex) + if (!response.ok) { + throw new Error(`failed to download index.yaml from ${repoIndex}`) + } + const blob = await response.blob() + const txt = await blob.text() + // logger.trace({ action: 'downloading index.yaml', txt, repoIndex }) + const indexObject = yaml.load(txt) as HelmIndex + logger.info({ action: 'index object is', indexObject }) + return indexObject + } + async pull(chart: string, exactChartVersion: string, deploymentType: string, deploymentVersion: string) { const cmd = `helm pull ${chart} --version ${exactChartVersion} ` + @@ -82,46 +130,6 @@ export class HelmTool { return result.stdout } - async refreshDbRepos() { - // Get list of helm repos from the edition - const editionRepos = this.helmRepos - // Get list of helm repos from the database - const dbRepos = await this.store.helmrepository.list() - await this.store.transaction(async (trx) => { - // For each helm repo in the edition, check if it is in the database - for (const repo of editionRepos) { - // check if the helm repo is already in the database - const repoInDb = dbRepos.find((helmRepo) => helmRepo.name === repo.name) - if (repoInDb) { - // update the helm repo in the database - await this.store.helmrepository.update( - { - data: { - active: true, - name: repo.name, - url: repo.url, - }, - id: repoInDb.id, - }, - trx - ) - } else { - // add the helm repo to the database - await this.store.helmrepository.create( - { - data: { - active: true, - name: repo.name, - url: repo.url, - }, - }, - trx - ) - } - } - }) - } - async remove(deploymentType: string) { logger.debug( { @@ -150,10 +158,11 @@ export class HelmTool { async start() { try { - await this.add() - await this.update() - await this.refreshDbRepos() + await this.addRepositories() + await this.updateRepositories() + await this.syncDbRepositories() await this.storeChartsLocally() + await this.updateDbHelmCharts() } catch (err: unknown) { logger.error({ err }, 'Error running helmTool.start()') throw err @@ -174,8 +183,132 @@ export class HelmTool { } } + // For the given helm repository, get the index.yaml + // file and populate the database with the helm charts, + // or update the helm charts if they already exist + async syncDbCharts(repo: HelmRepository) { + // Get the index.yaml file + logger.info({ action: 'syncing helm charts for', repo }) + logger.info({ action: 'getting repository index.yaml' }) + const repoindex = await this.getIndexYaml(repo) + logger.debug({ action: 'repoindex', repoindex }) + // From the repoindex make an array of charts + const indexcharts = Object.keys(repoindex.entries) + logger.debug({ action: 'indexcharts', indexcharts }) + // For each chart in the index.yaml + for (const chartKey of indexcharts) { + const chartVersions = repoindex.entries[chartKey] + logger.debug({ action: 'chartVersions', chartVersions }) + // Get the chart's helm repository details from the database + const dbRepository = await this.store.helmrepository.getByUrl({ + url: repo.url, + }) + // For each chart version in the index.yaml + for (const chart of chartVersions) { + const chartInDb = await this.store.helmchart.getExact({ + name: chart.name, + repository_id: dbRepository.id, + version: chart.version, + }) + if (chartInDb) { + // update the helm chart in the database + logger.debug({ action: 'updating helm chart', chartInDb }) + await this.store.helmchart.update({ + id: chartInDb.id, + data: { + active: true, + app_version: chart.appVersion, + description: chart.description, + digest: chart.digest, + icon: chart.icon, + keywords: chart.keywords, + kube_version: chart.kubeVersion, + name: chart.name, + repository_id: dbRepository.id, + urls: chart.urls, + verified: false, + version: chart.version, + }, + }) + } else { + // add the helm chart to the database + logger.debug({ action: 'adding helm chart', chart }) + await this.store.helmchart.create({ + data: { + active: true, + app_version: chart.appVersion, + description: chart.description, + digest: chart.digest, + icon: chart.icon, + keywords: chart.keywords, + kube_version: chart.kubeVersion, + name: chart.name, + repository_id: dbRepository.id, + urls: chart.urls, + verified: false, + version: chart.version, + }, + }) + } + } + } + } + + // For each helm repository that is in the edition, + // check if it is in the database. If it is, update it, if not add it + async syncDbRepositories() { + // Get list of helm repos from the edition + const editionRepos = this.helmRepos + // Get list of helm repos from the database + const dbRepos = await this.store.helmrepository.list() + await this.store.transaction(async (trx) => { + // For each helm repo in the edition, check if it is in the database + for (const repo of editionRepos) { + // check if the helm repo is already in the database + const repoInDb = dbRepos.find((helmRepo) => helmRepo.name === repo.name) + if (repoInDb) { + // update the helm repo in the database + await this.store.helmrepository.update( + { + data: { + active: true, + name: repo.name, + url: repo.url, + }, + id: repoInDb.id, + }, + trx + ) + } else { + // add the helm repo to the database + await this.store.helmrepository.create( + { + data: { + active: true, + name: repo.name, + url: repo.url, + }, + }, + trx + ) + } + } + }) + } + + // For each helm repository that is in the database, run syncDbCharts + async updateDbHelmCharts() { + // Get list of helm repositories from the database + const dbRepos = await this.store.helmrepository.list() + logger.debug({ action: 'dbRepos', dbRepos }) + // Run syncDbHelmChart for each repository + for (const repo of dbRepos) { + await this.syncDbCharts(repo) + } + } + // Add in updating the helm-chart table to update - async update() { + async updateRepositories() { logger.info({ action: 'updating helm repositories', }) diff --git a/src/store/helmchart.ts b/src/store/helmchart.ts index e7943c39..9ee40f4f 100644 --- a/src/store/helmchart.ts +++ b/src/store/helmchart.ts @@ -23,14 +23,39 @@ export class HelmChartStore { // * icon // * version // * keywords + // * urls public async create( { - data: { active, app_version, description, digest, repository_id, icon, name, version, keywords }, + data: { + active, + app_version, + description, + digest, + repository_id, + icon, + name, + version, + verified, + keywords, + kube_version, + urls, + }, }: { data: Pick< HelmChart, - 'active' | 'app_version' | 'description' | 'digest' | 'repository_id' | 'icon' | 'name' | 'version' | 'keywords' + | 'active' + | 'app_version' + | 'description' + | 'digest' + | 'repository_id' + | 'icon' + | 'name' + | 'version' + | 'verified' + | 'keywords' + | 'kube_version' + | 'urls' > }, trx?: Knex.Transaction @@ -45,7 +70,10 @@ export class HelmChartStore { icon, name, version, + verified, keywords, + kube_version, + urls, }) .returning('*') @@ -79,6 +107,42 @@ export class HelmChartStore { return result } + // Get an exact helm chart + // Returns 0 or 1 helm charts + // params: + // * name + // * version + // * repository_id + public async getExact( + { name, repository_id, version }: { name: string; repository_id: DatabaseIdentifier; version: string }, + trx?: Knex.Transaction + ) { + const [result] = await (trx || this.knex)(TABLES.helmchart) + .where({ + name, + repository_id, + version, + }) + .returning('*') + + return result + } + + // Get matching helm charts + // Returns 0, 1 or more helm charts + // params: + // * name + // * version + public async getMatching({ name }: { name: string }, trx?: Knex.Transaction) { + const result = await (trx || this.knex)(TABLES.helmchart) + .where({ + name, + }) + .returning('*') + + return result + } + // List all helm charts // params: public list(trx?: Knex.Transaction) { @@ -99,6 +163,7 @@ export class HelmChartStore { // * icon // * version // * keywords + // * urls public async update( { data, id }: { data: Partial>; id: DatabaseIdentifier }, trx?: Knex.Transaction diff --git a/src/store/helmrepository.ts b/src/store/helmrepository.ts index e05b4864..a00300d6 100644 --- a/src/store/helmrepository.ts +++ b/src/store/helmrepository.ts @@ -64,9 +64,21 @@ export class HelmRepositoryStore { return result } + // Get a helm repository by its url + // params: + // * url + public async getByUrl({ url }: { url: string }, trx?: Knex.Transaction) { + const [result] = await (trx || this.knex)(TABLES.helmrepository) + .where({ + url, + }) + .returning('*') + + return result + } + // List all helm repositories // params: none - public list(trx?: Knex.Transaction) { const orderBy = LIST_ORDER_BY_FIELDS.helmrepository return (trx || this.knex)(TABLES.helmrepository) diff --git a/src/store/model/model-types.ts b/src/store/model/model-types.ts index 7008cea4..38e8c653 100644 --- a/src/store/model/model-types.ts +++ b/src/store/model/model-types.ts @@ -115,7 +115,6 @@ export type HelmRepository = { } export type HelmChart = { - // short name of the chart without the repo name active: boolean app_version: string created_at: Date @@ -124,11 +123,17 @@ export type HelmChart = { icon?: string id: DatabaseIdentifier keywords?: string[] - kubeVersion?: string + kube_version?: string name: string refreshed_at?: Date repository_id: DatabaseIdentifier updated_at: Date + urls: string[] verified: boolean version: string } + +export type IndexYaml = { + apiVersion: string + entries: [apiVersion: string] +}