Skip to content

Commit

Permalink
feat(helmTool): save helm chart details in database
Browse files Browse the repository at this point in the history 
SXT-1015

Signed-off-by: Alex Marshall <[email protected]>
  • Loading branch information
@alexhq
alexhq committed Jul 26, 2023
1 parent 1e3e87b commit a67845a
Show file tree
Hide file tree
Showing 4 changed files with 271 additions and 50 deletions.
229 changes: 184 additions & 45 deletions src/helmTool.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,45 @@ import * as util from 'util'
import { ChartTable, Edition, HelmRepository } from './edition-type'
import { getLogger } from './logging'
import { Store } from './store'
import { response } from 'express'
import * as yaml from 'js-yaml'
import { HelmChart } from './store/model/model-types'

const logger = getLogger({
name: 'helmTool',
})
const exec = util.promisify(childProcess.exec)

type HelmIndex = {
apiVersion: string
entries: {
[key: string]: {
apiVersion: string
appVersion: string
created: string
dependencies: {
name: string
repository: string
version: string
alias?: string
condition?: string
[key: string]: unknown
}[]
description: string
digest: string
name: string
type: string
urls: string[]
keywords: string[]
kubeVersion?: string
version: string
icon?: string
[key: string]: unknown
}[]
}
generated: string
}

export class HelmTool {
chartTable: ChartTable
destinationDir: string
Expand All @@ -25,7 +58,7 @@ export class HelmTool {
logger.info({ destinationDir }, 'HelmTool created')
}

async add() {
async addRepositories() {
// runs helm add for all the repos in the editions object
// iterate through the repos array adding each one
const runHelmAdd = async (repo: HelmRepository) => {
Expand All @@ -40,6 +73,7 @@ export class HelmTool {
}
for (const repo of this.helmRepos) {
await runHelmAdd(repo)
await this.getIndexYaml(repo)
}
}

Expand All @@ -66,6 +100,22 @@ export class HelmTool {
return parseVal[0].version as string
}

// Given a helm repository, download the index.yaml file and return it as an object
async getIndexYaml(repo: HelmRepository) {
const repoIndex = `${repo.url}/index.yaml`
logger.info({ action: 'downloading index.yaml', repoIndex })
const response = await fetch(repoIndex)
if (!response.ok) {
throw new Error(`failed to download index.yaml from ${repoIndex}`)
}
const blob = await response.blob()
const txt = await blob.text()
// logger.trace({ action: 'downloading index.yaml', txt, repoIndex })
const indexObject = yaml.load(txt) as HelmIndex
logger.info({ action: 'index object is', indexObject })
return indexObject
}

async pull(chart: string, exactChartVersion: string, deploymentType: string, deploymentVersion: string) {
const cmd =
`helm pull ${chart} --version ${exactChartVersion} ` +
Expand All @@ -82,46 +132,6 @@ export class HelmTool {
return result.stdout
}

async refreshDbRepos() {
// Get list of helm repos from the edition
const editionRepos = this.helmRepos
// Get list of helm repos from the database
const dbRepos = await this.store.helmrepository.list()
await this.store.transaction(async (trx) => {
// For each helm repo in the edition, check if it is in the database
for (const repo of editionRepos) {
// check if the helm repo is already in the database
const repoInDb = dbRepos.find((helmRepo) => helmRepo.name === repo.name)
if (repoInDb) {
// update the helm repo in the database
await this.store.helmrepository.update(
{
data: {
active: true,
name: repo.name,
url: repo.url,
},
id: repoInDb.id,
},
trx
)
} else {
// add the helm repo to the database
await this.store.helmrepository.create(
{
data: {
active: true,
name: repo.name,
url: repo.url,
},
},
trx
)
}
}
})
}

async remove(deploymentType: string) {
logger.debug(
{
Expand Down Expand Up @@ -150,10 +160,11 @@ export class HelmTool {

async start() {
try {
await this.add()
await this.update()
await this.refreshDbRepos()
await this.addRepositories()
await this.updateRepositories()
await this.syncDbRepositories()
await this.storeChartsLocally()
await this.updateDbHelmCharts()
} catch (err: unknown) {
logger.error({ err }, 'Error running helmTool.start()')
throw err
Expand All @@ -174,8 +185,136 @@ export class HelmTool {
}
}

// // for each repo that is in the database, for each get the index yaml, and parse that into chart table
// async syncDbCharts() {
// // Get list of helm repos from the database
// const dbRepos = await this.store.helmrepository.list()
// // For each helm in the database get the index.yaml
// for (const repo of dbRepos) {
// // Get the index.yaml file
// const repoindex = await this.getIndexYaml(repo)
// console.log('syncDBcharts', repoindex)
// }
// }

// Take a helm repository and chart name.
// Get the helm repository index.yaml
// Populate the database with the helm chart information from the index.yaml
async syncDbCharts(repo: HelmRepository) {
// Get the index.yaml file
const repoindex = await this.getIndexYaml(repo)
console.log('repoindex', repoindex)
// From the repoindex make an array of charts
const indexcharts = Object.keys(repoindex.entries)
console.log('indexcharts', indexcharts)
// For each chart in the index.yaml
for (const chartKey of indexcharts) {
const chartVersions = repoindex.entries[chartKey]
console.log('chartVersions', chartVersions)
// Get the chart's helm repository details from the database
const dbRepository = await this.store.helmrepository.getByUrl({
url: repo.url,
})
// For each chart version in the index.yaml
for (const chart of chartVersions) {
const chartInDb = await this.store.helmchart.getExact({
name: chart.name,
repository_id: dbRepository.id,
version: chart.version,
})
if (chartInDb) {
// update the helm chart in the database
await this.store.helmchart.update({
id: chartInDb.id,
data: {
active: true,
app_version: chart.appVersion,
description: chart.description,
digest: chart.digest,
icon: chart.icon,
keywords: chart.keywords,
kube_version: chart.kubeVersion,
name: chart.name,
repository_id: dbRepository.id,
urls: chart.urls,
verified: false,
version: chart.version,
},
})
} else {
// add the helm chart to the database
await this.store.helmchart.create({
data: {
active: true,
app_version: chart.appVersion,
description: chart.description,
digest: chart.digest,
icon: chart.icon,
keywords: chart.keywords,
kube_version: chart.kubeVersion,
name: chart.name,
repository_id: dbRepository.id,
urls: chart.urls,
verified: false,
version: chart.version,
},
})
}
}
}
}

async syncDbRepositories() {
// Get list of helm repos from the edition
const editionRepos = this.helmRepos
// Get list of helm repos from the database
const dbRepos = await this.store.helmrepository.list()
await this.store.transaction(async (trx) => {
// For each helm repo in the edition, check if it is in the database
for (const repo of editionRepos) {
// check if the helm repo is already in the database
const repoInDb = dbRepos.find((helmRepo) => helmRepo.name === repo.name)
if (repoInDb) {
// update the helm repo in the database
await this.store.helmrepository.update(
{
data: {
active: true,
name: repo.name,
url: repo.url,
},
id: repoInDb.id,
},
trx
)
} else {
// add the helm repo to the database
await this.store.helmrepository.create(
{
data: {
active: true,
name: repo.name,
url: repo.url,
},
},
trx
)
}
}
})
}

async updateDbHelmCharts() {
// Get list of helm repos from the database
const dbRepos = await this.store.helmrepository.list()
// for each helm repo in the database, run syncDbHelmChart
for (const repo of dbRepos) {
await this.syncDbCharts(repo)
}
}

// Add in updating the helm-chart table to update
async update() {
async updateRepositories() {
logger.info({
action: 'updating helm repositories',
})
Expand Down
69 changes: 67 additions & 2 deletions src/store/helmchart.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,39 @@ export class HelmChartStore {
// * icon
// * version
// * keywords
// * urls

public async create(
{
data: { active, app_version, description, digest, repository_id, icon, name, version, keywords },
data: {
active,
app_version,
description,
digest,
repository_id,
icon,
name,
version,
verified,
keywords,
kube_version,
urls,
},
}: {
data: Pick<
HelmChart,
'active' | 'app_version' | 'description' | 'digest' | 'repository_id' | 'icon' | 'name' | 'version' | 'keywords'
| 'active'
| 'app_version'
| 'description'
| 'digest'
| 'repository_id'
| 'icon'
| 'name'
| 'version'
| 'verified'
| 'keywords'
| 'kube_version'
| 'urls'
>
},
trx?: Knex.Transaction
Expand All @@ -45,7 +70,10 @@ export class HelmChartStore {
icon,
name,
version,
verified,
keywords,
kube_version,
urls,
})
.returning('*')

Expand Down Expand Up @@ -79,6 +107,42 @@ export class HelmChartStore {
return result
}

// Get an exact helm chart
// Returns 0 or 1 helm charts
// params:
// * name
// * version
// * repository_id
public async getExact(
{ name, repository_id, version }: { name: string; repository_id: DatabaseIdentifier; version: string },
trx?: Knex.Transaction
) {
const [result] = await (trx || this.knex)<HelmChart>(TABLES.helmchart)
.where({
name,
repository_id,
version,
})
.returning<HelmChart[]>('*')

return result
}

// Get matching helm charts
// Returns 0, 1 or more helm charts
// params:
// * name
// * version
public async getMatching({ name }: { name: string }, trx?: Knex.Transaction) {
const result = await (trx || this.knex)<HelmChart>(TABLES.helmchart)
.where({
name,
})
.returning<HelmChart[]>('*')

return result
}

// List all helm charts
// params:
public list(trx?: Knex.Transaction) {
Expand All @@ -99,6 +163,7 @@ export class HelmChartStore {
// * icon
// * version
// * keywords
// * urls
public async update(
{ data, id }: { data: Partial<Omit<HelmChart, 'id'>>; id: DatabaseIdentifier },
trx?: Knex.Transaction
Expand Down
Loading

0 comments on commit a67845a

Please sign in to comment.