diff --git a/packaging/electron-builder.yml b/packaging/electron-builder.yml index 087d4fdbf57..34441f97ff5 100644 --- a/packaging/electron-builder.yml +++ b/packaging/electron-builder.yml @@ -4,11 +4,15 @@ productName: Rancher Desktop icon: ./resources/icons/logo-square-512.png appId: io.rancherdesktop.app asar: true +electronLanguages: [ en-US ] extraResources: - resources/ - '!resources/darwin/lima*.tgz' - '!resources/linux/lima*.tgz' +- '!resources/linux/staging/' +- '!resources/win32/staging/' - '!resources/host/' +- '!resources/**/*.js.map' files: - dist/app/**/* - '!**/node_modules/*/prebuilds/!(${platform}*)/*.node' diff --git a/pkg/rancher-desktop/backend/wsl.ts b/pkg/rancher-desktop/backend/wsl.ts index 9da50747331..9d0fd4a742d 100644 --- a/pkg/rancher-desktop/backend/wsl.ts +++ b/pkg/rancher-desktop/backend/wsl.ts @@ -903,21 +903,6 @@ export default class WSLBackend extends events.EventEmitter implements VMBackend } } - /** - * On Windows Trivy is run via WSL as there's no native port. - * Ensure that all relevant files are in the wsl mount, not the windows one. - */ - protected async installTrivy() { - // download-resources.sh installed trivy into the resources area - // This function moves it into /usr/local/bin/ so when trivy is - // invoked to run through wsl, it runs faster. - - const trivyExecPath = path.join(paths.resources, 'linux', 'internal', 'trivy'); - - await this.execCommand('mkdir', '-p', '/var/local/bin'); - await this.wslInstall(trivyExecPath, '/usr/local/bin'); - } - protected async installGuestAgent(kubeVersion: semver.SemVer | undefined, cfg: BackendSettings | undefined) { let guestAgentConfig: Record; const enableKubernetes = !!kubeVersion; @@ -947,10 +932,7 @@ export default class WSLBackend extends events.EventEmitter implements VMBackend }; } - const guestAgentPath = path.join(paths.resources, 'linux', 'internal', 'guestagent'); - await Promise.all([ - this.wslInstall(guestAgentPath, '/usr/local/bin/', 'rancher-desktop-guestagent'), this.writeFile('/etc/init.d/rancher-desktop-guestagent', SERVICE_GUEST_AGENT_INIT, 0o755), this.writeConf('rancher-desktop-guestagent', guestAgentConfig), ]); @@ -1525,7 +1507,6 @@ export default class WSLBackend extends events.EventEmitter implements VMBackend this.runWslProxy().catch(console.error); } }), - this.progressTracker.action('Installing image scanner', 100, this.installTrivy()), this.progressTracker.action('Installing CA certificates', 100, this.installCACerts()), this.progressTracker.action('Installing helpers', 50, this.installWSLHelpers()), ])); diff --git a/scripts/dependencies/go-source.ts b/scripts/dependencies/go-source.ts new file mode 100644 index 00000000000..6b6943515e3 --- /dev/null +++ b/scripts/dependencies/go-source.ts @@ -0,0 +1,137 @@ +import path from 'path'; + +import { AlpineLimaISOVersion, Dependency, DownloadContext } from 'scripts/lib/dependencies'; +import { simpleSpawn } from 'scripts/simple_process'; + +type GoDependencyOptions = { + /** + * The output file name, relative to the platform-specific resources directory. + * If this does not contain any directory separators ('/'), it is assumed to + * be a directory name (defaults to `bin`) and the leaf name of the source + * path is appended as the executable name. + */ + outputPath: string; + /** + * Additional environment for the go compiler; e.g. for GOARCH overrides. + */ + env?: NodeJS.ProcessEnv; +}; + +/** + * GoDependency represents a golang binary that is built from the local source + * code. + */ +export class GoDependency implements Dependency { + /** + * Construct a new GoDependency. + * @param sourcePath The path to be compiled, relative to .../src/go + * @param options Additional configuration option; if a string is given, this + * is the outputPath option, defaulting to `bin`. + */ + constructor(sourcePath: string, options: string | GoDependencyOptions = 'bin') { + this.sourcePath = sourcePath; + this.options = typeof options === 'string' ? { outputPath: options } : options; + } + + get name(): string { + if (this.options.outputPath.includes('/')) { + return path.basename(this.options.outputPath); + } + + return path.basename(this.sourcePath); + } + + sourcePath: string; + options: GoDependencyOptions; + + async download(context: DownloadContext): Promise { + // Rather than actually downloading anything, this builds the source code. + const sourceDir = path.join(process.cwd(), 'src', 'go', this.sourcePath); + const outFile = this.outFile(context); + + console.log(`Building go utility \x1B[1;33;40m${ this.name }\x1B[0m from ${ sourceDir } to ${ outFile }...`); + await simpleSpawn('go', ['build', '-ldflags', '-s -w', '-o', outFile, '.'], { + cwd: sourceDir, + env: this.environment(context), + }); + } + + environment(context: DownloadContext): NodeJS.ProcessEnv { + return { + ...process.env, + GOOS: context.goPlatform, + GOARCH: context.isM1 ? 'arm64' : 'amd64', + ...this.options.env ?? {}, + }; + } + + outFile(context: DownloadContext): string { + const suffix = context.platform === 'win32' ? '.exe' : ''; + let outputPath = `${ this.options.outputPath }${ suffix }`; + + if (!this.options.outputPath.includes('/')) { + outputPath = `${ this.options.outputPath }/${ this.name }${ suffix }`; + } + + return path.join(context.resourcesDir, context.platform, outputPath); + } + + getAvailableVersions(includePrerelease?: boolean | undefined): Promise { + throw new Error('Go dependencies do not have available versions.'); + } + + rcompareVersions(version1: string | AlpineLimaISOVersion, version2: string): 0 | 1 | -1 { + throw new Error('Go dependencies do not have available versions.'); + } +} + +export class RDCtl extends GoDependency { + constructor() { + super('rdctl'); + } + + dependencies(context: DownloadContext): string[] { + if (context.dependencyPlatform === 'wsl') { + // For the WSL copy depend on the Windows one to generate code + return ['rdctl:win32']; + } + + return []; + } + + override async download(context: DownloadContext): Promise { + // For WSL, don't re-generate the code; the win32 copy did it. + if (context.dependencyPlatform !== 'wsl') { + await simpleSpawn('node', ['scripts/ts-wrapper.js', + 'scripts/generateCliCode.ts', + 'pkg/rancher-desktop/assets/specs/command-api.yaml', + 'src/go/rdctl/pkg/options/generated/options.go']); + } + await super.download(context); + } +} + +export class WSLHelper extends GoDependency { + constructor() { + super('wsl-helper', { outputPath: 'internal', env: { CGO_ENABLED: '0' } }); + } + + dependencies(context: DownloadContext): string[] { + return ['mobyOpenAPISpec:win32']; + } +} + +export class NerdctlStub extends GoDependency { + constructor() { + super('nerdctl-stub'); + } + + override outFile(context: DownloadContext) { + // nerdctl-stub is the actual nerdctl binary to be run on linux; + // there is also a `nerdctl` wrapper in the same directory to make it + // easier to handle permissions for Linux-in-WSL. + const leafName = context.platform === 'win32' ? 'nerdctl.exe' : 'nerdctl-stub'; + + return path.join(context.resourcesDir, context.platform, 'bin', leafName); + } +} diff --git a/scripts/dependencies/tar-archives.ts b/scripts/dependencies/tar-archives.ts new file mode 100644 index 00000000000..85c30199d74 --- /dev/null +++ b/scripts/dependencies/tar-archives.ts @@ -0,0 +1,182 @@ +import crypto from 'crypto'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import stream from 'stream'; + +import tar from 'tar-stream'; + +import { AlpineLimaISOVersion, Dependency, DownloadContext } from 'scripts/lib/dependencies'; + +export class ExtensionProxyImage implements Dependency { + name = 'rdx-proxy.tar'; + dependencies(context: DownloadContext) { + return [`extension-proxy:linux`]; + } + + async download(context: DownloadContext): Promise { + // Build the extension proxy image. + const workDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'rd-build-rdx-pf-')); + + try { + const executablePath = path.join(context.resourcesDir, 'linux', 'staging', 'extension-proxy'); + const layerPath = path.join(workDir, 'layer.tar'); + const imagePath = path.join(context.resourcesDir, 'rdx-proxy.tar'); + + console.log('Building RDX proxying image...'); + + // Build the layer tarball + // tar streams don't implement piping to multiple writers, and stream.Duplex + // can't deal with it either; so we need to fully write out the file, then + // calculate the hash as a separate step. + const layer = tar.pack(); + const layerOutput = layer.pipe(fs.createWriteStream(layerPath)); + const executableStats = await fs.promises.stat(executablePath); + + await stream.promises.finished( + fs.createReadStream(executablePath) + .pipe(layer.entry({ + name: path.basename(executablePath), + mode: 0o755, + type: 'file', + mtime: new Date(0), + size: executableStats.size, + }))); + layer.finalize(); + await stream.promises.finished(layerOutput); + + // calculate the hash + const layerReader = fs.createReadStream(layerPath); + const layerHasher = layerReader.pipe(crypto.createHash('sha256')); + + await stream.promises.finished(layerReader); + + // Build the image tarball + const layerHash = layerHasher.digest().toString('hex'); + const image = tar.pack(); + const imageStream = fs.createWriteStream(imagePath); + const imageWritten = stream.promises.finished(imageStream); + + image.pipe(imageStream); + const addEntry = (name: string, input: Buffer | stream.Readable, size?: number) => { + if (Buffer.isBuffer(input)) { + size = input.length; + input = stream.Readable.from(input); + } + + return stream.promises.finished((input as stream.Readable).pipe(image.entry({ + name, + size, + type: 'file', + mtime: new Date(0), + }))); + }; + + image.entry({ name: layerHash, type: 'directory' }); + await addEntry(`${ layerHash }/VERSION`, Buffer.from('1.0')); + await addEntry(`${ layerHash }/layer.tar`, fs.createReadStream(layerPath), layerOutput.bytesWritten); + await addEntry(`${ layerHash }/json`, Buffer.from(JSON.stringify({ + id: layerHash, + config: { + ExposedPorts: { '80/tcp': {} }, + WorkingDir: '/', + Entrypoint: [`/${ path.basename(executablePath) }`], + }, + }))); + await addEntry(`${ layerHash }.json`, Buffer.from(JSON.stringify({ + architecture: context.isM1 ? 'arm64' : 'amd64', + config: { + ExposedPorts: { '80/tcp': {} }, + Entrypoint: [`/${ path.basename(executablePath) }`], + WorkingDir: '/', + }, + history: [], + os: 'linux', + rootfs: { + type: 'layers', + diff_ids: [`sha256:${ layerHash }`], + }, + }))); + await addEntry('manifest.json', Buffer.from(JSON.stringify([ + { + Config: `${ layerHash }.json`, + RepoTags: ['ghcr.io/rancher-sandbox/rancher-desktop/rdx-proxy:latest'], + Layers: [`${ layerHash }/layer.tar`], + }, + ]))); + image.finalize(); + await imageWritten; + console.log('Built RDX port proxy image'); + } finally { + await fs.promises.rm(workDir, { recursive: true }); + } + } + + getAvailableVersions(includePrerelease?: boolean | undefined): Promise { + throw new Error('extension-proxy does not have versions.'); + } + + rcompareVersions(version1: string | AlpineLimaISOVersion, version2: string | AlpineLimaISOVersion): 0 | 1 | -1 { + throw new Error('extension-proxy does not have versions.'); + } +} + +export class WSLDistroImage implements Dependency { + name = 'WSLDistroImage'; + dependencies(context: DownloadContext): string[] { + return ['WSLDistro:win32', 'guestagent:linux']; + } + + async download(context: DownloadContext): Promise { + const tarName = `distro-${ context.versions.WSLDistro }.tar`; + const pristinePath = path.join(context.resourcesDir, context.platform, 'staging', tarName); + const pristineFile = fs.createReadStream(pristinePath); + const extractor = tar.extract(); + const destPath = path.join(context.resourcesDir, context.platform, tarName); + const destFile = fs.createWriteStream(destPath); + const packer = tar.pack(); + + console.log('Building WSLDistro image...'); + + // Copy the pristine tar file to the destination. + packer.pipe(destFile); + extractor.on('entry', (header, stream, callback) => { + stream.pipe(packer.entry(header, callback)); + }); + await stream.promises.finished(pristineFile.pipe(extractor)); + + async function addFile(fromPath: string, name: string, options: Omit = {}) { + const { size } = await fs.promises.stat(fromPath); + const inputFile = fs.createReadStream(fromPath); + + console.log(`WSL Distro: Adding ${ fromPath } to ${ name }...`); + await stream.promises.finished(inputFile.pipe(packer.entry({ + name, + size, + mode: 0o755, + type: 'file', + mtime: new Date(0), + ...options, + }))); + } + + // Add extra files. + await addFile(path.join(context.resourcesDir, 'linux', 'staging', 'guestagent'), + 'usr/local/bin/rancher-desktop-guestagent'); + await addFile(path.join(context.resourcesDir, 'linux', 'staging', 'trivy'), + 'usr/local/bin/trivy'); + + // Finish the archive. + packer.finalize(); + await stream.promises.finished(packer); + console.log('Built WSLDistro image.'); + } + + getAvailableVersions(includePrerelease?: boolean | undefined): Promise { + throw new Error('WSLDistroImage does not have versions.'); + } + + rcompareVersions(version1: string | AlpineLimaISOVersion, version2: string | AlpineLimaISOVersion): 0 | 1 | -1 { + throw new Error('WSLDistroImage does not have versions.'); + } +} diff --git a/scripts/dependencies/tools.ts b/scripts/dependencies/tools.ts index 22b346c485f..4c1ab4bbc64 100644 --- a/scripts/dependencies/tools.ts +++ b/scripts/dependencies/tools.ts @@ -298,7 +298,8 @@ export class Trivy implements Dependency, GitHubDependency { const trivyURL = `${ trivyURLBase }/download/${ versionWithV }/${ trivyBasename }.tar.gz`; const checksumURL = `${ trivyURLBase }/download/${ versionWithV }/trivy_${ context.versions.trivy }_checksums.txt`; const trivySHA = await findChecksum(checksumURL, `${ trivyBasename }.tar.gz`); - const trivyPath = path.join(context.resourcesDir, 'linux', 'internal', 'trivy'); + const trivyDir = context.dependencyPlatform === 'wsl' ? 'staging' : 'internal'; + const trivyPath = path.join(context.resourcesDir, 'linux', trivyDir, 'trivy'); // trivy.tgz files are top-level tarballs - not wrapped in a labelled directory :( await downloadTarGZ(trivyURL, trivyPath, { expectedChecksum: trivySHA }); diff --git a/scripts/dependencies/wsl.ts b/scripts/dependencies/wsl.ts index cca83da5183..8f16ecda0d3 100644 --- a/scripts/dependencies/wsl.ts +++ b/scripts/dependencies/wsl.ts @@ -164,7 +164,7 @@ export class WSLDistro implements Dependency, GitHubDependency { const baseUrl = `https://github.com/${ this.githubOwner }/${ this.githubRepo }/releases/download`; const tarName = `distro-${ context.versions.WSLDistro }.tar`; const url = `${ baseUrl }/v${ context.versions.WSLDistro }/${ tarName }`; - const destPath = path.join(context.resourcesDir, context.platform, tarName); + const destPath = path.join(context.resourcesDir, context.platform, 'staging', tarName); await download(url, destPath, { access: fs.constants.W_OK }); } diff --git a/scripts/lib/build-utils.ts b/scripts/lib/build-utils.ts index ffa6f325e59..c5ef29f82c7 100644 --- a/scripts/lib/build-utils.ts +++ b/scripts/lib/build-utils.ts @@ -3,16 +3,12 @@ */ import childProcess from 'child_process'; -import crypto from 'crypto'; import fs from 'fs'; -import os from 'os'; import path from 'path'; -import stream from 'stream'; import util from 'util'; import spawn from 'cross-spawn'; import _ from 'lodash'; -import tar from 'tar-stream'; import webpack from 'webpack'; import babelConfig from 'babel.config'; @@ -232,223 +228,10 @@ export default { }); }, - /** Mapping from the platform name to the Go OS value. */ - mapPlatformToGoOS(platform: NodeJS.Platform) { - switch (platform) { - case 'darwin': - return 'darwin'; - case 'linux': - return 'linux'; - case 'win32': - return 'windows'; - default: - throw new Error(`Invalid platform "${ platform }"`); - } - }, - - mapArchToGoArch(arch: string) { - const result = ({ - x64: 'amd64', - arm64: 'arm64', - } as const)[arch]; - - if (!result) { - throw new Error(`Architecture ${ arch } is not supported.`); - } - - return result; - }, - get arch(): NodeJS.Architecture { return process.env.M1 ? 'arm64' : process.arch; }, - /** - * Build the WSL helper application for Windows. - */ - async buildWSLHelper(): Promise { - /** - * Build for a single platform - * @param platform The platform to build for. - */ - const buildPlatform = async(platform: 'linux' | 'win32') => { - const exeRoot = 'wsl-helper'; - const exeName = `${ exeRoot }${ platform === 'win32' ? '.exe' : '' }`; - const outFile = path.join(this.rootDir, 'resources', platform, 'internal', exeName); - - await this.spawn('go', 'build', '-ldflags', '-s -w', '-o', outFile, '.', { - cwd: path.join(this.rootDir, 'src', 'go', 'wsl-helper'), - env: { - ...process.env, - GOOS: this.mapPlatformToGoOS(platform), - CGO_ENABLED: '0', - }, - }); - }; - - await this.wait( - buildPlatform.bind(this, 'linux'), - buildPlatform.bind(this, 'win32'), - ); - }, - - /** - * Build the nerdctl stub. - */ - async buildNerdctlStub(os: 'windows' | 'linux'): Promise { - let platDir, parentDir, outFile; - - if (os === 'windows') { - platDir = 'win32'; - parentDir = path.join(this.rootDir, 'resources', platDir, 'bin'); - outFile = path.join(parentDir, 'nerdctl.exe'); - } else { - platDir = 'linux'; - parentDir = path.join(this.rootDir, 'resources', platDir, 'bin'); - // nerdctl-stub is the actual nerdctl binary to be run on linux; - // there is also a `nerdctl` wrapper in the same directory to make it - // easier to handle permissions for Linux-in-WSL. - outFile = path.join(parentDir, 'nerdctl-stub'); - } - // The linux build produces both nerdctl-stub and nerdctl - await this.spawn('go', 'build', '-ldflags', '-s -w', '-o', outFile, '.', { - cwd: path.join(this.rootDir, 'src', 'go', 'nerdctl-stub'), - env: { - ...process.env, - GOOS: os, - }, - }); - }, - - async buildExtensionProxyImage(): Promise { - const workDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'rd-build-rdx-pf-')); - - try { - const executablePath = path.join(workDir, 'extension-proxy'); - const layerPath = path.join(workDir, 'layer.tar'); - const imagePath = path.join(this.rootDir, 'resources', 'rdx-proxy.tar'); - - console.log('Building RDX proxying image...'); - - // Build the golang executable - await this.spawn('go', 'build', '-ldflags', '-s -w', '-o', executablePath, '.', { - cwd: path.join(this.rootDir, 'src', 'go', 'extension-proxy'), - env: { - ...process.env, - CGO_ENABLED: '0', - GOOS: 'linux', - GOARCH: this.mapArchToGoArch(this.arch), - }, - }); - - // Build the layer tarball - // tar streams don't implement piping to multiple writers, and stream.Duplex - // can't deal with it either; so we need to fully write out the file, then - // calculate the hash as a separate step. - const layer = tar.pack(); - const layerOutput = layer.pipe(fs.createWriteStream(layerPath)); - const executableStats = await fs.promises.stat(executablePath); - - await stream.promises.finished( - fs.createReadStream(executablePath) - .pipe(layer.entry({ - name: path.basename(executablePath), - mode: 0o755, - type: 'file', - mtime: new Date(0), - size: executableStats.size, - }))); - layer.finalize(); - await stream.promises.finished(layerOutput); - - // calculate the hash - const layerReader = fs.createReadStream(layerPath); - const layerHasher = layerReader.pipe(crypto.createHash('sha256')); - - await stream.promises.finished(layerReader); - - // Build the image tarball - const layerHash = layerHasher.digest().toString('hex'); - const image = tar.pack(); - const imageWritten = - stream.promises.finished( - image - .pipe(fs.createWriteStream(imagePath))); - const addEntry = (name: string, input: Buffer | stream.Readable, size?: number) => { - if (Buffer.isBuffer(input)) { - size = input.length; - input = stream.Readable.from(input); - } - - return stream.promises.finished((input as stream.Readable).pipe(image.entry({ - name, - size, - type: 'file', - mtime: new Date(0), - }))); - }; - - image.entry({ name: layerHash, type: 'directory' }); - await addEntry(`${ layerHash }/VERSION`, Buffer.from('1.0')); - await addEntry(`${ layerHash }/layer.tar`, fs.createReadStream(layerPath), layerOutput.bytesWritten); - await addEntry(`${ layerHash }/json`, Buffer.from(JSON.stringify({ - id: layerHash, - config: { - ExposedPorts: { '80/tcp': {} }, - WorkingDir: '/', - Entrypoint: [`/${ path.basename(executablePath) }`], - }, - }))); - await addEntry(`${ layerHash }.json`, Buffer.from(JSON.stringify({ - architecture: this.mapArchToGoArch(this.arch), - config: { - ExposedPorts: { '80/tcp': {} }, - Entrypoint: [`/${ path.basename(executablePath) }`], - WorkingDir: '/', - }, - history: [], - os: 'linux', - rootfs: { - type: 'layers', - diff_ids: [`sha256:${ layerHash }`], - }, - }))); - await addEntry('manifest.json', Buffer.from(JSON.stringify([ - { - Config: `${ layerHash }.json`, - RepoTags: ['ghcr.io/rancher-sandbox/rancher-desktop/rdx-proxy:latest'], - Layers: [`${ layerHash }/layer.tar`], - }, - ]))); - image.finalize(); - await imageWritten; - console.log('Built RDX port proxy image'); - } finally { - await fs.promises.rm(workDir, { recursive: true }); - } - }, - - /** - * Build a golang-based utility for the specified platform. - * @param name basename of the executable to build - * @param platform 'linux', 'windows', or 'darwin' - * @param childDir final folder destination either 'internal' or 'bin' - */ - async buildUtility(name: string, platform: NodeJS.Platform, childDir: string): Promise { - const target = platform === 'win32' ? `${ name }.exe` : name; - const parentDir = path.join(this.rootDir, 'resources', platform, childDir); - const outFile = path.join(parentDir, target); - - await this.spawn('go', 'build', '-ldflags', '-s -w', '-o', outFile, '.', { - cwd: path.join(this.rootDir, 'src', 'go', name), - env: { - ...process.env, - GOOS: this.mapPlatformToGoOS(platform), - GOARCH: this.mapArchToGoArch(this.arch), - }, - }); - }, - /** * Build the preload script. */ @@ -463,27 +246,4 @@ export default { return this.wait(() => this.buildJavaScript(this.webpackConfig)); }, - /** - * Build the things we build with go - */ - async buildGoUtilities() { - const tasks = []; - - if (os.platform().startsWith('win')) { - tasks.push(() => this.buildWSLHelper()); - tasks.push(() => this.buildNerdctlStub('windows')); - tasks.push(() => this.buildNerdctlStub('linux')); - tasks.push(() => this.buildUtility('vtunnel', 'linux', 'internal')); - tasks.push(() => this.buildUtility('vtunnel', 'win32', 'internal')); - tasks.push(() => this.buildUtility('rdctl', 'linux', 'bin')); - tasks.push(() => this.buildUtility('privileged-service', 'win32', 'internal')); - tasks.push(() => this.buildUtility('guestagent', 'linux', 'internal')); - } - tasks.push(() => this.buildUtility('rdctl', os.platform(), 'bin')); - tasks.push(() => this.buildUtility('docker-credential-none', os.platform(), 'bin')); - tasks.push(() => this.buildExtensionProxyImage()); - - return await this.wait(...tasks); - }, - }; diff --git a/scripts/lib/dependencies.ts b/scripts/lib/dependencies.ts index cfab1a05544..37e38166acd 100644 --- a/scripts/lib/dependencies.ts +++ b/scripts/lib/dependencies.ts @@ -72,6 +72,11 @@ export async function writeDependencyVersions(path: string, depVersions: Depende export interface Dependency { name: string, + /** + * Other dependencies this one requires. + * This must be in the form :, e.g. "kuberlr:linux" + */ + dependencies?: (context: DownloadContext) => string[], download(context: DownloadContext): Promise // Returns the available versions of the Dependency. // Includes prerelease versions if includePrerelease is true. diff --git a/scripts/lib/download.ts b/scripts/lib/download.ts index 523f370a985..d0759c46c09 100644 --- a/scripts/lib/download.ts +++ b/scripts/lib/download.ts @@ -10,6 +10,8 @@ import path from 'path'; import fetch from 'node-fetch'; +import { simpleSpawn } from 'scripts/simple_process'; + type ChecksumAlgorithm = 'sha1' | 'sha256' | 'sha512'; export type DownloadOptions = { @@ -193,9 +195,10 @@ export async function downloadTarGZ(url: string, destPath: string, options: Arch } args[0] = path.join(systemRoot, 'system32', 'tar.exe'); } - execFileSync(args[0], args.slice(1), { stdio: 'inherit' }); - fs.copyFileSync(path.join(workDir, fileToExtract), destPath); - fs.chmodSync(destPath, mode); + await simpleSpawn(args[0], args.slice(1)); + await fs.promises.mkdir(path.dirname(destPath), { recursive: true }); + await fs.promises.copyFile(path.join(workDir, fileToExtract), destPath); + await fs.promises.chmod(destPath, mode); } finally { fs.rmSync(workDir, { recursive: true, maxRetries: 10 }); } diff --git a/scripts/package.ts b/scripts/package.ts index 5cc3194ed9e..41f491deab3 100644 --- a/scripts/package.ts +++ b/scripts/package.ts @@ -118,6 +118,10 @@ class Builder { linux: 'linux', } as const)[process.platform as string]; + if (!electronPlatform) { + throw new Error(`Packaging for ${ process.platform } is not supported`); + } + switch (electronPlatform) { case 'linux': await this.createLinuxResources(finalBuildVersion); @@ -127,6 +131,23 @@ class Builder { break; } + // When there are files (e.g., extraFiles or extraResources) specified at both + // the top-level and platform-specific levels, we need to combine them + // and place the combined list at the top level. This approach enables us to have + // platform-specific exclusions, since the two lists are initially processed + // separately and then merged together afterward. + for (const key of ['files', 'extraFiles', 'extraResources'] as const) { + const section = config[electronPlatform]; + const items = config[key]; + const overrideItems = section?.[key]; + + if (!section || !Array.isArray(items) || !Array.isArray(overrideItems)) { + continue; + } + config[key] = items.concat(overrideItems); + delete section[key]; + } + _.set(config, 'extraMetadata.version', finalBuildVersion); await fs.promises.writeFile(configPath, yaml.stringify(config), 'utf-8'); diff --git a/scripts/postinstall.ts b/scripts/postinstall.ts index d408f57b99a..380b5500bfa 100644 --- a/scripts/postinstall.ts +++ b/scripts/postinstall.ts @@ -2,10 +2,10 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; -import buildUtils from './lib/build-utils'; - +import * as goUtils from 'scripts/dependencies/go-source'; import { Lima, LimaAndQemu, AlpineLimaISO } from 'scripts/dependencies/lima'; import { MobyOpenAPISpec } from 'scripts/dependencies/moby-openapi'; +import { ExtensionProxyImage, WSLDistroImage } from 'scripts/dependencies/tar-archives'; import * as tools from 'scripts/dependencies/tools'; import { Wix } from 'scripts/dependencies/wix'; import { @@ -16,6 +16,11 @@ import { } from 'scripts/lib/dependencies'; import { simpleSpawn } from 'scripts/simple_process'; +type DependencyWithContext = { + dependency: Dependency; + context: DownloadContext; +}; + // Dependencies that should be installed into places that users touch // (so users' WSL distros and hosts as of the time of writing). const userTouchedDependencies = [ @@ -27,6 +32,8 @@ const userTouchedDependencies = [ new tools.DockerProvidedCredHelpers(), new tools.ECRCredHelper(), new tools.SpinCLI(), + new goUtils.RDCtl(), + new goUtils.GoDependency('docker-credential-none'), ]; // Dependencies that are specific to unix hosts. @@ -39,15 +46,25 @@ const unixDependencies = [ // Dependencies that are specific to windows hosts. const windowsDependencies = [ new WSLDistro(), + new WSLDistroImage(), new HostResolverHost(), new Wix(), new HostSwitch(), + new goUtils.GoDependency('vtunnel', 'internal'), + new goUtils.GoDependency('privileged-service', 'internal'), + new goUtils.WSLHelper(), + new goUtils.NerdctlStub(), ]; // Dependencies that are specific to WSL. const wslDependencies = [ new HostResolverPeer(), new Moproxy(), + new goUtils.GoDependency('vtunnel', 'internal'), + new goUtils.RDCtl(), + new goUtils.GoDependency('guestagent', 'staging'), + new goUtils.WSLHelper(), + new goUtils.NerdctlStub(), ]; // Dependencies that are specific to WSL and Lima VMs. @@ -56,6 +73,8 @@ const vmDependencies = [ new tools.WasmShims(), new tools.CertManager(), new tools.SpinOperator(), + new goUtils.GoDependency('extension-proxy', { outputPath: 'staging', env: { CGO_ENABLED: '0' } }), + new ExtensionProxyImage(), ]; // Dependencies that are specific to hosts. @@ -65,38 +84,106 @@ const hostDependencies = [ new MobyOpenAPISpec(), ]; -function downloadDependencies(context: DownloadContext, dependencies: Dependency[]): Promise { - return Promise.all( - dependencies.map(dependency => dependency.download(context)), - ); +async function downloadDependencies(items: DependencyWithContext[]): Promise { + function specialize(item: DependencyWithContext) { + return `${ item.dependency.name }:${ item.context.platform }`; + } + // Dependencies might depend on other dependencies. Note that we may have + // multiple dependencies of the same name, but different platforms; therefore, + // all dependencies are keyed by :. + const dependenciesByName = Object.fromEntries(items.map(item => [specialize(item), item])); + const forwardDependencies = Object.fromEntries(items.map(item => [specialize(item), [] as string[]] as const)); + const reverseDependencies = Object.fromEntries(items.map(item => [specialize(item), [] as string[]] as const)); + const all = new Set(Object.keys(dependenciesByName)); + const running = new Set(); + const done = new Set(); + const promises: Promise[] = []; + + for (const item of items) { + const dependencies = item.dependency.dependencies?.(item.context) ?? []; + + forwardDependencies[specialize(item)].push(...dependencies); + for (const dependency of dependencies) { + if (dependency in reverseDependencies) { + reverseDependencies[dependency].push(specialize(item)); + } else { + throw new Error(`Dependency ${ item.dependency.name } depends on unknown dependency ${ dependency }`); + } + } + } + async function process(name: string) { + running.add(name); + const item = dependenciesByName[name]; + + await item.dependency.download(item.context); + done.add(name); + for (const dependent of reverseDependencies[name]) { + if (!running.has(dependent)) { + if (forwardDependencies[dependent].every(d => done.has(d))) { + promises.push(process(dependent)); + } + } + } + } + + for (const item of items.filter(d => (d.dependency.dependencies?.(d.context) ?? []).length === 0)) { + promises.push(process(specialize(item))); + } + + while (running.size > done.size) { + await Promise.all(promises); + } + + if (all.size > done.size) { + const remaining = Array.from(all).filter(d => !done.has(d)).sort(); + const message = [`${ remaining.length } dependencies are stuck:`]; + + for (const key of remaining) { + const deps = forwardDependencies[key].filter(d => !done.has(d)); + + message.push(` ${ key } depends on ${ deps }`); + } + throw new Error(message.join('\n')); + } + + return await Promise.all(promises); } async function runScripts(): Promise { // load desired versions of dependencies const depVersions = await readDependencyVersions(path.join('pkg', 'rancher-desktop', 'assets', 'dependencies.yaml')); const platform = os.platform(); + const dependencies: DependencyWithContext[] = []; if (platform === 'linux' || platform === 'darwin') { // download things that go on unix host const hostDownloadContext = buildDownloadContextFor(platform, depVersions); - await downloadDependencies(hostDownloadContext, [...userTouchedDependencies, ...unixDependencies, ...hostDependencies]); + for (const dependency of [...userTouchedDependencies, ...unixDependencies, ...hostDependencies]) { + dependencies.push({ dependency, context: hostDownloadContext }); + } // download things that go inside Lima VM const vmDownloadContext = buildDownloadContextFor('linux', depVersions); - await downloadDependencies(vmDownloadContext, vmDependencies); + dependencies.push(...vmDependencies.map(dependency => ({ dependency, context: vmDownloadContext }))); } else if (platform === 'win32') { // download things for windows const hostDownloadContext = buildDownloadContextFor('win32', depVersions); - await downloadDependencies(hostDownloadContext, [...userTouchedDependencies, ...windowsDependencies, ...hostDependencies]); + for (const dependency of [...userTouchedDependencies, ...windowsDependencies, ...hostDependencies]) { + dependencies.push({ dependency, context: hostDownloadContext }); + } // download things that go inside WSL distro const vmDownloadContext = buildDownloadContextFor('wsl', depVersions); - await downloadDependencies(vmDownloadContext, [...userTouchedDependencies, ...wslDependencies, ...vmDependencies]); + for (const dependency of [...userTouchedDependencies, ...wslDependencies, ...vmDependencies]) { + dependencies.push({ dependency, context: vmDownloadContext }); + } } + + await downloadDependencies(dependencies); } function buildDownloadContextFor(rawPlatform: DependencyPlatform, depVersions: DependencyVersions): DownloadContext { @@ -129,11 +216,6 @@ const keepScriptAlive = setTimeout(() => { }, 24 * 3600 * 1000); await runScripts(); await simpleSpawn('node', ['node_modules/electron-builder/out/cli/cli.js', 'install-app-deps']); - await simpleSpawn('node', ['scripts/ts-wrapper.js', - 'scripts/generateCliCode.ts', - 'pkg/rancher-desktop/assets/specs/command-api.yaml', - 'src/go/rdctl/pkg/options/generated/options.go']); - await buildUtils.buildGoUtilities(); exitCode = 0; } catch (e: any) { console.error('POSTINSTALL ERROR: ', e);