Merge pull request #7145 from mook-as/win32/embed-agent

Scripts: Add guestagent and trivy into wsl-distro tarball instead of installing at runtime

packaging/electron-builder.yml

@@ -4,11 +4,15 @@ productName: Rancher Desktop
 icon: ./resources/icons/logo-square-512.png
 appId: io.rancherdesktop.app
 asar: true
+electronLanguages: [ en-US ]
 extraResources:
 - resources/
 - '!resources/darwin/lima*.tgz'
 - '!resources/linux/lima*.tgz'
+- '!resources/linux/staging/'
+- '!resources/win32/staging/'
 - '!resources/host/'
+- '!resources/**/*.js.map'
 files:
 - dist/app/**/*
 - '!**/node_modules/*/prebuilds/!(${platform}*)/*.node'

pkg/rancher-desktop/backend/wsl.ts

@@ -903,21 +903,6 @@ export default class WSLBackend extends events.EventEmitter implements VMBackend
     }
   }
 
-  /**
-   * On Windows Trivy is run via WSL as there's no native port.
-   * Ensure that all relevant files are in the wsl mount, not the windows one.
-   */
-  protected async installTrivy() {
-    // download-resources.sh installed trivy into the resources area
-    // This function moves it into /usr/local/bin/ so when trivy is
-    // invoked to run through wsl, it runs faster.
-
-    const trivyExecPath = path.join(paths.resources, 'linux', 'internal', 'trivy');
-
-    await this.execCommand('mkdir', '-p', '/var/local/bin');
-    await this.wslInstall(trivyExecPath, '/usr/local/bin');
-  }
-
   protected async installGuestAgent(kubeVersion: semver.SemVer | undefined, cfg: BackendSettings | undefined) {
     let guestAgentConfig: Record<string, any>;
     const enableKubernetes = !!kubeVersion;
@@ -947,10 +932,7 @@ export default class WSLBackend extends events.EventEmitter implements VMBackend
       };
     }
 
-    const guestAgentPath = path.join(paths.resources, 'linux', 'internal', 'guestagent');
-
     await Promise.all([
-      this.wslInstall(guestAgentPath, '/usr/local/bin/', 'rancher-desktop-guestagent'),
       this.writeFile('/etc/init.d/rancher-desktop-guestagent', SERVICE_GUEST_AGENT_INIT, 0o755),
       this.writeConf('rancher-desktop-guestagent', guestAgentConfig),
     ]);
@@ -1525,7 +1507,6 @@ export default class WSLBackend extends events.EventEmitter implements VMBackend
                 this.runWslProxy().catch(console.error);
               }
             }),
-            this.progressTracker.action('Installing image scanner', 100, this.installTrivy()),
             this.progressTracker.action('Installing CA certificates', 100, this.installCACerts()),
             this.progressTracker.action('Installing helpers', 50, this.installWSLHelpers()),
           ]));

scripts/dependencies/go-source.ts

@@ -0,0 +1,137 @@
+import path from 'path';
+
+import { AlpineLimaISOVersion, Dependency, DownloadContext } from 'scripts/lib/dependencies';
+import { simpleSpawn } from 'scripts/simple_process';
+
+type GoDependencyOptions = {
+  /**
+   * The output file name, relative to the platform-specific resources directory.
+   * If this does not contain any directory separators ('/'), it is assumed to
+   * be a directory name (defaults to `bin`) and the leaf name of the source
+   * path is appended as the executable name.
+   */
+  outputPath: string;
+  /**
+   * Additional environment for the go compiler; e.g. for GOARCH overrides.
+   */
+  env?: NodeJS.ProcessEnv;
+};
+
+/**
+ * GoDependency represents a golang binary that is built from the local source
+ * code.
+ */
+export class GoDependency implements Dependency {
+  /**
+   * Construct a new GoDependency.
+   * @param sourcePath The path to be compiled, relative to .../src/go
+   * @param options Additional configuration option; if a string is given, this
+   * is the outputPath option, defaulting to `bin`.
+   */
+  constructor(sourcePath: string, options: string | GoDependencyOptions = 'bin') {
+    this.sourcePath = sourcePath;
+    this.options = typeof options === 'string' ? { outputPath: options } : options;
+  }
+
+  get name(): string {
+    if (this.options.outputPath.includes('/')) {
+      return path.basename(this.options.outputPath);
+    }
+
+    return path.basename(this.sourcePath);
+  }
+
+  sourcePath: string;
+  options: GoDependencyOptions;
+
+  async download(context: DownloadContext): Promise<void> {
+    // Rather than actually downloading anything, this builds the source code.
+    const sourceDir = path.join(process.cwd(), 'src', 'go', this.sourcePath);
+    const outFile = this.outFile(context);
+
+    console.log(`Building go utility \x1B[1;33;40m${ this.name }\x1B[0m from ${ sourceDir } to ${ outFile }...`);
+    await simpleSpawn('go', ['build', '-ldflags', '-s -w', '-o', outFile, '.'], {
+      cwd: sourceDir,
+      env: this.environment(context),
+    });
+  }
+
+  environment(context: DownloadContext): NodeJS.ProcessEnv {
+    return {
+      ...process.env,
+      GOOS:   context.goPlatform,
+      GOARCH: context.isM1 ? 'arm64' : 'amd64',
+      ...this.options.env ?? {},
+    };
+  }
+
+  outFile(context: DownloadContext): string {
+    const suffix = context.platform === 'win32' ? '.exe' : '';
+    let outputPath = `${ this.options.outputPath }${ suffix }`;
+
+    if (!this.options.outputPath.includes('/')) {
+      outputPath = `${ this.options.outputPath }/${ this.name }${ suffix }`;
+    }
+
+    return path.join(context.resourcesDir, context.platform, outputPath);
+  }
+
+  getAvailableVersions(includePrerelease?: boolean | undefined): Promise<string[]> {
+    throw new Error('Go dependencies do not have available versions.');
+  }
+
+  rcompareVersions(version1: string | AlpineLimaISOVersion, version2: string): 0 | 1 | -1 {
+    throw new Error('Go dependencies do not have available versions.');
+  }
+}
+
+export class RDCtl extends GoDependency {
+  constructor() {
+    super('rdctl');
+  }
+
+  dependencies(context: DownloadContext): string[] {
+    if (context.dependencyPlatform === 'wsl') {
+      // For the WSL copy depend on the Windows one to generate code
+      return ['rdctl:win32'];
+    }
+
+    return [];
+  }
+
+  override async download(context: DownloadContext): Promise<void> {
+    // For WSL, don't re-generate the code; the win32 copy did it.
+    if (context.dependencyPlatform !== 'wsl') {
+      await simpleSpawn('node', ['scripts/ts-wrapper.js',
+        'scripts/generateCliCode.ts',
+        'pkg/rancher-desktop/assets/specs/command-api.yaml',
+        'src/go/rdctl/pkg/options/generated/options.go']);
+    }
+    await super.download(context);
+  }
+}
+
+export class WSLHelper extends GoDependency {
+  constructor() {
+    super('wsl-helper', { outputPath: 'internal', env: { CGO_ENABLED: '0' } });
+  }
+
+  dependencies(context: DownloadContext): string[] {
+    return ['mobyOpenAPISpec:win32'];
+  }
+}
+
+export class NerdctlStub extends GoDependency {
+  constructor() {
+    super('nerdctl-stub');
+  }
+
+  override outFile(context: DownloadContext) {
+    // nerdctl-stub is the actual nerdctl binary to be run on linux;
+    // there is also a `nerdctl` wrapper in the same directory to make it
+    // easier to handle permissions for Linux-in-WSL.
+    const leafName = context.platform === 'win32' ? 'nerdctl.exe' : 'nerdctl-stub';
+
+    return path.join(context.resourcesDir, context.platform, 'bin', leafName);
+  }
+}

scripts/dependencies/tar-archives.ts

@@ -0,0 +1,182 @@
+import crypto from 'crypto';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import stream from 'stream';
+
+import tar from 'tar-stream';
+
+import { AlpineLimaISOVersion, Dependency, DownloadContext } from 'scripts/lib/dependencies';
+
+export class ExtensionProxyImage implements Dependency {
+  name = 'rdx-proxy.tar';
+  dependencies(context: DownloadContext) {
+    return [`extension-proxy:linux`];
+  }
+
+  async download(context: DownloadContext): Promise<void> {
+    // Build the extension proxy image.
+    const workDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'rd-build-rdx-pf-'));
+
+    try {
+      const executablePath = path.join(context.resourcesDir, 'linux', 'staging', 'extension-proxy');
+      const layerPath = path.join(workDir, 'layer.tar');
+      const imagePath = path.join(context.resourcesDir, 'rdx-proxy.tar');
+
+      console.log('Building RDX proxying image...');
+
+      // Build the layer tarball
+      // tar streams don't implement piping to multiple writers, and stream.Duplex
+      // can't deal with it either; so we need to fully write out the file, then
+      // calculate the hash as a separate step.
+      const layer = tar.pack();
+      const layerOutput = layer.pipe(fs.createWriteStream(layerPath));
+      const executableStats = await fs.promises.stat(executablePath);
+
+      await stream.promises.finished(
+        fs.createReadStream(executablePath)
+          .pipe(layer.entry({
+            name:  path.basename(executablePath),
+            mode:  0o755,
+            type:  'file',
+            mtime: new Date(0),
+            size:  executableStats.size,
+          })));
+      layer.finalize();
+      await stream.promises.finished(layerOutput);
+
+      // calculate the hash
+      const layerReader = fs.createReadStream(layerPath);
+      const layerHasher = layerReader.pipe(crypto.createHash('sha256'));
+
+      await stream.promises.finished(layerReader);
+
+      // Build the image tarball
+      const layerHash = layerHasher.digest().toString('hex');
+      const image = tar.pack();
+      const imageStream = fs.createWriteStream(imagePath);
+      const imageWritten = stream.promises.finished(imageStream);
+
+      image.pipe(imageStream);
+      const addEntry = (name: string, input: Buffer | stream.Readable, size?: number) => {
+        if (Buffer.isBuffer(input)) {
+          size = input.length;
+          input = stream.Readable.from(input);
+        }
+
+        return stream.promises.finished((input as stream.Readable).pipe(image.entry({
+          name,
+          size,
+          type:  'file',
+          mtime: new Date(0),
+        })));
+      };
+
+      image.entry({ name: layerHash, type: 'directory' });
+      await addEntry(`${ layerHash }/VERSION`, Buffer.from('1.0'));
+      await addEntry(`${ layerHash }/layer.tar`, fs.createReadStream(layerPath), layerOutput.bytesWritten);
+      await addEntry(`${ layerHash }/json`, Buffer.from(JSON.stringify({
+        id:     layerHash,
+        config: {
+          ExposedPorts: { '80/tcp': {} },
+          WorkingDir:   '/',
+          Entrypoint:   [`/${ path.basename(executablePath) }`],
+        },
+      })));
+      await addEntry(`${ layerHash }.json`, Buffer.from(JSON.stringify({
+        architecture: context.isM1 ? 'arm64' : 'amd64',
+        config:       {
+          ExposedPorts: { '80/tcp': {} },
+          Entrypoint:   [`/${ path.basename(executablePath) }`],
+          WorkingDir:   '/',
+        },
+        history: [],
+        os:      'linux',
+        rootfs:  {
+          type:     'layers',
+          diff_ids: [`sha256:${ layerHash }`],
+        },
+      })));
+      await addEntry('manifest.json', Buffer.from(JSON.stringify([
+        {
+          Config:   `${ layerHash }.json`,
+          RepoTags: ['ghcr.io/rancher-sandbox/rancher-desktop/rdx-proxy:latest'],
+          Layers:   [`${ layerHash }/layer.tar`],
+        },
+      ])));
+      image.finalize();
+      await imageWritten;
+      console.log('Built RDX port proxy image');
+    } finally {
+      await fs.promises.rm(workDir, { recursive: true });
+    }
+  }
+
+  getAvailableVersions(includePrerelease?: boolean | undefined): Promise<string[] | AlpineLimaISOVersion[]> {
+    throw new Error('extension-proxy does not have versions.');
+  }
+
+  rcompareVersions(version1: string | AlpineLimaISOVersion, version2: string | AlpineLimaISOVersion): 0 | 1 | -1 {
+    throw new Error('extension-proxy does not have versions.');
+  }
+}
+
+export class WSLDistroImage implements Dependency {
+  name = 'WSLDistroImage';
+  dependencies(context: DownloadContext): string[] {
+    return ['WSLDistro:win32', 'guestagent:linux'];
+  }
+
+  async download(context: DownloadContext): Promise<void> {
+    const tarName = `distro-${ context.versions.WSLDistro }.tar`;
+    const pristinePath = path.join(context.resourcesDir, context.platform, 'staging', tarName);
+    const pristineFile = fs.createReadStream(pristinePath);
+    const extractor = tar.extract();
+    const destPath = path.join(context.resourcesDir, context.platform, tarName);
+    const destFile = fs.createWriteStream(destPath);
+    const packer = tar.pack();
+
+    console.log('Building WSLDistro image...');
+
+    // Copy the pristine tar file to the destination.
+    packer.pipe(destFile);
+    extractor.on('entry', (header, stream, callback) => {
+      stream.pipe(packer.entry(header, callback));
+    });
+    await stream.promises.finished(pristineFile.pipe(extractor));
+
+    async function addFile(fromPath: string, name: string, options: Omit<tar.Headers, 'name' | 'size'> = {}) {
+      const { size } = await fs.promises.stat(fromPath);
+      const inputFile = fs.createReadStream(fromPath);
+
+      console.log(`WSL Distro: Adding ${ fromPath } to ${ name }...`);
+      await stream.promises.finished(inputFile.pipe(packer.entry({
+        name,
+        size,
+        mode:  0o755,
+        type:  'file',
+        mtime: new Date(0),
+        ...options,
+      })));
+    }
+
+    // Add extra files.
+    await addFile(path.join(context.resourcesDir, 'linux', 'staging', 'guestagent'),
+      'usr/local/bin/rancher-desktop-guestagent');
+    await addFile(path.join(context.resourcesDir, 'linux', 'staging', 'trivy'),
+      'usr/local/bin/trivy');
+
+    // Finish the archive.
+    packer.finalize();
+    await stream.promises.finished(packer);
+    console.log('Built WSLDistro image.');
+  }
+
+  getAvailableVersions(includePrerelease?: boolean | undefined): Promise<string[] | AlpineLimaISOVersion[]> {
+    throw new Error('WSLDistroImage does not have versions.');
+  }
+
+  rcompareVersions(version1: string | AlpineLimaISOVersion, version2: string | AlpineLimaISOVersion): 0 | 1 | -1 {
+    throw new Error('WSLDistroImage does not have versions.');
+  }
+}

scripts/dependencies/tools.ts

@@ -322,7 +322,8 @@ export class Trivy implements Dependency, GitHubDependency {
     const trivyURL = `${ trivyURLBase }/download/${ versionWithV }/${ trivyBasename }.tar.gz`;
     const checksumURL = `${ trivyURLBase }/download/${ versionWithV }/trivy_${ context.versions.trivy }_checksums.txt`;
     const trivySHA = await findChecksum(checksumURL, `${ trivyBasename }.tar.gz`);
-    const trivyPath = path.join(context.resourcesDir, 'linux', 'internal', 'trivy');
+    const trivyDir = context.dependencyPlatform === 'wsl' ? 'staging' : 'internal';
+    const trivyPath = path.join(context.resourcesDir, 'linux', trivyDir, 'trivy');
 
     // trivy.tgz files are top-level tarballs - not wrapped in a labelled directory :(
     await downloadTarGZ(trivyURL, trivyPath, { expectedChecksum: trivySHA });

scripts/dependencies/wsl.ts

@@ -164,7 +164,7 @@ export class WSLDistro implements Dependency, GitHubDependency {
     const baseUrl = `https://github.com/${ this.githubOwner }/${ this.githubRepo }/releases/download`;
     const tarName = `distro-${ context.versions.WSLDistro }.tar`;
     const url = `${ baseUrl }/v${ context.versions.WSLDistro }/${ tarName }`;
-    const destPath = path.join(context.resourcesDir, context.platform, tarName);
+    const destPath = path.join(context.resourcesDir, context.platform, 'staging', tarName);
 
     await download(url, destPath, { access: fs.constants.W_OK });
   }