Merge pull request #5303 from pablintino/devex-mco-sanitize-main

MCO-1685: Add mco-sanitize utility main logic

Author: openshift-merge-bot[bot]

devex/cmd/mco-sanitize/README.md

@@ -0,0 +1,158 @@
+# mco-sanitize
+
+A command-line tool that removes sensitive information from Machine Config Operator (MCO) must-gather reports while 
+preserving their structure for debugging and analysis purposes.
+
+## Overview
+
+`mco-sanitize` is designed to sanitize OpenShift must-gather reports by redacting sensitive data from Kubernetes 
+resources according to configurable rules. The tool maintains the original file structure and metadata while r
+eplacing sensitive content with redaction markers that preserve data length information for analysis.
+
+## Features
+
+- **Configurable Redaction**: Define which Kubernetes resource types and fields to sanitize via YAML configuration
+- **Parallel Processing**: Multi-threaded file processing for improved performance (defaults to CPU core count)
+- **Encrypted Output**: Automatically creates GPG-encrypted tar.gz archives of sanitized data
+- **Multiple File Format Support**: Handles YAML, JSON, and other text-based files
+- **Path-based Targeting**: Use dot-notation paths to target specific fields within resources
+- **Array Support**: Process all elements (`*`) or specific indices in arrays
+- **Namespace Filtering**: Optionally limit redaction to specific namespaces
+
+## Installation
+
+Build the tool from source:
+
+```bash
+go build -o mco-sanitize .
+```
+
+## Usage
+
+### Basic Usage
+
+```bash
+# Sanitize a must-gather directory
+./mco-sanitize --input /path/to/must-gather
+
+# Sanitize and create encrypted archive
+./mco-sanitize --input /path/to/must-gather --output /path/to/sanitized.tar.gz
+
+# Use custom worker count
+./mco-sanitize --input /path/to/must-gather --workers 8
+```
+
+### Command Line Options
+
+- `--input` (required): Path to the must-gather directory to sanitize
+- `--output` (optional): Path where the encrypted tar.gz output should be saved
+- `--workers` (optional): Number of worker threads (defaults to CPU core count)
+
+## Configuration
+
+### Default Configuration
+
+The tool includes a built-in default configuration that redacts sensitive fields from common MCO resources:
+
+```yaml
+redact:
+  - kind: MachineConfig
+    apiVersion: machineconfiguration.openshift.io/v1
+    paths:
+      - spec.config.storage.files.*.contents
+      - spec.config.systemd.units.*.contents
+      - spec.config.systemd.units.*.dropins.*.contents
+  - kind: ControllerConfig
+    apiVersion: machineconfiguration.openshift.io/v1
+    paths:
+      - spec.internalRegistryPullSecret
+      - spec.kubeAPIServerServingCAData
+      - spec.rootCAData
+      - spec.additionalTrustBundle
+```
+
+### Custom Configuration
+
+Override the default configuration using the `MCO_MUST_GATHER_SANITIZER_CFG` environment variable:
+
+```bash
+# Using a configuration file
+export MCO_MUST_GATHER_SANITIZER_CFG="/path/to/config.yaml"
+
+# Using base64-encoded configuration
+export MCO_MUST_GATHER_SANITIZER_CFG="cmVkYWN0Og0KIC0ga2luZDogU2VjcmV0..."
+```
+
+#### Configuration Format
+
+```yaml
+redact:
+  - kind: Pod                    # Kubernetes resource kind (required)
+    apiVersion: v1               # API version (optional, matches all if empty)
+    namespaces:                  # Limit to specific namespaces (optional)
+      - kube-system
+      - openshift-config
+    paths:                       # Fields to redact using dot notation
+      - spec.containers.*.env.*.value
+      - data.password
+      - metadata.annotations.secret-key
+```
+
+#### Path Syntax
+
+- Use dot notation to navigate object hierarchies: `spec.containers.0.image`
+- Use `*` for all array elements: `spec.containers.*.env.*.value`
+- Use numeric indices for specific array elements: `spec.containers.0.ports.1.containerPort`
+- Combine object and array navigation: `data.config.yaml.databases.*.password`
+
+## Encryption
+
+### Default Encryption
+
+By default, archives are encrypted using an embedded GPG public key. This ensures that sanitized data remains 
+secure during transport and storage.
+
+### Custom Encryption Key
+
+Provide your own GPG public key using the `MCO_MUST_GATHER_SANITIZER_KEY` environment variable:
+
+```bash
+# Using base64-encoded public key
+export MCO_MUST_GATHER_SANITIZER_KEY="$(base64 -w 0 < /path/to/public-key.asc)"
+```
+
+## Redaction Behavior
+
+When a field is redacted, it's replaced with a structured object containing:
+
+```yaml
+_REDACTED: "This field has been redacted"
+length: 1234  # Original content length in characters
+```
+
+This preserves:
+- The fact that sensitive data existed
+- The approximate size of the original data
+- The overall structure of the resource
+
+
+## Environment Variables
+
+| Variable | Description | Example |
+|----------|-------------|---------|
+| `MCO_MUST_GATHER_SANITIZER_CFG` | Custom configuration (file path or base64) | `/path/to/config.yaml` |
+| `MCO_MUST_GATHER_SANITIZER_KEY` | Custom GPG public key (base64-encoded) | `LS0tLS1CRUdJTi...` |
+
+## Development
+
+### Running Tests
+
+```bash
+go test ./...
+```
+
+### Adding New Redaction Rules
+
+1. Update the default configuration in `data/default-config.yaml`
+2. Add test cases in the `testdata/` directory
+3. Run tests to verify behavior

devex/cmd/mco-sanitize/archive.go

@@ -0,0 +1,134 @@
+// Assisted-by: Claude
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	_ "embed"
+	"encoding/base64"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/ProtonMail/go-crypto/openpgp"
+)
+
+const McoMustGatherSanitizerEncryptKeyEnvVar = "MCO_MUST_GATHER_SANITIZER_KEY"
+
+// pabrodri/pablintino's GPG key by default
+//
+//go:embed data/public-key.asc
+var defaultGpgKey []byte
+
+func Archive(src, target string) (err error) {
+	entityList, err := getEncryptionKey()
+	if err != nil {
+		return err
+	}
+	buf := &bytes.Buffer{}
+	if err := writeTar(src, buf); err != nil {
+		return err
+	}
+
+	targetFile, err := os.OpenFile(target, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o666)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if dstErr := targetFile.Close(); dstErr != nil {
+			err = errors.Join(err, dstErr)
+		}
+	}()
+	return encrypt(entityList, buf, targetFile)
+}
+
+func getEncryptionKey() (openpgp.EntityList, error) {
+	// Early load and fail if the key is not present/valid
+	content := os.Getenv(McoMustGatherSanitizerEncryptKeyEnvVar)
+	var gpgBuffer *bytes.Buffer
+
+	if content == "" {
+		gpgBuffer = bytes.NewBuffer(defaultGpgKey)
+	} else {
+		rawBytes, err := base64.StdEncoding.DecodeString(content)
+		if err != nil {
+			return nil, err
+		}
+		gpgBuffer = bytes.NewBuffer(rawBytes)
+	}
+
+	entityList, err := openpgp.ReadArmoredKeyRing(gpgBuffer)
+	if err != nil {
+		return nil, err
+	}
+	return entityList, err
+}
+
+func writeTar(src string, writer io.Writer) (err error) {
+	zr := gzip.NewWriter(writer)
+	defer func() {
+		if zrErr := zr.Close(); zrErr != nil {
+			err = errors.Join(err, zrErr)
+		}
+	}()
+	tw := tar.NewWriter(zr)
+	defer func() {
+		if twErr := tw.Close(); twErr != nil {
+			err = errors.Join(err, twErr)
+		}
+	}()
+
+	// walk through every file in the folder
+	return filepath.Walk(src, func(file string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		relPath, err := filepath.Rel(src, file)
+		if err != nil {
+			return err
+		}
+		header, err := tar.FileInfoHeader(fi, relPath)
+		if err != nil {
+			return err
+		}
+
+		header.Name = filepath.ToSlash(relPath)
+
+		if err := tw.WriteHeader(header); err != nil {
+			return err
+		}
+		// if not a dir, write file content
+		if !fi.IsDir() {
+			data, err := os.Open(file)
+			if err != nil {
+				return err
+			}
+			defer data.Close()
+			if err != nil {
+				return err
+			}
+			if _, err := io.Copy(tw, data); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
+
+func encrypt(entities []*openpgp.Entity, reader io.Reader, writer io.Writer) (err error) {
+	gpgWriter, err := openpgp.Encrypt(writer, entities, nil, &openpgp.FileHints{IsBinary: true}, nil)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if wCloseErr := gpgWriter.Close(); wCloseErr != nil {
+			err = errors.Join(err, wCloseErr)
+		}
+	}()
+	if _, err := io.Copy(gpgWriter, reader); err != nil {
+		return err
+	}
+	return err
+}

devex/cmd/mco-sanitize/archive_test.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestArchive_Success(t *testing.T) {
+	// Create a temporary directory with test files
+	tempDir := t.TempDir()
+	testFile := filepath.Join(tempDir, "test.txt")
+	require.NoError(t, os.WriteFile(testFile, []byte("test content"), 0644))
+
+	// Create a temporary output file
+	outputFile := filepath.Join(t.TempDir(), "output.tar.gz.gpg")
+
+	// Test the Archive function
+	err := Archive(tempDir, outputFile)
+
+	assert.NoError(t, err)
+	assert.FileExists(t, outputFile)
+
+	// Verify the output file is not empty
+	stat, err := os.Stat(outputFile)
+	require.NoError(t, err)
+	assert.Greater(t, stat.Size(), int64(0))
+
+	// Verify the file is GPG encrypted by checking for GPG binary markers
+	fileContent, err := os.ReadFile(outputFile)
+	require.NoError(t, err)
+
+	// GPG encrypted files start with packet type markers (high bit set, indicating packet type)
+	assert.True(t, len(fileContent) > 0 && fileContent[0] >= 0x80,
+		"File should be GPG encrypted (expected GPG packet marker >= 0x80, got 0x%02x)", fileContent[0])
+}
+
+func TestArchive_NonExistentSource(t *testing.T) {
+	nonExistentDir := "/non/existent/directory"
+	outputFile := filepath.Join(t.TempDir(), "output.tar.gz.gpg")
+
+	err := Archive(nonExistentDir, outputFile)
+
+	assert.Error(t, err)
+}
+
+func TestArchive_InvalidTarget(t *testing.T) {
+	tempDir := t.TempDir()
+	testFile := filepath.Join(tempDir, "test.txt")
+	require.NoError(t, os.WriteFile(testFile, []byte("test content"), 0644))
+
+	// Try to write to a directory that doesn't exist
+	invalidTarget := "/non/existent/directory/output.tar.gz.gpg"
+
+	err := Archive(tempDir, invalidTarget)
+
+	assert.Error(t, err)
+}
+
+func TestArchive_EmptyDirectory(t *testing.T) {
+	tempDir := t.TempDir()
+	outputFile := filepath.Join(t.TempDir(), "output.tar.gz.gpg")
+
+	err := Archive(tempDir, outputFile)
+
+	assert.NoError(t, err)
+	assert.FileExists(t, outputFile)
+}