orchestratord: Test defaults from our documentation

Author: def-

.gitignore

@@ -59,3 +59,4 @@ flake.lock
 /src/testdrive/ci/protobuf-include
 /known-docker-images.txt
 /test/sqllogictest/sqlite
+my-local-mz/

ci/nightly/pipeline.template.yml

@@ -2295,6 +2295,18 @@ steps:
     key: orchestratord-test
     steps:
       - id: orchestratord-defaults
+        label: "Orchestratord test (defaults from documentation)"
+        depends_on: build-aarch64
+        timeout_in_minutes: 120
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: orchestratord
+              run: defaults
+              ci-builder: stable
+        agents:
+          queue: hetzner-aarch64-16cpu-32gb
+
+      - id: orchestratord-default-properties
         label: "Orchestratord test (defaults for properties)"
         depends_on: build-aarch64
         timeout_in_minutes: 120

ci/test/lint-main/checks/check-mzcompose-files.sh

@@ -48,6 +48,7 @@ check_default_workflow_references_others() {
         -not -wholename "./test/testdrive-old-kafka-src-syntax/mzcompose.py" `# Other workflow is run separately` \
         -not -wholename "./test/terraform/mzcompose.py" `# Handled differently` \
         -not -wholename "./test/cluster-spec-sheet/mzcompose.py" `# Handled differently` \
+        -not -wholename "./test/orchestratord/mzcompose.py" `# Handled differently` \
     )
 
     for file in "${MZCOMPOSE_TEST_FILES[@]}"; do

misc/helm-charts/testing/materialize.yaml

@@ -31,6 +31,7 @@ spec:
   environmentdImageRef: materialize/environmentd:v0.163.0-dev.0
   backendSecretName: materialize-backend
   authenticatorKind: None
+  environmentId: 12345678-1234-1234-1234-123456789013
   #balancerdExternalCertificateSpec:
   #  dnsNames:
   #    - balancerd

test/orchestratord/mzcompose.py

@@ -17,6 +17,7 @@
 import json
 import os
 import random
+import shutil
 import signal
 import subprocess
 import time
@@ -25,6 +26,7 @@
 from enum import Enum
 from typing import Any
 
+import requests
 import yaml
 from semver.version import Version
 
@@ -41,7 +43,10 @@
 from materialize.mzcompose.services.orchestratord import Orchestratord
 from materialize.mzcompose.services.testdrive import Testdrive
 from materialize.util import all_subclasses
-from materialize.version_list import get_all_self_managed_versions
+from materialize.version_list import (
+    get_all_self_managed_versions,
+    get_self_managed_versions,
+)
 
 SERVICES = [
     Testdrive(),
@@ -1127,6 +1132,125 @@ class Action(Enum):
     UpgradeChain = "upgrade-chain"
 
 
+def workflow_defaults(c: Composition, parser: WorkflowArgumentParser) -> None:
+    current_version = get_tag()
+
+    # Following https://materialize.com/docs/self-managed/v25.2/installation/install-on-local-kind/
+    for version in reversed(get_self_managed_versions() + [current_version]):
+        dir = "my-local-mz"
+        if os.path.exists(dir):
+            shutil.rmtree(dir)
+        os.mkdir(dir)
+        spawn.runv(["kind", "delete", "cluster"])
+        spawn.runv(["kind", "create", "cluster"])
+
+        shutil.copyfile(
+            "misc/helm-charts/operator/values.yaml",
+            os.path.join(dir, "sample-values.yaml"),
+        )
+        files = {
+            "sample-postgres.yaml": "misc/helm-charts/testing/postgres.yaml",
+            "sample-minio.yaml": "misc/helm-charts/testing/minio.yaml",
+            "sample-materialize.yaml": "misc/helm-charts/testing/materialize.yaml",
+        }
+
+        for file, path in files.items():
+            if version == current_version:
+                shutil.copyfile(path, os.path.join(dir, file))
+            else:
+                url = f"https://raw.githubusercontent.com/MaterializeInc/materialize/refs/tags/{version}/{path}"
+                response = requests.get(url)
+                assert (
+                    response.status_code == 200
+                ), f"Failed to download {file} from {url}: {response.status_code}"
+                with open(os.path.join(dir, file), "wb") as f:
+                    f.write(response.content)
+
+        spawn.runv(
+            [
+                "helm",
+                "repo",
+                "add",
+                "materialize",
+                "https://materializeinc.github.io/materialize",
+            ]
+        )
+        spawn.runv(["helm", "repo", "update", "materialize"])
+        spawn.runv(
+            [
+                "helm",
+                "install",
+                "my-materialize-operator",
+                "materialize/materialize-operator",
+                "--namespace=materialize",
+                "--create-namespace",
+                "--version",
+                "v25.2.13",  # TODO: How to set this version?
+                "--set",
+                "observability.podMetrics.enabled=true",
+                "-f",
+                os.path.join(dir, "sample-values.yaml"),
+            ]
+        )
+        spawn.runv(
+            ["kubectl", "apply", "-f", os.path.join(dir, "sample-postgres.yaml")]
+        )
+        spawn.runv(["kubectl", "apply", "-f", os.path.join(dir, "sample-minio.yaml")])
+        spawn.runv(["kubectl", "get", "all", "-n", "materialize"])
+        spawn.runv(
+            [
+                "helm",
+                "repo",
+                "add",
+                "metrics-server",
+                "https://kubernetes-sigs.github.io/metrics-server/",
+            ]
+        )
+        spawn.runv(["helm", "repo", "update", "metrics-server"])
+        spawn.runv(
+            [
+                "helm",
+                "install",
+                "metrics-server",
+                "metrics-server/metrics-server",
+                "--namespace",
+                "kube-system",
+                "--set",
+                "args={--kubelet-insecure-tls,--kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP}",
+            ]
+        )
+        for i in range(120):
+            try:
+                spawn.capture(
+                    [
+                        "kubectl",
+                        "get",
+                        "crd",
+                        "materializes.materialize.cloud",
+                        "-n",
+                        "materialize",
+                        "-o",
+                        "name",
+                    ],
+                    stderr=subprocess.DEVNULL,
+                )
+                break
+
+            except subprocess.CalledProcessError:
+                pass
+            time.sleep(1)
+        else:
+            raise ValueError("Never completed")
+        spawn.runv(
+            ["kubectl", "apply", "-f", os.path.join(dir, "sample-materialize.yaml")]
+        )
+        # TODO: Verify that everything is there:
+        spawn.runv(["kubectl", "get", "all", "-n", "materialize-environment"])
+
+        # TODO: Remove the break once environmentId is set in older versions
+        break
+
+
 def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
     parser.add_argument(
         "--recreate-cluster",