build: Try to get ASan much greener, fix minor memory leaks

Author: def-

bin/ci-builder

@@ -47,6 +47,10 @@ fi
 cmd=$1 && shift
 flavor=$1 && shift
 
+if [[ ${CI_SANITIZER:-none} != none && $flavor = stable ]]; then
+    flavor=nightly
+fi
+
 rust_date=
 case "$flavor" in
     min)

ci/builder/Dockerfile

@@ -141,7 +141,7 @@ RUN apt-get update --fix-missing && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-ge
     libc6-dbg \
     libclang-common-18-dev \
     libclang-dev \
-    libclang-rt-18-dev \
+    libclang-rt-19-dev \
     libpq-dev \
     libxml2 \
     lld \

ci/builder/sanshim/aarch64/c++

@@ -12,7 +12,4 @@
 exec /usr/bin/clang++ -fsanitize="$CI_SANITIZER" \
   -fuse-ld=lld \
   --target=aarch64-unknown-linux-gnu \
-  --gcc-toolchain=/opt/x-tools/aarch64-unknown-linux-gnu/ \
-  --sysroot=/opt/x-tools/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot \
-  -L/opt/x-tools/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/lib \
   "$@"

ci/builder/sanshim/aarch64/cc

@@ -12,7 +12,4 @@
 exec /usr/bin/clang -fsanitize="$CI_SANITIZER" \
   -fuse-ld=lld \
   --target=aarch64-unknown-linux-gnu \
-  --gcc-toolchain=/opt/x-tools/aarch64-unknown-linux-gnu/ \
-  --sysroot=/opt/x-tools/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot \
-  -L/opt/x-tools/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/lib \
   "$@"

ci/builder/sanshim/x86_64/c++

@@ -12,7 +12,4 @@
 exec /usr/bin/clang++ -fsanitize="$CI_SANITIZER" \
   -fuse-ld=lld \
   --target=x86_64-unknown-linux-gnu \
-  --gcc-toolchain=/opt/x-tools/x86_64-unknown-linux-gnu/ \
-  --sysroot=/opt/x-tools/x86_64-unknown-linux-gnu/x86_64-unknown-linux-gnu/sysroot \
-  -L/opt/x-tools/x86_64-unknown-linux-gnu/x86_64-unknown-linux-gnu/sysroot/lib \
   "$@"

ci/builder/sanshim/x86_64/cc

@@ -12,7 +12,4 @@
 exec /usr/bin/clang -fsanitize="$CI_SANITIZER" \
   -fuse-ld=lld \
   --target=x86_64-unknown-linux-gnu \
-  --gcc-toolchain=/opt/x-tools/x86_64-unknown-linux-gnu/ \
-  --sysroot=/opt/x-tools/x86_64-unknown-linux-gnu/x86_64-unknown-linux-gnu/sysroot \
-  -L/opt/x-tools/x86_64-unknown-linux-gnu/x86_64-unknown-linux-gnu/sysroot/lib \
   "$@"

ci/mkpipeline.py

@@ -381,9 +381,12 @@ def fetch_hashes() -> None:
             )
     truncate_skip_length(pipeline)
     handle_sanitizer_skip(pipeline, args.sanitizer)
-    increase_agents_timeouts(pipeline, args.sanitizer, args.coverage)
     prioritize_pipeline(pipeline, args.priority)
     switch_jobs_to_aws(pipeline, args.priority)
+    # After `switch_jobs_to_aws`, so that the queue a step ends up on is the one
+    # that gets sized up. The other order lets the aarch64 to x86_64 fallback
+    # land on a smaller machine than the size-up asked for.
+    increase_agents_timeouts(pipeline, args.sanitizer, args.coverage)
     permit_rerunning_successful_steps(pipeline)
     set_retry_on_agent_lost(pipeline)
     set_default_agents_queue(pipeline)
@@ -496,56 +499,55 @@ def handle_sanitizer_skip(pipeline: Any, sanitizer: Sanitizer) -> None:
                 step["skip"] = True
 
 
+# The next size up for each agent queue, used when a run needs more machine
+# than the pipeline asks for.
+NEXT_LARGER_AGENT = {
+    "linux-aarch64-small": "linux-aarch64",
+    "linux-aarch64": "linux-aarch64-medium",
+    "linux-aarch64-medium": "linux-aarch64-large",
+    "linux-aarch64-large": "builder-linux-aarch64-mem",
+    "linux-x86_64-small": "linux-x86_64",
+    "linux-x86_64": "linux-x86_64-medium",
+    "linux-x86_64-medium": "linux-x86_64-large",
+    "linux-x86_64-large": "builder-linux-x86_64",
+    "hetzner-aarch64-2cpu-4gb": "hetzner-aarch64-4cpu-8gb",
+    "hetzner-aarch64-4cpu-8gb": "hetzner-aarch64-8cpu-16gb",
+    "hetzner-aarch64-8cpu-16gb": "hetzner-aarch64-16cpu-32gb",
+    "hetzner-x86-64-2cpu-4gb": "hetzner-x86-64-4cpu-8gb",
+    "hetzner-x86-64-4cpu-8gb": "hetzner-x86-64-8cpu-16gb",
+    "hetzner-x86-64-8cpu-16gb": "hetzner-x86-64-16cpu-32gb",
+    "hetzner-x86-64-12cpu-24gb": "hetzner-x86-64-dedi-16cpu-64gb",
+    "hetzner-x86-64-16cpu-32gb": "hetzner-x86-64-dedi-16cpu-64gb",
+    "hetzner-x86-64-16cpu-64gb": "hetzner-x86-64-dedi-32cpu-128gb",
+    "hetzner-x86-64-dedi-8cpu-32gb": "hetzner-x86-64-dedi-16cpu-64gb",
+    "hetzner-x86-64-dedi-16cpu-64gb": "hetzner-x86-64-dedi-32cpu-128gb",
+    "hetzner-x86-64-dedi-32cpu-128gb": "hetzner-x86-64-dedi-48cpu-192gb",
+}
+
+
 def increase_agents_timeouts(
     pipeline: Any, sanitizer: Sanitizer, coverage: bool
 ) -> None:
-    if sanitizer != Sanitizer.none or os.getenv("CI_SYSTEM_PARAMETERS", "") == "random":
+    # Most sanitizer runs, as well as random permutations of system parameters,
+    # are slower and need more memory. The default system parameters in CI are
+    # chosen to be efficient for execution, while a random permutation might
+    # take way longer and use more memory.
+    if sanitizer != Sanitizer.none:
+        sizes_up = 2
+    elif os.getenv("CI_SYSTEM_PARAMETERS", "") == "random":
+        sizes_up = 1
+    else:
+        sizes_up = 0
+
+    if sizes_up:
         for step in steps(pipeline):
-            # Most sanitizer runs, as well as random permutations of system
-            # parameters, are slower and need more memory. The default system
-            # parameters in CI are chosen to be efficient for execution, while
-            # a random permutation might take way longer and use more memory.
             if "timeout_in_minutes" in step:
                 step["timeout_in_minutes"] *= 10
 
             if "agents" in step:
                 agent = step["agents"].get("queue", None)
-                if agent == "linux-aarch64-small":
-                    agent = "linux-aarch64"
-                elif agent == "linux-aarch64":
-                    agent = "linux-aarch64-medium"
-                elif agent == "linux-aarch64-medium":
-                    agent = "linux-aarch64-large"
-                elif agent == "linux-aarch64-large":
-                    agent = "builder-linux-aarch64-mem"
-                elif agent == "linux-x86_64-small":
-                    agent = "linux-x86_64"
-                elif agent == "linux-x86_64":
-                    agent = "linux-x86_64-medium"
-                elif agent == "linux-x86_64-medium":
-                    agent = "linux-x86_64-large"
-                elif agent == "linux-x86_64-large":
-                    agent = "builder-linux-x86_64"
-                elif agent == "hetzner-aarch64-2cpu-4gb":
-                    agent = "hetzner-aarch64-4cpu-8gb"
-                elif agent == "hetzner-aarch64-4cpu-8gb":
-                    agent = "hetzner-aarch64-8cpu-16gb"
-                elif agent == "hetzner-aarch64-8cpu-16gb":
-                    agent = "hetzner-aarch64-16cpu-32gb"
-                elif agent == "hetzner-x86-64-2cpu-4gb":
-                    agent = "hetzner-x86-64-4cpu-8gb"
-                elif agent == "hetzner-x86-64-4cpu-8gb":
-                    agent = "hetzner-x86-64-8cpu-16gb"
-                elif agent == "hetzner-x86-64-8cpu-16gb":
-                    agent = "hetzner-x86-64-16cpu-32gb"
-                elif agent == "hetzner-x86-64-12cpu-24gb":
-                    agent = "hetzner-x86-64-dedi-16cpu-64gb"
-                elif agent == "hetzner-x86-64-16cpu-32gb":
-                    agent = "hetzner-x86-64-dedi-16cpu-64gb"
-                elif agent == "hetzner-x86-64-16cpu-64gb":
-                    agent = "hetzner-x86-64-dedi-32cpu-128gb"
-                elif agent == "hetzner-x86-64-dedi-32cpu-128gb":
-                    agent = "hetzner-x86-64-dedi-48cpu-192gb"
+                for _ in range(sizes_up):
+                    agent = NEXT_LARGER_AGENT.get(agent, agent)
                 step["agents"] = {"queue": agent}
 
     if coverage:

ci/nightly/pipeline.template.yml

@@ -36,7 +36,7 @@ steps:
 
       - id: build-x86_64-asan
         label: ":rust: Build x86_64 (ASan)"
-        command: bin/ci-builder run min bin/pyactivate -m ci.test.build
+        command: bin/ci-builder run nightly bin/pyactivate -m ci.test.build
         inputs:
           - "*"
         depends_on: []
@@ -45,14 +45,12 @@ steps:
           queue: builder-linux-x86_64
         env:
           CI_SANITIZER: address
-
         sanitizer: skip
         branches: "main"
-        skip: "currently broken"
 
       - id: build-aarch64-asan
         label: ":rust: Build aarch64 (ASan)"
-        command: bin/ci-builder run min bin/pyactivate -m ci.test.build
+        command: bin/ci-builder run nightly bin/pyactivate -m ci.test.build
         inputs:
           - "*"
         depends_on: []
@@ -63,7 +61,6 @@ steps:
           CI_SANITIZER: address
         sanitizer: skip
         branches: "main"
-        skip: "currently broken"
 
       - id: build-rust-latest-beta
         label: "Build with Latest Rust Beta"
@@ -179,6 +176,7 @@ steps:
       - id: feature-benchmark
         topics: [cockroach, copy-to-s3, iceberg, kafka, kafka-sink, mysql, postgres, sql-server]
         label: "Feature benchmark against merge base or 'latest'"
+        sanitizer: skip
         depends_on: build-x86_64
         timeout_in_minutes: 120
         parallelism: 12
@@ -194,6 +192,7 @@ steps:
       - id: scalability-benchmark-dml-dql
         topics: [postgres]
         label: "Scalability benchmark (read & write) against merge base or 'latest'"
+        sanitizer: skip
         depends_on: build-x86_64
         timeout_in_minutes: 120
         env:
@@ -220,6 +219,7 @@ steps:
       - id: scalability-benchmark-ddl
         topics: [postgres]
         label: "Scalability benchmark (DDL) against merge base or 'latest'"
+        sanitizer: skip
         depends_on: build-x86_64
         timeout_in_minutes: 120
         env:
@@ -249,6 +249,7 @@ steps:
       - id: scalability-benchmark-connection
         topics: [postgres]
         label: "Scalability benchmark (connection) against merge base or 'latest'"
+        sanitizer: skip
         depends_on: build-x86_64
         timeout_in_minutes: 120
         parallelism: 2
@@ -277,6 +278,7 @@ steps:
       - id: parallel-benchmark
         topics: [cockroach, kafka, mysql, postgres]
         label: "Parallel Benchmark"
+        sanitizer: skip
         depends_on: build-x86_64
         timeout_in_minutes: 120
         parallelism: 5
@@ -296,6 +298,7 @@ steps:
       - id: workload-replay
         topics: [iceberg, kafka, mysql, postgres, sql-server, ssh-tunnel]
         label: "Workload Replay (1% initial data)"
+        sanitizer: skip
         depends_on: build-x86_64
         timeout_in_minutes: 120
         parallelism: 5
@@ -314,6 +317,7 @@ steps:
       - id: console-scalability
         topics: [iceberg, kafka, mysql, postgres, sql-server, ssh-tunnel]
         label: "Console scalability (workload replay)"
+        sanitizer: skip
         depends_on: build-x86_64
         timeout_in_minutes: 120
         parallelism: 3
@@ -389,6 +393,7 @@ steps:
       - id: testdrive-replicas-4
         topics: [copy-to-s3, kafka, kafka-sink, mysql, postgres]
         label: ":racing_car: testdrive 4 replicas"
+        sanitizer: skip
         depends_on: build-aarch64
         timeout_in_minutes: 60
         parallelism: 2
@@ -415,6 +420,7 @@ steps:
       - id: testdrive-size-8
         topics: [copy-to-s3, kafka, kafka-sink, mysql, postgres]
         label: ":racing_car: testdrive with SIZE 8"
+        sanitizer: skip
         depends_on: build-aarch64
         timeout_in_minutes: 60
         parallelism: 2
@@ -456,6 +462,7 @@ steps:
       - id: azurite-testdrive-size-8
         topics: [azurite, copy-to-s3, kafka, kafka-sink, mysql, postgres]
         label: ":racing_car: testdrive with SIZE 8 and :azure: blob store"
+        sanitizer: skip
         depends_on: build-aarch64
         timeout_in_minutes: 60
         parallelism: 2
@@ -529,6 +536,7 @@ steps:
       - id: bounded-memory
         topics: [copy-to-s3, iceberg, kafka, mysql, postgres]
         label: "Bounded Memory"
+        sanitizer: skip
         depends_on: build-aarch64
         timeout_in_minutes: 60
         parallelism: 2
@@ -540,6 +548,7 @@ steps:
       - id: bounded-memory-search
         topics: [copy-to-s3, iceberg, kafka, mysql, postgres]
         label: "Bounded Memory Search (materialized only)"
+        sanitizer: skip
         depends_on: build-aarch64
         timeout_in_minutes: 60
         parallelism: 8

ci/test/cargo-test/mzcompose.py

@@ -86,10 +86,6 @@
 ]
 
 
-def flatten(xss):
-    return [x for xs in xss for x in xs]
-
-
 def pull_image(image: str) -> None:
     # Check if image exists locally before pulling
     image_exists = subprocess.run(
@@ -280,12 +276,9 @@ def run_sanitizer(
 ):
     cflags = [
         f"--target={target(Arch.host())}",
-        f"--gcc-toolchain=/opt/x-tools/{target(Arch.host())}/",
-        f"--sysroot=/opt/x-tools/{target(Arch.host())}/{target(Arch.host())}/sysroot",
     ] + rustc_flags.sanitizer_cflags[sanitizer]
     ldflags = cflags + [
         "-fuse-ld=lld",
-        f"-L/opt/x-tools/{target(Arch.host())}/{target(Arch.host())}/lib64",
     ]
     extra_env = {
         "CFLAGS": " ".join(cflags),
@@ -294,21 +287,26 @@ def run_sanitizer(
         "CXXSTDLIB": "stdc++",
         "CC": "cc",
         "CXX": "c++",
-        "CPP": "clang-cpp-18",
+        "CPP": "clang-cpp-19",
         "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER": "cc",
         "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER": "cc",
-        "PATH": f"/sanshim:/opt/x-tools/{target(Arch.host())}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+        "PATH": "/sanshim:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
         "RUSTFLAGS": (
             env.get("RUSTFLAGS", "") + " " + " ".join(rustc_flags.sanitizer[sanitizer])
         ),
         "TSAN_OPTIONS": "report_bugs=0",  # build-scripts fail
     }
+    # `bin/ci-builder run` forwards only an allowlist of host variables, so the
+    # sanitizer environment has to be set inside the container. `env` also
+    # resolves `cargo` through the `PATH` it just set, which is what puts the
+    # `/sanshim` compiler wrappers ahead of the real ones.
+    env_prefix = ["env", *(f"{key}={val}" for key, val in extra_env.items())]
     spawn.runv(
         [
             "bin/ci-builder",
             "run",
             "nightly",
-            *flatten([["--env", f"{key}={val}"] for key, val in extra_env.items()]),
+            *env_prefix,
             "cargo",
             "build",
             "--workspace",
@@ -330,9 +328,7 @@ def run_sanitizer(
                     "bin/ci-builder",
                     "run",
                     "nightly",
-                    *flatten(
-                        [["--env", f"{key}={val}"] for key, val in extra_env.items()]
-                    ),
+                    *env_prefix,
                     "cargo",
                     "nextest",
                     "run",

ci/test/dev_tag.py

@@ -12,13 +12,12 @@
 import os
 from pathlib import Path
 
-from materialize import ci_util, git, mzbuild
+from materialize import ci_util, mzbuild
 from materialize.rustc_flags import Sanitizer
 from materialize.xcompile import Arch
 
 
 def main() -> None:
-    mz_version = ci_util.get_mz_version()
     sanitizer = Sanitizer[os.getenv("CI_SANITIZER", "none")]
 
     repos = [
@@ -42,12 +41,7 @@ def main() -> None:
         repo.resolve_dependencies(image for image in repo if image.publish)
         for repo in repos
     ]
-    # Ideally we'd use SemVer metadata (e.g., `v1.0.0+metadata`), but `+` is not
-    # a valid character in Docker tags, so we use `--` instead.
-    suffix = "pr" if sanitizer == Sanitizer.none else f"pr-{sanitizer}"
-    mzbuild.publish_multiarch_images(
-        f'v{mz_version}--{suffix}.g{git.rev_parse("HEAD")}', deps
-    )
+    mzbuild.publish_multiarch_images(ci_util.dev_docker_tag(), deps)
 
 
 if __name__ == "__main__":