diff --git a/Cargo.lock b/Cargo.lock index 6cf6e9c85dff5..04f3e2e3fe6ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10057,7 +10057,7 @@ dependencies = [ [[package]] name = "rdkafka" version = "0.29.0" -source = "git+https://github.com/MaterializeInc/rust-rdkafka.git#4ebbcd8c765d556a91c66b63b443f4ea85340cde" +source = "git+https://github.com/MaterializeInc/rust-rdkafka.git#a8db73bf96b3f65a7b48ffe4fb490eab75537310" dependencies = [ "futures-channel", "futures-util", @@ -10074,7 +10074,7 @@ dependencies = [ [[package]] name = "rdkafka-sys" version = "4.3.0+2.5.0" -source = "git+https://github.com/MaterializeInc/rust-rdkafka.git#45a2a34c428a9c7f37bc79979f25c8041d00d992" +source = "git+https://github.com/MaterializeInc/rust-rdkafka.git#a8db73bf96b3f65a7b48ffe4fb490eab75537310" dependencies = [ "cmake", "libc", diff --git a/Cargo.toml b/Cargo.toml index 5ae0e0cdbb082..c4986181059c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -286,6 +286,7 @@ similar = { opt-level = 3 } [profile.release] # Compile time seems similar to "lto = false", runtime ~10% faster lto = "thin" +incremental = false # Emit full debug info, allowing us to easily analyze core dumps from # staging (and, in an emergency, also prod). diff --git a/ci/builder/Dockerfile b/ci/builder/Dockerfile index 1af9442a6fbbe..38377c5d14b8b 100644 --- a/ci/builder/Dockerfile +++ b/ci/builder/Dockerfile @@ -108,7 +108,71 @@ RUN userdel -r ubuntu ENTRYPOINT ["autouseradd", "--user", "materialize"] -# Stage 2: Build a full CI Builder image that can be used for any CI job. +# Stage 2. Build a cross-compiling toolchain that targets the oldest version of +# Linux that we support. +# +# TODO(parkmycar): This shouldn't be necessary anymore with Bazel. +FROM ubuntu:noble-20250619 as crosstool + +ARG ARCH_GCC +ARG ARCH_GO + +WORKDIR /scratch + +# The environment variables are necessary to convince `tzdata`'s install scripts +# not to hang. +# See: https://serverfault.com/questions/949991/how-to-install-tzdata-on-a-ubuntu-docker-image +RUN apt-get update --fix-missing && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + autoconf \ + automake \ + bison \ + bzip2 \ + ca-certificates \ + curl \ + file \ + flex \ + gawk \ + g++ \ + gcc \ + gnupg2 \ + help2man \ + libc-dev \ + libncurses-dev \ + libstdc++6 \ + libstdc++-13-dev \ + libtool-bin \ + llvm \ + make \ + patch \ + texinfo \ + unzip \ + xz-utils + +COPY crosstool.asc ./ + +# Faster uncompression +ARG XZ_OPT=-T0 + +RUN gpg --import crosstool.asc \ + && rm crosstool.asc \ + && echo "trusted-key 09f6dd5f1f30ef2e" >> ~/.gnupg/gpg.conf \ + && mkdir crosstool \ + && curl -fsSL https://github.com/crosstool-ng/crosstool-ng/releases/download/crosstool-ng-1.26.0/crosstool-ng-1.26.0.tar.xz > crosstool.tar.xz \ + && curl -fsSL https://github.com/crosstool-ng/crosstool-ng/releases/download/crosstool-ng-1.26.0/crosstool-ng-1.26.0.tar.xz.sig > crosstool.sig \ + && gpg --verify crosstool.sig crosstool.tar.xz \ + && tar -xf crosstool.tar.xz -C crosstool --strip-components=1 \ + && rm crosstool.sig crosstool.tar.xz \ + && (cd crosstool && ./configure && make install) \ + && rm -rf crosstool + +COPY crosstool-$ARCH_GCC.defconfig ./ + +RUN DEFCONFIG=crosstool-$ARCH_GCC.defconfig ct-ng defconfig \ + && rm crosstool-$ARCH_GCC.defconfig \ + && ct-ng build + +# Stage 3: Build a full CI Builder image that imports the cross-compiling +# toolchain and can be used for any CI job. FROM ci-builder-min as ci-builder-full ARG ARCH_GCC @@ -116,6 +180,10 @@ ARG ARCH_GO WORKDIR /workdir +# Import the cross toolchain. + +COPY --from=crosstool /opt/x-tools /opt/x-tools + # Install dependencies needed by any CI job. Not all of these are available in # the Ubuntu repositories. RUN apt-get update --fix-missing && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ @@ -136,9 +204,9 @@ RUN apt-get update --fix-missing && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-ge jq \ lcov \ libc-dbg \ - libclang-common-18-dev \ + libclang-common-20-dev \ libclang-dev \ - libclang-rt-18-dev \ + libclang-rt-20-dev \ libpq-dev \ lld \ llvm \ @@ -238,6 +306,11 @@ RUN mkdir rust \ && cargo install --root /usr/local --version "=0.3.6" --locked cargo-binutils \ && cargo install --root /usr/local --version "=0.13.1" --locked wasm-pack +# Link the system lld into the cross-compiling sysroot. + +RUN ln -s /usr/bin/lld /opt/x-tools/$ARCH_GCC-unknown-linux-gnu/bin/$ARCH_GCC-unknown-linux-gnu-ld.lld \ + && ln -s /usr/bin/lld /opt/x-tools/$ARCH_GCC-unknown-linux-gnu/bin/$ARCH_GCC-unknown-linux-gnu-lld + # Shims for sanitizers COPY sanshim/$ARCH_GCC /sanshim @@ -347,11 +420,27 @@ RUN curl -fsSL https://amazon-inspector-sbomgen.s3.amazonaws.com/1.8.1/linux/$AR # trustworthy on the first connection. COPY ssh_known_hosts /etc/ssh/ssh_known_hosts +# Set the necessary environment variables to point Cargo and C/C++ build systems +# at our cross-compiling toolchain. + +ENV AR=$ARCH_GCC-unknown-linux-gnu-ar +ENV LD=$ARCH_GCC-unknown-linux-gnu-lld +ENV RANLIB=$ARCH_GCC-unknown-linux-gnu-ranlib +ENV CPP=$ARCH_GCC-unknown-linux-gnu-cpp +ENV CC=$ARCH_GCC-unknown-linux-gnu-cc +ENV CXX=$ARCH_GCC-unknown-linux-gnu-c++ +ENV CXXSTDLIB=static=stdc++ ENV LDFLAGS="-fuse-ld=lld -static-libstdc++" -ENV RUSTFLAGS="-Clink-arg=-Wl,--compress-debug-sections=zlib -Clink-arg=-Wl,-O3 -Clink-arg=-fuse-ld=lld -Csymbol-mangling-version=v0 -Ctarget-cpu=$RUST_CPU_TARGET -Ctarget-feature=$RUST_TARGET_FEATURES --cfg=tokio_unstable" -ENV PATH=/opt/google-cloud-sdk/bin:$PATH +ENV RUSTFLAGS="-Clink-arg=-Wl,--compress-debug-sections=zlib -Clink-arg=-Wl,-O3 -Clink-arg=-fuse-ld=lld -L/opt/x-tools/$ARCH_GCC-unknown-linux-gnu/$ARCH_GCC-unknown-linux-gnu/sysroot/lib/ -Csymbol-mangling-version=v0 --cfg=tokio_unstable" +ENV TARGET_AR=$AR +ENV TARGET_CC=$CC +ENV TARGET_CXX=$CXX +ENV TARGET_CXXSTDLIB=static=stdc++ +ENV TARGET_RANLIB=$RANLIB +ENV PATH=/opt/google-cloud-sdk/bin:/opt/x-tools/$ARCH_GCC-unknown-linux-gnu/bin:$PATH +ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-unknown-linux-gnu-cc +ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-unknown-linux-gnu-cc ENV CARGO_TARGET_DIR=/mnt/build -ENV CARGO_INCREMENTAL=1 ENV HELM_PLUGINS=/usr/local/share/helm/plugins # Set up for a persistent volume to hold Cargo metadata, so that crate metadata diff --git a/ci/test/cargo-test/mzcompose.py b/ci/test/cargo-test/mzcompose.py index 248aebe7ab09e..3cca613001ab2 100644 --- a/ci/test/cargo-test/mzcompose.py +++ b/ci/test/cargo-test/mzcompose.py @@ -191,7 +191,7 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: "CXXSTDLIB": "stdc++", "CC": "cc", "CXX": "c++", - "CPP": "clang-cpp-18", + "CPP": "clang-cpp-20", "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER": "cc", "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER": "cc", "PATH": f"/sanshim:/opt/x-tools/{target(Arch.host())}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", diff --git a/misc/python/materialize/mzbuild.py b/misc/python/materialize/mzbuild.py index 56b576e26a160..adcfae23dd122 100644 --- a/misc/python/materialize/mzbuild.py +++ b/misc/python/materialize/mzbuild.py @@ -384,38 +384,35 @@ def generate_cargo_build_command( else ( rustc_flags.sanitizer[rd.sanitizer] if rd.sanitizer != Sanitizer.none - else ["--cfg=tokio_unstable"] + else ["--cfg=tokio_unstable", "-Clink-arg=-fuse-ld=lld", "-Clinker=clang++", "-Clinker-plugin-lto"] ) ) - cflags = ( - [ - f"--target={target(rd.arch)}", - f"--gcc-toolchain=/opt/x-tools/{target(rd.arch)}/", - "-fuse-ld=lld", - f"--sysroot=/opt/x-tools/{target(rd.arch)}/{target(rd.arch)}/sysroot", - f"-L/opt/x-tools/{target(rd.arch)}/{target(rd.arch)}/lib64", - ] - + rustc_flags.sanitizer_cflags[rd.sanitizer] - if rd.sanitizer != Sanitizer.none - else [] - ) - extra_env = ( - { - "CFLAGS": " ".join(cflags), - "CXXFLAGS": " ".join(cflags), - "LDFLAGS": " ".join(cflags), - "CXXSTDLIB": "stdc++", - "CC": "cc", - "CXX": "c++", - "CPP": "clang-cpp-18", - "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER": "cc", - "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER": "cc", - "PATH": f"/sanshim:/opt/x-tools/{target(rd.arch)}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", - "TSAN_OPTIONS": "report_bugs=0", # build-scripts fail - } - if rd.sanitizer != Sanitizer.none - else {} - ) + cflags = [ + f"--target={target(rd.arch)}", + f"--gcc-toolchain=/opt/x-tools/{target(rd.arch)}/", + "-flto=thin", + "-fuse-ld=lld", + f"--sysroot=/opt/x-tools/{target(rd.arch)}/{target(rd.arch)}/sysroot", + f"-L/opt/x-tools/{target(rd.arch)}/{target(rd.arch)}/lib64", + "-O3", + ] + if rd.sanitizer != Sanitizer.none: + cflags += rustc_flags.sanitizer_cflags[rd.sanitizer] + + extra_env = { + "CFLAGS": " ".join(cflags), + "CXXFLAGS": " ".join(cflags), + "LDFLAGS": " ".join(cflags), + "CXXSTDLIB": "stdc++", + "CC": "/sanshim/cc", + "CXX": "/sanshim/c++", + "CPP": "clang-cpp-20", + "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER": "/sanshim/cc", + "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER": "/sanshim/cc", + "PATH": f"/sanshim:/opt/x-tools/{target(rd.arch)}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + } + if rd.sanitizer != Sanitizer.none: + extra_env["TSAN_OPTIONS"] = "report_bugs=0" # build-scripts fail cargo_build = rd.build( "build", channel=None, rustflags=rustflags, extra_env=extra_env diff --git a/misc/python/materialize/rustc_flags.py b/misc/python/materialize/rustc_flags.py index 6353f83d3b68a..1c70b05465c25 100644 --- a/misc/python/materialize/rustc_flags.py +++ b/misc/python/materialize/rustc_flags.py @@ -62,30 +62,35 @@ def __str__(self) -> str: "-Cdebug-assertions=on", "-Clink-arg=-fuse-ld=lld", # access beyond end of merged section "-Clinker=clang++", + "--cfg=tokio_unstable", ], Sanitizer.hwaddress: [ "-Zsanitizer=hwaddress", "-Ctarget-feature=+tagged-globals", "-Clink-arg=-fuse-ld=lld", # access beyond end of merged section "-Clinker=clang++", + "--cfg=tokio_unstable", ], Sanitizer.cfi: [ "-Zsanitizer=cfi", "-Clto", # error: `-Zsanitizer=cfi` requires `-Clto` or `-Clinker-plugin-lto` "-Clink-arg=-fuse-ld=lld", # access beyond end of merged section "-Clinker=clang++", + "--cfg=tokio_unstable", ], Sanitizer.thread: [ "-Zsanitizer=thread", "-Clink-arg=-fuse-ld=lld", # access beyond end of merged section "-Clinker=clang++", + "--cfg=tokio_unstable", ], Sanitizer.leak: [ "-Zsanitizer=leak", "-Clink-arg=-fuse-ld=lld", # access beyond end of merged section "-Clinker=clang++", + "--cfg=tokio_unstable", ], - Sanitizer.undefined: ["-Clink-arg=-fsanitize=undefined", "-Clinker=clang++"], + Sanitizer.undefined: ["-Clink-arg=-fsanitize=undefined", "-Clinker=clang++", "--cfg=tokio_unstable"], } sanitizer_cflags = { diff --git a/src/ore/src/env.rs b/src/ore/src/env.rs index 100cb3b7a8383..4edc1cabe7e14 100644 --- a/src/ore/src/env.rs +++ b/src/ore/src/env.rs @@ -32,3 +32,7 @@ where Some(val) => val != "0" && val != "false" && val != "", } } + + + +