Merge branch 'dev' into fix/CORE-627

.bazelrc

@@ -9,7 +9,7 @@ common:clang-19 --extra_toolchains=@llvm_19_toolchain//:all
 
 build:system-clang --extra_toolchains=@local_config_cc_toolchains//:all
 build:system-clang --action_env=BAZEL_COMPILER=clang
-build:system-clang --cxxopt=-std=c++20 --host_cxxopt=-std=c++20
+build:system-clang --cxxopt=-std=c++23 --host_cxxopt=-std=c++23
 build:system-clang --linkopt -fuse-ld=lld
 # use a compiler name that doesn't symlink to ccache
 build:system-clang-18 --config=system-clang
@@ -89,11 +89,28 @@ build:secure --config=relro
 build:release --compilation_mode opt
 build:release --config=secure
 build:release --copt -mllvm --copt -inline-threshold=2500
-build:release --linkopt=-flto
 build:release --@seastar//:stack_guards=False
+build:release --//src/v/redpanda:lto=True
+build:release --copt -flto=thin --copt -ffat-lto-objects
 
 build:stamp --stamp --workspace_status_command=./bazel/stamp_vars.sh
 
+# =================================
+# Coverage
+# =================================
+coverage --config=coverage
+build:coverage --action_env=BAZEL_USE_LLVM_NATIVE_COVERAGE=1
+build:coverage --action_env=GCOV=llvm-profdata
+build:coverage --copt=-DNDEBUG
+build:coverage --define=dynamic_link_tests=true
+build:coverage --combined_report=lcov
+build:coverage --experimental_use_llvm_covmap
+build:coverage --experimental_generate_llvm_lcov
+build:coverage --experimental_split_coverage_postprocessing
+build:coverage --experimental_fetch_all_coverage_outputs
+build:coverage --collect_code_coverage
+build:coverage --instrumentation_filter="^//src/v[/:]"
+
 # =================================
 # Testing
 # =================================

.bazelversion

@@ -1 +1 @@
-7.4.0
+7.4.1

.github/workflows/jira_issue_manage.yml

@@ -16,6 +16,7 @@ jobs:
     permissions:
       id-token: write
       contents: read
+      issues: write
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5

.github/workflows/promote.yml

@@ -19,7 +19,7 @@ jobs:
           secret-ids: |
             ,sdlc/prod/github/buildkite_token
           parse-json-secrets: true
-      - uses: buildkite/trigger-pipeline-action@v2
+      - uses: buildkite/trigger-pipeline-action@v2.0.0
         with:
           buildkite_api_access_token: ${{ env.BUILDKITE_TOKEN }}
           pipeline: "redpanda/redpanda"

.github/workflows/render-pr-body-release-notes.yml

@@ -22,7 +22,7 @@ jobs:
           curl -s -S -f -L -o rpchangelog/requirements.txt https://vectorized-public.s3.us-west-2.amazonaws.com/rpchangelog/requirements.txt
           curl -s -S -f -L -o rpchangelog/rpchangelog.py https://vectorized-public.s3.us-west-2.amazonaws.com/rpchangelog/rpchangelog.py
           chmod +x rpchangelog/rpchangelog.py
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
           cache: 'pip'

.github/workflows/rpk-build.yml

@@ -13,10 +13,12 @@ on:
     branches: [dev]
     paths:
       - 'src/go/rpk/**'
+      - 'src/v/pandaproxy/schema_registry/protobuf/**'
       - '.github/workflows/rpk-build.yml'
   pull_request:
     paths:
       - 'src/go/rpk/**'
+      - 'src/v/pandaproxy/schema_registry/protobuf/**'
       - '.github/workflows/rpk-build.yml'
 jobs:
   test:

CMakeLists.txt

@@ -12,7 +12,7 @@ project(redpanda LANGUAGES CXX)
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
 
 set(CMAKE_C_STANDARD 11)
-set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD 23)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)

MODULE.bazel

@@ -41,7 +41,16 @@ bazel_dep(name = "googletest", version = "1.15.2")
 bazel_dep(name = "liburing", version = "2.5")
 bazel_dep(name = "lz4", version = "1.9.4")
 bazel_dep(name = "platforms", version = "0.0.10")
-bazel_dep(name = "protobuf", version = "27.3")
+bazel_dep(name = "protobuf", version = "29.0")
+single_version_override(
+    module_name = "protobuf",
+    patch_strip = 1,
+    # this patch is in upstream protobuf, so should be in an upcoming release.
+    patches = [
+        "//bazel/thirdparty:protobuf_capture_warning.patch",
+    ],
+)
+
 bazel_dep(name = "re2", version = "2024-07-02")
 bazel_dep(name = "rules_foreign_cc", version = "0.12.0")
 bazel_dep(name = "rules_go", version = "0.50.1")
@@ -114,7 +123,7 @@ single_version_override(
 # NOTE: We build our toolchains on ubuntu:jammy so you're going to need a distro at least that old.
 llvm = use_extension("@toolchains_llvm//toolchain/extensions:llvm.bzl", "llvm")
 
-CXX_STANDARD = "c++20"
+CXX_STANDARD = "c++23"
 
 llvm.toolchain(
     name = "llvm_19_toolchain",

bazel/build.bzl

@@ -51,6 +51,7 @@ def redpanda_cc_binary(
         local_defines = [],
         visibility = None,
         copts = [],
+        linkopts = [],
         deps = []):
     """
     Define a Redpanda C++ binary.
@@ -64,6 +65,7 @@ def redpanda_cc_binary(
         visibility = visibility,
         deps = deps,
         copts = redpanda_copts() + copts,
+        linkopts = linkopts,
         features = [
             "layering_check",
         ],

bazel/cert.bzl

@@ -0,0 +1,161 @@
+"""
+This module contains functions to generate a simple CA
+"""
+
+load("@bazel_skylib//rules:run_binary.bzl", "run_binary")
+
+openssl_env = {
+    "OPENSSL_CONF": "$(execpath @openssl//:openssl_data)/openssl.cnf",
+}
+
+# buildifier: disable=function-docstring-args
+def _redpanda_private_key(name, certificate):
+    private_key = certificate + ".key"
+
+    run_binary(
+        name = name + "_key_gen",
+        srcs = [
+            "@openssl//:openssl_data",
+        ],
+        outs = [private_key],
+        args = [
+            "ecparam",
+            "-name",
+            "prime256v1",
+            "-genkey",
+            "-noout",
+            "-out",
+            "$(execpath :{})".format(private_key),
+        ],
+        env = openssl_env,
+        tool = "@openssl//:openssl_exe",
+    )
+
+    return private_key
+
+def redpanda_selfsigned_cert(name, certificate, common_name, visibility = None):
+    """
+    Generate a Redpanda self-signed certificate.
+
+    Args:
+      name: name of the target
+      certificate: name to use for output files (crt, key, and csr)
+      common_name: the CN to use when setting the subject name
+      visibility: visibility setting
+    """
+
+    cert = certificate + ".crt"
+    subj = "/C=US/ST=California/L=San Francisco/O=Redpanda Data/OU=Core/CN=" + common_name
+
+    private_key = _redpanda_private_key(name, certificate)
+
+    run_binary(
+        name = name + "_crt_gen",
+        srcs = [
+            private_key,
+            "@openssl//:openssl_data",
+        ],
+        outs = [cert],
+        args = [
+            "req",
+            "-new",
+            "-x509",
+            "-sha256",
+            "-key",
+            "$(execpath :{})".format(private_key),
+            "-out",
+            "$(execpath :{})".format(cert),
+            "-subj",
+            subj,
+            "-addext",
+            "subjectAltName = IP:127.0.0.1",
+        ],
+        env = openssl_env,
+        tool = "@openssl//:openssl_exe",
+    )
+
+    native.filegroup(
+        name = name,
+        srcs = [private_key, cert],
+        visibility = visibility,
+    )
+
+def redpanda_signed_cert(name, certificate, common_name, ca, serial_number, visibility = None):
+    """
+    Generate a Redpanda signed certificate.
+
+    Args:
+      name: name of the target
+      certificate: name to use for output files (crt, key, and csr)
+      common_name: the CN to use when setting the subject name
+      ca: the certificate to be used as the signing CA
+      serial_number: the serial number of cert when issued by CA
+      visibility: visibility setting
+    """
+
+    subj = "/C=US/ST=California/L=San Francisco/O=Redpanda Data/OU=Core/CN=" + common_name
+
+    private_key = _redpanda_private_key(name, certificate)
+    csr = certificate + ".csr"
+    cert = certificate + ".crt"
+
+    run_binary(
+        name = name + "_csr_gen",
+        srcs = [
+            private_key,
+            "@openssl//:openssl_data",
+        ],
+        outs = [csr],
+        args = [
+            "req",
+            "-new",
+            "-sha256",
+            "-key",
+            "$(execpath :{})".format(private_key),
+            "-out",
+            "$(execpath :{})".format(csr),
+            "-subj",
+            subj,
+        ],
+        env = openssl_env,
+        tool = "@openssl//:openssl_exe",
+    )
+
+    ca_cert = ca + ".crt"
+    ca_private_key = ca + ".key"
+
+    run_binary(
+        name = name + "_crt_gen",
+        srcs = [
+            ca_cert,
+            ca_private_key,
+            csr,
+            "@openssl//:openssl_data",
+        ],
+        outs = [cert],
+        args = [
+            "x509",
+            "-req",
+            "-days",
+            "1000",
+            "-sha256",
+            "-set_serial",
+            "{}".format(serial_number),
+            "-in",
+            "$(execpath :{})".format(csr),
+            "-CA",
+            "$(execpaths :{})".format(ca_cert),
+            "-CAkey",
+            "$(execpaths :{})".format(ca_private_key),
+            "-out",
+            "$(execpath :{})".format(cert),
+        ],
+        env = openssl_env,
+        tool = "@openssl//:openssl_exe",
+    )
+
+    native.filegroup(
+        name = name,
+        srcs = [private_key, csr, cert],
+        visibility = visibility,
+    )

bazel/repositories.bzl

@@ -167,9 +167,9 @@ def data_dependency():
     http_archive(
         name = "seastar",
         build_file = "//bazel/thirdparty:seastar.BUILD",
-        sha256 = "8b71096e8c57a5a1a447638fc64b05686ecd7f079861de7a3fe3112fd0e4d6f7",
-        strip_prefix = "seastar-ef24a8bc3f7dca212dfe982911bb726e5d37cef5",
-        url = "https://github.com/redpanda-data/seastar/archive/ef24a8bc3f7dca212dfe982911bb726e5d37cef5.tar.gz",
+        sha256 = "254bd1c2d26bb2d9df8e5217e1c50a2f08bce58b0a6d13cdc692192ee8b79a56",
+        strip_prefix = "seastar-f194d34ef62b5a64364291b941f97a9cd19b7835",
+        url = "https://github.com/redpanda-data/seastar/archive/f194d34ef62b5a64364291b941f97a9cd19b7835.tar.gz",
         patches = ["//bazel/thirdparty:seastar-fortify-source.patch"],
         patch_args = ["-p1"],
     )

bazel/test.bzl

@@ -249,7 +249,10 @@ def redpanda_cc_btest_no_seastar(
         srcs = srcs,
         defines = defines,
         copts = redpanda_copts(),
-        deps = ["@boost//:test.so"] + deps,
+        deps = [
+            "//src/v/test_utils:boost_result_redirect",
+            "@boost//:test.so",
+        ] + deps,
     )
 
 def redpanda_test_cc_library(

bazel/thirdparty/krb5.BUILD

@@ -48,6 +48,10 @@ configure_make(
         "--disable-static",
         "--enable-asan=$(SANITIZERS)",
     ],
+    # Need to pass this additionally here because of a bug in the kerberos build where it doesn't properly pass the linker flag down
+    copts = [
+        "-fuse-ld=lld",
+    ],
     env = {
         "KRB5_BUILD_JOBS": "$(BUILD_JOBS)",
     },

bazel/thirdparty/openssl.BUILD

@@ -1,5 +1,6 @@
 load("@bazel_skylib//rules:common_settings.bzl", "int_flag", "string_flag")
-load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make")
+load("@bazel_skylib//rules:select_file.bzl", "select_file")
+load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make", "runnable_binary")
 
 # Make this build faster by setting `build --@openssl//:build_jobs=16` in user.bazelrc
 # if you have the cores to spare.
@@ -55,6 +56,12 @@ configure_make(
         "OPENSSL_BUILD_JOBS": "$(BUILD_JOBS)",
     },
     lib_source = ":srcs",
+    out_binaries = [
+        "openssl",
+    ],
+    out_data_dirs = [
+        "ssl",
+    ],
     out_shared_libs = [
         "libssl.so.3",
         "libcrypto.so.3",
@@ -64,3 +71,25 @@ configure_make(
         "//visibility:public",
     ],
 )
+
+filegroup(
+    name = "gen_dir",
+    srcs = [":openssl"],
+    output_group = "gen_dir",
+)
+
+select_file(
+    name = "openssl_data",
+    srcs = ":openssl",
+    subpath = "ssl",
+    visibility = [
+        "//visibility:public",
+    ],
+)
+
+runnable_binary(
+    name = "openssl_exe",
+    binary = "openssl",
+    foreign_cc_target = ":openssl",
+    visibility = ["//visibility:public"],
+)

bazel/thirdparty/protobuf_capture_warning.patch

@@ -0,0 +1,13 @@
+diff --git a/src/google/protobuf/map.cc b/src/google/protobuf/map.cc
+index 570b61bec86f..da6ceb99d993 100644
+--- a/src/google/protobuf/map.cc
++++ b/src/google/protobuf/map.cc
+@@ -116,7 +116,7 @@ void UntypedMapBase::ClearTable(const ClearInput input) {
+   ABSL_DCHECK_NE(num_buckets_, kGlobalEmptyTableSize);
+
+   if (alloc_.arena() == nullptr) {
+-    const auto loop = [=](auto destroy_node) {
++    const auto loop = [&, this](auto destroy_node) {
+       const TableEntryPtr* table = table_;
+       for (map_index_t b = index_of_first_non_null_, end = num_buckets_;
+            b < end; ++b) {