Define a generated registry for tracking linked-in extensions.

#test-continuous

This behaves similarly to the generated extension registry in other languages.  A global ref-counted (lock-free thread-safe) singleton registry can be lazily constructed from the linker array data produced by gencode.  For now, this is simply used to preserve custom options on the descriptors in our DefPool.

This design makes heavy use of various non-standard C extensions.  It expands on the pre-existing linker arrays used for extensions, adding weak constructors to register all of the extensions linked into each binary.  This was tested and works with clang and gcc, and a similar approach was implemented for MSVC.  For unsupported compilers or architectures, the generated registry will be unpopulated.

PiperOrigin-RevId: 825343000

Author: mkruskal-google

.github/workflows/test_upb.yml

@@ -30,6 +30,8 @@ jobs:
         config:
           - { name: "Fastbuild" }
           - { name: "Optimized", flags: "-c opt", continuous-only: true }
+          - { name: "GCC Optimized", flags: "-c opt --force_pic --java_runtime_version=remotejdk_11 --copt=\"-Wno-error=maybe-uninitialized\"", image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:8.0.1-12.2-12e21b8dda91028bc14212a3ab582c7c4d149fac" }
+          - { name: "GCC Static", flags: "-c opt --dynamic_mode=off --java_runtime_version=remotejdk_11 --copt=\"-Wno-error=maybe-uninitialized\"", image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:8.0.1-12.2-12e21b8dda91028bc14212a3ab582c7c4d149fac", continuous-only: true }
           - { name: "ASAN", flags: "--config=asan -c dbg", exclude-targets: "-//benchmarks:benchmark -//python/...", runner: ubuntu-22-4core }
           - { name: "UBSAN", flags: "--config=ubsan -c dbg", exclude-targets: "-//benchmarks:benchmark -//python/... -//lua/...", continuous-only: true }
           - { name: "32-bit", flags: "--copt=-m32 --linkopt=-m32", exclude-targets: "-//benchmarks:benchmark -//python/..." }
@@ -49,33 +51,12 @@ jobs:
         if: ${{ !matrix.config.continuous-only || inputs.continuous-run }}
         uses: protocolbuffers/protobuf-ci/bazel-docker@v5
         with:
-          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize:${{ matrix.config.bazel_version || '8.0.1' }}-12e21b8dda91028bc14212a3ab582c7c4d149fac
+          image: ${{ matrix.config.image || 'us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize:8.0.1-12e21b8dda91028bc14212a3ab582c7c4d149fac' }}
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: upb-bazel
           bazel: test //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/... ${{ matrix.config.flags }}
           exclude-targets: ${{ matrix.config.exclude-targets }}
 
-  linux-gcc:
-    strategy:
-      fail-fast: false   # Don't cancel all jobs if one fails.
-    name: GCC Optimized
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout pending changes
-        uses: protocolbuffers/protobuf-ci/checkout@v5
-        with:
-          ref: ${{ inputs.safe-checkout }}
-      - name: Run tests
-        uses: protocolbuffers/protobuf-ci/bazel-docker@v5
-        with:
-          image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:8.0.1-12.2-12e21b8dda91028bc14212a3ab582c7c4d149fac"
-          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
-          bazel-cache: "upb-bazel-gcc"
-          bazel: >-
-            test -c opt
-            --copt="-Wno-error=maybe-uninitialized" --java_runtime_version=remotejdk_11
-            //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/...
-
   windows:
     strategy:
       fail-fast: false   # Don't cancel all jobs if one fails.

cmake/installed_include_golden.txt

@@ -185,12 +185,14 @@ upb/message/value.h
 upb/mini_descriptor/build_enum.h
 upb/mini_descriptor/decode.h
 upb/mini_descriptor/link.h
+upb/mini_table/compat.h
 upb/mini_table/debug_string.h
 upb/mini_table/enum.h
 upb/mini_table/extension.h
 upb/mini_table/extension_registry.h
 upb/mini_table/field.h
 upb/mini_table/file.h
+upb/mini_table/generated_registry.h
 upb/mini_table/message.h
 upb/mini_table/sub.h
 upb/port/atomic.h

pkg/BUILD.bazel

@@ -249,6 +249,7 @@ cc_dist_library(
         "//upb/json",
         "//upb/message:compare",
         "//upb/message:copy",
+        "//upb/mini_table:compat",
         "//upb/mini_table:debug_string",
         "//upb/text",
         "//upb/text:debug",

upb/BUILD

@@ -98,6 +98,7 @@ cc_library(
         "//upb/message:internal",
         "//upb/mini_descriptor",
         "//upb/mini_table",
+        "//upb/mini_table:internal",
         "//upb/wire",
     ] + select({
         ":fasttable_enabled_setting": [
@@ -308,9 +309,13 @@ filegroup(
 filegroup(
     name = "test_srcs",
     srcs = [
+        "//upb/hash:test_srcs",
         "//upb/json:test_srcs",
+        "//upb/lex:test_srcs",
         "//upb/mem:test_srcs",
         "//upb/message:test_srcs",
+        "//upb/mini_descriptor:test_srcs",
+        "//upb/mini_table:test_srcs",
         "//upb/test:test_srcs",
         "//upb/util:test_srcs",
         "//upb/wire:test_srcs",

upb/generated_code_support.h

@@ -36,6 +36,7 @@
 #include "upb/mini_table/extension_registry.h"
 #include "upb/mini_table/field.h"
 #include "upb/mini_table/file.h"
+#include "upb/mini_table/internal/generated_registry.h"
 #include "upb/mini_table/message.h"
 #include "upb/mini_table/sub.h"
 #include "upb/wire/decode.h"

upb/hash/BUILD

@@ -65,5 +65,5 @@ filegroup(
             "**/*test.cc",
         ],
     ),
-    visibility = ["//pkg:__pkg__"],
+    visibility = ["//upb:__pkg__"],
 )

upb/lex/BUILD

@@ -72,5 +72,5 @@ filegroup(
             "**/*test.cc",
         ],
     ),
-    visibility = ["//pkg:__pkg__"],
+    visibility = ["//upb:__pkg__"],
 )

upb/mini_descriptor/BUILD

@@ -105,5 +105,5 @@ filegroup(
             "**/*test.cc",
         ],
     ),
-    visibility = ["//pkg:__pkg__"],
+    visibility = ["//upb:__pkg__"],
 )

upb/mini_table/BUILD

@@ -28,6 +28,7 @@ cc_library(
     name = "mini_table",
     srcs = [
         "extension_registry.c",
+        "generated_registry.c",
         "message.c",
     ],
     hdrs = [
@@ -36,6 +37,7 @@ cc_library(
         "extension_registry.h",
         "field.h",
         "file.h",
+        "generated_registry.h",
         "message.h",
         "sub.h",
     ],
@@ -84,6 +86,7 @@ cc_library(
         "internal/extension.h",
         "internal/field.h",
         "internal/file.h",
+        "internal/generated_registry.h",
         "internal/message.h",
         "internal/size_log2.h",
         "internal/sub.h",
@@ -92,7 +95,6 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         "//upb/base",
-        "//upb/hash",
         "//upb/mem",
         "//upb/message:types",
         "//upb/port",
@@ -131,6 +133,21 @@ cc_test(
     ],
 )
 
+cc_test(
+    name = "generated_registry_test",
+    srcs = ["generated_registry_test.cc"],
+    deps = [
+        ":mini_table",
+        "//src/google/protobuf:descriptor_upb_minitable_proto",
+        "//upb/test:custom_options_upb_minitable_proto",
+        "//upb/test:editions_test_upb_minitable_proto",
+        "//upb/test:test_multiple_files_upb_minitable_proto",
+        "@abseil-cpp//absl/synchronization",
+        "@googletest//:gtest",
+        "@googletest//:gtest_main",
+    ],
+)
+
 proto_library(
     name = "message_benchmark_proto",
     testonly = 1,

upb/mini_table/extension_registry.c

@@ -15,6 +15,8 @@
 #include "upb/hash/str_table.h"
 #include "upb/mem/arena.h"
 #include "upb/mini_table/extension.h"
+#include "upb/mini_table/generated_registry.h"
+#include "upb/mini_table/internal/generated_registry.h"
 #include "upb/mini_table/message.h"
 
 // Must be last.
@@ -89,25 +91,46 @@ upb_ExtensionRegistryStatus upb_ExtensionRegistry_AddArray(
   return status;
 }
 
-#ifdef UPB_LINKARR_DECLARE
-
-UPB_LINKARR_DECLARE(upb_AllExts, const upb_MiniTableExtension);
+const UPB_PRIVATE(upb_GeneratedExtensionListEntry) *
+    UPB_PRIVATE(upb_generated_extension_list) = NULL;
 
 bool upb_ExtensionRegistry_AddAllLinkedExtensions(upb_ExtensionRegistry* r) {
-  const upb_MiniTableExtension* start = UPB_LINKARR_START(upb_AllExts);
-  const upb_MiniTableExtension* stop = UPB_LINKARR_STOP(upb_AllExts);
-  for (const upb_MiniTableExtension* p = start; p < stop; p++) {
-    // Windows can introduce zero padding, so we have to skip zeroes.
-    if (upb_MiniTableExtension_Number(p) != 0) {
-      if (upb_ExtensionRegistry_Add(r, p) != kUpb_ExtensionRegistryStatus_Ok) {
+  const UPB_PRIVATE(upb_GeneratedExtensionListEntry)* entry =
+      UPB_PRIVATE(upb_generated_extension_list);
+  while (entry != NULL) {
+    // Comparing pointers to different objects is undefined behavior, so we
+    // convert them to uintptr_t and compare their values.
+    uintptr_t begin = (uintptr_t)entry->start;
+    uintptr_t end = (uintptr_t)entry->stop;
+    uintptr_t current = begin;
+    while (current < end) {
+      const upb_MiniTableExtension* ext =
+          (const upb_MiniTableExtension*)current;
+      // Sentinels and padding introduced by the linker can result in zeroed
+      // entries, so simply skip them.
+      if (upb_MiniTableExtension_Number(ext) == 0) {
+        // MSVC introduces padding that might not be sized exactly the same as
+        // upb_MiniTableExtension, so we can't iterate by sizeof.  This is a
+        // valid thing for any linker to do, so it's safer to just always do it.
+        current += UPB_ALIGN_OF(upb_MiniTableExtension);
+        continue;
+      }
+
+      if (upb_ExtensionRegistry_Add(r, ext) !=
+          kUpb_ExtensionRegistryStatus_Ok) {
         return false;
       }
+      current += sizeof(upb_MiniTableExtension);
     }
+    entry = entry->next;
   }
   return true;
 }
 
-#endif  // UPB_LINKARR_DECLARE
+const upb_ExtensionRegistry* upb_ExtensionRegistry_GetGenerated(
+    const upb_GeneratedRegistryRef* genreg) {
+  return genreg != NULL ? genreg->registry : NULL;
+}
 
 const upb_MiniTableExtension* upb_ExtensionRegistry_Lookup(
     const upb_ExtensionRegistry* r, const upb_MiniTable* t, uint32_t num) {