protobuf upgrade

README.md

@@ -52,7 +52,7 @@ Detailed instructions are provided below.
 ```
 python >= 3.8
 gcc >= 6.4
-protobuf >= 4.21.12
+protobuf >= 4.25.1
 cmake >= 3.13.4
 make >= 4.2.1 or ninja >= 1.10.2
 java >= 1.11 (optional)

docker/Dockerfile.llvm-project

@@ -30,10 +30,10 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"]
           apt-get install -qq -y --no-install-recommends \
                   autoconf automake ca-certificates clang cmake cppcheck \
                   curl default-jdk-headless gcc g++ git libncurses-dev \
-                  libtool make maven ninja-build openjdk-11-jdk-headless \
+                  libtool make maven ninja-build openjdk-21-jdk-headless \
                   python3 python3-dev python3-distutils python3-numpy \
                   python3-pip python3-pytest-xdist python3-setuptools \
-                  python3-typing-extensions zlib1g-dev && \
+                  python3-typing-extensions zlib1g-dev unzip zip && \
           rm -rf /var/lib/apt/lists/* && \
           ln -sf /usr/bin/pytest-3 /usr/bin/pytest; \
        elif [ "${distro}" = "rhel" ] || [ "${distro}" = "fedora" ]; then \
@@ -45,10 +45,10 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"]
           yum update -q -y && \
           yum install -q -y \
               autoconf automake ca-certificates clang cmake diffutils \
-              file java-11-openjdk-devel java-11-openjdk-headless \
+              file java-21-openjdk-devel java-21-openjdk-headless \
               gcc gcc-c++ git libtool make ncurses-devel ninja-build \
               python39 python39-devel python39-pip python39-setuptools \
-              python39-wheel tzdata-java zlib-devel && \
+              python39-wheel tzdata-java zlib-devel unzip zip which && \
 	  # Workaround broken ubi8 amd64 image installing python3.12 as
 	  # dependency of clang, which also breaks the /usr/bin/pip3
 	  # symlink creation
@@ -63,14 +63,106 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"]
     && ln -sf /usr/bin/python3 /usr/bin/python \
     && pip3 install --upgrade pip
 
-# Install protobuf
-ARG PROTOBUF_VERSION=21.12
-RUN git clone -b v${PROTOBUF_VERSION} --recursive https://github.com/protocolbuffers/protobuf.git \
-    && cd protobuf && ./autogen.sh \
-    && ./configure --enable-static=no \
-    && make -j${NPROC} install && ldconfig \
-    && cd python && python3 setup.py install --cpp_implementation \
-    && cd ../.. && rm -rf protobuf
+# # --- Install Bazelisk (Bazel version manager) --- doesn't work on s390x and ppc64le
+# RUN curl -L -o /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/bazelisk-linux-amd64 && \
+#     chmod +x /usr/local/bin/bazel && \
+#     echo "7.0.0" > ~/.bazelversion && \
+#     bazel --version
+
+
+# # --- Build and install protobuf using Bazel with Bzlmod ---
+# ARG PROTOBUF_VERSION=25.1
+# ARG INSTALL_PROTOBUF_PATH=/usr/local
+# ARG BUILD_TYPE=Release
+# ARG CORE_NUMBER=2
+# # ARG ORIGINAL_PATH=${pwd}
+
+# # Install dependencies
+# #RUN yum install -y wget git gcc-c++ make ninja-build python3 python3-pip cmake python3-devel
+
+# # Download and extract Abseil and Protobuf sources
+# RUN git clone --recurse-submodules -b v25.1 https://github.com/protocolbuffers/protobuf.git /workdir/protobuf-25.1
+# #RUN cd / && \
+#    # wget https://github.com/abseil/abseil-cpp/releases/download/20230802.2/abseil-cpp-20230802.2.tar.gz && \
+#    # tar -xvf abseil-cpp-20230802.2.tar.gz && \
+#    # wget https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz && \
+#    # tar -xvf protobuf-${PROTOBUF_VERSION}.tar.gz
+
+# # Build and install Protobuf (static libs, no tests)
+# RUN cd /workdir/protobuf-${PROTOBUF_VERSION} && \
+#     mkdir build_source && cd build_source && \
+#     cmake -G Ninja ../ \
+#           -DBUILD_SHARED_LIBS=OFF \
+#           -DCMAKE_INSTALL_PREFIX=$INSTALL_PROTOBUF_PATH \
+#           -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+#           -Dprotobuf_BUILD_TESTS=OFF \
+#           -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
+#           # -DABSL_ROOT_DIR="${ORIGINAL_PATH}/abseil-cpp-20230802.2"\
+#           -DCMAKE_CXX_STANDARD=17 \
+#           -DABSL_PROPAGATE_CXX_STD=on \
+#     && cmake --build . --target install --parallel $CORE_NUMBER
+
+# # RUN export PATH=$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$INSTALL_PROTOBUF_PATH/bin:$PATH && \
+# #     export LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" && \
+# #     export CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include"
+
+# # Set environment variables for protobuf
+# ENV PATH=${INSTALL_PROTOBUF_PATH}/bin:$PATH
+# ENV LD_LIBRARY_PATH=${INSTALL_PROTOBUF_PATH}/lib:$LD_LIBRARY_PATH
+# ENV CPATH=${INSTALL_PROTOBUF_PATH}/include:$CPATH
+# ENV LIBRARY_PATH=${INSTALL_PROTOBUF_PATH}/lib:$LIBRARY_PATH
+
+# # Build and install Python protobuf bindings with cpp implementation support
+# RUN cd /workdir/protobuf-${PROTOBUF_VERSION}/python && \
+#     python3 setup.py build --cpp_implementation && \
+#     python3 setup.py install && \
+#     python3 -m pip install --upgrade pip setuptools wheel && \
+#     PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp python3 setup.py bdist_wheel && \
+#     pip3 install dist/*.whl
+
+# # Verify versions
+# #RUN protoc --version && python3 -c "import google.protobuf; print(google.protobuf.__version__)"
+# RUN protoc --version && \
+#     python3 -c "from google.protobuf.internal import api_implementation; \
+#     print('protobuf version:', api_implementation.Version()); \
+#     print('cpp_implementation:', api_implementation.Type())
+
+# Install bazel 
+RUN mkdir /usr/local/bazel && cd /usr/local/bazel && \
+    curl -LO https://github.com/bazelbuild/bazel/releases/download/7.2.0/bazel-7.2.0-dist.zip  && \
+    unzip bazel-7.2.0-dist.zip &&  chmod -R +w . && \
+    EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk" EXTRA_BAZEL_ARGS="--tool_java_runtime_version=local_jdk" bash ./compile.sh && \
+    cp output/bazel /usr/local/bin/
+RUN bazel --version
+
+# Install rust, cargo, and cargo-bazel
+ARG RUST_VERSION=1.88
+ARG CARGO_BAZEL_VERSION=0.16.0
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
+    sh -s -- -y --default-toolchain none \
+    && . "${HOME}/.cargo/env" \
+    && rustup install ${RUST_VERSION} \
+    && cargo install cargo-bazel --version ${CARGO_BAZEL_VERSION}
+#ENV PATH="${HOME}/.cargo/bin:${PATH}"
+
+# Build and install protobuf using bazel
+WORKDIR /workdir
+RUN git clone --recursive https://github.com/protocolbuffers/protobuf.git
+RUN cd /workdir/protobuf && git checkout v32.0 && git submodule update --init --recursive
+RUN export CARGO_BAZEL_GENERATOR_URL=file:///root/.cargo/bin/cargo-bazel \
+    && export CARGO_BAZEL_REPIN=true \
+    && CC=clang CXX=clang++ && cd /workdir/protobuf \ 
+    #&& CC=clang CXX=clang++ bazel fetch //python/dist:binary_wheel \
+    && sed -i -e 's/\["arm64", "amd64"\]/\["arm64", "amd64", "s390x"\]/g' \
+           ${HOME}/.cache/bazel/_bazel_root/*/external/rules_buf/buf/internal/toolchain.bzl \
+    && CC=clang CXX=clang++ bazel build --enable_bzlmod :protobuf :protoc
+#RUN cd /workdir/protobuf && bazel build :protoc :protobuf
+RUN cp /workdir/protobuf/bazel-bin/protoc /usr/local/bin && \
+    cp -r /workdir/protobuf/bazel-bin/external/com_google_protobuf/python/google /usr/local/lib/python3.9/site-packages/google && \
+    cp -r /workdir/protobuf/bazel-bin/external/com_google_protobuf/libprotobuf.so* /usr/local/lib && \
+    ldconfig 
+
+RUN cd /workdir/protobuf && bazel build //python/dist:binary_wheel
 
 # Install jsoniter
 ARG JSONITER_VERSION=0.9.23

docs/BuildOnWindows.md

@@ -15,11 +15,12 @@ Build protobuf as a static library.
 
 [same-as-file]: <> (utils/install-protobuf.cmd)
 ```shell
-REM Check out protobuf v21.12
-set protobuf_version=21.12
+REM Check out protobuf v4.25.1
+set protobuf_version=4.25.1
 git clone -b v%protobuf_version% --recursive https://github.com/protocolbuffers/protobuf.git
 
 set root_dir=%cd%
+git submodule update --init --recursive
 md protobuf_build
 cd protobuf_build
 call cmake %root_dir%\protobuf\cmake -G "Ninja" ^
@@ -42,7 +43,7 @@ set PATH=%root_dir%\protobuf_install\bin;%PATH%
 
 If you wish to be able to run all the ONNX-MLIR tests, you will also need to install the matching version of protobuf through pip. Note that this is included in the requirements.txt file at the root of onnx-mlir, so if you plan on using it, you won't need to explicitly install protobuf.
 ```shell
-python3 -m pip install protobuf==4.21.12
+python3 -m pip install protobuf==4.25.1
 ```
 
 #### MLIR

docs/Prerequisite.md

@@ -6,7 +6,7 @@
 ```
 python >= 3.8
 gcc >= 6.4
-protobuf >= 4.21.12
+protobuf >= 4.25.1
 cmake >= 3.13.4
 make >= 4.2.1 or ninja >= 1.10.2
 java >= 1.11 (optional)

requirements.txt

@@ -3,6 +3,6 @@ lit~=15.0
 # and np.int which are used heavily in onnx-mlir.
 numpy==2.0.1
 onnx==1.17.0
-protobuf==4.21.12
+protobuf==4.25.1
 pytest==8.3.2
 pytest-xdist==3.6.1

utils/cppcheck_scan/cppcheck_suppressions.txt

@@ -15,9 +15,16 @@ preprocessorErrorDirective:/workdir/onnx-mlir/build/third_party/onnx/onnx/onnx-o
 #            <location file="/workdir/llvm-project/llvm/include/llvm-c/DataTypes.h" line="38" column="0"/>
 #        </error>
 preprocessorErrorDirective:/workdir/llvm-project/llvm/include/llvm-c/DataTypes.h:38
+# Build image has a newer protobuf than onnx-mlir source. This causes preprocessor issues when scanning with cppcheck.
+#        <error id="preprocessorErrorDirective" severity="error" msg="#error &quot;This file was generated by a newer version of protoc which is&quot;" verbose="#error &quot;This file was generated by a newer version of protoc which is&quot;">
+#            <location file="/workdir/onnx-mlir/build/third_party/onnx/onnx/onnx-data.pb.h" line="15" column="0"/>
+#        </error>
+preprocessorErrorDirective:/workdir/onnx-mlir/build/third_party/onnx/onnx/onnx-data.pb.h:15
+preprocessorErrorDirective:/workdir/onnx-mlir/build/third_party/onnx/onnx/onnx-ml.pb.h:15
+preprocessorErrorDirective:/workdir/onnx-mlir/build/third_party/onnx/onnx/onnx-operators-ml.pb.h:15
 
 # There is an unknown macro here somewhere. Configuration is required. If LLVM_DEPRECATED is a macro then please configure it.
 # <error id="unknownMacro" severity="error" msg="There is an unknown macro here somewhere. Configuration is required. If LLVM_DEPRECATED is a macro then please configure it." verbose="There is an unknown macro here somewhere. Configuration is required. If LLVM_DEPRECATED is a macro then please configure it." file0="/workdir/onnx-mlir/third_party/stablehlo/stablehlo/integrations/cpp/builder/AttrTypeBuilderUtilTest.cpp">
 #            <location file="/workdir/llvm-project/mlir/include/mlir/IR/OperationSupport.h" line="805" column="3"/>
 #        </error>
-unknownMacro:/workdir/llvm-project/mlir/include/mlir/IR/OperationSupport.h:805 
+unknownMacro:/workdir/llvm-project/mlir/include/mlir/IR/OperationSupport.h:805 

utils/install-protobuf.cmd

@@ -1,8 +1,9 @@
-REM Check out protobuf v21.12
-set protobuf_version=21.12
+REM Check out protobuf v4.25.1
+set protobuf_version=4.25.1
 git clone -b v%protobuf_version% --recursive https://github.com/protocolbuffers/protobuf.git
 
 set root_dir=%cd%
+git submodule update --init --recursive
 md protobuf_build
 cd protobuf_build
 call cmake %root_dir%\protobuf\cmake -G "Ninja" ^

utils/install-protobuf.sh

@@ -1,10 +1,34 @@
-# Check out protobuf
-PROTOBUF_VERSION=21.12
-git clone -b v${PROTOBUF_VERSION} --depth 1 --recursive https://github.com/protocolbuffers/protobuf.git
+# Exit immediately if a command exits with a non-zero status.
+set -e
 
-cd protobuf
-./autogen.sh
-./configure --enable-static=no
-make -j$(sysctl -n hw.logicalcpu) install
-cd python
-python3 setup.py install --cpp_implementation
+export INSTALL_PROTOBUF_PATH=/usr/local # Changed to a dedicated install directory
+export BUILD_TYPE=Release
+export CORE_NUMBER=1
+
+# Build protobuf from source with -fPIC on Unix-like system
+ORIGINAL_PATH=$(pwd)
+cd ..
+wget https://github.com/abseil/abseil-cpp/releases/download/20230802.2/abseil-cpp-20230802.2.tar.gz
+tar -xvf abseil-cpp-20230802.2.tar.gz
+
+wget https://github.com/protocolbuffers/protobuf/releases/download/v25.1/protobuf-25.1.tar.gz
+tar -xvf protobuf-25.1.tar.gz
+cd protobuf-25.1
+mkdir build_source && cd build_source
+cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=$INSTALL_PROTOBUF_PATH -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DABSL_ROOT_DIR="${ORIGINAL_PATH}/../abseil-cpp-20230802.2" -DCMAKE_CXX_STANDARD=17 -DABSL_PROPAGATE_CXX_STD=on ..
+if [ "$INSTALL_PROTOBUF_PATH" == "/usr" ]; then
+    # Don't use sudo for root
+    if [[ "$(id -u)" == "0" ]]; then
+      cmake --build . --target install --parallel $CORE_NUMBER
+    else
+      # install Protobuf on default system path so it needs sudo permission
+      sudo cmake --build . --target install --parallel $CORE_NUMBER
+    fi
+else
+    cmake --build . --target install --parallel $CORE_NUMBER
+    export PATH=$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$INSTALL_PROTOBUF_PATH/bin:$PATH
+    export LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib"
+    export CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include"
+fi
+protoc --version
+cd $ORIGINAL_PATH