diff --git a/README.md b/README.md index 93a2e32626..5be537080c 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Detailed instructions are provided below. ``` python >= 3.8 gcc >= 6.4 -protobuf >= 4.21.12 +protobuf >= 4.25.1 cmake >= 3.13.4 make >= 4.2.1 or ninja >= 1.10.2 java >= 1.11 (optional) diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project index 62f1af44e4..90915d696d 100644 --- a/docker/Dockerfile.llvm-project +++ b/docker/Dockerfile.llvm-project @@ -30,10 +30,10 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"] apt-get install -qq -y --no-install-recommends \ autoconf automake ca-certificates clang cmake cppcheck \ curl default-jdk-headless gcc g++ git libncurses-dev \ - libtool make maven ninja-build openjdk-11-jdk-headless \ + libtool make maven ninja-build openjdk-21-jdk-headless \ python3 python3-dev python3-distutils python3-numpy \ python3-pip python3-pytest-xdist python3-setuptools \ - python3-typing-extensions zlib1g-dev && \ + python3-typing-extensions zlib1g-dev unzip zip && \ rm -rf /var/lib/apt/lists/* && \ ln -sf /usr/bin/pytest-3 /usr/bin/pytest; \ elif [ "${distro}" = "rhel" ] || [ "${distro}" = "fedora" ]; then \ @@ -45,10 +45,10 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"] yum update -q -y && \ yum install -q -y \ autoconf automake ca-certificates clang cmake diffutils \ - file java-11-openjdk-devel java-11-openjdk-headless \ + file java-21-openjdk-devel java-21-openjdk-headless \ gcc gcc-c++ git libtool make ncurses-devel ninja-build \ python39 python39-devel python39-pip python39-setuptools \ - python39-wheel tzdata-java zlib-devel && \ + python39-wheel tzdata-java zlib-devel unzip zip which && \ # Workaround broken ubi8 amd64 image installing python3.12 as # dependency of clang, which also breaks the /usr/bin/pip3 # symlink creation @@ -63,14 +63,106 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"] && ln -sf /usr/bin/python3 /usr/bin/python \ && pip3 install --upgrade pip -# Install protobuf -ARG PROTOBUF_VERSION=21.12 -RUN git clone -b v${PROTOBUF_VERSION} --recursive https://github.com/protocolbuffers/protobuf.git \ - && cd protobuf && ./autogen.sh \ - && ./configure --enable-static=no \ - && make -j${NPROC} install && ldconfig \ - && cd python && python3 setup.py install --cpp_implementation \ - && cd ../.. && rm -rf protobuf +# # --- Install Bazelisk (Bazel version manager) --- doesn't work on s390x and ppc64le +# RUN curl -L -o /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/bazelisk-linux-amd64 && \ +# chmod +x /usr/local/bin/bazel && \ +# echo "7.0.0" > ~/.bazelversion && \ +# bazel --version + + +# # --- Build and install protobuf using Bazel with Bzlmod --- +# ARG PROTOBUF_VERSION=25.1 +# ARG INSTALL_PROTOBUF_PATH=/usr/local +# ARG BUILD_TYPE=Release +# ARG CORE_NUMBER=2 +# # ARG ORIGINAL_PATH=${pwd} + +# # Install dependencies +# #RUN yum install -y wget git gcc-c++ make ninja-build python3 python3-pip cmake python3-devel + +# # Download and extract Abseil and Protobuf sources +# RUN git clone --recurse-submodules -b v25.1 https://github.com/protocolbuffers/protobuf.git /workdir/protobuf-25.1 +# #RUN cd / && \ +# # wget https://github.com/abseil/abseil-cpp/releases/download/20230802.2/abseil-cpp-20230802.2.tar.gz && \ +# # tar -xvf abseil-cpp-20230802.2.tar.gz && \ +# # wget https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz && \ +# # tar -xvf protobuf-${PROTOBUF_VERSION}.tar.gz + +# # Build and install Protobuf (static libs, no tests) +# RUN cd /workdir/protobuf-${PROTOBUF_VERSION} && \ +# mkdir build_source && cd build_source && \ +# cmake -G Ninja ../ \ +# -DBUILD_SHARED_LIBS=OFF \ +# -DCMAKE_INSTALL_PREFIX=$INSTALL_PROTOBUF_PATH \ +# -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ +# -Dprotobuf_BUILD_TESTS=OFF \ +# -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ +# # -DABSL_ROOT_DIR="${ORIGINAL_PATH}/abseil-cpp-20230802.2"\ +# -DCMAKE_CXX_STANDARD=17 \ +# -DABSL_PROPAGATE_CXX_STD=on \ +# && cmake --build . --target install --parallel $CORE_NUMBER + +# # RUN export PATH=$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$INSTALL_PROTOBUF_PATH/bin:$PATH && \ +# # export LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" && \ +# # export CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include" + +# # Set environment variables for protobuf +# ENV PATH=${INSTALL_PROTOBUF_PATH}/bin:$PATH +# ENV LD_LIBRARY_PATH=${INSTALL_PROTOBUF_PATH}/lib:$LD_LIBRARY_PATH +# ENV CPATH=${INSTALL_PROTOBUF_PATH}/include:$CPATH +# ENV LIBRARY_PATH=${INSTALL_PROTOBUF_PATH}/lib:$LIBRARY_PATH + +# # Build and install Python protobuf bindings with cpp implementation support +# RUN cd /workdir/protobuf-${PROTOBUF_VERSION}/python && \ +# python3 setup.py build --cpp_implementation && \ +# python3 setup.py install && \ +# python3 -m pip install --upgrade pip setuptools wheel && \ +# PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp python3 setup.py bdist_wheel && \ +# pip3 install dist/*.whl + +# # Verify versions +# #RUN protoc --version && python3 -c "import google.protobuf; print(google.protobuf.__version__)" +# RUN protoc --version && \ +# python3 -c "from google.protobuf.internal import api_implementation; \ +# print('protobuf version:', api_implementation.Version()); \ +# print('cpp_implementation:', api_implementation.Type()) + +# Install bazel +RUN mkdir /usr/local/bazel && cd /usr/local/bazel && \ + curl -LO https://github.com/bazelbuild/bazel/releases/download/7.2.0/bazel-7.2.0-dist.zip && \ + unzip bazel-7.2.0-dist.zip && chmod -R +w . && \ + EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk" EXTRA_BAZEL_ARGS="--tool_java_runtime_version=local_jdk" bash ./compile.sh && \ + cp output/bazel /usr/local/bin/ +RUN bazel --version + +# Install rust, cargo, and cargo-bazel +ARG RUST_VERSION=1.88 +ARG CARGO_BAZEL_VERSION=0.16.0 +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- -y --default-toolchain none \ + && . "${HOME}/.cargo/env" \ + && rustup install ${RUST_VERSION} \ + && cargo install cargo-bazel --version ${CARGO_BAZEL_VERSION} +#ENV PATH="${HOME}/.cargo/bin:${PATH}" + +# Build and install protobuf using bazel +WORKDIR /workdir +RUN git clone --recursive https://github.com/protocolbuffers/protobuf.git +RUN cd /workdir/protobuf && git checkout v32.0 && git submodule update --init --recursive +RUN export CARGO_BAZEL_GENERATOR_URL=file:///root/.cargo/bin/cargo-bazel \ + && export CARGO_BAZEL_REPIN=true \ + && CC=clang CXX=clang++ && cd /workdir/protobuf \ + #&& CC=clang CXX=clang++ bazel fetch //python/dist:binary_wheel \ + && sed -i -e 's/\["arm64", "amd64"\]/\["arm64", "amd64", "s390x"\]/g' \ + ${HOME}/.cache/bazel/_bazel_root/*/external/rules_buf/buf/internal/toolchain.bzl \ + && CC=clang CXX=clang++ bazel build --enable_bzlmod :protobuf :protoc +#RUN cd /workdir/protobuf && bazel build :protoc :protobuf +RUN cp /workdir/protobuf/bazel-bin/protoc /usr/local/bin && \ + cp -r /workdir/protobuf/bazel-bin/external/com_google_protobuf/python/google /usr/local/lib/python3.9/site-packages/google && \ + cp -r /workdir/protobuf/bazel-bin/external/com_google_protobuf/libprotobuf.so* /usr/local/lib && \ + ldconfig + +RUN cd /workdir/protobuf && bazel build //python/dist:binary_wheel # Install jsoniter ARG JSONITER_VERSION=0.9.23 diff --git a/docs/BuildOnWindows.md b/docs/BuildOnWindows.md index 2bc48d3e3d..dea603def1 100644 --- a/docs/BuildOnWindows.md +++ b/docs/BuildOnWindows.md @@ -15,11 +15,12 @@ Build protobuf as a static library. [same-as-file]: <> (utils/install-protobuf.cmd) ```shell -REM Check out protobuf v21.12 -set protobuf_version=21.12 +REM Check out protobuf v4.25.1 +set protobuf_version=4.25.1 git clone -b v%protobuf_version% --recursive https://github.com/protocolbuffers/protobuf.git set root_dir=%cd% +git submodule update --init --recursive md protobuf_build cd protobuf_build call cmake %root_dir%\protobuf\cmake -G "Ninja" ^ @@ -42,7 +43,7 @@ set PATH=%root_dir%\protobuf_install\bin;%PATH% If you wish to be able to run all the ONNX-MLIR tests, you will also need to install the matching version of protobuf through pip. Note that this is included in the requirements.txt file at the root of onnx-mlir, so if you plan on using it, you won't need to explicitly install protobuf. ```shell -python3 -m pip install protobuf==4.21.12 +python3 -m pip install protobuf==4.25.1 ``` #### MLIR diff --git a/docs/Prerequisite.md b/docs/Prerequisite.md index b65fa3cce0..8717ee069f 100644 --- a/docs/Prerequisite.md +++ b/docs/Prerequisite.md @@ -6,7 +6,7 @@ ``` python >= 3.8 gcc >= 6.4 -protobuf >= 4.21.12 +protobuf >= 4.25.1 cmake >= 3.13.4 make >= 4.2.1 or ninja >= 1.10.2 java >= 1.11 (optional) diff --git a/requirements.txt b/requirements.txt index d5390063e5..a3550aa449 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,6 @@ lit~=15.0 # and np.int which are used heavily in onnx-mlir. numpy==2.0.1 onnx==1.17.0 -protobuf==4.21.12 +protobuf==4.25.1 pytest==8.3.2 pytest-xdist==3.6.1 diff --git a/utils/cppcheck_scan/cppcheck_suppressions.txt b/utils/cppcheck_scan/cppcheck_suppressions.txt index 2ead27ce6b..ed35454f4b 100644 --- a/utils/cppcheck_scan/cppcheck_suppressions.txt +++ b/utils/cppcheck_scan/cppcheck_suppressions.txt @@ -15,9 +15,16 @@ preprocessorErrorDirective:/workdir/onnx-mlir/build/third_party/onnx/onnx/onnx-o # # preprocessorErrorDirective:/workdir/llvm-project/llvm/include/llvm-c/DataTypes.h:38 +# Build image has a newer protobuf than onnx-mlir source. This causes preprocessor issues when scanning with cppcheck. +# +# +# +preprocessorErrorDirective:/workdir/onnx-mlir/build/third_party/onnx/onnx/onnx-data.pb.h:15 +preprocessorErrorDirective:/workdir/onnx-mlir/build/third_party/onnx/onnx/onnx-ml.pb.h:15 +preprocessorErrorDirective:/workdir/onnx-mlir/build/third_party/onnx/onnx/onnx-operators-ml.pb.h:15 # There is an unknown macro here somewhere. Configuration is required. If LLVM_DEPRECATED is a macro then please configure it. # # # -unknownMacro:/workdir/llvm-project/mlir/include/mlir/IR/OperationSupport.h:805 \ No newline at end of file +unknownMacro:/workdir/llvm-project/mlir/include/mlir/IR/OperationSupport.h:805 diff --git a/utils/install-protobuf.cmd b/utils/install-protobuf.cmd index 9112848113..5ce4386300 100644 --- a/utils/install-protobuf.cmd +++ b/utils/install-protobuf.cmd @@ -1,8 +1,9 @@ -REM Check out protobuf v21.12 -set protobuf_version=21.12 +REM Check out protobuf v4.25.1 +set protobuf_version=4.25.1 git clone -b v%protobuf_version% --recursive https://github.com/protocolbuffers/protobuf.git set root_dir=%cd% +git submodule update --init --recursive md protobuf_build cd protobuf_build call cmake %root_dir%\protobuf\cmake -G "Ninja" ^ diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index 70e133e48d..419a00d9a3 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -1,10 +1,34 @@ -# Check out protobuf -PROTOBUF_VERSION=21.12 -git clone -b v${PROTOBUF_VERSION} --depth 1 --recursive https://github.com/protocolbuffers/protobuf.git +# Exit immediately if a command exits with a non-zero status. +set -e -cd protobuf -./autogen.sh -./configure --enable-static=no -make -j$(sysctl -n hw.logicalcpu) install -cd python -python3 setup.py install --cpp_implementation +export INSTALL_PROTOBUF_PATH=/usr/local # Changed to a dedicated install directory +export BUILD_TYPE=Release +export CORE_NUMBER=1 + +# Build protobuf from source with -fPIC on Unix-like system +ORIGINAL_PATH=$(pwd) +cd .. +wget https://github.com/abseil/abseil-cpp/releases/download/20230802.2/abseil-cpp-20230802.2.tar.gz +tar -xvf abseil-cpp-20230802.2.tar.gz + +wget https://github.com/protocolbuffers/protobuf/releases/download/v25.1/protobuf-25.1.tar.gz +tar -xvf protobuf-25.1.tar.gz +cd protobuf-25.1 +mkdir build_source && cd build_source +cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=$INSTALL_PROTOBUF_PATH -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DABSL_ROOT_DIR="${ORIGINAL_PATH}/../abseil-cpp-20230802.2" -DCMAKE_CXX_STANDARD=17 -DABSL_PROPAGATE_CXX_STD=on .. +if [ "$INSTALL_PROTOBUF_PATH" == "/usr" ]; then + # Don't use sudo for root + if [[ "$(id -u)" == "0" ]]; then + cmake --build . --target install --parallel $CORE_NUMBER + else + # install Protobuf on default system path so it needs sudo permission + sudo cmake --build . --target install --parallel $CORE_NUMBER + fi +else + cmake --build . --target install --parallel $CORE_NUMBER + export PATH=$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$INSTALL_PROTOBUF_PATH/bin:$PATH + export LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" + export CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include" +fi +protoc --version +cd $ORIGINAL_PATH \ No newline at end of file