From 6a7d306cfbcbac05d588ec2b47eced21e3037d32 Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Wed, 6 Aug 2025 22:00:12 -0500 Subject: [PATCH 01/38] protobuf upgrade Signed-off-by: Sunny Anand --- README.md | 2 +- docker/Dockerfile.llvm-project | 23 ++++++++++++++++------- docs/BuildOnWindows.md | 6 +++--- docs/Prerequisite.md | 2 +- requirements.txt | 2 +- utils/install-protobuf.cmd | 5 +++-- utils/install-protobuf.sh | 25 ++++++++++++++++--------- 7 files changed, 41 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 93a2e32626..5be537080c 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Detailed instructions are provided below. ``` python >= 3.8 gcc >= 6.4 -protobuf >= 4.21.12 +protobuf >= 4.25.1 cmake >= 3.13.4 make >= 4.2.1 or ninja >= 1.10.2 java >= 1.11 (optional) diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project index 62f1af44e4..e1c349b6e1 100644 --- a/docker/Dockerfile.llvm-project +++ b/docker/Dockerfile.llvm-project @@ -64,13 +64,22 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"] && pip3 install --upgrade pip # Install protobuf -ARG PROTOBUF_VERSION=21.12 -RUN git clone -b v${PROTOBUF_VERSION} --recursive https://github.com/protocolbuffers/protobuf.git \ - && cd protobuf && ./autogen.sh \ - && ./configure --enable-static=no \ - && make -j${NPROC} install && ldconfig \ - && cd python && python3 setup.py install --cpp_implementation \ - && cd ../.. && rm -rf protobuf +ARG PROTOBUF_VERSION=4.25.1 +RUN git clone --recursive https://github.com/protocolbuffers/protobuf.git \ + && cd protobuf \ + && git checkout v${PROTOBUF_VERSION} \ + && git submodule update --init --recursive \ + && mkdir build_source && cd build_source \ + && cmake -G Ninja ../ \ + -Dprotobuf_BUILD_SHARED_LIBS=OFF \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -Dprotobuf_BUILD_TESTS=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + && cmake --build . --target install && ldconfig \ + && cd ../python && python3 setup.py install --cpp_implementation \ + && protoc --version \ + && echo "protobuf installed" # Install jsoniter ARG JSONITER_VERSION=0.9.23 diff --git a/docs/BuildOnWindows.md b/docs/BuildOnWindows.md index dd47c23563..2c323ca760 100644 --- a/docs/BuildOnWindows.md +++ b/docs/BuildOnWindows.md @@ -15,8 +15,8 @@ Build protobuf as a static library. [same-as-file]: <> (utils/install-protobuf.cmd) ```shell -REM Check out protobuf v21.12 -set protobuf_version=21.12 +REM Check out protobuf v4.25.1 +set protobuf_version=4.25.1 git clone -b v%protobuf_version% --recursive https://github.com/protocolbuffers/protobuf.git set root_dir=%cd% @@ -42,7 +42,7 @@ set PATH=%root_dir%\protobuf_install\bin;%PATH% If you wish to be able to run all the ONNX-MLIR tests, you will also need to install the matching version of protobuf through pip. Note that this is included in the requirements.txt file at the root of onnx-mlir, so if you plan on using it, you won't need to explicitly install protobuf. ```shell -python3 -m pip install protobuf==4.21.12 +python3 -m pip install protobuf==4.25.1 ``` #### MLIR diff --git a/docs/Prerequisite.md b/docs/Prerequisite.md index b65fa3cce0..8717ee069f 100644 --- a/docs/Prerequisite.md +++ b/docs/Prerequisite.md @@ -6,7 +6,7 @@ ``` python >= 3.8 gcc >= 6.4 -protobuf >= 4.21.12 +protobuf >= 4.25.1 cmake >= 3.13.4 make >= 4.2.1 or ninja >= 1.10.2 java >= 1.11 (optional) diff --git a/requirements.txt b/requirements.txt index d5390063e5..a3550aa449 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,6 @@ lit~=15.0 # and np.int which are used heavily in onnx-mlir. numpy==2.0.1 onnx==1.17.0 -protobuf==4.21.12 +protobuf==4.25.1 pytest==8.3.2 pytest-xdist==3.6.1 diff --git a/utils/install-protobuf.cmd b/utils/install-protobuf.cmd index 9112848113..5ce4386300 100644 --- a/utils/install-protobuf.cmd +++ b/utils/install-protobuf.cmd @@ -1,8 +1,9 @@ -REM Check out protobuf v21.12 -set protobuf_version=21.12 +REM Check out protobuf v4.25.1 +set protobuf_version=4.25.1 git clone -b v%protobuf_version% --recursive https://github.com/protocolbuffers/protobuf.git set root_dir=%cd% +git submodule update --init --recursive md protobuf_build cd protobuf_build call cmake %root_dir%\protobuf\cmake -G "Ninja" ^ diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index 70e133e48d..25f5e46110 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -1,10 +1,17 @@ -# Check out protobuf -PROTOBUF_VERSION=21.12 -git clone -b v${PROTOBUF_VERSION} --depth 1 --recursive https://github.com/protocolbuffers/protobuf.git - +# Check out protobuf source code and build and install it +PROTOBUF_VERSION=4.25.1 +git clone -b v${PROTOBUF_VERSION} --depth 1 --recursive https://github.com/protocolbuffers/protobuf.git \ cd protobuf -./autogen.sh -./configure --enable-static=no -make -j$(sysctl -n hw.logicalcpu) install -cd python -python3 setup.py install --cpp_implementation +git submodule update --init --recursive +mkdir build_source && cd build_source +cmake -G Ninja ../ \ + -Dprotobuf_BUILD_SHARED_LIBS=OFF \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -Dprotobuf_BUILD_TESTS=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + .. +cmake --build . --target install && ldconfig +cd ../python && python3 setup.py install --cpp_implementation +protoc --version +echo "protobuf installed" From e745797fd56563d60281504502d1922276028248 Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Wed, 6 Aug 2025 22:09:08 -0500 Subject: [PATCH 02/38] fix git error for macos Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index 25f5e46110..e447fa7f51 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -1,6 +1,6 @@ # Check out protobuf source code and build and install it PROTOBUF_VERSION=4.25.1 -git clone -b v${PROTOBUF_VERSION} --depth 1 --recursive https://github.com/protocolbuffers/protobuf.git \ +git clone -b v${PROTOBUF_VERSION} --depth 1 --recursive https://github.com/protocolbuffers/protobuf.git cd protobuf git submodule update --init --recursive mkdir build_source && cd build_source From 2bc50fa1a672fad49cf2963aa4e87d7e406ee711 Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Wed, 6 Aug 2025 23:43:43 -0500 Subject: [PATCH 03/38] updates Signed-off-by: Sunny Anand --- docker/Dockerfile.llvm-project | 2 +- utils/install-protobuf.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project index e1c349b6e1..969a0aca23 100644 --- a/docker/Dockerfile.llvm-project +++ b/docker/Dockerfile.llvm-project @@ -64,7 +64,7 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"] && pip3 install --upgrade pip # Install protobuf -ARG PROTOBUF_VERSION=4.25.1 +ARG PROTOBUF_VERSION=25.1 RUN git clone --recursive https://github.com/protocolbuffers/protobuf.git \ && cd protobuf \ && git checkout v${PROTOBUF_VERSION} \ diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index e447fa7f51..4b5a4f6819 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -10,6 +10,8 @@ cmake -G Ninja ../ \ -Dprotobuf_BUILD_TESTS=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_CXX_STANDARD=17 \ + -DABSL_PROPAGATE_CXX_STD=ON \ .. cmake --build . --target install && ldconfig cd ../python && python3 setup.py install --cpp_implementation From c4a7ae1f4b14c115ab6bfbcc74c4c4a2aa1b07d4 Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Thu, 7 Aug 2025 00:26:51 -0500 Subject: [PATCH 04/38] updates Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index 4b5a4f6819..9557aebcd4 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -1,12 +1,14 @@ # Check out protobuf source code and build and install it -PROTOBUF_VERSION=4.25.1 +set -e +PROTOBUF_VERSION=25.1 +INSTALL_PROTOBUF_PATH=~/work git clone -b v${PROTOBUF_VERSION} --depth 1 --recursive https://github.com/protocolbuffers/protobuf.git cd protobuf git submodule update --init --recursive mkdir build_source && cd build_source cmake -G Ninja ../ \ -Dprotobuf_BUILD_SHARED_LIBS=OFF \ - -DCMAKE_INSTALL_PREFIX=/usr \ + -DCMAKE_INSTALL_PREFIX=$INSTALL_PROTOBUF_PATH \ -Dprotobuf_BUILD_TESTS=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ @@ -15,5 +17,6 @@ cmake -G Ninja ../ \ .. cmake --build . --target install && ldconfig cd ../python && python3 setup.py install --cpp_implementation +export PATH=$INSTALL_PROTOBUF_PATH/protobuf/include:$INSTALL_PROTOBUF_PATH/protobuf/lib:$INSTALL_PROTOBUF_PATH/protobuf/bin:$PATH protoc --version echo "protobuf installed" From f32f22a59a5ac5bd9f1c9362b63c7dbe9ee4717a Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Thu, 7 Aug 2025 00:41:13 -0500 Subject: [PATCH 05/38] updates Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index 9557aebcd4..3118f56e75 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -15,7 +15,7 @@ cmake -G Ninja ../ \ -DCMAKE_CXX_STANDARD=17 \ -DABSL_PROPAGATE_CXX_STD=ON \ .. -cmake --build . --target install && ldconfig +cmake --build . --target install cd ../python && python3 setup.py install --cpp_implementation export PATH=$INSTALL_PROTOBUF_PATH/protobuf/include:$INSTALL_PROTOBUF_PATH/protobuf/lib:$INSTALL_PROTOBUF_PATH/protobuf/bin:$PATH protoc --version From bae4408ea6fdb7f40c627aa7799da7596e2b4e4b Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Thu, 7 Aug 2025 01:20:01 -0500 Subject: [PATCH 06/38] updates Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index 3118f56e75..224dcdd08e 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -16,7 +16,7 @@ cmake -G Ninja ../ \ -DABSL_PROPAGATE_CXX_STD=ON \ .. cmake --build . --target install -cd ../python && python3 setup.py install --cpp_implementation +cd ~/work/protobuf/python && python3 setup.py install --cpp_implementation export PATH=$INSTALL_PROTOBUF_PATH/protobuf/include:$INSTALL_PROTOBUF_PATH/protobuf/lib:$INSTALL_PROTOBUF_PATH/protobuf/bin:$PATH protoc --version echo "protobuf installed" From a1d9a3ae084348040cbed1a71c066e2fe3ef21cb Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Sun, 10 Aug 2025 12:36:40 -0500 Subject: [PATCH 07/38] updates osx Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index 224dcdd08e..1a5727b2ef 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -1,7 +1,8 @@ +# Exit immediately if a command exits with a non-zero status. +set -e # Check out protobuf source code and build and install it -set -e PROTOBUF_VERSION=25.1 -INSTALL_PROTOBUF_PATH=~/work +INSTALL_PROTOBUF_PATH=~/work/protobuf_install # Changed to a dedicated install directory git clone -b v${PROTOBUF_VERSION} --depth 1 --recursive https://github.com/protocolbuffers/protobuf.git cd protobuf git submodule update --init --recursive @@ -16,7 +17,28 @@ cmake -G Ninja ../ \ -DABSL_PROPAGATE_CXX_STD=ON \ .. cmake --build . --target install -cd ~/work/protobuf/python && python3 setup.py install --cpp_implementation -export PATH=$INSTALL_PROTOBUF_PATH/protobuf/include:$INSTALL_PROTOBUF_PATH/protobuf/lib:$INSTALL_PROTOBUF_PATH/protobuf/bin:$PATH + +# Verify that protoc is installed correctly before proceeding +echo "Verifying protoc installation at $INSTALL_PROTOBUF_PATH/bin/protoc..." +if [ -f "$INSTALL_PROTOBUF_PATH/bin/protoc" ]; then + echo "protoc found." + "$INSTALL_PROTOBUF_PATH/bin/protoc" --version +else + echo "Error: protoc not found at $INSTALL_PROTOBUF_PATH/bin/protoc. Installation might have failed." + exit 1 +fi + +# Now navigate and run the python setup.py +# Use a subshell to temporarily modify PATH and LDFLAGS for this specific command, +# ensuring our installed protoc and libraries are found first. +(cd ~/work/protobuf/python && \ + PATH="$INSTALL_PROTOBUF_PATH/bin:$PATH" \ + LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" \ + CPPFLAGS="-I/$INSTALL_PROTOBUF_PATH/include" \ + python3 setup.py install --cpp_implementation) + +# Update the main shell's PATH for subsequent commands like 'protoc --version' +export PATH="$INSTALL_PROTOBUF_PATH/bin:$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$PATH" + protoc --version echo "protobuf installed" From 7efe63f6d8154dee0f517cf3f86f1137614920c1 Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Sun, 10 Aug 2025 12:45:15 -0500 Subject: [PATCH 08/38] updates osx Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index 1a5727b2ef..942ff5fdf6 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -35,6 +35,7 @@ fi PATH="$INSTALL_PROTOBUF_PATH/bin:$PATH" \ LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" \ CPPFLAGS="-I/$INSTALL_PROTOBUF_PATH/include" \ + CXXFLAGS="-std=c++17" python3 setup.py install --cpp_implementation) # Update the main shell's PATH for subsequent commands like 'protoc --version' From a3d4701c19ea3d94055e955f90db85f05cbe2b6d Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Sun, 10 Aug 2025 12:54:27 -0500 Subject: [PATCH 09/38] updates osx Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index 942ff5fdf6..0a6c441655 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -31,12 +31,15 @@ fi # Now navigate and run the python setup.py # Use a subshell to temporarily modify PATH and LDFLAGS for this specific command, # ensuring our installed protoc and libraries are found first. +# Pass library/include paths directly to setup.py (cd ~/work/protobuf/python && \ PATH="$INSTALL_PROTOBUF_PATH/bin:$PATH" \ LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" \ CPPFLAGS="-I/$INSTALL_PROTOBUF_PATH/include" \ CXXFLAGS="-std=c++17" - python3 setup.py install --cpp_implementation) + python3 setup.py install --cpp_implementation \ + --library-dirs="$INSTALL_PROTOBUF_PATH/lib" \ + --include-dirs="$INSTALL_PROTOBUF_PATH/include") # Update the main shell's PATH for subsequent commands like 'protoc --version' export PATH="$INSTALL_PROTOBUF_PATH/bin:$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$PATH" From c69a66f2a3e5826a1f3ceece481fa5f13d4400dc Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Sun, 10 Aug 2025 13:02:19 -0500 Subject: [PATCH 10/38] updates osx Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index 0a6c441655..68cf7b5a7a 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -38,8 +38,7 @@ fi CPPFLAGS="-I/$INSTALL_PROTOBUF_PATH/include" \ CXXFLAGS="-std=c++17" python3 setup.py install --cpp_implementation \ - --library-dirs="$INSTALL_PROTOBUF_PATH/lib" \ - --include-dirs="$INSTALL_PROTOBUF_PATH/include") + build_ext --library-dirs="$INSTALL_PROTOBUF_PATH/lib" --include-dirs="$INSTALL_PROTOBUF_PATH/include") # Update the main shell's PATH for subsequent commands like 'protoc --version' export PATH="$INSTALL_PROTOBUF_PATH/bin:$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$PATH" From f0c3c9b7e931a85751be03fc7ba8ac4ba2a8400c Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Sun, 10 Aug 2025 15:30:02 -0500 Subject: [PATCH 11/38] updates osx Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index 68cf7b5a7a..bded0cd8e5 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -36,7 +36,8 @@ fi PATH="$INSTALL_PROTOBUF_PATH/bin:$PATH" \ LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" \ CPPFLAGS="-I/$INSTALL_PROTOBUF_PATH/include" \ - CXXFLAGS="-std=c++17" + CC="clang -std=c++17" \ + CXX="clang++ -std=c++17" \ python3 setup.py install --cpp_implementation \ build_ext --library-dirs="$INSTALL_PROTOBUF_PATH/lib" --include-dirs="$INSTALL_PROTOBUF_PATH/include") From a269c63913b2a00eba9f16b787df211dd78f252f Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Sun, 10 Aug 2025 18:04:38 -0500 Subject: [PATCH 12/38] updates osx Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index bded0cd8e5..bdd3f0264f 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -35,7 +35,7 @@ fi (cd ~/work/protobuf/python && \ PATH="$INSTALL_PROTOBUF_PATH/bin:$PATH" \ LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" \ - CPPFLAGS="-I/$INSTALL_PROTOBUF_PATH/include" \ + CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include" \ CC="clang -std=c++17" \ CXX="clang++ -std=c++17" \ python3 setup.py install --cpp_implementation \ From f88f3bd73f1dd824f6809c5bb49019bdd9017549 Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Sun, 10 Aug 2025 18:19:35 -0500 Subject: [PATCH 13/38] updates osx Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index bdd3f0264f..669ebb892e 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -34,8 +34,8 @@ fi # Pass library/include paths directly to setup.py (cd ~/work/protobuf/python && \ PATH="$INSTALL_PROTOBUF_PATH/bin:$PATH" \ - LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" \ - CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include" \ + LDFLAGS="-std=c++17 -L$INSTALL_PROTOBUF_PATH/lib" \ + CPPFLAGS="-std=c++17 -I$INSTALL_PROTOBUF_PATH/include" \ CC="clang -std=c++17" \ CXX="clang++ -std=c++17" \ python3 setup.py install --cpp_implementation \ From ab79c148070a0331de6806e113b1ab2c73ffd6fc Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Sun, 10 Aug 2025 18:28:58 -0500 Subject: [PATCH 14/38] updates osx Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index 669ebb892e..688b01d38e 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -38,6 +38,7 @@ fi CPPFLAGS="-std=c++17 -I$INSTALL_PROTOBUF_PATH/include" \ CC="clang -std=c++17" \ CXX="clang++ -std=c++17" \ + CFLAGS="-std=c++17" \ python3 setup.py install --cpp_implementation \ build_ext --library-dirs="$INSTALL_PROTOBUF_PATH/lib" --include-dirs="$INSTALL_PROTOBUF_PATH/include") From c1b610602d85952a229a4c0b14e5c00e071916d7 Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Sun, 10 Aug 2025 18:58:08 -0500 Subject: [PATCH 15/38] updates osx Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index 688b01d38e..f0bb2a696e 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -37,7 +37,7 @@ fi LDFLAGS="-std=c++17 -L$INSTALL_PROTOBUF_PATH/lib" \ CPPFLAGS="-std=c++17 -I$INSTALL_PROTOBUF_PATH/include" \ CC="clang -std=c++17" \ - CXX="clang++ -std=c++17" \ + CXXFLAGS="clang++ -std=c++17" \ CFLAGS="-std=c++17" \ python3 setup.py install --cpp_implementation \ build_ext --library-dirs="$INSTALL_PROTOBUF_PATH/lib" --include-dirs="$INSTALL_PROTOBUF_PATH/include") From c543bae1b08d842b7236d71bfa5befd572ceaa34 Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Sun, 10 Aug 2025 22:08:39 -0500 Subject: [PATCH 16/38] updates osx Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index f0bb2a696e..b393ce767e 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -32,13 +32,21 @@ fi # Use a subshell to temporarily modify PATH and LDFLAGS for this specific command, # ensuring our installed protoc and libraries are found first. # Pass library/include paths directly to setup.py +if [ -d "$HOME/work/protobuf/python/build" ]; then + echo "Removing existing build directory..." + rm -rf "$HOME/work/protobuf/python/build" +else + echo "No build directory to remove." +fi + (cd ~/work/protobuf/python && \ PATH="$INSTALL_PROTOBUF_PATH/bin:$PATH" \ - LDFLAGS="-std=c++17 -L$INSTALL_PROTOBUF_PATH/lib" \ - CPPFLAGS="-std=c++17 -I$INSTALL_PROTOBUF_PATH/include" \ - CC="clang -std=c++17" \ - CXXFLAGS="clang++ -std=c++17" \ + CC="clang" \ + CXX="clang++" \ CFLAGS="-std=c++17" \ + CXXFLAGS="-std=c++17" \ + LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" \ + CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include" \ python3 setup.py install --cpp_implementation \ build_ext --library-dirs="$INSTALL_PROTOBUF_PATH/lib" --include-dirs="$INSTALL_PROTOBUF_PATH/include") From 00b384ee3aebda12c6471107f1afbf9130bec299 Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Sun, 10 Aug 2025 22:33:15 -0500 Subject: [PATCH 17/38] updates osx Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index b393ce767e..b6b6b1fe7c 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -39,17 +39,24 @@ else echo "No build directory to remove." fi -(cd ~/work/protobuf/python && \ - PATH="$INSTALL_PROTOBUF_PATH/bin:$PATH" \ - CC="clang" \ - CXX="clang++" \ - CFLAGS="-std=c++17" \ - CXXFLAGS="-std=c++17" \ - LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" \ - CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include" \ +cd ~/work/protobuf/python +# Temporarily modify setup.py to remove the hardcoded -std=c++14 flag +# Use a backup extension like '.bak' for macOS sed. +sed -i.bak 's/extra_compile_args.append(\'-std=c++14\')/#extra_compile_args.append(\'-std=c++14\') # Commented out by install script/g' setup.py + + +( export PATH="$INSTALL_PROTOBUF_PATH/bin:$PATH" && \ + export CC="clang" && \ + export CXX="clang++" && \ + export CFLAGS="-std=c++17" && \ + export CXXFLAGS="-std=c++17" && \ + export LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" &&\ + export CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include" &&\ python3 setup.py install --cpp_implementation \ build_ext --library-dirs="$INSTALL_PROTOBUF_PATH/lib" --include-dirs="$INSTALL_PROTOBUF_PATH/include") +mv setup.py.bak setup.py + # Update the main shell's PATH for subsequent commands like 'protoc --version' export PATH="$INSTALL_PROTOBUF_PATH/bin:$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$PATH" From 90e06b471760d206c3fb1ae330f5e9ac4784b47f Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Sun, 10 Aug 2025 22:56:30 -0500 Subject: [PATCH 18/38] updates osx Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index b6b6b1fe7c..9b687e8fab 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -42,8 +42,7 @@ fi cd ~/work/protobuf/python # Temporarily modify setup.py to remove the hardcoded -std=c++14 flag # Use a backup extension like '.bak' for macOS sed. -sed -i.bak 's/extra_compile_args.append(\'-std=c++14\')/#extra_compile_args.append(\'-std=c++14\') # Commented out by install script/g' setup.py - +sed -i '.bak' 's/extra_compile_args\.append('\''-std=c++14'\'')/#extra_compile_args.append('\''-std=c++14'\'') # Commented out by install script/g' setup.py ( export PATH="$INSTALL_PROTOBUF_PATH/bin:$PATH" && \ export CC="clang" && \ From 6b935399b2753825bec21963f67d1dfa5daad33f Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Mon, 11 Aug 2025 00:05:04 -0500 Subject: [PATCH 19/38] updates osx Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index 9b687e8fab..02e1d86954 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -51,8 +51,7 @@ sed -i '.bak' 's/extra_compile_args\.append('\''-std=c++14'\'')/#extra_compile_a export CXXFLAGS="-std=c++17" && \ export LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" &&\ export CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include" &&\ - python3 setup.py install --cpp_implementation \ - build_ext --library-dirs="$INSTALL_PROTOBUF_PATH/lib" --include-dirs="$INSTALL_PROTOBUF_PATH/include") + pip3 install .) mv setup.py.bak setup.py From b12664a3357373abc3d7194ceadb350b4fefae0b Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Mon, 11 Aug 2025 01:26:07 -0500 Subject: [PATCH 20/38] updates osx Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index 02e1d86954..2f3be25db4 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -42,7 +42,7 @@ fi cd ~/work/protobuf/python # Temporarily modify setup.py to remove the hardcoded -std=c++14 flag # Use a backup extension like '.bak' for macOS sed. -sed -i '.bak' 's/extra_compile_args\.append('\''-std=c++14'\'')/#extra_compile_args.append('\''-std=c++14'\'') # Commented out by install script/g' setup.py +sed -i '.bak' 's/extra_compile_args\.append('\''-std=c++14'\'')/extra_compile_args.append('\''-std=c++17'\'')/g' setup.py ( export PATH="$INSTALL_PROTOBUF_PATH/bin:$PATH" && \ export CC="clang" && \ @@ -51,7 +51,8 @@ sed -i '.bak' 's/extra_compile_args\.append('\''-std=c++14'\'')/#extra_compile_a export CXXFLAGS="-std=c++17" && \ export LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" &&\ export CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include" &&\ - pip3 install .) + python3 setup.py install --cpp_implementation \ + build_ext --library-dirs="$INSTALL_PROTOBUF_PATH/lib" --include-dirs="$INSTALL_PROTOBUF_PATH/include") mv setup.py.bak setup.py @@ -59,4 +60,4 @@ mv setup.py.bak setup.py export PATH="$INSTALL_PROTOBUF_PATH/bin:$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$PATH" protoc --version -echo "protobuf installed" +echo "protobuf installed" \ No newline at end of file From c1bc4ac37d954a42650d8a4636732903bda98e9c Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Mon, 11 Aug 2025 08:58:35 -0500 Subject: [PATCH 21/38] updates osx Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index 2f3be25db4..1685ed997a 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -40,7 +40,7 @@ else fi cd ~/work/protobuf/python -# Temporarily modify setup.py to remove the hardcoded -std=c++14 flag +# Temporarily modify setup.py to remove the hardcoded -std=c++14 flag with 17 # Use a backup extension like '.bak' for macOS sed. sed -i '.bak' 's/extra_compile_args\.append('\''-std=c++14'\'')/extra_compile_args.append('\''-std=c++17'\'')/g' setup.py From 2594aebf768b2b1e942c34fee6c77ee900c72d3f Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Mon, 11 Aug 2025 10:41:10 -0500 Subject: [PATCH 22/38] updates osx Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index 1685ed997a..dd5218819a 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -42,7 +42,7 @@ fi cd ~/work/protobuf/python # Temporarily modify setup.py to remove the hardcoded -std=c++14 flag with 17 # Use a backup extension like '.bak' for macOS sed. -sed -i '.bak' 's/extra_compile_args\.append('\''-std=c++14'\'')/extra_compile_args.append('\''-std=c++17'\'')/g' setup.py +sed -i '.bak' "s/extra_compile_args\.append('-std=c++14')/extra_compile_args.append('-std=c++17')/g" setup.py ( export PATH="$INSTALL_PROTOBUF_PATH/bin:$PATH" && \ export CC="clang" && \ @@ -54,7 +54,7 @@ sed -i '.bak' 's/extra_compile_args\.append('\''-std=c++14'\'')/extra_compile_ar python3 setup.py install --cpp_implementation \ build_ext --library-dirs="$INSTALL_PROTOBUF_PATH/lib" --include-dirs="$INSTALL_PROTOBUF_PATH/include") -mv setup.py.bak setup.py +#mv setup.py.bak setup.py # Update the main shell's PATH for subsequent commands like 'protoc --version' export PATH="$INSTALL_PROTOBUF_PATH/bin:$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$PATH" From 28d335f7ed76d4b0c8317d93be4871ffda92ee51 Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Mon, 11 Aug 2025 12:00:44 -0500 Subject: [PATCH 23/38] updates osx Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 83 +++++++++++++-------------------------- 1 file changed, 27 insertions(+), 56 deletions(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index dd5218819a..7d441867fe 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -1,63 +1,34 @@ # Exit immediately if a command exits with a non-zero status. set -e -# Check out protobuf source code and build and install it -PROTOBUF_VERSION=25.1 -INSTALL_PROTOBUF_PATH=~/work/protobuf_install # Changed to a dedicated install directory -git clone -b v${PROTOBUF_VERSION} --depth 1 --recursive https://github.com/protocolbuffers/protobuf.git -cd protobuf -git submodule update --init --recursive -mkdir build_source && cd build_source -cmake -G Ninja ../ \ - -Dprotobuf_BUILD_SHARED_LIBS=OFF \ - -DCMAKE_INSTALL_PREFIX=$INSTALL_PROTOBUF_PATH \ - -Dprotobuf_BUILD_TESTS=OFF \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DCMAKE_CXX_STANDARD=17 \ - -DABSL_PROPAGATE_CXX_STD=ON \ - .. -cmake --build . --target install -# Verify that protoc is installed correctly before proceeding -echo "Verifying protoc installation at $INSTALL_PROTOBUF_PATH/bin/protoc..." -if [ -f "$INSTALL_PROTOBUF_PATH/bin/protoc" ]; then - echo "protoc found." - "$INSTALL_PROTOBUF_PATH/bin/protoc" --version -else - echo "Error: protoc not found at $INSTALL_PROTOBUF_PATH/bin/protoc. Installation might have failed." - exit 1 -fi +export INSTALL_PROTOBUF_PATH=~/work/protobuf_install # Changed to a dedicated install directory +export BUILD_TYPE=Release +export CORE_NUMBER=1 -# Now navigate and run the python setup.py -# Use a subshell to temporarily modify PATH and LDFLAGS for this specific command, -# ensuring our installed protoc and libraries are found first. -# Pass library/include paths directly to setup.py -if [ -d "$HOME/work/protobuf/python/build" ]; then - echo "Removing existing build directory..." - rm -rf "$HOME/work/protobuf/python/build" +# Build protobuf from source with -fPIC on Unix-like system +ORIGINAL_PATH=$(pwd) +cd .. +wget https://github.com/abseil/abseil-cpp/releases/download/20230802.2/abseil-cpp-20230802.2.tar.gz +tar -xvf abseil-cpp-20230802.2.tar.gz + +wget https://github.com/protocolbuffers/protobuf/releases/download/v25.1/protobuf-25.1.tar.gz +tar -xvf protobuf-25.1.tar.gz +cd protobuf-25.1 +mkdir build_source && cd build_source +cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=$INSTALL_PROTOBUF_PATH -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DABSL_ROOT_DIR="${ORIGINAL_PATH}/../abseil-cpp-20230802.2" -DCMAKE_CXX_STANDARD=17 -DABSL_PROPAGATE_CXX_STD=on .. +if [ "$INSTALL_PROTOBUF_PATH" == "/usr" ]; then + # Don't use sudo for root + if [[ "$(id -u)" == "0" ]]; then + cmake --build . --target install --parallel $CORE_NUMBER + else + # install Protobuf on default system path so it needs sudo permission + sudo cmake --build . --target install --parallel $CORE_NUMBER + fi else - echo "No build directory to remove." + cmake --build . --target install --parallel $CORE_NUMBER + export PATH=$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$INSTALL_PROTOBUF_PATH/bin:$PATH + export LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" + export CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include" fi - -cd ~/work/protobuf/python -# Temporarily modify setup.py to remove the hardcoded -std=c++14 flag with 17 -# Use a backup extension like '.bak' for macOS sed. -sed -i '.bak' "s/extra_compile_args\.append('-std=c++14')/extra_compile_args.append('-std=c++17')/g" setup.py - -( export PATH="$INSTALL_PROTOBUF_PATH/bin:$PATH" && \ - export CC="clang" && \ - export CXX="clang++" && \ - export CFLAGS="-std=c++17" && \ - export CXXFLAGS="-std=c++17" && \ - export LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" &&\ - export CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include" &&\ - python3 setup.py install --cpp_implementation \ - build_ext --library-dirs="$INSTALL_PROTOBUF_PATH/lib" --include-dirs="$INSTALL_PROTOBUF_PATH/include") - -#mv setup.py.bak setup.py - -# Update the main shell's PATH for subsequent commands like 'protoc --version' -export PATH="$INSTALL_PROTOBUF_PATH/bin:$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$PATH" - protoc --version -echo "protobuf installed" \ No newline at end of file +cd $ORIGINAL_PATH \ No newline at end of file From 6750e534346297046c5f79a3a726a61203ff4440 Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Mon, 11 Aug 2025 14:03:33 -0500 Subject: [PATCH 24/38] add cppcheck suppression Signed-off-by: Sunny Anand --- utils/cppcheck_scan/cppcheck_suppressions.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/utils/cppcheck_scan/cppcheck_suppressions.txt b/utils/cppcheck_scan/cppcheck_suppressions.txt index 75f4e90bcd..b7346d4203 100644 --- a/utils/cppcheck_scan/cppcheck_suppressions.txt +++ b/utils/cppcheck_scan/cppcheck_suppressions.txt @@ -15,3 +15,10 @@ preprocessorErrorDirective:/workdir/onnx-mlir/build/third_party/onnx/onnx/onnx-o # # preprocessorErrorDirective:/workdir/llvm-project/llvm/include/llvm-c/DataTypes.h:38 +# Build image has a newer protobuf than onnx-mlir source. This causes preprocessor issues when scanning with cppcheck. +# +# +# +preprocessorErrorDirective:/workdir/onnx-mlir/build/third_party/onnx/onnx/onnx-data.pb.h:15 +preprocessorErrorDirective:/workdir/onnx-mlir/build/third_party/onnx/onnx/onnx-ml.pb.h:15 +preprocessorErrorDirective:/workdir/onnx-mlir/build/third_party/onnx/onnx/onnx-operators-ml.pb.h:15 \ No newline at end of file From 7a2474ba89e9cd8946c6263846afb5ddf9f268bf Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Mon, 11 Aug 2025 21:57:54 -0500 Subject: [PATCH 25/38] update s390x Signed-off-by: Sunny Anand --- docker/Dockerfile.llvm-project | 40 +++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project index 969a0aca23..bafb4c956a 100644 --- a/docker/Dockerfile.llvm-project +++ b/docker/Dockerfile.llvm-project @@ -65,21 +65,31 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"] # Install protobuf ARG PROTOBUF_VERSION=25.1 -RUN git clone --recursive https://github.com/protocolbuffers/protobuf.git \ - && cd protobuf \ - && git checkout v${PROTOBUF_VERSION} \ - && git submodule update --init --recursive \ - && mkdir build_source && cd build_source \ - && cmake -G Ninja ../ \ - -Dprotobuf_BUILD_SHARED_LIBS=OFF \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -Dprotobuf_BUILD_TESTS=OFF \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - && cmake --build . --target install && ldconfig \ - && cd ../python && python3 setup.py install --cpp_implementation \ - && protoc --version \ - && echo "protobuf installed" +ARG INSTALL_PROTOBUF_PATH=/workdir/protobuf_install +ARG BUILD_TYPE=Release +ARG CORE_NUMBER=2 +ARG ORIGINAL_PATH=${pwd} +RUN cd .. && \ + wget https://github.com/abseil/abseil-cpp/releases/download/20230802.2/abseil-cpp-20230802.2.tar.gz && \ + tar -xvf abseil-cpp-20230802.2.tar.gz && \ + wget https://github.com/protocolbuffers/protobuf/releases/download/v25.1/protobuf-25.1.tar.gz && \ + tar -xvf protobuf-25.1.tar.gz +RUN cd /protobuf-25.1 && \ + mkdir build_source && cd build_source && \ + cmake -G Ninja ../ \ + -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_INSTALL_PREFIX=$INSTALL_PROTOBUF_PATH \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -Dprotobuf_BUILD_TESTS=OFF \ + -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ + -DABSL_ROOT_DIR="${ORIGINAL_PATH}/abseil-cpp-20230802.2"\ + -DCMAKE_CXX_STANDARD=17 \ + -DABSL_PROPAGATE_CXX_STD=on \ + && cmake --build . --target install --parallel $CORE_NUMBER + +RUN export PATH=$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$INSTALL_PROTOBUF_PATH/bin:$PATH && \ + export LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" && \ + export CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include" # Install jsoniter ARG JSONITER_VERSION=0.9.23 From f87d82d227c4360c6a996115308e6c2fce8db044 Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Mon, 11 Aug 2025 22:01:30 -0500 Subject: [PATCH 26/38] update s390x Signed-off-by: Sunny Anand --- docker/Dockerfile.llvm-project | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project index bafb4c956a..4e58f6fdd8 100644 --- a/docker/Dockerfile.llvm-project +++ b/docker/Dockerfile.llvm-project @@ -33,7 +33,7 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"] libtool make maven ninja-build openjdk-11-jdk-headless \ python3 python3-dev python3-distutils python3-numpy \ python3-pip python3-pytest-xdist python3-setuptools \ - python3-typing-extensions zlib1g-dev && \ + python3-typing-extensions zlib1g-dev wget tar && \ rm -rf /var/lib/apt/lists/* && \ ln -sf /usr/bin/pytest-3 /usr/bin/pytest; \ elif [ "${distro}" = "rhel" ] || [ "${distro}" = "fedora" ]; then \ @@ -48,7 +48,7 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"] file java-11-openjdk-devel java-11-openjdk-headless \ gcc gcc-c++ git libtool make ncurses-devel ninja-build \ python39 python39-devel python39-pip python39-setuptools \ - python39-wheel tzdata-java zlib-devel && \ + python39-wheel tzdata-java zlib-devel wget tar && \ # Workaround broken ubi8 amd64 image installing python3.12 as # dependency of clang, which also breaks the /usr/bin/pip3 # symlink creation From 92bdbced704f605aa66e04efb20ef4959c42fa1e Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Mon, 11 Aug 2025 23:29:11 -0500 Subject: [PATCH 27/38] update s390x Signed-off-by: Sunny Anand --- docker/Dockerfile.llvm-project | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project index 4e58f6fdd8..1942ce4f9b 100644 --- a/docker/Dockerfile.llvm-project +++ b/docker/Dockerfile.llvm-project @@ -65,7 +65,7 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"] # Install protobuf ARG PROTOBUF_VERSION=25.1 -ARG INSTALL_PROTOBUF_PATH=/workdir/protobuf_install +ARG INSTALL_PROTOBUF_PATH=/usr/local ARG BUILD_TYPE=Release ARG CORE_NUMBER=2 ARG ORIGINAL_PATH=${pwd} From 9beeb47b591eef5dc81c9476ceb70f387da0fa0f Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Wed, 13 Aug 2025 10:00:41 -0500 Subject: [PATCH 28/38] updates Signed-off-by: Sunny Anand --- utils/install-protobuf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/install-protobuf.sh b/utils/install-protobuf.sh index 7d441867fe..419a00d9a3 100755 --- a/utils/install-protobuf.sh +++ b/utils/install-protobuf.sh @@ -1,7 +1,7 @@ # Exit immediately if a command exits with a non-zero status. set -e -export INSTALL_PROTOBUF_PATH=~/work/protobuf_install # Changed to a dedicated install directory +export INSTALL_PROTOBUF_PATH=/usr/local # Changed to a dedicated install directory export BUILD_TYPE=Release export CORE_NUMBER=1 From 137e940dd480c61c3bb553a5eb4bb91cb03d2ef6 Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Wed, 13 Aug 2025 11:17:02 -0500 Subject: [PATCH 29/38] updates Signed-off-by: Sunny Anand --- docs/BuildOnWindows.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/BuildOnWindows.md b/docs/BuildOnWindows.md index 2c323ca760..f31916ba0b 100644 --- a/docs/BuildOnWindows.md +++ b/docs/BuildOnWindows.md @@ -20,6 +20,7 @@ set protobuf_version=4.25.1 git clone -b v%protobuf_version% --recursive https://github.com/protocolbuffers/protobuf.git set root_dir=%cd% +git submodule update --init --recursive md protobuf_build cd protobuf_build call cmake %root_dir%\protobuf\cmake -G "Ninja" ^ From c972e44460e232c6c6e21464bd05037485c7cf74 Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Thu, 14 Aug 2025 00:35:49 -0500 Subject: [PATCH 30/38] revert to using cpp implementation using whl files - can't use bazel as it fails to build on s390x Signed-off-by: Sunny Anand --- docker/Dockerfile.llvm-project | 60 +++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project index 1942ce4f9b..ebe3000774 100644 --- a/docker/Dockerfile.llvm-project +++ b/docker/Dockerfile.llvm-project @@ -63,18 +63,33 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"] && ln -sf /usr/bin/python3 /usr/bin/python \ && pip3 install --upgrade pip -# Install protobuf +# # --- Install Bazelisk (Bazel version manager) --- doesn't work on s390x and ppc64le +# RUN curl -L -o /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/bazelisk-linux-amd64 && \ +# chmod +x /usr/local/bin/bazel && \ +# echo "7.0.0" > ~/.bazelversion && \ +# bazel --version + + +# --- Build and install protobuf using Bazel with Bzlmod --- ARG PROTOBUF_VERSION=25.1 ARG INSTALL_PROTOBUF_PATH=/usr/local ARG BUILD_TYPE=Release ARG CORE_NUMBER=2 -ARG ORIGINAL_PATH=${pwd} -RUN cd .. && \ - wget https://github.com/abseil/abseil-cpp/releases/download/20230802.2/abseil-cpp-20230802.2.tar.gz && \ - tar -xvf abseil-cpp-20230802.2.tar.gz && \ - wget https://github.com/protocolbuffers/protobuf/releases/download/v25.1/protobuf-25.1.tar.gz && \ - tar -xvf protobuf-25.1.tar.gz -RUN cd /protobuf-25.1 && \ +# ARG ORIGINAL_PATH=${pwd} + +# Install dependencies +RUN yum install -y wget git gcc-c++ make ninja-build python3 python3-pip cmake python3-devel + +# Download and extract Abseil and Protobuf sources +RUN git clone --recurse-submodules -b v25.1 https://github.com/protocolbuffers/protobuf.git protobuf-25.1 +#RUN cd / && \ + # wget https://github.com/abseil/abseil-cpp/releases/download/20230802.2/abseil-cpp-20230802.2.tar.gz && \ + # tar -xvf abseil-cpp-20230802.2.tar.gz && \ + # wget https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz && \ + # tar -xvf protobuf-${PROTOBUF_VERSION}.tar.gz + +# Build and install Protobuf (static libs, no tests) +RUN cd /protobuf-${PROTOBUF_VERSION} && \ mkdir build_source && cd build_source && \ cmake -G Ninja ../ \ -DBUILD_SHARED_LIBS=OFF \ @@ -82,14 +97,35 @@ RUN cd /protobuf-25.1 && \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ -Dprotobuf_BUILD_TESTS=OFF \ -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ - -DABSL_ROOT_DIR="${ORIGINAL_PATH}/abseil-cpp-20230802.2"\ + # -DABSL_ROOT_DIR="${ORIGINAL_PATH}/abseil-cpp-20230802.2"\ -DCMAKE_CXX_STANDARD=17 \ -DABSL_PROPAGATE_CXX_STD=on \ && cmake --build . --target install --parallel $CORE_NUMBER -RUN export PATH=$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$INSTALL_PROTOBUF_PATH/bin:$PATH && \ - export LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" && \ - export CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include" +# RUN export PATH=$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$INSTALL_PROTOBUF_PATH/bin:$PATH && \ +# export LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" && \ +# export CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include" + +# Set environment variables for protobuf +ENV PATH=${INSTALL_PROTOBUF_PATH}/bin:$PATH +ENV LD_LIBRARY_PATH=${INSTALL_PROTOBUF_PATH}/lib:$LD_LIBRARY_PATH +ENV CPATH=${INSTALL_PROTOBUF_PATH}/include:$CPATH +ENV LIBRARY_PATH=${INSTALL_PROTOBUF_PATH}/lib:$LIBRARY_PATH + +# Build and install Python protobuf bindings with cpp implementation support +RUN cd /protobuf-${PROTOBUF_VERSION}/python && \ + python3 setup.py build --cpp_implementation && \ + python3 setup.py install && \ + python3 -m pip install --upgrade pip setuptools wheel && \ + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp python3 setup.py bdist_wheel && \ + pip3 install dist/*.whl + +# Verify versions +#RUN protoc --version && python3 -c "import google.protobuf; print(google.protobuf.__version__)" +RUN protoc --version && \ + python3 -c "from google.protobuf.internal import api_implementation; \ + print('protobuf version:', api_implementation.Version()); \ + print('cpp_implementation:', api_implementation.Type()) # Install jsoniter ARG JSONITER_VERSION=0.9.23 From e08b36c254815c03ecbc223c9fb9ad1d487ba70f Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Thu, 14 Aug 2025 00:41:34 -0500 Subject: [PATCH 31/38] comment the install Signed-off-by: Sunny Anand --- docker/Dockerfile.llvm-project | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project index ebe3000774..4050d57f85 100644 --- a/docker/Dockerfile.llvm-project +++ b/docker/Dockerfile.llvm-project @@ -78,7 +78,7 @@ ARG CORE_NUMBER=2 # ARG ORIGINAL_PATH=${pwd} # Install dependencies -RUN yum install -y wget git gcc-c++ make ninja-build python3 python3-pip cmake python3-devel +#RUN yum install -y wget git gcc-c++ make ninja-build python3 python3-pip cmake python3-devel # Download and extract Abseil and Protobuf sources RUN git clone --recurse-submodules -b v25.1 https://github.com/protocolbuffers/protobuf.git protobuf-25.1 From 577f5bf5ed9905793769a2298433bdab8043cb53 Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Thu, 14 Aug 2025 00:46:25 -0500 Subject: [PATCH 32/38] update the clone path Signed-off-by: Sunny Anand --- docker/Dockerfile.llvm-project | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project index 4050d57f85..61d1b307f9 100644 --- a/docker/Dockerfile.llvm-project +++ b/docker/Dockerfile.llvm-project @@ -81,7 +81,7 @@ ARG CORE_NUMBER=2 #RUN yum install -y wget git gcc-c++ make ninja-build python3 python3-pip cmake python3-devel # Download and extract Abseil and Protobuf sources -RUN git clone --recurse-submodules -b v25.1 https://github.com/protocolbuffers/protobuf.git protobuf-25.1 +RUN git clone --recurse-submodules -b v25.1 https://github.com/protocolbuffers/protobuf.git /workdir/protobuf-25.1 #RUN cd / && \ # wget https://github.com/abseil/abseil-cpp/releases/download/20230802.2/abseil-cpp-20230802.2.tar.gz && \ # tar -xvf abseil-cpp-20230802.2.tar.gz && \ @@ -89,7 +89,7 @@ RUN git clone --recurse-submodules -b v25.1 https://github.com/protocolbuffers/p # tar -xvf protobuf-${PROTOBUF_VERSION}.tar.gz # Build and install Protobuf (static libs, no tests) -RUN cd /protobuf-${PROTOBUF_VERSION} && \ +RUN cd /workdir/protobuf-${PROTOBUF_VERSION} && \ mkdir build_source && cd build_source && \ cmake -G Ninja ../ \ -DBUILD_SHARED_LIBS=OFF \ From 59518c55384136e2fb8670412d78a40b33477cfc Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Thu, 14 Aug 2025 08:01:17 -0500 Subject: [PATCH 33/38] update the clone path Signed-off-by: Sunny Anand --- docker/Dockerfile.llvm-project | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project index 61d1b307f9..c782390a05 100644 --- a/docker/Dockerfile.llvm-project +++ b/docker/Dockerfile.llvm-project @@ -113,7 +113,7 @@ ENV CPATH=${INSTALL_PROTOBUF_PATH}/include:$CPATH ENV LIBRARY_PATH=${INSTALL_PROTOBUF_PATH}/lib:$LIBRARY_PATH # Build and install Python protobuf bindings with cpp implementation support -RUN cd /protobuf-${PROTOBUF_VERSION}/python && \ +RUN cd /workdir/protobuf-${PROTOBUF_VERSION}/python && \ python3 setup.py build --cpp_implementation && \ python3 setup.py install && \ python3 -m pip install --upgrade pip setuptools wheel && \ From 3bdd1d35e3bb48c94c77253c1d8e9c5ad6f7f95d Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Tue, 7 Oct 2025 22:34:20 -0500 Subject: [PATCH 34/38] update with bazel and no abseil patching Signed-off-by: Sunny Anand --- docker/Dockerfile.llvm-project | 146 ++++++++++++++++++++------------- 1 file changed, 88 insertions(+), 58 deletions(-) diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project index c782390a05..08f12d52e0 100644 --- a/docker/Dockerfile.llvm-project +++ b/docker/Dockerfile.llvm-project @@ -33,7 +33,7 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"] libtool make maven ninja-build openjdk-11-jdk-headless \ python3 python3-dev python3-distutils python3-numpy \ python3-pip python3-pytest-xdist python3-setuptools \ - python3-typing-extensions zlib1g-dev wget tar && \ + python3-typing-extensions zlib1g-dev unzip zip && \ rm -rf /var/lib/apt/lists/* && \ ln -sf /usr/bin/pytest-3 /usr/bin/pytest; \ elif [ "${distro}" = "rhel" ] || [ "${distro}" = "fedora" ]; then \ @@ -48,7 +48,7 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"] file java-11-openjdk-devel java-11-openjdk-headless \ gcc gcc-c++ git libtool make ncurses-devel ninja-build \ python39 python39-devel python39-pip python39-setuptools \ - python39-wheel tzdata-java zlib-devel wget tar && \ + python39-wheel tzdata-java zlib-devel unzip zip which && \ # Workaround broken ubi8 amd64 image installing python3.12 as # dependency of clang, which also breaks the /usr/bin/pip3 # symlink creation @@ -70,62 +70,92 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"] # bazel --version -# --- Build and install protobuf using Bazel with Bzlmod --- -ARG PROTOBUF_VERSION=25.1 -ARG INSTALL_PROTOBUF_PATH=/usr/local -ARG BUILD_TYPE=Release -ARG CORE_NUMBER=2 -# ARG ORIGINAL_PATH=${pwd} - -# Install dependencies -#RUN yum install -y wget git gcc-c++ make ninja-build python3 python3-pip cmake python3-devel - -# Download and extract Abseil and Protobuf sources -RUN git clone --recurse-submodules -b v25.1 https://github.com/protocolbuffers/protobuf.git /workdir/protobuf-25.1 -#RUN cd / && \ - # wget https://github.com/abseil/abseil-cpp/releases/download/20230802.2/abseil-cpp-20230802.2.tar.gz && \ - # tar -xvf abseil-cpp-20230802.2.tar.gz && \ - # wget https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz && \ - # tar -xvf protobuf-${PROTOBUF_VERSION}.tar.gz - -# Build and install Protobuf (static libs, no tests) -RUN cd /workdir/protobuf-${PROTOBUF_VERSION} && \ - mkdir build_source && cd build_source && \ - cmake -G Ninja ../ \ - -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_INSTALL_PREFIX=$INSTALL_PROTOBUF_PATH \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -Dprotobuf_BUILD_TESTS=OFF \ - -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ - # -DABSL_ROOT_DIR="${ORIGINAL_PATH}/abseil-cpp-20230802.2"\ - -DCMAKE_CXX_STANDARD=17 \ - -DABSL_PROPAGATE_CXX_STD=on \ - && cmake --build . --target install --parallel $CORE_NUMBER - -# RUN export PATH=$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$INSTALL_PROTOBUF_PATH/bin:$PATH && \ -# export LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" && \ -# export CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include" - -# Set environment variables for protobuf -ENV PATH=${INSTALL_PROTOBUF_PATH}/bin:$PATH -ENV LD_LIBRARY_PATH=${INSTALL_PROTOBUF_PATH}/lib:$LD_LIBRARY_PATH -ENV CPATH=${INSTALL_PROTOBUF_PATH}/include:$CPATH -ENV LIBRARY_PATH=${INSTALL_PROTOBUF_PATH}/lib:$LIBRARY_PATH - -# Build and install Python protobuf bindings with cpp implementation support -RUN cd /workdir/protobuf-${PROTOBUF_VERSION}/python && \ - python3 setup.py build --cpp_implementation && \ - python3 setup.py install && \ - python3 -m pip install --upgrade pip setuptools wheel && \ - PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp python3 setup.py bdist_wheel && \ - pip3 install dist/*.whl - -# Verify versions -#RUN protoc --version && python3 -c "import google.protobuf; print(google.protobuf.__version__)" -RUN protoc --version && \ - python3 -c "from google.protobuf.internal import api_implementation; \ - print('protobuf version:', api_implementation.Version()); \ - print('cpp_implementation:', api_implementation.Type()) +# # --- Build and install protobuf using Bazel with Bzlmod --- +# ARG PROTOBUF_VERSION=25.1 +# ARG INSTALL_PROTOBUF_PATH=/usr/local +# ARG BUILD_TYPE=Release +# ARG CORE_NUMBER=2 +# # ARG ORIGINAL_PATH=${pwd} + +# # Install dependencies +# #RUN yum install -y wget git gcc-c++ make ninja-build python3 python3-pip cmake python3-devel + +# # Download and extract Abseil and Protobuf sources +# RUN git clone --recurse-submodules -b v25.1 https://github.com/protocolbuffers/protobuf.git /workdir/protobuf-25.1 +# #RUN cd / && \ +# # wget https://github.com/abseil/abseil-cpp/releases/download/20230802.2/abseil-cpp-20230802.2.tar.gz && \ +# # tar -xvf abseil-cpp-20230802.2.tar.gz && \ +# # wget https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz && \ +# # tar -xvf protobuf-${PROTOBUF_VERSION}.tar.gz + +# # Build and install Protobuf (static libs, no tests) +# RUN cd /workdir/protobuf-${PROTOBUF_VERSION} && \ +# mkdir build_source && cd build_source && \ +# cmake -G Ninja ../ \ +# -DBUILD_SHARED_LIBS=OFF \ +# -DCMAKE_INSTALL_PREFIX=$INSTALL_PROTOBUF_PATH \ +# -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ +# -Dprotobuf_BUILD_TESTS=OFF \ +# -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ +# # -DABSL_ROOT_DIR="${ORIGINAL_PATH}/abseil-cpp-20230802.2"\ +# -DCMAKE_CXX_STANDARD=17 \ +# -DABSL_PROPAGATE_CXX_STD=on \ +# && cmake --build . --target install --parallel $CORE_NUMBER + +# # RUN export PATH=$INSTALL_PROTOBUF_PATH/include:$INSTALL_PROTOBUF_PATH/lib:$INSTALL_PROTOBUF_PATH/bin:$PATH && \ +# # export LDFLAGS="-L$INSTALL_PROTOBUF_PATH/lib" && \ +# # export CPPFLAGS="-I$INSTALL_PROTOBUF_PATH/include" + +# # Set environment variables for protobuf +# ENV PATH=${INSTALL_PROTOBUF_PATH}/bin:$PATH +# ENV LD_LIBRARY_PATH=${INSTALL_PROTOBUF_PATH}/lib:$LD_LIBRARY_PATH +# ENV CPATH=${INSTALL_PROTOBUF_PATH}/include:$CPATH +# ENV LIBRARY_PATH=${INSTALL_PROTOBUF_PATH}/lib:$LIBRARY_PATH + +# # Build and install Python protobuf bindings with cpp implementation support +# RUN cd /workdir/protobuf-${PROTOBUF_VERSION}/python && \ +# python3 setup.py build --cpp_implementation && \ +# python3 setup.py install && \ +# python3 -m pip install --upgrade pip setuptools wheel && \ +# PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp python3 setup.py bdist_wheel && \ +# pip3 install dist/*.whl + +# # Verify versions +# #RUN protoc --version && python3 -c "import google.protobuf; print(google.protobuf.__version__)" +# RUN protoc --version && \ +# python3 -c "from google.protobuf.internal import api_implementation; \ +# print('protobuf version:', api_implementation.Version()); \ +# print('cpp_implementation:', api_implementation.Type()) + +# Install bazel +RUN mkdir /usr/local/bazel && cd /usr/local/bazel && \ + curl -LO https://github.com/bazelbuild/bazel/releases/download/6.5.0/bazel-6.5.0-dist.zip && \ + unzip bazel-6.5.0-dist.zip && chmod -R +w . && \ + EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk" EXTRA_BAZEL_ARGS="--tool_java_runtime_version=local_jdk" bash ./compile.sh && \ + cp output/bazel /usr/local/bin/ +RUN bazel --version + +# Install rust, cargo, and cargo-bazel +ARG RUST_VERSION=1.88 +ARG CARGO_BAZEL_VERSION=0.16.0 +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- -y --default-toolchain none \ + && . "${HOME}/.cargo/env" \ + && rustup install ${RUST_VERSION} \ + && cargo install cargo-bazel --version ${CARGO_BAZEL_VERSION} +#ENV PATH="${HOME}/.cargo/bin:${PATH}" + +# Build and install protobuf using bazel +WORKDIR /workdir +RUN git clone --recursive https://github.com/protocolbuffers/protobuf.git +RUN cd /workdir/protobuf && git checkout v32.0 && git submodule update --init --recursive +RUN cd /workdir/protobuf && bazel build :protoc :protobuf +RUN cp /workdir/protobuf/bazel-bin/protoc /usr/local/bin && \ + cp -r /workdir/protobuf/bazel-bin/external/com_google_protobuf/python/google /usr/local/lib/python3.9/site-packages/google && \ + cp -r /workdir/protobuf/bazel-bin/external/com_google_protobuf/libprotobuf.so* /usr/local/lib && \ + ldconfig + +RUN cd /workdir/protobuf && bazel build //python/dist:binary_wheel # Install jsoniter ARG JSONITER_VERSION=0.9.23 From 6110c52ea0ff432ab4ade4d3c1a01964b8d238ee Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Tue, 7 Oct 2025 23:33:33 -0500 Subject: [PATCH 35/38] update CARGO URL and add s390x to toolchain Signed-off-by: Sunny Anand --- docker/Dockerfile.llvm-project | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project index 08f12d52e0..29bb4a9396 100644 --- a/docker/Dockerfile.llvm-project +++ b/docker/Dockerfile.llvm-project @@ -149,7 +149,14 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ WORKDIR /workdir RUN git clone --recursive https://github.com/protocolbuffers/protobuf.git RUN cd /workdir/protobuf && git checkout v32.0 && git submodule update --init --recursive -RUN cd /workdir/protobuf && bazel build :protoc :protobuf +RUN export CARGO_BAZEL_GENERATOR_URL=file:///root/.cargo/bin/cargo-bazel \ + && export CARGO_BAZEL_REPIN=true \ + && CC=clang CXX=clang++ && cd /workdir/protobuf \ + && CC=clang CXX=clang++ bazel fetch //python/dist:binary_wheel \ + && sed -i -e 's/\["arm64", "amd64"\]/\["arm64", "amd64", "s390x"\]/g' \ + ${HOME}/.cache/bazel/_bazel_root/*/external/rules_buf/buf/internal/toolchain.bzl \ + && CC=clang CXX=clang++ bazel build //:protobuf //:protoc +#RUN cd /workdir/protobuf && bazel build :protoc :protobuf RUN cp /workdir/protobuf/bazel-bin/protoc /usr/local/bin && \ cp -r /workdir/protobuf/bazel-bin/external/com_google_protobuf/python/google /usr/local/lib/python3.9/site-packages/google && \ cp -r /workdir/protobuf/bazel-bin/external/com_google_protobuf/libprotobuf.so* /usr/local/lib && \ From 09dcc207b0bd911214653c1a079ee43e92cd951f Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Wed, 8 Oct 2025 10:07:50 -0500 Subject: [PATCH 36/38] update bazel version to support latest protobuf install via bazel Signed-off-by: Sunny Anand --- docker/Dockerfile.llvm-project | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project index 29bb4a9396..a26b37eeb7 100644 --- a/docker/Dockerfile.llvm-project +++ b/docker/Dockerfile.llvm-project @@ -129,8 +129,8 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"] # Install bazel RUN mkdir /usr/local/bazel && cd /usr/local/bazel && \ - curl -LO https://github.com/bazelbuild/bazel/releases/download/6.5.0/bazel-6.5.0-dist.zip && \ - unzip bazel-6.5.0-dist.zip && chmod -R +w . && \ + curl -LO https://github.com/bazelbuild/bazel/releases/download/7.2.0/bazel-7.2.0-dist.zip && \ + unzip bazel-7.2.0-dist.zip && chmod -R +w . && \ EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk" EXTRA_BAZEL_ARGS="--tool_java_runtime_version=local_jdk" bash ./compile.sh && \ cp output/bazel /usr/local/bin/ RUN bazel --version From b8519f4d49d3e84324bf639f2c94fb0f7c5a10b3 Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Wed, 8 Oct 2025 10:24:43 -0500 Subject: [PATCH 37/38] update java for bazel Signed-off-by: Sunny Anand --- docker/Dockerfile.llvm-project | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project index a26b37eeb7..7df13dc260 100644 --- a/docker/Dockerfile.llvm-project +++ b/docker/Dockerfile.llvm-project @@ -30,7 +30,7 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"] apt-get install -qq -y --no-install-recommends \ autoconf automake ca-certificates clang cmake cppcheck \ curl default-jdk-headless gcc g++ git libncurses-dev \ - libtool make maven ninja-build openjdk-11-jdk-headless \ + libtool make maven ninja-build openjdk-21-jdk-headless \ python3 python3-dev python3-distutils python3-numpy \ python3-pip python3-pytest-xdist python3-setuptools \ python3-typing-extensions zlib1g-dev unzip zip && \ @@ -45,7 +45,7 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"] yum update -q -y && \ yum install -q -y \ autoconf automake ca-certificates clang cmake diffutils \ - file java-11-openjdk-devel java-11-openjdk-headless \ + file java-21-openjdk-devel java-21-openjdk-headless \ gcc gcc-c++ git libtool make ncurses-devel ninja-build \ python39 python39-devel python39-pip python39-setuptools \ python39-wheel tzdata-java zlib-devel unzip zip which && \ From e8d7341b533287cecd69b307d180eb7569ffe603 Mon Sep 17 00:00:00 2001 From: Sunny Anand Date: Wed, 8 Oct 2025 10:54:51 -0500 Subject: [PATCH 38/38] update python for bazel Signed-off-by: Sunny Anand --- docker/Dockerfile.llvm-project | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project index 7df13dc260..90915d696d 100644 --- a/docker/Dockerfile.llvm-project +++ b/docker/Dockerfile.llvm-project @@ -152,10 +152,10 @@ RUN cd /workdir/protobuf && git checkout v32.0 && git submodule update --init -- RUN export CARGO_BAZEL_GENERATOR_URL=file:///root/.cargo/bin/cargo-bazel \ && export CARGO_BAZEL_REPIN=true \ && CC=clang CXX=clang++ && cd /workdir/protobuf \ - && CC=clang CXX=clang++ bazel fetch //python/dist:binary_wheel \ + #&& CC=clang CXX=clang++ bazel fetch //python/dist:binary_wheel \ && sed -i -e 's/\["arm64", "amd64"\]/\["arm64", "amd64", "s390x"\]/g' \ ${HOME}/.cache/bazel/_bazel_root/*/external/rules_buf/buf/internal/toolchain.bzl \ - && CC=clang CXX=clang++ bazel build //:protobuf //:protoc + && CC=clang CXX=clang++ bazel build --enable_bzlmod :protobuf :protoc #RUN cd /workdir/protobuf && bazel build :protoc :protobuf RUN cp /workdir/protobuf/bazel-bin/protoc /usr/local/bin && \ cp -r /workdir/protobuf/bazel-bin/external/com_google_protobuf/python/google /usr/local/lib/python3.9/site-packages/google && \