Skip to content
This repository was archived by the owner on May 25, 2024. It is now read-only.

Commit f877036

Browse files
pichuancopybara-github
pichuan
authored and
copybara-github
committed
Update build-prereq.sh to directly build pyclif binary from the latest clif repo.
PiperOrigin-RevId: 360735027
1 parent 46bff37 commit f877036

File tree

11 files changed

+180
-33
lines changed

11 files changed

+180
-33
lines changed

Diff for: ‎.bazelrc

+3
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@ build --define=use_fast_cpp_protos=true
4141
build --copt="-Wno-maybe-uninitialized"
4242
build --copt="-Wno-unused-function"
4343

44+
# Set c++ version.
45+
build --cxxopt="-std=c++17"
46+
4447
# These are errors coming from the protobuf library itself. We'd like to see
4548
# them in our own build.
4649
# build --copt="-Wno-write-strings"

Diff for: ‎README.md

+7-3
Original file line numberDiff line numberDiff line change
@@ -59,18 +59,22 @@ pip install --user google-nucleus==0.3.2
5959

6060
## Building from source
6161

62-
For Ubuntu 14, Ubuntu 16, Ubuntu 18 and Debian 9 systems, building from source
63-
is easy. Simply type
62+
For Ubuntu 18, building from source is easy. Simply type
6463

6564
```shell
6665
source install.sh
6766
```
6867

68+
This will call `build_clif.sh`, which will build CLIF from scratch as well.
69+
6970
For all other systems, you will need to first install CLIF by following the
7071
instructions at
7172
[https://github.com/google/clif#installation](https://github.com/google/clif#installation)
7273
before running install.sh. You'll need to run this command with Python 3.6 or
73-
3.7.
74+
3.7. If you don't want to build CLIF binaries on your own, you can consider
75+
using pre-built CLIF binaries (see
76+
[an example here](https://github.com/google/nucleus/blob/v0.5.6/install.sh#L143-L152)). Note that we don't plan to update these pre-built CLIF binaries, so we
77+
recommend building CLIF binaries from scratch.
7478

7579
Note that install.sh extensively depends on apt-get, so it is unlikely to run
7680
without extensive modifications on non-Debian-based systems.

Diff for: ‎WORKSPACE

+11
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,17 @@ http_archive(
6161
],
6262
)
6363

64+
http_archive(
65+
name = "com_google_glog",
66+
sha256 = "1ee310e5d0a19b9d584a855000434bb724aa744745d5b8ab1855c85bff8a8e21",
67+
strip_prefix = "glog-028d37889a1e80e8a07da1b8945ac706259e5fd8",
68+
urls = [
69+
"https://mirror.bazel.build/github.com/google/glog/archive/028d37889a1e80e8a07da1b8945ac706259e5fd8.tar.gz",
70+
"https://github.com/google/glog/archive/028d37889a1e80e8a07da1b8945ac706259e5fd8.tar.gz",
71+
],
72+
)
73+
74+
6475
# bazel_skylib is now a required dependency of com_google_protobuf.
6576
http_archive(
6677
name = "bazel_skylib",

Diff for: ‎build_clif.sh

+135
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
#!/bin/bash
2+
# Copyright 2020 Google LLC.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions
6+
# are met:
7+
#
8+
# 1. Redistributions of source code must retain the above copyright notice,
9+
# this list of conditions and the following disclaimer.
10+
#
11+
# 2. Redistributions in binary form must reproduce the above copyright
12+
# notice, this list of conditions and the following disclaimer in the
13+
# documentation and/or other materials provided with the distribution.
14+
#
15+
# 3. Neither the name of the copyright holder nor the names of its
16+
# contributors may be used to endorse or promote products derived from this
17+
# software without specific prior written permission.
18+
#
19+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22+
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23+
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24+
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25+
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26+
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27+
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28+
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29+
# POSSIBILITY OF SUCH DAMAGE.
30+
#
31+
#
32+
# This script is only maintained for Ubuntu 18.04.
33+
34+
set -eux -o pipefail
35+
36+
echo ========== This script is only maintained for Ubuntu 18.04.
37+
echo ========== See https://github.com/google/clif for how to build on different Unix distributions.
38+
39+
UBUNTU_VERSION=18.04
40+
ABSL_VERSION=20200923
41+
PROTOBUF_VERSION=3.13.0
42+
PYTHON_VERSION=3.6
43+
44+
APT_ARGS=(
45+
"-qq"
46+
"-y"
47+
)
48+
49+
50+
sudo apt-get update "${APT_ARGS[@]}"
51+
sudo apt-get install "${APT_ARGS[@]}" --no-install-recommends \
52+
autoconf \
53+
automake \
54+
cmake \
55+
curl \
56+
gpg-agent \
57+
g++ \
58+
libtool \
59+
make \
60+
pkg-config \
61+
software-properties-common \
62+
wget \
63+
unzip
64+
65+
# Configure LLVM 11 apt repository
66+
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - && \
67+
sudo add-apt-repository "deb http://apt.llvm.org/$(lsb_release -sc)/ llvm-toolchain-$(lsb_release -sc)-11 main"
68+
69+
# Install CLIF dependencies
70+
sudo apt-get update "${APT_ARGS[@]}"
71+
sudo apt-get install "${APT_ARGS[@]}" \
72+
clang-11 \
73+
libclang-11-dev \
74+
libgoogle-glog-dev \
75+
libgtest-dev \
76+
libllvm11 \
77+
llvm-11-dev \
78+
python3-dev \
79+
python3-pyparsing \
80+
zlib1g-dev
81+
82+
# Configure deadsnakes PPA with the more recent versions of python packaged for
83+
# Ubuntu. See https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa
84+
sudo apt-get update "${APT_ARGS[@]}" && \
85+
sudo apt-get install "${APT_ARGS[@]}" \
86+
"python$PYTHON_VERSION-dev" \
87+
"python$PYTHON_VERSION-distutils"
88+
89+
# Install latest version of pip since the version on ubuntu could be outdated
90+
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
91+
"python$PYTHON_VERSION" get-pip.py && \
92+
rm get-pip.py
93+
94+
# Compile and install absl-cpp from source
95+
wget "https://github.com/abseil/abseil-cpp/archive/$ABSL_VERSION.tar.gz" && \
96+
tar -xf "$ABSL_VERSION.tar.gz" && \
97+
mkdir "abseil-cpp-$ABSL_VERSION/build" && \
98+
cd "abseil-cpp-$ABSL_VERSION/build" && \
99+
cmake .. -DCMAKE_POSITION_INDEPENDENT_CODE=true && \
100+
sudo make install && \
101+
rm -rf "/abseil-cpp-$ABSL_VERSION" "/$ABSL_VERSION.tar.gz"
102+
103+
# Compile and install protobuf from source
104+
wget "https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-cpp-$PROTOBUF_VERSION.tar.gz" && \
105+
tar -xf "protobuf-cpp-$PROTOBUF_VERSION.tar.gz" && \
106+
cd "protobuf-$PROTOBUF_VERSION" && \
107+
# Configure and install C++ libraries
108+
./autogen.sh && \
109+
./configure && \
110+
make -j"$(nproc)" && \
111+
sudo make install && \
112+
sudo ldconfig && \
113+
rm -rf "/protobuf-$PROTOBUF_VERSION" "/protobuf-cpp-$PROTOBUF_VERSION.tar.gz"
114+
115+
# Install googletest
116+
cd /usr/src/gtest && \
117+
sudo cmake . && \
118+
sudo make install
119+
120+
# Install python runtime and test dependencies
121+
"python$PYTHON_VERSION" -m pip install \
122+
absl-py \
123+
parameterized \
124+
protobuf=="$PROTOBUF_VERSION"
125+
126+
sudo "python$PYTHON_VERSION" -m pip uninstall -y pyparsing && \
127+
"python$PYTHON_VERSION" -m pip install -Iv 'pyparsing==2.2.0'
128+
129+
DV_PLATFORM="ubuntu-${UBUNTU_VERSION}"
130+
131+
sudo ln -sf /usr/bin/python\$PYTHON_VERSION /usr/local/bin/python3
132+
133+
cd && rm -rf clif && git clone https://github.com/google/clif.git && \
134+
cd clif && \
135+
sudo ./INSTALL.sh

Diff for: ‎install.sh

+14-27
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ python3 -m pip install --user 'setuptools==49.6.0'
8686
python3 -m pip install --user 'keras_preprocessing==1.1.2' --no-deps
8787
python3 -m pip install --user 'h5py==2.10.0'
8888
python3 -m pip install --user enum34
89-
python3 -m pip install --user 'protobuf==3.9.2'
89+
python3 -m pip install --user 'protobuf==3.13.0'
9090

9191
# Install Bazel
9292
################################################################################
@@ -122,35 +122,22 @@ ensure_wanted_bazel_version "${NUCLEUS_BAZEL_VERSION}"
122122

123123
note_build_stage "Install CLIF binary"
124124

125-
if [[ -e /usr/local/clif/bin/pyclif ]];
125+
if [[ -e /usr/local/bin/pyclif ]];
126126
then
127127
echo "CLIF already installed."
128128
else
129-
# Figure out which linux installation we are on to fetch an appropriate
130-
# version of the pre-built CLIF binary. Note that we only support now Ubuntu
131-
# 14, 16, and 18.
132-
case "$(lsb_release -d)" in
133-
*Ubuntu*18.*.*) PLATFORM="ubuntu-18" ;;
134-
*Ubuntu*16.*.*) PLATFORM="ubuntu-16" ;;
135-
*Ubuntu*14.*.*) PLATFORM="ubuntu-14" ;;
136-
*Debian*9.*) PLATFORM="debian" ;;
137-
*Debian*rodete*) PLATFORM="debian" ;;
138-
*) echo "CLIF is not installed on this machine and a prebuilt binary is not
139-
available for this platform. Please install CLIF at
140-
https://github.com/google/clif before continuing."
141-
exit 1
142-
esac
143-
144-
PACKAGE_CURL_PATH="https://storage.googleapis.com/deepvariant/packages"
145-
OSS_CLIF_CURL_ROOT="${PACKAGE_CURL_PATH}/oss_clif"
146-
OSS_CLIF_PKG="oss_clif.${PLATFORM}.latest.tgz"
147-
148-
if [[ ! -f "/tmp/${OSS_CLIF_PKG}" ]]; then
149-
curl "${OSS_CLIF_CURL_ROOT}/${OSS_CLIF_PKG}" > /tmp/${OSS_CLIF_PKG}
150-
fi
151-
152-
(cd / && sudo tar xzf "/tmp/${OSS_CLIF_PKG}")
153-
sudo ldconfig # Reload shared libraries.
129+
# Build clif binary from scratch. Might not be ideal because it installs a
130+
# bunch of dependencies, but this works fine when we used this in a Dockerfile
131+
# because we don't do build-prereq.sh in the final image.
132+
time ./build_clif.sh
133+
# TODO(b/181283422): Figure out why these symbolic links are needed and see if
134+
# we can do this better.
135+
sudo mkdir -p /usr/clang/bin/
136+
sudo ln -sf /usr/local/bin/clif-matcher /usr/clang/bin/clif-matcher
137+
sudo mkdir -p /usr/local/clif/bin
138+
sudo ln -sf /usr/local/bin/pyclif* /usr/local/clif/bin/
139+
DIST_PACKAGES_DIR=$(python3 -c "import site; print(site.getsitepackages()[0])")
140+
sudo ln -sf ${DIST_PACKAGES_DIR}/clif/python /usr/local/clif/
154141
fi
155142

156143
# Download and build TensorFlow

Diff for: ‎nucleus/io/BUILD

+3
Original file line numberDiff line numberDiff line change
@@ -1021,6 +1021,7 @@ cc_library(
10211021
deps = [
10221022
"//nucleus/platform:types",
10231023
"@com_google_absl//absl/memory",
1024+
"@com_google_absl//absl/strings:cord",
10241025
"@org_tensorflow//tensorflow/core:lib",
10251026
"@org_tensorflow//tensorflow/core/platform/cloud:gcs_file_system",
10261027
],
@@ -1044,6 +1045,7 @@ cc_library(
10441045
hdrs = ["tfrecord_writer.h"],
10451046
deps = [
10461047
"@com_google_absl//absl/memory",
1048+
"@com_google_absl//absl/strings:cord",
10471049
"@org_tensorflow//tensorflow/core:lib",
10481050
"@org_tensorflow//tensorflow/core/platform/cloud:gcs_file_system",
10491051
],
@@ -1055,6 +1057,7 @@ cc_library(
10551057
hdrs = ["gfile.h"],
10561058
deps = [
10571059
"@com_google_absl//absl/memory",
1060+
"@com_google_absl//absl/strings:cord",
10581061
"@org_tensorflow//tensorflow/core:lib",
10591062
],
10601063
)

Diff for: ‎nucleus/pip_package/build_pip_package.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ set -x
3030
# egg_files/PKG-INFO.
3131
NUCLEUS_VERSION="0.5.8"
3232
PACKAGE_NAME="google_nucleus-${NUCLEUS_VERSION}"
33-
PYTHON_VERSION="3.5"
33+
PYTHON_VERSION="3.6"
3434

3535
TMPDIR=$(mktemp -d -t tmp.XXXXXXXXXXX)
3636
TOPDIR="${TMPDIR}/${PACKAGE_NAME}"

Diff for: ‎nucleus/util/BUILD

+2
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,7 @@ cc_library(
132132
srcs = ["math.cc"],
133133
hdrs = ["math.h"],
134134
deps = [
135+
"@com_google_absl//absl/strings:cord",
135136
"@org_tensorflow//tensorflow/core:lib",
136137
],
137138
)
@@ -368,6 +369,7 @@ cc_library(
368369
deps = [
369370
":proto_ptr",
370371
"@clif//:cpp_runtime",
372+
"@com_google_absl//absl/strings:cord",
371373
"@com_google_protobuf//:proto_api",
372374
"@com_google_protobuf//:protobuf",
373375
"@org_tensorflow//tensorflow/core:lib",

Diff for: ‎nucleus/vendor/BUILD

+1
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ cc_library(
6060
deps = [
6161
":statusor",
6262
"@clif//:cpp_runtime",
63+
"@com_google_absl//absl/strings:cord",
6364
"@local_config_python//:python_headers",
6465
],
6566
)

Diff for: ‎third_party/clif.BUILD

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ cc_library(
1919
hdrs = glob(["clif/python/*.h"]),
2020
visibility = ["//visibility:public"],
2121
deps = [
22+
"@com_google_glog//:glog",
2223
"@com_google_protobuf//:protobuf",
2324
"@local_config_python//:python_headers",
2425
],

Diff for: ‎third_party/clif.bzl

+2-2
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ CLIF_PROTO = "@clif//:proto"
99
# Label for our OSS CLIF C++ runtime headers and sources.
1010
CLIF_CPP_RUNTIME = "@clif//:cpp_runtime"
1111

12-
# The CLIF generated code only compiles with C++11.
13-
EXTRA_CC_FLAGS = ["-std=c++11"]
12+
# The CLIF generated code only compiles with C++17.
13+
EXTRA_CC_FLAGS = ["-std=c++17"]
1414

1515
_PROTO_LIBRARY_SUFFIX = "_pyclif"
1616

0 commit comments

Comments
 (0)