Add Amazon Linux 2 Build

- "Install_amazon-linux-2.sh" will install required build tools and development libraries. - Adds "Makefile_amazon-linux-2" to describe the required dependency compilation order. - Updates individual recipes for compiling each dependency to generally be more consistent. - Fixes all dependencies to a specific version.
facebook · Apr 30, 2021 · 065f705 · 065f705
1 parent 5c4f50e
commit 065f705
Show file tree

Hide file tree

Showing 14 changed files with 266 additions and 86 deletions.
diff --git a/mcrouter/scripts/Makefile_amazon-linux-2 b/mcrouter/scripts/Makefile_amazon-linux-2
@@ -0,0 +1,58 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+RECIPES_DIR := ./recipes
+
+all: mcrouter
+
+# Boost available from Amazon Linux is not recent enough.
+# There *are* more recent versions available via EPEL, but they introduce python dependency conflicts.
+.boost-done:
+	${RECIPES_DIR}/boost.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+# The version of fmt from the package repositories is insufficient to satisfy the "folly" build.
+.fmt-done: .boost-done
+	${RECIPES_DIR}/fmtlib.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+# The version of gflags in the repo is insufficient for compiling mcrouter
+.gflags-done: .boost-done
+	${RECIPES_DIR}/gflags.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+# The version of glog in the repo is insufficient for compiling folly.
+# Additionally, the most recent (HEAD / v0.5.0.rc2) revision from upstream is incompatible for compiling mcrouter.
+# Fortunately, v0.4.0 works for both, so we just check out that revision.
+.glog-done: .gflags-done .boost-done
+	${RECIPES_DIR}/glog.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+.zstd-done:
+	${RECIPES_DIR}/zstd.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+.folly-done: .zstd-done .glog-done .gflags-done .boost-done .fmt-done
+	${RECIPES_DIR}/folly.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+.fizz-done: .folly-done .glog-done .gflags-done .boost-done
+	${RECIPES_DIR}/fizz.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+.wangle-done: .folly-done .fizz-done .glog-done .gflags-done .boost-done
+	${RECIPES_DIR}/wangle.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+.fbthrift-done: .folly-done .fizz-done .wangle-done .fmt-done .glog-done .gflags-done .boost-done
+	${RECIPES_DIR}/fbthrift.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+deps: .fbthrift-done .folly-done .fizz-done .wangle-done .fmt-done .zstd-done .glog-done .gflags-done .boost-done
+	touch $@
+
+mcrouter: deps
+	${RECIPES_DIR}/mcrouter.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
diff --git a/mcrouter/scripts/common.sh b/mcrouter/scripts/common.sh
@@ -14,7 +14,7 @@ PKG_DIR="$1"
 INSTALL_DIR="$2"
 INSTALL_AUX_DIR="$3"
 shift $#
-MAKE_ARGS="$@"
+
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 
 mkdir -p "$PKG_DIR" "$INSTALL_DIR" 

diff --git a/mcrouter/scripts/get_and_build_by_make.sh b/mcrouter/scripts/get_and_build_by_make.sh
@@ -7,15 +7,24 @@
 set -ex
 
 MAKE_FILE="$1"
-TARGET="$2"
-PKG_DIR="${3%/}"/pkgs
-INSTALL_DIR="${3%/}"/install
-INSTALL_AUX_DIR="${3%/}"/install/aux
+TARGET="$3"
+PKG_DIR="${2%/}"/pkgs
+INSTALL_DIR="${2%/}"/install
+INSTALL_AUX_DIR="${2%/}"/install/aux
 
 [ -n "$MAKE_FILE" ] || ( echo "Make file missing"; exit 1 )
 [ -n "$TARGET" ] || ( echo "Target missing"; exit 1 )
 
 mkdir -p "$PKG_DIR" "$INSTALL_DIR" "$INSTALL_AUX_DIR"
+mkdir -p "$INSTALL_DIR/lib"
+
+# The recipes/mcrouter.sh build script fails at convincing the compiler and linker
+# to look at $INSTALL_DIR/lib64
+# As a workaround, we just upfront link lib64 -> lib so all dependency artifacts
+# end up in $INSTALL_DIR/lib which that build script *can* find.
+if [ ! -e "$INSTALL_DIR/lib64" ]; then
+    ln -sf "$INSTALL_DIR/lib" "$INSTALL_DIR/lib64"
+fi
 
 cd "$(dirname "$0")" || ( echo "cd fail"; exit 1 )
 
@@ -24,6 +33,6 @@ export REPO_BASE_DIR
 
 export LDFLAGS="-ljemalloc $LDFLAGS"
 
-make "$TARGET" -j3 -f "$MAKE_FILE" PKG_DIR="$PKG_DIR" INSTALL_DIR="$INSTALL_DIR" INSTALL_AUX_DIR="$INSTALL_AUX_DIR"
+make "$TARGET" -j "$(nproc)" -f "$MAKE_FILE" PKG_DIR="$PKG_DIR" INSTALL_DIR="$INSTALL_DIR" INSTALL_AUX_DIR="$INSTALL_AUX_DIR"
 
 printf "%s\n" "make $TARGET for $MAKE_FILE done"
diff --git a/mcrouter/scripts/install_amazon-linux-2.sh b/mcrouter/scripts/install_amazon-linux-2.sh
@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -ex
+
+BASE_DIR="$1"
+TARGET="${2:-all}"
+
+[ -n "$BASE_DIR" ] || ( echo "Base dir missing"; exit 1 )
+
+sudo yum install -y epel-release
+
+sudo yum groupinstall -y "Development Tools"
+sudo yum install -y \
+    autoconf \
+    binutils-devel \
+    bison \
+    bzip2-devel \
+    cmake3 \
+    double-conversion-devel \
+    flex \
+    gcc-c++\
+    git \
+    gtest-devel \
+    jemalloc-devel \
+    libevent-devel \
+    libsodium-devel \
+    libtool \
+    libunwind-devel \
+    lz4-devel \
+    make \
+    openssl-devel \
+    python-devel \
+    ragel \
+    snappy-devel \
+    xz-devel \
+    zlib-devel
+
+# The above dependencies provide the build time requirements
+# for compiling "mcrouter" as well as a number of other dependencies.
+#
+# In order to package and deploy the resulting artifact, we need to ship
+# mcrouter along with the compiled dynamic library dependencies.
+# In addition, we need to install into the system the non development
+# version of some libraries.  This comment is here to provide an example
+# for what is necessary to provide a runtime environment.
+
+#sudo yum install -y \
+#  bzip2 \
+#  double-conversion \
+#  jemalloc \
+#  libevent \
+#  libsodium \
+#  libunwind \
+#  lz4 \
+#  snappy \
+#  openssl \
+#  xz-libz \
+#  zlib \
+
+
+# Set CC and CXX to unambiguously choose compiler.
+export CC=/usr/bin/gcc
+export CXX=/usr/bin/c++
+
+sudo ln -sf /usr/bin/cmake3 /usr/bin/cmake
+
+# Automake available by default is 1.13 and unsupported for mcrouter.
+# Install automake-1.15 from Fedora
+yum info automake-1.15-4.fc23 || sudo yum install -y "http://archives.fedoraproject.org/pub/archive/fedora/linux/releases/23/Everything/x86_64/os/Packages/a/automake-1.15-4.fc23.noarch.rpm"
+
+cd "$(dirname "$0")" || ( echo "cd fail"; exit 1 )
+
+./get_and_build_by_make.sh "Makefile_amazon-linux-2" "$BASE_DIR" "$TARGET"
diff --git a/mcrouter/scripts/recipes/boost.sh b/mcrouter/scripts/recipes/boost.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+source common.sh
+
+if [ ! -d "$PKG_DIR/boost" ]; then
+    wget https://boostorg.jfrog.io/artifactory/main/release/1.76.0/source/boost_1_76_0.tar.gz
+    tar xzf boost_1_76_0.tar.gz
+    mv boost_1_76_0 boost
+    rm -f boost*.tar.gz
+fi
+
+cd "$PKG_DIR/boost" || die "cd fail"
+
+./bootstrap.sh --prefix="$INSTALL_DIR"
+./b2 -j "$(nproc)" --prefix="$INSTALL_DIR" install
diff --git a/mcrouter/scripts/recipes/fbthrift.sh b/mcrouter/scripts/recipes/fbthrift.sh
@@ -6,20 +6,17 @@
 
 source common.sh
 
-if [[ ! -d "$PKG_DIR/fbthrift" ]]; then
+if [ ! -d "$PKG_DIR/fbthrift" ]; then
   git clone https://github.com/facebook/fbthrift
-  cd "$PKG_DIR/fbthrift" || die "cd fail"
-  if [[ -f "$REPO_BASE_DIR/mcrouter/FBTHRIFT_COMMIT" ]]; then
-    FBTHRIFT_COMMIT="$(head -n 1 "$REPO_BASE_DIR/mcrouter/FBTHRIFT_COMMIT")"
-    echo "FBTHRIFT_COMMIT file found: using fbthrift commit $FBTHRIFT_COMMIT"
-    git checkout "$FBTHRIFT_COMMIT"
-  else
-    echo "No FBTHRIFT_COMMIT file, using fbthrift HEAD=$(git rev-parse HEAD)"
-  fi
 fi
 
+cd "$PKG_DIR/fbthrift" || die "cd fail"
+
+# Use a known compatible version
+git checkout v2021.04.26.00
+
 cd "$PKG_DIR/fbthrift/build" || die "cd fbthrift failed"
 
 CXXFLAGS="$CXXFLAGS -fPIC" \
 cmake .. -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR"
-make $MAKE_ARGS && make install $MAKE_ARGS
+make -j "$(nproc)" && make install
diff --git a/mcrouter/scripts/recipes/fizz.sh b/mcrouter/scripts/recipes/fizz.sh
@@ -6,19 +6,16 @@
 
 source common.sh
 
-if [[ ! -d fizz ]]; then
+if [ ! -d "$PKG_DIR/fizz" ]; then
   git clone https://github.com/facebookincubator/fizz
-  cd "$PKG_DIR/fizz" || die "cd fail"
-  if [[ -f "$REPO_BASE_DIR/mcrouter/FIZZ_COMMIT" ]]; then
-    FIZZ_COMMIT="$(head -n 1 "$REPO_BASE_DIR/mcrouter/FIZZ_COMMIT")"
-    echo "FIZZ_COMMIT file found: using fizz commit $FIZZ_COMMIT"
-    git checkout "$FIZZ_COMMIT"
-  else
-    echo "No FIZZ_COMMIT file, using fizz HEAD=$(git rev-parse HEAD)"
-  fi
 fi
 
+cd "$PKG_DIR/fizz" || die "cd fail"
+
+# Use a known compatible version
+git checkout v2021.04.26.00
+
 cd "$PKG_DIR/fizz/fizz/" || die "cd fail"
 
 cmake . -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DBUILD_TESTS=OFF
-make $MAKE_ARGS && make install $MAKE_ARGS
+make -j "$(nproc)" && make install
diff --git a/mcrouter/scripts/recipes/fmtlib.sh b/mcrouter/scripts/recipes/fmtlib.sh
@@ -6,15 +6,19 @@
 
 source common.sh
 
-if [[ ! -d "$PKG_DIR/fmt" ]]; then
+if [ ! -d "$PKG_DIR/fmt" ]; then
   git clone https://github.com/fmtlib/fmt
-  cd "$PKG_DIR/fmt" || die "cd failed"
-  mkdir "$PKG_DIR/fmt/build"
 fi
 
+cd "$PKG_DIR/fmt" || die "cd failed"
+
+# Use a known compatible version
+git checkout 7.1.3
+
+mkdir "$PKG_DIR/fmt/build"
 cd "$PKG_DIR/fmt/build" || die "cd fmt failed"
 
 CXXFLAGS="$CXXFLAGS -fPIC" \
   cmake .. -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR"
-make $MAKE_ARGS && make install $MAKE_ARGS
+make -j "$(nproc)" && make install
 
diff --git a/mcrouter/scripts/recipes/folly.sh b/mcrouter/scripts/recipes/folly.sh
@@ -6,46 +6,20 @@
 
 source common.sh
 
-if [[ ! -d folly ]]; then
+if [ ! -d "$PKG_DIR/folly" ]; then
   git clone https://github.com/facebook/folly
-  cd "$PKG_DIR/folly" || die "cd fail"
-  if [[ -f "$REPO_BASE_DIR/mcrouter/FOLLY_COMMIT" ]]; then
-    FOLLY_COMMIT="$(head -n 1 "$REPO_BASE_DIR/mcrouter/FOLLY_COMMIT")"
-    echo "FOLLY_COMMIT file found: using folly commit $FOLLY_COMMIT"
-    git checkout "$FOLLY_COMMIT"
-  else
-    echo "No FOLLY_COMMIT file, using folly HEAD=$(git rev-parse HEAD)"
-  fi
 fi
 
-if [ ! -d /usr/include/double-conversion ]; then
-  if [ ! -d "$PKG_DIR/double-conversion" ]; then
-    cd "$PKG_DIR" || die "cd fail"
-    git clone https://github.com/google/double-conversion.git
-  fi
-  cd "$PKG_DIR/double-conversion" || die "cd fail"
+cd "$PKG_DIR/folly" || die "cd fail"
 
-  # Workaround double-conversion CMakeLists.txt changes that
-  # are incompatible with cmake-2.8
-  git checkout ea970f69edacf66bd3cba2892be284b76e9599b0
-  cmake . -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR"
-  make $MAKE_ARGS && make install $MAKE_ARGS
+# Use a known compatible version
+git checkout v2021.04.26.00
 
-  export LDFLAGS="-L$INSTALL_DIR/lib -ldl $LDFLAGS"
-  export CPPFLAGS="-I$INSTALL_DIR/include $CPPFLAGS"
-fi
-
-if [ ! -d "$PKG_DIR/zstd" ]; then
-  cd "$PKG_DIR" || die "cd fail"
-  git clone https://github.com/facebook/zstd
-
-  cd "$PKG_DIR/zstd" || die "cd fail"
-
-  # Checkout zstd-1.4.9 release
-  git checkout e4558ffd1dc49399faf4ee5d85abed4386b4dcf5
-  cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" build/cmake/
-  make $MAKE_ARGS && make install $MAKE_ARGS
-fi
+# There is an issue when compiling folly on aarch64 when libunwind is available.
+# The build configuration does expose a direct way to avoid using libunwind, and we
+# need to have it installed for other dependencies, so we just edit the cmake config
+# file to force this variable to OFF.
+sed -i 's/set(FOLLY_HAVE_LIBUNWIND ON)/set(FOLLY_HAVE_LIBUNWIND OFF)/' CMake/folly-deps.cmake
 
 cd "$PKG_DIR/folly/folly/" || die "cd fail"
 
@@ -56,4 +30,4 @@ CXXFLAGS="$CXXFLAGS -fPIC" \
     -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \
     -DCMAKE_INCLUDE_PATH="$INSTALL_DIR/lib" \
     -DCMAKE_LIBRARY_PATH="$INSTALL_DIR/lib"
-make $MAKE_ARGS && make install $MAKE_ARGS
+make -j "$(nproc)" && make install
diff --git a/mcrouter/scripts/recipes/gflags.sh b/mcrouter/scripts/recipes/gflags.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+source common.sh
+
+if [ ! -d "$PKG_DIR/gflags" ]; then
+    git clone https://github.com/gflags/gflags.git
+fi
+
+cd "$PKG_DIR/gflags" || die "cd fail"
+
+# Use a known compatible version
+# There hasn't been a release in years, this is just the (currently) most
+# recent commit.
+git checkout 827c769e5fc98e0f2a34c47cef953cc6328abced
+
+LDFLAGS="-Wl,-rpath=$INSTALL_DIR/lib,--enable-new-dtags -L$INSTALL_DIR/lib $LDFLAGS" \
+    CPPFLAGS="-I$INSTALL_DIR/include -DGOOGLE_GLOG_DLL_DECL='' $CPPFLAGS" \
+    cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DBUILD_SHARED_LIBS=YES -S . -B build -G "Unix Makefiles"
+
+cmake --build build --target install
diff --git a/mcrouter/scripts/recipes/glog.sh b/mcrouter/scripts/recipes/glog.sh
@@ -8,12 +8,16 @@ source common.sh
 
 if [ ! -d "$PKG_DIR/glog" ]; then
     git clone https://github.com/google/glog.git
-    cd "$PKG_DIR/glog" || die "cd fail"
-
-    autoreconf --install
-    LDFLAGS="-Wl,-rpath=$INSTALL_DIR/lib,--enable-new-dtags \
-             -L$INSTALL_DIR/lib $LDFLAGS" \
-        CPPFLAGS="-I$INSTALL_DIR/include -DGOOGLE_GLOG_DLL_DECL='' $CPPFLAGS" \
-        ./configure --prefix="$INSTALL_DIR" && \
-        make $MAKE_ARGS && make install $MAKE_ARGS
 fi
+
+cd "$PKG_DIR/glog" || die "cd fail"
+
+# Use a known compatible version
+# N.B. More recent versions (v0.5.0.rcx) failed to work with mcrouter.
+git reset --hard v0.4.0
+
+autoreconf --install
+LDFLAGS="-Wl,-rpath=$INSTALL_DIR/lib,--enable-new-dtags -L$INSTALL_DIR/lib $LDFLAGS" \
+    CPPFLAGS="-I$INSTALL_DIR/include -DGOOGLE_GLOG_DLL_DECL='' $CPPFLAGS" \
+    ./configure --prefix="$INSTALL_DIR" &&
+    make -j "$(nproc)" && make install