From b79d6977aab86847c5a11a81ca8853f5eb724b8c Mon Sep 17 00:00:00 2001 From: Paul Groudas Date: Mon, 3 May 2021 15:46:14 -0400 Subject: [PATCH 1/3] Fix Makefile I am unsure as to why, but the dependency expressed as "mcrouter: deps" doesn't seem to work correctly. When invoking the equivalent of "make mcrouter", I expect it to build all the transitive dependencies of "deps", but it only builds the first dependency that it determines it needs to build, and then exits with a return value of "2". I'm just reverting this to the previous style that is redundant but works correctly. --- mcrouter/scripts/Makefile_amazon-linux-2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcrouter/scripts/Makefile_amazon-linux-2 b/mcrouter/scripts/Makefile_amazon-linux-2 index 250091eed..8ab5eaca6 100644 --- a/mcrouter/scripts/Makefile_amazon-linux-2 +++ b/mcrouter/scripts/Makefile_amazon-linux-2 @@ -53,7 +53,7 @@ all: mcrouter deps: .fbthrift-done .folly-done .fizz-done .wangle-done .fmt-done .zstd-done .glog-done .gflags-done .boost-done touch $@ -mcrouter: deps +mcrouter: .fbthrift-done .folly-done .fizz-done .wangle-done .fmt-done .zstd-done .glog-done .gflags-done .boost-done ${RECIPES_DIR}/mcrouter.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR) touch $@ From abbe369fce7091ac1d9297c2d255f58ce7f878d3 Mon Sep 17 00:00:00 2001 From: Paul Groudas Date: Mon, 3 May 2021 15:48:24 -0400 Subject: [PATCH 2/3] "git clean" during checkout of dependency This repo defines "recipes" for dependencies which are essentially build scripts. Some of the build systems used (or at least the way they are used) will cache some state between invocations, which is confusing when testing different values for things like "LDFLAGS" and "LD_LIBRARY_PATH". Specifically, the cmake based build scripts. This change ensures that when force re-running a given recipe, we do a full clean of the repo. n.b.: we are *not* cleaning the target install dir. --- mcrouter/scripts/common.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/mcrouter/scripts/common.sh b/mcrouter/scripts/common.sh index 9d2050918..9dac59ddd 100644 --- a/mcrouter/scripts/common.sh +++ b/mcrouter/scripts/common.sh @@ -13,6 +13,7 @@ function gitEnsureTreeish { git checkout "$1" -- # Ensures that there are edited files in git index or working directory git reset --hard HEAD + git clean -fdx . } [ -n "$1" ] || die "PKG_DIR missing" From 06fc2300572a50dbcf1949048948fa7a8883b32f Mon Sep 17 00:00:00 2001 From: Paul Groudas Date: Mon, 3 May 2021 15:52:13 -0400 Subject: [PATCH 3/3] Build jemalloc from source. This introduces "jemalloc" as a library dependency that we build from source. While this library is available from our runtime system's package respository, mcrouter would log an error at startup implying that there is a page size mis-match between the current kernel, and the system on which jemalloc was compiled. As this package available via the EPEL[1] and not Amazon Linux directly, this isn't entirely surprising. Compiling "jemalloc" is very straightforward, but getting all the dependencies to correctly link against it during their builds was a bit challenging. Ultimately I made the following changes: 1. The default value we set for *all* recipes is: LDFLAGS="-L$INSTALL_DIR/lib -ljemalloc - "-ljemalloc" means "link against the 'jemalloc'" library. - "-L$INSTALL_DIR/lib" search "$INSTALL_DIR/lib" when looking for libraries. 2. I removed redundant LDFLAGS from each recipe following the change to the default. 3. Some builds, notable those that produce binary executables (not just shared libraries), also need to include "-Wl,-rpath=$INSTALL_DIR/lib", which embeds metadata about the dynamic libraries' location into the executable. Conceptually, this doesn't make sense to me, as we are expecting users to manage their own library path, but not setting this causes the builds to fail, and I have a limited appetite for debugging and understanding these build tools. [1]: https://fedoraproject.org/wiki/EPEL --- mcrouter/scripts/Makefile_amazon-linux-2 | 32 ++++++++++++------- mcrouter/scripts/get_and_build_by_make.sh | 2 +- .../scripts/install_deps_amazon-linux-2.sh | 2 -- mcrouter/scripts/recipes/fbthrift.sh | 1 + mcrouter/scripts/recipes/fmtlib.sh | 3 +- mcrouter/scripts/recipes/gflags.sh | 2 +- mcrouter/scripts/recipes/glog.sh | 2 +- mcrouter/scripts/recipes/jemalloc.sh | 19 +++++++++++ mcrouter/scripts/recipes/mcrouter.sh | 1 - 9 files changed, 45 insertions(+), 19 deletions(-) create mode 100755 mcrouter/scripts/recipes/jemalloc.sh diff --git a/mcrouter/scripts/Makefile_amazon-linux-2 b/mcrouter/scripts/Makefile_amazon-linux-2 index 8ab5eaca6..fe46e64b9 100644 --- a/mcrouter/scripts/Makefile_amazon-linux-2 +++ b/mcrouter/scripts/Makefile_amazon-linux-2 @@ -7,53 +7,63 @@ RECIPES_DIR := ./recipes all: mcrouter +# jemalloc is available from the package repositories, but then we get this warning at mcrouter startup: +# : Error in munmap(): Invalid argument which corresponds to https://github.com/jemalloc/jemalloc/issues/467 +# As we build on the same system we deploy to (our amazon linux AMI), I hope that this results with consistent +# page sizes and no error. +.jemalloc-done: + # We set "LDFLAGS" in the "get_and_build_by_make.sh" script that tells all + # projects to link against jemalloc. We unset that when we actually compile jemalloc. + LDFLAGS="" ${RECIPES_DIR}/jemalloc.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR) + touch $@ + # Boost available from Amazon Linux is not recent enough. # There *are* more recent versions available via EPEL, but they introduce python dependency conflicts. -.boost-done: +.boost-done: .jemalloc-done ${RECIPES_DIR}/boost.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR) touch $@ # The version of fmt from the package repositories is insufficient to satisfy the "folly" build. -.fmt-done: .boost-done +.fmt-done: .boost-done .jemalloc-done ${RECIPES_DIR}/fmtlib.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR) touch $@ # The version of gflags in the repo is insufficient for compiling mcrouter -.gflags-done: .boost-done +.gflags-done: .boost-done .jemalloc-done ${RECIPES_DIR}/gflags.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR) touch $@ # The version of glog in the repo is insufficient for compiling folly. # Additionally, the most recent (HEAD / v0.5.0.rc2) revision from upstream is incompatible for compiling mcrouter. # Fortunately, v0.4.0 works for both, so we just check out that revision. -.glog-done: .gflags-done .boost-done +.glog-done: .gflags-done .boost-done .jemalloc-done ${RECIPES_DIR}/glog.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR) touch $@ -.zstd-done: +.zstd-done: .jemalloc-done ${RECIPES_DIR}/zstd.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR) touch $@ -.folly-done: .zstd-done .glog-done .gflags-done .boost-done .fmt-done +.folly-done: .zstd-done .glog-done .gflags-done .boost-done .fmt-done .jemalloc-done ${RECIPES_DIR}/folly.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR) touch $@ -.fizz-done: .folly-done .glog-done .gflags-done .boost-done +.fizz-done: .folly-done .glog-done .gflags-done .boost-done .jemalloc-done ${RECIPES_DIR}/fizz.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR) touch $@ -.wangle-done: .folly-done .fizz-done .glog-done .gflags-done .boost-done +.wangle-done: .folly-done .fizz-done .glog-done .gflags-done .boost-done .jemalloc-done ${RECIPES_DIR}/wangle.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR) touch $@ -.fbthrift-done: .folly-done .fizz-done .wangle-done .fmt-done .glog-done .gflags-done .boost-done +.fbthrift-done: .folly-done .fizz-done .wangle-done .fmt-done .glog-done .gflags-done .boost-done .jemalloc-done ${RECIPES_DIR}/fbthrift.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR) touch $@ -deps: .fbthrift-done .folly-done .fizz-done .wangle-done .fmt-done .zstd-done .glog-done .gflags-done .boost-done +deps: .fbthrift-done .folly-done .fizz-done .wangle-done .fmt-done .zstd-done .glog-done .gflags-done .boost-done .jemalloc-done touch $@ -mcrouter: .fbthrift-done .folly-done .fizz-done .wangle-done .fmt-done .zstd-done .glog-done .gflags-done .boost-done +mcrouter: .fbthrift-done .folly-done .fizz-done .wangle-done .fmt-done .zstd-done .glog-done .gflags-done .boost-done .jemalloc-done ${RECIPES_DIR}/mcrouter.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR) touch $@ diff --git a/mcrouter/scripts/get_and_build_by_make.sh b/mcrouter/scripts/get_and_build_by_make.sh index d4a9876c4..50e4f9942 100755 --- a/mcrouter/scripts/get_and_build_by_make.sh +++ b/mcrouter/scripts/get_and_build_by_make.sh @@ -36,7 +36,7 @@ cd "$(dirname "$0")" || ( echo "cd fail"; exit 1 ) REPO_BASE_DIR="$(cd ../../ && pwd)" || die "Couldn't determine repo top dir" export REPO_BASE_DIR -export LDFLAGS="-ljemalloc $LDFLAGS" +export LDFLAGS="-L$INSTALL_DIR/lib -ljemalloc $LDFLAGS" # Set CC and CXX to unambiguously choose compiler. export CC=/usr/bin/gcc export CXX=/usr/bin/c++ diff --git a/mcrouter/scripts/install_deps_amazon-linux-2.sh b/mcrouter/scripts/install_deps_amazon-linux-2.sh index 58a0ab937..c5c6783f2 100755 --- a/mcrouter/scripts/install_deps_amazon-linux-2.sh +++ b/mcrouter/scripts/install_deps_amazon-linux-2.sh @@ -20,7 +20,6 @@ sudo yum install -y \ gcc-c++\ git \ gtest-devel \ - jemalloc-devel \ libevent-devel \ libsodium-devel \ libtool \ @@ -46,7 +45,6 @@ sudo yum install -y \ #sudo yum install -y \ # bzip2 \ # double-conversion \ -# jemalloc \ # libevent \ # libsodium \ # libunwind \ diff --git a/mcrouter/scripts/recipes/fbthrift.sh b/mcrouter/scripts/recipes/fbthrift.sh index f1cc43a5b..de864046a 100755 --- a/mcrouter/scripts/recipes/fbthrift.sh +++ b/mcrouter/scripts/recipes/fbthrift.sh @@ -18,5 +18,6 @@ gitEnsureTreeish v2021.04.26.00 cd "$PKG_DIR/fbthrift/build" || die "cd fbthrift failed" CXXFLAGS="$CXXFLAGS -fPIC" \ +LDFLAGS="-Wl,-rpath=$INSTALL_DIR/lib $LDFLAGS" \ cmake .. -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" make -j "$(nproc)" && make install diff --git a/mcrouter/scripts/recipes/fmtlib.sh b/mcrouter/scripts/recipes/fmtlib.sh index e7e064c45..741b88a7e 100755 --- a/mcrouter/scripts/recipes/fmtlib.sh +++ b/mcrouter/scripts/recipes/fmtlib.sh @@ -15,10 +15,9 @@ cd "$PKG_DIR/fmt" || die "cd failed" # Use a known compatible version gitEnsureTreeish 7.1.3 -mkdir "$PKG_DIR/fmt/build" +mkdir -p "$PKG_DIR/fmt/build" cd "$PKG_DIR/fmt/build" || die "cd fmt failed" CXXFLAGS="$CXXFLAGS -fPIC" \ cmake .. -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" make -j "$(nproc)" && make install - diff --git a/mcrouter/scripts/recipes/gflags.sh b/mcrouter/scripts/recipes/gflags.sh index dd790490b..b8fa9d890 100755 --- a/mcrouter/scripts/recipes/gflags.sh +++ b/mcrouter/scripts/recipes/gflags.sh @@ -17,7 +17,7 @@ cd "$PKG_DIR/gflags" || die "cd fail" # recent commit. gitEnsureTreeish 827c769e5fc98e0f2a34c47cef953cc6328abced -LDFLAGS="-Wl,-rpath=$INSTALL_DIR/lib,--enable-new-dtags -L$INSTALL_DIR/lib $LDFLAGS" \ +LDFLAGS="-Wl,-rpath=$INSTALL_DIR/lib,--enable-new-dtags $LDFLAGS" \ CPPFLAGS="-I$INSTALL_DIR/include -DGOOGLE_GLOG_DLL_DECL='' $CPPFLAGS" \ cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DBUILD_SHARED_LIBS=YES -S . -B build -G "Unix Makefiles" diff --git a/mcrouter/scripts/recipes/glog.sh b/mcrouter/scripts/recipes/glog.sh index 27d9f8cd3..962418dd6 100755 --- a/mcrouter/scripts/recipes/glog.sh +++ b/mcrouter/scripts/recipes/glog.sh @@ -17,7 +17,7 @@ cd "$PKG_DIR/glog" || die "cd fail" gitEnsureTreeish v0.4.0 autoreconf --install -LDFLAGS="-Wl,-rpath=$INSTALL_DIR/lib,--enable-new-dtags -L$INSTALL_DIR/lib $LDFLAGS" \ +LDFLAGS="-Wl,-rpath=$INSTALL_DIR/lib,--enable-new-dtags $LDFLAGS" \ CPPFLAGS="-I$INSTALL_DIR/include -DGOOGLE_GLOG_DLL_DECL='' $CPPFLAGS" \ ./configure --prefix="$INSTALL_DIR" && make -j "$(nproc)" && make install diff --git a/mcrouter/scripts/recipes/jemalloc.sh b/mcrouter/scripts/recipes/jemalloc.sh new file mode 100755 index 000000000..a870fa046 --- /dev/null +++ b/mcrouter/scripts/recipes/jemalloc.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +source common.sh + +if [ ! -d "$PKG_DIR/jemalloc" ]; then + git clone https://github.com/jemalloc/jemalloc +fi + +cd "$PKG_DIR/jemalloc" || die "cd failed" + +# Use a known compatible version +gitEnsureTreeish 5.2.1 + +./autogen.sh --prefix="$INSTALL_DIR" +make -j "$(nproc)" && make install diff --git a/mcrouter/scripts/recipes/mcrouter.sh b/mcrouter/scripts/recipes/mcrouter.sh index 3f1bc76c6..7ea3ae5bb 100755 --- a/mcrouter/scripts/recipes/mcrouter.sh +++ b/mcrouter/scripts/recipes/mcrouter.sh @@ -11,7 +11,6 @@ cd "$SCRIPT_DIR/../.." || die "cd fail" autoreconf --install LD_LIBRARY_PATH="$INSTALL_DIR/lib:$LD_LIBRARY_PATH" \ LD_RUN_PATH="$INSTALL_DIR/lib:$LD_RUN_PATH" \ - LDFLAGS="-L$INSTALL_DIR/lib $LDFLAGS" \ CPPFLAGS="-I$INSTALL_DIR/include $CPPFLAGS" \ FBTHRIFT_BIN="$INSTALL_DIR/bin/" \ ./configure --prefix="$INSTALL_DIR" --with-boost-libdir="$INSTALL_DIR/lib"