Build jemalloc from source.

This introduces "jemalloc" as a library dependency that we build from
source.

While this library is available from our runtime system's package
respository, mcrouter would log an error at startup implying that there
is a page size mis-match between the current kernel, and the system on
which jemalloc was compiled.

As this package available via the EPEL[1] and not Amazon Linux directly,
this isn't entirely surprising.

Compiling "jemalloc" is very straightforward, but getting all the
dependencies to correctly link against it during their builds was a bit
challenging.  Ultimately I made the following changes:

1. The default value we set for *all* recipes is:

        LDFLAGS="-L$INSTALL_DIR/lib -ljemalloc

- "-ljemalloc" means "link against the 'jemalloc'" library.
- "-L$INSTALL_DIR/lib" search "$INSTALL_DIR/lib" when looking for
libraries.

2. I removed redundant LDFLAGS from each recipe following the change to
the default.

3. Some builds, notable those that produce binary executables (not just
shared libraries), also need to include "-Wl,-rpath=$INSTALL_DIR/lib",
which embeds metadata about the dynamic libraries' location into the
executable.  Conceptually, this doesn't make sense to me, as we are
expecting users to manage their own library path, but not setting this
causes the builds to fail, and I have a limited appetite for debugging
and understanding these build tools.

[1]: https://fedoraproject.org/wiki/EPEL

mcrouter/scripts/Makefile_amazon-linux-2

@@ -7,53 +7,63 @@ RECIPES_DIR := ./recipes
 
 all: mcrouter
 
+# jemalloc is available from the package repositories, but then we get this warning at mcrouter startup:
+# <jemalloc>: Error in munmap(): Invalid argument which corresponds to https://github.com/jemalloc/jemalloc/issues/467
+# As we build on the same system we deploy to (our amazon linux AMI), I hope that this results with consistent
+# page sizes and no error.
+.jemalloc-done:
+    # We set "LDFLAGS" in the "get_and_build_by_make.sh" script that tells all
+    # projects to link against jemalloc.  We unset that when we actually compile jemalloc.
+	LDFLAGS="" ${RECIPES_DIR}/jemalloc.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
 # Boost available from Amazon Linux is not recent enough.
 # There *are* more recent versions available via EPEL, but they introduce python dependency conflicts.
-.boost-done:
+.boost-done: .jemalloc-done
 	${RECIPES_DIR}/boost.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
 	touch $@
 
 # The version of fmt from the package repositories is insufficient to satisfy the "folly" build.
-.fmt-done: .boost-done
+.fmt-done: .boost-done .jemalloc-done
 	${RECIPES_DIR}/fmtlib.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
 	touch $@
 
 # The version of gflags in the repo is insufficient for compiling mcrouter
-.gflags-done: .boost-done
+.gflags-done: .boost-done .jemalloc-done
 	${RECIPES_DIR}/gflags.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
 	touch $@
 
 # The version of glog in the repo is insufficient for compiling folly.
 # Additionally, the most recent (HEAD / v0.5.0.rc2) revision from upstream is incompatible for compiling mcrouter.
 # Fortunately, v0.4.0 works for both, so we just check out that revision.
-.glog-done: .gflags-done .boost-done
+.glog-done: .gflags-done .boost-done .jemalloc-done
 	${RECIPES_DIR}/glog.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
 	touch $@
 
-.zstd-done:
+.zstd-done: .jemalloc-done
 	${RECIPES_DIR}/zstd.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
 	touch $@
 
-.folly-done: .zstd-done .glog-done .gflags-done .boost-done .fmt-done
+.folly-done: .zstd-done .glog-done .gflags-done .boost-done .fmt-done .jemalloc-done
 	${RECIPES_DIR}/folly.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
 	touch $@
 
-.fizz-done: .folly-done .glog-done .gflags-done .boost-done
+.fizz-done: .folly-done .glog-done .gflags-done .boost-done .jemalloc-done
 	${RECIPES_DIR}/fizz.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
 	touch $@
 
-.wangle-done: .folly-done .fizz-done .glog-done .gflags-done .boost-done
+.wangle-done: .folly-done .fizz-done .glog-done .gflags-done .boost-done .jemalloc-done
 	${RECIPES_DIR}/wangle.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
 	touch $@
 
-.fbthrift-done: .folly-done .fizz-done .wangle-done .fmt-done .glog-done .gflags-done .boost-done
+.fbthrift-done: .folly-done .fizz-done .wangle-done .fmt-done .glog-done .gflags-done .boost-done .jemalloc-done
 	${RECIPES_DIR}/fbthrift.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
 	touch $@
 
-deps: .fbthrift-done .folly-done .fizz-done .wangle-done .fmt-done .zstd-done .glog-done .gflags-done .boost-done
+deps: .fbthrift-done .folly-done .fizz-done .wangle-done .fmt-done .zstd-done .glog-done .gflags-done .boost-done .jemalloc-done
 	touch $@
 
-mcrouter: .fbthrift-done .folly-done .fizz-done .wangle-done .fmt-done .zstd-done .glog-done .gflags-done .boost-done
+mcrouter: .fbthrift-done .folly-done .fizz-done .wangle-done .fmt-done .zstd-done .glog-done .gflags-done .boost-done .jemalloc-done
 	${RECIPES_DIR}/mcrouter.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
 	touch $@
 

mcrouter/scripts/get_and_build_by_make.sh

@@ -36,7 +36,7 @@ cd "$(dirname "$0")" || ( echo "cd fail"; exit 1 )
 REPO_BASE_DIR="$(cd ../../ && pwd)" || die "Couldn't determine repo top dir"
 export REPO_BASE_DIR
 
-export LDFLAGS="-ljemalloc $LDFLAGS"
+export LDFLAGS="-L$INSTALL_DIR/lib -ljemalloc $LDFLAGS"
 # Set CC and CXX to unambiguously choose compiler.
 export CC=/usr/bin/gcc
 export CXX=/usr/bin/c++

mcrouter/scripts/install_deps_amazon-linux-2.sh

@@ -20,7 +20,6 @@ sudo yum install -y \
     gcc-c++\
     git \
     gtest-devel \
-    jemalloc-devel \
     libevent-devel \
     libsodium-devel \
     libtool \
@@ -46,7 +45,6 @@ sudo yum install -y \
 #sudo yum install -y \
 #  bzip2 \
 #  double-conversion \
-#  jemalloc \
 #  libevent \
 #  libsodium \
 #  libunwind \

mcrouter/scripts/recipes/fbthrift.sh

@@ -18,5 +18,6 @@ gitEnsureTreeish v2021.04.26.00
 cd "$PKG_DIR/fbthrift/build" || die "cd fbthrift failed"
 
 CXXFLAGS="$CXXFLAGS -fPIC" \
+LDFLAGS="-Wl,-rpath=$INSTALL_DIR/lib $LDFLAGS" \
 cmake .. -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR"
 make -j "$(nproc)" && make install

mcrouter/scripts/recipes/fmtlib.sh

@@ -15,10 +15,9 @@ cd "$PKG_DIR/fmt" || die "cd failed"
 # Use a known compatible version
 gitEnsureTreeish 7.1.3
 
-mkdir "$PKG_DIR/fmt/build"
+mkdir -p "$PKG_DIR/fmt/build"
 cd "$PKG_DIR/fmt/build" || die "cd fmt failed"
 
 CXXFLAGS="$CXXFLAGS -fPIC" \
   cmake .. -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR"
 make -j "$(nproc)" && make install
-

mcrouter/scripts/recipes/gflags.sh

@@ -17,7 +17,7 @@ cd "$PKG_DIR/gflags" || die "cd fail"
 # recent commit.
 gitEnsureTreeish 827c769e5fc98e0f2a34c47cef953cc6328abced
 
-LDFLAGS="-Wl,-rpath=$INSTALL_DIR/lib,--enable-new-dtags -L$INSTALL_DIR/lib $LDFLAGS" \
+LDFLAGS="-Wl,-rpath=$INSTALL_DIR/lib,--enable-new-dtags $LDFLAGS" \
     CPPFLAGS="-I$INSTALL_DIR/include -DGOOGLE_GLOG_DLL_DECL='' $CPPFLAGS" \
     cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DBUILD_SHARED_LIBS=YES -S . -B build -G "Unix Makefiles"
 

mcrouter/scripts/recipes/glog.sh

@@ -17,7 +17,7 @@ cd "$PKG_DIR/glog" || die "cd fail"
 gitEnsureTreeish v0.4.0
 
 autoreconf --install
-LDFLAGS="-Wl,-rpath=$INSTALL_DIR/lib,--enable-new-dtags -L$INSTALL_DIR/lib $LDFLAGS" \
+LDFLAGS="-Wl,-rpath=$INSTALL_DIR/lib,--enable-new-dtags $LDFLAGS" \
     CPPFLAGS="-I$INSTALL_DIR/include -DGOOGLE_GLOG_DLL_DECL='' $CPPFLAGS" \
     ./configure --prefix="$INSTALL_DIR" &&
     make -j "$(nproc)" && make install

mcrouter/scripts/recipes/jemalloc.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+source common.sh
+
+if [ ! -d "$PKG_DIR/jemalloc" ]; then
+  git clone https://github.com/jemalloc/jemalloc
+fi
+
+cd "$PKG_DIR/jemalloc" || die "cd failed"
+
+# Use a known compatible version
+gitEnsureTreeish 5.2.1
+
+./autogen.sh --prefix="$INSTALL_DIR"
+make -j "$(nproc)" && make install

mcrouter/scripts/recipes/mcrouter.sh

@@ -11,7 +11,6 @@ cd "$SCRIPT_DIR/../.." || die "cd fail"
 autoreconf --install
 LD_LIBRARY_PATH="$INSTALL_DIR/lib:$LD_LIBRARY_PATH" \
     LD_RUN_PATH="$INSTALL_DIR/lib:$LD_RUN_PATH" \
-    LDFLAGS="-L$INSTALL_DIR/lib $LDFLAGS" \
     CPPFLAGS="-I$INSTALL_DIR/include $CPPFLAGS" \
     FBTHRIFT_BIN="$INSTALL_DIR/bin/" \
     ./configure --prefix="$INSTALL_DIR" --with-boost-libdir="$INSTALL_DIR/lib"