Merge pull request #1 from clubhouse/pgroudas/amazon-linux-support

.gitignore

@@ -0,0 +1 @@
+/.idea/

README.md

@@ -12,27 +12,9 @@ See https://github.com/facebook/mcrouter/wiki to get started.
 
 ## Quick start guide
 
-### New! Ubuntu package available
-
-Currently, we support Ubuntu Bionic (18.04) amd64.
-Here is how to install it:
-
-Add the repo key:
-
-    $ wget -O - https://facebook.github.io/mcrouter/debrepo/bionic/PUBLIC.KEY | sudo apt-key add
-
-Add the following line to apt sources file /etc/apt/sources.list
-
-    deb https://facebook.github.io/mcrouter/debrepo/bionic bionic contrib
-
-Update the local repo cache:
-
-    $ sudo apt-get update
-
-Install mcrouter:
-
-    $ sudo apt-get install mcrouter
+### Building on Amazon Linux 2
 
+See [INSTALL.md](/mcrouter/scripts/INSTALL.md)
 
 ### Installing From Source
 
@@ -87,6 +69,15 @@ the simplest mcrouter setup is:
 Documentation: https://github.com/facebook/mcrouter/wiki
 Engineering discussions and support: https://www.facebook.com/groups/mcrouter
 
+## Development
+
+This is an internal fork, aimed only to add packaging support for Amazon Linux 2.
+
+Branches:
+
+- upstream: This should track facebook/master
+- main: PRs can target this for review
+
 ## License
 
 Copyright (c) Facebook, Inc. and its affiliates.

mcrouter/scripts/INSTALL.md

@@ -0,0 +1,13 @@
+N.B These scripts *only* support Amazon Linux 2
+
+1. Install compilers and system-provided libraries:
+
+        ./install_deps_amazon-linux-2.sh
+
+2. Compile mcrouter and dependencies
+
+        MY_INSTALL_DIR=path/to/install/dir
+        TARGET=mcrouter
+        ./get_and_build_by_make $MY_INSTALL_DIR mcrouter
+
+You can substitute individual dependencies as the value of TARGET in order to debug. 

mcrouter/scripts/Makefile_amazon-linux-2

@@ -0,0 +1,62 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+RECIPES_DIR := ./recipes
+
+all: mcrouter
+
+# Boost available from Amazon Linux is not recent enough.
+# There *are* more recent versions available via EPEL, but they introduce python dependency conflicts.
+.boost-done:
+	${RECIPES_DIR}/boost.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+# The version of fmt from the package repositories is insufficient to satisfy the "folly" build.
+.fmt-done: .boost-done
+	${RECIPES_DIR}/fmtlib.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+# The version of gflags in the repo is insufficient for compiling mcrouter
+.gflags-done: .boost-done
+	${RECIPES_DIR}/gflags.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+# The version of glog in the repo is insufficient for compiling folly.
+# Additionally, the most recent (HEAD / v0.5.0.rc2) revision from upstream is incompatible for compiling mcrouter.
+# Fortunately, v0.4.0 works for both, so we just check out that revision.
+.glog-done: .gflags-done .boost-done
+	${RECIPES_DIR}/glog.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+.zstd-done:
+	${RECIPES_DIR}/zstd.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+.folly-done: .zstd-done .glog-done .gflags-done .boost-done .fmt-done
+	${RECIPES_DIR}/folly.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+.fizz-done: .folly-done .glog-done .gflags-done .boost-done
+	${RECIPES_DIR}/fizz.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+.wangle-done: .folly-done .fizz-done .glog-done .gflags-done .boost-done
+	${RECIPES_DIR}/wangle.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+.fbthrift-done: .folly-done .fizz-done .wangle-done .fmt-done .glog-done .gflags-done .boost-done
+	${RECIPES_DIR}/fbthrift.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+deps: .fbthrift-done .folly-done .fizz-done .wangle-done .fmt-done .zstd-done .glog-done .gflags-done .boost-done
+	touch $@
+
+mcrouter: deps
+	${RECIPES_DIR}/mcrouter.sh $(PKG_DIR) $(INSTALL_DIR) $(INSTALL_AUX_DIR)
+	touch $@
+
+archive: mcrouter
+	touch $@
+	tar czfv "mcrouter.$$(git rev-parse --short=12 HEAD).$$(uname -m).tar.gz" -C $(INSTALL_DIR) .

mcrouter/scripts/common.sh

@@ -7,17 +7,26 @@ set -ex
 
 function die { printf "%s: %s\n" "$0" "$@"; exit 1; }
 
+# Ensures a git repository working tree is set to the specific commit
+function gitEnsureTreeish {
+    # In the case of a commit or tag, this will put the repo into DETACHED HEAD state
+    git checkout "$1" --
+    # Ensures that there are edited files in git index or working directory
+    git reset --hard HEAD
+}
+
 [ -n "$1" ] || die "PKG_DIR missing"
 [ -n "$2" ] || die "INSTALL_DIR missing"
 
 PKG_DIR="$1"
 INSTALL_DIR="$2"
 INSTALL_AUX_DIR="$3"
 shift $#
-MAKE_ARGS="$@"
+
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+export SCRIPT_DIR
 
-mkdir -p "$PKG_DIR" "$INSTALL_DIR" 
+mkdir -p "$PKG_DIR" "$INSTALL_DIR"
 
 if [ -n "$3" ]; then
 	mkdir -p "$INSTALL_AUX_DIR"

mcrouter/scripts/get_and_build_by_make.sh

@@ -6,24 +6,41 @@
 
 set -ex
 
-MAKE_FILE="$1"
 TARGET="$2"
-PKG_DIR="${3%/}"/pkgs
-INSTALL_DIR="${3%/}"/install
-INSTALL_AUX_DIR="${3%/}"/install/aux
+PKG_DIR="${1%/}"/pkgs
+INSTALL_DIR="${1%/}"/install
+INSTALL_AUX_DIR="${1%/}"/install/aux
 
-[ -n "$MAKE_FILE" ] || ( echo "Make file missing"; exit 1 )
 [ -n "$TARGET" ] || ( echo "Target missing"; exit 1 )
 
+SYSTEM_MEM="$(free -g | grep "^Mem:" | tr -s '[:blank:]' | cut -d ' ' -f 2)"
+if (( ${SYSTEM_MEM:-0} <= 25 )); then
+    echo "mcrouter needs a lot of memory to compile, and is known to exhaust memory on 16gb systems." >&2
+    echo "On EC2 consider an instance with 32 GB of memory, like an m6g.2xlarge or m5a.2xlarge" >&2
+    exit 1
+fi
+
 mkdir -p "$PKG_DIR" "$INSTALL_DIR" "$INSTALL_AUX_DIR"
+mkdir -p "$INSTALL_DIR/lib"
+
+# The recipes/mcrouter.sh build script fails at convincing the compiler and linker
+# to look at $INSTALL_DIR/lib64
+# As a workaround, we just upfront link lib64 -> lib so all dependency artifacts
+# end up in $INSTALL_DIR/lib which that build script *can* find.
+if [ ! -e "$INSTALL_DIR/lib64" ]; then
+    ln -sf "$INSTALL_DIR/lib" "$INSTALL_DIR/lib64"
+fi
 
 cd "$(dirname "$0")" || ( echo "cd fail"; exit 1 )
 
 REPO_BASE_DIR="$(cd ../../ && pwd)" || die "Couldn't determine repo top dir"
 export REPO_BASE_DIR
 
 export LDFLAGS="-ljemalloc $LDFLAGS"
+# Set CC and CXX to unambiguously choose compiler.
+export CC=/usr/bin/gcc
+export CXX=/usr/bin/c++
 
-make "$TARGET" -j3 -f "$MAKE_FILE" PKG_DIR="$PKG_DIR" INSTALL_DIR="$INSTALL_DIR" INSTALL_AUX_DIR="$INSTALL_AUX_DIR"
+make "$TARGET" -j "$(nproc)" -f "Makefile_amazon-linux-2" PKG_DIR="$PKG_DIR" INSTALL_DIR="$INSTALL_DIR" INSTALL_AUX_DIR="$INSTALL_AUX_DIR"
 
 printf "%s\n" "make $TARGET for $MAKE_FILE done"