diff --git a/.gitignore b/.gitignore index e76ac3e60..ebb163ae7 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ CMakeCache.txt *.o-* *.a *.log +*.tgz leveldb*/ NuRaft*/ @@ -54,7 +55,6 @@ plots/ blocks.dat test_db - # System files .DS_Store .dirstamp @@ -72,3 +72,6 @@ build/tests # E2E Test results testruns/ + +# Virtualenv +.py_venv/ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..5e065199b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +setuptools +eth-hash +matplotlib +numpy +pylint diff --git a/scripts/activate-venv.sh b/scripts/activate-venv.sh new file mode 100755 index 000000000..44a43cf97 --- /dev/null +++ b/scripts/activate-venv.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +if ! (return 0 2>/dev/null); then + echo -e "\nThis script must be sourced, see below\n" + echo -e "Usage: 'source scripts/activate-venv.sh'\n" + exit 1 +fi + +ENV_NAME=".py_venv" +PWD=$(pwd) + +check_venv_location() { + # see if directory for venv exists + if [[ $# -lt 2 ]]; then + echo -e "\nUsage: check_venv_location ENV_NAME ENV_LOCATION\n" + return 1 + fi + ENV_NAME="$1"; ENV_LOCATION="$2" + # if venv does not exist, send error message and return + if [[ ! -d "$ENV_LOCATION" ]]; then + echo -e "\nVirtual environment '${ENV_NAME}' not found at location: '${ENV_LOCATION}'" + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + echo -e "run 'sudo ./scripts/install-build-tools.sh' to create it.\n" + elif [[ "$OSTYPE" == "darwin"* ]]; then + echo -e "run './scripts/install-build-tools.sh' to create it.\n" + fi + return 1 + fi + return 0 +} + +activate_venv() { + # activate the virtual environment (we know venv exists) + if [[ $# -lt 2 ]]; then + echo -e "\nUsage: activate_venv ENV_NAME ENV_LOCATION\n" + return 1 + fi + ENV_NAME="$1"; ENV_LOCATION="$2" + if source "${ENV_LOCATION}/bin/activate"; then + echo -e "\nVirtual environment '${ENV_NAME}' activated." + echo -e "Run 'deactivate' to exit the virtual environment.\n" + return 0 + else + echo -e "Failed to activate virtual environment '${ENV_NAME}'\n" + return 1 + fi +} + +# see if we are in a git repository +if git rev-parse --show-toplevel >/dev/null 2>&1; then + ROOT="$(git rev-parse --show-toplevel)" + ENV_LOCATION="${ROOT}/${ENV_NAME}" + [[ "$ROOT" == "/" ]] && ENV_LOCATION="/${ENV_NAME}" + if check_venv_location "$ENV_NAME" "$ENV_LOCATION"; then + activate_venv "$ENV_NAME" "$ENV_LOCATION" && return 0 + fi +# try to find venv in current or parent directories +else + # figure out env location; don't cd up beyond root + ROOT="$(cd "$(dirname "$0")" && pwd)" + while [[ $ROOT != "/" ]]; do + ENV_LOCATION="${ROOT}/${ENV_NAME}" + if check_venv_location "$ENV_NAME" "$ENV_LOCATION"; then + activate_venv "$ENV_NAME" "$ENV_LOCATION" && return 0 + fi + ROOT="$(cd "${ROOT}"/.. && pwd)" + done + ENV_LOCATION="/${ENV_NAME}" + if check_venv_location "$ENV_NAME" "$ENV_LOCATION"; then + activate_venv "$ENV_NAME" "$ENV_LOCATION" && return 0 + fi +fi + +echo -e "Virtual environment '${ENV_NAME}' not found and/or failed to activate." +return 1 diff --git a/scripts/install-build-tools.sh b/scripts/install-build-tools.sh index 592e8929b..355ab2072 100755 --- a/scripts/install-build-tools.sh +++ b/scripts/install-build-tools.sh @@ -6,6 +6,7 @@ green="\033[0;32m" cyan="\033[0;36m" end="\033[0m" +# for debugging, 'set -x' can be added to trace the commands set -e SUDO='' @@ -14,36 +15,212 @@ if (( $EUID != 0 )); then SUDO='sudo' fi +# Supporting these versions for buildflow +PYTHON_VERSIONS=("3.10" "3.11" "3.12") +echo "Python3 versions supported: ${PYTHON_VERSIONS[@]}" + +# check if supported version of python3 is already installed, and save the version +PY_INSTALLED='' +for PY_VERS in "${PYTHON_VERSIONS[@]}"; do + if which "python${PY_VERS}" &> /dev/null; then + # save the installed version if found + PY_INSTALLED=${PY_VERS} + echo "Detected compatible python${PY_VERS} installed" + break + fi +done + +ROOT="$(cd "$(dirname "$0")"/.. && pwd)" +ENV_NAME=".py_venv" + +# make a virtual environement to install python packages +create_venv_install_python() { + PY_LOC=$1 + if [[ -z "$PY_LOC" ]]; then + echo "Python path not provided" + exit 1 + fi + PY_VERSION=$2 + if [[ -z "$PY_VERSION" ]]; then + echo "python version not provided" + exit 1 + fi + # remove existing virtual environment if it exists - global var + ENV_LOCATION="${ROOT}/${ENV_NAME}" + if [[ -d "${ENV_LOCATION}" ]]; then + if rm -rf -- "${ENV_LOCATION}"; then + echo "Removed existing virtual environment" + else + echo "Failed to remove existing virtual environment"; exit 1 + fi + fi + # install pip for linux + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + if ! $SUDO apt install -y python3-pip; then + echo "Failed to install python3-pip" + wget https://bootstrap.pypa.io/get-pip.py + $SUDO python${PY_VERSION} get-pip.py + rm get-pip.py + fi + # add deadsnakes to download the python venv module + $SUDO add-apt-repository -y ppa:deadsnakes/ppa + # make sure deadsnakes is available + DEADSNAKES_AVAIL=$(wget -q --spider http://ppa.launchpad.net/deadsnakes/ppa/ubuntu/dists/focal/Release; echo $?) + if [[ $DEADSNAKES_AVAIL -ne 0 ]]; then + echo "Failed to add deadsnakes which is needed to install python3" + exit 1 + fi + # install python3 venv module for linux + if ! $SUDO apt install -y "python${PY_VERSION}-venv"; then + echo "Failed to install python${PY_VERSION}-venv" + exit 1 + else + echo "Installed python${PY_VERSION}-venv" + fi + fi + # create virtual environment with specific python version + if "${PY_LOC}" -m venv "${ENV_NAME}"; then + echo "Virtual environment '${ENV_NAME}' created" + else + echo "Virtual environment creation failed"; exit 1 + fi + # activate virtual environment + source "${ROOT}/scripts/activate-venv.sh" + # install python packages + if pip install -r "${ROOT}/requirements.txt"; then + echo "Success installing python packages" + else + echo "Failure installing python packages" + deactivate + exit 1 + fi + deactivate +} + +echo "OS Type: $OSTYPE" +# macOS install with homebrew if [[ "$OSTYPE" == "darwin"* ]]; then + + # macOS does not support running shell scripts as root with homebrew + if [[ $EUID -eq 0 ]]; then + echo -e "Mac users should run this script without 'sudo'. Exiting..." + exit 1 + fi + CPUS=$(sysctl -n hw.ncpu) # ensure development environment is set correctly for clang $SUDO xcode-select -switch /Library/Developer/CommandLineTools - brew install llvm@14 googletest google-benchmark lcov make wget cmake curl - CLANG_TIDY=/usr/local/bin/clang-tidy - if [ ! -L "$CLANG_TIDY" ]; then - $SUDO ln -s $(brew --prefix)/opt/llvm@14/bin/clang-tidy /usr/local/bin/clang-tidy + + if ! brew --version &>/dev/null; then + echo -e "${cyan}Homebrew is required to install dependencies.${end}" + exit 1 fi + brew install llvm@14 googletest google-benchmark lcov make wget cmake bash bc + brew upgrade bash + + BREW_ROOT=$(brew --prefix) + + CLANG_TIDY=/usr/local/bin/clang-tidy + if [[ ! -L "$CLANG_TIDY" ]]; then + $SUDO ln -s "${BREW_ROOT}/opt/llvm@14/bin/clang-tidy" /usr/local/bin/clang-tidy + fi GMAKE=/usr/local/bin/gmake - if [ ! -L "$GMAKE" ]; then + if [[ ! -L "$GMAKE" ]]; then $SUDO ln -s $(xcode-select -p)/usr/bin/gnumake /usr/local/bin/gmake fi -fi -if [[ "$OSTYPE" == "linux-gnu"* ]]; then - $SUDO apt update - $SUDO apt install -y build-essential wget cmake libgtest-dev libbenchmark-dev lcov git software-properties-common rsync unzip + # install valid python version if not installed yet + if [[ -z "$PY_INSTALLED" ]]; then + PY_VERS=${PYTHON_VERSIONS[0]} + FULL_PY="python${PY_VERS}" + + MAX_RETRIES=2 + while [[ $MAX_RETRIES -gt 0 ]]; do + # try to install python version from homebrew and verify installation + if brew install "${FULL_PY}"; then + echo "${FULL_PY} installed successfully" + PY_INSTALLED=${PY_VERS} + break + fi + MAX_RETRIES=$((MAX_RETRIES - 1)) + sleep 1 + done + if [[ $MAX_RETRIES -eq 0 ]]; then + echo "Python3 install with homebrew failed, attempted on ${FULL_PY}" + exit 1 + fi + fi +# Linux install with apt +elif [[ "$OSTYPE" == "linux-gnu"* ]]; then + # avoids getting stuck on interactive prompts which is essential for CI/CD + export DEBIAN_FRONTEND=noninteractive + $SUDO apt update -y + $SUDO apt install -y build-essential wget cmake libgtest-dev libbenchmark-dev \ + lcov git software-properties-common rsync unzip bc + + # Add LLVM GPG key (apt-key is deprecated in Ubuntu 21.04+ so using gpg) + wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | \ + gpg --dearmor -o /usr/share/keyrings/llvm-archive-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/llvm-archive-keyring.gpg] http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main" | \ + $SUDO tee /etc/apt/sources.list.d/llvm.list - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | $SUDO apt-key add - - $SUDO add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main" + $SUDO apt update -y $SUDO apt install -y clang-format-14 clang-tidy-14 - $SUDO ln -s -f $(which clang-format-14) /usr/local/bin/clang-format - $SUDO ln -s -f $(which clang-tidy-14) /usr/local/bin/clang-tidy + $SUDO ln -sf $(which clang-format-14) /usr/local/bin/clang-format + $SUDO ln -sf $(which clang-tidy-14) /usr/local/bin/clang-tidy + + # install valid python version if not installed yet + if [[ -z "$PY_INSTALLED" ]]; then + PY_VERS=${PYTHON_VERSIONS[0]} + FULL_PY="python${PY_VERS}" + + # try to install python version from apt and verify installation + $SUDO apt install -y software-properties-common + $SUDO add-apt-repository -y ppa:deadsnakes/ppa + $SUDO apt update -y + + DEADSNAKES_AVAIL=$(wget -q --spider http://ppa.launchpad.net/deadsnakes/ppa/ubuntu/dists/focal/Release; echo $?) + if [[ $DEADSNAKES_AVAIL -ne 0 ]]; then + echo "Failed to add deadsnakes which is needed to install python3" + exit 1 + fi + + MAX_RETRIES=2 + while [[ $MAX_RETRIES -gt 0 ]]; do + # install python3 valid version and venv module + if $SUDO apt install -y ${FULL_PY}; then + echo "${FULL_PY} installed successfully" + PY_INSTALLED=${PY_VERS} + break + fi + MAX_RETRIES=$((MAX_RETRIES - 1)) + sleep 1 + done + if [[ $MAX_RETRIES -eq 0 ]]; then + echo "Python3 install with apt and deadsnakes failed, attempted on ${FULL_PY}" + exit 1 + fi + fi +fi + +# leave if no valid python version is installed +if ! which "python${PY_INSTALLED}" &> /dev/null; then + echo "Python${PY_INSTALLED} not found in user path" + exit 1 +else + # create virtual environment and install python packages for the valid python version + PYTHON_PATH=$(which "python${PY_INSTALLED}") + create_venv_install_python "${PYTHON_PATH}" ${PY_INSTALLED} fi +echo "To activate the virtual env to run python, run 'source ./scripts/activate-venv.sh'" PYTHON_TIDY=/usr/local/bin/run-clang-tidy.py -if [ ! -f "${PYTHON_TIDY}" ]; then +if [[ ! -f "${PYTHON_TIDY}" ]]; then echo -e "${green}Copying run-clang-tidy to /usr/local/bin${end}" wget https://raw.githubusercontent.com/llvm/llvm-project/e837ce2a32369b2e9e8e5d60270c072c7dd63827/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py $SUDO mv run-clang-tidy.py /usr/local/bin fi + +echo "Build environment setup complete." +echo "Next run './scripts/setup-dependencies.sh'." diff --git a/scripts/lint.sh b/scripts/lint.sh index dbabe0ce1..c6e4b0936 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -1,6 +1,7 @@ #!/bin/bash set -e +# Usage: ./scripts/lint.sh echo "Linting..." check_files=$(git ls-files \ @@ -43,4 +44,8 @@ if [ -z ${BUILD_DIR+x} ]; then fi fi -python3 /usr/local/bin/run-clang-tidy.py -p ${BUILD_DIR} "tests/.*/.*\.cpp|src/.*/.*\.cpp|tools/.*/.*\.cpp" +# use python from the virtual environment for clang-tidy +if source "./scripts/activate-venv.sh"; then + python /usr/local/bin/run-clang-tidy.py -p ${BUILD_DIR} "tests/.*/.*\.cpp|src/.*/.*\.cpp|tools/.*/.*\.cpp" + deactivate +fi diff --git a/scripts/native-system-benchmark.sh b/scripts/native-system-benchmark.sh index 3c1ff23f0..1b7923fa2 100755 --- a/scripts/native-system-benchmark.sh +++ b/scripts/native-system-benchmark.sh @@ -187,7 +187,9 @@ on_int() { if [[ -z "$_failed" ]]; then printf 'Generating plots\n' - python "$RT"/scripts/plot.py "$TESTDIR" + source "${RT}/scripts/activate-venv.sh" + python "$RT"/scripts/plot-samples.py -d "$TESTDIR" + deactivate fi printf 'Terminating any remaining processes\n' diff --git a/scripts/setup-dependencies.sh b/scripts/setup-dependencies.sh index 1dfdf50a7..ea4b4c771 100755 --- a/scripts/setup-dependencies.sh +++ b/scripts/setup-dependencies.sh @@ -10,10 +10,9 @@ set -e # install in a custom prefix rather than /usr/local. by default, this # chooses "prefix" directory alongside "scripts" directory. - PREFIX="$(cd "$(dirname "$0")"/.. && pwd)/prefix" echo "Will install local dependencies in the following prefix: $PREFIX" -mkdir -p $PREFIX $PREFIX/lib $PREFIX/include +mkdir -p "$PREFIX"/{lib,include} CMAKE_BUILD_TYPE="Debug" if [[ "$BUILD_RELEASE" == "1" ]]; then diff --git a/tools/bench/parsec/evm/contracts/package.json b/tools/bench/parsec/evm/contracts/package.json index 25ca7ff69..9bbc0555d 100644 --- a/tools/bench/parsec/evm/contracts/package.json +++ b/tools/bench/parsec/evm/contracts/package.json @@ -4,7 +4,7 @@ "description": "Load generator for PARSEC EVM", "main": "index.js", "scripts": { - "build": "npx hardhat compile && python3 gen_header.py && clang-format -i cpp_header/contracts.hpp" + "build": "npx hardhat compile && source scripts/activate-venv.sh && python gen_header.py && deactivate && clang-format -i cpp_header/contracts.hpp" }, "author": "MIT Digital Currency Initiative, Federal Reserve Bank of Boston", "license": "MIT",