added standalone pylint.sh script; improved install-build-tools.sh ov…

…er prev commit Signed-off-by: Morgan Rockett <[email protected]>
rockett-m · Jun 17, 2024 · 53ecb23 · 53ecb23
1 parent 7323f53
commit 53ecb23
Show file tree

Hide file tree

Showing 5 changed files with 190 additions and 63 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -18,7 +18,7 @@ env:
 jobs:
   build-release:
     name: Build Release Candidate
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       BUILD_RELEASE: 1
     steps:
@@ -30,7 +30,7 @@ jobs:
       - name: Setup Local Dependencies
         run: ./scripts/setup-dependencies.sh
       - name: Build
-        run: scripts/build.sh
+        run: ./scripts/build.sh
   lint:
     name: Lint
     runs-on: ubuntu-20.04
@@ -43,16 +43,17 @@ jobs:
       - name: Setup Local Dependencies
         run: ./scripts/setup-dependencies.sh
       - name: Build
-        run: scripts/build.sh
+        run: ./scripts/build.sh
       - name: Lint
-        run: scripts/lint.sh
+        run: ./scripts/lint.sh
   pylint:
     name: Pylint
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     continue-on-error: true
-    timeout-minutes: 5
+    timeout-minutes: 10
     strategy:
       matrix:
+        # pylint is not supported on python 3.13 yet
         python-version: ["3.10", "3.11", "3.12"]
     steps:
       - uses: actions/checkout@v2
@@ -62,18 +63,16 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: |
-          python3 -m pip install --upgrade pip
-          if [ -f requirements_py.txt ]; then pip install -r requirements_py.txt; fi
-          pip install pylint
+      - name: Setup Build Env
+        run: sudo ./scripts/install-build-tools.sh
       - name: Lint with Pylint
         run: |
-          # In the future we should have (minimum score of 8.0/10.0 or 9.0/10.0)
-          pylint --rcfile=.pylintrc $(git ls-files '*.py') --fail-under=5.0
+          set -x
+          MIN_CODE_QUALITY=5.0
+          ./scripts/pylint.sh $MIN_CODE_QUALITY
   unit-and-integration-test:
     name: Unit and Integration Tests
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 30
     steps:
       - uses: actions/checkout@v2
@@ -84,9 +83,9 @@ jobs:
       - name: Setup Local Dependencies
         run: ./scripts/setup-dependencies.sh
       - name: Build
-        run: scripts/build.sh
+        run: ./scripts/build.sh
       - name: Run Unit Tests
-        run: scripts/test.sh
+        run: ./scripts/test.sh
       - name: Shorten SHA
         id: vars
         run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
@@ -101,7 +100,7 @@ jobs:
           retention-days: 7
   doxygen:
     name: doxygen
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
         with:

diff --git a/.gitignore b/.gitignore
@@ -47,7 +47,6 @@ ethash*/
 lua-*/
 benchmark-results/
 CMakeFiles/
-Python-*/
 plots/
 .deps/
 .libs/

diff --git a/README.md b/README.md
@@ -42,7 +42,7 @@ There are two UHS-based architectures as follows:
    - Maximum demonstrated throughput ~1.7M transactions per second.
    - Geo-replicated latency <1 second.
 
-Read the [2PC &amp; Atomizer architecture guide](docs/uhs-architectures.md) for a detailed description of the system components and implementation of each architecture.
+Read the [2PC & Atomizer architecture guide](docs/uhs-architectures.md) for a detailed description of the system components and implementation of each architecture.
 
 ## Parallel Architecture for Scalably Executing smart Contracts ("PArSEC")
 
@@ -87,8 +87,13 @@ If you just want to run the system, see "Run the Code" below.
    Note that this script is just a convenience to install system-wide dependencies we expect.
    As a result, it uses the system package manager, requires `sudo`, and should only be run **once**.
    ```console
+   sudo ./scripts/install-build-tools.sh
+   ```
+   Note: Running Homebrew as root on mac via shell script is not supported, so run without sudo and when prompted, enter the root password.
+   ```
    ./scripts/install-build-tools.sh
    ```
+
 2. Setup project dependencies
    This script builds and installs a local copy of several build-dependencies which are not widely packaged.
    Because it installs to a local, configurable prefix (defaulting to `./prefix`), it does not need root permissions to run.
@@ -116,7 +121,7 @@ See the [live deployment](https://mit-dci.github.io/opencbdc-tx-pages/) to brows
 
 ## UHS-based Architectures (2PC & Atomizer)
 
-See the [2PC &amp; Atomizer User Guide](docs/2pc_atomizer_user_guide.md)
+See the [2PC & Atomizer User Guide](docs/2pc_atomizer_user_guide.md)
 
 ## PArSEC Architecture
 
@@ -159,7 +164,7 @@ Review results and logs at `testruns/<testrun-uuid>/`
 ## Linting
 
 ### General
-This script checks for newlines at the end of files. 
+This script checks for newlines at the end of all tracked git files except images. 
 Then it runs clang-format and clang-tidy on `.cpp` files in the following directories:
  `src`, `tests`, `cmake-tests`, `tools`.
 ```console
@@ -168,6 +173,7 @@ Then it runs clang-format and clang-tidy on `.cpp` files in the following direct
 
 ### Python
 Lint all python files according to ruleset defined in `.pylintrc`.
+Optional code quality value >= 5.0 and <= 10.0 can be entered as a threshold of failure.
 ```console
-pylint --rcfile=.pylintrc $(git ls-files '*.py') --fail-under=8.0
+./scripts/pylint.sh 8.0
 ```
diff --git a/scripts/install-build-tools.sh b/scripts/install-build-tools.sh
@@ -6,6 +6,7 @@ green="\033[0;32m"
 cyan="\033[0;36m"
 end="\033[0m"
 
+# for debugging, 'set -x' can be added to trace the commands
 set -e
 
 SUDO=''
@@ -14,40 +15,74 @@ if (( $EUID != 0 )); then
     SUDO='sudo'
 fi
 
-# Function to check if a version of Python >= 3.10 is installed
-check_python_version() {
-  if command -v python3 &>/dev/null; then
-    # Get the version of Python
-    PYTHON_VERSION=$(python3 --version | awk '{print $2}')
-    IFS='.' read -r -a version_parts <<< "$PYTHON_VERSION"
-    echo "Python version: ${version_parts[0]}.${version_parts[1]}"
-    # >= 3.10
-    if (( ${version_parts[0]} >= 3 && ${version_parts[1]} >= 10 )); then
-      return 0
-    fi
-  fi
-  return 1
-}
+# Supporting these versions for buildflow
+PYTHON_VERSIONS=("3.12" "3.11" "3.10")
+echo "Python3 versions supported: ${PYTHON_VERSIONS[@]}"
+echo "OS Type: $OSTYPE"
+
+# 'Python 3.10.12' -> 'Python 3.10'
+PYTHON3_DEFAULT=`python3 --version | grep -Eo 'Python 3.[0-9]{1,2}'`
+# 'Python 3.10' -> '3.10'
+PYTHON3_DEFAULT_VERS=`python3 --version | grep -Eo '3.[0-9]{1,2}'`
 
+# macOS install with homebrew
 if [[ "$OSTYPE" == "darwin"* ]]; then
+
+  # macOS does not support running shell scripts as root with homebrew
+  if [ $EUID -eq 0 ]; then
+    echo -e "${cyan}Please run this script without 'sudo' on ${OSTYPE} arch. ${end}"
+    exit 1
+  fi
+
   CPUS=$(sysctl -n hw.ncpu)
   # ensure development environment is set correctly for clang
   $SUDO xcode-select -switch /Library/Developer/CommandLineTools
-  # see if homebrew is installed and install if not
-  if ! [[ -f /opt/homebrew/bin/brew ]]; then
-    echo -e "${green}Installing Homebrew...${end}"
-    /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+
+  if ! command -v brew &>/dev/null; then
+    echo -e "${cyan}Homebrew is required to install dependencies.${end}"
+    exit 1
   fi
-  export PATH="/opt/homebrew/bin:$PATH"
 
-  brew install llvm@14 googletest google-benchmark lcov make wget cmake curl bash python3 pylint python-matplotlib
+  brew install llvm@14 googletest google-benchmark lcov make wget cmake bash bc
   brew upgrade bash
 
-  # Add Python 3 to PATH
-  echo "export PATH=\"/opt/homebrew/opt/python@3/bin:\$PATH\"" >> ~/.bash_profile
-  # Make python3 default
-  echo "alias python=python3" >> ~/.bash_profile
-  . ~/.bash_profile
+  # Check if the default python3 version is supported by our buildflow
+  if (echo "${PYTHON3_DEFAULT}" | grep -E 'Python 3.1[0-2]'); then
+    FULL_PY="python@${PYTHON3_DEFAULT_VERS}"
+    if brew search ${FULL_PY} | grep "${FULL_PY}"; then
+      brew install pylint python-matplotlib
+    fi
+    echo "Successfully installed dependencies for default Python3 version: ${PYTHON3_DEFAULT}"
+  # User could have valid python versions installed, but could be using complex env so won't check
+  # Install one of the compatible python versions
+  else
+    PY_INSTALLED=''
+    for PY_VERS in "${PYTHON_VERSIONS[@]}"; do
+      FULL_PY="python@${PY_VERS}"
+      if brew search ${FULL_PY} | grep "${FULL_PY}"; then
+        echo "Installing $FULL_PY and dependencies"
+        brew install ${FULL_PY} pylint python-matplotlib
+        PY_INSTALLED=${PY_VERS}
+        echo "Successfully installed dependencies for ${FULL_PY}"
+        break
+      fi
+    done
+
+    # post-install checks
+    if [[ "$PY_INSTALLED" == '' ]]; then
+      echo "Python3 install with brew failed, attempted on these versions: ${PYTHON_VERSIONS[@]}"
+      exit 1
+    elif [[ $(echo "${PYTHON3_DEFAULT_VERS} < 3.10" | bc -l) -eq 1 || \
+          $(echo "${PYTHON3_DEFAULT_VERS} >= 3.13" | bc -l) -eq 1 ]]; then
+      echo "Your default Python3 version: '$DEFAULT_PYTHON3' is not supported by our buildflow"
+      echo "Please set your default python3 to the recently installed version '${PY_INSTALLED}'."
+      echo "This will involve updating the PATH variable typically set in .bash_profile / .zshrc / .bashrc file(s)."
+      echo "If python aliases are used, they will have be updated to reflect the new version."
+      echo "Finally, source the shell file to update the PATH variable, and rerun this script."
+      exit 1
+    fi
+
+  fi
 
   CLANG_TIDY=/usr/local/bin/clang-tidy
   if [ ! -L "$CLANG_TIDY" ]; then
@@ -57,30 +92,82 @@ if [[ "$OSTYPE" == "darwin"* ]]; then
   if [ ! -L "$GMAKE" ]; then
     $SUDO ln -s $(xcode-select -p)/usr/bin/gnumake /usr/local/bin/gmake
   fi
-
+# Linux install with apt
 elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
-  $SUDO apt update
-  $SUDO apt install -y build-essential wget cmake libgtest-dev libbenchmark-dev lcov git software-properties-common rsync unzip
+  # avoids getting stuck on interactive prompts which is essential for CI/CD
+  export DEBIAN_FRONTEND=noninteractive
+  $SUDO apt update -y
+  $SUDO apt upgrade -y
+  $SUDO apt install -y build-essential wget cmake libgtest-dev libbenchmark-dev \
+    lcov git software-properties-common rsync unzip bc
 
-  wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | $SUDO apt-key add -
-  $SUDO add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main"
+  # Add LLVM GPG key (apt-key is deprecated in Ubuntu 21.04+ so using gpg)
+  wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | \
+    gpg --dearmor -o /usr/share/keyrings/llvm-archive-keyring.gpg
+  echo "deb [signed-by=/usr/share/keyrings/llvm-archive-keyring.gpg] http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main" | \
+    $SUDO tee /etc/apt/sources.list.d/llvm.list
+
+  $SUDO apt update -y
   $SUDO apt install -y clang-format-14 clang-tidy-14
-  $SUDO ln -s -f $(which clang-format-14) /usr/local/bin/clang-format
-  $SUDO ln -s -f $(which clang-tidy-14) /usr/local/bin/clang-tidy
+  $SUDO ln -sf $(which clang-format-14) /usr/local/bin/clang-format
+  $SUDO ln -sf $(which clang-tidy-14) /usr/local/bin/clang-tidy
+
+  # Install Python and necessary packages
+  $SUDO add-apt-repository -y ppa:deadsnakes/ppa
+  $SUDO apt update -y
 
-  # Python 3.10+ not installed so get the latest version
-  if ! check_python_version; then
-    $SUDO add-apt-repository ppa:deadsnakes/ppa
-    $SUDO apt update
+  # If default python3 version is already installed and supported, then just install pylint and matplotlib
+  # not going to look through all the user's python versions to see if any are in supported versions
+  if (echo "${PYTHON3_DEFAULT}" | grep -E 'Python 3.1[0-2]'); then
+    FULL_PY="python${PYTHON3_DEFAULT_VERS}"
+    # Check if the desired Python version is available in the apt repository
+    if apt list -a ${FULL_PY} &> /dev/null; then
+      # install packages for the default python3 version which is valid
+      $SUDO apt install -y python${PY_VERS}-distutils
+      $SUDO ${FULL_PY} -m pip install pylint matplotlib
+    fi
+    echo "Successfully installed dependencies for default Python3 version: ${PYTHON3_DEFAULT}"
+  # User could have valid python versions installed, but could be using complex env so won't check
+  # Install one of the compatible python versions
+  else
+    PY_INSTALLED=''
+    for PY_VERS in "${PYTHON_VERSIONS[@]}"; do
+      # Check if the desired Python version is available in the apt repository
+      FULL_PY="python${PY_VERS}"
+      # Check if the desired Python version is available in the apt repository
+      if apt list -a ${FULL_PY} &> /dev/null; then
+        echo "Installing ${FULL_PY} and dependencies"
+        $SUDO apt install -y python${PY_VERS} python${PY_VERS}-distutils
+        # for dependency installation on specific python version
+        wget https://bootstrap.pypa.io/get-pip.py
+        $SUDO ${FULL_PY} get-pip.py && rm -f get-pip.py
+        $SUDO ${FULL_PY} -m pip install pylint matplotlib
+        PY_INSTALLED="python${PY_VERS}"
+        echo "Successfully installed dependencies for ${FULL_PY}"
+        break
+      fi
+    done
+
+    # post-install checks
+    if [[ "$PY_INSTALLED" == '' ]]; then
+      echo "Python3 install with apt failed, attempted on these versions: ${PYTHON_VERSIONS[@]}"
+      exit 1
 
-    latest_python_version=$(apt-cache search python3. | grep -o 'python3\.[0-9]*' | sort -V | tail -n 1)
-    $SUDO apt install -y $latest_python_version
+    # elif (( $(echo "${PYTHON3_DEFAULT_VERS} < 3.10" | bc -l) -eq 1)) || (( $(echo "${PYTHON3_DEFAULT_VERS} >= 3.13" | bc -l) -eq 1)); then
+
+    elif [[ $(echo "${PYTHON3_DEFAULT_VERS} < 3.10" | bc -l) -eq 1 || \
+          $(echo "${PYTHON3_DEFAULT_VERS} >= 3.13" | bc -l) -eq 1 ]]; then
+      echo "Your default Python3 version: '$DEFAULT_PYTHON3' is not supported by our buildflow"
+      echo "Please set your default python3 to the recently installed version '${PY_INSTALLED}'."
+      echo "This will can be done by the following commands in the terminal:"
+      echo "'sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PY_VERS} 1'"
+      echo "'sudo update-alternatives --config python3'"
+      exit 1
+    fi
 
-    $SUDO update-alternatives --install /usr/bin/python3 python3 /usr/bin/$(ls /usr/bin/ | grep -E '^python3\.[0-9]+$' | sort -V | tail -n 1) 1
-    $SUDO update-alternatives --config python3
   fi
+
 fi
-python3 --version
 
 PYTHON_TIDY=/usr/local/bin/run-clang-tidy.py
 if [ ! -f "${PYTHON_TIDY}" ]; then

diff --git a/scripts/pylint.sh b/scripts/pylint.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# depedenent on 'sudo scripts/install-build-tools.sh' to install pylint
+if ! command -v pylint &>/dev/null; then
+  echo "pylint is not installed."
+  echo "Run 'sudo ./scripts/install-build-tools.sh' to install pylint."
+  exit 1;
+fi
+
+MIN_CODE_QUALITY=8.0
+if [ -n "$1" ]; then
+  # set minimum quality to user input (int/float) if provided and (5.0 <= input <= 10.0)
+  if [[ $1 =~ ^[0-9]*([\.][0-9])?$ ]]; then
+    if (( $(echo "$1 >= 5.0" | bc -l) )) && (( $(echo "$1 <= 10.0" | bc -l) )); then
+      MIN_CODE_QUALITY=$1
+    else
+      # In the future, we want code quality to be at minimum 8.0/10.0
+      echo "Code quality score must be between 5.0 and 10.0, inclusive."
+      echo "Recommended code quality score is >= 8.0."
+      exit 1
+    fi
+  else
+    echo "Code quality score must be an integer or floating point number."
+    exit 1
+  fi
+fi
+
+echo "Linting Python code with minimum quality of $MIN_CODE_QUALITY/10.0..."
+pylint --rcfile=.pylintrc --fail-under=$MIN_CODE_QUALITY $(git ls-files '*.py')
+
+if [ $? -ne 0 ]; then
+  echo "Linting failed, please fix the issues above."
+  exit 1
+else
+  echo "Linting passed."
+fi