Ci installer

.flake8

@@ -55,14 +55,28 @@ max-line-length = 100
 # One can omit the description on the __init__ docstring and simply
 # document the parameters, but then flake8 complains about the
 # structure of the __init__ doctsring. The D205 and D400 errors are
-# precisely those complains. We disable these in order to have a
+# precisely those complaints. We disable these in order to have a
 # sane way of documenting classes with Sphinx' autoclass.
 
 
 ignore = B902, D205, D400, D401, D100, W503
 
 # D103 - Missing docstring in public function
 #
-# Ignore docstrings requirement in tests
+# Ignore docstrings requirement in tests.
+#
+# D101, D102, D103, D107
+#
+# We don't quite document the installer.
+#
+# E402 - module level import not at top of file
+#
+# In the installer, we need to patch Textual for non-unicode terminal
+# support, and that needs to be the first thing done before we import
+# other stuff.
+#
 
-per-file-ignores = tests/*:D103
+per-file-ignores =
+    tests/*: D103
+    tests/installer/*: D101, D102, D103, D104, D105, D107
+    tests/installer/main.py: E402, D101, D102, D103, D107

.github/dependabot.yml

@@ -16,4 +16,5 @@ updates:
       - dependency-name: "pystache"
       - dependency-name: "typeguard"
       - dependency-name: "packaging"
+      - dependency-name: "requests"
 

psij-ci-run

@@ -2,9 +2,20 @@
 
 usage() {
     cat <<EOF
-Usage: ./psij-ci-run [--help] [--repeat]
-    --help          Displays this message
-    --repeat        If specified, run tests every 24 hours in a loop.
+Usage: ./psij-ci-run [--help] [--repeat] [--reschedule HH:MM] [--log]
+    --help
+        Displays this message.
+    --repeat
+        If specified, run tests every 24 hours in a loop.
+    --reschedule HH:MM
+        If specified, reschedule the tests at HH:MM using the
+        at command.
+    --log
+        Disables output and redirects to a log instead. Used
+        when running the tests automatically.
+    --tee
+        Logs output as if --log was specified, but also prints
+        on stdout.
 EOF
 }
 
@@ -16,12 +27,17 @@ failifempty() {
     fi
 }
 
+MYPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+cd "$MYPATH"
+
 if [ -f psij-ci-env ]; then
     source psij-ci-env
 fi
 
 REPEAT=0
 RESCHEDULE=0
+NO_OUT=0
+TEE=0
 
 while [ "$1" != "" ]; do
     case "$1" in
@@ -38,13 +54,43 @@ while [ "$1" != "" ]; do
             TIME="$2"
             shift 2
             ;;
+        --log)
+            NO_OUT=1
+            shift
+            ;;
+        --tee)
+            TEE=1
+            shift
+            ;;
         *)
             echo "Unrecognized command line option: $1."
             usage
             exit 1
+            ;;
     esac
 done
 
+tee() {
+    while read LINE; do
+        echo "$LINE" >&3
+        echo "$LINE" >>./testing.log
+    done
+}
+
+if [ "$NO_OUT" == "1" ]; then
+    exec 1<&-
+    exec 2<&-
+
+    exec 1>>./testing.log
+    exec 2>&1
+elif [ "$TEE" == "1" ]; then
+    exec 3>&1-
+    exec 2<&-
+
+    exec 1> >(tee)
+    exec 2>&1
+fi
+
 if python --version 2>&1 | egrep -q 'Python 3\..*' >/dev/null 2>&1 ; then
     PYTHON="python"
 else
@@ -68,7 +114,7 @@ if [ "$REPEAT" == "1" ]; then
         sleep $TO_SLEEP
     done
 elif [ "$RESCHEDULE" == "1" ]; then
-    CMD="./psij-ci-run --reschedule $TIME >> testing.log 2>&1"
+    CMD="./psij-ci-run --reschedule $TIME --log"
     echo "$CMD" | at $TIME
     $PYTHON tests/ci_runner.py $MODE
 else

psij-ci-setup

@@ -1,11 +1,22 @@
 #!/bin/bash
 
+MIN_PYTHON_VERSION="3.8"
+
+set -o pipefail
+
 if pip --version 2>&1 | egrep -q 'python 3\..*' >/dev/null 2>&1 ; then
     PIP="pip"
 else
     PIP="pip3"
 fi
 
+if python --version 2>&1 | egrep -q 'Python 3\..*' >/dev/null 2>&1 ; then
+    PYTHON="python"
+else
+    PYTHON="python3"
+fi
+
+
 existing_error_trailer() {
     echo "If you are certain that you want to install multiple entries,   "
     echo "you can re-run this script with the \"-f\" flag.                "
@@ -50,7 +61,7 @@ cron_existing_error() {
 }
 
 cron_install() {
-    CMD="$CMD >> testing.log 2>&1"
+    CMD="$CMD --log"
     LINE="$MINUTE $HOUR * * * cd \"$MYPATH\" && $CMD"
     echo
     echo "================================================================"
@@ -109,7 +120,7 @@ at_existing_error() {
 }
 
 at_install() {
-    CMD="$CMD --reschedule $HOUR:$MINUTE >> testing.log 2>&1"
+    CMD="$CMD --reschedule $HOUR:$MINUTE --log"
     echo
     echo "================================================================"
     echo "The following will be executed:"
@@ -141,7 +152,7 @@ screen_existing_error() {
 }
 
 screen_install() {
-    CMD="$CMD --repeat >> testing.log 2>&1"
+    CMD="$CMD --repeat --log"
     echo
     echo "================================================================"
     echo "WARNING: Screen sessions do not persist across reboots. Please  "
@@ -166,7 +177,7 @@ manual_existing_error() {
 }
 
 manual_install() {
-    CMD="$CMD --repeat >> testing.log 2>&1"
+    CMD="$CMD --repeat --log"
     echo
     echo "================================================================"
     echo "Please run the following command in the background:             "
@@ -271,14 +282,100 @@ check_key() {
     fi
 }
 
+check_python_version() {
+    VERSION=`$PYTHON --version | awk '{print $2}'`
+    FIRST=`echo -e "$VERSION\n$MIN_PYTHON_VERSION" | sort -V|head -n 1`
+    if [ "$FIRST" == "$VERSION" ]; then
+        echo "Error: PSI/J requires Python $MIN_PYTHON_VERSION or above. Your current version is $VERSION."
+        exit 2
+    fi
+}
+
+check_pip_version() {
+    PYTHON_VERSION=`$PYTHON --version | awk '{print $2}' | cut -d '.' -f 1,2`
+    PIP_PYTHON_VERSION=`$PIP --version | sed -nE 's/.*\(python\s(.*)\)/\1/p'`
+
+    if [ "$PYTHON_VERSION" != "$PIP_PYTHON_VERSION" ]; then
+        echo -e "\e[33mWarning: The installed $PIP comes from Python $PIP_PYTHON_VERSION while your Python"
+        echo -e "interpreter has version $PYTHON_VERSION. If this causes problems, please use a "
+        echo -e "virtual environment.\e[0m"
+    fi
+}
+
+not_cached() {
+    if [ ! -d .packages ]; then
+        return 0
+    fi
+    if [ ! -f .packages/req_csum.txt ]; then
+        return 0
+    fi
+    REQ_CSUM=`cat .packages/req_csum.txt`
+    PYTHON=`which $PYTHON`
+    CRT_CSUM=`echo $PYTHON | cat - requirements*.txt | md5sum`
+
+    if [ "$REQ_CSUM" != "$CRT_CSUM" ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+dots_for_lines() {
+    while read LINE; do
+        echo -n .
+    done
+}
+
+install_deps() {
+    if not_cached; then
+        echo -n "Installing dependencies..."
+
+        exec 3> >(dots_for_lines >> /dev/stdout)
+        OUT=`$PIP install --target .packages --upgrade -r requirements-tests.txt --no-compile 2>&1 1>&3`
+
+        if [ "$?" != "0" ]; then
+            echo "FAILED"
+            echo "$OUT"
+            exit 2
+        else
+            echo "Done"
+            CSUM=`echo $PYTHON | cat - requirements*.txt | md5sum`
+            echo "$CSUM" >.packages/req_csum.txt
+        fi
+    fi
+
+    export PYTHONPATH=`pwd`/.packages:$PYTHONPATH
+}
+
+
 FORCE=0
+PLAIN=0
 
-if [ "$1" == "-f" ]; then
-    FORCE=1
+while [ "$1" != "" ]; do
+    if [ "$1" == "-f" ]; then
+        FORCE=1
+    elif [ "$1" == "--plain" ]; then
+        PLAIN=1
+    else
+        echo "Unknown option \"$1\""
+        exit 1
+    fi
     shift
-fi
+done
+
+check_python_version
+check_pip_version
 
 MYPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+cd "$MYPATH"
+
+if [ "$PLAIN" != "1" ]; then
+    install_deps
+
+    echo "Running TUI installer. To run the normal installer, use \"psij-ci-setup --plain\""
+
+    exec $PYTHON tests/run_installer.py
+fi
 
 echo
 echo "================================================================"
@@ -297,21 +394,8 @@ check_email
 
 check_key
 
-cd "$MYPATH"
-
-echo -n "Installing dependencies..."
-
-OUT=`$PIP install --target .packages --upgrade -r requirements-tests.txt 2>&1`
-
-if [ "$?" != "0" ]; then
-    echo "FAILED"
-    echo $OUT
-    exit 2
-else
-    echo "Done"
-fi
+install_deps
 
-export PYTHONPATH=`pwd`/.packages:$PYTHONPATH
 
 HOUR=`echo $(($RANDOM % 24))`
 MINUTE=`echo $(($RANDOM % 60))`

requirements-tests.txt

@@ -8,3 +8,6 @@ requests >= 2.25.1
 pytest-cov
 pytest-timeout
 filelock >= 3.4, < 3.19
+textual==3.2.0
+tree-sitter
+tree-sitter-bash