spy: add option to use webspy dir from pyinstaller bundle (+build)

build: include conf and webspy dir in tar.gz

ci: add WEBSPY=1 to debian

.github/workflows/build.yml

@@ -61,23 +61,28 @@ jobs:
       - uses: actions/checkout@v4
       - name: Run apt, venv, pip, build and upload artifacts
         run: |
-          DEBIAN_FRONTEND=noninteractive apt-get update -y
-          apt-get install -y upx-ucl || {
+          DEB="upx-ucl_4.2.2-3_amd64.deb"  #upx-ucl_3.96-2_amd64.deb
+          export DEBIAN_FRONTEND=noninteractive
+          apt-get update -y
+          if ! DEBIAN_FRONTEND=noninteractive apt-get install -y upx-ucl; then
             apt-get install -y libucl1 &&
-            wget -q http://ftp.us.debian.org/debian/pool/main/u/upx-ucl/upx-ucl_3.96-3+b1_amd64.deb && 
-            dpkg -i upx-ucl_3.96-3+b1_amd64.deb;
-          }
+            wget -q http://ftp.us.debian.org/debian/pool/main/u/upx-ucl/$DEB &&
+            dpkg -i $DEB;
+          fi
           python3 -m venv venv && . venv/bin/activate
           pip3 install --upgrade pip wheel setuptools
           pip3 install -r requirements.txt
           pip3 install pyinstaller sysv-ipc geoip2 flask
           ./build.sh && mkdir ./build-debian && mv -f *.tar.gz *.sha512sum ./build-debian
+      - if: ${{ matrix.debian-release }} == 'bookworm'
+        run: WEBSPY=1 ./build.sh && mkdir ./build-ubuntu && mv -f *.tar.gz *.sha512sum ./build-ubuntu
       - uses: actions/upload-artifact@v4
         with:
           name: build-debian-artifacts
           path: ./build-debian/*
           if-no-files-found: error
 
+# disabled, EOL
 #  centos7-container-build-job:
 #    name: Build for CentOS 7
 #    runs-on: ubuntu-latest

.github/workflows/docker.yml

@@ -1,3 +1,5 @@
+# unused
+
 name: Docker build and push
 
 on:

.github/workflows/pylint.yml

@@ -1,3 +1,5 @@
+# disabled, run manually
+
 name: Pylint
 
 # on: [push]

README.md

@@ -41,7 +41,7 @@ Supported: Debian 10, 11, 12 and Ubuntu 20.04, 22.04, 24.04, CentOS Stream, Alma
 
 EOL: [CentOS 7](https://github.com/silv3rr/pyspy/releases/download/slv-pyspy-v20230708/pyspy-centos7-python3.6-x86_x64.tar.gz)
 
-All files on [Releases](../../releases) tab. Click **Show all .. assets** link at the bottom.
+Get all files from [Releases](../../releases) tab. Click **Show all .. assets** link at the bottom.
 
 ## apt
 
@@ -76,7 +76,7 @@ To set gl path, color and refresh for cli, web ip/port etc edit 'spy.conf'. By d
 
 All options are explained at the bottom of conf. Make sure 'ipc_key' matches glftpd.
 
-To change how flask webspy looks, you can edit static/style.css and html in templates dir.
+To change how flask webspy looks, edit webspy/static/style.css and html in webspy/templates dir.
 
 ## Build
 

build.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # Usage:
-#   ./build.sh _WITH_SPY _WITH_GEOIP _WITH_HTTPD _WITH_FLASK
+#   ./build.sh _WITH_GEOIP _WITH_HTTPD _WITH_FLASK _WITH_BUNDLE
 
 # Required: Python 3.7.3+, sysv-ipc, pyinstaller
 #   python3 -m venv venv && . venv/bin/activate && \
@@ -15,10 +15,16 @@ PYINSTALLER=1
 PACK=1
 REQS="$(cut -d= -f1 requirements.txt 2>/dev/null)"
 ARGS="--hidden-import sysv_ipc"
-OPTS="_WITH_SPY _WITH_GEOIP _WITH_HTTPD _WITH_FLASK"
+OPTS="_WITH_GEOIP _WITH_HTTPD _WITH_FLASK _WITH_BUNDLE"
 
 if [ "${WEBSPY:-0}" -eq 1 ]; then
-  ARGS=" $ARGS --add-data webspy:. "
+  PKG_SUFFIX="-web"
+  ARGS=" $ARGS --add-data webspy:./webspy "
+  # use webspy dir from bundule
+  # shellcheck disable=SC2086
+  if echo $OPTS | grep -q "_WITH_BUNDLE"; then
+    sed -i "s/^\(_WITH_BUNDLE\) *= *.*$/\1 = True/" "$PYSRC"
+  fi
 fi
 
 if [ ! -s requirements.txt ] || [ -z "$REQS" ]; then
@@ -49,6 +55,7 @@ for a in "$@"; do
 done
 
 # disable flask dev
+# shellcheck disable=SC2086
 if echo $OPTS | grep -q "_WITH_FLASK"; then
   sed -i "s/^\(FLASK_OPTIONS\['debug']\) *= *.*$/\1 = False/" "$PYSRC"
 fi
@@ -113,15 +120,18 @@ if [ "$PYINSTALLER" -eq 1 ]; then
     echo "ERROR: pyinstaller not found, try 'apt install python3-pyinstaller' or 'pip install pyinstaller'"
     exit 1
   }
+  # shellcheck disable=SC2086
   pyinstaller spy.py $ARGS --clean --noconfirm --onefile &&
     if [ -e "dist/spy" ]; then
       printf "\nresult: OK "
       ls -la dist/spy
+      echo
       if [ "$PACK" -eq 1 ]; then
+        # shellcheck disable=SC1091
         . /etc/os-release
-        PACKNAME="pyspy-${ID:-linux}${VERSION_ID}-python${PYVER:-3}-x86_x64"
+        PACKNAME="pyspy-${ID:-linux}${VERSION_ID}-python${PYVER:-3}-x86_x64${PKG_SUFFIX}"
         printf "Creating %s.tar.gz...\n" "$PACKNAME"
-        tar -C ./dist -cvf "${PACKNAME}.tar.gz" spy >/dev/null &&
+        tar -C ./dist -cvf "${PACKNAME}.tar.gz" ../spy.conf spy ../webspy >/dev/null &&
           sha512sum "${PACKNAME}.tar.gz" >"${PACKNAME}.sha512sum" && echo "shasum: OK" || echo "ERROR: shasum"
       fi
     else

docker-build-distro.sh

@@ -1,27 +1,31 @@
 #!/bin/sh
 
 # from github workflow
+# runs latest docker image by default
 
 UBUNTU="
   python3 -m venv venv && . venv/bin/activate
   pip3 install --upgrade pip wheel setuptools
   pip3 install -r requirements.txt
   pip3 install pyinstaller sysv-ipc geoip2 flask
-  WEBSPY=0 ./build.sh && mkdir -p ./build-ubuntu && mv -f *.tar.gz *.sha512sum ./build-ubuntu
+  ./build.sh && mkdir -p ./build-ubuntu && mv -f *.tar.gz *.sha512sum ./build-ubuntu
 "
+DEB=upx-ucl_4.2.2-3_amd64.deb  #upx-ucl_3.96-2_amd64.deb
 DEBIAN="
-  DEBIAN_FRONTEND=noninteractive apt-get update -y;
+  export DEBIAN_FRONTEND=noninteractive
+  apt-get update -y
   apt-get install -y wget python3 python3-venv python3-pip
-  apt-get install -y upx-ucl || {
+  if ! apt-get install -y upx-ucl; then
     apt-get install -y libucl1 &&
-    wget -q http://ftp.us.debian.org/debian/pool/main/u/upx-ucl/upx-ucl_3.96-3+b1_amd64.deb &&
-    dpkg -i upx-ucl_3.96-3+b1_amd64.deb;
-  };
+    wget -q http://ftp.us.debian.org/debian/pool/main/u/upx-ucl/$DEB &&
+    dpkg -i $DEB;
+  fi
   python3 -m venv venv && . venv/bin/activate
   pip3 install --upgrade pip wheel setuptools
-  pip3 install -r requirements.txt
-  pip3 install pyinstaller sysv-ipc geoip2 flask
-  WEBSPY=0 ./build.sh && mkdir -p ./build-debian && mv -f *.tar.gz *.sha512sum ./build-debian
+  pip3 install --force -r requirements.txt
+  pip3 install --force pyinstaller sysv-ipc geoip2 flask
+  ./build.sh && mkdir -p ./build-debian && mv -f *.tar.gz *.sha512sum ./build-debian
+  WEBSPY=1 ./build.sh && mkdir -p ./build-debian && mv -f *.tar.gz *.sha512sum ./build-debian
 "
 #CENTOS="
 #  yum install -y gcc python3-devel python3-pip python3-virtualenv
@@ -31,15 +35,15 @@ DEBIAN="
 #  pip3 install --upgrade pip wheel setuptools
 #  pip3 install -r requirements.txt
 #  pip3 install pyinstaller sysv-ipc geoip2
-#  WEBSPY=0 ./build.sh  && mkdir -p ./build-centos && mv -f *.tar.gz *.sha512sum ./build-centos
+#  ./build.sh  && mkdir -p ./build-centos && mv -f *.tar.gz *.sha512sum ./build-centos
 #"
 ALPINE="
   apk add python3 python3-dev py3-pip py3-virtualenv gcc musl-dev upx
   python3 -m venv venv && . venv/bin/activate
   pip3 install --upgrade pip wheel setuptools
   pip3 install -r requirements.txt
   pip3 install pyinstaller sysv-ipc geoip2 flask
-  WEBSPY=0 ./build.sh && mkdir -p ./build-alpine && mv -f *.tar.gz *.sha512sum ./build-alpine
+  ./build.sh && mkdir -p ./build-alpine && mv -f *.tar.gz *.sha512sum ./build-alpine
 "
 RHEL="
   dnf install -y gcc python3-devel python3-pip
@@ -55,16 +59,20 @@ RHEL="
 func_docker_run() {
   image=$1
   shift
-  docker run -it --rm  --workdir /build -v "$PWD:/build" "$image" sh -c "$*"
+  #docker run -it --workdir /build -v "$PWD:/build" "$image" bash
+  docker run --rm  --workdir /build -v "$PWD:/build" "$image" sh -c "$*"
 }
 
+TAG="${2:-latest}"
+
+# shellcheck disable=SC2046
 case $1 in
-  ubuntu) func_docker_run ubuntu24.04 "$UBUNTU" ;;
-  debian) func_docker_run debian:12 "$DEBIAN" ;;
-  centos7) func_docker_run centos:7 "$CENTOS" ;;
-  centos-stream) func_docker_run quay.io/centos/centos:stream9 "$RHEL" ;;
-  alma) func_docker_run almalinux:9.4 "$RHEL" ;;
-  rocky) func_docker_run rockylinu:9.3 "$RHEL" ;;
-  alpine) func_docker_run alpine:3.18 "$ALPINE" ;;
-  *) echo "$0 $(grep -E ' *[a-z]+\).*;;' "$0" | cut -d')' -f1 | tr -d '\n')" ;;
+  ubuntu) func_docker_run "ubuntu:$TAG" "$UBUNTU" ;;
+  debian) func_docker_run "debian:$TAG" "$DEBIAN" ;;
+  #centos7) func_docker_run centos:7 "$CENTOS" ;;
+  centos-stream) func_docker_run "quay.io/centos/centos:$TAG" "$RHEL" ;;
+  alma) func_docker_run "almalinux:$TAG" "$RHEL" ;;
+  rocky) func_docker_run "rockylinux:$TAG" "$RHEL" ;;
+  alpine) func_docker_run  "alpine:$TAG" "$ALPINE" ;;
+  *) echo "USAGE: $0" $(grep -Pow ' \K[a-z-]+\)' docker-build-distro.sh | cut -d')' -f1) ;;
 esac

spy.conf

@@ -2,7 +2,7 @@
 # SPY.PY CONFIGURATION #
 ########################
 
-glrootpath = /jail/glftpd
+glrootpath = /glftpd
 #maxusers = 50
 maxusers = -1
 ipc_key = 0x0000DEAD

spy.py

@@ -30,6 +30,7 @@
 _WITH_GEOIP = False
 _WITH_HTTPD = True
 _WITH_FLASK = True
+_WITH_BUNDLE = False
 
 PYINSTALLER = False
 
@@ -47,11 +48,13 @@
 if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
     PYINSTALLER = True
 
-VERSION = "20230705"
+VERSION = "20240803"
 
 SCRIPT_PATH = os.path.abspath(__file__) if os.path.abspath(__file__) else os.path.realpath(sys.argv[0])
 if PYINSTALLER:
     SCRIPT_PATH = os.path.realpath(sys.argv[0])
+    if _WITH_BUNDLE:
+        BUNDLE_DIR = os.path.abspath(os.path.dirname(__file__)) if os.path.dirname(__file__) else sys._MEIPASS
 SCRIPT = os.path.basename(sys.argv[0])
 SCRIPT_NAME = os.path.splitext(SCRIPT)[0]
 SCRIPT_DIR = os.path.dirname(SCRIPT_PATH)
@@ -1327,6 +1330,14 @@ def create_app() -> object:
     if _WITH_FLASK and FLASK_MODE == 1:
         tmpl_path = os.path.join(SCRIPT_DIR, 'webspy/templates/')
         static_path = os.path.join(SCRIPT_DIR, 'webspy/static/')
+        # if webspy dir does not exist, use incl dir from bundle
+        if PYINSTALLER and _WITH_BUNDLE:
+            try:
+                os.path.isdir(tmpl_path)
+                os.path.isdir(static_path)
+            except OSError:
+                tmpl_path = os.path.join(BUNDLE_DIR, 'webspy/templates/')
+                static_path = os.path.join(BUNDLE_DIR, 'webspy/static/')
         app = flask.Flask(__name__, template_folder=tmpl_path, static_folder=static_path, static_url_path='/static')
         app.secret_key = 'SECRET_KEY'
         if FLASK_PROXY: