From fefbf2e29c1b1ff6ecaa11932d66b754256ebd87 Mon Sep 17 00:00:00 2001 From: Plamen Dimitrov Date: Tue, 3 Jan 2023 13:09:18 +0200 Subject: [PATCH] Provide a minimal CI workflow using GH actions The biggest obstacle here was codecov's GH action which had some really obscure logging and whose default XML exporting did not work out of the box this time, extending our customized coverage analysis further. --- .github/workflows/ci.yml | 92 ++++++++++++++++++++++++++++++++++++++ .travis.yml | 81 --------------------------------- tests/coverage_analysis.sh | 3 ++ 3 files changed, 95 insertions(+), 81 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..0d82fa6c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,92 @@ +name: GH Actions + +on: + pull_request: + workflow_dispatch: + push: + branches: + - master + +jobs: + + original-travis-ci-job: + name: Linux ${{ matrix.install_variant }} with Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + env: + DISPLAY: ':99.0' + + strategy: + matrix: + python-version: [3.7, 3.8, 3.9] + install_variant: ["pip"] + include: + - python-version: 3.7 + install_variant: "rpm.fedora.31" + - python-version: 3.8 + install_variant: "rpm.fedora.32" + - python-version: 3.6 + install_variant: "deb.ubuntu.bionic" + - python-version: 3.7 + install_variant: "deb.ubuntu.focal" + fail-fast: false + + steps: + - run: echo "Job triggered by a ${{ github.event_name }} event on branch is ${{ github.ref }} in repository is ${{ github.repository }}, runner on ${{ runner.os }}" + - name: Check out repository code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Cache large ML models + uses: actions/cache@v3 + with: + path: ~/.cache/torch/hub/checkpoints + key: fasterrcnn-258fb6c6+maskrcnn-bf2d0c1e + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: 'packaging/pip_requirements.txt' + if: matrix.install_variant == 'pip' + - name: Display Python version + run: python -c "import sys; print(sys.version)" + if: matrix.install_variant == 'pip' + - name: Install non-python dependencies using system packager + run: | + # ocr + sudo apt-get -y install pkg-config tesseract-ocr libtesseract-dev + # xdotool + sudo apt-get -y install xdotool x11-apps imagemagick + # vncdotool + sudo apt-get -y install xfonts-base x11vnc + # pyautogui + sudo apt-get -y install scrot + if: matrix.install_variant == 'pip' + - name: Install any dependencies and build the package + run: pip --default-timeout=60 install -r packaging/pip_requirements.txt + if: matrix.install_variant == 'pip' + - run: sleep 3 + - name: Prepare virtual screen (fake display) + run: | + sudo apt-get -y install libx11-dev libxtst-dev xvfb vim-common + sudo apt-get install -y x11-utils libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0 + Xvfb :99 -screen 0 1024x768x24 &> /tmp/xvfb.log & + sudo touch ~/.Xauthority + sudo xauth -f ~/.Xauthority add ${HOST}:99 . $(xxd -l 16 -p /dev/urandom) + sudo chmod a+rxw ~/.Xauthority + # give xvfb some time to start + sleep 3 + - name: Run current semi-isolation semi-integration tests (to be separated in the future) + run: | + if [[ ${{ matrix.python-version }} == '3.9' ]]; then export DISABLE_AUTOPY=1; fi + if [[ ${{ matrix.install_variant }} != "pip" ]]; then cd packaging && bash packager_docker.sh; + else cd tests && bash coverage_analysis.sh; fi + env: + INSTALL_VARIANT: ${{ matrix.install_variant }} + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + files: tests/coverage.xml + verbose: true + if: matrix.install_variant == 'pip' + - run: echo "🥑 This job's status is ${{ job.status }}." diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f5a4d4fb..00000000 --- a/.travis.yml +++ /dev/null @@ -1,81 +0,0 @@ -os: linux -dist: bionic -language: python -python: -- 3.6 -- 3.7 -- 3.8 -- pypy3.5 -env: -- INSTALL_VARIANT=pip -jobs: - include: - - python: 3.7 - env: INSTALL_VARIANT=rpm.fedora.31 - - python: 3.8 - env: INSTALL_VARIANT=rpm.fedora.32 - - python: 3.6 - env: INSTALL_VARIANT=deb.ubuntu.bionic - - python: 3.7 - env: INSTALL_VARIANT=deb.ubuntu.focal - allow_failures: - # the pypi setup is limited in terms of Qt4 and Tesseract dependencies - - python: pypy3.5 -services: -# docker images are not cached by travis due to vm recreation for each build -- docker -cache: - pip: true - directories: - # cache large DL models - - $HOME/.cache/torch/checkpoints -addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - # ocr - - pkg-config - - tesseract-ocr - - libtesseract-dev - # virtual screen - - libx11-dev - - libxtst-dev - # xdotool - - xdotool - - x11-apps - - imagemagick - # vncdotool - - xfonts-base - - x11vnc - # pyautogui - - scrot -# install any dependencies and build package -install: -- if [[ $INSTALL_VARIANT == "pip" ]]; then travis_retry pip --default-timeout=60 install -r packaging/pip_requirements.txt; fi -before_script: -- export DISPLAY=:99.0 -- "/sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile - --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1024x768x24" -- sleep 3 # give xvfb some time to start -- if [[ $TRAVIS_PYTHON_VERSION == 'pypy3.5' ]]; then export DISABLE_AUTOPY=1; fi -- if [[ $TRAVIS_PYTHON_VERSION == 'pypy3.5' ]]; then export DISABLE_PYQT=1; fi -- if [[ $TRAVIS_PYTHON_VERSION == 'pypy3.5' ]]; then export DISABLE_OPENCV=1; fi -# run tests -script: -- if [[ $INSTALL_VARIANT != "pip" ]]; then cd packaging && bash packager_docker.sh; - else cd tests && bash coverage_analysis.sh; fi -- if [[ $INSTALL_VARIANT == "pip" ]]; then codecov; fi -# use this to disable email notifications (e.g. while testing configuration) -#notifications: -#email: false -before_deploy: "cd .. && cd packaging" -deploy: - provider: pypi - username: "__token__" - password: - secure: xIlFxzEDn+Wnl6EsYbTvR2wiuU0AJbqoc+qtrvY8lpvuE5WDB+b6YNOIyWQ6WBjyMhQrget5/zl36AyB44H6WbJO/9n9RcfpeTpqm/UzSqbLxFrZGqW1k7iQVRNhb+DBi3KfsWGteczFRUQDycExp5qiBjMC3nSq7vhawmFFESPFRVRHoAp0naa6l/dsICPTDHFwcVR1adzJMarFByJ4AkenHPzYi5gFDSPtk0JAHFBNy1l2WlSYNtBX2Umo0GJgEAu5SPem9fnp1CZ+4pOHxqoEuszQvDE7ICZRLYDmOAe29qi+FUgJkUabOF0yPRzeXmFyLeomf2iqpn95O2HKx/+B/2P7QVpleNhyVI490KMASkISWgbd0EYb8vZQ9gdH+k1DtYd/Gi9Tn+1gbLORq3gHnXYQ9t9LEKS5kGEEkWQjG6L4eqINrawLxfTvzihTXzPRWyur0pTV/qPKtZfJeCyNMC1/2mDeitaWO17CeU/Y/qbYc6qlbW/hOnGSH6ODVVQP1Gh87/JHzzK3iIx2QmyubHfFVeixoVm/zf5F5u/DNxsAZm0Cuu1BzSGpEwjgVbjoMQxTsD8HINvEqcG9dRw+L7R6ym9E0cPp+PGXf4NNc8f0GK0yUHSIoUCylIvfe8JbGLKYKQ3UhLFN76lLdSInCV+z9lDCTLwO1zsAka0= - distributions: "sdist bdist_wheel" - skip_existing: true - on: - tags: true diff --git a/tests/coverage_analysis.sh b/tests/coverage_analysis.sh index 947910b4..8f434f24 100755 --- a/tests/coverage_analysis.sh +++ b/tests/coverage_analysis.sh @@ -9,4 +9,7 @@ readonly submit="${SUBMIT:-0}" $coverage run --source="$libpath" -m unittest discover -v -s ../tests/ # use -i to ignore errors from pythong cache files and other traced dependencies $coverage report -m -i +# codecov is poorly documented and even in verbose mode and various directory changes +# won't tell us where it runs the xml command and why it ends up not finding it +$coverage xml if [[ $submit == 1 ]]; then codecov; fi