From 15de0d06b579b13976a87c8f97924c57d8a12531 Mon Sep 17 00:00:00 2001 From: Brian Gow Date: Wed, 14 Apr 2021 14:39:35 -0400 Subject: [PATCH 01/13] Create physionet-build Github CI action [WIP] This github action will replace the shippable.yml for build-test. --- .github/workflows/physionet-build-test.yml | 66 ++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/physionet-build-test.yml diff --git a/.github/workflows/physionet-build-test.yml b/.github/workflows/physionet-build-test.yml new file mode 100644 index 0000000000..e64bb501ae --- /dev/null +++ b/.github/workflows/physionet-build-test.yml @@ -0,0 +1,66 @@ +# a github action on the dev branch that performs a build - test in a docker container +name: physionet-build-test + +on: + push: + branches: + - dev + +jobs: + container: + runs-on: ubuntu-latest + container: debian:10 + steps: + - name: checkout physionet-build repo + uses: actions/checkout@v2 + - name: update packages + run: | + sudo apt-get update --yes + - name: install and configure needed software + run: | + sudo apt-get install python3-dev build-essential postgresql zip wget python-virtualenv --yes + sudo service postgresql start + virtualenv -p python3.7 env3 + source env3/bin/activate + ln -sT .env.example .env + - name: install repo dependencies + run: | + pip install -r requirements.txt + - name: install libseccomp-dev for syscall filtering + run: | + sudo apt-get install libseccomp-dev + - name: setup postgres + run: | + sudo -u postgres psql -c "create user physionet with superuser password 'password';" -U postgres + sudo -u postgres psql -c "create database physionet;" -U postgres + - name: install and setup wfdb + run: | + wget https://github.com/bemoody/wfdb/archive/10.6.2.tar.gz -O wfdb.tar.gz + tar -xf wfdb.tar.gz + (cd wfdb-* && ./configure --without-netfiles && sudo make -C lib install && sudo make -C data install) + - name: run linker for newly installed software + run: | + sudo ldconfig + - name: install and setup lightwave + run: | + wget https://github.com/bemoody/lightwave/archive/bfe908a7f53434df61fd2444bf8c235e3e6226fc.tar.gz -O lightwave.tar.gz + tar -xf lightwave.tar.gz + (cd lightwave-* && make CGIDIR=/usr/local/bin sandboxed-server) + wget https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz + - name: install web driver - geckodriver + run: | + mkdir geckodriver + tar -xf geckodriver-v0.23.0-linux64.tar.gz -C geckodriver + export PATH=$PATH:$PWD/geckodriver + - name: setup and test physionet + run: | + cd physionet-django + python manage.py makemigrations --dry-run --no-input --check + python manage.py resetdb + python manage.py loaddemo + python manage.py test --verbosity=3 --keepdb + - name: check the amount of code being tested and print/report the result + run: | + coverage run --source='.' manage.py test --keepdb + coverage report -m + From a0633939229f85625ee78a3161abd80751eb19a7 Mon Sep 17 00:00:00 2001 From: Brian Gow Date: Wed, 14 Apr 2021 15:10:31 -0400 Subject: [PATCH 02/13] Remove excess tabs Remove excess tabs to prevent errors. --- .github/workflows/physionet-build-test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/physionet-build-test.yml b/.github/workflows/physionet-build-test.yml index e64bb501ae..e8545dfa90 100644 --- a/.github/workflows/physionet-build-test.yml +++ b/.github/workflows/physionet-build-test.yml @@ -5,7 +5,7 @@ on: push: branches: - dev - + jobs: container: runs-on: ubuntu-latest @@ -63,4 +63,3 @@ jobs: run: | coverage run --source='.' manage.py test --keepdb coverage report -m - From 901f91d9d4f7aaff96294243a410f1a286ec782a Mon Sep 17 00:00:00 2001 From: Tom Pollard Date: Wed, 14 Apr 2021 16:33:58 -0400 Subject: [PATCH 03/13] build on pr --- .github/workflows/physionet-build-test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/physionet-build-test.yml b/.github/workflows/physionet-build-test.yml index e8545dfa90..685ef70a85 100644 --- a/.github/workflows/physionet-build-test.yml +++ b/.github/workflows/physionet-build-test.yml @@ -5,6 +5,9 @@ on: push: branches: - dev + pull_request: + branches: + - dev jobs: container: From ad7dead414145de669f4edcca9796df5df9e3cb3 Mon Sep 17 00:00:00 2001 From: briangow <4754434+briangow@users.noreply.github.com> Date: Wed, 14 Apr 2021 17:03:33 -0400 Subject: [PATCH 04/13] remove sudo fromcommands since Docker runs as root anyhow. --- .github/workflows/physionet-build-test.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/physionet-build-test.yml b/.github/workflows/physionet-build-test.yml index 685ef70a85..6915cc5a4a 100644 --- a/.github/workflows/physionet-build-test.yml +++ b/.github/workflows/physionet-build-test.yml @@ -18,11 +18,11 @@ jobs: uses: actions/checkout@v2 - name: update packages run: | - sudo apt-get update --yes + apt-get update --yes - name: install and configure needed software run: | - sudo apt-get install python3-dev build-essential postgresql zip wget python-virtualenv --yes - sudo service postgresql start + apt-get install python3-dev build-essential postgresql zip wget python-virtualenv --yes + service postgresql start virtualenv -p python3.7 env3 source env3/bin/activate ln -sT .env.example .env @@ -31,19 +31,19 @@ jobs: pip install -r requirements.txt - name: install libseccomp-dev for syscall filtering run: | - sudo apt-get install libseccomp-dev + apt-get install libseccomp-dev - name: setup postgres run: | - sudo -u postgres psql -c "create user physionet with superuser password 'password';" -U postgres - sudo -u postgres psql -c "create database physionet;" -U postgres + psql -c "create user physionet with superuser password 'password';" -U postgres + psql -c "create database physionet;" -U postgres - name: install and setup wfdb run: | wget https://github.com/bemoody/wfdb/archive/10.6.2.tar.gz -O wfdb.tar.gz tar -xf wfdb.tar.gz - (cd wfdb-* && ./configure --without-netfiles && sudo make -C lib install && sudo make -C data install) + (cd wfdb-* && ./configure --without-netfiles && make -C lib install && make -C data install) - name: run linker for newly installed software run: | - sudo ldconfig + ldconfig - name: install and setup lightwave run: | wget https://github.com/bemoody/lightwave/archive/bfe908a7f53434df61fd2444bf8c235e3e6226fc.tar.gz -O lightwave.tar.gz From 50daa549156c09be1910e08ec5c104298c98e5d2 Mon Sep 17 00:00:00 2001 From: briangow <4754434+briangow@users.noreply.github.com> Date: Wed, 14 Apr 2021 19:49:02 -0400 Subject: [PATCH 05/13] sets default shell --- .github/workflows/physionet-build-test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/physionet-build-test.yml b/.github/workflows/physionet-build-test.yml index 6915cc5a4a..b2f701747c 100644 --- a/.github/workflows/physionet-build-test.yml +++ b/.github/workflows/physionet-build-test.yml @@ -13,6 +13,9 @@ jobs: container: runs-on: ubuntu-latest container: debian:10 + defaults: + run: + shell: bash steps: - name: checkout physionet-build repo uses: actions/checkout@v2 From 12771296abf5df50fca1f51b0d014b31575d1b3a Mon Sep 17 00:00:00 2001 From: briangow <4754434+briangow@users.noreply.github.com> Date: Wed, 14 Apr 2021 21:02:23 -0400 Subject: [PATCH 06/13] add python3-pip --- .github/workflows/physionet-build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/physionet-build-test.yml b/.github/workflows/physionet-build-test.yml index b2f701747c..e6efec0939 100644 --- a/.github/workflows/physionet-build-test.yml +++ b/.github/workflows/physionet-build-test.yml @@ -24,7 +24,7 @@ jobs: apt-get update --yes - name: install and configure needed software run: | - apt-get install python3-dev build-essential postgresql zip wget python-virtualenv --yes + apt-get install python3-dev python3-pip build-essential postgresql zip wget python-virtualenv --yes service postgresql start virtualenv -p python3.7 env3 source env3/bin/activate From c182dd524bc2f85864f8f59a1de1b8a4b7c691dc Mon Sep 17 00:00:00 2001 From: briangow <4754434+briangow@users.noreply.github.com> Date: Wed, 14 Apr 2021 21:20:36 -0400 Subject: [PATCH 07/13] call pip3 instead of pip --- .github/workflows/physionet-build-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/physionet-build-test.yml b/.github/workflows/physionet-build-test.yml index e6efec0939..6418c58fd9 100644 --- a/.github/workflows/physionet-build-test.yml +++ b/.github/workflows/physionet-build-test.yml @@ -31,7 +31,8 @@ jobs: ln -sT .env.example .env - name: install repo dependencies run: | - pip install -r requirements.txt + which pip3 + pip3 install -r requirements.txt - name: install libseccomp-dev for syscall filtering run: | apt-get install libseccomp-dev From 472962f313d2941a99b48334969002b091268db9 Mon Sep 17 00:00:00 2001 From: briangow <4754434+briangow@users.noreply.github.com> Date: Thu, 15 Apr 2021 06:55:54 -0400 Subject: [PATCH 08/13] Lucas/Tom fix for Postgres error. Add path to manage.py. --- .github/workflows/physionet-build-test.yml | 31 +++++++++++++++------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/.github/workflows/physionet-build-test.yml b/.github/workflows/physionet-build-test.yml index 6418c58fd9..a73761cc24 100644 --- a/.github/workflows/physionet-build-test.yml +++ b/.github/workflows/physionet-build-test.yml @@ -13,6 +13,17 @@ jobs: container: runs-on: ubuntu-latest container: debian:10 + services: + postgres: + image: postgres:11.9 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: github_actions + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 defaults: run: shell: bash @@ -25,7 +36,6 @@ jobs: - name: install and configure needed software run: | apt-get install python3-dev python3-pip build-essential postgresql zip wget python-virtualenv --yes - service postgresql start virtualenv -p python3.7 env3 source env3/bin/activate ln -sT .env.example .env @@ -38,8 +48,10 @@ jobs: apt-get install libseccomp-dev - name: setup postgres run: | - psql -c "create user physionet with superuser password 'password';" -U postgres - psql -c "create database physionet;" -U postgres + psql -c "create database physionet;" + env: + POSTGRES_HOST: postgres + POSTGRES_PORT: 5432 - name: install and setup wfdb run: | wget https://github.com/bemoody/wfdb/archive/10.6.2.tar.gz -O wfdb.tar.gz @@ -58,15 +70,14 @@ jobs: run: | mkdir geckodriver tar -xf geckodriver-v0.23.0-linux64.tar.gz -C geckodriver - export PATH=$PATH:$PWD/geckodriver + echo "$PWD/geckodriver" >> $GITHUB_PATH - name: setup and test physionet run: | - cd physionet-django - python manage.py makemigrations --dry-run --no-input --check - python manage.py resetdb - python manage.py loaddemo - python manage.py test --verbosity=3 --keepdb + python physionet-django/manage.py makemigrations --dry-run --no-input --check + python physionet-django/manage.py resetdb + python physionet-django/manage.py loaddemo + python physionet-django/manage.py test --verbosity=3 --keepdb - name: check the amount of code being tested and print/report the result run: | - coverage run --source='.' manage.py test --keepdb + coverage run --source='.' physionet-django/manage.py test --keepdb coverage report -m From 7df8c326a3c11be92175cdf78c8f51d594fac89b Mon Sep 17 00:00:00 2001 From: briangow <4754434+briangow@users.noreply.github.com> Date: Thu, 15 Apr 2021 07:07:30 -0400 Subject: [PATCH 09/13] removed extraneous postgres code. --- .github/workflows/physionet-build-test.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/physionet-build-test.yml b/.github/workflows/physionet-build-test.yml index a73761cc24..3587a1dd8c 100644 --- a/.github/workflows/physionet-build-test.yml +++ b/.github/workflows/physionet-build-test.yml @@ -41,17 +41,10 @@ jobs: ln -sT .env.example .env - name: install repo dependencies run: | - which pip3 pip3 install -r requirements.txt - name: install libseccomp-dev for syscall filtering run: | apt-get install libseccomp-dev - - name: setup postgres - run: | - psql -c "create database physionet;" - env: - POSTGRES_HOST: postgres - POSTGRES_PORT: 5432 - name: install and setup wfdb run: | wget https://github.com/bemoody/wfdb/archive/10.6.2.tar.gz -O wfdb.tar.gz From 657609978bf86a28a5f53928c7d18ac152aa5cbc Mon Sep 17 00:00:00 2001 From: briangow <4754434+briangow@users.noreply.github.com> Date: Thu, 15 Apr 2021 07:15:29 -0400 Subject: [PATCH 10/13] add sudo for lightwave install --- .github/workflows/physionet-build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/physionet-build-test.yml b/.github/workflows/physionet-build-test.yml index 3587a1dd8c..f6493495cf 100644 --- a/.github/workflows/physionet-build-test.yml +++ b/.github/workflows/physionet-build-test.yml @@ -35,7 +35,7 @@ jobs: apt-get update --yes - name: install and configure needed software run: | - apt-get install python3-dev python3-pip build-essential postgresql zip wget python-virtualenv --yes + apt-get install sudo python3-dev python3-pip build-essential postgresql zip wget python-virtualenv --yes virtualenv -p python3.7 env3 source env3/bin/activate ln -sT .env.example .env From 7241f0bd3bfbcfc098f40e8caedd4c358cb063b5 Mon Sep 17 00:00:00 2001 From: briangow <4754434+briangow@users.noreply.github.com> Date: Thu, 15 Apr 2021 07:41:22 -0400 Subject: [PATCH 11/13] missing django fix - install requirements within env3 --- .github/workflows/physionet-build-test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/physionet-build-test.yml b/.github/workflows/physionet-build-test.yml index f6493495cf..4a70c8af7b 100644 --- a/.github/workflows/physionet-build-test.yml +++ b/.github/workflows/physionet-build-test.yml @@ -36,12 +36,11 @@ jobs: - name: install and configure needed software run: | apt-get install sudo python3-dev python3-pip build-essential postgresql zip wget python-virtualenv --yes - virtualenv -p python3.7 env3 - source env3/bin/activate ln -sT .env.example .env - name: install repo dependencies run: | - pip3 install -r requirements.txt + virtualenv -p python3.7 env3 + (source env3/bin/activate && pip3 install -r requirements.txt) - name: install libseccomp-dev for syscall filtering run: | apt-get install libseccomp-dev From 454b1214e207a1691bc46c5d3775c2c068ff5785 Mon Sep 17 00:00:00 2001 From: briangow <4754434+briangow@users.noreply.github.com> Date: Fri, 16 Apr 2021 10:19:13 -0400 Subject: [PATCH 12/13] setup postgres from command line. add virtual env to path. --- .github/workflows/physionet-build-test.yml | 35 ++++++++++------------ 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/.github/workflows/physionet-build-test.yml b/.github/workflows/physionet-build-test.yml index 4a70c8af7b..abe4e464c8 100644 --- a/.github/workflows/physionet-build-test.yml +++ b/.github/workflows/physionet-build-test.yml @@ -3,7 +3,7 @@ name: physionet-build-test on: push: - branches: + branches: - dev pull_request: branches: @@ -12,18 +12,8 @@ on: jobs: container: runs-on: ubuntu-latest - container: debian:10 - services: - postgres: - image: postgres:11.9 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: github_actions - ports: - - 5432:5432 - # needed because the postgres container does not provide a healthcheck - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + container: + image: debian:10 defaults: run: shell: bash @@ -38,12 +28,21 @@ jobs: apt-get install sudo python3-dev python3-pip build-essential postgresql zip wget python-virtualenv --yes ln -sT .env.example .env - name: install repo dependencies + # add virtual env path to github_path so each run: process will see it run: | - virtualenv -p python3.7 env3 - (source env3/bin/activate && pip3 install -r requirements.txt) + export VIRTUAL_ENV=/env3 + virtualenv -p python3.7 $VIRTUAL_ENV + echo "$VIRTUAL_ENV/bin" >> $GITHUB_PATH + source $VIRTUAL_ENV/bin/activate + pip3 install -r requirements.txt - name: install libseccomp-dev for syscall filtering run: | apt-get install libseccomp-dev + - name: setup postgres + run: | + service postgresql start + sudo -u postgres psql -c "create user physionet with superuser password 'password';" -U postgres + sudo -u postgres psql -c "create database physionet;" -U postgres - name: install and setup wfdb run: | wget https://github.com/bemoody/wfdb/archive/10.6.2.tar.gz -O wfdb.tar.gz @@ -58,18 +57,16 @@ jobs: tar -xf lightwave.tar.gz (cd lightwave-* && make CGIDIR=/usr/local/bin sandboxed-server) wget https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz - - name: install web driver - geckodriver + - name: install web driver - geckodriver run: | mkdir geckodriver tar -xf geckodriver-v0.23.0-linux64.tar.gz -C geckodriver echo "$PWD/geckodriver" >> $GITHUB_PATH - - name: setup and test physionet + - name: setup and test physionet, check amount of code tested run: | python physionet-django/manage.py makemigrations --dry-run --no-input --check python physionet-django/manage.py resetdb python physionet-django/manage.py loaddemo python physionet-django/manage.py test --verbosity=3 --keepdb - - name: check the amount of code being tested and print/report the result - run: | coverage run --source='.' physionet-django/manage.py test --keepdb coverage report -m From 77f176817b1f362d44c1bc677f29f092a7abf069 Mon Sep 17 00:00:00 2001 From: briangow <4754434+briangow@users.noreply.github.com> Date: Tue, 20 Apr 2021 14:21:28 -0400 Subject: [PATCH 13/13] final version as tested in my forked branch. no signficant updates since last commit. --- .github/workflows/physionet-build-test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/physionet-build-test.yml b/.github/workflows/physionet-build-test.yml index abe4e464c8..5baef030d6 100644 --- a/.github/workflows/physionet-build-test.yml +++ b/.github/workflows/physionet-build-test.yml @@ -12,8 +12,7 @@ on: jobs: container: runs-on: ubuntu-latest - container: - image: debian:10 + container: debian:10 defaults: run: shell: bash