From e5a85c5f2271bb85a5e69f4502f51af3421d4b54 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Mon, 21 Oct 2024 14:09:32 +0200 Subject: [PATCH 01/14] copy lint workflow from taca, excluding prettier --- .github/workflows/lint-code.yml | 113 ++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 .github/workflows/lint-code.yml diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml new file mode 100644 index 00000000..592b9157 --- /dev/null +++ b/.github/workflows/lint-code.yml @@ -0,0 +1,113 @@ +name: Lint code +on: [push, pull_request] + +jobs: + # Use ruff to check for code style violations + ruff-check: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ruff + - name: ruff --> Check for style violations + # Configured in pyproject.toml + run: ruff check . + + # Use ruff to check code formatting + ruff-format: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ruff + - name: ruff --> Check code formatting + run: ruff format --check . + + # Use mypy for static type checking + mypy-check: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install mypy + # Start by installing type stubs + - name: mypy --> Install stubs + run: echo -e "y" | mypy --install-types . || exit 0 + - name: mypy --> Static type checking + # Configured in pyprojet.toml + run: mypy . + + # Use pipreqs to check for missing dependencies + pipreqs-check: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-dev.txt + + - name: Run pipreqs + run: | + pipreqs --savepath pipreqs.txt taca 2>&1 | tee pipreqs_output.log + if grep -q 'WARNING: Package .* does not exist or network problems' pipreqs_output.log; then + missing_packages=$(grep 'WARNING: Package .* does not exist or network problems' pipreqs_output.log | sed -E 's/.*Package "(.*)" does not exist.*/\1/') + echo "ERROR: Add unresolved packages to requirements. Missing package(s): $missing_packages. Example: ' @ git+https://github.com//.git'" + exit 1 + fi + + - name: Compare requirements + run: | + # Extract and sort package names + awk -F'(=|==|>|>=|<|<=| @ )' '{print $1}' requirements.txt | tr '[:upper:]' '[:lower:]' | sort -u > requirements.compare + awk -F'(=|==|>|>=|<|<=| @ )' '{print $1}' pipreqs.txt | tr '[:upper:]' '[:lower:]' | sort -u > pipreqs.compare + + # Compare package lists + if cmp -s requirements.compare pipreqs.compare + then + echo "Requirements are the same" + + exit 0 + else + echo "Requirements are different" + echo "" + + echo "=== current requirements.txt ===" + echo "" + cat requirements.compare + echo "" + + echo "=== pipreqs requirements ===" + echo "" + cat pipreqs.compare + + exit 1 + fi From 4563dd2bf30fb316f655fef98c96822001ce1386 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Mon, 21 Oct 2024 14:09:57 +0200 Subject: [PATCH 02/14] add dev requirements --- requirements_dev.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requirements_dev.txt b/requirements_dev.txt index 7cb6656b..71074423 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1 +1,5 @@ +ipdb +mypy +pipreqs +ruff selenium From 23029aae3beef1eff86161b99526d8a300d37f63 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Mon, 21 Oct 2024 14:11:10 +0200 Subject: [PATCH 03/14] add .vscode to .gitignore and alphabetize --- .gitignore | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 62cdbed4..a4efc29a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,17 +1,18 @@ -d3/ -jquery/ +.DS_Store +.vscode +*.orig *.pyc +*.swp *~ build/ +d3/ dist/ -status.egg-info/ -run_dir/settings.yaml -tests/test_items.yaml -.DS_Store -run_dir/qc_reports/ +jquery/ Pipfile Pipfile.lock -run_dir/settings/ -*.swp -*.orig run_dir/genosqlrc.yaml +run_dir/qc_reports/ +run_dir/settings.yaml +run_dir/settings/ +status.egg-info/ +tests/test_items.yaml From 1a3be9fc7914c83662dda69a02e67a5a29734695 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Mon, 21 Oct 2024 14:15:48 +0200 Subject: [PATCH 04/14] add more to .gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a4efc29a..f206da9d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ +__pycache__/ .DS_Store -.vscode +.ruff_cache/ +.vscode/ *.orig *.pyc *.swp From 759104ea91632a11bc1e93a6b78af40cbb47b39d Mon Sep 17 00:00:00 2001 From: kedhammar Date: Mon, 21 Oct 2024 14:16:10 +0200 Subject: [PATCH 05/14] introduce pyproject.toml for ci tool settings --- pyproject.toml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..8a625b82 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,25 @@ +# === LINTING ================================================================ + +[tool.ruff.lint] +select = [ + # Ruff default rules + # ------------------------------ + "E4", # pycodestyle Imports + "E7", # pycodestyle Statements + "E9", # pycodestyle Runtime + "F", # Pyflakes + + # Additional Comment + # ------------------------------------------------------ + "I", # isort Best-practice sorting of imports + "UP", # pyupgrade Make sure syntax is up-to-date +] +ignore = [ + "E402", # Module level import not at top of file + "E722", # Do not use bare 'except' + "E741", # Ambiguous variable name +] + +[tool.mypy] +ignore_missing_imports = true +follow_imports = 'skip' \ No newline at end of file From 1374f199e3b9cddacbf3f55c987deed33e234cbd Mon Sep 17 00:00:00 2001 From: kedhammar Date: Mon, 21 Oct 2024 14:39:03 +0200 Subject: [PATCH 06/14] add more tolerant requirements check --- .github/workflows/lint-code.yml | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 592b9157..2b624096 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -77,7 +77,7 @@ jobs: - name: Run pipreqs run: | - pipreqs --savepath pipreqs.txt taca 2>&1 | tee pipreqs_output.log + pipreqs --savepath pipreqs.txt . 2>&1 | tee pipreqs_output.log if grep -q 'WARNING: Package .* does not exist or network problems' pipreqs_output.log; then missing_packages=$(grep 'WARNING: Package .* does not exist or network problems' pipreqs_output.log | sed -E 's/.*Package "(.*)" does not exist.*/\1/') echo "ERROR: Add unresolved packages to requirements. Missing package(s): $missing_packages. Example: ' @ git+https://github.com//.git'" @@ -87,27 +87,18 @@ jobs: - name: Compare requirements run: | # Extract and sort package names - awk -F'(=|==|>|>=|<|<=| @ )' '{print $1}' requirements.txt | tr '[:upper:]' '[:lower:]' | sort -u > requirements.compare - awk -F'(=|==|>|>=|<|<=| @ )' '{print $1}' pipreqs.txt | tr '[:upper:]' '[:lower:]' | sort -u > pipreqs.compare + awk -F'(=|==|>|>=|<|<=| @ )' '{gsub("-", "_", $1); print $1}' requirements.txt | tr '[:upper:]' '[:lower:]' | sort -u > requirements.compare + awk -F'(=|==|>|>=|<|<=| @ )' '{gsub("-", "_", $1); print $1}' pipreqs.txt | tr '[:upper:]' '[:lower:]' | sort -u > pipreqs.compare # Compare package lists - if cmp -s requirements.compare pipreqs.compare - then - echo "Requirements are the same" - + missing_packages=$(comm -23 pipreqs.compare requirements.compare) + if [ -z "$missing_packages" ]; then + echo "All packages in pipreqs are listed in requirements" exit 0 else - echo "Requirements are different" - echo "" - - echo "=== current requirements.txt ===" - echo "" - cat requirements.compare + echo "Some packages in pipreqs are not listed in requirements" echo "" - - echo "=== pipreqs requirements ===" - echo "" - cat pipreqs.compare - + echo "=== Missing packages ===" + echo "$missing_packages" exit 1 fi From 93a21cc9f777b450ff15a365916f54dd7b294ba4 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Mon, 21 Oct 2024 14:39:12 +0200 Subject: [PATCH 07/14] update and alphabetize requirements --- requirements.txt | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3203012a..d2f1cd9a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,24 +1,29 @@ +atlassian-python-api backports.ssl-match-hostname>=3.5.0.1 +cffi CouchDB==1.1 ecdsa>=0.13 genologics @ git+https://github.com/SciLifeLab/genologics.git genologics_sql @ git+https://github.com/SciLifeLab/genologics_sql.git httplib2>=0.10.3 ibmcloudant>=0.9.1 -cffi -matplotlib>=2.0.0 markdown +matplotlib>=2.0.0 nest_asyncio +nose numpy>=1.12.1 oauth2>=1.9.0.post1 openpyxl -atlassian-python-api +pandas +psycopg2 pycryptodome>=3.6.1 pyparsing>=2.2.0 python-dateutil>=2.7.5 pytz>=2018.9 PyYAML requests>=2.14.2 +selenium +setuptools simplejson>=3.10.0 six>=1.10.0 slack_sdk From 107178d4178ad2f7bca507b9f7e6fef59159e535 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Mon, 21 Oct 2024 14:45:29 +0200 Subject: [PATCH 08/14] mypy exclude build --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8a625b82..88d697af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,4 +22,5 @@ ignore = [ [tool.mypy] ignore_missing_imports = true -follow_imports = 'skip' \ No newline at end of file +follow_imports = 'skip' +exclude = 'build' \ No newline at end of file From dce3cba3ae9c0b345af74c37d59d1fd362a3c8ed Mon Sep 17 00:00:00 2001 From: kedhammar Date: Mon, 21 Oct 2024 14:57:42 +0200 Subject: [PATCH 09/14] ignore mypy for now, can't be bothered --- .github/workflows/lint-code.yml | 21 --------------------- pyproject.toml | 5 ----- requirements_dev.txt | 1 - 3 files changed, 27 deletions(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 2b624096..91668a4f 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -37,27 +37,6 @@ jobs: - name: ruff --> Check code formatting run: ruff format --check . - # Use mypy for static type checking - mypy-check: - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install mypy - # Start by installing type stubs - - name: mypy --> Install stubs - run: echo -e "y" | mypy --install-types . || exit 0 - - name: mypy --> Static type checking - # Configured in pyprojet.toml - run: mypy . - # Use pipreqs to check for missing dependencies pipreqs-check: runs-on: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml index 88d697af..4f14b00c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,8 +19,3 @@ ignore = [ "E722", # Do not use bare 'except' "E741", # Ambiguous variable name ] - -[tool.mypy] -ignore_missing_imports = true -follow_imports = 'skip' -exclude = 'build' \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index 71074423..8a7fce3a 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,5 +1,4 @@ ipdb -mypy pipreqs ruff selenium From 53e2bad7b7701c7df97ae64631d2fa01af8008ac Mon Sep 17 00:00:00 2001 From: kedhammar Date: Mon, 21 Oct 2024 14:59:52 +0200 Subject: [PATCH 10/14] use py3.12 for linting jobs --- .github/workflows/lint-code.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 91668a4f..035941d6 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -11,7 +11,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.12" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -29,7 +29,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.12" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -46,7 +46,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.12" - name: Install dependencies run: | From 0c49d0ffffc3ab2dd02386054d0479b496d24fe2 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 22 Oct 2024 13:30:21 +0200 Subject: [PATCH 11/14] include pip .txt env in conda .yml and remove duplicates --- conda_requirements.yml | 19 ++++++++++++------- requirements.txt | 2 -- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/conda_requirements.yml b/conda_requirements.yml index d4b14013..0af3fb94 100644 --- a/conda_requirements.yml +++ b/conda_requirements.yml @@ -1,9 +1,14 @@ +channels: + - conda-forge + - defaults dependencies: - python=3.12 - - 'conda-forge::pango>=1.42.0' - - 'conda-forge::pandas>=1.3.2' - - conda-forge::psycopg2 - - conda-forge::open-fonts - - conda-forge::xorg-libxrender - - conda-forge::xorg-libxext - - conda-forge::xorg-libxau + - pango>=1.42.0 + - pandas>=1.3.2 + - psycopg2 + - open-fonts + - xorg-libxrender + - xorg-libxext + - xorg-libxau + - pip: + - '-r requirements.txt' diff --git a/requirements.txt b/requirements.txt index d2f1cd9a..9a489695 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,8 +14,6 @@ nose numpy>=1.12.1 oauth2>=1.9.0.post1 openpyxl -pandas -psycopg2 pycryptodome>=3.6.1 pyparsing>=2.2.0 python-dateutil>=2.7.5 From 42dadb9573abcb661870b15cef33d1b6ebcd5e6f Mon Sep 17 00:00:00 2001 From: kedhammar Date: Wed, 23 Oct 2024 10:53:57 +0200 Subject: [PATCH 12/14] remove nose from reqs, only present in outdated tests --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9a489695..d56ea4b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,6 @@ ibmcloudant>=0.9.1 markdown matplotlib>=2.0.0 nest_asyncio -nose numpy>=1.12.1 oauth2>=1.9.0.post1 openpyxl From 72cf543d42194d8dbdc811c142c5d08bafa2aa69 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Wed, 23 Oct 2024 10:54:16 +0200 Subject: [PATCH 13/14] remove defaults channel from conda --- conda_requirements.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/conda_requirements.yml b/conda_requirements.yml index 0af3fb94..6fd2603c 100644 --- a/conda_requirements.yml +++ b/conda_requirements.yml @@ -1,6 +1,5 @@ channels: - conda-forge - - defaults dependencies: - python=3.12 - pango>=1.42.0 From 75694e18b340352621111bd2654c62c269afaa4f Mon Sep 17 00:00:00 2001 From: kedhammar Date: Wed, 23 Oct 2024 10:54:34 +0200 Subject: [PATCH 14/14] remove pipreqs from CI --- .github/workflows/lint-code.yml | 45 --------------------------------- requirements_dev.txt | 1 - 2 files changed, 46 deletions(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 035941d6..1dad5e52 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -36,48 +36,3 @@ jobs: pip install ruff - name: ruff --> Check code formatting run: ruff format --check . - - # Use pipreqs to check for missing dependencies - pipreqs-check: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.12" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r requirements-dev.txt - - - name: Run pipreqs - run: | - pipreqs --savepath pipreqs.txt . 2>&1 | tee pipreqs_output.log - if grep -q 'WARNING: Package .* does not exist or network problems' pipreqs_output.log; then - missing_packages=$(grep 'WARNING: Package .* does not exist or network problems' pipreqs_output.log | sed -E 's/.*Package "(.*)" does not exist.*/\1/') - echo "ERROR: Add unresolved packages to requirements. Missing package(s): $missing_packages. Example: ' @ git+https://github.com//.git'" - exit 1 - fi - - - name: Compare requirements - run: | - # Extract and sort package names - awk -F'(=|==|>|>=|<|<=| @ )' '{gsub("-", "_", $1); print $1}' requirements.txt | tr '[:upper:]' '[:lower:]' | sort -u > requirements.compare - awk -F'(=|==|>|>=|<|<=| @ )' '{gsub("-", "_", $1); print $1}' pipreqs.txt | tr '[:upper:]' '[:lower:]' | sort -u > pipreqs.compare - - # Compare package lists - missing_packages=$(comm -23 pipreqs.compare requirements.compare) - if [ -z "$missing_packages" ]; then - echo "All packages in pipreqs are listed in requirements" - exit 0 - else - echo "Some packages in pipreqs are not listed in requirements" - echo "" - echo "=== Missing packages ===" - echo "$missing_packages" - exit 1 - fi diff --git a/requirements_dev.txt b/requirements_dev.txt index 8a7fce3a..92c89dee 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,3 @@ ipdb -pipreqs ruff selenium