fix-ci (#107)

* merging mg-scripts

* -e

* add branch run-ci

* run-ci -> merge_mg-scripts

* rm extra brench

* mv tests files outside src

* global changes to mv to tests/

* some missing sequence_processing_pipeline/tests/

* update package_root

* ./ -> ../

* comment out cover-package

* ../ -> ./

* fixing some tests

* fixing some more tests

* my_state.json

* data/configuration.json -> configuration.json

* making sure 211021_A00000_0000_SAMPLE files are not rm

* minor change

* fix errors

* trying to fix some tests

* fix some tests

* knigt_lab_processing -> knight_lab_processing

* cover-package

* rm extra comments

Author: antgonza

.coveragerc

@@ -0,0 +1,3 @@
+[run]
+branch = True
+relative_files = True

.github/workflows/qiita-plugin-ci.yml

@@ -2,7 +2,7 @@
 
 on:
   push:
-    branches: [dev]
+    branches: [master]
   pull_request:
 
 jobs:
@@ -90,11 +90,10 @@ jobs:
           pip --quiet install https://github.com/qiita-spots/qtp-job-output-folder/archive/refs/heads/main.zip
 
           # pip --quiet install .
-          pip install .
+          export QP_KLP_CONFIG_FP=`pwd`/tests/configuration.json
+          pip install -e .
           pip --quiet install coveralls
 
-          export QP_KLP_CONFIG_FP=`pwd`/configuration.json
-
           configure_qtp_job_output_folder --env-script "source /home/runner/.profile; conda activate klp" --ca-cert $QIITA_ROOTCA_CERT
           configure_klp --env-script "source /home/runner/.profile; export QP_KLP_CONFIG_FP=$QP_KLP_CONFIG_FP; conda activate klp" --ca-cert $QIITA_ROOTCA_CERT
 
@@ -131,17 +130,18 @@ jobs:
           sleep 10
           cat /tmp/supervisord.log
 
-      - name: Main tests
+      - name: qp-knight-lab-processing & mg-scripts tests
         shell: bash -l {0}
         env:
           COVER_PACKAGE: ${{ matrix.cover_package }}
         run: |
           conda activate klp
           export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt
           export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg
-          export QP_KLP_CONFIG_FP=`pwd`/configuration.json
+          export QP_KLP_CONFIG_FP=`pwd`/tests/configuration.json
           export PYTHONWARNINGS="ignore:Certificate for localhost has no \`subjectAltName\`"
-          nosetests --with-doctest --with-coverage -v --cover-package=qp_klp
+          nosetests --with-doctest --with-coverage -v --cover-package qp_klp,sequence_processing_pipeline
+
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
@@ -160,4 +160,4 @@ jobs:
       - name: lint
         run: |
           pip install -q flake8
-          flake8 qp_klp
+          flake8 .

.gitignore

@@ -129,4 +129,7 @@ dmypy.json
 .pyre/
 
 # ignore local changes
-qp_klp/tests/data/sample-sheets/*/*/*.csv
+tests/data/sample-sheets/*/*/*.csv
+
+# test output
+tests/data/output_dir/

LICENSE

@@ -1,6 +1,9 @@
 BSD 3-Clause License
 
-Copyright (c) 2014--, Qiita Spots
+Copyright (c) 2014, Qiita Spots
+Copyright (c) 2020, biocore
+Copyright (c) 2021, jdereus
+
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

README.md

@@ -0,0 +1,68 @@
+# Sequence Processing Pipeline
+
+A Jupyter notebook to assist wet lab shotgun pipeline.
+A packaged Python-based implementation of Knight Lab's sequencing process pipeline.
+
+## Installation
+
+To install this package, first clone this repository from GitHub:
+
+```bash
+git clone https://github.com/biocore/mg-scripts.git
+```
+
+Create a Python3 Conda environment in which to run the notebook:
+
+```bash
+conda create --yes -n spp python='python=3.9' scikit-learn pandas numpy nose pep8 flake8 matplotlib jupyter notebook 'seaborn>=0.7.1' pip openpyxl 'seqtk>=1.4' click scipy fastq-pair
+```
+
+Activate the Conda environment:
+
+```bash
+source activate sp_pipeline
+```
+
+Change directory to the cloned repository folder and install:
+
+```bash
+cd mg-scripts
+pip install -e .
+```
+
+This will automatically install https://github.com/biocore/metagenomics_pooling_notebook.git, a dependency of mg-scripts and the sequence_processing_pipeline.
+
+## Running Unittests
+
+Change directory to the downloaded repository folder:
+
+```bash
+cd mg-scripts
+nosetests --with-coverage --cover-inclusive --cover-package sequence_processing_pipeline
+```
+
+## Getting Started
+
+Review Pipeline.py and main.py to learn how to import and access package functionality:
+
+```bash
+cd mg-scripts/sequence_processing_pipeline
+more Pipeline.py
+more main.py
+```
+
+Adjust configuration settings as needed:
+
+```bash
+vi tests/data/configuration.json
+```
+
+Please note that the setting 'minimap2_databases' is expected to be a list of paths to individual .mmi files for QCJob.
+For NuQCJob, minimap2_databases is expected to be the path to a directory containing two subdirectories: 'metagenomic'
+and 'metatranscriptomic'. Each directory should contain or symlink to the appropriate .mmi files needed for that Assay
+type.
+
+Additional TellSeq-related notes:
+'spades-cloudspades-0.1', 'tellread-release-novaseqX' or similar directories must be placed in a location available to SPP.
+Their paths should be made known to SPP in the configuration files. (See examples for details).
+Additional scripts found in sequence_processing_pipeline/contrib were contributed by Daniel and Omar and can be similarly located and configured.

pyproject.toml

@@ -2,21 +2,28 @@
 requires = ["setuptools", "setuptools-scm"]
 build-backend = "setuptools.build_meta"
 
+[tool.hatch.build.targets.wheel]
+packages = ["qp_klp", "sequence_processing_pipeline"]
+
+[tool.poetry]
+packages = [
+    { include = "src/qp_klp", from = "./", to = "qp_klp" },
+    { include = "src/sequence_processing_pipeline", from = "./", to = "sequence_processing_pipeline" },
+]
+
 [project]
-name = "qp_klp"
+name = "knight_lab_processing"
 # version strings must comply with PEP 440:
 # https://peps.python.org/pep-0440/
-version = "2022.4"
-authors = [
-    {name = "Qiita Development Team", email = "[email protected]"},
-]
+version = "2025.4"
+authors = [{ name = "Qiita Development Team", email = "[email protected]" }]
 description = "Qiita Plugin: Knight Lab Processing"
 readme = "README.rst"
 # ">=" required as most available versions of 3.9 will be "3.9.17" or
 # similar.
 requires-python = ">=3.9"
 keywords = ["bioinformatics", "qiita", "sequence_processing_pipeline"]
-license = {text = "BSD"}
+license = { text = "BSD" }
 classifiers = [
     "Development Status :: 5 - Production/Stable",
     "License :: OSI Approved :: BSD License",
@@ -27,17 +34,26 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: Implementation :: CPython",
     "Operating System :: POSIX :: Linux",
-    "Operating System :: MacOS :: MacOS X"
-    ]
+    "Operating System :: MacOS :: MacOS X",
+]
 dependencies = [
     "nose>=0.10.1",
     "click>=3.3",
     "future",
     "pandas",
+    'requests',
+    'flake8',
+    'nose',
+    'coverage',
+    'pgzip',
+    'jinja2',
+    'numpy',
+    'cython',
     "qiita-files@https://github.com/qiita-spots/qiita-files/archive/master.zip",
     "qiita_client@https://github.com/qiita-spots/qiita_client/archive/master.zip",
-    "sequence-processing-pipeline@https://github.com/biocore/mg-scripts/archive/master.zip"
+    'metapool@https://github.com/biocore/metagenomics_pooling_notebook/archive/master.zip',
 ]
 [project.scripts]
 configure_klp = "qp_klp.scripts.configure_klp:config"
 start_klp = "qp_klp.scripts.start_klp:execute"
+demux = "sequence_processing_pipeline.scripts.cli:demux"