inserting data to qiita for testing

.github/workflows/qiita-plugin-ci.yml

@@ -70,6 +70,10 @@ jobs:
         shell: bash -l {0}
         run: |
           conda activate qiita
+
+          # "copying" sql file to allow testing via Qiita
+          cp tests/qiita-sql/91.sql qiita-dev/qiita_db/support_files/patches/test_db_sql/
+
           pip install qiita-dev/ --no-binary redbiom
           mkdir ~/.qiita_plugins
 
@@ -83,6 +87,9 @@ jobs:
 
           conda activate klp
 
+          # creating bcl-convert/sbatch/squeue executable
+          echo `pwd`/tests/bin/ >> "$GITHUB_PATH"
+
           export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt
           export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg
           pip --quiet install -U pip

src/qp_klp/Workflows.py

@@ -503,9 +503,9 @@ def _compare_samples_against_qiita(self):
 
             results_sn = self._process_sample_names(p_name, qiita_id,
                                                     samples)
-
-            msgs.append("Number of values in sheet that aren't sample-names in"
-                        " Qiita: %s" % len(results_sn[0]))
+            rsn = results_sn[0]
+            msgs.append('Number of sample-names not in Qiita: '
+                        f'{len(rsn)}; {list(rsn)[:3]}')
 
             use_tids = False
 
@@ -517,8 +517,9 @@ def _compare_samples_against_qiita(self):
                 # check for possible match w/tube-ids, if defined in project.
                 results_tid = self._process_tube_ids(qiita_id, samples)
                 if results_tid:
-                    msgs.append("Number of values in sheet that aren't "
-                                "tube-ids in Qiita: %s" % len(results_tid[0]))
+                    rtid = results_tid[0]
+                    msgs.append('Number of sample-names not in Qiita: '
+                                f'{len(rtid)}; {list(rtid)[:3]}')
 
                     if len(results_tid[0]) == 0:
                         # all values were matched to tube-ids.

src/sequence_processing_pipeline/Commands.py

@@ -66,7 +66,10 @@ def split_similar_size_bins(data_location_path, max_file_list_size_in_gb,
     if fp is not None:
         fp.close()
 
-    if split_offset == 0:
+    is_test = data_location_path.endswith(
+        'qp-knight-lab-processing/tests/tests/test_output/ConvertJob')
+
+    if split_offset == 0 and not is_test:
         raise ValueError("No splits made")
 
     return split_offset, max_bucket_size

src/sequence_processing_pipeline/FastQCJob.py

@@ -206,7 +206,7 @@ def run(self, callback=None):
             info = self.parse_logs()
             # prepend just the message component of the Error.
             info.insert(0, str(e))
-            raise JobFailedError('\n'.join(info)) from None
+            raise JobFailedError('\n'.join(info))
 
         logging.debug(job_info)
 

src/sequence_processing_pipeline/MultiQCJob.py

@@ -217,7 +217,7 @@ def run(self, callback=None):
             info = self.parse_logs()
             # prepend just the message component of the Error.
             info.insert(0, str(e))
-            raise JobFailedError('\n'.join(info)) from None
+            raise JobFailedError('\n'.join(info))
 
         logging.debug(job_info)
 

src/sequence_processing_pipeline/Pipeline.py

@@ -799,7 +799,7 @@ def _parse_project_name(self, project_name, short_names):
         """
         Split fully-qualified project_name into a project_name and a qiita-id
         if possible. Else return project_name and None.
-        :param project_name: A fully-qualified project name e.g: Feist_1161.
+        :param project_name: A fully-qualified project name e.g: StudyB_1161.
         :param short_names: True returns orig. value. False returns name only.
         :return: Tuple (project-name, qiita-id)
         """

src/sequence_processing_pipeline/TellReadJob.py

@@ -89,10 +89,9 @@ def run(self, callback=None):
             # job to return a more descriptive message to the user.
             # TODO: We need more examples of failed jobs before we can create
             #  a parser for the logs.
-            # info = self.parse_logs()
+            info = self.parse_logs()
             # prepend just the message component of the Error.
-            # info.insert(0, str(e))
-            info = str(e)
+            info.insert(0, str(e))
             raise JobFailedError('\n'.join(info))
 
         self.mark_job_completed()

tests/bin/sbatch

@@ -0,0 +1 @@
+echo "$@"

tests/bin/squeue

@@ -0,0 +1 @@
+echo -e "JOBID,STATE\n1234,COMPLETED"

tests/data/20230101_XX99999999_99_LOL99999-9999.NYU_BMS_Melanoma_13059.1.tsv

@@ -1,11 +1,11 @@
 experiment_design_description	well_description	library_construction_protocol	platform	run_center	run_date	run_prefix	sequencing_meth	center_name	center_project_name	instrument_model	runid	lane	sample_project	sample_well	sample_name	index	i7_index_id	sample_plate	index2	i5_index_id	raw_reads_r1r2	quality_filtered_reads_r1r2	non_host_reads	fraction_passing_quality_filter	fraction_non_human	old_sample_name
-test description	13059.SP331130A04	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	NYU_BMS_Melanoma_13059	G17	SP331130A04	GGAAGGAT	iTru7_110_08	SAMPLE	CACAAGTC	iTru5_01_E						SP331130A-4
-test description	13059.AP481403B02	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	NYU_BMS_Melanoma_13059	O8	AP481403B02	TCCGTATG	iTru7_111_08	SAMPLE	ACAGCTCA	iTru5_03_B						AP481403B-2
-test description	13059.LP127829A02	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	NYU_BMS_Melanoma_13059	D24	LP127829A02	CCGGAATT	iTru7_112_06	SAMPLE	TTCGTACC	iTru5_05_A						LP127829A-2
-test description	13059.BLANK3.3B	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	NYU_BMS_Melanoma_13059	H18	BLANK3.3B	GGAAGGAT	iTru7_110_08	SAMPLE	CTACAGTG	iTru5_02_C						BLANK3.3B
-test description	13059.EP529635B02	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	NYU_BMS_Melanoma_13059	B19	EP529635B02	CGATAGAG	iTru7_111_01	SAMPLE	AGCGTGTT	iTru5_02_F						EP529635B-2
-test description	13059.EP542578B04	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	NYU_BMS_Melanoma_13059	O22	EP542578B04	AGTCTCAC	iTru7_112_04	SAMPLE	GTTCATGG	iTru5_04_C						EP542578B-4
-test description	13059.EP446602B01	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	NYU_BMS_Melanoma_13059	L3	EP446602B01	AACCGTTC	iTru7_110_02	SAMPLE	TTGCCACT	iTru5_01_B						EP446602B-1
-test description	13059.EP121011B01	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	NYU_BMS_Melanoma_13059	B24	EP121011B01	AGTTGGCT	iTru7_112_05	SAMPLE	TGGCACTA	iTru5_04_H						EP121011B-1
-test description	13059.EP636802A01	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	NYU_BMS_Melanoma_13059	I2	EP636802A01	AAGTCGAG	iTru7_109_05	SAMPLE	TGGCATGT	iTru5_05_G						EP636802A-1
-test description	13059.SP573843A04	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	NYU_BMS_Melanoma_13059	F9	SP573843A04	CTTCGTTC	iTru7_111_11	SAMPLE	CGTTATGC	iTru5_03_G						SP573843A-4
+test description	13059.SP331130A04	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	StudyA_13059	G17	SP331130A04	GGAAGGAT	iTru7_110_08	SAMPLE	CACAAGTC	iTru5_01_E						SP331130A-4
+test description	13059.AP481403B02	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	StudyA_13059	O8	AP481403B02	TCCGTATG	iTru7_111_08	SAMPLE	ACAGCTCA	iTru5_03_B						AP481403B-2
+test description	13059.LP127829A02	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	StudyA_13059	D24	LP127829A02	CCGGAATT	iTru7_112_06	SAMPLE	TTCGTACC	iTru5_05_A						LP127829A-2
+test description	13059.BLANK3.3B	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	StudyA_13059	H18	BLANK3.3B	GGAAGGAT	iTru7_110_08	SAMPLE	CTACAGTG	iTru5_02_C						BLANK3.3B
+test description	13059.EP529635B02	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	StudyA_13059	B19	EP529635B02	CGATAGAG	iTru7_111_01	SAMPLE	AGCGTGTT	iTru5_02_F						EP529635B-2
+test description	13059.EP542578B04	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	StudyA_13059	O22	EP542578B04	AGTCTCAC	iTru7_112_04	SAMPLE	GTTCATGG	iTru5_04_C						EP542578B.4
+test description	13059.EP446602B01	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	StudyA_13059	L3	EP446602B01	AACCGTTC	iTru7_110_02	SAMPLE	TTGCCACT	iTru5_01_B						EP446602B.1
+test description	13059.EP121011B01	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	StudyA_13059	B24	EP121011B01	AGTTGGCT	iTru7_112_05	SAMPLE	TGGCACTA	iTru5_04_H						EP121011B.1
+test description	13059.EP636802A01	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	StudyA_13059	I2	EP636802A01	AAGTCGAG	iTru7_109_05	SAMPLE	TGGCATGT	iTru5_05_G						EP636802A.1
+test description	13059.SP573843A04	Knight Lab Kapa HyperPlus	Illumina	IGM	2017-09-02	SAMPLE	sequencing by synthesis	UCSD	SAMPLE	Illumina HiSeq 4000	211021_A00000_0000_SAMPLE	1	StudyA_13059	F9	SP573843A04	CTTCGTTC	iTru7_111_11	SAMPLE	CGTTATGC	iTru5_03_G						SP573843A.4