rm create qiime mapping file (#3043)

* Version 092020 (#3034)

* inject study_type in EBI and improvements to current automatic processing pipeline (#3023)

* inject study_type in ebi and improvements to current automatic proecssing pipeline

* addressing @ElDeveloper comments

* some general fixes/additions for next release (#3026)

* some general fixes/additions for next release

* adding test for not None job.release_validator_job

* fix #2839

* fix #2868 (#3028)

* fix #2868

* 2nd round

* fix errors

* more changes

* fix errors

* fix ProcessingJobTest

* fix PY_PATCH

* add missing TRN.add

* encapsulated_query -> perform_as_transaction

* fix #3022 (#3030)

* fix #3022

* adding tests

* fix #2320 (#3031)

* fix #2320

* adding prints to debug

* children -> 1

* APIArtifactHandlerTest -> APIArtifactHandlerTests

* configure_biom

* qdb.util.activate_or_update_plugins

* improving code

* almost there

* add values.template

* fix filepaths

* filepaths -> files

* fixing errors

* add prep.artifact insertion

* addressing @ElDeveloper comments

* fix artifact_definition active command

* != -> ==

* Added three tutorial sections to the Qiita documentation (#3032)

* Added three tutorial sections to the Qiita documentation: 'Retrieving Public Data for Own Analysis' and 'Processing public data retrieved with redbiom' to the redbiom tab, and 'Statistical Analysis to Justify Clinical Trial Sample Size Tutorial' to the analyzing samples tab.

* Update redbiom.rst

* Update redbiom.rst

* Update redbiom.rst

* Further updates to redbiom.rst and the Stats tutorial.

* update redbiom.rst

* Finished proof-reading

* Placed all three tutorials/sections together under Introduction to the download and analysis of public Qiita data

* added a new introduction, with links to the three sections

* Added figures to stats tutorial and contexts explanation

* Added figures to stats tutorial and contexts explanation

* Apply suggestions from code review [skip ci]

Co-authored-by: Yoshiki Vázquez Baeza <[email protected]>

Co-authored-by: Antonio Gonzalez <[email protected]>
Co-authored-by: Yoshiki Vázquez Baeza <[email protected]>

* 092020 (#3033)

* 092020

* connect artifact with job

* rm INSERT qiita.artifact_processing_job

* Apply suggestions from code review [skip ci]

Co-authored-by: Yoshiki Vázquez Baeza <[email protected]>

Co-authored-by: Yoshiki Vázquez Baeza <[email protected]>

Co-authored-by: Daniel McDonald <[email protected]>
Co-authored-by: Mirte Kuijpers <[email protected]>
Co-authored-by: Yoshiki Vázquez Baeza <[email protected]>

* rm create_qiime_mapping_file

* fixing some tests

* fixing more tests

* fix even more tests

* rm npt.assert_warns

* qiime-map -> sample-file

* update not_merged_samples.txt

* adding @ElDeveloper changes

Co-authored-by: Daniel McDonald <[email protected]>
Co-authored-by: Mirte Kuijpers <[email protected]>
Co-authored-by: Yoshiki Vázquez Baeza <[email protected]>

Author: 4 people

qiita_db/analysis.py

@@ -1040,13 +1040,16 @@ def _build_mapping_file(self, samples, rename_dup_samples=False):
         with qdb.sql_connection.TRN:
             all_ids = set()
             to_concat = []
+            sample_infos = dict()
             for aid, samps in samples.items():
-                pt = qdb.artifact.Artifact(aid).prep_templates[0]
-                qiime_map_fp = pt.qiime_map_fp
+                artifact = qdb.artifact.Artifact(aid)
+                si = artifact.study.sample_template
+                if si not in sample_infos:
+                    sample_infos[si] = si.to_dataframe()
+                pt = artifact.prep_templates[0]
+                pt_df = pt.to_dataframe()
 
-                # Parse the mapping file
-                qm = qdb.metadata_template.util.load_template_to_dataframe(
-                    qiime_map_fp, index='#SampleID')
+                qm = pt_df.join(sample_infos[si], lsuffix="_prep")
 
                 # if we are not going to merge the duplicated samples
                 # append the aid to the sample name
@@ -1076,15 +1079,6 @@ def _build_mapping_file(self, samples, rename_dup_samples=False):
 
             merged_map = pd.concat(to_concat)
 
-            # forcing QIIME column order
-            cols = merged_map.columns.values.tolist()
-            cols.remove('BarcodeSequence')
-            cols.remove('LinkerPrimerSequence')
-            cols.remove('Description')
-            cols = (['BarcodeSequence', 'LinkerPrimerSequence'] + cols +
-                    ['Description'])
-            merged_map = merged_map[cols]
-
             # Save the mapping file
             _, base_fp = qdb.util.get_mountpoint(self._table)[0]
             mapping_fp = join(base_fp, "%d_analysis_mapping.txt" % self._id)

qiita_db/handlers/prep_template.py

@@ -66,21 +66,25 @@ def get(self, prep_id):
             'investigation_type': prep info investigation type
             'study': study that the prep info belongs to
             'status': prep info status
-            'qiime-map': the path to the qiime mapping file
+            'sample-file': the path to the sample information file
             'prep-file': the path to the prep info file
         """
         with qdb.sql_connection.TRN:
             pt = _get_prep_template(prep_id)
             prep_files = [fp for _, fp in pt.get_filepaths()
                           if 'qiime' not in basename(fp)]
             artifact = pt.artifact.id if pt.artifact is not None else None
+            sid = pt.study_id
             response = {
                 'data_type': pt.data_type(),
                 'artifact': artifact,
                 'investigation_type': pt.investigation_type,
-                'study': pt.study_id,
+                'study': sid,
                 'status': pt.status,
-                'qiime-map': pt.qiime_map_fp,
+                # get_filepaths returns an ordered list of [filepath_id,
+                # filepath] and we want the last pair
+                'sample-file': qdb.study.Study(
+                    sid).sample_template.get_filepaths()[0][1],
                 # The first element in the prep_files is the newest
                 # prep information file - hence the correct one
                 'prep-file': prep_files[0]

qiita_db/handlers/tests/test_prep_template.py

@@ -54,8 +54,8 @@ def test_get(self):
         self.assertEqual(obs['investigation_type'], 'Metagenomics')
         self.assertEqual(obs['study'], 1)
         self.assertEqual(obs['status'], 'private')
-        self.assertTrue(obs['qiime-map'].startswith(
-            path_builder('1_prep_1_qiime_')))
+        self.assertTrue(obs['sample-file'].startswith(
+            path_builder('1_')))
         self.assertTrue(obs['prep-file'].startswith(
             path_builder('1_prep_1_')))
 

qiita_db/metadata_template/prep_template.py

@@ -9,11 +9,8 @@
 from os.path import join
 from time import strftime
 from copy import deepcopy
-import warnings
 from skbio.util import find_duplicates
 
-import pandas as pd
-
 from qiita_core.exceptions import IncompetentQiitaDeveloperError
 import qiita_db as qdb
 from .constants import (PREP_TEMPLATE_COLUMNS, TARGET_GENE_DATA_TYPES,
@@ -519,116 +516,6 @@ def generate_files(self, samples=None, columns=None):
             fp_id = qdb.util.convert_to_id("prep_template", "filepath_type")
             self.add_filepath(fp, fp_id=fp_id)
 
-            # creating QIIME mapping file
-            self.create_qiime_mapping_file()
-
-    def create_qiime_mapping_file(self):
-        """This creates the QIIME mapping file and links it in the db.
-
-        Returns
-        -------
-        filepath : str
-            The filepath of the created QIIME mapping file
-
-        Raises
-        ------
-        ValueError
-            If the prep template is not a subset of the sample template
-        QiitaDBWarning
-            If the QIIME-required columns are not present in the template
-
-        Notes
-        -----
-        We cannot ensure that the QIIME-required columns are present in the
-        metadata map. However, we have to generate a QIIME-compliant mapping
-        file. Since the user may need a QIIME mapping file, but not these
-        QIIME-required columns, we are going to create them and
-        populate them with the value XXQIITAXX.
-        """
-        with qdb.sql_connection.TRN:
-            rename_cols = {
-                'barcode': 'BarcodeSequence',
-                'primer': 'LinkerPrimerSequence',
-                'description': 'Description',
-            }
-
-            if 'reverselinkerprimer' in self.categories():
-                rename_cols['reverselinkerprimer'] = 'ReverseLinkerPrimer'
-                new_cols = ['BarcodeSequence', 'LinkerPrimerSequence',
-                            'ReverseLinkerPrimer']
-            else:
-                new_cols = ['BarcodeSequence', 'LinkerPrimerSequence']
-
-            # Retrieve the latest sample template
-            # Since we sorted the filepath retrieval, the first result contains
-            # the filepath that we want. `retrieve_filepaths` returns a
-            # 3-tuple, in which the fp is the second element
-            sample_template_fp = qdb.util.retrieve_filepaths(
-                "sample_template_filepath", "study_id", self.study_id,
-                sort='descending')[0]['fp']
-
-            # reading files via pandas
-            st = qdb.metadata_template.util.load_template_to_dataframe(
-                sample_template_fp)
-            pt = self.to_dataframe()
-
-            st_sample_names = set(st.index)
-            pt_sample_names = set(pt.index)
-
-            if not pt_sample_names.issubset(st_sample_names):
-                raise ValueError(
-                    "Prep template is not a sub set of the sample template, "
-                    "file: %s - samples: %s"
-                    % (sample_template_fp,
-                       ', '.join(pt_sample_names-st_sample_names)))
-
-            mapping = pt.join(st, lsuffix="_prep")
-            mapping.rename(columns=rename_cols, inplace=True)
-
-            # Pre-populate the QIIME-required columns with the value XXQIITAXX
-            index = mapping.index
-            placeholder = ['XXQIITAXX'] * len(index)
-            missing = []
-            for val in rename_cols.values():
-                if val not in mapping:
-                    missing.append(val)
-                    mapping[val] = pd.Series(placeholder, index=index)
-
-            if missing:
-                warnings.warn(
-                    "Some columns required to generate a QIIME-compliant "
-                    "mapping file are not present in the template. A "
-                    "placeholder value (XXQIITAXX) has been used to populate "
-                    "these columns. Missing columns: %s"
-                    % ', '.join(sorted(missing)),
-                    qdb.exceptions.QiitaDBWarning)
-
-            # Gets the orginal mapping columns and readjust the order to comply
-            # with QIIME requirements
-            cols = mapping.columns.values.tolist()
-            cols.remove('BarcodeSequence')
-            cols.remove('LinkerPrimerSequence')
-            cols.remove('Description')
-            new_cols.extend(cols)
-            new_cols.append('Description')
-            mapping = mapping[new_cols]
-
-            # figuring out the filepath for the QIIME map file
-            _id, fp = qdb.util.get_mountpoint('templates')[0]
-            filepath = join(fp, '%d_prep_%d_qiime_%s.txt' % (self.study_id,
-                            self.id, strftime("%Y%m%d-%H%M%S")))
-
-            # Save the mapping file
-            mapping.to_csv(filepath, index_label='#SampleID', na_rep='',
-                           sep='\t', encoding='utf-8')
-
-            # adding the fp to the object
-            self.add_filepath(
-                filepath,
-                fp_id=qdb.util.convert_to_id("qiime_map", "filepath_type"))
-
-            return filepath
-
     @property
     def status(self):
         """The status of the prep template

qiita_db/metadata_template/test/test_prep_template.py

@@ -954,9 +954,9 @@ def _common_creation_checks(self, pt, fp_count, name):
         for s_id in exp_sample_ids:
             self.assertEqual(pt[s_id]._to_dict(), exp_dict[s_id])
 
-        # prep and qiime files have been created
+        # prep files have been created
         filepaths = pt.get_filepaths()
-        self.assertEqual(len(filepaths), 2)
+        self.assertEqual(len(filepaths), 1)
 
     def test_validate_restrictions(self):
         PT = qdb.metadata_template.prep_template.PrepTemplate
@@ -1019,28 +1019,9 @@ def test_generate_files(self):
         fp_count = qdb.util.get_count("qiita.filepath")
         self.tester.generate_files()
         obs = qdb.util.get_count("qiita.filepath")
-        # We just make sure that the count has been increased by 2, since
+        # We just make sure that the count has been increased by 1, since
         # the contents of the files have been tested elsewhere.
-        self.assertEqual(obs, fp_count + 2)
-
-    def test_create_qiime_mapping_file(self):
-        pt = qdb.metadata_template.prep_template.PrepTemplate(1)
-
-        # creating prep template file
-        _id, fp = qdb.util.get_mountpoint('templates')[0]
-
-        obs_fp = pt.create_qiime_mapping_file()
-        exp_fp = join(fp, '1_prep_1_qiime_19700101-000000.txt')
-
-        obs = pd.read_csv(obs_fp, sep='\t', infer_datetime_format=False,
-                          parse_dates=False, index_col=False, comment='\t')
-        exp = pd.read_csv(
-            exp_fp, sep='\t', infer_datetime_format=False,
-            parse_dates=False, index_col=False, comment='\t')
-        obs = obs.reindex(sorted(obs.columns), axis=1)
-        exp = exp.reindex(sorted(exp.columns), axis=1)
-
-        assert_frame_equal(obs, exp, check_like=True)
+        self.assertEqual(obs, fp_count + 1)
 
     def test_create_data_type_id(self):
         """Creates a new PrepTemplate passing the data_type_id"""
@@ -1119,9 +1100,9 @@ def test_create_warning(self):
         for s_id in exp_sample_ids:
             self.assertEqual(pt[s_id]._to_dict(), exp_dict[s_id])
 
-        # prep and qiime files have been created
+        # prep files have been created
         filepaths = pt.get_filepaths()
-        self.assertEqual(len(filepaths), 2)
+        self.assertEqual(len(filepaths), 1)
 
         # cleaning
         qdb.metadata_template.prep_template.PrepTemplate.delete(pt.id)

qiita_db/metadata_template/test/test_sample_template.py

@@ -1365,9 +1365,9 @@ def test_generate_files(self):
         fp_count = qdb.util.get_count("qiita.filepath")
         self.tester.generate_files()
         obs = qdb.util.get_count("qiita.filepath")
-        # We just make sure that the count has been increased by 6, since
+        # We just make sure that the count has been increased by 3, since
         # the contents of the files have been tested elsewhere.
-        self.assertEqual(obs, fp_count + 5)
+        self.assertEqual(obs, fp_count + 3)
 
     def test_to_file(self):
         """to file writes a tab delimited file with all the metadata"""