Dev updates 2021 (#3063)

* rm key

* use test_ssh_key

* flake8

* add chmod

* ssh_key -> ssh_key_fp

* fix #3057

* adding JS for download

* minor improvements

* addressing @ElDeveloper comment

* fix #3049

* fix #3060

* flake8

* addressing @ElDeveloper comments

* Delete 81.sql

* No

* Yes

* No

* Yes

* Replace qiita-spots to qiita

Co-authored-by: Antonio Gonzalez <[email protected]>

Author: ElDeveloper

README.rst

@@ -19,12 +19,12 @@ typically limiting for researchers studying microbial ecology.
 
 Qiita is currently in beta status. We are very open to community
 contributions and feedback. If you're interested in contributing to Qiita,
-see `CONTRIBUTING.md <https://github.com/biocore/qiita/blob/master/CONTRIBUTING.md>`__.
+see `CONTRIBUTING.md <https://github.com/qiita-spots/qiita/blob/master/CONTRIBUTING.md>`__.
 If you'd like to report bugs or request features, you can do that in the
-`Qiita issue tracker <https://github.com/biocore/qiita/issues>`__.
+`Qiita issue tracker <https://github.com/qiita-spots/qiita/issues>`__.
 
 To install and configure your own Qiita server, see
-`INSTALL.md <https://github.com/biocore/qiita/blob/master/INSTALL.md>`__.
+`INSTALL.md <https://github.com/qiita-spots/qiita/blob/master/INSTALL.md>`__.
 
 For more specific details about qiita visit `the Qiita main site tutorial <https://qiita.microbio.me/static/doc/html/qiita-philosophy/index.html>`__.
 
@@ -71,9 +71,7 @@ future.
   existing studies by submitting a fix proposals to the authors of the study.
 
 
-.. |Build Status| image:: https://travis-ci.org/biocore/qiita.png?branch=master
-   :target: https://travis-ci.org/biocore/qiita
-.. |Coverage Status| image:: https://codecov.io/gh/biocore/qiita/branch/master/graph/badge.svg
-   :target: https://codecov.io/gh/biocore/qiita
-.. |Gitter| image:: https://badges.gitter.im/Join%20Chat.svg
-   :target: https://gitter.im/biocore/qiita?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+.. |Build Status| image:: https://travis-ci.org/qiita-spots/qiita.png?branch=master
+   :target: https://travis-ci.org/qiita-spots/qiita
+.. |Coverage Status| image:: https://codecov.io/gh/qiita-spots/qiita/branch/master/graph/badge.svg
+   :target: https://codecov.io/gh/qiita-spots/qiita

qiita_db/metadata_template/base_metadata_template.py

@@ -558,20 +558,20 @@ def _clean_validate_template(cls, md_template, study_id,
         error = []
         if pgsql_reserved:
             error.append(
-                "The following column names in the template contain PgSQL "
-                "reserved words: %s." % ", ".join(pgsql_reserved))
+                "These column names are PgSQL reserved words, replace them: "
+                "~~ %s ~~." % ", ".join(pgsql_reserved))
         if invalid:
             error.append(
-                "The following column names in the template contain invalid "
-                "chars: %s." % ", ".join(invalid))
+                "These column names contain invalid chars, remove or replace "
+                "them: ~~ %s ~~." % ", ".join(invalid))
         if forbidden:
             error.append(
-                "The following column names in the template contain invalid "
-                "values: %s." % ", ".join(forbidden))
+                "These column names are not valid in this information file, "
+                "remove them: ~~ %s ~~." % ", ".join(forbidden))
         if qiime2_reserved:
             error.append(
-                "The following column names in the template contain QIIME2 "
-                "reserved words: %s." % ", ".join(pgsql_reserved))
+                "These columns are QIIME2 reserved words, replace them: "
+                " ~~ %s ~~." % ", ".join(pgsql_reserved))
 
         if error:
             raise qdb.exceptions.QiitaDBColumnError(
@@ -1160,13 +1160,15 @@ def to_file(self, fp, samples=None):
             df.to_csv(fp, index_label='sample_name', na_rep="", sep='\t',
                       encoding='utf-8')
 
-    def _common_to_dataframe_steps(self):
+    def _common_to_dataframe_steps(self, samples=None):
         """Perform the common to_dataframe steps
 
         Returns
         -------
         pandas DataFrame
             The metadata in the template,indexed on sample id
+        samples list of string, optional
+            A list of the sample names we actually want to retrieve
         """
         with qdb.sql_connection.TRN:
             # Retrieve all the information from the database
@@ -1175,7 +1177,12 @@ def _common_to_dataframe_steps(self):
                      FROM qiita.{0}
                      WHERE sample_id != '{1}'""".format(
                         self._table_name(self._id), QIITA_COLUMN_NAME)
-            qdb.sql_connection.TRN.add(sql)
+            if samples is None:
+                qdb.sql_connection.TRN.add(sql)
+            else:
+                sql += ' AND sample_id IN %s'
+                qdb.sql_connection.TRN.add(sql, [tuple(samples)])
+
             # this query is going to return a tuple
             # (sample_id, dict of columns/values); however it's important to
             # notice that we can't assure that all column/values pairs are the

qiita_db/metadata_template/constants.py

@@ -17,6 +17,7 @@
     'EBI': Restriction(columns={'collection_timestamp': datetime,
                                 'physical_specimen_location': str,
                                 'taxon_id': int,
+                                'description': str,
                                 'scientific_name': str},
                        error_msg="EBI submission disabled"),
     # The following columns are required for the official main QIITA site

qiita_db/metadata_template/sample_template.py

@@ -333,15 +333,17 @@ def biosample_accessions(self, value):
         """
         self._update_accession_numbers('biosample_accession', value)
 
-    def to_dataframe(self, add_ebi_accessions=False):
+    def to_dataframe(self, add_ebi_accessions=False, samples=None):
         """Returns the metadata template as a dataframe
 
         Parameters
         ----------
         add_ebi_accessions : bool, optional
             If this should add the ebi accessions
+        samples list of string, optional
+            A list of the sample names we actually want to retrieve
         """
-        df = self._common_to_dataframe_steps()
+        df = self._common_to_dataframe_steps(samples=samples)
 
         if add_ebi_accessions:
             accessions = self.ebi_sample_accessions

qiita_db/metadata_template/test/test_sample_template.py

@@ -1963,8 +1963,7 @@ def test_to_dataframe(self):
                '1.SKM4.640180', '1.SKM5.640177', '1.SKM6.640187',
                '1.SKM7.640188', '1.SKM8.640201', '1.SKM9.640192'}
         self.assertEqual(set(obs.index), exp)
-
-        self.assertEqual(set(obs.columns), {
+        exp_columns = {
             'physical_specimen_location', 'physical_specimen_remaining',
             'dna_extracted', 'sample_type', 'collection_timestamp',
             'host_subject_id', 'description', 'latitude', 'longitude',
@@ -1973,7 +1972,15 @@ def test_to_dataframe(self):
             'water_content_soil', 'elevation', 'temp', 'tot_nitro',
             'samp_salinity', 'altitude', 'env_biome', 'country', 'ph',
             'anonymized_name', 'tot_org_carb', 'description_duplicate',
-            'env_feature', 'scientific_name', 'qiita_study_id'})
+            'env_feature', 'scientific_name', 'qiita_study_id'}
+        self.assertEqual(set(obs.columns), exp_columns)
+
+        # test limiting samples produced
+        exp_samples = set(['1.SKD4.640185', '1.SKD5.640186'])
+        obs = self.tester.to_dataframe(samples=exp_samples)
+        self.assertEqual(len(obs), 2)
+        self.assertEqual(set(obs.index), exp_samples)
+        self.assertEqual(set(obs.columns), exp_columns)
 
         # test with add_ebi_accessions as True
         obs = self.tester.to_dataframe(True)

qiita_db/processing_job.py

@@ -1763,8 +1763,8 @@ def shape(self):
         if self.command.name == 'Validate':
             # Validate only has two options to calculate it's size: template (a
             # job that has a preparation linked) or analysis (is from an
-            # analysis).
-            if 'template' in parameters:
+            # analysis). However, 'template' can be present and be None
+            if 'template' in parameters and parameters['template'] is not None:
                 try:
                     pt = qdb.metadata_template.prep_template.PrepTemplate(
                         parameters['template'])

qiita_pet/handlers/artifact_handlers/base_handlers.py

@@ -160,7 +160,7 @@ def artifact_summary_get_request(user, artifact_id):
             buttons.append(btn_base % ('revert to sandbox', 'sandbox',
                                        'Revert to sandbox'))
 
-        if user.level == 'admin':
+        if user.level == 'admin' and not study.autoloaded:
             if artifact.can_be_submitted_to_ebi:
                 buttons.append(
                     '<a class="btn btn-primary btn-sm" '

qiita_pet/handlers/artifact_handlers/tests/test_base_handlers.py

@@ -257,6 +257,22 @@ def test_artifact_summary_get_request(self):
                'errored_summary_jobs': []}
         self.assertEqual(obs, exp)
 
+        # the buttons shouldn't be present when the study is autoloaded
+        study = a.study
+        study.autoloaded = True
+        exp['buttons'] = ('<button onclick="if (confirm(\'Are you sure you '
+                          'want to make public artifact id: 2?\')) { '
+                          'set_artifact_visibility(\'public\', 2) }" '
+                          'class="btn btn-primary btn-sm">Make public'
+                          '</button> <button onclick="if (confirm(\'Are you '
+                          'sure you want to revert to sandbox artifact id: '
+                          '2?\')) { set_artifact_visibility(\'sandbox\', 2) '
+                          '}" class="btn btn-primary btn-sm">Revert to '
+                          'sandbox</button> ' + private_download_button % 2)
+        obs = artifact_summary_get_request(User('[email protected]'), 2)
+        self.assertEqual(obs, exp)
+        study.autoloaded = False
+
         # analysis artifact
         obs = artifact_summary_get_request(user, 8)
         exp = {'name': 'noname',

qiita_pet/handlers/download.py

@@ -46,10 +46,7 @@ def _check_permissions(self, sid):
                 study_info['message'], self.current_user.email, sid))
         return Study(sid)
 
-    def _generate_files(self, header_name, accessions, filename):
-        text = "sample_name\t%s\n%s" % (header_name, '\n'.join(
-            ["%s\t%s" % (k, v) for k, v in accessions.items()]))
-
+    def _finish_generate_files(self, filename, text):
         self.set_header('Content-Description', 'text/csv')
         self.set_header('Expires', '0')
         self.set_header('Cache-Control', 'no-cache')
@@ -58,6 +55,12 @@ def _generate_files(self, header_name, accessions, filename):
         self.write(text)
         self.finish()
 
+    def _generate_files(self, header_name, accessions, filename):
+        text = "sample_name\t%s\n%s" % (header_name, '\n'.join(
+            ["%s\t%s" % (k, v) for k, v in accessions.items()]))
+
+        self._finish_generate_files(filename, text)
+
     def _list_dir_files_nginx(self, dirpath):
         """Generates a nginx list of files in the given dirpath for nginx
 
@@ -327,6 +330,25 @@ def get(self, prep_template_id):
             'ebi_experiment_accessions_study_%s_prep_%s.tsv' % (sid, pid))
 
 
+class DownloadSampleInfoPerPrep(BaseHandlerDownload):
+    @authenticated
+    @coroutine
+    @execute_as_transaction
+    def get(self, prep_template_id):
+        pid = int(prep_template_id)
+        pt = PrepTemplate(pid)
+        sid = pt.study_id
+
+        self._check_permissions(sid)
+
+        st = SampleTemplate(sid)
+
+        text = st.to_dataframe(samples=list(pt)).to_csv(None, sep='\t')
+
+        self._finish_generate_files(
+            'sample_information_from_prep_%s.tsv' % pid, text)
+
+
 class DownloadUpload(BaseHandlerDownload):
     @authenticated
     @coroutine

qiita_pet/handlers/upload.py

@@ -9,7 +9,7 @@
 from tornado.web import authenticated, HTTPError
 
 from os.path import join, exists
-from os import remove
+from os import remove, chmod
 from json import loads, dumps
 
 from collections import defaultdict
@@ -142,6 +142,7 @@ def post(self, study_id):
 
         with open(ssh_key_fp, 'wb') as f:
             f.write(ssh_key)
+        chmod(ssh_key_fp, 0o600)
 
         qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
         if method == 'list':