Merge pull request #3159 from antgonza/fix-Overflow-download-analysis-metadata

fix Overflow in download analysis metadata

Author: wasade

qiita_db/handlers/analysis.py

@@ -6,7 +6,9 @@
 # The full license is in the file LICENSE, distributed with this software.
 # -----------------------------------------------------------------------------
 
+from tornado import gen
 from tornado.web import HTTPError
+from json import dumps
 
 import qiita_db as qdb
 from .oauth2 import OauthBaseHandler, authenticate_oauth
@@ -44,7 +46,7 @@ def _get_analysis(a_id):
 
 class APIAnalysisMetadataHandler(OauthBaseHandler):
     @authenticate_oauth
-    def get(self, analysis_id):
+    async def get(self, analysis_id):
         """Retrieves the analysis metadata
 
         Parameters
@@ -56,15 +58,37 @@ def get(self, analysis_id):
         -------
         dict
             The contents of the analysis keyed by sample id
+
+        Notes
+        -----
+        This response needed to be broken in chunks because we were hitting
+        the max size of a respose: 2G; based on: https://bit.ly/3CPvyjd
         """
+        chunk_len = 1024 * 1024 * 1  # 1 MiB
+
+        response = None
         with qdb.sql_connection.TRN:
             a = _get_analysis(analysis_id)
             mf_fp = qdb.util.get_filepath_information(
                 a.mapping_file)['fullpath']
-            response = None
             if mf_fp is not None:
                 df = qdb.metadata_template.util.load_template_to_dataframe(
                     mf_fp, index='#SampleID')
-                response = df.to_dict(orient='index')
+                response = dumps(df.to_dict(orient='index'))
+
+        if response is not None:
+            crange = range(chunk_len, len(response)+chunk_len, chunk_len)
+            for i, (win) in enumerate(crange):
+                # sending the chunk and flushing
+                chunk = response[i*chunk_len:win]
+                self.write(chunk)
+                await self.flush()
+
+                # cleaning chuck and pause the coroutine so other handlers
+                # can run, note that this is required/important based on the
+                # original implementation in https://bit.ly/3CPvyjd
+                del chunk
+                await gen.sleep(0.000000001)  # 1 nanosecond
 
-        self.write(response)
+        else:
+            self.write(None)

scripts/qiita-test-install

@@ -232,11 +232,11 @@ class QiitaConfig(TestCase):
                            string_acceptable_version))
 
     def test_redbiom_version(self):
-        acceptable_version = (0, 3, 5)
+        acceptable_version = (0, 3, 8)
         string_acceptable_version = '.'.join(map(str, acceptable_version))
         version = tuple(map(int, redbiom_lib_version.split('.')))
 
-        self.assertTrue(acceptable_version == version,
+        self.assertTrue(acceptable_version >= version,
                         'Unsupported redbiom version. You have %s but the '
                         'minimum required version is %s'
                         % ('.'.join(map(str, version)),

setup.py

@@ -105,7 +105,7 @@
       install_requires=['psycopg2', 'click', 'bcrypt', 'pandas',
                         'biom-format', 'tornado<6.0', 'toredis', 'redis',
                         'scp', 'pyparsing', 'h5py',  'natsort', 'nose', 'pep8',
-                        'networkx', 'humanize', 'scikit-bio', 'wtforms',
+                        'networkx', 'humanize', 'scikit-bio', 'wtforms<3.0.0',
                         'openpyxl', 'sphinx-bootstrap-theme', 'Sphinx', 'nltk',
                         'gitpython', 'redbiom', 'pyzmq', 'sphinx_rtd_theme',
                         'paramiko', 'seaborn',  'matplotlib', 'scipy', 'nose',