Implement extracting files using new endpoint

src/server/oasisapi/analyses/v2_api/viewsets.py

@@ -21,7 +21,7 @@
 from ...data_files.v2_api.serializers import DataFileSerializer
 from ...decorators import requires_sql_reader
 from ...files.v2_api.serializers import RelatedFileSerializer, FileSQLSerializer, NestedRelatedFileSerializer
-from ...files.v1_api.views import handle_related_file, handle_json_data, handle_related_file_sql
+from ...files.v1_api.views import handle_related_file, handle_json_data, handle_related_file_sql, handle_get_related_file_tar
 from ...filters import TimeStampedFilter, CsvMultipleChoiceFilter, CsvModelMultipleChoiceFilter
 from ...permissions.group_auth import VerifyGroupAccessModelViewSet, verify_user_is_in_obj_groups
 from ...portfolios.models import Portfolio
@@ -31,7 +31,7 @@
     RUN_MODE_PARAM,
     SUBTASK_STATUS_PARAM,
     SUBTASK_SLUG_PARAM,
-    LIST_FILE_PARAM,
+    FILENAME_PARAM,
 )
 
 
@@ -421,14 +421,27 @@ def input_file(self, request, pk=None, version=None):
         return handle_related_file(self.get_object(), 'input_file', request, ['application/x-gzip', 'application/gzip', 'application/x-tar', 'application/tar'])
 
 
-    @swagger_auto_schema(methods=['get'], manual_parameters=[LIST_FILE_PARAM,])
+    @swagger_auto_schema(methods=['get'])
     @action(methods=['get'], detail=True)
     def list_input_files(self, request, pk=None, version=None):
         """
         get:
         List the files in `input_file`.
         """
-        return handle_related_file(self.get_object(), 'input_file', request,  ['application/x-gzip', 'application/gzip', 'application/x-tar', 'application/tar'])
+        return handle_get_related_file_tar(self.get_object(), 'input_file', request,  ['application/x-gzip', 'application/gzip', 'application/x-tar', 'application/tar'])
+
+
+    @swagger_auto_schema(methods=['get'], responses={200: FILE_RESPONSE}, manual_parameters=[FILENAME_PARAM])
+    @action(methods=['get'], detail=True)
+    def extract_input_file(self, request, pk=None, version=None):
+        """
+        get:
+        Gets the portfolios `input_file` contents
+
+        delete:
+        Disassociates the portfolios `input_file` contents
+        """
+        return handle_get_related_file_tar(self.get_object(), 'input_file', request, ['application/x-gzip', 'application/gzip', 'application/x-tar', 'application/tar'])
 
 
     @swagger_auto_schema(methods=['get'], responses={200: FILE_RESPONSE})
@@ -515,14 +528,14 @@ def output_file(self, request, pk=None, version=None):
         """
         return handle_related_file(self.get_object(), 'output_file', request, ['application/x-gzip', 'application/gzip', 'application/x-tar', 'application/tar'])
 
-    @swagger_auto_schema(methods=['get'], manual_parameters=[LIST_FILE_PARAM,])
+    @swagger_auto_schema(methods=['get'])
     @action(methods=['get'], detail=True)
     def list_output_files(self, request, pk=None, version=None):
         """
         get:
         List the files in `output_file`.
         """
-        return handle_related_file(self.get_object(), 'output_file', request,  ['application/x-gzip', 'application/gzip', 'application/x-tar', 'application/tar'])
+        return handle_get_related_file_tar(self.get_object(), 'output_file', request,  ['application/x-gzip', 'application/gzip', 'application/x-tar', 'application/tar'])
 
     @requires_sql_reader
     @swagger_auto_schema(methods=['get'], responses={200: NestedRelatedFileSerializer})

src/server/oasisapi/files/models.py

@@ -36,6 +36,13 @@ def list_tar_file(RelatedFile):
     return files
 
 
+def extract_file_from_tar(RelatedFile, fname):
+    if not RelatedFile:
+        return None 
+
+    tarf = tarfile.open(fileobj=BytesIO(RelatedFile.read()), mode='r')
+    return tarf.extractfile(fname)
+
 def random_file_name(instance, filename):
     if getattr(instance, "store_as_filename", False):
         return filename

src/server/oasisapi/files/v1_api/views.py

@@ -1,5 +1,6 @@
 import json
 import io
+import os
 from tempfile import TemporaryFile
 
 from django.conf import settings
@@ -10,7 +11,7 @@
 
 from oasis_data_manager.df_reader.config import get_df_reader
 from oasis_data_manager.df_reader.exceptions import InvalidSQLException
-from ..models import RelatedFile, list_tar_file
+from ..models import RelatedFile, list_tar_file, extract_file_from_tar
 from .serializers import RelatedFileSerializer, EXPOSURE_ARGS
 from ...permissions.group_auth import verify_user_is_in_obj_groups
 
@@ -49,8 +50,6 @@ def _handle_get_related_file(parent, field, request):
     verify_user_is_in_obj_groups(request.user, f, 'You do not have permission to read this file')
     file_format = request.GET.get('file_format', None)
 
-    list_files = request.query_params.get('file_mode', False)
-
     if 'converted' in request.GET:
         if not (f.converted_file and f.conversion_state == RelatedFile.ConversionState.DONE):
             raise Http404()
@@ -89,11 +88,6 @@ def _handle_get_related_file(parent, field, request):
         response['Content-Disposition'] = 'attachment; filename="{}{}"'.format(download_name, '.csv')
         return response
 
-    if list_files: 
-        files = list_tar_file(f)
-
-        # todo: change this to a proper response
-        return Response(files)
 
     # Original Fallback method - Reutrn data 'as is'
     response = StreamingHttpResponse(_get_chunked_content(file_obj), content_type=f.content_type)
@@ -189,6 +183,31 @@ def handle_json_data(parent, field, request, serializer):
         return _handle_delete_related_file(parent, field, request)
 
 
+def handle_get_related_file_tar(parent, field, request, content_types):
+    f = getattr(parent, field)
+    if not f:
+        raise Http404()
+
+    verify_user_is_in_obj_groups(request.user, f, 'You do not have permission to read this file')
+
+    if 'list' in request.path: 
+        files = list_tar_file(f)
+
+        # todo: change this to a proper response
+        return Response(files)
+    elif 'extract' in request.path:
+        filename = request.GET.get('filename', None)
+        if not filename:
+            raise Http404
+        output_buffer = extract_file_from_tar(f, filename)
+        output_buffer.seek(0)
+
+        content_type='text/csv' # todo: find content_type
+        response = StreamingHttpResponse(output_buffer, content_type=content_type)
+        response['Content-Disposition'] = f'attachment; filename="{os.path.basename(filename)}"'
+        return response
+
+
 def handle_related_file_sql(parent, field, request, sql, m2m_file_pk=None):
     requested_format = request.GET.get('file_format', None)
     f = getattr(parent, field)

src/server/oasisapi/schemas/custom_swagger.py

@@ -7,7 +7,7 @@
     'SUBTASK_STATUS_PARAM',
     'SUBTASK_SLUG_PARAM',
     'FILE_VALIDATION_PARAM',
-    'LIST_FILE_PARAM', 
+    'FILENAME_PARAM', 
 ]
 
 from drf_yasg import openapi
@@ -111,10 +111,10 @@
     type=openapi.TYPE_BOOLEAN,
 )
 
-LIST_FILE_PARAM = openapi.Parameter(
-    'file_mode', 
+FILENAME_PARAM = openapi.Parameter(
+    'filename', 
     openapi.IN_QUERY, 
-    required=False, 
-    description="List file, if directory or tar, list contents.", 
-    type=openapi.TYPE_BOOLEAN, 
+    required=True, 
+    description="Filename to extract from tarfile.", 
+    type=openapi.TYPE_STRING, 
 )