Merge branch 'feature/unoserver-celery' into feature/buff-worms

 Closes #391
 [ENG-7610]

Author: felliott

docs/conf.py

@@ -30,7 +30,7 @@
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest']
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'stevedore.sphinxext']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']

docs/index.rst

@@ -26,6 +26,7 @@ Guide
    overview
    integrations
    code
+   plugins
 
 Project info
 ------------

docs/plugins.rst

@@ -0,0 +1,20 @@
+
+Plugins
+-------
+
+Auth Providers
+==============
+
+.. list-plugins:: mfr.providers
+   :detailed:
+
+Exporters
+=========
+
+.. list-plugins:: mfr.exporters
+
+Renderers
+=========
+
+.. list-plugins:: mfr.renderers
+

mfr/core/extension.py

@@ -1,6 +1,15 @@
 import abc
+import asyncio
+import time
+from dataclasses import dataclass, field
 
+from waterbutler.core.streams import StringStream
+from waterbutler.core.utils import make_provider
+
+from mfr.server import settings
 from mfr.core.metrics import MetricsRecord
+from mfr.core.provider import ProviderMetadata
+from mfr.tasks.serializer import serializable
 
 
 class BaseExporter(metaclass=abc.ABCMeta):
@@ -42,19 +51,28 @@ def _get_module_name(self):
             .replace('mfr.extensions.', '', 1) \
             .replace('.export', '', 1)
 
-
+@serializable
+@dataclass
 class BaseRenderer(metaclass=abc.ABCMeta):
-
-    def __init__(self, metadata, file_path, url, assets_url, export_url):
-        self.metadata = metadata
-        self.file_path = file_path
-        self.url = url
-        self.assets_url = f'{assets_url}/{self._get_module_name()}'
-        self.export_url = export_url
-        self.renderer_metrics = MetricsRecord('renderer')
+    metadata: ProviderMetadata
+    file_path: str
+    url: str
+    assets_url: str
+    export_url: str
+    renderer_metrics: MetricsRecord = field(default=None)
+    metrics: MetricsRecord = field(default=None)
+
+    def __post_init__(self):
+        self.assets_url = f'{self.assets_url}/{self._get_module_name()}'
+        self.renderer_metrics = MetricsRecord('renderer',)
         if self._get_module_name():
             self.metrics = self.renderer_metrics.new_subrecord(self._get_module_name())
 
+        if name := self.metadata.name:
+            self.cache_file_path_str = f'/export/{self.metadata.unique_key}.{name}'
+        else:
+            self.cache_file_path_str = f'/export/{self.metadata.unique_key}'
+
         self.renderer_metrics.merge({
             'class': self._get_module_name(),
             'ext': self.metadata.ext,
@@ -73,10 +91,55 @@ def __init__(self, metadata, file_path, url, assets_url, export_url):
         except AttributeError:
             pass
 
+    @property
+    def cache_provider(self):
+        return make_provider(
+            settings.CACHE_PROVIDER_NAME,
+            {},  # User information which can be left blank
+            settings.CACHE_PROVIDER_CREDENTIALS,
+            settings.CACHE_PROVIDER_SETTINGS
+        )
+
+    async def get_cache_file_path(self):
+        return await self.cache_provider.validate_path(self.cache_file_path_str)
+
     @abc.abstractmethod
-    def render(self):
+    def _render(self) -> str:
         pass
 
+    async def render(self):
+        if self.use_celery or self.cache_result:
+            self.cache_file_path = await self.cache_provider.validate_path(self.cache_file_path_str)
+        if not self.use_celery:
+            rendition = await self.do_render()
+            return StringStream(rendition)
+        else:
+            from mfr.tasks.render import render
+            result = render.delay(self)
+            for i in range(100 * 60 * 10):
+                if not result.ready():
+                    time.sleep(0.01)
+                else:
+                    return await self.cache_provider.download(self.cache_file_path)
+
+            return None
+
+    async def do_render(self):
+        if self.use_celery or self.cache_result:
+            file_path_task = asyncio.ensure_future(self.get_cache_file_path())
+        rendition = await asyncio.get_running_loop().run_in_executor(None, self._render)
+        if self.use_celery or self.cache_result:
+            upload_task = asyncio.ensure_future(
+                self.cache_provider.upload(
+                    StringStream(rendition),
+                    await file_path_task
+                )
+            )
+            if self.use_celery:
+                await upload_task
+        return rendition
+
+    @property
     @abc.abstractmethod
     def file_required(self):
         """Does the rendering html need the raw file content to display correctly?
@@ -85,10 +148,15 @@ def file_required(self):
         """
         pass
 
+    @property
     @abc.abstractmethod
-    def cache_result(self):
+    def cache_result(self) -> bool:
         pass
 
+    @property
+    def use_celery(self) -> bool:
+        return False
+
     def _get_module_name(self):
         return self.__module__ \
             .replace('mfr.extensions.', '', 1) \

mfr/core/metrics.py

@@ -1,4 +1,7 @@
 import copy
+from dataclasses import dataclass, field
+
+from mfr.tasks.serializer import serializable
 
 
 def _merge_dicts(a, b, path=None):
@@ -93,16 +96,17 @@ def _set_dotted_key(store, key, value):
             current = current[part]
         current[parts[-1]] = value
 
-
+@serializable
+@dataclass
 class MetricsRecord(MetricsBase):
     """An extension to MetricsBase that carries a category and list of submetrics.  When
     serialized, will include the serialized child metrics
     """
+    category: str
+    subrecords: list = field(default_factory=list)
 
-    def __init__(self, category):
+    def __post_init__(self):
         super().__init__()
-        self.category = category
-        self.subrecords = []
 
     @property
     def key(self):
@@ -121,20 +125,23 @@ def serialize(self):
     def new_subrecord(self, name):
         """Create a new MetricsSubRecord object with our category and save it to the subrecords
         list."""
-        subrecord = MetricsSubRecord(self.category, name)
+        subrecord = MetricsSubRecord(category=self.category, name=name)
         self.subrecords.append(subrecord)
         return subrecord
 
-
+@serializable
+@dataclass
 class MetricsSubRecord(MetricsRecord):
     """An extension to MetricsRecord that carries a name in addition to a category.  Will identify
     itself as {category}_{name}.  Can create its own subrecord whose category will be this
     subrecord's ``name``.
     """
+    name: str = field(default=None)
 
-    def __init__(self, category, name):
-        super().__init__(category)
-        self.name = name
+    def __post_init__(self):
+        super().__post_init__()
+        if self.name is None:
+            raise TypeError('name must be provided')
 
     @property
     def key(self):
@@ -153,6 +160,6 @@ def new_subrecord(self, name):
             print(child.key)       # foo_bar
             print(grandchild.key)  # bar_baz
         """
-        subrecord = MetricsSubRecord(self.name, name)
+        subrecord = MetricsSubRecord(category=self.name, name=name)
         self.subrecords.append(subrecord)
         return subrecord

mfr/core/provider.py

@@ -1,11 +1,14 @@
 import abc
+from dataclasses import dataclass
+
 import markupsafe
 
 import furl
 
 from mfr.core import exceptions
 from mfr.server import settings
 from mfr.core.metrics import MetricsRecord
+from mfr.tasks.serializer import serializable
 
 
 class BaseProvider(metaclass=abc.ABCMeta):
@@ -47,16 +50,15 @@ def metadata(self):
     def download(self):
         pass
 
-
+@serializable
+@dataclass
 class ProviderMetadata:
-
-    def __init__(self, name, ext, content_type, unique_key, download_url, stable_id=None):
-        self.name = name
-        self.ext = ext
-        self.content_type = content_type
-        self.unique_key = unique_key
-        self.download_url = download_url
-        self.stable_id = stable_id
+    name: str
+    ext: str
+    content_type: str
+    unique_key: str
+    download_url: str
+    stable_id: str = None
 
     def serialize(self):
         return {

mfr/core/utils.py

@@ -100,7 +100,13 @@ def make_renderer(name, metadata, file_path, url, assets_url, export_url):
             namespace='mfr.renderers',
             name=normalized_name,
             invoke_on_load=True,
-            invoke_args=(metadata, file_path, url, assets_url, export_url),
+            invoke_kwds={
+                'metadata': metadata,
+                'file_path': file_path,
+                'url': url,
+                'assets_url': assets_url,
+                'export_url': export_url
+            },
         ).driver
     except RuntimeError:
         raise exceptions.MakeRendererError(

mfr/extensions/audio/render.py

@@ -13,7 +13,7 @@ class AudioRenderer(extension.BaseRenderer):
             os.path.join(os.path.dirname(__file__), 'templates')
         ]).get_template('viewer.mako')
 
-    def render(self):
+    def _render(self):
         safe_url = escape_url_for_template(self.url)
         return self.TEMPLATE.render(base=self.assets_url, url=safe_url)
 

mfr/extensions/codepygments/render.py

@@ -30,7 +30,7 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.metrics.add('pygments_version', pygments.__version__)
 
-    def render(self):
+    def _render(self):
         file_size = os.path.getsize(self.file_path)
         if file_size > settings.MAX_SIZE:
             raise exceptions.FileTooLargeError(

mfr/extensions/image/render.py

@@ -15,7 +15,7 @@ class ImageRenderer(extension.BaseRenderer):
             os.path.join(os.path.dirname(__file__), 'templates')
         ]).get_template('viewer.mako')
 
-    def render(self):
+    def _render(self):
         self.metrics.add('needs_export', False)
         if self.metadata.ext in settings.EXPORT_EXCLUSIONS:
             download_url = munge_url_for_localdev(self.url)