MEDUSA_URL_PREFIX, and exception handling + logging fixes when running in multiprocessing

django_medusa/log.py

@@ -0,0 +1,48 @@
+from django.conf import settings
+import logging
+
+
+class ProxyLogHandler(logging.Handler):
+    def __init__(self, logger):
+        logging.Handler.__init__(self)
+        self.__logger = logger
+
+    def emit(self, record):
+        self.__logger.handle(record)
+
+listener = None
+
+def get_logger():
+    if not settings.MEDUSA_MULTITHREAD:
+        return get_base_logger()
+
+    from logutils.queue import QueueHandler, QueueListener
+    from multiprocessing import Queue
+
+    mplogger = logging.getLogger(__name__ + '.__multiprocessing__')
+    if not getattr(mplogger, 'setup_done', False):
+        base = get_base_logger()
+        logqueue = Queue()
+
+        mplogger.setLevel(logging.DEBUG)
+        mplogger.addHandler(QueueHandler(logqueue))
+        mplogger.setup_done = True
+        mplogger.propagate = False
+
+        global listener
+        listener = QueueListener(logqueue, ProxyLogHandler(get_base_logger()))
+        listener.start()
+
+    return mplogger
+
+def finalize_logger():
+    global listener
+
+    if listener is not None:
+        mplogger = get_logger()
+        mplogger.setup_done = False
+        listener.stop()
+        listener = None
+
+def get_base_logger():
+    return logging.getLogger(__name__)

django_medusa/management/commands/staticsitegen.py

@@ -1,4 +1,6 @@
+from django.conf import settings
 from django.core.management.base import BaseCommand
+from django.core.urlresolvers import set_script_prefix
 from django_medusa.renderers import StaticSiteRenderer
 from django_medusa.utils import get_static_renderers
 
@@ -12,8 +14,18 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         StaticSiteRenderer.initialize_output()
 
-        for Renderer in get_static_renderers():
-            r = Renderer()
-            r.generate()
+        renderers = [Renderer() for Renderer in get_static_renderers()]
+        for renderer in renderers:
+            # memoize this first to avoid script prefix pollution
+            renderer.paths
+
+        # Set script prefix here
+        url_prefix = getattr(settings, 'MEDUSA_URL_PREFIX', None)
+        if url_prefix is not None:
+            set_script_prefix(url_prefix)
+
+        # And now generate stuff
+        for renderer in renderers:
+            renderer.generate()
 
         StaticSiteRenderer.finalize_output()

django_medusa/renderers/appengine.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
 from django.conf import settings
 from django.test.client import Client
+from ..log import get_logger
 from .base import BaseStaticSiteRenderer
 import os
 
@@ -10,55 +11,53 @@
     'htm', 'html', 'css', 'xml', 'json', 'js', 'yaml', 'txt'
 )
 
-
 # Unfortunately split out from the class at the moment to allow rendering with
 # several processes via `multiprocessing`.
 # TODO: re-implement within the class if possible?
-def _gae_render_path(args):
-    client, path, view = args
-    if not client:
-        client = Client()
-    if path:
-        DEPLOY_DIR = settings.MEDUSA_DEPLOY_DIR
-        realpath = path
-        if path.startswith("/"):
-            realpath = realpath[1:]
 
-        if path.endswith("/"):
-            needs_ext = True
-        else:
-            needs_ext = False
 
-        output_dir = os.path.abspath(os.path.join(
-            DEPLOY_DIR,
-            "deploy",
-            os.path.dirname(realpath)
-        ))
-        if not os.path.exists(output_dir):
-            os.makedirs(output_dir)
-        outpath = os.path.join(DEPLOY_DIR, "deploy", realpath)
+class GAEStaticSiteRenderer(BaseStaticSiteRenderer):
+    """
+    A variation of BaseStaticSiteRenderer that deploys directly to S3
+    rather than to the local filesystem.
+
+    Settings:
+      * GAE_APP_ID
+      * MEDUSA_DEPLOY_DIR
+    """
+    def render_path(self, path=None, view=None):
+        if not path:
+            return None
+
+        DEPLOY_DIR = settings.MEDUSA_DEPLOY_DIR
 
-        resp = client.get(path)
-        if resp.status_code != 200:
-            raise Exception
+        resp = self._render(path)
 
-        mimetype = resp['Content-Type'].split(";", 1)[0]
+        # Force get_outpath to always use index.html by passing text/html
+        # mimetype
+        rel_outpath = os.path.join("deploy",
+                                   self.get_outpath(path, 'text/html'))
+        outpath = os.path.join(self.DEPLOY_DIR, rel_outpath)
 
-        if needs_ext:
-            outpath += "index.html"
+        # Ensure the directories exist
+        try:
+            os.makedirs(os.path.dirname(outpath))
+        except OSError:
+            pass
+
+        self.logger.info("Saving file to %s", outpath)
 
-        print(outpath)
         with open(outpath, 'w') as f:
             f.write(resp.content)
 
-        rel_outpath = outpath.replace(
-            os.path.abspath(DEPLOY_DIR) + "/",
-            ""
-        )
-        if ((not needs_ext) and path.endswith(STANDARD_EXTENSIONS))\
-        or (mimetype == "text/html"):
-            # Either has obvious extension OR it's a regular HTML file.
+        mimetype = resp['Content-Type'].split(';', 1)[0]
+
+        if (mimetype == 'text/html' or
+            (not path.endswith('/') and
+             outpath.endswith(STANDARD_EXTENSIONS))):
+            # Either has obvious extension OR it's a regular HTML file
             return None
+
         return "# req since this url does not end in an extension and also\n"\
                "# has non-html mime: %s\n"\
                "- url: %s\n"\
@@ -68,22 +67,10 @@ def _gae_render_path(args):
                     mimetype, path, rel_outpath, rel_outpath, mimetype
                )
 
-
-class GAEStaticSiteRenderer(BaseStaticSiteRenderer):
-    """
-    A variation of BaseStaticSiteRenderer that deploys directly to S3
-    rather than to the local filesystem.
-
-    Settings:
-      * GAE_APP_ID
-      * MEDUSA_DEPLOY_DIR
-    """
-    def render_path(self, path=None, view=None):
-        return _gae_render_path((self.client, path, view))
-
     @classmethod
     def initialize_output(cls):
-        print("Initializing output directory with `app.yaml`.")
+        super(GAEStaticSiteRenderer, cls).initialize_output()
+        cls.logger.info("Initializing output directory with `app.yaml`")
 
         # Initialize the MEDUSA_DEPLOY_DIR with an `app.yaml` and `deploy`
         # directory which stores the static files on disk.
@@ -113,7 +100,7 @@ def initialize_output(cls):
 
     @classmethod
     def finalize_output(cls):
-        print("Finalizing `app.yaml`.")
+        cls.logger.info("Finalizing app.yaml")
 
         DEPLOY_DIR = settings.MEDUSA_DEPLOY_DIR
         app_yaml = os.path.abspath(os.path.join(
@@ -156,34 +143,17 @@ def finalize_output(cls):
         )
         app_yaml_f.close()
 
-        print("You should now be able to deploy this to Google App Engine")
-        print("by performing the following command:")
-        print("appcfg.py update %s" % os.path.abspath(DEPLOY_DIR))
+        cls.logger.info("You should now be able to deploy this to "
+                        "Google App Engine by performing the following "
+                        "command:\n"
+                        "appcfg.py update %s", os.path.abspath(DEPLOY_DIR))
+
+        super(GAEStaticSiteRenderer, cls).finalize_output()
 
     def generate(self):
         DEPLOY_DIR = settings.MEDUSA_DEPLOY_DIR
 
-        # Generate the site
-        if getattr(settings, "MEDUSA_MULTITHREAD", False):
-            # Upload up to ten items at once via `multiprocessing`.
-            from multiprocessing import Pool
-
-            print("Uploading with up to 10 upload processes...")
-            pool = Pool(10)
-
-            handlers = pool.map(
-                _gae_render_path,
-                ((None, path, None) for path in self.paths),
-                chunksize=5
-            )
-            pool.close()
-            pool.join()
-        else:
-            # Use standard, serial upload.
-            self.client = Client()
-            handlers = []
-            for path in self.paths:
-                handlers.append(self.render_path(path=path))
+        handlers = super(GAEStaticSiteRenderer, self).generate()
 
         DEPLOY_DIR = settings.MEDUSA_DEPLOY_DIR
         app_yaml = os.path.abspath(os.path.join(

django_medusa/renderers/base.py

@@ -1,3 +1,10 @@
+from __future__ import print_function
+from django.conf import settings
+from django.test.client import Client
+from django_medusa.log import get_logger, finalize_logger
+import mimetypes
+import os
+
 __all__ = ['COMMON_MIME_MAPS', 'BaseStaticSiteRenderer']
 
 
@@ -15,13 +22,20 @@
     "text/css": ".css",
 }
 
+class RenderError(Exception):
+    """
+    Exception thrown during a rendering error.
+    """
+    pass
 
 class BaseStaticSiteRenderer(object):
     """
     This default renderer writes the given URLs (defined in get_paths())
     into static files on the filesystem by getting the view's response
     through the Django testclient.
     """
+    def __init__(self):
+        self.client = None
 
     @classmethod
     def initialize_output(cls):
@@ -33,7 +47,9 @@ def initialize_output(cls):
         Management command calls this once before iterating over all
         renderer instances.
         """
-        pass
+        # Store logger on BaseStaticSiteRenderer so that all derivative classes
+        # can access this instance.
+        BaseStaticSiteRenderer.logger = get_logger()
 
     @classmethod
     def finalize_output(cls):
@@ -46,7 +62,8 @@ def finalize_output(cls):
         Management command calls this once after iterating over all
         renderer instances.
         """
-        pass
+        finalize_logger()
+        BaseStaticSiteRenderer.logger = None
 
     def get_paths(self):
         """ Override this in a subclass to define the URLs to process """
@@ -61,9 +78,81 @@ def paths(self):
             self._paths = p
         return p
 
+    def _render(self, path=None, view=None):
+        client = self.client
+
+        if not client:
+            client = Client()
+
+        response = client.get(path)
+        if response.status_code != 200:
+            raise RenderError(
+                "Path {0} did not return status 200".format(path))
+
+        return response
+
+    @classmethod
+    def get_outpath(cls, path, content_type):
+        # Get non-absolute path
+        path = path[1:] if path.startswith('/') else path
+
+        # Resolves to a file, not a directory
+        if not path.endswith('/'):
+            return path
+
+        return os.path.join(path, cls.get_dirsuffix(content_type))
+
+    @classmethod
+    def get_dirsuffix(cls, content_type):
+        mime = content_type.split(';', 1)[0]
+
+        return ('index' +
+                (COMMON_MIME_MAPS.get(mime, mimetypes.guess_extension(mime)) or
+                 '.html'))
+
     def render_path(self, path=None, view=None):
         raise NotImplementedError
 
     def generate(self):
-        for path in self.paths:
-            self.render_path(path)
+        arglist = ((path, None) for path in self.paths)
+
+        if getattr(settings, "MEDUSA_MULTITHREAD", False):
+            from multiprocessing import Pool, cpu_count, Queue
+
+            generator = PageGenerator(self)
+
+            self.logger.info("Generating with up to %s processes...",
+                             cpu_count())
+            pool = Pool(cpu_count())
+            retval = pool.map(generator, arglist, chunksize=1)
+            pool.close()
+
+        else:
+            self.client = Client()
+            generator = PageGenerator(self)
+
+            retval = map(generator, arglist)
+
+        return retval
+
+
+class PageGenerator(object):
+    """
+    Helper class to bounce things back into the renderer instance, since
+    multiprocessing is unable to transfer a bound method object into a pickle.
+    """
+    def __init__(self, renderer):
+        self.renderer = renderer
+
+    def __call__(self, args):
+        path = args[0]
+        logger = self.renderer.logger
+
+        try:
+            logger.info("Generating %s...", path)
+            retval = self.renderer.render_path(*args)
+            logger.info("Generated %s successfully", path)
+            return retval
+
+        except:
+            logger.error("Could not generate %s", path, exc_info=True)