Integrate dockerfile_lint

README.md

@@ -65,6 +65,10 @@ in Firewall. For Fedora 21 this can be done so:
     firewall-cmd --permanent --add-port=8000/tcp
     ./manage.py runserver 0.0.0.0:8000
 
+For dockerfile_lint support, you'll need to install it:
+
+    sudo npm install -g git+https://github.com/redhataccess/dockerfile_lint
+
 Manipulate with the data
 ------------------------
 

dbs/admin.py

@@ -1,8 +1,9 @@
 from __future__ import absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement
 
 from django.contrib import admin
-from .models import TaskData, Task, Rpm, Registry, YumRepo, Image, ImageRegistryRelation
+from .models import TaskData, TaskLint, Task, Rpm, Registry, YumRepo, Image, ImageRegistryRelation
 
+admin.site.register(TaskLint)
 admin.site.register(TaskData)
 admin.site.register(Task)
 admin.site.register(Rpm)

dbs/api/core.py

@@ -8,7 +8,7 @@
 from django.core.exceptions import ObjectDoesNotExist
 from django.shortcuts import get_object_or_404
 
-from dbs.models import Task, TaskData, Dockerfile, Image
+from dbs.models import Task, TaskData, TaskLint, Dockerfile, Image
 from dbs.task_api import TaskApi
 from dbs.utils import chain_dict_get
 
@@ -21,6 +21,13 @@ class ErrorDuringRequest(Exception):
     """ indicate that there was an error during processing request; e.g. 404, invalid sth... """
 
 
+def lint_output_callback(task_id, lint, **kwargs):
+    t = Task.objects.get(id=task_id)
+    tl = TaskLint(lint=lint["html_markup"])
+    tl.save()
+    t.task_lint = tl
+    t.save()
+
 def new_image_callback(task_id, response_tuple):
     try:
         response_hash, df, build_log = response_tuple
@@ -59,6 +66,7 @@ def new_image_callback(task_id, response_tuple):
 
         t.status = Task.STATUS_SUCCESS
     else:
+        logger.debug("task failed: %s" % repr (response_tuple))
         t.status = Task.STATUS_FAILED
     t.save()
 
@@ -76,10 +84,12 @@ def build(post_args, **kwargs):
              type=Task.TYPE_BUILD, owner=owner, task_data=td)
     t.save()
 
+    lint_callback = partial(lint_output_callback, t.id)
     callback = partial(new_image_callback, t.id)
 
     post_args.update({'build_image': "buildroot-fedora", 'local_tag': local_tag,
-                 'callback': callback})
+                      'callback': callback,
+                      'lint_callback': lint_callback})
     task_id = builder_api.build_docker_image(**post_args)
     t.celery_id = task_id
     t.save()

dbs/lint.py

@@ -0,0 +1,171 @@
+from __future__ import absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement
+
+import git
+import shutil
+import subprocess
+import tempfile
+import os
+import json
+
+def html_escape(s):
+    # In python3 we can use html.escape(s, quote=False)
+    return s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
+
+class DockerfileLint:
+    rules = "/usr/lib/node_modules/dockerfile_lint/sample_rules.yaml"
+
+    PF_CLASSES = { 'error':
+                   { 'alert': '<div class="alert alert-danger">',
+                     'icon': """
+  <span class="pficon-layered">
+    <span class="pficon pficon-error-octagon"></span>
+    <span class="pficon pficon-error-exclamation"></span>
+  </span>""" },
+
+                   'warn':
+                   { 'alert': '<div class="alert alert-warning">',
+                     'icon': """
+  <span class="pficon-layered">
+    <span class="pficon pficon-warning-triangle"></span>
+    <span class="pficon pficon-warning-exclamation"></span>
+  </span>""" },
+
+                   'info':
+                   { 'alert': '<div class="alert alert-info">',
+                     'icon': """
+  <span class="pficon pficon-info"></span>"""}
+                   }
+
+    def __init__ (self, git_url, git_path=None, git_commit=None):
+        self._git_url = git_url
+        self._git_path = git_path
+        self._git_commit = git_commit
+        self._temp_dir = None
+
+    def __del__ (self):
+        if self._temp_dir:
+            try:
+                shutil.rmtree(self._temp_dir)
+            except (IOError, OSError, AttributeError) as exc:
+                pass
+
+    def _get_dockerfile (self):
+        self._temp_dir = tempfile.mkdtemp ()
+        git.Repo.clone_from (self._git_url, self._temp_dir)
+        if self._git_path:
+            if self._git_path.endswith('Dockerfile'):
+                git_df_dir = os.path.dirname(self._git_path)
+                df_path = os.path.abspath(os.path.join(self._temp_dir,
+                                                       git_df_dir))
+            else:
+                df_path = os.path.abspath(os.path.join(self._temp_dir,
+                                                       self._git_path))
+        else:
+            df_path = self._temp_dir
+
+        self._Dockerfile = os.path.join(df_path, "Dockerfile")
+
+    def _run_dockerfile_lint (self):
+        with open ("/dev/null", "rw") as devnull:
+            dfl = subprocess.Popen (["dockerfile_lint",
+                                     "-j",
+                                     "-r", self.rules,
+                                     "-f", self._Dockerfile],
+                                    stdin=devnull,
+                                    stdout=subprocess.PIPE,
+                                    stderr=devnull,
+                                    close_fds=True)
+
+        (stdout, stderr) = dfl.communicate ()
+        jsonstr = stdout.decode ()
+        self._lint = json.loads (jsonstr)
+
+    def _mark_dockerfile (self):
+        out = ""
+        with open(self._Dockerfile, "r") as df:
+            dflines = df.readlines ()
+            dflines.append ("\n") # Extra line to bind 'absent' messages to
+            lastline = len (dflines)
+            msgs_by_linenum = {}
+            for severity in ["error", "warn", "info"]:
+                for msg in self._lint[severity]["data"]:
+                    if "line" in msg:
+                        linenum = msg["line"]
+                    else:
+                        linenum = lastline
+
+                    msgs = msgs_by_linenum.get (linenum, [])
+                    msgs.append (msg)
+                    msgs_by_linenum[linenum] = msgs
+
+            linenum = 1
+            for line in dflines:
+                msgs = msgs_by_linenum.get (linenum, [])
+                linenum += 1
+                if not msgs:
+                    continue
+
+                display_line = False
+                msgout = ""
+                for msg in msgs:
+                    if "line" in msg:
+                        display_line = True
+
+                    level = msg["level"]
+                    classes = self.PF_CLASSES.get (level)
+                    if not classes:
+                        continue
+
+                    msgout += classes["alert"] + classes["icon"] + "\n"
+                    msgout += ("  <strong>" +
+                               html_escape (msg["message"]) +
+                               "</strong> ")
+                    description = msg.get ("description", "None")
+                    if description != "None":
+                        msgout += html_escape (description)
+                    url = msg.get ("reference_url", "None")
+                    if url != "None":
+                        if type (url) == list:
+                            url = reduce (lambda a, b: a + b, url)
+
+                        msgout += (' <a href="%s\" class="alert-link">'
+                                   '(more info)</a>' % url)
+                    msgout += "\n</div>\n"
+
+                if display_line:
+                    out += (("<pre>%d:" % linenum) +
+                            html_escape (line).rstrip () + "</pre>\n")
+
+                out += msgout
+
+        if out == "":
+            out = """
+<div class='alert alert-success'>
+  <span class='pficon pficon-ok'></span>
+  <strong>Looks good!</strong> This Dockerfile has no output from dockerfile_lint.
+</div>
+"""
+
+        self._html_markup = out
+
+    def run (self):
+        self._get_dockerfile ()
+        self._run_dockerfile_lint ()
+        self._mark_dockerfile ()
+
+        if self._temp_dir:
+            try:
+                shutil.rmtree(self._temp_dir)
+                self._temp_dir = None
+            except (IOError, OSError) as exc:
+                pass
+
+        return self._html_markup
+
+    def get_json (self):
+        return self._lint
+
+if __name__ == "__main__":
+    git_url = "https://github.com/TomasTomecek/docker-hello-world.git"
+    lint = DockerfileLint (git_url)
+    print (lint.run ())

dbs/models.py

@@ -20,6 +20,12 @@ def __unicode__(self):
         return json.dumps(json.loads(self.json), indent=4)
 
 
+class TaskLint(models.Model):
+    lint = models.TextField()
+
+    def __unicode__(self):
+        return self.lint
+
 
 class Task(models.Model):
     STATUS_PENDING  = 1
@@ -48,6 +54,7 @@ class Task(models.Model):
     type            = models.IntegerField(choices=_TYPE_NAMES.items())
     owner           = models.CharField(max_length=38)
     task_data       = models.ForeignKey(TaskData)
+    task_lint       = models.ForeignKey(TaskLint, null=True, blank=True)
     log             = models.TextField(blank=True, null=True)
 
     class Meta:

dbs/task_api.py

@@ -21,7 +21,11 @@ def watch_task(task, callback, kwargs=None):
 
     :return: None
     """
-    response = task.wait()
+    try:
+        response = task.wait()
+    except Exception as exc:
+        response = exc
+
     if kwargs:
         callback(response, **kwargs)
     else:
@@ -33,7 +37,7 @@ class TaskApi(object):
 
     def build_docker_image(self, build_image, git_url, local_tag, git_dockerfile_path=None, git_commit=None,
                            parent_registry=None, target_registries=None, tag=None, repos=None,
-                           callback=None, kwargs=None):
+                           callback=None, lint_callback=None, kwargs=None):
         """
         build docker image from supplied git repo
 
@@ -63,14 +67,43 @@ def build_docker_image(self, build_image, git_url, local_tag, git_dockerfile_pat
                        'git_commit': git_commit,
                        'git_dockerfile_path': git_dockerfile_path,
                        'repos': repos}
-        task_info = tasks.build_image.apply_async(args=args, kwargs=task_kwargs,
-                                                   link=tasks.submit_results.s())
-        task_id = task_info.task_id
-        if callback:
-            t = Thread(target=watch_task, args=(task_info, callback, kwargs))
-            #w.daemon = True
-            t.start()
-        return task_id
+
+        # The linter task, which runs dockerfile_lint
+        linter_task = tasks.linter.s(git_url,
+                                     git_dockerfile_path,
+                                     git_commit)
+
+        # This task builds the image
+        build_image_task = tasks.build_image.subtask((build_image,
+                                                      git_url,
+                                                      local_tag),
+                                                     **task_kwargs)
+
+        # This task submits the results
+        submit_results_task = tasks.submit_results.s()
+
+        # Chain the tasks together in the right order and start them
+        task_chain = (linter_task |
+                      build_image_task |
+                      submit_results_task).apply_async()
+
+        # Call lint_callback when the linter task is done
+        linter = task_chain.parent.parent # 3rd from last task
+        lint_watcher = Thread(target=watch_task,
+                              args=(linter,
+                                    lint_callback,
+                                    kwargs))
+        lint_watcher.start()
+
+        # Call callback when the entire chain is done
+        chain_watcher = Thread(target=watch_task,
+                               args=(task_chain,
+                                     callback,
+                                     kwargs))
+        chain_watcher.start()
+
+        # Return the celery task ID of the chain
+        return task_chain.task_id
 
     def find_dockerfiles_in_git(self):
         raise NotImplemented()