Render markdown by converting through asciidoc with pandoc

Use pandoc to convert markdown to asciidoc, then convert the asciidoc to
html using asciidoctor.

Fixes #1328

Author: gavinwahl

core/asciidoc.py

@@ -1,6 +1,9 @@
 import subprocess
 
 
+ASCIIDOCTOR_COMMAND = ["asciidoctor", "-r", "asciidoctor_boost", "-e", "-o", "-", "-"]
+
+
 def convert_adoc_to_html(input):
     """
     Converts an AsciiDoc file to HTML.
@@ -15,7 +18,7 @@ def convert_adoc_to_html(input):
     :param input: The contents of the AsciiDoc file
     """
     result = subprocess.run(
-        ["asciidoctor", "-r", "asciidoctor_boost", "-e", "-o", "-", "-"],
+        ASCIIDOCTOR_COMMAND,
         check=True,
         capture_output=True,
         text=True,
@@ -24,3 +27,35 @@ def convert_adoc_to_html(input):
 
     # Get the output from the command
     return result.stdout
+
+
+def convert_md_to_html(input):
+    # first convert gfm to asciidoctor
+    pandoc = subprocess.Popen(
+        ["pandoc", "-f", "gfm", "-t", "asciidoctor"],
+        stdout=subprocess.PIPE,
+        stdin=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    # then convert asciidoctor to html
+    ad = subprocess.Popen(
+        ASCIIDOCTOR_COMMAND,
+        stdin=pandoc.stdout,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    pandoc.stdin.write(input)
+    pandoc.stdin.close()
+    pandoc_stderr = pandoc.stderr.read()
+    stdout, ad_stderr = ad.communicate()
+    pandoc.wait()
+    assert (
+        ad.returncode == 0
+    ), f"asciidoctor command returned {ad.returncode}: {ad_stderr}"
+    assert (
+        pandoc.returncode == 0
+    ), f"pandoc command returned {pandoc.returncode}: {pandoc_stderr}"
+
+    return stdout

core/tests/test_asciidoc.py

@@ -1,10 +1,11 @@
+import textwrap
 from os import getcwd, makedirs
 from unittest.mock import patch
 
 import pytest
 
 
-from core.asciidoc import convert_adoc_to_html
+from core.asciidoc import convert_adoc_to_html, convert_md_to_html
 
 
 def test_convert_adoc_to_html_subprocess():
@@ -40,3 +41,37 @@ def test_convert_adoc_to_html_content_file():
         makedirs("/tmp/asciidocs", exist_ok=True)
         open("/tmp/asciidocs/tmp.html", "w").write(output)
     assert output == expected_output
+
+
+@pytest.mark.asciidoctor
+def test_convert_md_to_html():
+    input = textwrap.dedent(
+        """
+        header
+        header
+        ------
+        text
+        """
+    )
+    output = textwrap.dedent(
+        """
+        <div class="sect1">
+        <h2 id="_header_header">header header</h2>
+        <div class="sectionbody">
+        <div class="paragraph">
+        <p>text</p>
+        </div>
+        </div>
+        </div>
+        """
+    )
+
+    assert convert_md_to_html(input).strip() == output.strip()
+
+
+@pytest.mark.asciidoctor
+def test_convert_md_to_html_huge_input():
+    """
+    Make sure we don't run into pipe buffering issues with large inputs.
+    """
+    convert_md_to_html("asdf\n" * 500000)

docker/Dockerfile

@@ -35,7 +35,7 @@ RUN yarn build
 # Final image.
 FROM python:3.11-slim AS release
 
-RUN apt update && apt install -y git libpq-dev ruby ruby-dev && rm -rf /var/lib/apt/lists/*
+RUN apt update && apt install -y git libpq-dev ruby ruby-dev pandoc && rm -rf /var/lib/apt/lists/*
 
 # Install Asciidoctor
 RUN gem install asciidoctor asciidoctor-boost

libraries/models.py

@@ -10,13 +10,12 @@
 from django.db.models.functions import Upper
 
 from core.htmlhelper import get_release_notes_for_library_version
-from core.markdown import process_md
 from core.models import RenderedContent
-from core.asciidoc import convert_adoc_to_html
+from core.asciidoc import convert_adoc_to_html, convert_md_to_html
 from libraries.managers import IssueManager
 from mailing_list.models import EmailData
 
-from .utils import generate_random_string, write_content_to_tempfile
+from .utils import generate_random_string
 
 
 class Category(models.Model):
@@ -278,8 +277,7 @@ def get_description(self, client, tag="develop"):
                 if file_path.endswith(".adoc"):
                     body_content = convert_adoc_to_html(content.decode("utf-8"))
                 else:
-                    temp_file = write_content_to_tempfile(content)
-                    _, body_content = process_md(temp_file.name)
+                    body_content = convert_md_to_html(content.decode("utf-8"))
                 static_content_cache.set(cache_key, body_content)
                 RenderedContent.objects.update_or_create(
                     cache_key=cache_key,