generate-tex.py

#!/usr/bin/env python3

import yaml
import glob
from jinja2 import Template
from os.path import basename, splitext
from subprocess import check_output
from collections import defaultdict


def convert_markdown_to_tex(markdown):
    return check_output(
        ["pandoc", "--from=markdown", "--to=latex"],
        universal_newlines=True,
        input=markdown
    ).replace('??', '\+').rstrip("\n") # if line breaks are needed


def convert_map_values_to_tex(map):
    if map is not None:
        map.update({k: convert_markdown_to_tex(v) for k, v in map.items()})
        return map


def convert_list_entries_to_tex(list):
    if list is not None:
        return [convert_markdown_to_tex(e) for e in list]


def convert_map_list_to_tex(list_of_maps):
    if list_of_maps is not None:
        return [convert_map_values_to_tex(e) for e in list_of_maps]

##### query cards

with open('templates/query-card-template.tex', 'r') as f:
    query_card_template = Template(f.read())

# with open('templates/short-description-template.tex', 'r') as f:
#     short_description_template = Template(f.read())

# with open('templates/standalone-query-card.tex', 'r') as f:
#     standalone_query_card_template = Template(f.read())

# with open('templates/parameters', 'r') as f:
#     parameters_template = Template(f.read())

all_choke_points = set()
all_queries = set()
query_choke_point = defaultdict(list)      # queries -> cps
choke_point_references = defaultdict(list) # cps -> queries

for filename in glob.glob("query-specifications/*.yaml"):
    if "deprecated" in basename(filename):
        continue
    print("Processing query specification: %s" % filename)
    query_id = splitext(basename(filename))[0]

    with open(filename, 'r') as f:
        doc = yaml.load(f, Loader=yaml.FullLoader)

    number = doc['number']
    number_string = "%02d" % (number)
    workload = doc['workload']
    title = doc['title']
    description_markdown = doc['description']
    operation = doc['operation']

    query_tuple = (query_id, workload, operation, number)
    all_queries.add(query_tuple)

    choke_points = list(map(lambda e: str(e), doc.get('choke_points', [])))
    for choke_point in choke_points:
        choke_point_references[choke_point].append(query_tuple)
        all_choke_points.add(choke_point)

    query_choke_point[query_id] = choke_points

    # currently, there are no off-the-shelf solutions for Markdown to TeX conversion in Python 3,
    # so we use Pandoc -- it's hands down the best Markdown to Tex converter you can get anyways
    description_tex = convert_markdown_to_tex(description_markdown)
    
    # optional arguments 
    composition = doc.get('compositions')
    parameters = doc.get('parameters')
    results = doc.get('result')
    sort = doc.get('sort')
    limit = doc.get('limit')
    relevance_markdown = doc.get('relevance')
    groupby = doc.get('groupby')

    relevance_tex = None if relevance_markdown is None else convert_markdown_to_tex(relevance_markdown)

    # parameter_file_text = parameters_template.render(
    #     parameters = parameters,
    # )

    # with open("parameters/%s.parameters" % query_id, 'w') as parameter_file:
    #     parameter_file.write(parameter_file_text)

    query_card_text = query_card_template.render(
        number        = number,
        workload      = workload,
        operation     = operation,
        number_string = number_string,
        query_id      = query_id,
        title         = title,
        description   = description_tex,
        composition   = convert_map_list_to_tex(composition),
        parameters    = convert_map_list_to_tex(parameters),
        result        = convert_map_list_to_tex(results),
        sort          = convert_map_list_to_tex(sort),
        limit         = limit,
        choke_points  = choke_points,
        groupby       = groupby,
        # relevance     = relevance_tex, # Comment out temporarily
    )

    with open("query-cards/%s.tex" % query_id, 'w') as query_card_file:
        query_card_file.write(query_card_text)

    ##### short descriptions
    # short_description_text = short_description_template.render(
    #     number        = number,
    #     title         = title,
    #     description   = description_tex,
    # )
    # short_description_text = short_description_text.replace("\n\n", "\n")

    # with open("short-descriptions/%s.tex" % query_id, 'w') as short_description_file:
    #     short_description_file.write(short_description_text)

    ##### standalone query cards
    # standalone_query_card_text = standalone_query_card_template.render(
    #     query_id=query_id
    # )
    # with open("standalone-query-cards/%s.tex" % query_id, 'w') as standalone_query_card_file:
    #     standalone_query_card_file.write(standalone_query_card_text)

##### choke points

with open('templates/choke-point-query-mapping-template.tex', 'r') as f:
    choke_point_template = Template(f.read())

for choke_point in choke_point_references:
    choke_point_filename = choke_point.replace('.', '-')

    queries = choke_point_references[choke_point]
    queries_sorted = sorted(queries, key=lambda tup: tup[0])
    choke_point_text = choke_point_template.render(
        queries = queries_sorted,
    )

    with open("choke-point-query-mapping/cp-%s.tex" % choke_point_filename, 'w') as choke_point_file:
        choke_point_file.write(choke_point_text)

##### table for choke points and queries

# print('Processing choke points... ', end='')

# all_queries_sorted = sorted(all_queries, key=lambda tup: tup[0])
# all_choke_points_sorted = sorted(all_choke_points)

# with open('templates/choke-points-queries.tex', 'r') as f:
#     choke_points_queries_template = Template(f.read())

# choke_points_queries_text = choke_points_queries_template.render(
#     choke_point_references = choke_point_references,
#     query_choke_point = query_choke_point,
#     queries = all_queries_sorted,
#     choke_points = all_choke_points_sorted,
# )

# with open("choke-points/choke-points-queries.tex", 'w') as choke_points_queries_template:
#     choke_points_queries_template.write(choke_points_queries_text)

# #### CSV for choke points and queries (to be pasted in a spreadsheet)

# with open('templates/choke-points-queries.csv', 'r') as f:
#     choke_points_queries_template = Template(f.read())

# choke_points_queries_text = choke_points_queries_template.render(
#     choke_point_references = choke_point_references,
#     query_choke_point = query_choke_point,
#     queries = all_queries_sorted,
#     choke_points = all_choke_points_sorted,
# )

# with open("choke-points/choke-points-queries.csv", 'w') as choke_points_queries_template:
#     choke_points_queries_template.write(choke_points_queries_text)

print('Done')