[WIP] tools: add scripts for automating tests of old tool revisions

tools/compare-conda.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+import os
+import csv
+import json
+import argparse
+import subprocess
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("count")
+
+args = parser.parse_args()
+
+tools = [
+    'odin_ii', 'tree-sitter-verilog', 'zachjs-sv2v', 'verilator', 'iverilog',
+    'yosys', 'slang'
+]
+
+packages = {}
+
+versions = {}
+
+for tool in tools:
+    cmd = 'conda search {} -c symbiflow --json'.format(tool)
+    proc = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)
+    log, _ = proc.communicate()
+    packages[tool] = json.loads(log)
+
+for p in packages:
+    versions[p] = []
+    for r in packages[p][p]:
+        if r['version'] not in versions[p]:
+            versions[p].append(r['version'])
+
+os.system('rm -rf ./tests/generated')
+os.system('make generate-tests')
+
+last_ver = {}
+
+used_versions = []
+
+for i in range(1, int(args.count) + 1):
+    os.system('rm -rf ./out')
+
+    for tool in tools:
+        try:
+            ver = versions[tool][-i]
+        except IndexError:
+            ver = last_ver[tool]
+
+        last_ver[tool] = ver
+
+        os.system('conda install -y -c symbiflow {}={}'.format(tool, ver))
+
+        print('{}={} installed'.format(tool, ver))
+
+    used_versions.append(last_ver.copy())
+
+    os.system('make -j12')
+    os.system('cp ./out/report/report.csv report-{}.csv'.format(i))
+
+with open('out.csv', 'w', newline='') as csvfile:
+    writer = csv.writer(csvfile)
+    writer.writerow(tools)
+    for ver in used_versions:
+        row = []
+        for v in ver:
+            row.append(ver[v])
+
+        writer.writerow(row)

tools/compare.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+rm -rf ./tests/generated
+make generate-tests
+
+for i in $(seq 0 $1); do
+	DATE=$(date --date="`date` -$i week" '+%Y-%m-%d')
+
+	rm -rf ./out/
+	rm -rf ./third_party/tools
+
+	git submodule update
+
+	echo $DATE
+
+	for d in ./third_party/tools/*/ ; do
+		pushd $d
+		git checkout `git rev-list -n 1 --first-parent --before="$DATE" HEAD`
+		popd
+	done
+
+	make runners -j`nproc`
+
+	make -j`nproc`
+
+	cp ./out/report/report.csv $DATE.csv
+done
+
+./tools/csv-analyzer *.csv

tools/csv-analyzer

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+
+import matplotlib.pyplot as plt
+import argparse
+import csv
+import os
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("files", metavar='N', type=str, nargs='+')
+
+args = parser.parse_args()
+
+results = {}
+
+runner_lst = []
+
+for f in args.files:
+    name = os.path.splitext(os.path.basename(f))[0]
+    results[name] = {}
+    with open(f, newline='') as csvfile:
+        reader = csv.DictReader(csvfile)
+        runners = reader.fieldnames[3:]
+        runner_lst += runners
+        for runner in runners:
+            results[name][runner] = 0
+        for row in reader:
+            for runner in runners:
+                if row[runner] == 'True':
+                    results[name][runner] += 1
+
+runner_lst = set(runner_lst)
+
+dates = []
+tools = {}
+
+for d in results:
+    dates.append(d)
+    for runner in runner_lst:
+        val = int(results[d].get(runner, '0'))
+        try:
+            tools[runner].append(val)
+        except KeyError:
+            tools[runner] = [val]
+
+fig, ax = plt.subplots()
+
+for tool in tools:
+    ax.plot(dates, tools[tool], label=tool)
+
+ax.legend()
+plt.setp(ax.get_xticklabels(), rotation=45)
+ax.grid(True)
+plt.savefig('out.png', bbox_inches='tight', dpi=300)
+
+with open('out.csv', 'w', newline='') as csvfile:
+    writer = csv.writer(csvfile)
+    writer.writerow(['date', *runner_lst])
+    for d in results:
+        row = [d]
+        for runner in runner_lst:
+            row.append(results[d].get(runner, '0'))
+        writer.writerow(row)

tools/runners.mk

@@ -61,7 +61,8 @@ tree-sitter-verilog: $(INSTALL_DIR)/lib/tree-sitter-verilog.so
 
 $(INSTALL_DIR)/lib/tree-sitter-verilog.so:
 	mkdir -p $(INSTALL_DIR)/lib
-	cd $(RDIR)/tree-sitter-verilog && npm install
+	cd $(RDIR)/tree-sitter-verilog && npm install tree-sitter-cli
+	cd $(RDIR)/tree-sitter-verilog && ./node_modules/tree-sitter-cli/tree-sitter generate
 	/usr/bin/env python3 -c "from tree_sitter import Language; Language.build_library(\"$@\", [\"$(abspath $(RDIR)/tree-sitter-verilog)\"])"
 
 # sv-parser