diff --git a/tools/compare-conda.py b/tools/compare-conda.py new file mode 100755 index 0000000000000..be6793f7f7cdc --- /dev/null +++ b/tools/compare-conda.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +import os +import csv +import json +import argparse +import subprocess + +parser = argparse.ArgumentParser() + +parser.add_argument("count") + +args = parser.parse_args() + +tools = [ + 'odin_ii', 'tree-sitter-verilog', 'zachjs-sv2v', 'verilator', 'iverilog', + 'yosys', 'slang' +] + +packages = {} + +versions = {} + +for tool in tools: + cmd = 'conda search {} -c symbiflow --json'.format(tool) + proc = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) + log, _ = proc.communicate() + packages[tool] = json.loads(log) + +for p in packages: + versions[p] = [] + for r in packages[p][p]: + if r['version'] not in versions[p]: + versions[p].append(r['version']) + +os.system('rm -rf ./tests/generated') +os.system('make generate-tests') + +last_ver = {} + +used_versions = [] + +for i in range(1, int(args.count) + 1): + os.system('rm -rf ./out') + + for tool in tools: + try: + ver = versions[tool][-i] + except IndexError: + ver = last_ver[tool] + + last_ver[tool] = ver + + os.system('conda install -y -c symbiflow {}={}'.format(tool, ver)) + + print('{}={} installed'.format(tool, ver)) + + used_versions.append(last_ver.copy()) + + os.system('make -j12') + os.system('cp ./out/report/report.csv report-{}.csv'.format(i)) + +with open('out.csv', 'w', newline='') as csvfile: + writer = csv.writer(csvfile) + writer.writerow(tools) + for ver in used_versions: + row = [] + for v in ver: + row.append(ver[v]) + + writer.writerow(row) diff --git a/tools/compare.sh b/tools/compare.sh new file mode 100755 index 0000000000000..9fb9b75ab2b34 --- /dev/null +++ b/tools/compare.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +rm -rf ./tests/generated +make generate-tests + +for i in $(seq 0 $1); do + DATE=$(date --date="`date` -$i week" '+%Y-%m-%d') + + rm -rf ./out/ + rm -rf ./third_party/tools + + git submodule update + + echo $DATE + + for d in ./third_party/tools/*/ ; do + pushd $d + git checkout `git rev-list -n 1 --first-parent --before="$DATE" HEAD` + popd + done + + make runners -j`nproc` + + make -j`nproc` + + cp ./out/report/report.csv $DATE.csv +done + +./tools/csv-analyzer *.csv diff --git a/tools/csv-analyzer b/tools/csv-analyzer new file mode 100755 index 0000000000000..7ff857b884dcc --- /dev/null +++ b/tools/csv-analyzer @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +import matplotlib.pyplot as plt +import argparse +import csv +import os + +parser = argparse.ArgumentParser() + +parser.add_argument("files", metavar='N', type=str, nargs='+') + +args = parser.parse_args() + +results = {} + +runner_lst = [] + +for f in args.files: + name = os.path.splitext(os.path.basename(f))[0] + results[name] = {} + with open(f, newline='') as csvfile: + reader = csv.DictReader(csvfile) + runners = reader.fieldnames[3:] + runner_lst += runners + for runner in runners: + results[name][runner] = 0 + for row in reader: + for runner in runners: + if row[runner] == 'True': + results[name][runner] += 1 + +runner_lst = set(runner_lst) + +dates = [] +tools = {} + +for d in results: + dates.append(d) + for runner in runner_lst: + val = int(results[d].get(runner, '0')) + try: + tools[runner].append(val) + except KeyError: + tools[runner] = [val] + +fig, ax = plt.subplots() + +for tool in tools: + ax.plot(dates, tools[tool], label=tool) + +ax.legend() +plt.setp(ax.get_xticklabels(), rotation=45) +ax.grid(True) +plt.savefig('out.png', bbox_inches='tight', dpi=300) + +with open('out.csv', 'w', newline='') as csvfile: + writer = csv.writer(csvfile) + writer.writerow(['date', *runner_lst]) + for d in results: + row = [d] + for runner in runner_lst: + row.append(results[d].get(runner, '0')) + writer.writerow(row) diff --git a/tools/runners.mk b/tools/runners.mk index 780b00b8d4604..9ca94852600dd 100644 --- a/tools/runners.mk +++ b/tools/runners.mk @@ -61,7 +61,8 @@ tree-sitter-verilog: $(INSTALL_DIR)/lib/tree-sitter-verilog.so $(INSTALL_DIR)/lib/tree-sitter-verilog.so: mkdir -p $(INSTALL_DIR)/lib - cd $(RDIR)/tree-sitter-verilog && npm install + cd $(RDIR)/tree-sitter-verilog && npm install tree-sitter-cli + cd $(RDIR)/tree-sitter-verilog && ./node_modules/tree-sitter-cli/tree-sitter generate /usr/bin/env python3 -c "from tree_sitter import Language; Language.build_library(\"$@\", [\"$(abspath $(RDIR)/tree-sitter-verilog)\"])" # sv-parser