core: auto generated markdown tables

.github/workflows/benchmark.yml

@@ -1,11 +1,7 @@
 name: Run App Integration Tests
 
 on:
-  push:
-    branches:
-      - main
-  pull_request:
-    types: [assigned, opened, synchronize, reopened]
+  workflow_dispatch:
 
 jobs:
   run:
@@ -33,6 +29,16 @@ jobs:
           FAL_TARGET_NODE: ${{ secrets.FAL_TARGET_NODE }}
         run: |
           python -m benchmarks /tmp \
-            --session-id="${{ github.run_id }}" \
+            --session-id=latest \
             --target-node=$FAL_TARGET_NODE
 
+      - name: Regenerate tables
+        run: python benchmarks/regenerate_tables.py /tmp/latest.json
+
+      - name: Commit and push changes
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: "perf: update benchmarks"
+          commit_options: "--no-verify"
+          commit_user_name: "Fal Bot"
+          commit_user_email: "[email protected]"

README.md

@@ -2,3 +2,13 @@
 
 A set of benchmarks targeting different stable diffusion implementations to have a
 better understanding of their performance and scalability.
+
+## Benchmarks
+
+Running on an A100 80G SXM hosted at [fal.ai](https://fal.ai).
+
+<!-- START TABLE -->
+|                  | mean (s) | median (s) | min (s) | max (s) |
+|------------------|----------|------------|---------|---------|
+| Diffusers SD1.5  |   1.610s |     1.606s |  1.595s |  1.630s |
+<!-- END TABLE -->

benchmarks/__main__.py

@@ -1,8 +1,10 @@
-import json
 import argparse
-from rich.progress import track
-from pathlib import Path
+import json
+from dataclasses import asdict
 from datetime import datetime
+from pathlib import Path
+
+from rich.progress import track
 
 from benchmarks import diffusers
 from benchmarks.settings import BenchmarkSettings, InputParameters
@@ -36,7 +38,7 @@ def main() -> None:
     )
     parameters = InputParameters(prompt="A photo of a cat", steps=50)
 
-    results = []
+    timings = []
     for benchmark in track(ALL_BENCHMARKS, description="Running benchmarks..."):
         print(f"Running benchmark: {benchmark['name']}")
         function = benchmark["function"].on(_scheduler="nomad")
@@ -52,13 +54,19 @@ def main() -> None:
             parameters=parameters,
             **benchmark["kwargs"],
         )
-        results.append(
+        timings.append(
             {
                 "name": benchmark["name"],
                 "timings": benchmark_results.timings,
             }
         )
 
+    results = {
+        "settings": asdict(settings),
+        "parameters": asdict(parameters),
+        "timings": timings,
+    }
+
     with open(options.results_dir / f"{options.session_id}.json", "w") as results_file:
         json.dump(results, results_file)
 

benchmarks/diffusers.py

@@ -1,6 +1,8 @@
-import fal
 from functools import partial
-from benchmarks.settings import BenchmarkSettings, InputParameters, BenchmarkResults
+
+import fal
+
+from benchmarks.settings import BenchmarkResults, BenchmarkSettings, InputParameters
 
 
 @fal.function(

benchmarks/settings.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 import time
+from collections.abc import Callable
 from dataclasses import dataclass
-from typing import Callable, Any
+from typing import Any
 
 
 @dataclass

benchmarks/update_table.py

@@ -0,0 +1,51 @@
+import json
+import statistics
+from argparse import ArgumentParser
+from pathlib import Path
+
+README_PATH = Path(__file__).parent.parent / "README.md"
+TABLE_HEADER = "|                  | mean (s) | median (s) | min (s) | max (s) |\n"
+TABLE_DIVIDER = "|------------------|----------|------------|---------|---------|\n"
+TABLE_ROW_FORMAT = (
+    "| {name:16} | {mean:7.3f}s | {median:9.3f}s | {min:6.3f}s | {max:6.3f}s |\n"
+)
+START_MARKER = "<!-- START TABLE -->\n"
+END_MARKER = "<!-- END TABLE -->\n"
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument("results_file", type=Path)
+    parser.add_argument("--document-path", type=Path, default=README_PATH)
+
+    options = parser.parse_args()
+    with open(options.results_file) as f:
+        results = json.load(f)
+
+    with open(options.document_path) as f:
+        lines = f.readlines()
+
+    table_lines = [TABLE_HEADER, TABLE_DIVIDER]
+    for timing in results["timings"]:
+        benchmark_name = timing["name"]
+        benchmark_timings = timing["timings"]
+        table_lines.append(
+            TABLE_ROW_FORMAT.format(
+                name=benchmark_name,
+                mean=statistics.mean(benchmark_timings),
+                median=statistics.median(benchmark_timings),
+                min=min(benchmark_timings),
+                max=max(benchmark_timings),
+            )
+        )
+
+    start_index = lines.index(START_MARKER) + 1
+    end_index = lines.index(END_MARKER)
+    lines[start_index:end_index] = table_lines
+
+    with open(options.document_path, "w") as f:
+        f.writelines(lines)
+
+
+if __name__ == "__main__":
+    main()