benchmarks: add xformers option

fal-ai · Nov 4, 2023 · aa3a059 · aa3a059
1 parent ae04c62
commit aa3a059
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -11,12 +11,14 @@ Running on an A100 80G SXM hosted at [fal.ai](https://fal.ai).
 ### SD1.5 Benchmarks
 |                  | mean (s) | median (s) | min (s) | max (s) | speed (it/s) |
 |------------------|----------|------------|---------|---------|--------------|
-| Diffusers (fp16, SDPA) |   1.654s |     1.654s |  1.632s |  1.675s |   30.24 it/s |
+| Diffusers (fp16, SDPA) |   1.591s |     1.590s |  1.581s |  1.601s |   31.44 it/s |
+| Diffusers (fp16, xformers) |   1.758s |     1.759s |  1.746s |  1.772s |   28.43 it/s |
 
 ### SDXL Benchmarks
 |                  | mean (s) | median (s) | min (s) | max (s) | speed (it/s) |
 |------------------|----------|------------|---------|---------|--------------|
-| Diffusers (fp16, SDPA) |   5.912s |     5.912s |  5.906s |  5.918s |    8.46 it/s |
+| Diffusers (fp16, SDPA) |   5.933s |     5.933s |  5.924s |  5.943s |    8.43 it/s |
+| Diffusers (fp16, xformers) |   5.724s |     5.724s |  5.714s |  5.731s |    8.74 it/s |
 
 <!-- END TABLE -->
 

diff --git a/benchmarks/diffusers.py b/benchmarks/diffusers.py
@@ -11,13 +11,15 @@
         "diffusers==0.21.4",
         "torch==2.1.0",
         "transformers==4.35.0",
+        "xformers==0.0.22.post7",
     ],
     machine_type="GPU",
 )
 def diffusers_any(
     benchmark_settings: BenchmarkSettings,
     parameters: InputParameters,
     model_name: str,
+    enable_xformers: bool = False,
 ) -> BenchmarkResults:
     import torch
     from diffusers import DiffusionPipeline
@@ -28,6 +30,9 @@ def diffusers_any(
         use_safetensors=True,
     )
     pipeline.to("cuda")
+    if enable_xformers:
+        pipeline.enable_xformers_memory_efficient_attention()
+
     return benchmark_settings.apply(
         partial(
             pipeline,
@@ -46,6 +51,15 @@ def diffusers_any(
             "model_name": "runwayml/stable-diffusion-v1-5",
         },
     },
+    {
+        "name": "Diffusers (fp16, xformers)",
+        "category": "SD1.5",
+        "function": diffusers_any,
+        "kwargs": {
+            "model_name": "runwayml/stable-diffusion-v1-5",
+            "enable_xformers": True,
+        },
+    },
     {
         "name": "Diffusers (fp16, SDPA)",
         "category": "SDXL",
@@ -54,4 +68,13 @@ def diffusers_any(
             "model_name": "stabilityai/stable-diffusion-xl-base-1.0",
         },
     },
+    {
+        "name": "Diffusers (fp16, xformers)",
+        "category": "SDXL",
+        "function": diffusers_any,
+        "kwargs": {
+            "model_name": "stabilityai/stable-diffusion-xl-base-1.0",
+            "enable_xformers": True,
+        },
+    },
 ]