fix bechmark.py

NevermindNilas · NevermindNilas · commit e29faadacfb2 · 2024-10-19T15:29:45.000+03:00
diff --git a/BENCHMARKS.MD b/BENCHMARKS.MD
@@ -9,6 +9,50 @@ V2 -> V3, Added new options and a different video with more duplicated frames fo
 V1 -> V2, Longer video for more accurate results, More time in between runs to cool of the gpu, more options ( dedup )
 
 #### Testing Methodology V4
+# Benchmark Results
+
+## System Information
+
+- **CPU:** 13th Gen Intel(R) Core(TM) i7-13700K
+- **Total RAM:** 31.77 GB
+- **Graphics Card 0:** Intel(R) UHD Graphics 770
+- **Graphics Card 1:** NVIDIA GeForce RTX 3090
+
+**Version:** 2.0.0 ( Pre-Release )
+**Testing Methodology:** V4.4
+
+## Upscale Results
+
+| Model                      | Time (s) | FPS   |
+|----------------------------|----------|-------|
+| shufflecugan               | 30.65    | 23.49 |
+| compact                    | 50.81    | 14.17 |
+| ultracompact               | 29.01    | 24.82 |
+| superultracompact          | 9.96     | 72.31 |
+| span                       | 57.32    | 12.56 |
+| compact-tensorrt           | 44.26    | 21.69 |
+| ultracompact-tensorrt      | 24.63    | 38.97 |
+| superultracompact-tensorrt | 9.80     | 97.99 |
+| shufflecugan-tensorrt      | 23.42    | 40.98 |
+| span-tensorrt              | 36.82    | 26.08 |
+
+## Interpolate Results
+
+| Model                           | Time (s) | FPS    |
+|---------------------------------|----------|--------|
+| rife4.6                         | 16.46    | 145.78 |
+| rife4.6-ensemble                | 22.20    | 86.50  |
+| rife4.22                        | 23.34    | 102.82 |
+| rife4.22-ensemble               | 18.85    | 101.83 |
+| rife4.22-lite                   | 17.16    | 139.84 |
+| rife4.22-lite-ensemble          | 14.44    | 132.97 |
+| rife4.6-tensorrt                | 8.83     | 271.90 |
+| rife4.6-tensorrt-ensemble       | 12.38    | 155.06 |
+| rife4.22-tensorrt               | 15.50    | 154.86 |
+| rife4.22-tensorrt-ensemble      | 12.72    | 150.94 |
+| rife4.22-lite-tensorrt          | 11.55    | 207.73 |
+| rife4.22-lite-tensorrt-ensemble | 9.54     | 201.18 |
+
 # Toji
 ![benchmarkResults](https://github.com/user-attachments/assets/195e38da-b3d8-45af-868c-def2b3034e7f)
 
diff --git a/CHANGELOG.MD b/CHANGELOG.MD
@@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Improved `Input/Output` handling which should result in a much more dynamic encoding selection for batch processes.
 - Slight adjustment to chained processes should yield in a small performance boost.
 - Lazifying TAS has improved start up times by up to `1s` in certain workloads, this should improve the responsiveness of TAS by a margin of `30%` on average.
+- Improved and streamlined TAS benchmarkig.
 
 #### Removals:
 - Vidgear package
@@ -37,6 +38,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - `16bit` workflow wouldn't work due to undeclared variable.
 - Fixed an issue where RIFE TRT would produce unwanted results on the 2nd frame of the output video when CPU usage was high.
 - Fixed an issue where RIFE TRT would display duplicated frames past `2x` interpolation.
+- Fixed `benchmark.py` issues caused by backend changes.
 
 #### Breaking Changes:
 - If you are updating from `TAS 1.9.8, 1.9.9, 1.9.10`, YOU will have to delete all `RIFE` related Engines and regenerate them.
diff --git a/benchmark.py b/benchmark.py
@@ -17,11 +17,11 @@
 if not os.path.exists(mainPath):
     os.makedirs(mainPath)
 
-ffmpegLogPath = os.path.join(mainPath, "ffmpegLog.txt")
-logTxtPath = os.path.join(mainPath, "log.txt")
+ffmpegLogPath = os.path.join(mainPath, "ffmpeg.log")
+tasLogPath = os.path.join(mainPath, "TAS.log")
 
 
-def runAllBenchmarks(executor, version, inputVideo=None):
+def runAllBenchmarks(executor, version, inputVideo=None, systemInfo=None):
     print(
         "Running all benchmarks. Depending on your system, this may take a while. Please be patient and keep the terminal in focus at all time."
     )
@@ -33,8 +33,6 @@ def runAllBenchmarks(executor, version, inputVideo=None):
         "Interpolate": runInterpolateBenchmark(inputVideo, executor),
     }
 
-    systemInfo = parseSystemInfo()
-
     with open("benchmarkResults.json", "w") as f:
         json.dump(
             {
@@ -80,7 +78,9 @@ def getClip(executor):
     outputPath = "output/test.mp4"
     cmd = executor + ["--input", CLIPURL, "--output", outputPath]
     subprocess.Popen(cmd, shell=False).wait()
-    return os.path.abspath(outputPath)
+    systemInfo = parseSystemInfo()
+
+    return os.path.abspath(outputPath), systemInfo
 
 
 def runUpscaleBenchmark(inputVideo, executor):
@@ -94,9 +94,10 @@ def runUpscaleBenchmark(inputVideo, executor):
             "--upscale",
             "--upscale_method",
             method,
+            "--static",
             "--benchmark",
             "--outpoint",
-            "16" if "-tensorrt" in method else "12",
+            "20" if "-tensorrt" in method else "15",
         ]
         print(f"Running command: {' '.join(cmd)}")  # Debugging line
         subprocess.run(cmd, check=True, cwd=os.path.dirname(os.path.abspath(__file__)))
@@ -143,10 +144,13 @@ def runInterpolateBenchmark(inputVideo, executor):
             "--interpolate_method",
             method,
             "--benchmark",
-            "--ensemble",
+            # "--ensemble",
             "--outpoint",
             "20",
         ]
+
+        if method not in ["rife4.22", "rife4.22-lite"]:
+            cmd += ["--ensemble"]
         print(f"Running command: {' '.join(cmd)}")  # Debugging line
         subprocess.run(cmd, cwd=os.path.dirname(os.path.abspath(__file__)))
 
@@ -158,24 +162,24 @@ def runInterpolateBenchmark(inputVideo, executor):
 
 
 def parseFPS():
-    with open(ffmpegLogPath, "r") as file:
-        output = file.read()
-    matches = re.findall(r"fps=\s*([\d.]+)", output)
-    # Filter out fps values that are 0.0 or 0
-    filtered = [float(fps) for fps in matches if float(fps) > 0]
-    if filtered:
-        highestFPS = max(filtered)
-        averageFPS = round(sum(filtered) / len(filtered), 2)
-        print("Highest FPS:", highestFPS, "Average FPS:", averageFPS)
-        return ("Highest FPS:", highestFPS, "Average FPS:", averageFPS)
-    else:
-        print("Couldn't identify FPS value. Skipping...")
-        return None
+    with open(tasLogPath, "r") as file:
+        for line in file:
+            if "Total Execution Time" in line:
+                match = re.search(
+                    r"Total Execution Time: ([\d.]+) seconds - FPS: ([\d.]+)", line
+                )
+                if match:
+                    total_execution_time = float(match.group(1))
+                    fps = float(match.group(2))
+                    print(f"Total Execution Time: {total_execution_time} seconds")
+                    print(f"FPS: {fps}")
+                    return total_execution_time, fps
+    return None, None
 
 
 def parseSystemInfo():
     systemInfo = {}
-    with open(logTxtPath, "r") as file:
+    with open(tasLogPath, "r") as file:
         lines = file.readlines()
         start = lines.index("============== System Checker ==============\n") + 1
         end = lines.index("============== Arguments Checker ==============\n")
@@ -189,7 +193,7 @@ def parseSystemInfo():
 if __name__ == "__main__":
     TIMESLEEP = 2
     CLIPURL = "https://www.youtube.com/watch?v=kpeUMAVJCig"
-    TESTINGVERSION = "V4.3"
+    TESTINGVERSION = "V4.4"
 
     upscaleMethods = [
         "shufflecugan",
@@ -224,7 +228,7 @@ def parseSystemInfo():
 
     currentTest = 0
     executor, version = getExe()
-    inputVideo = getClip(executor)
+    inputVideo, systemInfo = getClip(executor)
 
     # Define the questions
     questions = [
@@ -274,4 +278,4 @@ def parseSystemInfo():
     print(f"Using {' '.join(executor)} version {version}")
     print("Current working directory:", os.getcwd())
 
-    runAllBenchmarks(executor, version, inputVideo)
+    runAllBenchmarks(executor, version, inputVideo, systemInfo)
diff --git a/src/utils/jsonToTable.py b/src/utils/jsonToTable.py
@@ -1,104 +1,47 @@
 import json
-import pandas as pd
-import matplotlib.pyplot as plt
-import argparse
-
-from matplotlib.gridspec import GridSpec
-
-def jsonToTable(input: str) -> None:
-    with open(input, "r") as file:
-        data = json.load(file)
-
-    fig = plt.figure(figsize=(16, 12))
-    gs = GridSpec(2, 2, width_ratios=[1, 1.5], height_ratios=[1, 1])
-    fig.suptitle("Benchmark Results", fontsize=16)
-
-    ax1 = fig.add_subplot(gs[:, 0])
-    systemInfo = pd.DataFrame.from_dict(
-        data["System Info"], orient="index", columns=["Value"]
-    )
-    systemInfo.index.name = "Property"
-    systemInfo = systemInfo.reset_index()
-
-    versionInfo = pd.DataFrame(
-        {
-            "Property": ["Version", "Testing Methodology"],
-            "Value": [data["Version"], data["Testing Methodology"]],
-        }
-    )
-    systemInfo = pd.concat([versionInfo, systemInfo]).reset_index(drop=True)
-
-    ax1.axis("off")
-    ax1.set_title("System Information")
-    table1 = ax1.table(
-        cellText=systemInfo.values,
-        colLabels=systemInfo.columns,
-        cellLoc="left",
-        loc="center",
-    )
-    table1.auto_set_font_size(False)
-    table1.set_fontsize(9)
-    table1.scale(1, 1.5)
-
-    ax2 = fig.add_subplot(gs[0, 1])
-    upscaleData = data["Results"]["Upscale"]
-    upscaleDf = pd.DataFrame.from_dict(
-        upscaleData,
-        orient="index",
-        columns=["Highest FPS", "Highest FPS Value", "Average FPS", "Average FPS Value"],
-    )
-    upscaleDf = upscaleDf.drop(["Highest FPS", "Average FPS"], axis=1)
-    upscaleDf.index.name = "Model"
-    upscaleDf = upscaleDf.reset_index()
-    upscaleDf["Model"] = upscaleDf["Model"].str.title()
-
-    ax2.axis("off")
-    ax2.set_title("Upscale Results")
-    table2 = ax2.table(
-        cellText=upscaleDf.values, colLabels=upscaleDf.columns, cellLoc="left", loc="center"
-    )
-    table2.auto_set_font_size(False)
-    table2.set_fontsize(9)
-    table2.scale(1, 1.5)
-
-    ax3 = fig.add_subplot(gs[1, 1])
-    interpolateData = data["Results"]["Interpolate"]
-    interpolateDf = pd.DataFrame.from_dict(
-        interpolateData,
-        orient="index",
-        columns=["Highest FPS", "Highest FPS Value", "Average FPS", "Average FPS Value"],
-    )
-    interpolateDf = interpolateDf.drop(["Highest FPS", "Average FPS"], axis=1)
-    interpolateDf.index.name = "Model"
-    interpolateDf = interpolateDf.reset_index()
-    interpolateDf["Model"] = interpolateDf["Model"].str.title()
-
-    ax3.axis("off")
-    ax3.set_title("Interpolate Results")
-    table3 = ax3.table(
-        cellText=interpolateDf.values,
-        colLabels=interpolateDf.columns,
-        cellLoc="left",
-        loc="center",
-    )
-    table3.auto_set_font_size(False)
-    table3.set_fontsize(9)
-    table3.scale(1, 1.5)
-
-    plt.tight_layout()
-    plt.savefig("benchmarkResults.png", dpi=300, bbox_inches="tight")
-
-    plt.show()
-
-if __name__ == "__main__":
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--input",
-        type=str,
-        help="Path to the JSON file containing the benchmark results",
-        required=True,
-    )
-
-    args = parser.parse_args()
-    jsonToTable(args.input)
+import random
+import string
+
+jsonPath = r"G:\TheAnimeScripter\benchmarkResults.json"
+
+
+def jsonToMarkdownTable(jsonData):
+    markdown = "# Benchmark Results\n\n"
+
+    markdown += "## System Information\n\n"
+    for key, value in jsonData["System Info"].items():
+        markdown += f"- **{key}:** {value}\n"
+    markdown += "\n"
+
+    markdown += f"**Version:** {jsonData['Version']}  \n"
+    markdown += f"**Testing Methodology:** {jsonData['Testing Methodology']}\n\n"
+
+    for category, models in jsonData["Results"].items():
+        markdown += f"## {category} Results\n\n"
+        markdown += "| Model | Time (s) | FPS |\n"
+        markdown += "|-------|-----------|-----|\n"
+
+        for model, results in models.items():
+            timeMs, fps = results
+            markdown += f"| {model} | {timeMs:.2f} | {fps:.2f} |\n"
+
+        markdown += "\n"
+
+    return markdown
+
+
+def generateRandomFilename(extension="md"):
+    randomString = "".join(random.choices(string.ascii_lowercase + string.digits, k=8))
+    return f"{randomString}.{extension}"
+
+
+with open(jsonPath, "r") as file:
+    jsonData = json.load(file)
+
+markdownOutput = jsonToMarkdownTable(jsonData)
+
+randomFilename = generateRandomFilename()
+with open(randomFilename, "w") as file:
+    file.write(markdownOutput)
+
+print(f"{randomFilename} has been generated successfully.")
