From 3fc3eeab64d76bb213804d67518f481d29281e4d Mon Sep 17 00:00:00 2001 From: Lisa Glover Date: Thu, 22 Jan 2026 12:00:49 -0800 Subject: [PATCH 01/10] Removed the conditional benchmark attributes from the VTS library and created benchmark classes in the Benckmark project. --- .../MonteCarloSimulationBenchmarks.cs | 32 +++++++++++++ .../ParallelMonteCarloSimulationBenchmarks.cs | 35 ++++++++++++++ src/Vts.Benchmark/NLog.config | 14 ++++++ src/Vts.Benchmark/Program.cs | 20 ++++---- src/Vts.sln | 46 +------------------ src/Vts/MonteCarlo/MonteCarloSimulation.cs | 6 --- .../ParallelMonteCarloSimulation.cs | 19 -------- src/Vts/Vts.csproj | 8 +--- 8 files changed, 94 insertions(+), 86 deletions(-) create mode 100644 src/Vts.Benchmark/Benchmarks/MonteCarloSimulationBenchmarks.cs create mode 100644 src/Vts.Benchmark/Benchmarks/ParallelMonteCarloSimulationBenchmarks.cs create mode 100644 src/Vts.Benchmark/NLog.config diff --git a/src/Vts.Benchmark/Benchmarks/MonteCarloSimulationBenchmarks.cs b/src/Vts.Benchmark/Benchmarks/MonteCarloSimulationBenchmarks.cs new file mode 100644 index 000000000..fd55fca82 --- /dev/null +++ b/src/Vts.Benchmark/Benchmarks/MonteCarloSimulationBenchmarks.cs @@ -0,0 +1,32 @@ +using BenchmarkDotNet.Attributes; +using NLog; +using System.Collections.Generic; +using Vts.MonteCarlo; + +namespace Vts.Benchmark.Benchmarks; + +public class MonteCarloSimulationBenchmarks +{ + private MonteCarloSimulation _simulation; + + [ParamsSource(nameof(SimulationInputs))] + public SimulationInput Input { get; set; } + + public IEnumerable SimulationInputs() + { + yield return new SimulationInput { N = 100 }; + // we can add more SimulationInput instances with different configurations if needed + } + + [GlobalSetup] + public void Setup() + { + LogManager.SuspendLogging(); // logging can interfere with benchmark results, it is also suspended in NLog config for benchmarks + + _simulation = new MonteCarloSimulation(Input); + } + + [Benchmark] + public SimulationOutput RunSimulation() + => _simulation.Run(); +} \ No newline at end of file diff --git a/src/Vts.Benchmark/Benchmarks/ParallelMonteCarloSimulationBenchmarks.cs b/src/Vts.Benchmark/Benchmarks/ParallelMonteCarloSimulationBenchmarks.cs new file mode 100644 index 000000000..0d60d2af5 --- /dev/null +++ b/src/Vts.Benchmark/Benchmarks/ParallelMonteCarloSimulationBenchmarks.cs @@ -0,0 +1,35 @@ +using BenchmarkDotNet.Attributes; +using NLog; +using System.Collections.Generic; +using Vts.MonteCarlo; + +namespace Vts.Benchmark.Benchmarks; + +public class ParallelMonteCarloSimulationBenchmarks +{ + private ParallelMonteCarloSimulation _simulation; + + [ParamsSource(nameof(SimulationInputs))] + public SimulationInput Input { get; set; } + + public IEnumerable SimulationInputs() + { + yield return new SimulationInput { N = 100 }; + // we can add more SimulationInput instances with different configurations if needed + } + + [Params(4)] // we can add more parameters separated by commas for different CPU counts + public int NumberOfCpUs { get; set; } + + [GlobalSetup] + public void Setup() + { + LogManager.SuspendLogging(); + + _simulation = new ParallelMonteCarloSimulation(Input, NumberOfCpUs); + } + + [Benchmark] + public SimulationOutput RunSingleInParallel() + => _simulation.RunSingleInParallel(); +} \ No newline at end of file diff --git a/src/Vts.Benchmark/NLog.config b/src/Vts.Benchmark/NLog.config new file mode 100644 index 000000000..a6478e968 --- /dev/null +++ b/src/Vts.Benchmark/NLog.config @@ -0,0 +1,14 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/Vts.Benchmark/Program.cs b/src/Vts.Benchmark/Program.cs index e5079574b..68163b0af 100644 --- a/src/Vts.Benchmark/Program.cs +++ b/src/Vts.Benchmark/Program.cs @@ -9,7 +9,7 @@ using System; using System.IO; using System.Text.RegularExpressions; -using Vts.MonteCarlo; +using Vts.Benchmark.Benchmarks; namespace Vts.Benchmark { @@ -17,15 +17,13 @@ public static class Program { public static void Main(string[] args) { - // Set up execution of Monte Carlo Command Line application and run benchmark + // Set up execution of Monte Carlo Command Line application and run benchmark - We might need to figure this out later // collecting estimate of the Mean time of execution and Standard Deviation. // Output to CSV file. - // Notes: 1) Build Vts.Benchmark in Benchmark configuration (the Post-Processor - // Program.cs will show compile errors on the CommandLine.Switch statements but this - // is okay since not included in the benchmark - // 2) Run with Debug tab -> Start Without Debugging + // Notes: 1) Build Vts.Benchmark in Release configuration + // 2) Run with Debug tab -> Start Without Debugging var config = new ManualConfig() - .AddJob(new Job("Benchmark").WithCustomBuildConfiguration("Benchmark")) + .AddJob(new Job("Benchmark")) .AddLogger(ConsoleLogger.Default) .AddColumn(TargetMethodColumn.Method, StatisticColumn.Mean, @@ -35,11 +33,14 @@ public static void Main(string[] args) .AddExporter(CsvExporter.Default, HtmlExporter.Default, MarkdownExporter.GitHub) .AddAnalyser(EnvironmentAnalyser.Default); config.UnionRule = ConfigUnionRule.Union; - var summary = BenchmarkRunner.Run(config); + // This line is used to run the non-parallel benchmark, we could add a parameter to switch between them + var summary = BenchmarkRunner.Run(config); + // This line is used to run the parallel benchmark + //var summary = BenchmarkRunner.Run(config); Console.WriteLine(summary); // Read CSV file for Mean and Standard Deviation (StDev) data var inputPath = Path.GetFullPath(Directory.GetCurrentDirectory()); - const string csvFile = "\\BenchmarkDotNet.Artifacts\\results\\Vts.MonteCarlo.MonteCarloSimulation-report.csv"; + const string csvFile = "\\BenchmarkDotNet.Artifacts\\results\\Vts.Benchmark.Benchmarks.MonteCarloSimulationBenchmarks-report.csv"; var filePath = Path.GetFullPath(inputPath + csvFile); using var reader = new StreamReader(filePath); // read header line @@ -67,6 +68,7 @@ public static void Main(string[] args) Console.ForegroundColor = ConsoleColor.Cyan; Console.WriteLine(" is within 1-SD of prior mean = {0:F}", priorMean); } + Console.ForegroundColor = ConsoleColor.White; } } } diff --git a/src/Vts.sln b/src/Vts.sln index 5756d6d37..b921b82fb 100644 --- a/src/Vts.sln +++ b/src/Vts.sln @@ -25,10 +25,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Vts.Benchmark", "Vts.Benchm EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution - Benchmark|Any CPU = Benchmark|Any CPU - Benchmark|Mixed Platforms = Benchmark|Mixed Platforms - Benchmark|Win32 = Benchmark|Win32 - Benchmark|x86 = Benchmark|x86 Debug|Any CPU = Debug|Any CPU Debug|Mixed Platforms = Debug|Mixed Platforms Debug|Win32 = Debug|Win32 @@ -43,10 +39,6 @@ Global ReleaseWhiteList|x86 = ReleaseWhiteList|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.Benchmark|Any CPU.ActiveCfg = Debug|Any CPU - {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.Benchmark|Mixed Platforms.ActiveCfg = Debug|Any CPU - {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.Benchmark|Win32.ActiveCfg = Debug|Any CPU - {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.Benchmark|x86.ActiveCfg = Debug|Any CPU {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.Debug|Any CPU.Build.0 = Debug|Any CPU {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU @@ -66,14 +58,6 @@ Global {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.ReleaseWhiteList|Mixed Platforms.Build.0 = Release|Any CPU {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.ReleaseWhiteList|Win32.ActiveCfg = Release|Any CPU {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.ReleaseWhiteList|x86.ActiveCfg = Release|Any CPU - {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.Benchmark|Any CPU.ActiveCfg = Benchmark|Any CPU - {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.Benchmark|Any CPU.Build.0 = Benchmark|Any CPU - {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.Benchmark|Mixed Platforms.ActiveCfg = Benchmark|Any CPU - {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.Benchmark|Mixed Platforms.Build.0 = Benchmark|Any CPU - {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.Benchmark|Win32.ActiveCfg = Benchmark|Any CPU - {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.Benchmark|Win32.Build.0 = Benchmark|Any CPU - {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.Benchmark|x86.ActiveCfg = Benchmark|Any CPU - {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.Benchmark|x86.Build.0 = Benchmark|Any CPU {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.Debug|Any CPU.Build.0 = Debug|Any CPU {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU @@ -93,14 +77,6 @@ Global {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.ReleaseWhiteList|Mixed Platforms.Build.0 = Release|Any CPU {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.ReleaseWhiteList|Win32.ActiveCfg = Release|Any CPU {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.ReleaseWhiteList|x86.ActiveCfg = Release|Any CPU - {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.Benchmark|Any CPU.ActiveCfg = Release|Any CPU - {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.Benchmark|Any CPU.Build.0 = Release|Any CPU - {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.Benchmark|Mixed Platforms.ActiveCfg = Debug|Any CPU - {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.Benchmark|Mixed Platforms.Build.0 = Debug|Any CPU - {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.Benchmark|Win32.ActiveCfg = Debug|Any CPU - {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.Benchmark|Win32.Build.0 = Debug|Any CPU - {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.Benchmark|x86.ActiveCfg = Debug|Any CPU - {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.Benchmark|x86.Build.0 = Debug|Any CPU {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.Debug|Any CPU.Build.0 = Debug|Any CPU {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU @@ -120,10 +96,6 @@ Global {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.ReleaseWhiteList|Mixed Platforms.Build.0 = Release|Any CPU {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.ReleaseWhiteList|Win32.ActiveCfg = Release|Any CPU {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.ReleaseWhiteList|x86.ActiveCfg = Release|Any CPU - {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.Benchmark|Any CPU.ActiveCfg = Debug|Any CPU - {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.Benchmark|Mixed Platforms.ActiveCfg = Debug|Any CPU - {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.Benchmark|Win32.ActiveCfg = Debug|Any CPU - {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.Benchmark|x86.ActiveCfg = Debug|Any CPU {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.Debug|Any CPU.Build.0 = Debug|Any CPU {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU @@ -141,10 +113,6 @@ Global {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.ReleaseWhiteList|Mixed Platforms.Build.0 = Release|Any CPU {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.ReleaseWhiteList|Win32.ActiveCfg = Release|Any CPU {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.ReleaseWhiteList|x86.ActiveCfg = Release|Any CPU - {36DE9584-F033-4EC0-A18F-1AE60EE55754}.Benchmark|Any CPU.ActiveCfg = Debug|Any CPU - {36DE9584-F033-4EC0-A18F-1AE60EE55754}.Benchmark|Mixed Platforms.ActiveCfg = Debug|Any CPU - {36DE9584-F033-4EC0-A18F-1AE60EE55754}.Benchmark|Win32.ActiveCfg = Debug|Any CPU - {36DE9584-F033-4EC0-A18F-1AE60EE55754}.Benchmark|x86.ActiveCfg = Debug|Any CPU {36DE9584-F033-4EC0-A18F-1AE60EE55754}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {36DE9584-F033-4EC0-A18F-1AE60EE55754}.Debug|Any CPU.Build.0 = Debug|Any CPU {36DE9584-F033-4EC0-A18F-1AE60EE55754}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU @@ -162,10 +130,6 @@ Global {36DE9584-F033-4EC0-A18F-1AE60EE55754}.ReleaseWhiteList|Mixed Platforms.Build.0 = Release|Any CPU {36DE9584-F033-4EC0-A18F-1AE60EE55754}.ReleaseWhiteList|Win32.ActiveCfg = Release|Any CPU {36DE9584-F033-4EC0-A18F-1AE60EE55754}.ReleaseWhiteList|x86.ActiveCfg = Release|Any CPU - {8A1194A0-C773-483B-BD78-9D013105CEA1}.Benchmark|Any CPU.ActiveCfg = Debug|Any CPU - {8A1194A0-C773-483B-BD78-9D013105CEA1}.Benchmark|Mixed Platforms.ActiveCfg = Debug|Any CPU - {8A1194A0-C773-483B-BD78-9D013105CEA1}.Benchmark|Win32.ActiveCfg = Debug|Any CPU - {8A1194A0-C773-483B-BD78-9D013105CEA1}.Benchmark|x86.ActiveCfg = Debug|Any CPU {8A1194A0-C773-483B-BD78-9D013105CEA1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {8A1194A0-C773-483B-BD78-9D013105CEA1}.Debug|Any CPU.Build.0 = Debug|Any CPU {8A1194A0-C773-483B-BD78-9D013105CEA1}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU @@ -183,14 +147,6 @@ Global {8A1194A0-C773-483B-BD78-9D013105CEA1}.ReleaseWhiteList|Mixed Platforms.Build.0 = Release|Any CPU {8A1194A0-C773-483B-BD78-9D013105CEA1}.ReleaseWhiteList|Win32.ActiveCfg = Release|Any CPU {8A1194A0-C773-483B-BD78-9D013105CEA1}.ReleaseWhiteList|x86.ActiveCfg = Release|Any CPU - {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.Benchmark|Any CPU.ActiveCfg = Release|Any CPU - {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.Benchmark|Any CPU.Build.0 = Release|Any CPU - {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.Benchmark|Mixed Platforms.ActiveCfg = Debug|Any CPU - {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.Benchmark|Mixed Platforms.Build.0 = Debug|Any CPU - {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.Benchmark|Win32.ActiveCfg = Debug|Any CPU - {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.Benchmark|Win32.Build.0 = Debug|Any CPU - {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.Benchmark|x86.ActiveCfg = Debug|Any CPU - {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.Benchmark|x86.Build.0 = Debug|Any CPU {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.Debug|Any CPU.Build.0 = Debug|Any CPU {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU @@ -228,7 +184,7 @@ Global {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702} = {D6A3DE7A-7E93-4932-AA8F-C612FD97830E} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - EnterpriseLibraryConfigurationToolBinariesPath = packages\Unity.2.1.505.2\lib\NET35 SolutionGuid = {BB1B236A-1BE9-476A-A4B3-8C0F51F1FDA7} + EnterpriseLibraryConfigurationToolBinariesPath = packages\Unity.2.1.505.2\lib\NET35 EndGlobalSection EndGlobal diff --git a/src/Vts/MonteCarlo/MonteCarloSimulation.cs b/src/Vts/MonteCarlo/MonteCarloSimulation.cs index 421118212..4b66a8ac0 100644 --- a/src/Vts/MonteCarlo/MonteCarloSimulation.cs +++ b/src/Vts/MonteCarlo/MonteCarloSimulation.cs @@ -7,9 +7,6 @@ using Vts.MonteCarlo.Controllers; using Vts.MonteCarlo.Extensions; using Vts.MonteCarlo.Factories; -#if BENCHMARK -using BenchmarkDotNet.Attributes; -#endif namespace Vts.MonteCarlo { @@ -179,9 +176,6 @@ public void SetOutputPathForDatabases(string outputPath) /// Run the simulation /// /// SimulationOutput -#if BENCHMARK - [Benchmark] -#endif public virtual SimulationOutput Run() { _isCancelled = false; diff --git a/src/Vts/MonteCarlo/ParallelMonteCarloSimulation.cs b/src/Vts/MonteCarlo/ParallelMonteCarloSimulation.cs index 0e9e32754..8ec932ed7 100644 --- a/src/Vts/MonteCarlo/ParallelMonteCarloSimulation.cs +++ b/src/Vts/MonteCarlo/ParallelMonteCarloSimulation.cs @@ -7,9 +7,6 @@ using Vts.Common.Logging; using Vts.IO; using Vts.MonteCarlo.Factories; -#if BENCHMARK -using BenchmarkDotNet.Attributes; -#endif namespace Vts.MonteCarlo { @@ -26,21 +23,10 @@ public class ParallelMonteCarloSimulation /// /// SimulationInput class passed in /// -#if BENCHMARK - [ParamsSource(nameof(SimulationInputs))] -#endif public SimulationInput Input { get; set; } /// /// number of CPUs /// -#if BENCHMARK - public IEnumerable SimulationInputs() - { - yield return new SimulationInput { N = 100 }; - } - - [Params(4)] -#endif public int NumberOfCPUs { get; set; } /// /// simulation statistics @@ -54,10 +40,8 @@ public IEnumerable SimulationInputs() /// number of parallel CPUs to be run public ParallelMonteCarloSimulation(SimulationInput input, int numberOfCpus) { -#if !BENCHMARK Input = input; NumberOfCPUs = numberOfCpus; -#endif } /// @@ -69,9 +53,6 @@ public ParallelMonteCarloSimulation() : this(new SimulationInput(), 2) { } /// Method to run single MC simulation in parallel /// /// array of SimulationOutput -#if BENCHMARK - [Benchmark] -#endif public SimulationOutput RunSingleInParallel() { var threads = NumberOfCPUs; diff --git a/src/Vts/Vts.csproj b/src/Vts/Vts.csproj index 710853952..1a9c45262 100644 --- a/src/Vts/Vts.csproj +++ b/src/Vts/Vts.csproj @@ -18,7 +18,7 @@ 12.2.0.0 12.2.0.0 - Debug;Release;Benchmark + Debug;Release license.md logo.png readme.md @@ -27,9 +27,6 @@ Virtual Tissue Simulator True - - true - Vts.xml @@ -124,9 +121,6 @@ - - - From 6248acf8fac5a414ffa8dd740b0fc54862b8129f Mon Sep 17 00:00:00 2001 From: Lisa Glover Date: Thu, 22 Jan 2026 19:54:26 -0800 Subject: [PATCH 02/10] Removed the ReleaseWhiteList build configuration because it is only used in the WPF application --- src/Vts.sln | 45 +-------------------------------------------- 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/src/Vts.sln b/src/Vts.sln index b921b82fb..a5813861d 100644 --- a/src/Vts.sln +++ b/src/Vts.sln @@ -33,10 +33,6 @@ Global Release|Mixed Platforms = Release|Mixed Platforms Release|Win32 = Release|Win32 Release|x86 = Release|x86 - ReleaseWhiteList|Any CPU = ReleaseWhiteList|Any CPU - ReleaseWhiteList|Mixed Platforms = ReleaseWhiteList|Mixed Platforms - ReleaseWhiteList|Win32 = ReleaseWhiteList|Win32 - ReleaseWhiteList|x86 = ReleaseWhiteList|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.Debug|Any CPU.ActiveCfg = Debug|Any CPU @@ -53,11 +49,6 @@ Global {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.Release|Win32.ActiveCfg = Release|Any CPU {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.Release|Win32.Build.0 = Release|Any CPU {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.Release|x86.ActiveCfg = Release|Any CPU - {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.ReleaseWhiteList|Any CPU.ActiveCfg = Release|Any CPU - {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.ReleaseWhiteList|Mixed Platforms.ActiveCfg = Release|Any CPU - {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.ReleaseWhiteList|Mixed Platforms.Build.0 = Release|Any CPU - {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.ReleaseWhiteList|Win32.ActiveCfg = Release|Any CPU - {8E45BF42-5B0B-4AEF-9E63-B2966622FC23}.ReleaseWhiteList|x86.ActiveCfg = Release|Any CPU {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.Debug|Any CPU.Build.0 = Debug|Any CPU {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU @@ -71,12 +62,6 @@ Global {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.Release|Mixed Platforms.Build.0 = Release|Any CPU {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.Release|Win32.ActiveCfg = Release|Any CPU {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.Release|x86.ActiveCfg = Release|Any CPU - {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.ReleaseWhiteList|Any CPU.ActiveCfg = Release|Any CPU - {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.ReleaseWhiteList|Any CPU.Build.0 = Release|Any CPU - {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.ReleaseWhiteList|Mixed Platforms.ActiveCfg = Release|Any CPU - {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.ReleaseWhiteList|Mixed Platforms.Build.0 = Release|Any CPU - {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.ReleaseWhiteList|Win32.ActiveCfg = Release|Any CPU - {DBF062BD-84EE-4ADF-ADB7-A27B10EAB13F}.ReleaseWhiteList|x86.ActiveCfg = Release|Any CPU {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.Debug|Any CPU.Build.0 = Debug|Any CPU {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU @@ -90,12 +75,6 @@ Global {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.Release|Mixed Platforms.Build.0 = Release|Any CPU {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.Release|Win32.ActiveCfg = Release|Any CPU {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.Release|x86.ActiveCfg = Release|Any CPU - {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.ReleaseWhiteList|Any CPU.ActiveCfg = Release|Any CPU - {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.ReleaseWhiteList|Any CPU.Build.0 = Release|Any CPU - {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.ReleaseWhiteList|Mixed Platforms.ActiveCfg = Release|Any CPU - {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.ReleaseWhiteList|Mixed Platforms.Build.0 = Release|Any CPU - {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.ReleaseWhiteList|Win32.ActiveCfg = Release|Any CPU - {2C5074CA-FBA8-46F1-A430-5437D67B63E5}.ReleaseWhiteList|x86.ActiveCfg = Release|Any CPU {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.Debug|Any CPU.Build.0 = Debug|Any CPU {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU @@ -108,11 +87,6 @@ Global {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.Release|Mixed Platforms.Build.0 = Release|Any CPU {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.Release|Win32.ActiveCfg = Release|Any CPU {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.Release|x86.ActiveCfg = Release|Any CPU - {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.ReleaseWhiteList|Any CPU.ActiveCfg = Release|Any CPU - {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.ReleaseWhiteList|Mixed Platforms.ActiveCfg = Release|Any CPU - {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.ReleaseWhiteList|Mixed Platforms.Build.0 = Release|Any CPU - {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.ReleaseWhiteList|Win32.ActiveCfg = Release|Any CPU - {169F5A43-1079-4E1F-87A2-C5E8E7A69BF9}.ReleaseWhiteList|x86.ActiveCfg = Release|Any CPU {36DE9584-F033-4EC0-A18F-1AE60EE55754}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {36DE9584-F033-4EC0-A18F-1AE60EE55754}.Debug|Any CPU.Build.0 = Debug|Any CPU {36DE9584-F033-4EC0-A18F-1AE60EE55754}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU @@ -125,11 +99,6 @@ Global {36DE9584-F033-4EC0-A18F-1AE60EE55754}.Release|Mixed Platforms.Build.0 = Release|Any CPU {36DE9584-F033-4EC0-A18F-1AE60EE55754}.Release|Win32.ActiveCfg = Release|Any CPU {36DE9584-F033-4EC0-A18F-1AE60EE55754}.Release|x86.ActiveCfg = Release|Any CPU - {36DE9584-F033-4EC0-A18F-1AE60EE55754}.ReleaseWhiteList|Any CPU.ActiveCfg = Release|Any CPU - {36DE9584-F033-4EC0-A18F-1AE60EE55754}.ReleaseWhiteList|Mixed Platforms.ActiveCfg = Release|Any CPU - {36DE9584-F033-4EC0-A18F-1AE60EE55754}.ReleaseWhiteList|Mixed Platforms.Build.0 = Release|Any CPU - {36DE9584-F033-4EC0-A18F-1AE60EE55754}.ReleaseWhiteList|Win32.ActiveCfg = Release|Any CPU - {36DE9584-F033-4EC0-A18F-1AE60EE55754}.ReleaseWhiteList|x86.ActiveCfg = Release|Any CPU {8A1194A0-C773-483B-BD78-9D013105CEA1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {8A1194A0-C773-483B-BD78-9D013105CEA1}.Debug|Any CPU.Build.0 = Debug|Any CPU {8A1194A0-C773-483B-BD78-9D013105CEA1}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU @@ -142,11 +111,6 @@ Global {8A1194A0-C773-483B-BD78-9D013105CEA1}.Release|Mixed Platforms.Build.0 = Release|Any CPU {8A1194A0-C773-483B-BD78-9D013105CEA1}.Release|Win32.ActiveCfg = Release|Any CPU {8A1194A0-C773-483B-BD78-9D013105CEA1}.Release|x86.ActiveCfg = Release|Any CPU - {8A1194A0-C773-483B-BD78-9D013105CEA1}.ReleaseWhiteList|Any CPU.ActiveCfg = Release|Any CPU - {8A1194A0-C773-483B-BD78-9D013105CEA1}.ReleaseWhiteList|Mixed Platforms.ActiveCfg = Release|Any CPU - {8A1194A0-C773-483B-BD78-9D013105CEA1}.ReleaseWhiteList|Mixed Platforms.Build.0 = Release|Any CPU - {8A1194A0-C773-483B-BD78-9D013105CEA1}.ReleaseWhiteList|Win32.ActiveCfg = Release|Any CPU - {8A1194A0-C773-483B-BD78-9D013105CEA1}.ReleaseWhiteList|x86.ActiveCfg = Release|Any CPU {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.Debug|Any CPU.Build.0 = Debug|Any CPU {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU @@ -163,13 +127,6 @@ Global {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.Release|Win32.Build.0 = Release|Any CPU {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.Release|x86.ActiveCfg = Release|Any CPU {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.Release|x86.Build.0 = Release|Any CPU - {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.ReleaseWhiteList|Any CPU.ActiveCfg = Release|Any CPU - {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.ReleaseWhiteList|Mixed Platforms.ActiveCfg = Release|Any CPU - {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.ReleaseWhiteList|Mixed Platforms.Build.0 = Release|Any CPU - {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.ReleaseWhiteList|Win32.ActiveCfg = Release|Any CPU - {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.ReleaseWhiteList|Win32.Build.0 = Release|Any CPU - {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.ReleaseWhiteList|x86.ActiveCfg = Release|Any CPU - {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702}.ReleaseWhiteList|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -184,7 +141,7 @@ Global {3B0676C4-5CF1-4A0B-97E4-0CDF2B035702} = {D6A3DE7A-7E93-4932-AA8F-C612FD97830E} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {BB1B236A-1BE9-476A-A4B3-8C0F51F1FDA7} EnterpriseLibraryConfigurationToolBinariesPath = packages\Unity.2.1.505.2\lib\NET35 + SolutionGuid = {BB1B236A-1BE9-476A-A4B3-8C0F51F1FDA7} EndGlobalSection EndGlobal From c47ae63f4b88dd351eab0efece6ed51e617ba253 Mon Sep 17 00:00:00 2001 From: Lisa Glover Date: Thu, 22 Jan 2026 20:57:38 -0800 Subject: [PATCH 03/10] Added a command line argument to run in parallel and set the filename accordingly. Added launch settings to allow passing the parallel argument. --- .../MonteCarloSimulationBenchmarks.cs | 2 +- .../ParallelMonteCarloSimulationBenchmarks.cs | 2 +- src/Vts.Benchmark/Program.cs | 38 +++++++++++++------ .../Properties/launchSettings.json | 8 ++++ 4 files changed, 36 insertions(+), 14 deletions(-) create mode 100644 src/Vts.Benchmark/Properties/launchSettings.json diff --git a/src/Vts.Benchmark/Benchmarks/MonteCarloSimulationBenchmarks.cs b/src/Vts.Benchmark/Benchmarks/MonteCarloSimulationBenchmarks.cs index fd55fca82..1d6594a11 100644 --- a/src/Vts.Benchmark/Benchmarks/MonteCarloSimulationBenchmarks.cs +++ b/src/Vts.Benchmark/Benchmarks/MonteCarloSimulationBenchmarks.cs @@ -12,7 +12,7 @@ public class MonteCarloSimulationBenchmarks [ParamsSource(nameof(SimulationInputs))] public SimulationInput Input { get; set; } - public IEnumerable SimulationInputs() + public static IEnumerable SimulationInputs() { yield return new SimulationInput { N = 100 }; // we can add more SimulationInput instances with different configurations if needed diff --git a/src/Vts.Benchmark/Benchmarks/ParallelMonteCarloSimulationBenchmarks.cs b/src/Vts.Benchmark/Benchmarks/ParallelMonteCarloSimulationBenchmarks.cs index 0d60d2af5..df3d29f1d 100644 --- a/src/Vts.Benchmark/Benchmarks/ParallelMonteCarloSimulationBenchmarks.cs +++ b/src/Vts.Benchmark/Benchmarks/ParallelMonteCarloSimulationBenchmarks.cs @@ -12,7 +12,7 @@ public class ParallelMonteCarloSimulationBenchmarks [ParamsSource(nameof(SimulationInputs))] public SimulationInput Input { get; set; } - public IEnumerable SimulationInputs() + public static IEnumerable SimulationInputs() { yield return new SimulationInput { N = 100 }; // we can add more SimulationInput instances with different configurations if needed diff --git a/src/Vts.Benchmark/Program.cs b/src/Vts.Benchmark/Program.cs index 68163b0af..2175fab98 100644 --- a/src/Vts.Benchmark/Program.cs +++ b/src/Vts.Benchmark/Program.cs @@ -15,13 +15,20 @@ namespace Vts.Benchmark { public static class Program { + /// + /// Set up execution of Monte Carlo Command Line application and run benchmark + /// collecting estimate of the Mean time of execution and Standard Deviation. + /// Output to CSV file. + /// Notes: 1) Build Vts.Benchmark in Release configuration + /// 2) Run with Debug tab -> Start Without Debugging + /// + /// command like parameters public static void Main(string[] args) { - // Set up execution of Monte Carlo Command Line application and run benchmark - We might need to figure this out later - // collecting estimate of the Mean time of execution and Standard Deviation. - // Output to CSV file. - // Notes: 1) Build Vts.Benchmark in Release configuration - // 2) Run with Debug tab -> Start Without Debugging + // check for -p or --parallel argument to run parallel benchmark + var runInParallel = false || (args.Length > 0 && (args[0] == "-p" || args[0] == "--parallel")); + + // configure BenchmarkDotNet var config = new ManualConfig() .AddJob(new Job("Benchmark")) .AddLogger(ConsoleLogger.Default) @@ -33,16 +40,23 @@ public static void Main(string[] args) .AddExporter(CsvExporter.Default, HtmlExporter.Default, MarkdownExporter.GitHub) .AddAnalyser(EnvironmentAnalyser.Default); config.UnionRule = ConfigUnionRule.Union; - // This line is used to run the non-parallel benchmark, we could add a parameter to switch between them - var summary = BenchmarkRunner.Run(config); - // This line is used to run the parallel benchmark - //var summary = BenchmarkRunner.Run(config); + + // run the benchmark + var summary = runInParallel ? + // This line is used to run the parallel benchmark + BenchmarkRunner.Run(config) : + // This line is used to run the non-parallel benchmark + BenchmarkRunner.Run(config); + Console.WriteLine(summary); // Read CSV file for Mean and Standard Deviation (StDev) data var inputPath = Path.GetFullPath(Directory.GetCurrentDirectory()); - const string csvFile = "\\BenchmarkDotNet.Artifacts\\results\\Vts.Benchmark.Benchmarks.MonteCarloSimulationBenchmarks-report.csv"; - var filePath = Path.GetFullPath(inputPath + csvFile); - using var reader = new StreamReader(filePath); + const string filePath = "\\BenchmarkDotNet.Artifacts\\results\\"; + var csvFile = runInParallel ? + $"{typeof(ParallelMonteCarloSimulationBenchmarks).FullName}-report.csv" : + $"{typeof(MonteCarloSimulationBenchmarks).FullName}-report.csv"; + var fullPath = Path.GetFullPath(inputPath + filePath + csvFile); + using var reader = new StreamReader(fullPath); // read header line reader.ReadLine(); // read data line diff --git a/src/Vts.Benchmark/Properties/launchSettings.json b/src/Vts.Benchmark/Properties/launchSettings.json new file mode 100644 index 000000000..973e821e5 --- /dev/null +++ b/src/Vts.Benchmark/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "Vts.Benchmark": { + "commandName": "Project", + "commandLineArgs": "" //-p or -parallel to run benchmarks in parallel + } + } +} \ No newline at end of file From 405b16dc2c6cedacb04ddeeb41f9c78d067461ff Mon Sep 17 00:00:00 2001 From: Lisa Glover Date: Thu, 22 Jan 2026 21:13:33 -0800 Subject: [PATCH 04/10] Fixed a typo --- src/Vts.Benchmark/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Vts.Benchmark/Program.cs b/src/Vts.Benchmark/Program.cs index 2175fab98..3d03950b1 100644 --- a/src/Vts.Benchmark/Program.cs +++ b/src/Vts.Benchmark/Program.cs @@ -22,7 +22,7 @@ public static class Program /// Notes: 1) Build Vts.Benchmark in Release configuration /// 2) Run with Debug tab -> Start Without Debugging /// - /// command like parameters + /// command line parameters public static void Main(string[] args) { // check for -p or --parallel argument to run parallel benchmark From 468d6ae43ee071c21a6077da3a4765ce2ca0aeb5 Mon Sep 17 00:00:00 2001 From: Lisa Glover Date: Fri, 23 Jan 2026 13:45:26 -0800 Subject: [PATCH 05/10] Updated the launch settings to remove the comments and set up 2 launch profiles --- src/Vts.Benchmark/Properties/launchSettings.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Vts.Benchmark/Properties/launchSettings.json b/src/Vts.Benchmark/Properties/launchSettings.json index 973e821e5..9400096d7 100644 --- a/src/Vts.Benchmark/Properties/launchSettings.json +++ b/src/Vts.Benchmark/Properties/launchSettings.json @@ -1,8 +1,11 @@ { "profiles": { - "Vts.Benchmark": { - "commandName": "Project", - "commandLineArgs": "" //-p or -parallel to run benchmarks in parallel + "BenchmarkMonteCarlo": { + "commandName": "Project" + }, + "BenchmarkMonteCarloParallel": { + "commandLineArgs": "-p", + "commandName": "Project" } } } \ No newline at end of file From 124e3d003e24b981bc69d77dd2cf12ea64eb0253 Mon Sep 17 00:00:00 2001 From: hayakawa Date: Fri, 23 Jan 2026 14:25:33 -0800 Subject: [PATCH 06/10] Added priorMean for parallelMonteCarlo benchmarking, modified check to see if calculated mean is within 3-standard deviations of the prior mean (should happend 99.7% of the time), and modified header to indicate pull down now enables running Monte Carlo or parallel Monte Carlo in benchmark. A little code clean up. --- src/Vts.Benchmark/Program.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Vts.Benchmark/Program.cs b/src/Vts.Benchmark/Program.cs index 3d03950b1..0bac0553d 100644 --- a/src/Vts.Benchmark/Program.cs +++ b/src/Vts.Benchmark/Program.cs @@ -20,13 +20,14 @@ public static class Program /// collecting estimate of the Mean time of execution and Standard Deviation. /// Output to CSV file. /// Notes: 1) Build Vts.Benchmark in Release configuration - /// 2) Run with Debug tab -> Start Without Debugging + /// 2) Pull down enables running BenchmarkMonteCarlo or BenchmarkMonteCarloParallel + /// 3) Run with Debug tab -> Start Without Debugging /// /// command line parameters public static void Main(string[] args) { // check for -p or --parallel argument to run parallel benchmark - var runInParallel = false || (args.Length > 0 && (args[0] == "-p" || args[0] == "--parallel")); + var runInParallel = args.Length > 0 && (args[0] == "-p" || args[0] == "--parallel"); // configure BenchmarkDotNet var config = new ManualConfig() @@ -67,20 +68,20 @@ public static void Main(string[] args) var mean = double.Parse(Regex.Match(values[1], @"-?\d+(?:\.\d+)?").Value); // has 'ms' appended var standardDeviation = double.Parse(Regex.Match(values[3], @"-?\d+(?:\.\d+)?").Value); // has 'ms' appended // write out result of current run and standard deviation compared against prior mean - const double priorMean = 84.0; // ms + var priorMean = runInParallel ? 8.25 : 84.0; // ms // output 1 sigma results Console.ForegroundColor = ConsoleColor.Cyan; - Console.Write("SUMMARY: (Mean = {0:F} ms) +/- (SD = {1:F} ms)", mean, standardDeviation); - // output if result is larger than established mean + 1-SD - if (mean > standardDeviation + priorMean) + Console.Write("SUMMARY: (Mean = {0:F} ms) +/- (3*SD = {1:F} ms)", mean, 3 *standardDeviation); + // check if mean is within 3 standard deviation about the prior mean + if (mean < priorMean - 3 * standardDeviation || mean > priorMean + 3* standardDeviation) { Console.ForegroundColor = ConsoleColor.Red; - Console.WriteLine(" is 1-SD > prior mean = {0:F}", priorMean); + Console.WriteLine(" is not within 3-SD of prior mean = {0:F}", priorMean); } else { Console.ForegroundColor = ConsoleColor.Cyan; - Console.WriteLine(" is within 1-SD of prior mean = {0:F}", priorMean); + Console.WriteLine(" is within 3-SD of prior mean = {0:F}", priorMean); } Console.ForegroundColor = ConsoleColor.White; } From e6c52325d6c8f0367038aee5cf2dfa2dfde94e1b Mon Sep 17 00:00:00 2001 From: Lisa Glover Date: Tue, 27 Jan 2026 17:20:05 -0800 Subject: [PATCH 07/10] Added code to pull the previous mean value if there is an existing CSV file, for comparison with the new mean. Created helper methods so they could be accessed multiple times and catch any errors if the CSV file is missing or open. --- src/Helper.cs | 74 ++++++++++++++ src/Vts.Benchmark/Helpers/CsvTools.cs | 90 ++++++++++++++++ src/Vts.Benchmark/Program.cs | 141 ++++++++++++++------------ 3 files changed, 238 insertions(+), 67 deletions(-) create mode 100644 src/Helper.cs create mode 100644 src/Vts.Benchmark/Helpers/CsvTools.cs diff --git a/src/Helper.cs b/src/Helper.cs new file mode 100644 index 000000000..ff2891fa0 --- /dev/null +++ b/src/Helper.cs @@ -0,0 +1,74 @@ +using System; +using System.IO; +using System.Text.RegularExpressions; + + +public static class Helpers +{ + /// + /// Constructs the full path to the CSV file generated by BenchmarkDotNet based on the execution mode. + /// + /// + /// A boolean indicating whether the benchmark is running in parallel mode. + /// If true, the path for the parallel benchmark CSV file is returned. + /// Otherwise, the path for the non-parallel benchmark CSV file is returned. + /// + /// + /// The full path to the CSV file containing the benchmark results. + /// + private static string GetCsvFullPath(bool runInParallel) + { + var inputPath = Path.GetFullPath(Directory.GetCurrentDirectory()); + const string filePath = "\\BenchmarkDotNet.Artifacts\\results\\"; + var csvFile = runInParallel ? + $"{typeof(ParallelMonteCarloSimulationBenchmarks).FullName}-report.csv" : + $"{typeof(MonteCarloSimulationBenchmarks).FullName}-report.csv"; + return Path.GetFullPath(inputPath + filePath + csvFile); + } + + /// + /// Reads values from a CSV file located at the specified path. + /// + /// The full path to the CSV file. + /// + /// An array of strings representing the values read from the CSV file, or null + /// if the file does not exist. + /// + private static string[] ReadValuesFromCsv(string fullPath) + { + if (!File.Exists(fullPath)) + { + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine(@"CSV results file not found: {0}", fullPath); + Console.ForegroundColor = ConsoleColor.White; + return null; + } + + using var reader = new StreamReader(fullPath); + // read header line + reader.ReadLine(); + // read data line + var line = reader.ReadLine(); + var values = line?.Split(','); + return values; + } + + + /// + /// Extracts a double value from a specified index in an array of strings. + /// + /// The array of strings containing the values to parse. + /// The index of the value to extract within the array. + /// + /// The parsed double value if successful; otherwise, null if the value cannot be parsed + /// or if the index is out of bounds. + /// + private static double? ReadValue(string[] values, int index) + { + if (values.Length < 4) return null; + // excise double from string of type "###.## ms" + var valueMatch = Regex.Match(values[index], @"-?\d+(?:\.\d+)?").Value; + if (!double.TryParse(valueMatch, out var value)) return null; + return value; + } +} diff --git a/src/Vts.Benchmark/Helpers/CsvTools.cs b/src/Vts.Benchmark/Helpers/CsvTools.cs new file mode 100644 index 000000000..2d0953c81 --- /dev/null +++ b/src/Vts.Benchmark/Helpers/CsvTools.cs @@ -0,0 +1,90 @@ +using System; +using System.IO; +using System.Text.RegularExpressions; +using Vts.Benchmark.Benchmarks; + +namespace Vts.Benchmark.Helpers; + +/// +/// Provides utility methods and helper functions for benchmarking tasks +/// within the Vts.Benchmark.Helpers namespace. +/// +public static class CsvTools +{ + /// + /// Constructs the full path to the CSV file generated by BenchmarkDotNet based on the execution mode. + /// + /// + /// A boolean indicating whether the benchmark is running in parallel mode. + /// If true, the path for the parallel benchmark CSV file is returned. + /// Otherwise, the path for the non-parallel benchmark CSV file is returned. + /// + /// + /// The full path to the CSV file containing the benchmark results. + /// + internal static string GetCsvFullPath(bool runInParallel) + { + var inputPath = Path.GetFullPath(Directory.GetCurrentDirectory()); + const string filePath = "\\BenchmarkDotNet.Artifacts\\results\\"; + var csvFile = runInParallel ? + $"{typeof(ParallelMonteCarloSimulationBenchmarks).FullName}-report.csv" : + $"{typeof(MonteCarloSimulationBenchmarks).FullName}-report.csv"; + return Path.GetFullPath(inputPath + filePath + csvFile); + } + + /// + /// Reads values from a CSV file located at the specified path. + /// + /// The full path to the CSV file. + /// + /// An array of strings representing the values read from the CSV file, or null + /// if the file does not exist. + /// + internal static string[] ReadValuesFromCsv(string fullPath) + { + if (!File.Exists(fullPath)) + { + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine(@"CSV results file not found: {0}", fullPath); + Console.ForegroundColor = ConsoleColor.White; + return null; + } + + try + { + using var reader = new StreamReader(fullPath); + // read header line + reader.ReadLine(); + // read data line + var line = reader.ReadLine(); + var values = line?.Split(','); + return values; + } + catch (Exception e) + { + Console.ForegroundColor = ConsoleColor.DarkRed; + Console.WriteLine(@"Unable to access the CSV file: {0} {1}", fullPath, e.Message); + Console.ForegroundColor = ConsoleColor.White; + return null; + } + } + + + /// + /// Extracts a double value from a specified index in an array of strings. + /// + /// The array of strings containing the values to parse. + /// The index of the value to extract within the array. + /// + /// The parsed double value if successful; otherwise, null if the value cannot be parsed + /// or if the index is out of bounds. + /// + internal static double? ReadValue(string[] values, int index) + { + if (values.Length < 4) return null; + // excise double from string of type "###.## ms" + var valueMatch = Regex.Match(values[index], @"-?\d+(?:\.\d+)?").Value; + if (!double.TryParse(valueMatch, out var value)) return null; + return value; + } +} diff --git a/src/Vts.Benchmark/Program.cs b/src/Vts.Benchmark/Program.cs index 0bac0553d..9cadbc541 100644 --- a/src/Vts.Benchmark/Program.cs +++ b/src/Vts.Benchmark/Program.cs @@ -7,83 +7,90 @@ using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; using System; -using System.IO; -using System.Text.RegularExpressions; using Vts.Benchmark.Benchmarks; -namespace Vts.Benchmark +namespace Vts.Benchmark; + +public static class Program { - public static class Program + /// + /// Set up execution of Monte Carlo Command Line application and run benchmark + /// collecting estimate of the Mean time of execution and Standard Deviation. + /// Output to CSV file. + /// Notes: 1) Build Vts.Benchmark in Release configuration + /// 2) Pull down enables running BenchmarkMonteCarlo or BenchmarkMonteCarloParallel + /// 3) Run with Debug tab -> Start Without Debugging + /// + /// command line parameters + public static void Main(string[] args) { - /// - /// Set up execution of Monte Carlo Command Line application and run benchmark - /// collecting estimate of the Mean time of execution and Standard Deviation. - /// Output to CSV file. - /// Notes: 1) Build Vts.Benchmark in Release configuration - /// 2) Pull down enables running BenchmarkMonteCarlo or BenchmarkMonteCarloParallel - /// 3) Run with Debug tab -> Start Without Debugging - /// - /// command line parameters - public static void Main(string[] args) - { - // check for -p or --parallel argument to run parallel benchmark - var runInParallel = args.Length > 0 && (args[0] == "-p" || args[0] == "--parallel"); + // check for -p or --parallel argument to run parallel benchmark + var runInParallel = args.Length > 0 && (args[0] == "-p" || args[0] == "--parallel"); - // configure BenchmarkDotNet - var config = new ManualConfig() - .AddJob(new Job("Benchmark")) - .AddLogger(ConsoleLogger.Default) - .AddColumn(TargetMethodColumn.Method, - StatisticColumn.Mean, - StatisticColumn.Error, - StatisticColumn.StdDev, - StatisticColumn.Median) - .AddExporter(CsvExporter.Default, HtmlExporter.Default, MarkdownExporter.GitHub) - .AddAnalyser(EnvironmentAnalyser.Default); - config.UnionRule = ConfigUnionRule.Union; + // configure BenchmarkDotNet + var config = new ManualConfig() + .AddJob(new Job("Benchmark")) + .AddLogger(ConsoleLogger.Default) + .AddColumn(TargetMethodColumn.Method, + StatisticColumn.Mean, + StatisticColumn.Error, + StatisticColumn.StdDev, + StatisticColumn.Median) + .AddExporter(CsvExporter.Default, HtmlExporter.Default, MarkdownExporter.GitHub) + .AddAnalyser(EnvironmentAnalyser.Default); + config.UnionRule = ConfigUnionRule.Union; - // run the benchmark - var summary = runInParallel ? - // This line is used to run the parallel benchmark - BenchmarkRunner.Run(config) : - // This line is used to run the non-parallel benchmark - BenchmarkRunner.Run(config); - - Console.WriteLine(summary); - // Read CSV file for Mean and Standard Deviation (StDev) data - var inputPath = Path.GetFullPath(Directory.GetCurrentDirectory()); - const string filePath = "\\BenchmarkDotNet.Artifacts\\results\\"; - var csvFile = runInParallel ? - $"{typeof(ParallelMonteCarloSimulationBenchmarks).FullName}-report.csv" : - $"{typeof(MonteCarloSimulationBenchmarks).FullName}-report.csv"; - var fullPath = Path.GetFullPath(inputPath + filePath + csvFile); - using var reader = new StreamReader(fullPath); - // read header line - reader.ReadLine(); - // read data line - var line = reader.ReadLine(); - if (line == null) return; - var values = line.Split(','); - // excise double from string of type "###.## ms" - var mean = double.Parse(Regex.Match(values[1], @"-?\d+(?:\.\d+)?").Value); // has 'ms' appended - var standardDeviation = double.Parse(Regex.Match(values[3], @"-?\d+(?:\.\d+)?").Value); // has 'ms' appended - // write out result of current run and standard deviation compared against prior mean - var priorMean = runInParallel ? 8.25 : 84.0; // ms - // output 1 sigma results - Console.ForegroundColor = ConsoleColor.Cyan; - Console.Write("SUMMARY: (Mean = {0:F} ms) +/- (3*SD = {1:F} ms)", mean, 3 *standardDeviation); - // check if mean is within 3 standard deviation about the prior mean - if (mean < priorMean - 3 * standardDeviation || mean > priorMean + 3* standardDeviation) + // If the previous CSV file exists, get the prior mean using helper methods + var fullPath = Helpers.CsvTools.GetCsvFullPath(runInParallel); + var values = Helpers.CsvTools.ReadValuesFromCsv(fullPath); + // write out result of current run and standard deviation compared against prior mean + var priorMean = runInParallel ? 8.25 : 84.0; // put default values here in case there is no file + if (values != null) + { + var meanValue = Helpers.CsvTools.ReadValue(values, 1); // the mean is the 2nd value + if (meanValue == null) { - Console.ForegroundColor = ConsoleColor.Red; - Console.WriteLine(" is not within 3-SD of prior mean = {0:F}", priorMean); + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine(@"Could not read results from file"); + Console.ForegroundColor = ConsoleColor.White; } else { - Console.ForegroundColor = ConsoleColor.Cyan; - Console.WriteLine(" is within 3-SD of prior mean = {0:F}", priorMean); + priorMean = (double)meanValue; + Console.WriteLine(@"Prior mean is {0:F}", priorMean); } - Console.ForegroundColor = ConsoleColor.White; } + + // run the benchmark + var summary = runInParallel ? + // This line is used to run the parallel benchmark + BenchmarkRunner.Run(config) : + // This line is used to run the non-parallel benchmark + BenchmarkRunner.Run(config); + + Console.WriteLine(summary); + + // Read CSV file for Mean and Standard Deviation (StDev) data using helper methods + fullPath = Helpers.CsvTools.GetCsvFullPath(runInParallel); + values = Helpers.CsvTools.ReadValuesFromCsv(fullPath); + if (values == null) return; + var mean = Helpers.CsvTools.ReadValue(values, 1); // the mean is the 2nd value + var standardDeviation = Helpers.CsvTools.ReadValue(values, 3); //the standard deviation is the 4th value + + // output 1 sigma results + Console.ForegroundColor = ConsoleColor.Cyan; + Console.Write(@"SUMMARY: (Mean = {0:F} ms) +/- (3*SD = {1:F} ms)", mean, 3 * standardDeviation); + // check if mean is within 3 standard deviation about the prior mean + if (mean < priorMean - 3 * standardDeviation || mean > priorMean + 3 * standardDeviation) + { + Console.ForegroundColor = ConsoleColor.DarkRed; + Console.WriteLine(@" is not within 3-SD of prior mean = {0:F}", priorMean); + } + else + { + Console.ForegroundColor = ConsoleColor.Cyan; + Console.WriteLine(@" is within 3-SD of prior mean = {0:F}", priorMean); + } + Console.ForegroundColor = ConsoleColor.White; } -} +} \ No newline at end of file From 677d4ac368e8aef610fafd386206c01b224b5d49 Mon Sep 17 00:00:00 2001 From: Lisa Glover Date: Tue, 27 Jan 2026 17:26:30 -0800 Subject: [PATCH 08/10] Removed the helper file that was in the wrong place --- src/Helper.cs | 74 --------------------------------------------------- 1 file changed, 74 deletions(-) delete mode 100644 src/Helper.cs diff --git a/src/Helper.cs b/src/Helper.cs deleted file mode 100644 index ff2891fa0..000000000 --- a/src/Helper.cs +++ /dev/null @@ -1,74 +0,0 @@ -using System; -using System.IO; -using System.Text.RegularExpressions; - - -public static class Helpers -{ - /// - /// Constructs the full path to the CSV file generated by BenchmarkDotNet based on the execution mode. - /// - /// - /// A boolean indicating whether the benchmark is running in parallel mode. - /// If true, the path for the parallel benchmark CSV file is returned. - /// Otherwise, the path for the non-parallel benchmark CSV file is returned. - /// - /// - /// The full path to the CSV file containing the benchmark results. - /// - private static string GetCsvFullPath(bool runInParallel) - { - var inputPath = Path.GetFullPath(Directory.GetCurrentDirectory()); - const string filePath = "\\BenchmarkDotNet.Artifacts\\results\\"; - var csvFile = runInParallel ? - $"{typeof(ParallelMonteCarloSimulationBenchmarks).FullName}-report.csv" : - $"{typeof(MonteCarloSimulationBenchmarks).FullName}-report.csv"; - return Path.GetFullPath(inputPath + filePath + csvFile); - } - - /// - /// Reads values from a CSV file located at the specified path. - /// - /// The full path to the CSV file. - /// - /// An array of strings representing the values read from the CSV file, or null - /// if the file does not exist. - /// - private static string[] ReadValuesFromCsv(string fullPath) - { - if (!File.Exists(fullPath)) - { - Console.ForegroundColor = ConsoleColor.Yellow; - Console.WriteLine(@"CSV results file not found: {0}", fullPath); - Console.ForegroundColor = ConsoleColor.White; - return null; - } - - using var reader = new StreamReader(fullPath); - // read header line - reader.ReadLine(); - // read data line - var line = reader.ReadLine(); - var values = line?.Split(','); - return values; - } - - - /// - /// Extracts a double value from a specified index in an array of strings. - /// - /// The array of strings containing the values to parse. - /// The index of the value to extract within the array. - /// - /// The parsed double value if successful; otherwise, null if the value cannot be parsed - /// or if the index is out of bounds. - /// - private static double? ReadValue(string[] values, int index) - { - if (values.Length < 4) return null; - // excise double from string of type "###.## ms" - var valueMatch = Regex.Match(values[index], @"-?\d+(?:\.\d+)?").Value; - if (!double.TryParse(valueMatch, out var value)) return null; - return value; - } -} From 4702e5b937809451e6aa215ee2b6f51976a1f4dd Mon Sep 17 00:00:00 2001 From: hayakawa Date: Wed, 28 Jan 2026 11:43:06 -0800 Subject: [PATCH 09/10] Settled on displaying 2-Standard Deviation results because this seems to be the norm given it is 95.4%. Also added header Notes item to describe the use of prior run mean data as the validation mean if it exists, otherwise in prior mean defined in code is used. --- src/Vts.Benchmark/Program.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Vts.Benchmark/Program.cs b/src/Vts.Benchmark/Program.cs index 9cadbc541..c4b22894a 100644 --- a/src/Vts.Benchmark/Program.cs +++ b/src/Vts.Benchmark/Program.cs @@ -20,6 +20,8 @@ public static class Program /// Notes: 1) Build Vts.Benchmark in Release configuration /// 2) Pull down enables running BenchmarkMonteCarlo or BenchmarkMonteCarloParallel /// 3) Run with Debug tab -> Start Without Debugging + /// 4) Prior run mean data (in BenchmarkDotNet.Artifacts) is used as validation mean. + /// If no prior run data, prior mean set in code is used as validation mean. /// /// command line parameters public static void Main(string[] args) @@ -79,17 +81,17 @@ public static void Main(string[] args) // output 1 sigma results Console.ForegroundColor = ConsoleColor.Cyan; - Console.Write(@"SUMMARY: (Mean = {0:F} ms) +/- (3*SD = {1:F} ms)", mean, 3 * standardDeviation); - // check if mean is within 3 standard deviation about the prior mean - if (mean < priorMean - 3 * standardDeviation || mean > priorMean + 3 * standardDeviation) + Console.Write(@"SUMMARY: (Mean = {0:F} ms) +/- (2-SD = {1:F} ms)", mean, 2 * standardDeviation); + // check if mean is within 1 standard deviation about the prior mean + if (mean < priorMean - 2* standardDeviation || mean > priorMean + 2 * standardDeviation) { Console.ForegroundColor = ConsoleColor.DarkRed; - Console.WriteLine(@" is not within 3-SD of prior mean = {0:F}", priorMean); + Console.WriteLine(@" is not within 2-SD (95.4%) of prior mean = {0:F}", priorMean); } else { Console.ForegroundColor = ConsoleColor.Cyan; - Console.WriteLine(@" is within 3-SD of prior mean = {0:F}", priorMean); + Console.WriteLine(@" is within 2-SD (95.4%) of prior mean = {0:F}", priorMean); } Console.ForegroundColor = ConsoleColor.White; } From 8daa12b31270346d710830be7dc0f8d36b99adde Mon Sep 17 00:00:00 2001 From: hayakawa Date: Wed, 28 Jan 2026 12:37:30 -0800 Subject: [PATCH 10/10] Modified the number photons N=10,000 to give more reasonable timing estimates. The units of both the mean values for the single MC (7.76ms) and parallel MC (4.35ms) are now in ms and make more sense. Curiously the single takes about 10 wallclock minutes to run and the parallel runs in under a minute. Not sure what overhead the benchmark software is doing to cause this difference. But since the estimated mean values are in comparable units, the results are more trusted. --- src/Vts.Benchmark/Benchmarks/MonteCarloSimulationBenchmarks.cs | 2 +- .../Benchmarks/ParallelMonteCarloSimulationBenchmarks.cs | 2 +- src/Vts.Benchmark/Program.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Vts.Benchmark/Benchmarks/MonteCarloSimulationBenchmarks.cs b/src/Vts.Benchmark/Benchmarks/MonteCarloSimulationBenchmarks.cs index 1d6594a11..5e8b6c664 100644 --- a/src/Vts.Benchmark/Benchmarks/MonteCarloSimulationBenchmarks.cs +++ b/src/Vts.Benchmark/Benchmarks/MonteCarloSimulationBenchmarks.cs @@ -14,7 +14,7 @@ public class MonteCarloSimulationBenchmarks public static IEnumerable SimulationInputs() { - yield return new SimulationInput { N = 100 }; + yield return new SimulationInput { N = 10000 }; // we can add more SimulationInput instances with different configurations if needed } diff --git a/src/Vts.Benchmark/Benchmarks/ParallelMonteCarloSimulationBenchmarks.cs b/src/Vts.Benchmark/Benchmarks/ParallelMonteCarloSimulationBenchmarks.cs index df3d29f1d..34ab0aa35 100644 --- a/src/Vts.Benchmark/Benchmarks/ParallelMonteCarloSimulationBenchmarks.cs +++ b/src/Vts.Benchmark/Benchmarks/ParallelMonteCarloSimulationBenchmarks.cs @@ -14,7 +14,7 @@ public class ParallelMonteCarloSimulationBenchmarks public static IEnumerable SimulationInputs() { - yield return new SimulationInput { N = 100 }; + yield return new SimulationInput { N = 10000 }; // we can add more SimulationInput instances with different configurations if needed } diff --git a/src/Vts.Benchmark/Program.cs b/src/Vts.Benchmark/Program.cs index c4b22894a..5b27f3ff5 100644 --- a/src/Vts.Benchmark/Program.cs +++ b/src/Vts.Benchmark/Program.cs @@ -46,7 +46,7 @@ public static void Main(string[] args) var fullPath = Helpers.CsvTools.GetCsvFullPath(runInParallel); var values = Helpers.CsvTools.ReadValuesFromCsv(fullPath); // write out result of current run and standard deviation compared against prior mean - var priorMean = runInParallel ? 8.25 : 84.0; // put default values here in case there is no file + var priorMean = runInParallel ? 4.35 : 7.76; // [ms] put default values here in case there is no file if (values != null) { var meanValue = Helpers.CsvTools.ReadValue(values, 1); // the mean is the 2nd value