* [wasm] Add console version of browser-bench sample
Example run:
> v8 --expose_wasm runtime.js -- --run Wasm.Console.Bench.Sample.dll -t Json:non,Exceptions:Inline
console.info: Arguments: --run,Wasm.Console.Bench.Sample.dll,-t,Json:non,Exceptions:Inline
console.debug: MONO_WASM: Initializing mono runtime
console.debug: MONO_WASM: ICU data archive(s) loaded, disabling invariant mode
console.debug: mono_wasm_runtime_ready
fe00e07a-5519-4dfe-b35a-
f867dbaf2e28
console.info: Initializing.....
Benchmark started
Json, non-ASCII text serialize count: 23, per call: 7.565217391304348ms, total: 0.174s
...
Exceptions, TryCatchFilterInline count: 1766666, per call: 4.
301888415806949E-05ms, total: 0.076s
Summary
Json, non-ASCII text serialize: 7.478260869565218ms
Json, non-ASCII text deserialize: 10.789473684210526ms
Exceptions, TryCatchFilterInline: 4.
301888415806949E-05ms
.md
| measurement | time |
|-:|-:|
| Json, non-ASCII text serialize | 7.4783ms |
| Json, non-ASCII text deserialize | 10.7895ms |
| Exceptions, TryCatchFilterInline | 0.0430us |
* Don't generate v8 script for browser project
* Fix option set
* Changes for linux/mac
* Add link to bench sample README
* Pass ARGS to the console version
Add info about filtering and options to the README
* Show how the table will looks like
* Do not sign the assembly
To avoid
CSC : error CS8002: Referenced assembly 'Mono.Options, Version=6.0.0.0, Culture=neutral, PublicKeyToken=null' does not have a strong name.
* Add examples how to run it on windows
* Fix interpolated string
* Improve formatting
* Remove unwanted space
<Exec Command="$(_Dotnet) publish /p:Configuration=$(Configuration) /p:TargetArchitecture=wasm /p:TargetOS=Browser $(_AOTFlag) $(_SampleProject)" />
</Target>
<Target Name="RunSampleWithV8" DependsOnTargets="BuildSampleInTree">
- <Exec Command="cd bin/$(Configuration)/AppBundle && v8 --expose_wasm runtime.js -- $(DOTNET_MONO_LOG_LEVEL) --run Wasm.Console.Sample.dll" IgnoreExitCode="true" />
+ <Exec WorkingDirectory="bin/$(Configuration)/AppBundle" Command="v8 --expose_wasm runtime.js -- $(DOTNET_MONO_LOG_LEVEL) --run $(_SampleAssembly) $(Args)" IgnoreExitCode="true" />
</Target>
<Target Name="CheckServe">
<Exec Command="dotnet tool install -g dotnet-serve" IgnoreExitCode="true" />
readonly List<Result> results = new();
public Regex pattern;
+ public virtual bool BrowserOnly => false;
+
public async Task<string> RunBatch(List<Result> results, int measurementIdx, int milliseconds = 5000)
{
var measurement = Measurements[measurementIdx];
--- /dev/null
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Sample
+{
+ public partial class Test
+ {
+ public static int Main(string[] args)
+ {
+ return 0;
+ }
+ }
+}
--- /dev/null
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+using Mono.Options;
+
+namespace Sample
+{
+ public partial class Test
+ {
+ static string tasksArg;
+
+ static List<string> ProcessArguments(string[] args)
+ {
+ var help = false;
+ var options = new OptionSet {
+ "Simple mono wasm benchmark",
+ "",
+ "Copyright 2021 Microsoft Corporation",
+ "",
+ "Options:",
+ { "h|help|?",
+ "Show this message and exit",
+ v => help = v != null },
+ { "t|tasks=",
+ "Filter comma separated tasks and its measurements matching, TASK[:REGEX][,TASK[:REGEX],...]. Example: -t Json:non,Exceptions:Inline",
+ v => tasksArg = v },
+ };
+
+ var remaining = options.Parse(args);
+
+ if (help || remaining.Count > 0)
+ {
+ options.WriteOptionDescriptions(Console.Out);
+
+ Environment.Exit(0);
+ }
+
+ return remaining;
+ }
+
+ public static async Task<int> Main(string[] args)
+ {
+ ProcessArguments(args);
+
+ if (tasksArg != null)
+ SetTasks(tasksArg);
+
+ string output;
+
+ instance.formatter = new PlainFormatter();
+ instance.tasks.RemoveAll(t => t.BrowserOnly);
+
+ if (instance.tasks.Count < 1)
+ {
+ Console.WriteLine("No task(s) to run");
+ Environment.Exit(0);
+ }
+
+ do
+ {
+ output = await instance.RunTasks();
+ Console.Write(output);
+ } while (!string.IsNullOrEmpty(output));
+
+ return 0;
+ }
+ }
+}
--- /dev/null
+TOP=../../../../../..
+
+include ../../wasm.mk
+
+ifneq ($(AOT),)
+override MSBUILD_ARGS+=/p:RunAOTCompilation=true
+endif
+
+PROJECT_NAME=Wasm.Console.Bench.Sample.csproj
+CONSOLE_DLL=Wasm.Console.Bench.Sample.dll
+
+run: run-console
--- /dev/null
+<Project Sdk="Microsoft.NET.Sdk">
+ <PropertyGroup>
+ <WasmCopyAppZipToHelixTestDir Condition="'$(ArchiveTests)' == 'true'">true</WasmCopyAppZipToHelixTestDir>
+ <WasmMainJSPath>$(MonoProjectRoot)\wasm\runtime-test.js</WasmMainJSPath>
+ <WasmGenerateRunV8Script>true</WasmGenerateRunV8Script>
+ <SuppressTrimAnalysisWarnings>true</SuppressTrimAnalysisWarnings>
+ </PropertyGroup>
+
+ <PropertyGroup>
+ <_SampleProject>Wasm.Console.Bench.Sample.csproj</_SampleProject>
+ <_SampleAssembly>Wasm.Console.Bench.Sample.dll</_SampleAssembly>
+ <SignAssembly>False</SignAssembly>
+ </PropertyGroup>
+
+ <Target Name="RunSample" DependsOnTargets="RunSampleWithV8" />
+
+ <ItemGroup>
+ <Compile Include="../*.cs" />
+ <Compile Remove="../Browser.cs" />
+ <PackageReference Include="Mono.Options" Version="6.12.0.148" />
+ </ItemGroup>
+</Project>
namespace Sample
{
- public class Test
+ public partial class Test
{
- public static void Main(string[] args)
- {
- }
-
- BenchTask[] tasks =
+ List<BenchTask> tasks = new()
{
new ExceptionsTask(),
new JsonTask (),
new WebSocketTask()
};
- static Test instance = new Test ();
+ static Test instance = new Test();
+ Formatter formatter = new HTMLFormatter();
[MethodImpl(MethodImplOptions.NoInlining)]
public static Task<string> RunBenchmark()
{
- return instance.RunTasks ();
+ return instance.RunTasks();
}
[MethodImpl(MethodImplOptions.NoInlining)]
var idx = names[i].IndexOf(':');
string name;
- if (idx == -1) {
+ if (idx == -1)
+ {
name = names[i];
pattern = null;
}
else
{
name = names[i].Substring(0, idx);
- pattern = new Regex (names[i][(idx + 1)..]);
+ pattern = new Regex(names[i][(idx + 1)..]);
}
var taskType = Type.GetType($"Sample.{name}Task");
var task = (BenchTask)Activator.CreateInstance(taskType);
task.pattern = pattern;
- tasksList.Add (task);
+ tasksList.Add(task);
}
- instance.tasks = tasksList.ToArray ();
+ instance.tasks = tasksList;
}
int taskCounter = 0;
List<BenchTask.Result> results = new();
bool resultsReturned;
- bool NextTask ()
+ bool NextTask()
{
bool hasMeasurement;
- do {
- if (taskCounter == tasks.Length)
+ do
+ {
+ if (taskCounter == tasks.Count)
return false;
Task = tasks[taskCounter];
return true;
}
- bool NextMeasurement ()
+ bool NextMeasurement()
{
runIdx = 0;
if (resultsReturned)
return "";
- if (taskCounter == 0) {
- NextTask ();
- return "Benchmark started<br>";
+ if (taskCounter == 0)
+ {
+ NextTask();
+ return $"Benchmark started{formatter.NewLine}";
}
if (measurementIdx == -1)
return ResultsSummary();
- if (runIdx >= Task.Measurements [measurementIdx].NumberOfRuns && !NextMeasurement() && !NextTask ())
- return ResultsSummary();
+ if (runIdx >= Task.Measurements[measurementIdx].NumberOfRuns && !NextMeasurement() && !NextTask())
+ return ResultsSummary();
runIdx++;
- return await Task.RunBatch(results, measurementIdx);
+ return $"{await Task.RunBatch(results, measurementIdx)}{formatter.NewLine}";
}
- string ResultsSummary ()
+ string ResultsSummary()
{
- Dictionary<string, double> minTimes = new Dictionary<string, double> ();
+ Dictionary<string, double> minTimes = new Dictionary<string, double>();
StringBuilder sb = new();
foreach (var result in results)
{
double t;
var key = $"{result.taskName}, {result.measurementName}";
- t = result.span.TotalMilliseconds/result.steps;
+ t = result.span.TotalMilliseconds / result.steps;
if (minTimes.ContainsKey(key))
- t = Math.Min (minTimes[key], t);
+ t = Math.Min(minTimes[key], t);
minTimes[key] = t;
}
- sb.Append("<h4>Summary</h4>");
+ sb.Append($"{formatter.NewLine}Summary{formatter.NewLine}");
foreach (var key in minTimes.Keys)
{
- sb.Append($"{key}: {minTimes [key]}ms<br>");
+ sb.Append($"{key}: {minTimes[key]}ms{formatter.NewLine}");
}
- sb.Append("<h4>.md</h4><tt>| measurement | time |<br>|-:|-:|<br>");
+ sb.Append($"{formatter.NewLine}.md{formatter.NewLine}{formatter.CodeStart}| measurement | time |{formatter.NewLine}|-:|-:|{formatter.NewLine}");
foreach (var key in minTimes.Keys)
{
var time = minTimes[key];
time *= 1000;
unit = "us";
}
- sb.Append($"| {key.Replace('_',' '),38} | {time,10:F4}{unit} |<br>".Replace (" ", " "));
+ sb.Append($"| {key.Replace('_', ' '),38} | {time,10:F4}{unit} |{formatter.NewLine}".Replace(" ", formatter.NonBreakingSpace));
}
- sb.Append("</tt>");
+ sb.Append($"{formatter.CodeEnd}");
resultsReturned = true;
return sb.ToString();
}
}
+
+ public abstract class Formatter
+ {
+ public abstract string NewLine { get; }
+ public abstract string NonBreakingSpace { get; }
+ public abstract string CodeStart { get; }
+ public abstract string CodeEnd { get; }
+ }
+
+ public class PlainFormatter : Formatter
+ {
+ override public string NewLine => "\n";
+ override public string NonBreakingSpace => " ";
+ override public string CodeStart => "";
+ override public string CodeEnd => "";
+ }
+
+ public class HTMLFormatter : Formatter
+ {
+ override public string NewLine => "<br/>";
+ override public string NonBreakingSpace => " ";
+ override public string CodeStart => "<code>";
+ override public string CodeEnd => "</code>";
+ }
}
--- /dev/null
+## Simple wasm benchmark sample app
+
+Runs various performance measurements. It is intended as a quick tool to measure mono/wasm performance
+in dotnet/runtime repo, using in-tree runtime. Can be used to check performance impact of runtime changes
+and the summary is provided in [.md](https://guides.github.com/features/mastering-markdown/) markdown format,
+suitable for commit messages and PR descriptions.
+
+Browser and console versions are available.
+
+### Running the benchmark
+
+To run the benchmark on linux/mac:
+
+ > make build
+ > make run
+
+can be used in the `browser-bench/` and also in the `browser-bench/Console/`.
+
+To run the benchmark on windows:
+
+ > dotnet build /t:RunSample
+
+Example console output:
+
+ > make run
+ console.debug: MONO_WASM: Initializing mono runtime
+ console.debug: MONO_WASM: ICU data archive(s) loaded, disabling invariant mode
+ console.debug: mono_wasm_runtime_ready fe00e07a-5519-4dfe-b35a-f867dbaf2e28
+ console.info: Initializing.....
+ Benchmark started
+ Exceptions, NoExceptionHandling count: 8344090, per call: 6.807213249138013E-05ms, total: 0.568s
+ Exceptions, NoExceptionHandling count: 8500347, per call: 6.940893118833855E-05ms, total: 0.59s
+ ...
+ .md
+ | measurement | time |
+ |-:|-:|
+ | Exceptions, NoExceptionHandling | 0.0680us |
+ | Exceptions, TryCatch | 0.0723us |
+ | Exceptions, TryCatchThrow | 0.0035ms |
+ | Exceptions, TryCatchFilter | 0.0848us |
+ | Exceptions, TryCatchFilterInline | 0.0659us |
+ | Exceptions, TryCatchFilterThrow | 0.0046ms |
+ | Exceptions, TryCatchFilterThrowApplies | 0.0036ms |
+ | Json, non-ASCII text serialize | 15.3855ms |
+ | Json, non-ASCII text deserialize | 24.7299ms |
+ | Json, small serialize | 0.2980ms |
+ | Json, small deserialize | 0.5080ms |
+ | Json, large serialize | 95.3333ms |
+ | Json, large deserialize | 141.4737ms |
+
+The `.md` output will look like this:
+
+| measurement | time |
+|-:|-:|
+| Exceptions, NoExceptionHandling | 0.0680us |
+| Exceptions, TryCatch | 0.0723us |
+| Exceptions, TryCatchThrow | 0.0035ms |
+| Exceptions, TryCatchFilter | 0.0848us |
+| Exceptions, TryCatchFilterInline | 0.0659us |
+| Exceptions, TryCatchFilterThrow | 0.0046ms |
+| Exceptions, TryCatchFilterThrowApplies | 0.0036ms |
+| Json, non-ASCII text serialize | 15.3855ms |
+| Json, non-ASCII text deserialize | 24.7299ms |
+| Json, small serialize | 0.2980ms |
+| Json, small deserialize | 0.5080ms |
+| Json, large serialize | 95.3333ms |
+| Json, large deserialize | 141.4737ms |
+
+Multiple results can be also easily combined together in text editor to have a table with measurements next to each other for comparison.
+
+### Filter jobs/measurements
+
+The `-t` option can be used to run subset of jobs/measurements like this, 1st line linux/mac, 2nd line windows:
+
+ > make run ARGS="-t Json:non,Exceptions:Inline"
+ > dotnet build /v:n /t:RunSample /p:Args="-t Json:non%2cExceptions:Inline"
+
+Note the escaped `,` character (`%2c`) in the dotnet property above.
+
+### Console options
+
+ > make run ARGS=-h
+ ...
+ Simple mono wasm benchmark
+
+ Copyright 2021 Microsoft Corporation
+
+ Options:
+ -h, --help, -? Show this message and exit
+ -t, --tasks=VALUE Filter comma separated tasks and its measurements
+ matching, TASK[:REGEX][,TASK[:REGEX],...].
+ Example: -t Json:non,Exceptions:Inline
<ItemGroup>
<WasmExtraFilesToDeploy Include="index.html" />
+ <Compile Remove="Console/Console.cs" />
</ItemGroup>
<PropertyGroup>
<_SampleProject>Wasm.Browser.Bench.Sample.csproj</_SampleProject>
+ <_SampleAssembly> Wasm.Browser.Bench.Sample.dll</_SampleAssembly>
</PropertyGroup>
<Target Name="RunSample" DependsOnTargets="RunSampleWithBrowser" />
public override string Name => "WebSocket";
public override Measurement[] Measurements => measurements;
+ public override bool BrowserOnly => true;
Measurement[] measurements;
promise.then(ret => {
document.getElementById("out").innerHTML += ret;
if (ret.length > 0) {
- document.getElementById("out").innerHTML += "<br>";
setTimeout(function () { yieldBench(); }, 0);
} else {
- document.getElementById("out").innerHTML += "<br>Finished";
+ document.getElementById("out").innerHTML += "Finished";
}
});
}
endif
PROJECT_NAME=Wasm.Console.Sample.csproj
+CONSOLE_DLL=Wasm.Console.Sample.dll
run: run-console
<PropertyGroup>
<_SampleProject>Wasm.Console.Sample.csproj</_SampleProject>
+ <_SampleAssembly>Wasm.Console.Sample.dll</_SampleAssembly>
</PropertyGroup>
<Target Name="RunSample" DependsOnTargets="RunSampleWithV8" />
echo "The tool dotnet-serve could not be found. Install with: $(DOTNET) tool install --global dotnet-serve"; \
exit 1; \
else \
- $(DOTNET) serve -d:bin/$(CONFIG)/AppBundle -p:8000; \
+ $(DOTNET) serve -d:bin/$(CONFIG)/AppBundle -o -p:8000; \
fi
run-console:
- cd bin/$(CONFIG)/AppBundle && ~/.jsvu/v8 --stack-trace-limit=1000 --single-threaded --expose_wasm runtime.js -- $(DOTNET_MONO_LOG_LEVEL) --run Wasm.Console.Sample.dll
+ cd bin/$(CONFIG)/AppBundle && ~/.jsvu/v8 --stack-trace-limit=1000 --single-threaded --expose_wasm runtime.js -- $(DOTNET_MONO_LOG_LEVEL) --run $(CONSOLE_DLL) $(ARGS)
To build and run the samples with AOT, add `/p:RunAOTCompilation=true` to the above command lines.
+* bench sample
+
+Also check [bench](../sample/wasm/browser-bench/README.md) sample to measure mono/wasm runtime performance.
+
### Upgrading Emscripten
Bumping Emscripten version involves these steps:
projects / platform / upstream / dotnet / runtime.git / commitdiff