Skip to content

Commit

Permalink
feat: add doc
Browse files Browse the repository at this point in the history
  • Loading branch information
@gfyrag
gfyrag committed Sep 10, 2024
1 parent ad4a2a7 commit 58957ac
Show file tree
Hide file tree
Showing 6 changed files with 145 additions and 5 deletions.
9 changes: 6 additions & 3 deletions components/ledger/pkg/benchmarks/benchmark.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,11 @@ type RunConfiguration struct {
Env string
Script string
Features FeatureConfiguration
Ledger ledger.Ledger
}

func (c RunConfiguration) String() string {
return fmt.Sprintf("%s/%s/%s", c.Env, c.Script, c.Features)
return fmt.Sprintf("%s/%s/%s/%s/%s", c.Env, c.Script, c.Features, c.Ledger.Bucket, c.Ledger.Name)
}

type Runner interface {
Expand Down Expand Up @@ -67,10 +68,11 @@ func (b *Benchmark) Run(ctx context.Context) error {
Bucket: uuid.NewString()[:8],
}
ledgerConfiguration.SetDefaults()
env, err := envFactory.Create(ctx, ledger.Ledger{
l := ledger.Ledger{
Configuration: ledgerConfiguration,
Name: uuid.NewString()[:8],
})
}
env, err := envFactory.Create(ctx, l)
if err != nil {
return err
}
Expand All @@ -82,6 +84,7 @@ func (b *Benchmark) Run(ctx context.Context) error {
Env: envName,
Script: scriptName,
Features: features,
Ledger: l,
}, func(iterator Iterator) *Report {
for iterator.Next() {
id := int(cpt.Add(1))
Expand Down
73 changes: 72 additions & 1 deletion components/ledger/test/performance/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,75 @@ go test -bench=. -run ^$ -tags it \
--stack.url=XXX \
--stack.client-id=XXX \
--stack.client-secret=XXX
```
```

## Results

The output is a standard go bench output.

Example:
```
goos: darwin
goarch: arm64
pkg: github.com/formancehq/ledger/test/performance
BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=SYNC,PCV=SYNC-8 84 15693529 ns/op 113.0 ms/transaction 62.81 t/s
BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=SYNC,PCV=SYNC-8 90 12960446 ns/op 100.0 ms/transaction 71.65 t/s
BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=DISABLED,PCV=SYNC-8 87 13522271 ns/op 103.0 ms/transaction 70.18 t/s
BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=DISABLED,PCV=SYNC-8 92 13460427 ns/op 102.0 ms/transaction 71.03 t/s
BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=SYNC,PCV=DISABLED-8 66 15225778 ns/op 111.0 ms/transaction 59.71 t/s
BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=SYNC,PCV=DISABLED-8 80 16243343 ns/op 119.0 ms/transaction 60.42 t/s
BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=DISABLED,PCV=DISABLED-8 55 19083905 ns/op 140.0 ms/transaction 48.15 t/s
BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=DISABLED,PCV=DISABLED-8 57 22134052 ns/op 155.0 ms/transaction 43.34 t/s
BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=SYNC,PCV=SYNC-8 75 15552648 ns/op 120.0 ms/transaction 58.70 t/s
BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=SYNC,PCV=SYNC-8 68 17866844 ns/op 134.0 ms/transaction 51.82 t/s
BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=DISABLED,PCV=SYNC-8 81 13593212 ns/op 107.0 ms/transaction 66.39 t/s
BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=DISABLED,PCV=SYNC-8 84 13560348 ns/op 104.0 ms/transaction 67.69 t/s
BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=SYNC,PCV=DISABLED-8 86 14518456 ns/op 108.0 ms/transaction 67.05 t/s
BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=SYNC,PCV=DISABLED-8 90 13041432 ns/op 101.0 ms/transaction 71.27 t/s
BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=DISABLED,PCV=DISABLED-8 76 14276990 ns/op 108.0 ms/transaction 66.02 t/s
BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=DISABLED,PCV=DISABLED-8 88 13430946 ns/op 103.0 ms/transaction 69.66 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=SYNC,PCV=SYNC-8 72 15435962 ns/op 120.0 ms/transaction 58.49 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=SYNC,PCV=SYNC-8 84 15247506 ns/op 114.0 ms/transaction 62.61 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=DISABLED,PCV=SYNC-8 78 16336328 ns/op 122.0 ms/transaction 59.34 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=DISABLED,PCV=SYNC-8 62 17001069 ns/op 125.0 ms/transaction 54.60 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=SYNC,PCV=DISABLED-8 62 18296797 ns/op 137.0 ms/transaction 51.63 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=SYNC,PCV=DISABLED-8 78 15578895 ns/op 118.0 ms/transaction 60.62 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=DISABLED,PCV=DISABLED-8 57 18337382 ns/op 146.0 ms/transaction 45.01 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=DISABLED,PCV=DISABLED-8 66 17246153 ns/op 133.0 ms/transaction 52.94 t/s
```

There is the format:
```
BenchmarkWrite/<env>/<transaction-type>/<features>/<bucket>/<ledger>-<cpu count> <iteration count> <ns by op> <average transaction duration> <tps>
```

Where parameters are :
* env:
* core: the test use directly the core service of the ledger
* testserver: the test use a full running server
* remote : the test use a remote api
* transaction-type (see [scripts](./write_test.go))
* world->bank : A transaction from `@world` to `@bank`
* world->any : A transaction from `@world` to `@dst:<iteration>`
* any(unbounded)->any : A transaction from `@src:<iteration>` to `@dst:<iteration>`
* features: features enabled on the server (affect only servers supporting the feature)
* All combination of ledger features are tested. So the field `<features>` will contains the list of the feature. For better visibility, features names are abbreviated. There is the code :
* HL => HASH_LOG : Possible configuration are `DISABLED` or `SYNC`
* PCEV => POST_COMMIT_EFFECTIVE_VOLUMES : Possible configuration are `DISABLED` or `SYNC`
* PCV => POST_COMMIT_VOLUMES : Possible configuration are `DISABLED` or `SYNC`

If the flag `--include-ledger-in-results` is specified, the line will contain two additional arguments :
* bucket: bucket name where the ledger is created
* ledger: ledger name

Example:
```
BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=SYNC,PCV=SYNC/5372990c/363e57d5-8 90 13808911 ns/op 104.0 ms/transaction 68.89 t/s
...
```

Other columns of the benchmark output are the metrics.
In addition to the standard metrics, we have two specific metrics :
* average transaction duration
* tps : transactions per second
2 changes: 2 additions & 0 deletions components/ledger/test/performance/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ var (
stackURL string
stackClientID string
stackClientSecret string
includeLedgerInformationInResults bool

envFactories = make(map[string]benchmarks.EnvFactory)
)
Expand All @@ -31,6 +32,7 @@ func init() {
flag.StringVar(&stackURL, "stack.url", "", "Stack URL")
flag.StringVar(&stackClientID, "stack.client-id", "", "Stack client id")
flag.StringVar(&stackClientSecret, "stack.client-secret", "", "Stack client secret")
flag.BoolVar(&includeLedgerInformationInResults, "include-ledger-in-results", false, "Include ledger information in results")
}

func TestMain(m *testing.M) {
Expand Down
29 changes: 29 additions & 0 deletions components/ledger/test/performance/report_sandbox_stateless
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
goos: darwin
goarch: arm64
pkg: github.com/formancehq/ledger/test/performance
BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=SYNC,PCV=SYNC-8 82 14422190 ns/op 111.0 ms/transaction 62.51 t/s
BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=SYNC,PCV=SYNC-8 85 14416520 ns/op 109.0 ms/transaction 65.49 t/s
BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=DISABLED,PCV=SYNC-8 60 18009485 ns/op 139.0 ms/transaction 43.79 t/s
BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=DISABLED,PCV=SYNC-8 60 17852348 ns/op 137.0 ms/transaction 50.35 t/s
BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=SYNC,PCV=DISABLED-8 69 15683993 ns/op 124.0 ms/transaction 55.75 t/s
BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=SYNC,PCV=DISABLED-8 74 13948827 ns/op 108.0 ms/transaction 63.77 t/s
BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=DISABLED,PCV=DISABLED-8 81 14422006 ns/op 113.0 ms/transaction 62.93 t/s
BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=DISABLED,PCV=DISABLED-8 100 11897418 ns/op 92.00 ms/transaction 78.41 t/s
BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=SYNC,PCV=SYNC-8 79 14825957 ns/op 115.0 ms/transaction 61.39 t/s
BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=SYNC,PCV=SYNC-8 84 13216647 ns/op 104.0 ms/transaction 68.50 t/s
BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=DISABLED,PCV=SYNC-8 79 14308195 ns/op 112.0 ms/transaction 62.62 t/s
BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=DISABLED,PCV=SYNC-8 92 13698061 ns/op 103.0 ms/transaction 68.69 t/s
BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=SYNC,PCV=DISABLED-8 82 14185600 ns/op 110.0 ms/transaction 64.49 t/s
BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=SYNC,PCV=DISABLED-8 99 13251108 ns/op 100.0 ms/transaction 71.89 t/s
BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=DISABLED,PCV=DISABLED-8 80 17452899 ns/op 126.0 ms/transaction 56.74 t/s
BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=DISABLED,PCV=DISABLED-8 91 13061339 ns/op 101.0 ms/transaction 70.54 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=SYNC,PCV=SYNC-8 73 14089435 ns/op 116.0 ms/transaction 60.47 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=SYNC,PCV=SYNC-8 91 14123212 ns/op 107.0 ms/transaction 68.45 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=DISABLED,PCV=SYNC-8 60 17551044 ns/op 117.0 ms/transaction 50.95 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=DISABLED,PCV=SYNC-8 86 12961788 ns/op 100.0 ms/transaction 69.83 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=SYNC,PCV=DISABLED-8 81 14172736 ns/op 110.0 ms/transaction 63.76 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=SYNC,PCV=DISABLED-8 79 14202664 ns/op 111.0 ms/transaction 63.88 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=DISABLED,PCV=DISABLED-8 80 15340831 ns/op 117.0 ms/transaction 60.75 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=DISABLED,PCV=DISABLED-8 80 15331845 ns/op 117.0 ms/transaction 62.30 t/s
PASS
ok github.com/formancehq/ledger/test/performance 58.811s
29 changes: 29 additions & 0 deletions components/ledger/test/performance/report_sandbox_v2.0.17
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
goos: darwin
goarch: arm64
pkg: github.com/formancehq/ledger/test/performance
BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=SYNC,PCV=SYNC-8 57 22688839 ns/op 157.0 ms/transaction 42.66 t/s
BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=SYNC,PCV=SYNC-8 64 17240260 ns/op 124.0 ms/transaction 53.59 t/s
BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=DISABLED,PCV=SYNC-8 72 15970057 ns/op 122.0 ms/transaction 58.48 t/s
BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=DISABLED,PCV=SYNC-8 69 15281248 ns/op 115.0 ms/transaction 60.43 t/s
BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=SYNC,PCV=DISABLED-8 85 25605594 ns/op 160.0 ms/transaction 46.20 t/s
BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=SYNC,PCV=DISABLED-8 90 13135869 ns/op 102.0 ms/transaction 71.72 t/s
BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=DISABLED,PCV=DISABLED-8 88 13456556 ns/op 103.0 ms/transaction 71.16 t/s
BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=DISABLED,PCV=DISABLED-8 87 14881287 ns/op 109.0 ms/transaction 66.04 t/s
BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=SYNC,PCV=SYNC-8 72 14482659 ns/op 108.0 ms/transaction 63.77 t/s
BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=SYNC,PCV=DISABLED-8 86 14145120 ns/op 108.0 ms/transaction 67.07 t/s
BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=SYNC,PCV=SYNC-8 73 14135062 ns/op 105.0 ms/transaction 65.53 t/s
BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=SYNC,PCV=DISABLED-8 81 14169088 ns/op 112.0 ms/transaction 63.81 t/s
BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=DISABLED,PCV=SYNC-8 86 14559848 ns/op 109.0 ms/transaction 65.47 t/s
BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=DISABLED,PCV=DISABLED-8 84 14062040 ns/op 107.0 ms/transaction 66.17 t/s
BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=DISABLED,PCV=SYNC-8 86 14191338 ns/op 106.0 ms/transaction 67.66 t/s
BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=DISABLED,PCV=DISABLED-8 82 13660147 ns/op 106.0 ms/transaction 67.21 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=SYNC,PCV=SYNC-8 68 15796684 ns/op 127.0 ms/transaction 55.65 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=SYNC,PCV=SYNC-8 68 17922036 ns/op 136.0 ms/transaction 51.84 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=DISABLED,PCV=SYNC-8 68 17231234 ns/op 131.0 ms/transaction 53.30 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=DISABLED,PCV=SYNC-8 69 17769751 ns/op 135.0 ms/transaction 53.79 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=SYNC,PCV=DISABLED-8 69 16701367 ns/op 128.0 ms/transaction 55.27 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=SYNC,PCV=DISABLED-8 67 17351054 ns/op 131.0 ms/transaction 53.16 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=DISABLED,PCV=DISABLED-8 70 15975102 ns/op 122.0 ms/transaction 57.59 t/s
BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=DISABLED,PCV=DISABLED-8 72 16131747 ns/op 121.0 ms/transaction 57.66 t/s
PASS
ok github.com/formancehq/ledger/test/performance 63.983s
8 changes: 7 additions & 1 deletion components/ledger/test/performance/write_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,14 @@ func BenchmarkWrite(b *testing.B) {
configuration benchmarks.RunConfiguration,
f func(benchmarks.Iterator) *benchmarks.Report,
) {
b.Run(configuration.String(), func(b *testing.B) {
testName := fmt.Sprintf("%s/%s/%s", configuration.Env, configuration.Script, configuration.Features)
if includeLedgerInformationInResults {
testName = fmt.Sprintf("%s/%s/%s", testName, configuration.Ledger.Bucket, configuration.Ledger.Name)
}

b.Run(testName, func(b *testing.B) {
var report *benchmarks.Report

b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
report = f(benchmarks.IteratorFn(pb.Next))
Expand Down

0 comments on commit 58957ac

Please sign in to comment.