feat: Add benchmarks (#450)

Co-authored-by: Maxence Maireaux <[email protected]>

README.md

@@ -1,4 +1,4 @@
-# [Formance Stack](https://formance.com) • [![Ledger Stars](https://img.shields.io/github/stars/formancehq/ledger?label=Ledger%20stars)](https://github.com/formancehq/ledger/stargazers) [![License MIT](https://img.shields.io/badge/license-mit-purple)](https://github.com/formancehq/ledger/blob/main/LICENSE) [![YCombinator](https://img.shields.io/badge/Backed%20by-Y%20Combinator-%23f26625)](https://www.ycombinator.com/companies/formance-fka-numary) [![slack](https://img.shields.io/badge/slack-formance-brightgreen.svg?logo=slack)](https://bit.ly/formance-slack)
+[Formance Stack](https://formance.com) • [![Ledger Stars](https://img.shields.io/github/stars/formancehq/ledger?label=Ledger%20stars)](https://github.com/formancehq/ledger/stargazers) [![License MIT](https://img.shields.io/badge/license-mit-purple)](https://github.com/formancehq/ledger/blob/main/LICENSE) [![YCombinator](https://img.shields.io/badge/Backed%20by-Y%20Combinator-%23f26625)](https://www.ycombinator.com/companies/formance-fka-numary) [![slack](https://img.shields.io/badge/slack-formance-brightgreen.svg?logo=slack)](https://bit.ly/formance-slack)
 
 Formance is a highly modular developer platform to build and operate complex money flows of any size and shapes. It comes with several components, that can be used as a whole as the Formance Stack or separately as standalone micro-services and libraries:
 

components/ledger/internal/engine/command/lock.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/formancehq/stack/libs/go-libs/collectionutils"
 	"github.com/formancehq/stack/libs/go-libs/logging"
@@ -33,6 +34,7 @@ type Accounts struct {
 type lockIntent struct {
 	accounts Accounts
 	acquired chan struct{}
+	at       time.Time
 }
 
 func (intent *lockIntent) tryLock(ctx context.Context, chain *DefaultLocker) bool {
@@ -105,6 +107,7 @@ func (defaultLocker *DefaultLocker) Lock(ctx context.Context, accounts Accounts)
 	intent := &lockIntent{
 		accounts: accounts,
 		acquired: make(chan struct{}),
+		at:       time.Now(),
 	}
 
 	recheck := func() {

components/ledger/libs/otlp/otlpmetrics/module.go

@@ -2,7 +2,6 @@ package otlpmetrics
 
 import (
 	"context"
-	"fmt"
 	"time"
 
 	"github.com/formancehq/stack/libs/go-libs/logging"
@@ -64,11 +63,9 @@ func MetricsModule(cfg ModuleConfig) fx.Option {
 		otlp.LoadResource(cfg.ServiceName, cfg.ResourceAttributes),
 		fx.Decorate(fx.Annotate(func(mp *sdkmetric.MeterProvider) metric.MeterProvider { return mp }, fx.As(new(metric.MeterProvider)))),
 		fx.Provide(fx.Annotate(func(options ...sdkmetric.Option) *sdkmetric.MeterProvider {
-			fmt.Println("run meter provider with options", options)
 			return sdkmetric.NewMeterProvider(options...)
 		}, fx.ParamTags(metricsProviderOptionKey))),
 		fx.Invoke(func(lc fx.Lifecycle, metricProvider *sdkmetric.MeterProvider, options ...runtime.Option) {
-			fmt.Println("start meter provider")
 			// set global propagator to tracecontext (the default is no-op).
 			otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
 				b3.New(), propagation.TraceContext{})) // B3 format is common and used by zipkin. Always enabled right now.

libs/go-libs/otlp/otlpmetrics/module.go

@@ -2,7 +2,6 @@ package otlpmetrics
 
 import (
 	"context"
-	"fmt"
 	"time"
 
 	"github.com/formancehq/stack/libs/go-libs/logging"
@@ -64,11 +63,9 @@ func MetricsModule(cfg ModuleConfig) fx.Option {
 		otlp.LoadResource(cfg.ServiceName, cfg.ResourceAttributes),
 		fx.Decorate(fx.Annotate(func(mp *sdkmetric.MeterProvider) metric.MeterProvider { return mp }, fx.As(new(metric.MeterProvider)))),
 		fx.Provide(fx.Annotate(func(options ...sdkmetric.Option) *sdkmetric.MeterProvider {
-			fmt.Println("run meter provider with options", options)
 			return sdkmetric.NewMeterProvider(options...)
 		}, fx.ParamTags(metricsProviderOptionKey))),
 		fx.Invoke(func(lc fx.Lifecycle, metricProvider *sdkmetric.MeterProvider, options ...runtime.Option) {
-			fmt.Println("start meter provider")
 			// set global propagator to tracecontext (the default is no-op).
 			otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
 				b3.New(), propagation.TraceContext{})) // B3 format is common and used by zipkin. Always enabled right now.

tests/benchmarks/.gitignore

@@ -0,0 +1,4 @@
+k6
+results
+.env
+.env

tests/benchmarks/README.md

@@ -0,0 +1,14 @@
+# Benchmarks
+
+## Deps
+- xk6
+- Task
+## Launch benchmarks
+For v1:
+```bash
+task run:v1
+```
+For v2:
+```bash
+task run:v2
+```

tests/benchmarks/Taskfile.yaml

@@ -0,0 +1,35 @@
+version: '3'
+
+tasks:
+  tools:
+    dir: ./tools
+    cmds:
+      - docker compose up -d
+      - sleep 20
+
+  build:scripts:
+    dir: ./scripts
+    cmds:
+      - npm run build
+
+  build:k6:
+    cmds:
+      - xk6 build --with extension=$(pwd)/extension --with github.com/grafana/[email protected]
+
+  run:
+    deps:
+      - tools
+      - build:scripts
+      - build:k6
+    cmds:
+      - PGPASSWORD=ledger psql -h 127.0.0.1 -U ledger -c "DROP DATABASE IF EXISTS ledgerv2;"
+      - PGPASSWORD=ledger psql -h 127.0.0.1 -U ledger -c "CREATE DATABASE ledgerv2;"
+      - TEST_ID=$(date +%s)-v2 K6_PROMETHEUS_RW_TREND_AS_NATIVE_HISTOGRAM=true K6_PROMETHEUS_RW_SERVER_URL=http://127.0.0.1:9090/api/v1/write ./k6 run --summary-trend-stats="avg,min,med,max,p(90),p(95),p(99)" --out xk6-prometheus-rw scripts/dist/ledger-v2.js
+    env:
+      POSTGRES_DSN: postgresql://ledger:ledger@postgres:5432/ledgerv2?sslmode=disable
+      DOCKER_NETWORK: tools_benchmarks
+
+  cleanup:
+    dir: ./tools
+    cmds:
+      - docker compose down -v

tests/benchmarks/extension/extension.go

@@ -0,0 +1,182 @@
+package extension
+
+import (
+	"errors"
+	"fmt"
+	"github.com/google/uuid"
+	_ "github.com/lib/pq"
+	dockertest "github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+	"github.com/sirupsen/logrus"
+	"go.k6.io/k6/js/modules"
+	"net/http"
+	"os"
+)
+
+type LedgerConfiguration struct {
+	Version     string `json:"version"`
+	PostgresDSN string `json:"postgresDSN"`
+	Network     string `json:"network"`
+	TestID      string `json:"testID"`
+}
+
+type Ledger struct {
+	URL string `json:"url"`
+}
+
+func (c *LedgerConfiguration) resolve() error {
+	if c.TestID == "" {
+		c.TestID = os.Getenv("TEST_ID")
+		if c.TestID == "" {
+			c.TestID = uuid.NewString()
+		}
+	}
+	if c.Version == "" {
+		c.Version = os.Getenv("LEDGER_VERSION")
+		if c.Version == "" {
+			c.Version = "latest"
+		}
+	}
+	if c.PostgresDSN == "" {
+		c.PostgresDSN = os.Getenv("POSTGRES_DSN")
+		if c.PostgresDSN == "" {
+			return errors.New("missing postgres dsn")
+		}
+	}
+	if c.Network == "" {
+		c.Network = os.Getenv("DOCKER_NETWORK")
+	}
+	return nil
+}
+
+type Extension struct {
+	pool     *dockertest.Pool
+	resource *dockertest.Resource
+	logger   *logrus.Logger
+	testID   string `json:"testID"`
+}
+
+func (c *Extension) StartLedger(configuration LedgerConfiguration) *Ledger {
+	var err error
+	if err := configuration.resolve(); err != nil {
+		panic(err)
+	}
+
+	logger := c.logger.WithFields(map[string]interface{}{
+		"version": configuration.Version,
+		"testid":  configuration.TestID,
+	})
+
+	defer func() {
+		if e := recover(); e != nil {
+			if c.pool != nil && c.resource != nil {
+				if err := c.pool.Purge(c.resource); err != nil {
+					logger.Errorf("unable to clean docker resource: %s", err)
+				}
+			}
+			panic(e)
+		}
+	}()
+
+	logger.Infof("Connecting to docker server...")
+	c.pool, err = dockertest.NewPool("")
+	if err != nil {
+		panic(err)
+	}
+
+	err = c.pool.Client.Ping()
+	if err != nil {
+		panic(err)
+	}
+
+	var networkID string
+	if configuration.Network != "" {
+		logger.Infof("Searching network named '%s'...", configuration.Network)
+		network, err := c.pool.NetworksByName(configuration.Network)
+		if err != nil {
+			panic(err)
+		}
+		networkID = network[0].Network.ID
+	}
+
+	options := make([]func(config *docker.HostConfig), 0)
+	options = append(options, func(config *docker.HostConfig) {
+		config.AutoRemove = false
+		config.CPUShares = 4
+		config.Memory = 1024 * 1024 * 1024 * 8
+	})
+
+	logger.Infof("Starting ledger container...")
+	c.resource, err = c.pool.RunWithOptions(&dockertest.RunOptions{
+		Name:         fmt.Sprintf("ledger-%s", configuration.TestID),
+		Repository:   "ghcr.io/formancehq/ledger",
+		Tag:          configuration.Version,
+		Env:          envVars(configuration),
+		NetworkID:    networkID,
+		ExposedPorts: []string{"3068/tcp"},
+	}, options...)
+	if err != nil {
+		panic(err)
+	}
+
+	logger.Infof("Ledger container started with id : %s", c.resource.Container.ID)
+
+	logger.Infof("Checking ledger is alive...")
+	if err := c.pool.Retry(func() error {
+		_, err := http.Get("http://localhost:" + c.resource.GetPort("3068/tcp"))
+		return err
+	}); err != nil {
+		panic(err)
+	}
+
+	logger.Infof("Ledger properly started!")
+
+	return &Ledger{
+		URL: "http://localhost:" + c.resource.GetPort("3068/tcp"),
+	}
+}
+
+func (c *Extension) StopLedger() {
+	if os.Getenv("NO_CLEANUP") == "true" {
+		return
+	}
+	c.logger.Infof("Shutting down ledger container...")
+	if err := c.pool.Client.KillContainer(docker.KillContainerOptions{
+		ID:     c.resource.Container.ID,
+		Signal: 15,
+	}); err != nil {
+		panic(err)
+	}
+
+	exitCode, err := c.pool.Client.WaitContainer(c.resource.Container.ID)
+	if err != nil {
+		panic(err)
+	}
+	if exitCode != 0 {
+		panic(fmt.Errorf("unexpected status code %d when stopping the ledger", exitCode))
+	}
+
+	if err := c.pool.Purge(c.resource); err != nil {
+		panic(err)
+	}
+
+	c.logger.Infof("Ledger stopped!")
+}
+
+func envVars(configuration LedgerConfiguration) []string {
+	return []string{
+		"BIND=:3068",
+		"STORAGE_DRIVER=postgres",
+		"STORAGE_POSTGRES_CONN_STRING=" + configuration.PostgresDSN,
+		"STORAGE_POSTGRES_MAX_OPEN_CONNS=50",
+		"STORAGE_POSTGRES_MAX_IDLE_CONNS=50",
+		"DEBUG=false",
+	}
+}
+
+func init() {
+	ext := &Extension{
+		logger: logrus.New(),
+	}
+	modules.Register("k6/x/formancehq/benchmarks", ext)
+}

tests/benchmarks/extension/go.mod

@@ -0,0 +1,61 @@
+module github.com/formancehq/stack/tests/benchmarks/extension
+
+go 1.20
+
+require (
+	github.com/google/uuid v1.3.0
+	github.com/ory/dockertest/v3 v3.10.0
+	github.com/prometheus/client_golang v1.14.1-0.20221122130035-8b6e68085b10
+	github.com/sirupsen/logrus v1.9.3
+	go.k6.io/k6 v0.45.1-0.20230719100510-ac9c6bc85d13
+	golang.org/x/mod v0.12.0
+)
+
+require (
+	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
+	github.com/containerd/continuity v0.4.1 // indirect
+	github.com/dlclark/regexp2 v1.9.0 // indirect
+	github.com/docker/cli v24.0.4+incompatible // indirect
+	github.com/docker/docker v24.0.4+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/dop251/goja v0.0.0-20230621100801-7749907a8a20 // indirect
+	github.com/fatih/color v1.15.0 // indirect
+	github.com/go-sourcemap/sourcemap v2.1.4-0.20211119122758-180fcef48034+incompatible // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/imdario/mergo v0.3.16 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/mstoykov/atlas v0.0.0-20220811071828-388f114305dd // indirect
+	github.com/onsi/ginkgo v1.16.5 // indirect
+	github.com/onsi/gomega v1.27.8 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.2 // indirect
+	github.com/opencontainers/runc v1.1.8 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/serenize/snaker v0.0.0-20201027110005-a7ad2135616e // indirect
+	github.com/spf13/afero v1.1.2 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
+	golang.org/x/text v0.10.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.11.0 // indirect
+	gopkg.in/guregu/null.v3 v3.3.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+)