From 3429e6deb5c0e8eeefd4dff05c7f25602a37d5cf Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Mon, 9 Sep 2024 10:53:55 +0200 Subject: [PATCH] feat: introduce balances table --- .../controller/ledger/writer/store.go | 2 + .../controller/ledger/writer/writer.go | 53 +++++-- .../bucket/migrations/11-stateless.sql | 132 +++++++++--------- .../internal/storage/ledger/balances.go | 30 ++++ .../internal/storage/ledger/balances_test.go | 15 ++ .../ledger/internal/storage/ledger/moves.go | 34 ++--- .../ledger/test/performance/report_after_pcv | 59 ++++++++ .../ledger/test/performance/report_before_pcv | 59 ++++++++ tests/integration/go.mod | 4 +- tests/integration/go.sum | 2 + 10 files changed, 289 insertions(+), 101 deletions(-) create mode 100644 components/ledger/test/performance/report_after_pcv create mode 100644 components/ledger/test/performance/report_before_pcv diff --git a/components/ledger/internal/controller/ledger/writer/store.go b/components/ledger/internal/controller/ledger/writer/store.go index 31471a845f..edbd153cc3 100644 --- a/components/ledger/internal/controller/ledger/writer/store.go +++ b/components/ledger/internal/controller/ledger/writer/store.go @@ -6,6 +6,7 @@ import ( ledger "github.com/formancehq/ledger/internal" "github.com/formancehq/stack/libs/go-libs/metadata" "github.com/uptrace/bun" + "math/big" ) //go:generate mockgen -source store.go -destination store_generated.go -package writer . TX @@ -28,6 +29,7 @@ type TX interface { Commit(ctx context.Context) error Rollback(ctx context.Context) error QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) + AddToBalance(ctx context.Context, addr, asset string, amount *big.Int) (*big.Int, error) } //go:generate mockgen -source store.go -destination store_generated.go -package writer . Store diff --git a/components/ledger/internal/controller/ledger/writer/writer.go b/components/ledger/internal/controller/ledger/writer/writer.go index 223ed46f21..40b5fc9424 100644 --- a/components/ledger/internal/controller/ledger/writer/writer.go +++ b/components/ledger/internal/controller/ledger/writer/writer.go @@ -12,6 +12,7 @@ import ( "github.com/pkg/errors" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "math/big" "slices" ) @@ -94,6 +95,7 @@ func (w *Writer) CreateTransaction(ctx context.Context, parameters Parameters, r log, err := w.withTX(ctx, parameters, func(sqlTX TX) (*ledger.Log, error) { if len(machineInit.BoundedSources) > 0 { + // TODO: remove lock, and use SELECT FOR UPDATE _, latency, err := tracer.TraceWithLatency(ctx, "LockAccounts", func(ctx context.Context) (*struct{}, error) { return nil, sqlTX.LockAccounts(ctx, machineInit.BoundedSources...) }, func(ctx context.Context, _ *struct{}) { @@ -167,19 +169,19 @@ func (w *Writer) CreateTransaction(ctx context.Context, parameters Parameters, r } } - for _, account := range transaction.GetMoves().InvolvedAccounts() { - _, latency, err = tracer.TraceWithLatency(ctx, "LockAccounts", func(ctx context.Context) (struct{}, error) { - return struct{}{}, sqlTX.LockAccounts(ctx, account) - }, func(ctx context.Context, _ struct{}) { - trace.SpanFromContext(ctx).SetAttributes( - attribute.StringSlice("accounts", transaction.GetMoves().InvolvedAccounts()), - ) - }) - if err != nil { - return nil, errors.Wrapf(err, "failed to acquire account lock on %s", account) - } - logger.WithField("latency", latency.String()).Debugf("account locked: %s", account) - } + //for _, account := range transaction.GetMoves().InvolvedAccounts() { + // _, latency, err = tracer.TraceWithLatency(ctx, "LockAccounts", func(ctx context.Context) (struct{}, error) { + // return struct{}{}, sqlTX.LockAccounts(ctx, account) + // }, func(ctx context.Context, _ struct{}) { + // trace.SpanFromContext(ctx).SetAttributes( + // attribute.StringSlice("accounts", transaction.GetMoves().InvolvedAccounts()), + // ) + // }) + // if err != nil { + // return nil, errors.Wrapf(err, "failed to acquire account lock on %s", account) + // } + // logger.WithField("latency", latency.String()).Debugf("account locked: %s", account) + //} _, latency, err = tracer.TraceWithLatency(ctx, "InsertMoves", func(ctx context.Context) (struct{}, error) { return struct{}{}, sqlTX.InsertMoves(ctx, transaction.GetMoves()...) @@ -187,9 +189,32 @@ func (w *Writer) CreateTransaction(ctx context.Context, parameters Parameters, r if err != nil { return nil, errors.Wrap(err, "failed to insert moves") } - logger.WithField("latency", latency.String()).Debugf("moves inserted") + // TODO: merge move for the same account + for _, move := range transaction.GetMoves() { + newBalance, latency, err := tracer.TraceWithLatency(ctx, "AddToBalance", func(ctx context.Context) (*big.Int, error) { + return sqlTX.AddToBalance(ctx, move.Account, move.Asset, move.Amount) + }, func(ctx context.Context, newBalance *big.Int) { + trace.SpanFromContext(ctx).SetAttributes( + attribute.String("new_balance", newBalance.String()), + attribute.String("account", move.Account), + attribute.String("asset", move.Asset), + attribute.String("amount", move.Amount.String()), + ) + }) + if err != nil { + return nil, errors.Wrap(err, "failed to update balance") + } + logger. + WithField("latency", latency.String()). + WithField("account", move.Account). + WithField("asset", move.Asset). + WithField("amount", move.Amount). + WithField("new_balance", newBalance.String()). + Debugf("balance updated") + } + // notes(gfyrag): force date to be zero to let postgres fill it // todo: clean that return pointer.For(ledger.NewTransactionLogWithDate(*transaction, result.AccountMetadata, time.Time{})), err diff --git a/components/ledger/internal/storage/bucket/migrations/11-stateless.sql b/components/ledger/internal/storage/bucket/migrations/11-stateless.sql index 5d885ec3e2..588b002a42 100644 --- a/components/ledger/internal/storage/bucket/migrations/11-stateless.sql +++ b/components/ledger/internal/storage/bucket/migrations/11-stateless.sql @@ -12,86 +12,80 @@ alter table "{{.Bucket}}".transactions alter column id type bigint; --- create function "{{.Bucket}}".insert_moves_from_transaction() returns trigger +-- create function "{{.Bucket}}".set_effective_volumes() +-- returns trigger -- security definer -- language plpgsql -- as -- $$ --- declare --- posting jsonb; -- begin --- for posting in (select jsonb_array_elements(new.postings::jsonb)) loop --- perform "{{.Bucket}}".insert_posting(new.seq, new.ledger, new.inserted_at, new.timestamp, posting, '{}'::jsonb); --- end loop; +-- new.post_commit_effective_volumes = coalesce(( +-- select ( +-- (post_commit_effective_volumes).inputs + case when new.is_source then 0 else new.amount end, +-- (post_commit_effective_volumes).outputs + case when new.is_source then new.amount else 0 end +-- ) +-- from "{{.Bucket}}".moves +-- where accounts_seq = new.accounts_seq +-- and asset = new.asset +-- and ledger = new.ledger +-- and (effective_date < new.effective_date or (effective_date = new.effective_date and seq < new.seq)) +-- order by effective_date desc, seq desc +-- limit 1 +-- ), ( +-- case when new.is_source then 0 else new.amount end, +-- case when new.is_source then new.amount else 0 end +-- )); -- -- return new; -- end; -- $$; -- --- create trigger "{{.Bucket}}_project_moves_for_transaction" --- after insert --- on "{{.Bucket}}"."transactions" +-- create trigger "{{.Bucket}}_set_effective_volumes" +-- before insert +-- on "{{.Bucket}}"."moves" -- for each row --- execute procedure "{{.Bucket}}".insert_moves_from_transaction(); - -create function "{{.Bucket}}".set_effective_volumes() - returns trigger - security definer - language plpgsql -as -$$ -begin - new.post_commit_effective_volumes = coalesce(( - select ( - (post_commit_effective_volumes).inputs + case when new.is_source then 0 else new.amount end, - (post_commit_effective_volumes).outputs + case when new.is_source then new.amount else 0 end - ) - from "{{.Bucket}}".moves - where accounts_seq = new.accounts_seq - and asset = new.asset - and ledger = new.ledger - and (effective_date < new.effective_date or (effective_date = new.effective_date and seq < new.seq)) - order by effective_date desc, seq desc - limit 1 - ), ( - case when new.is_source then 0 else new.amount end, - case when new.is_source then new.amount else 0 end - )); - - return new; -end; -$$; - -create trigger "{{.Bucket}}_set_effective_volumes" -before insert -on "{{.Bucket}}"."moves" -for each row -execute procedure "{{.Bucket}}".set_effective_volumes(); +-- execute procedure "{{.Bucket}}".set_effective_volumes(); +-- +-- create function "{{.Bucket}}".update_effective_volumes() +-- returns trigger +-- security definer +-- language plpgsql +-- as +-- $$ +-- begin +-- update "{{.Bucket}}".moves +-- set post_commit_effective_volumes = +-- ( +-- (post_commit_effective_volumes).inputs + case when new.is_source then 0 else new.amount end, +-- (post_commit_effective_volumes).outputs + case when new.is_source then new.amount else 0 end +-- ) +-- where accounts_seq = new.accounts_seq +-- and asset = new.asset +-- and effective_date > new.effective_date +-- and ledger = new.ledger; +-- +-- return new; +-- end; +-- $$; +-- +-- create trigger "{{.Bucket}}_update_effective_volumes" +-- after insert +-- on "{{.Bucket}}"."moves" +-- for each row +-- execute procedure "{{.Bucket}}".update_effective_volumes(); -create function "{{.Bucket}}".update_effective_volumes() - returns trigger - security definer - language plpgsql -as -$$ -begin - update "{{.Bucket}}".moves - set post_commit_effective_volumes = - ( - (post_commit_effective_volumes).inputs + case when new.is_source then 0 else new.amount end, - (post_commit_effective_volumes).outputs + case when new.is_source then new.amount else 0 end - ) - where accounts_seq = new.accounts_seq - and asset = new.asset - and effective_date > new.effective_date - and ledger = new.ledger; +-- todo: need to populate balances with existing data +create table "{{.Bucket}}".balances ( + ledger varchar, + account varchar, + asset varchar, + balance numeric, - return new; -end; -$$; + primary key (ledger, account, asset) +); -create trigger "{{.Bucket}}_update_effective_volumes" - after insert - on "{{.Bucket}}"."moves" - for each row -execute procedure "{{.Bucket}}".update_effective_volumes(); \ No newline at end of file +alter table "{{.Bucket}}".moves +alter column post_commit_volumes +drop not null, +alter column post_commit_effective_volumes +drop not null; \ No newline at end of file diff --git a/components/ledger/internal/storage/ledger/balances.go b/components/ledger/internal/storage/ledger/balances.go index a59b24e1e6..83cccd7bc7 100644 --- a/components/ledger/internal/storage/ledger/balances.go +++ b/components/ledger/internal/storage/ledger/balances.go @@ -13,6 +13,15 @@ import ( "github.com/uptrace/bun" ) +type Balances struct { + bun.BaseModel `bun:"balances"` + + Ledger string `bun:"ledger,type:varchar"` + Account string `bun:"account,type:varchar"` + Asset string `bun:"asset,type:varchar"` + Balance *big.Int `bun:"balance,type:numeric"` +} + func (s *Store) GetAggregatedBalances(ctx context.Context, q ledgercontroller.GetAggregatedBalanceQuery) (ledger.BalancesByAssets, error) { var ( @@ -155,3 +164,24 @@ func (s *Store) GetBalance(ctx context.Context, address, asset string) (*big.Int return v.Balance, nil } + +func (s *Store) AddToBalance(ctx context.Context, addr, asset string, amount *big.Int) (*big.Int, error) { + r := &Balances{ + Ledger: s.ledgerName, + Account: addr, + Asset: asset, + Balance: amount, + } + _, err := s.db.NewInsert(). + Model(r). + ModelTableExpr(s.PrefixWithBucket("balances")). + On("conflict (ledger, account, asset) do update"). + Set("balance = balances.balance + excluded.balance"). + Returning("balance"). + Exec(ctx) + if err != nil { + return nil, err + } + + return r.Balance, err +} diff --git a/components/ledger/internal/storage/ledger/balances_test.go b/components/ledger/internal/storage/ledger/balances_test.go index 18733a23c3..7839f5f6f4 100644 --- a/components/ledger/internal/storage/ledger/balances_test.go +++ b/components/ledger/internal/storage/ledger/balances_test.go @@ -159,3 +159,18 @@ func TestGetBalancesAggregated(t *testing.T) { }, ret) }) } + +func TestAddToBalance(t *testing.T) { + t.Parallel() + + store := newLedgerStore(t) + ctx := logging.TestingContext() + + balance, err := store.AddToBalance(ctx, "world", "USD/2", big.NewInt(-100)) + require.NoError(t, err) + require.Equal(t, int64(-100), balance.Int64()) + + balance, err = store.AddToBalance(ctx, "world", "USD/2", big.NewInt(50)) + require.NoError(t, err) + require.Equal(t, int64(-50), balance.Int64()) +} diff --git a/components/ledger/internal/storage/ledger/moves.go b/components/ledger/internal/storage/ledger/moves.go index 90b32b1528..5ab23fb1ea 100644 --- a/components/ledger/internal/storage/ledger/moves.go +++ b/components/ledger/internal/storage/ledger/moves.go @@ -42,8 +42,10 @@ func (s *Store) InsertMoves(ctx context.Context, moves ...ledger.Move) error { } })) + // todo: rewrite, we just need to basically insert moves _, err := s.db.NewInsert(). With("_rows", s.db.NewValues(mappedMoves)). + //todo: we should already have the sequence when using UpsertAccount With("_account_sequences", s.db.NewSelect(). Column("seq", "address"). @@ -54,22 +56,22 @@ func (s *Store) InsertMoves(ctx context.Context, moves ...ledger.Move) error { s.db.NewSelect(). ColumnExpr("_rows.*"). ColumnExpr("_account_sequences.seq as accounts_seq"). - ColumnExpr("("+ - "coalesce(((last_move_by_seq.post_commit_volumes).inputs), 0) + case when is_source then 0 else amount end, "+ - "coalesce(((last_move_by_seq.post_commit_volumes).outputs), 0) + case when is_source then amount else 0 end"+ - ")::"+s.PrefixWithBucket("volumes")+" as post_commit_volumes"). + //ColumnExpr("("+ + // "coalesce(((last_move_by_seq.post_commit_volumes).inputs), 0) + case when is_source then 0 else amount end, "+ + // "coalesce(((last_move_by_seq.post_commit_volumes).outputs), 0) + case when is_source then amount else 0 end"+ + // ")::"+s.PrefixWithBucket("volumes")+" as post_commit_volumes"). Join("join _account_sequences on _account_sequences.address = address"). - Join("left join lateral ("+ - s.db.NewSelect(). - ColumnExpr("last_move.post_commit_volumes"). - ModelTableExpr(s.PrefixWithBucketUsingModel(Move{})+" as last_move"). - Where("_rows.account_address = last_move.account_address"). - Where("_rows.asset = last_move.asset"). - Where("_rows.ledger = last_move.ledger"). - Order("seq desc"). - Limit(1). - String()+ - ") last_move_by_seq on true"). + //Join("left join lateral ("+ + // s.db.NewSelect(). + // ColumnExpr("last_move.post_commit_volumes"). + // ModelTableExpr(s.PrefixWithBucketUsingModel(Move{})+" as last_move"). + // Where("_rows.account_address = last_move.account_address"). + // Where("_rows.asset = last_move.asset"). + // Where("_rows.ledger = last_move.ledger"). + // Order("seq desc"). + // Limit(1). + // String()+ + // ") last_move_by_seq on true"). Table("_rows"), ). Model(&Move{}). @@ -84,7 +86,7 @@ func (s *Store) InsertMoves(ctx context.Context, moves ...ledger.Move) error { "insertion_date", "effective_date", "accounts_seq", - "post_commit_volumes", + //"post_commit_volumes", ). ModelTableExpr(s.PrefixWithBucketUsingModel(Move{})). Table("_computed_rows"). diff --git a/components/ledger/test/performance/report_after_pcv b/components/ledger/test/performance/report_after_pcv new file mode 100644 index 0000000000..b5d70c6f40 --- /dev/null +++ b/components/ledger/test/performance/report_after_pcv @@ -0,0 +1,59 @@ +goos: darwin +goarch: arm64 +pkg: github.com/formancehq/ledger/test/performance +BenchmarkWrite/world_to_bank/testserver 282 4402464 ns/op 3.000 ms/transaction 227.1 t/s +--- BENCH: BenchmarkWrite/world_to_bank/testserver + server.go:176: Start testing server + server.go:177: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33882/907979e5-5f0c-4486-a348-358fdbf935f1?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:270: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33882/907979e5-5f0c-4486-a348-358fdbf935f1?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:149: Longest transaction: 1 (19.819084ms) + server.go:179: Stop testing server + server.go:176: Start testing server + server.go:177: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33882/643d4972-7f55-4f60-ba20-d69916427d27?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:270: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33882/643d4972-7f55-4f60-ba20-d69916427d27?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:149: Longest transaction: 1 (11.789833ms) + server.go:179: Stop testing server + ... [output truncated] +BenchmarkWrite/world_to_bank/core 308 9504116 ns/op 9.000 ms/transaction 105.2 t/s +--- BENCH: BenchmarkWrite/world_to_bank/core + write_test.go:149: Longest transaction: 1 (6.893833ms) + write_test.go:149: Longest transaction: 1 (9.1045ms) + write_test.go:149: Longest transaction: 107 (143.853708ms) +BenchmarkWrite/world_to_not_existing_destination/testserver 238 4856353 ns/op 4.000 ms/transaction 205.7 t/s +--- BENCH: BenchmarkWrite/world_to_not_existing_destination/testserver + server.go:176: Start testing server + server.go:177: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33882/98a25619-91a1-4bce-af7f-567baaa1639c?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:270: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33882/98a25619-91a1-4bce-af7f-567baaa1639c?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:149: Longest transaction: 1 (10.344792ms) + server.go:179: Stop testing server + server.go:176: Start testing server + server.go:177: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33882/77ac79de-09de-4a5c-a557-41384fd19354?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:270: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33882/77ac79de-09de-4a5c-a557-41384fd19354?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:149: Longest transaction: 49 (15.510917ms) + server.go:179: Stop testing server + ... [output truncated] +BenchmarkWrite/world_to_not_existing_destination/core 290 4512354 ns/op 4.000 ms/transaction 221.5 t/s +--- BENCH: BenchmarkWrite/world_to_not_existing_destination/core + write_test.go:149: Longest transaction: 1 (9.88ms) + write_test.go:149: Longest transaction: 26 (14.769875ms) + write_test.go:149: Longest transaction: 5 (19.295167ms) +BenchmarkWrite/not_existing_source_to_not_existing_destination/testserver 250 4776138 ns/op 4.000 ms/transaction 209.3 t/s +--- BENCH: BenchmarkWrite/not_existing_source_to_not_existing_destination/testserver + server.go:176: Start testing server + server.go:177: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33882/93861f4a-24c6-4acf-b753-e4aab423d5b0?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:270: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33882/93861f4a-24c6-4acf-b753-e4aab423d5b0?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:149: Longest transaction: 1 (13.181166ms) + server.go:179: Stop testing server + server.go:176: Start testing server + server.go:177: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33882/34f64a46-4faf-4f57-ab69-824bf9626e6c?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:270: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33882/34f64a46-4faf-4f57-ab69-824bf9626e6c?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:149: Longest transaction: 1 (18.103875ms) + server.go:179: Stop testing server + ... [output truncated] +BenchmarkWrite/not_existing_source_to_not_existing_destination/core 265 4495656 ns/op 4.000 ms/transaction 222.4 t/s +--- BENCH: BenchmarkWrite/not_existing_source_to_not_existing_destination/core + write_test.go:149: Longest transaction: 1 (8.498291ms) + write_test.go:149: Longest transaction: 36 (10.763875ms) + write_test.go:149: Longest transaction: 175 (12.974833ms) +PASS +ok github.com/formancehq/ledger/test/performance 19.318s diff --git a/components/ledger/test/performance/report_before_pcv b/components/ledger/test/performance/report_before_pcv new file mode 100644 index 0000000000..06be3e111e --- /dev/null +++ b/components/ledger/test/performance/report_before_pcv @@ -0,0 +1,59 @@ +goos: darwin +goarch: arm64 +pkg: github.com/formancehq/ledger/test/performance +BenchmarkWrite/world_to_bank/testserver 253 5281339 ns/op 4.000 ms/transaction 189.3 t/s +--- BENCH: BenchmarkWrite/world_to_bank/testserver + server.go:176: Start testing server + server.go:177: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33881/6f36ddca-4445-47ea-90d8-136c75a07b11?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:270: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33881/6f36ddca-4445-47ea-90d8-136c75a07b11?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:149: Longest transaction: 1 (17.257416ms) + server.go:179: Stop testing server + server.go:176: Start testing server + server.go:177: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33881/5e587752-610b-409f-a3eb-ee954d70a2b6?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:270: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33881/5e587752-610b-409f-a3eb-ee954d70a2b6?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:149: Longest transaction: 1 (11.969709ms) + server.go:179: Stop testing server + ... [output truncated] +BenchmarkWrite/world_to_bank/core 261 5260516 ns/op 4.000 ms/transaction 190.1 t/s +--- BENCH: BenchmarkWrite/world_to_bank/core + write_test.go:149: Longest transaction: 1 (12.220375ms) + write_test.go:149: Longest transaction: 1 (11.347458ms) + write_test.go:149: Longest transaction: 126 (14.59925ms) +BenchmarkWrite/world_to_not_existing_destination/testserver 231 5804882 ns/op 5.000 ms/transaction 172.3 t/s +--- BENCH: BenchmarkWrite/world_to_not_existing_destination/testserver + server.go:176: Start testing server + server.go:177: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33881/5a27d6df-bf21-4712-9ba9-2760ab5cff7a?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:270: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33881/5a27d6df-bf21-4712-9ba9-2760ab5cff7a?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:149: Longest transaction: 1 (13.981958ms) + server.go:179: Stop testing server + server.go:176: Start testing server + server.go:177: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33881/8e130bf2-2612-4906-8416-3766d40301ee?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:270: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33881/8e130bf2-2612-4906-8416-3766d40301ee?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:149: Longest transaction: 1 (11.551417ms) + server.go:179: Stop testing server + ... [output truncated] +BenchmarkWrite/world_to_not_existing_destination/core 248 5191606 ns/op 4.000 ms/transaction 192.6 t/s +--- BENCH: BenchmarkWrite/world_to_not_existing_destination/core + write_test.go:149: Longest transaction: 1 (10.171791ms) + write_test.go:149: Longest transaction: 69 (11.001625ms) + write_test.go:149: Longest transaction: 187 (14.4985ms) +BenchmarkWrite/not_existing_source_to_not_existing_destination/testserver 225 4827764 ns/op 4.000 ms/transaction 207.0 t/s +--- BENCH: BenchmarkWrite/not_existing_source_to_not_existing_destination/testserver + server.go:176: Start testing server + server.go:177: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33881/bbb3b017-9bea-4318-89d4-d28794f2f038?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:270: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33881/bbb3b017-9bea-4318-89d4-d28794f2f038?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:149: Longest transaction: 1 (14.75375ms) + server.go:179: Stop testing server + server.go:176: Start testing server + server.go:177: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33881/ccc5db1b-e650-4b27-848c-e9d7aca98d1a?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:270: Starting application with flags: serve --bind :0 --postgres-uri postgresql://root:root@127.0.0.1:33881/ccc5db1b-e650-4b27-848c-e9d7aca98d1a?sslmode=disable --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s --postgres-max-idle-conns 100 --postgres-max-open-conns 100 --postgres-conn-max-idle-time 1m0s + write_test.go:149: Longest transaction: 1 (16.87175ms) + server.go:179: Stop testing server + ... [output truncated] +BenchmarkWrite/not_existing_source_to_not_existing_destination/core 243 4912499 ns/op 4.000 ms/transaction 203.5 t/s +--- BENCH: BenchmarkWrite/not_existing_source_to_not_existing_destination/core + write_test.go:149: Longest transaction: 1 (10.010416ms) + write_test.go:149: Longest transaction: 1 (13.101166ms) + write_test.go:149: Longest transaction: 1 (12.813792ms) +PASS +ok github.com/formancehq/ledger/test/performance 17.832s diff --git a/tests/integration/go.mod b/tests/integration/go.mod index ce6b3f4325..f1bb5cf93c 100644 --- a/tests/integration/go.mod +++ b/tests/integration/go.mod @@ -246,8 +246,8 @@ require ( go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect go.opentelemetry.io/otel/log v0.3.0 // indirect go.opentelemetry.io/otel/metric v1.29.0 // indirect - go.opentelemetry.io/otel/sdk v1.28.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.28.0 // indirect + go.opentelemetry.io/otel/sdk v1.29.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.29.0 // indirect go.opentelemetry.io/otel/trace v1.29.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.temporal.io/api v1.18.1 // indirect diff --git a/tests/integration/go.sum b/tests/integration/go.sum index d1dc40f23b..5cc2db7745 100644 --- a/tests/integration/go.sum +++ b/tests/integration/go.sum @@ -1415,8 +1415,10 @@ go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdga go.opentelemetry.io/otel/sdk v1.2.0/go.mod h1:jNN8QtpvbsKhgaC6V5lHiejMoKD+V8uadoSafgHPx1U= go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= go.opentelemetry.io/otel/sdk/metric v1.28.0 h1:OkuaKgKrgAbYrrY0t92c+cC+2F6hsFNnCQArXCKlg08= go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg= +go.opentelemetry.io/otel/sdk/metric v1.29.0/go.mod h1:6zZLdCl2fkauYoZIOn/soQIDSWFmNSRcICarHfuhNJQ= go.opentelemetry.io/otel/trace v1.2.0/go.mod h1:N5FLswTubnxKxOJHM7XZC074qpeEdLy3CgAVsdMucK0= go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ=