test(ledger): add test server

Earthfile

@@ -1,7 +1,7 @@
 VERSION 0.8
 PROJECT FormanceHQ/stack
 
-IMPORT github.com/formancehq/earthly:tags/v0.14.0 AS core
+IMPORT github.com/formancehq/earthly:tags/v0.15.0 AS core
 
 sources:
     FROM core+base-image

components/Earthfile

@@ -1,6 +1,6 @@
 VERSION 0.8
 
-IMPORT github.com/formancehq/earthly:tags/v0.14.0 AS core
+IMPORT github.com/formancehq/earthly:tags/v0.15.0 AS core
 IMPORT .. AS stack
 
 deploy:

components/fctl/Earthfile

@@ -1,6 +1,6 @@
 VERSION 0.8
 
-IMPORT github.com/formancehq/earthly:tags/v0.14.0 AS core
+IMPORT github.com/formancehq/earthly:tags/v0.15.0 AS core
 IMPORT ../.. AS stack
 IMPORT ../../releases AS releases
 IMPORT .. AS components

components/ledger/Earthfile

@@ -1,6 +1,6 @@
 VERSION 0.8
 
-IMPORT github.com/formancehq/earthly:tags/v0.14.0 AS core
+IMPORT github.com/formancehq/earthly:tags/v0.15.0 AS core
 IMPORT ../.. AS stack
 IMPORT .. AS components
 
@@ -45,11 +45,47 @@ build-image:
 
 tests:
     FROM core+builder-image
+    RUN go install github.com/onsi/ginkgo/v2/ginkgo@latest
+
     COPY (+sources/*) /src
     WORKDIR /src/components/ledger
     COPY --dir --pass-args (+generate/*) .
-    WITH DOCKER --pull=postgres:15-alpine
-        DO --pass-args core+GO_TESTS
+
+    ARG includeIntegrationTests="true"
+    ARG coverage=""
+    ARG debug=false
+
+    ENV DEBUG=$debug
+    ENV CGO_ENABLED=1 # required for -race
+    RUN apk add gcc musl-dev
+
+    LET goFlags="-race"
+    IF [ "$coverage" = "true" ]
+        SET goFlags="$goFlags -covermode=atomic"
+        SET goFlags="$goFlags -coverpkg=github.com/formancehq/stack/components/ledger/internal/..."
+        SET goFlags="$goFlags,github.com/formancehq/stack/components/ledger/cmd/..."
+        SET goFlags="$goFlags -coverprofile cover.out"
+    END
+    IF [ "$includeIntegrationTests" = "true" ]
+        SET goFlags="$goFlags -tags it"
+        WITH DOCKER \
+            --pull=postgres:15-alpine \
+            --pull=clickhouse/clickhouse-server:head \
+            --pull=elasticsearch:8.14.3
+            RUN --mount type=cache,id=gopkgcache,target=${GOPATH}/pkg/mod \
+                --mount type=cache,id=gobuildcache,target=/root/.cache/go-build \
+                ginkgo -r -p $goFlags
+        END
+    ELSE
+        RUN --mount type=cache,id=gopkgcache,target=${GOPATH}/pkg/mod \
+            --mount type=cache,id=gobuildcache,target=/root/.cache/go-build \
+            ginkgo -r -p $goFlags
+    END
+    IF [ "$coverage" = "true" ]
+        # exclude files suffixed with _generated.go, these are mocks used by tests
+        RUN cat cover.out | grep -v "_generated.go" > cover2.out
+        RUN mv cover2.out cover.out
+        SAVE ARTIFACT cover.out AS LOCAL cover.out
     END
 
 deploy:
@@ -69,7 +105,7 @@ lint:
     COPY (+sources/*) /src
     COPY --pass-args +tidy/go.* .
     WORKDIR /src/components/ledger
-    DO --pass-args stack+GO_LINT
+    DO --pass-args stack+GO_LINT --ADDITIONAL_ARGUMENTS="--build-tags it"
     SAVE ARTIFACT cmd AS LOCAL cmd
     SAVE ARTIFACT internal AS LOCAL internal
     SAVE ARTIFACT pkg AS LOCAL pkg
@@ -138,4 +174,16 @@ tidy:
     DO --pass-args stack+GO_TIDY
 
 release:
-    BUILD --pass-args stack+goreleaser --path=components/ledger
+    BUILD --pass-args stack+goreleaser --path=components/ledger
+
+generate-sdk:
+    FROM node:20-alpine
+    RUN apk update && apk add yq git
+    WORKDIR /src
+    COPY (stack+speakeasy/speakeasy) /bin/speakeasy
+    ARG version=v0.0.0
+    COPY openapi/v2.yaml openapi.yaml
+    COPY --dir pkg/client client
+    RUN --secret SPEAKEASY_API_KEY speakeasy generate sdk -s ./openapi.yaml -o ./client -l go
+
+    SAVE ARTIFACT client AS LOCAL ./pkg/client

components/ledger/benchmarks/ledger_test.go

@@ -1,3 +1,5 @@
+//go:build it
+
 package benchmarks
 
 import (

components/ledger/cmd/container.go

@@ -18,7 +18,7 @@ func resolveOptions(cmd *cobra.Command, userOptions ...fx.Option) []fx.Option {
 	options := make([]fx.Option, 0)
 	options = append(options, fx.NopLogger)
 
-	numscriptCacheMaxCountFlag, _ := cmd.Flags().GetInt(numscriptCacheMaxCountFlag)
+	numscriptCacheMaxCountFlag, _ := cmd.Flags().GetInt(NumscriptCacheMaxCountFlag)
 	ledgerBatchSizeFlag, _ := cmd.Flags().GetInt(ledgerBatchSizeFlag)
 
 	options = append(options,

components/ledger/cmd/root.go

@@ -17,7 +17,7 @@ import (
 )
 
 const (
-	bindFlag = "bind"
+	BindFlag = "bind"
 )
 
 var (
@@ -49,7 +49,7 @@ func NewRootCommand() *cobra.Command {
 
 	root.AddCommand(NewDocCommand())
 
-	root.PersistentFlags().String(bindFlag, "0.0.0.0:3068", "API bind address")
+	root.PersistentFlags().String(BindFlag, "0.0.0.0:3068", "API bind address")
 
 	service.AddFlags(root.PersistentFlags())
 	otlpmetrics.AddFlags(root.PersistentFlags())

components/ledger/cmd/serve.go

@@ -19,21 +19,21 @@ import (
 )
 
 const (
-	ballastSizeInBytesFlag     = "ballast-size"
-	numscriptCacheMaxCountFlag = "numscript-cache-max-count"
+	BallastSizeInBytesFlag     = "ballast-size"
+	NumscriptCacheMaxCountFlag = "numscript-cache-max-count"
 	ledgerBatchSizeFlag        = "ledger-batch-size"
-	readOnlyFlag               = "read-only"
-	autoUpgradeFlag            = "auto-upgrade"
+	ReadOnlyFlag               = "read-only"
+	AutoUpgradeFlag            = "auto-upgrade"
 )
 
 func NewServe() *cobra.Command {
 	cmd := &cobra.Command{
 		Use: "serve",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			readOnly, _ := cmd.Flags().GetBool(readOnlyFlag)
-			autoUpgrade, _ := cmd.Flags().GetBool(autoUpgradeFlag)
-			ballastSize, _ := cmd.Flags().GetUint(ballastSizeInBytesFlag)
-			bind, _ := cmd.Flags().GetString(bindFlag)
+			readOnly, _ := cmd.Flags().GetBool(ReadOnlyFlag)
+			autoUpgrade, _ := cmd.Flags().GetBool(AutoUpgradeFlag)
+			ballastSize, _ := cmd.Flags().GetUint(BallastSizeInBytesFlag)
+			bind, _ := cmd.Flags().GetString(BindFlag)
 
 			return service.New(cmd.OutOrStdout(), resolveOptions(
 				cmd,
@@ -67,11 +67,11 @@ func NewServe() *cobra.Command {
 			)...).Run(cmd)
 		},
 	}
-	cmd.Flags().Uint(ballastSizeInBytesFlag, 0, "Ballast size in bytes, default to 0")
-	cmd.Flags().Int(numscriptCacheMaxCountFlag, 1024, "Numscript cache max count")
+	cmd.Flags().Uint(BallastSizeInBytesFlag, 0, "Ballast size in bytes, default to 0")
+	cmd.Flags().Int(NumscriptCacheMaxCountFlag, 1024, "Numscript cache max count")
 	cmd.Flags().Int(ledgerBatchSizeFlag, 50, "ledger batch size")
-	cmd.Flags().Bool(readOnlyFlag, false, "Read only mode")
-	cmd.Flags().Bool(autoUpgradeFlag, false, "Automatically upgrade all schemas")
+	cmd.Flags().Bool(ReadOnlyFlag, false, "Read only mode")
+	cmd.Flags().Bool(AutoUpgradeFlag, false, "Automatically upgrade all schemas")
 	return cmd
 }
 

components/ledger/go.mod

@@ -4,11 +4,18 @@ go 1.22
 
 toolchain go1.22.6
 
+replace github.com/formancehq/stack/ledger/client => ./pkg/client
+
+replace github.com/formancehq/stack/libs/go-libs => ../../libs/go-libs
+
+replace github.com/formancehq/stack/libs/core => ../../libs/core
+
 require (
-	github.com/ThreeDotsLabs/watermill v1.3.5
-	github.com/alitto/pond v1.8.3
+	github.com/ThreeDotsLabs/watermill v1.3.7
+	github.com/alitto/pond v1.9.2
 	github.com/antlr/antlr4/runtime/Go/antlr v1.4.10
 	github.com/bluele/gcache v0.0.2
+	github.com/formancehq/stack/ledger/client v0.0.0-00010101000000-000000000000
 	github.com/formancehq/stack/libs/core v0.0.0-00010101000000-000000000000
 	github.com/formancehq/stack/libs/go-libs v0.0.0-00010101000000-000000000000
 	github.com/go-chi/chi/v5 v5.1.0
@@ -18,17 +25,18 @@ require (
 	github.com/jackc/pgx/v5 v5.6.0
 	github.com/lib/pq v1.10.9
 	github.com/logrusorgru/aurora v2.0.3+incompatible
+	github.com/onsi/ginkgo/v2 v2.20.2
 	github.com/pborman/uuid v1.2.1
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0
-	github.com/uptrace/bun v1.2.1
-	github.com/uptrace/bun/dialect/pgdialect v1.2.1
-	go.opentelemetry.io/otel v1.28.0
-	go.opentelemetry.io/otel/metric v1.28.0
-	go.opentelemetry.io/otel/trace v1.28.0
+	github.com/uptrace/bun v1.2.3
+	github.com/uptrace/bun/dialect/pgdialect v1.2.3
+	go.opentelemetry.io/otel v1.29.0
+	go.opentelemetry.io/otel/metric v1.29.0
+	go.opentelemetry.io/otel/trace v1.29.0
 	go.uber.org/fx v1.22.2
 	go.uber.org/mock v0.4.0
 )
@@ -43,7 +51,6 @@ require (
 	github.com/ThreeDotsLabs/watermill-http/v2 v2.3.0 // indirect
 	github.com/ThreeDotsLabs/watermill-kafka/v3 v3.0.1 // indirect
 	github.com/ThreeDotsLabs/watermill-nats/v2 v2.0.2 // indirect
-	github.com/ajg/form v1.5.1 // indirect
 	github.com/aws/aws-msk-iam-sasl-signer-go v1.0.0 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.30.4 // indirect
 	github.com/aws/aws-sdk-go-v2/config v1.27.28 // indirect
@@ -64,26 +71,27 @@ require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dnwe/otelsarama v0.0.0-20231212173111-631a0a53d5d4 // indirect
 	github.com/docker/cli v26.1.4+incompatible // indirect
-	github.com/docker/docker v27.1.2+incompatible // indirect
+	github.com/docker/docker v27.1.1+incompatible // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/eapache/go-resiliency v1.7.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
+	github.com/ericlagergren/decimal v0.0.0-20221120152707-495c53812d05 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/go-chi/chi v4.1.2+incompatible // indirect
-	github.com/go-chi/render v1.0.2 // indirect
+	github.com/go-chi/chi v4.0.2+incompatible // indirect
+	github.com/go-chi/render v1.0.1 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/go-sql-driver/mysql v1.8.1 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
+	github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
-	github.com/gorilla/schema v1.4.1 // indirect
+	github.com/gorilla/schema v1.2.0 // indirect
 	github.com/gorilla/securecookie v1.1.1 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -113,17 +121,18 @@ require (
 	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
-	github.com/onsi/ginkgo/v2 v2.20.0 // indirect
+	github.com/onsi/gomega v1.34.2 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/opencontainers/runc v1.1.13 // indirect
 	github.com/ory/dockertest/v3 v3.11.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
+	github.com/puzpuzpuz/xsync/v3 v3.4.0 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/riandyrn/otelchi v0.9.0 // indirect
-	github.com/rs/cors v1.11.0 // indirect
+	github.com/rs/cors v1.10.1 // indirect
 	github.com/shirou/gopsutil/v4 v4.24.6 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/tklauser/go-sysconf v0.3.14 // indirect
@@ -166,7 +175,7 @@ require (
 	golang.org/x/net v0.28.0 // indirect
 	golang.org/x/oauth2 v0.20.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/sys v0.23.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
 	golang.org/x/text v0.17.0 // indirect
 	golang.org/x/tools v0.24.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
@@ -177,7 +186,3 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
-
-replace github.com/formancehq/stack/libs/go-libs => ../../libs/go-libs
-
-replace github.com/formancehq/stack/libs/core => ../../libs/core