Merge branch 'main' into update-run-locally-docs

.github/workflows/main.yml

@@ -43,7 +43,7 @@ jobs:
           GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
   GoReleaser:
     runs-on: "formance-runner"
-    if: contains(github.event.pull_request.labels.*.name, 'build-images')
+    if: contains(github.event.pull_request.labels.*.name, 'build-images') || github.ref == 'refs/heads/main'
     steps:
       - uses: 'actions/checkout@v4'
         with:

components/gateway/go.mod

@@ -110,7 +110,7 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/cel-go v0.15.1 // indirect
 	github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect
-	github.com/google/uuid v1.3.1 // indirect
+	github.com/google/uuid v1.3.1
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/gorilla/schema v1.2.0 // indirect
 	github.com/gorilla/securecookie v1.1.1 // indirect

components/gateway/internal/audit/messages/audit.go

@@ -2,7 +2,6 @@ package messages
 
 import (
 	"net/http"
-	"net/url"
 	"time"
 
 	"github.com/formancehq/stack/libs/go-libs/publish"
@@ -17,7 +16,7 @@ const (
 
 type HttpRequest struct {
 	Method string      `json:"method"`
-	Url    *url.URL    `json:"path"`
+	Path   string      `json:"path"`
 	Host   string      `json:"host"`
 	Header http.Header `json:"header"`
 	Body   string      `json:"body,omitempty"`

components/gateway/pkg/plugins/audit.go

@@ -287,7 +287,7 @@ func (a *Audit) provisionKafkaPublisher() error {
 func (a Audit) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
 	request := messages.HttpRequest{
 		Method: r.Method,
-		Url:    r.URL,
+		Path:   r.URL.Path,
 		Host:   r.Host,
 		Header: r.Header,
 		Body:   "",

components/ledger/Earthfile

@@ -15,7 +15,7 @@ sources:
     COPY main.go .
     SAVE ARTIFACT /src
 
-generate-mocks:
+generate:
     FROM core+builder-image
     RUN apk update && apk add openjdk11
     DO --pass-args core+GO_INSTALL --package=go.uber.org/mock/mockgen@latest
@@ -52,7 +52,7 @@ tests:
     FROM core+builder-image
     COPY (+sources/*) /src
     WORKDIR /src/components/ledger
-    COPY --dir --pass-args (+generate-mocks/*) .
+    COPY --dir --pass-args (+generate/*) .
     WITH DOCKER --pull=postgres:15-alpine
         DO --pass-args core+GO_TESTS
     END

components/ledger/internal/account.go

@@ -52,6 +52,11 @@ func (v ExpandedAccount) Copy() ExpandedAccount {
 	return v
 }
 
-const AccountPattern = "^[a-zA-Z_]+[a-zA-Z0-9_:]*$"
+const AccountSegmentRegex = "[a-zA-Z0-9_]+"
+const AccountPattern = "^" + AccountSegmentRegex + "(:" + AccountSegmentRegex + ")*$"
 
 var AccountRegexp = regexp.MustCompile(AccountPattern)
+
+func ValidateAddress(addr string) bool {
+	return AccountRegexp.Match([]byte(addr))
+}

components/ledger/internal/account_test.go

@@ -0,0 +1,57 @@
+package ledger
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestValidateAddress(t *testing.T) {
+	t.Parallel()
+
+	type testCase struct {
+		name       string
+		address    string
+		shouldBeOk bool
+	}
+
+	testsCases := []testCase{
+		{
+			name:       "nominal",
+			address:    "foo:bar",
+			shouldBeOk: true,
+		},
+		{
+			name:       "short segment",
+			address:    "a:b",
+			shouldBeOk: true,
+		},
+		{
+			name:       "only one segment",
+			address:    "a",
+			shouldBeOk: true,
+		},
+		{
+			name:       "using underscore as first char",
+			address:    "_a",
+			shouldBeOk: true,
+		},
+		{
+			name:       "using digits",
+			address:    "_0123",
+			shouldBeOk: true,
+		},
+		{
+			name:       "using empty segment",
+			address:    "a:",
+			shouldBeOk: false,
+		},
+	}
+
+	for _, testCase := range testsCases {
+		testCase := testCase
+		t.Run(testCase.name, func(t *testing.T) {
+			require.Equal(t, testCase.shouldBeOk, ValidateAddress(testCase.address))
+		})
+	}
+}

components/ledger/internal/machine/script/NumScript.g4

@@ -45,7 +45,7 @@ SAVE: 'save';
 NUMBER: [0-9]+;
 PERCENT: '%';
 VARIABLE_NAME: '$' [a-z_]+ [a-z0-9_]*;
-ACCOUNT: '@' [a-zA-Z_]+ [a-zA-Z0-9_:]*;
+ACCOUNT: '@' [a-zA-Z0-9_]+ [a-zA-Z0-9_:]*;
 ASSET: [A-Z/0-9]+;
 
 monetary: LBRACK asset=expression amt=NUMBER RBRACK;

components/ledger/internal/machine/vm/machine_test.go

@@ -1554,7 +1554,7 @@ func TestSetVarsFromJSON(t *testing.T) {
 			vars: map[string]string{
 				"dest": "invalid-acc",
 			},
-			expectedError: fmt.Errorf("invalid JSON value for variable $dest of type account: value invalid-acc: accounts should respect pattern ^[a-zA-Z_]+[a-zA-Z0-9_:]*$"),
+			expectedError: fmt.Errorf("invalid JSON value for variable $dest of type account: value invalid-acc: accounts should respect pattern ^[a-zA-Z0-9_]+(:[a-zA-Z0-9_]+)*$"),
 		},
 	} {
 		tc := tc

components/ledger/internal/machine/vm/run_test.go

@@ -281,11 +281,11 @@ var runTestCases = []runTestCase{
 		script: `
 		send [USD/2 100] (
 		  	source = @users:001 allowing unbounded overdraft
-		  	destination = @users:002
+		  	destination = @123:users:002
 		)`,
 		expectResult: Result{
 			Postings: []ledger.Posting{
-				ledger.NewPosting("users:001", "users:002", "USD/2", big.NewInt(100)),
+				ledger.NewPosting("users:001", "123:users:002", "USD/2", big.NewInt(100)),
 			},
 			Metadata:        metadata.Metadata{},
 			AccountMetadata: map[string]metadata.Metadata{},

components/ledger/internal/posting.go

@@ -4,7 +4,6 @@ import (
 	"database/sql/driver"
 	"encoding/json"
 	"math/big"
-	"regexp"
 
 	"github.com/pkg/errors"
 )
@@ -58,14 +57,6 @@ func (p *Postings) Scan(value interface{}) error {
 	}
 }
 
-// Account addresses are composed of segments separated by colons.
-// Each segment contains only the following characters: a to z (lower or upper case) and/or digits and/or the special character "_".
-var addressRegexp = regexp.MustCompile(`^\w+(:\w+)*$`)
-
-func ValidateAddress(addr string) bool {
-	return addressRegexp.Match([]byte(addr))
-}
-
 func (p Postings) Validate() (int, error) {
 	for i, p := range p {
 		if p.Amount.Cmp(Zero) < 0 {

components/ledger/libs/Earthfile

@@ -0,0 +1,21 @@
+VERSION --arg-scope-and-set --pass-args 0.7
+
+ARG core=github.com/formancehq/earthly:v0.5.2
+IMPORT $core AS core
+
+FROM core+base-image
+
+sources:
+    WORKDIR src
+    COPY go.* .
+    COPY *.go .
+    SAVE ARTIFACT /src
+
+tidy:
+    FROM core+builder-image
+    COPY (+sources/*) /src
+    WORKDIR /src
+    RUN --mount=type=cache,id=gomod,target=${GOPATH}/pkg/mod \
+        --mount=type=cache,id=gobuild,target=/root/.cache/go-build \
+        go mod tidy
+    SAVE ARTIFACT go.* AS LOCAL ./

components/ledger/libs/go.mod

@@ -48,6 +48,7 @@ require (
 	go.opentelemetry.io/otel/sdk/metric v0.39.0
 	go.opentelemetry.io/otel/trace v1.16.0
 	go.uber.org/fx v1.19.1
+	go.uber.org/zap v1.24.0
 )
 
 require (
@@ -144,7 +145,6 @@ require (
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/dig v1.16.1 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
-	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/crypto v0.6.0 // indirect
 	golang.org/x/mod v0.8.0 // indirect
 	golang.org/x/net v0.10.0 // indirect

components/ledger/libs/logging/zapadapter.go

@@ -0,0 +1,67 @@
+package logging
+
+// https://github.com/garsue/watermillzap/blob/master/adapter.go
+import (
+	"github.com/ThreeDotsLabs/watermill"
+	"go.uber.org/zap"
+)
+
+// Logger implements watermill.LoggerAdapter with *zap.Logger.
+type ZapLoggerAdapter struct {
+	backend *zap.Logger
+	fields  watermill.LogFields
+}
+
+// NewLogger returns new watermill.LoggerAdapter using passed *zap.Logger as backend.
+func NewZapLoggerAdapter(z *zap.Logger) watermill.LoggerAdapter {
+	return &ZapLoggerAdapter{backend: z}
+}
+
+// Error writes error log with message, error and some fields.
+func (l *ZapLoggerAdapter) Error(msg string, err error, fields watermill.LogFields) {
+	fields = l.fields.Add(fields)
+	fs := make([]zap.Field, 0, len(fields)+1)
+	fs = append(fs, zap.Error(err))
+	for k, v := range fields {
+		fs = append(fs, zap.Any(k, v))
+	}
+	l.backend.Error(msg, fs...)
+}
+
+// Info writes info log with message and some fields.
+func (l *ZapLoggerAdapter) Info(msg string, fields watermill.LogFields) {
+	fields = l.fields.Add(fields)
+	fs := make([]zap.Field, 0, len(fields)+1)
+	for k, v := range fields {
+		fs = append(fs, zap.Any(k, v))
+	}
+	l.backend.Info(msg, fs...)
+}
+
+// Debug writes debug log with message and some fields.
+func (l *ZapLoggerAdapter) Debug(msg string, fields watermill.LogFields) {
+	fields = l.fields.Add(fields)
+	fs := make([]zap.Field, 0, len(fields)+1)
+	for k, v := range fields {
+		fs = append(fs, zap.Any(k, v))
+	}
+	l.backend.Debug(msg, fs...)
+}
+
+// Trace writes debug log instead of trace log because zap does not support trace level logging.
+func (l *ZapLoggerAdapter) Trace(msg string, fields watermill.LogFields) {
+	fields = l.fields.Add(fields)
+	fs := make([]zap.Field, 0, len(fields)+1)
+	for k, v := range fields {
+		fs = append(fs, zap.Any(k, v))
+	}
+	l.backend.Debug(msg, fs...)
+}
+
+// With returns new LoggerAdapter with passed fields.
+func (l *ZapLoggerAdapter) With(fields watermill.LogFields) watermill.LoggerAdapter {
+	return &ZapLoggerAdapter{
+		backend: l.backend,
+		fields:  l.fields.Add(fields),
+	}
+}

components/ledger/libs/publish/kafka.go

@@ -88,7 +88,7 @@ func ProvideSaramaOption(options ...SaramaOption) fx.Option {
 
 type clientId string
 
-func newSaramaConfig(clientId clientId, version sarama.KafkaVersion, options ...SaramaOption) *sarama.Config {
+func NewSaramaConfig(clientId clientId, version sarama.KafkaVersion, options ...SaramaOption) *sarama.Config {
 	config := sarama.NewConfig()
 	config.ClientID = string(clientId)
 	config.Version = version
@@ -100,7 +100,7 @@ func newSaramaConfig(clientId clientId, version sarama.KafkaVersion, options ...
 	return config
 }
 
-func newKafkaPublisher(logger watermill.LoggerAdapter, config *sarama.Config, marshaller kafka.Marshaler, brokers ...string) (*kafka.Publisher, error) {
+func NewKafkaPublisher(logger watermill.LoggerAdapter, config *sarama.Config, marshaller kafka.Marshaler, brokers ...string) (*kafka.Publisher, error) {
 	return kafka.NewPublisher(kafka.PublisherConfig{
 		Brokers:               brokers,
 		Marshaler:             marshaller,
@@ -109,7 +109,7 @@ func newKafkaPublisher(logger watermill.LoggerAdapter, config *sarama.Config, ma
 	}, logger)
 }
 
-func newKafkaSubscriber(logger watermill.LoggerAdapter, config *sarama.Config,
+func NewKafkaSubscriber(logger watermill.LoggerAdapter, config *sarama.Config,
 	unmarshaler kafka.Unmarshaler, consumerGroup string, brokers ...string) (*kafka.Subscriber, error) {
 	return kafka.NewSubscriber(kafka.SubscriberConfig{
 		Brokers:               brokers,
@@ -126,9 +126,9 @@ func kafkaModule(clientId clientId, consumerGroup string, brokers ...string) fx.
 		fx.Supply(sarama.V1_0_0_0),
 		fx.Supply(fx.Annotate(kafka.DefaultMarshaler{}, fx.As(new(kafka.Marshaler)))),
 		fx.Supply(fx.Annotate(kafka.DefaultMarshaler{}, fx.As(new(kafka.Unmarshaler)))),
-		fx.Provide(fx.Annotate(newSaramaConfig, fx.ParamTags(``, ``, `group:"saramaOptions"`))),
+		fx.Provide(fx.Annotate(NewSaramaConfig, fx.ParamTags(``, ``, `group:"saramaOptions"`))),
 		fx.Provide(func(lc fx.Lifecycle, logger watermill.LoggerAdapter, marshaller kafka.Marshaler, config *sarama.Config) (*kafka.Publisher, error) {
-			ret, err := newKafkaPublisher(logger, config, marshaller, brokers...)
+			ret, err := NewKafkaPublisher(logger, config, marshaller, brokers...)
 			if err != nil {
 				return nil, err
 			}
@@ -140,7 +140,7 @@ func kafkaModule(clientId clientId, consumerGroup string, brokers ...string) fx.
 			return ret, nil
 		}),
 		fx.Provide(func(lc fx.Lifecycle, logger watermill.LoggerAdapter, unmarshaler kafka.Unmarshaler, config *sarama.Config) (*kafka.Subscriber, error) {
-			ret, err := newKafkaSubscriber(logger, config, unmarshaler, consumerGroup, brokers...)
+			ret, err := NewKafkaSubscriber(logger, config, unmarshaler, consumerGroup, brokers...)
 			if err != nil {
 				return nil, err
 			}

components/ledger/libs/publish/logging.go

@@ -34,6 +34,12 @@ func (w watermillLoggerAdapter) With(fields watermill.LogFields) watermill.Logge
 	}
 }
 
+func NewWatermillLoggerAdapter(logger logging.Logger) watermill.LoggerAdapter {
+	return watermillLoggerAdapter{
+		Logger: logger,
+	}
+}
+
 var _ watermill.LoggerAdapter = &watermillLoggerAdapter{}
 
 func defaultLoggingModule() fx.Option {

components/ledger/libs/publish/nats.go

@@ -12,7 +12,7 @@ import (
 	"go.uber.org/fx"
 )
 
-func newNatsConn(config wNats.PublisherConfig) (*nats.Conn, error) {
+func NewNatsConn(config wNats.PublisherConfig) (*nats.Conn, error) {
 	if err := config.Validate(); err != nil {
 		return nil, err
 	}
@@ -25,11 +25,11 @@ func newNatsConn(config wNats.PublisherConfig) (*nats.Conn, error) {
 	return conn, nil
 }
 
-func newNatsPublisherWithConn(conn *nats.Conn, logger watermill.LoggerAdapter, config wNats.PublisherConfig) (*wNats.Publisher, error) {
+func NewNatsPublisherWithConn(conn *nats.Conn, logger watermill.LoggerAdapter, config wNats.PublisherConfig) (*wNats.Publisher, error) {
 	return wNats.NewPublisherWithNatsConn(conn, config.GetPublisherPublishConfig(), logger)
 }
 
-func newNatsSubscriberWithConn(conn *nats.Conn, logger watermill.LoggerAdapter, config wNats.SubscriberConfig) (*wNats.Subscriber, error) {
+func NewNatsSubscriberWithConn(conn *nats.Conn, logger watermill.LoggerAdapter, config wNats.SubscriberConfig) (*wNats.Subscriber, error) {
 	return wNats.NewSubscriberWithNatsConn(conn, config.GetSubscriberSubscriptionConfig(), logger)
 }
 
@@ -42,10 +42,10 @@ func NatsModule(clientID, url, serviceName string) fx.Option {
 		nats.Name(clientID),
 	}
 	return fx.Options(
-		fx.Provide(newNatsConn),
+		fx.Provide(NewNatsConn),
 		fx.Provide(newNatsDefaultCallbacks),
-		fx.Provide(newNatsPublisherWithConn),
-		fx.Provide(newNatsSubscriberWithConn),
+		fx.Provide(NewNatsPublisherWithConn),
+		fx.Provide(NewNatsSubscriberWithConn),
 		fx.Provide(func(natsCallbacks NATSCallbacks) wNats.PublisherConfig {
 			natsOptions = append(natsOptions,
 				nats.ConnectHandler(natsCallbacks.ConnectedCB),

components/operator/apis/stack/v1beta3/migration_types.go

@@ -23,6 +23,7 @@ import (
 // MigrationSpec defines the desired state of Migration
 type MigrationSpec struct {
 	Configuration   string `json:"configuration"`
+	CurrentVersion  string `json:"currentVersion"`
 	Version         string `json:"version"`
 	Module          string `json:"module"`
 	TargetedVersion string `json:"targetedVersion"`