feat: move all payments amount to big int instead of int64 (#455)

components/operator/internal/controllers/stack/testdata/monopod-latest/results/migrations-stack.formance.com-v1beta3/payments-v0.9.1-pre-upgrade.yaml

@@ -0,0 +1,18 @@
+apiVersion: stack.formance.com/v1beta3
+kind: Migration
+metadata:
+    annotations:
+        reloader.stakater.com/auto: "true"
+    generation: 1
+    labels:
+        stack: "true"
+    name: payments-v0.9.1-pre-upgrade
+    namespace: monopod-latest
+spec:
+    configuration: monopod-latest
+    module: payments
+    postUpgrade: false
+    targetedVersion: v0.9.1
+    version: monopod-latest
+status:
+    terminated: true

components/operator/internal/controllers/stack/testdata/monopod-ledgerv1/results/migrations-stack.formance.com-v1beta3/payments-v0.9.1-pre-upgrade.yaml

@@ -0,0 +1,18 @@
+apiVersion: stack.formance.com/v1beta3
+kind: Migration
+metadata:
+    annotations:
+        reloader.stakater.com/auto: "true"
+    generation: 1
+    labels:
+        stack: "true"
+    name: payments-v0.9.1-pre-upgrade
+    namespace: monopod-ledgerv1
+spec:
+    configuration: monopod-ledgerv1
+    module: payments
+    postUpgrade: false
+    targetedVersion: v0.9.1
+    version: monopod-ledgerv1
+status:
+    terminated: true

components/operator/internal/controllers/stack/testdata/monopod-search-before-v0.7.0/results/migrations-stack.formance.com-v1beta3/payments-v0.9.1-pre-upgrade.yaml

@@ -0,0 +1,18 @@
+apiVersion: stack.formance.com/v1beta3
+kind: Migration
+metadata:
+    annotations:
+        reloader.stakater.com/auto: "true"
+    generation: 1
+    labels:
+        stack: "true"
+    name: payments-v0.9.1-pre-upgrade
+    namespace: monopod-search-before-v0-7-0
+spec:
+    configuration: monopod-search-before-v0-7-0
+    module: payments
+    postUpgrade: false
+    targetedVersion: v0.9.1
+    version: monopod-search-before-v0-7-0
+status:
+    terminated: true

components/operator/internal/controllers/stack/testdata/multipod-latest/results/migrations-stack.formance.com-v1beta3/payments-v0.9.1-pre-upgrade.yaml

@@ -0,0 +1,18 @@
+apiVersion: stack.formance.com/v1beta3
+kind: Migration
+metadata:
+    annotations:
+        reloader.stakater.com/auto: "true"
+    generation: 1
+    labels:
+        stack: "true"
+    name: payments-v0.9.1-pre-upgrade
+    namespace: multipod-latest
+spec:
+    configuration: multipod-latest
+    module: payments
+    postUpgrade: false
+    targetedVersion: v0.9.1
+    version: multipod-latest
+status:
+    terminated: true

components/operator/internal/handlers/handler_payments.go

@@ -125,6 +125,15 @@ func init() {
 					return paymentsServices(ctx, env)
 				},
 			},
+			"v0.9.1": {
+				PreUpgrade: func(ctx modules.Context) error {
+					// Add payment accounts
+					return paymentsPreUpgradeMigration(ctx)
+				},
+				Services: func(ctx modules.ModuleContext) modules.Services {
+					return paymentsServices(ctx, env)
+				},
+			},
 		},
 	})
 }

components/orchestration/internal/workflow/stages/send/run.go

@@ -2,7 +2,6 @@ package send
 
 import (
 	"fmt"
-	"math/big"
 	"strings"
 
 	"github.com/formancehq/formance-sdk-go/pkg/models/shared"
@@ -72,7 +71,7 @@ func runPaymentToWallet(ctx workflow.Context, source *PaymentSource, destination
 	}
 	if amount == nil {
 		amount = &shared.Monetary{
-			Amount: big.NewInt(payment.InitialAmount),
+			Amount: payment.InitialAmount,
 			Asset:  payment.Asset,
 		}
 	}
@@ -95,7 +94,7 @@ func savePayment(ctx workflow.Context, paymentID string) (*shared.Payment, error
 	reference := paymentAccountName(paymentID)
 	_, err = activities.CreateTransaction(internal.InfiniteRetryContext(ctx), internalLedger, shared.PostTransaction{
 		Postings: []shared.Posting{{
-			Amount:      big.NewInt(payment.InitialAmount),
+			Amount:      payment.InitialAmount,
 			Asset:       payment.Asset,
 			Destination: paymentAccountName(paymentID),
 			Source:      "world",
@@ -123,7 +122,7 @@ func runPaymentToAccount(ctx workflow.Context, source *PaymentSource, destinatio
 	}
 	if amount == nil {
 		amount = &shared.Monetary{
-			Amount: big.NewInt(payment.InitialAmount),
+			Amount: payment.InitialAmount,
 			Asset:  payment.Asset,
 		}
 	}
@@ -196,9 +195,8 @@ func runWalletToPayment(ctx workflow.Context, source *WalletSource, destination
 		return err
 	}
 
-	v := amount.Amount.Int64()
 	if err := activities.StripeTransfer(internal.InfiniteRetryContext(ctx), shared.StripeTransferRequest{
-		Amount:      &v,
+		Amount:      amount.Amount,
 		Asset:       &amount.Asset,
 		Destination: &stripeConnectID,
 	}); err != nil {
@@ -362,9 +360,8 @@ func runAccountToPayment(ctx workflow.Context, source *LedgerAccountSource, dest
 		return err
 	}
 
-	v := amount.Amount.Int64()
 	if err := activities.StripeTransfer(internal.InfiniteRetryContext(ctx), shared.StripeTransferRequest{
-		Amount:      &v,
+		Amount:      amount.Amount,
 		Asset:       &amount.Asset,
 		Destination: &stripeConnectID,
 	}); err != nil {

components/orchestration/internal/workflow/stages/send/run_test.go

@@ -124,7 +124,7 @@ var (
 				Returns: []any{
 					&shared.PaymentResponse{
 						Data: shared.Payment{
-							InitialAmount: 100,
+							InitialAmount: big.NewInt(100),
 							Asset:         "USD",
 							Provider:      shared.ConnectorStripe,
 							Status:        shared.PaymentStatusSucceeded,
@@ -243,7 +243,7 @@ var (
 				Returns: []any{
 					&shared.PaymentResponse{
 						Data: shared.Payment{
-							InitialAmount: 100,
+							InitialAmount: big.NewInt(100),
 							Asset:         "USD",
 							Provider:      shared.ConnectorStripe,
 							Status:        shared.PaymentStatusSucceeded,
@@ -344,7 +344,7 @@ var (
 				Returns: []any{
 					&shared.PaymentResponse{
 						Data: shared.Payment{
-							InitialAmount: 100,
+							InitialAmount: big.NewInt(100),
 							Asset:         "USD",
 							Provider:      shared.ConnectorStripe,
 							Status:        shared.PaymentStatusSucceeded,
@@ -688,7 +688,7 @@ var (
 				Activity: activities.StripeTransferActivity,
 				Args: []any{
 					mock.Anything, shared.StripeTransferRequest{
-						Amount:      ptrInt64(100),
+						Amount:      big.NewInt(100),
 						Asset:       ptrString("USD"),
 						Destination: ptrString("abcd"),
 					},
@@ -1025,7 +1025,7 @@ var (
 				Activity: activities.StripeTransferActivity,
 				Args: []any{
 					mock.Anything, shared.StripeTransferRequest{
-						Amount:      ptrInt64(100),
+						Amount:      big.NewInt(100),
 						Asset:       ptrString("USD"),
 						Destination: ptrString("abcd"),
 					},

components/payments/internal/app/api/balances.go

@@ -3,6 +3,7 @@ package api
 import (
 	"context"
 	"encoding/json"
+	"math/big"
 	"net/http"
 	"strconv"
 	"strings"
@@ -24,7 +25,7 @@ type balancesResponse struct {
 	CreatedAt     time.Time `json:"createdAt"`
 	LastUpdatedAt time.Time `json:"lastUpdatedAt"`
 	Currency      string    `json:"currency"`
-	Balance       int64     `json:"balance"`
+	Balance       *big.Int  `json:"balance"`
 }
 
 func listBalancesForAccount(repo balancesRepository) http.HandlerFunc {

components/payments/internal/app/api/connector.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"encoding/json"
+	"math/big"
 	"net/http"
 	"time"
 
@@ -254,16 +255,16 @@ func reset[Config models.ConnectorConfigObject](connectorManager *integration.Co
 }
 
 type transferRequest struct {
-	Amount      int64  `json:"amount"`
-	Source      string `json:"source"`
-	Destination string `json:"destination"`
-	Asset       string `json:"asset"`
+	Amount      *big.Int `json:"amount"`
+	Source      string   `json:"source"`
+	Destination string   `json:"destination"`
+	Asset       string   `json:"asset"`
 
 	currency string
 }
 
 func (req *transferRequest) validate() error {
-	if req.Amount <= 0 {
+	if req.Amount.Cmp(big.NewInt(0)) <= 0 {
 		return errors.New("amount must be greater than 0")
 	}
 
@@ -346,13 +347,13 @@ func initiateTransfer[Config models.ConnectorConfigObject](connectorManager *int
 }
 
 type listTransfersResponseElement struct {
-	ID          string  `json:"id"`
-	Source      string  `json:"source"`
-	Destination string  `json:"destination"`
-	Amount      int64   `json:"amount"`
-	Currency    string  `json:"asset"`
-	Status      string  `json:"status"`
-	Error       *string `json:"error"`
+	ID          string   `json:"id"`
+	Source      string   `json:"source"`
+	Destination string   `json:"destination"`
+	Amount      *big.Int `json:"amount"`
+	Currency    string   `json:"asset"`
+	Status      string   `json:"status"`
+	Error       *string  `json:"error"`
 }
 
 func listTransfers[Config models.ConnectorConfigObject](connectorManager *integration.ConnectorManager[Config],

components/payments/internal/app/api/payments.go

@@ -3,6 +3,7 @@ package api
 import (
 	"context"
 	"encoding/json"
+	"math/big"
 	"net/http"
 	"strings"
 	"time"
@@ -27,7 +28,7 @@ type paymentResponse struct {
 	Type                 string                   `json:"type"`
 	Provider             models.ConnectorProvider `json:"provider"`
 	Status               models.PaymentStatus     `json:"status"`
-	InitialAmount        int64                    `json:"initialAmount"`
+	InitialAmount        *big.Int                 `json:"initialAmount"`
 	Scheme               models.PaymentScheme     `json:"scheme"`
 	Asset                string                   `json:"asset"`
 	CreatedAt            time.Time                `json:"createdAt"`

components/payments/internal/app/connectors/bankingcircle/client/accounts.go

@@ -23,12 +23,12 @@ type Account struct {
 	OwnedByCompanyID string `json:"ownedByCompanyId"`
 	ProtectionType   string `json:"protectionType"`
 	Balances         []struct {
-		Type                     string  `json:"type"`
-		Currency                 string  `json:"currency"`
-		BeginOfDayAmount         float64 `json:"beginOfDayAmount"`
-		FinancialDate            string  `json:"financialDate"`
-		IntraDayAmount           float64 `json:"intraDayAmount"`
-		LastTransactionTimestamp string  `json:"lastTransactionTimestamp"`
+		Type                     string      `json:"type"`
+		Currency                 string      `json:"currency"`
+		BeginOfDayAmount         json.Number `json:"beginOfDayAmount"`
+		FinancialDate            string      `json:"financialDate"`
+		IntraDayAmount           json.Number `json:"intraDayAmount"`
+		LastTransactionTimestamp string      `json:"lastTransactionTimestamp"`
 	} `json:"balances"`
 }
 

components/payments/internal/app/connectors/bankingcircle/client/payments.go

@@ -34,8 +34,8 @@ type Payment struct {
 		} `json:"viban"`
 		InstructedDate interface{} `json:"instructedDate"`
 		DebitAmount    struct {
-			Currency string  `json:"currency"`
-			Amount   float64 `json:"amount"`
+			Currency string      `json:"currency"`
+			Amount   json.Number `json:"amount"`
 		} `json:"debitAmount"`
 		DebitValueDate time.Time   `json:"debitValueDate"`
 		FxRate         interface{} `json:"fxRate"`
@@ -46,8 +46,8 @@ type Payment struct {
 		DebtorName    interface{} `json:"debtorName"`
 		DebtorAddress interface{} `json:"debtorAddress"`
 		Amount        struct {
-			Currency string  `json:"currency"`
-			Amount   float64 `json:"amount"`
+			Currency string      `json:"currency"`
+			Amount   json.Number `json:"amount"`
 		} `json:"amount"`
 		ValueDate             interface{} `json:"valueDate"`
 		ChargeBearer          interface{} `json:"chargeBearer"`
@@ -70,8 +70,8 @@ type Payment struct {
 			Country              string `json:"country"`
 		} `json:"viban"`
 		CreditAmount struct {
-			Currency string  `json:"currency"`
-			Amount   float64 `json:"amount"`
+			Currency string      `json:"currency"`
+			Amount   json.Number `json:"amount"`
 		} `json:"creditAmount"`
 		CreditValueDate time.Time   `json:"creditValueDate"`
 		FxRate          interface{} `json:"fxRate"`

components/payments/internal/app/connectors/bankingcircle/task_fetch_accounts.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"math/big"
 	"time"
 
 	"github.com/formancehq/payments/internal/app/connectors/bankingcircle/client"
@@ -92,6 +93,15 @@ func ingestAccountsBatch(
 		})
 
 		for _, balance := range account.Balances {
+			var amount big.Float
+			_, ok := amount.SetString(balance.IntraDayAmount.String())
+			if !ok {
+				return fmt.Errorf("failed to parse amount %s", balance.IntraDayAmount)
+			}
+
+			var amountInt big.Int
+			amount.Mul(&amount, big.NewFloat(100)).Int(&amountInt)
+
 			balanceBatch = append(balanceBatch, &models.Balance{
 				AccountID: models.AccountID{
 					Reference: account.AccountID,
@@ -101,7 +111,7 @@ func ingestAccountsBatch(
 				// TODO(polo): do a complete pass on all connectors to
 				// normalize the currencies
 				Currency:      balance.Currency + "/2",
-				Balance:       int64(balance.IntraDayAmount * 100),
+				Balance:       &amountInt,
 				CreatedAt:     now,
 				LastUpdatedAt: now,
 			})

components/payments/internal/app/connectors/bankingcircle/task_fetch_payments.go

@@ -3,6 +3,8 @@ package bankingcircle
 import (
 	"context"
 	"encoding/json"
+	"fmt"
+	"math/big"
 
 	"github.com/formancehq/payments/internal/app/connectors/bankingcircle/client"
 	"github.com/formancehq/payments/internal/app/ingestion"
@@ -50,6 +52,15 @@ func ingestBatch(
 
 		paymentType := matchPaymentType(paymentEl.Classification)
 
+		var amount big.Float
+		_, ok := amount.SetString(paymentEl.Transfer.Amount.Amount.String())
+		if !ok {
+			return fmt.Errorf("failed to parse amount %s", paymentEl.Transfer.Amount.Amount.String())
+		}
+
+		var amountInt big.Int
+		amount.Mul(&amount, big.NewFloat(100)).Int(&amountInt)
+
 		batchElement := ingestion.PaymentBatchElement{
 			Payment: &models.Payment{
 				ID: models.PaymentID{
@@ -63,7 +74,7 @@ func ingestBatch(
 				Type:      paymentType,
 				Status:    matchPaymentStatus(paymentEl.Status),
 				Scheme:    models.PaymentSchemeOther,
-				Amount:    int64(paymentEl.Transfer.Amount.Amount * 100),
+				Amount:    &amountInt,
 				Asset:     models.PaymentAsset(paymentEl.Transfer.Amount.Currency + "/2"),
 				RawData:   raw,
 			},