feat(ledger): add capability to sort transactions by timestamp (#1606)

formancehq · Jul 19, 2024 · 7a8cb98 · 7a8cb98
1 parent d706e50
commit 7a8cb98
Show file tree

Hide file tree

Showing 13 changed files with 213 additions and 66 deletions.
diff --git a/components/ledger/internal/api/v2/controllers_transactions.go b/components/ledger/internal/api/v2/controllers_transactions.go
@@ -57,7 +57,16 @@ func getTransactions(w http.ResponseWriter, r *http.Request) {
 		if err != nil {
 			return nil, err
 		}
-		return pointer.For(ledgerstore.NewGetTransactionsQuery(*options)), nil
+		q := ledgerstore.NewGetTransactionsQuery(*options)
+
+		if r.URL.Query().Get("order") == "effective" {
+			q.Column = "timestamp"
+		}
+		if r.URL.Query().Get("reverse") == "true" {
+			q.Order = bunpaginate.OrderAsc
+		}
+
+		return pointer.For(q), nil
 	})
 	if err != nil {
 		sharedapi.BadRequest(w, ErrValidation, err)

diff --git a/components/ledger/internal/api/v2/controllers_transactions_test.go b/components/ledger/internal/api/v2/controllers_transactions_test.go
@@ -567,7 +567,7 @@ func TestGetTransactions(t *testing.T) {
 		name              string
 		queryParams       url.Values
 		body              string
-		expectQuery       ledgerstore.PaginatedQueryOptions[ledgerstore.PITFilterWithVolumes]
+		expectQuery       ledgerstore.GetTransactionsQuery
 		expectStatusCode  int
 		expectedErrorCode string
 	}
@@ -576,90 +576,90 @@ func TestGetTransactions(t *testing.T) {
 	testCases := []testCase{
 		{
 			name: "nominal",
-			expectQuery: ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
+			expectQuery: ledgerstore.NewGetTransactionsQuery(ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
 				PITFilter: ledgerstore.PITFilter{
 					PIT: &now,
 				},
-			}),
+			})),
 		},
 		{
 			name: "using metadata",
 			body: `{"$match": {"metadata[roles]": "admin"}}`,
-			expectQuery: ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
+			expectQuery: ledgerstore.NewGetTransactionsQuery(ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
 				PITFilter: ledgerstore.PITFilter{
 					PIT: &now,
 				},
 			}).
-				WithQueryBuilder(query.Match("metadata[roles]", "admin")),
+				WithQueryBuilder(query.Match("metadata[roles]", "admin"))),
 		},
 		{
 			name: "using startTime",
 			body: fmt.Sprintf(`{"$gte": {"start_time": "%s"}}`, now.Format(time.DateFormat)),
-			expectQuery: ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
+			expectQuery: ledgerstore.NewGetTransactionsQuery(ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
 				PITFilter: ledgerstore.PITFilter{
 					PIT: &now,
 				},
 			}).
-				WithQueryBuilder(query.Gte("start_time", now.Format(time.DateFormat))),
+				WithQueryBuilder(query.Gte("start_time", now.Format(time.DateFormat)))),
 		},
 		{
 			name: "using endTime",
 			body: fmt.Sprintf(`{"$lte": {"end_time": "%s"}}`, now.Format(time.DateFormat)),
-			expectQuery: ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
+			expectQuery: ledgerstore.NewGetTransactionsQuery(ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
 				PITFilter: ledgerstore.PITFilter{
 					PIT: &now,
 				},
 			}).
-				WithQueryBuilder(query.Lte("end_time", now.Format(time.DateFormat))),
+				WithQueryBuilder(query.Lte("end_time", now.Format(time.DateFormat)))),
 		},
 		{
 			name: "using account",
 			body: `{"$match": {"account": "xxx"}}`,
-			expectQuery: ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
+			expectQuery: ledgerstore.NewGetTransactionsQuery(ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
 				PITFilter: ledgerstore.PITFilter{
 					PIT: &now,
 				},
 			}).
-				WithQueryBuilder(query.Match("account", "xxx")),
+				WithQueryBuilder(query.Match("account", "xxx"))),
 		},
 		{
 			name: "using reference",
 			body: `{"$match": {"reference": "xxx"}}`,
-			expectQuery: ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
+			expectQuery: ledgerstore.NewGetTransactionsQuery(ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
 				PITFilter: ledgerstore.PITFilter{
 					PIT: &now,
 				},
 			}).
-				WithQueryBuilder(query.Match("reference", "xxx")),
+				WithQueryBuilder(query.Match("reference", "xxx"))),
 		},
 		{
 			name: "using destination",
 			body: `{"$match": {"destination": "xxx"}}`,
-			expectQuery: ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
+			expectQuery: ledgerstore.NewGetTransactionsQuery(ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
 				PITFilter: ledgerstore.PITFilter{
 					PIT: &now,
 				},
 			}).
-				WithQueryBuilder(query.Match("destination", "xxx")),
+				WithQueryBuilder(query.Match("destination", "xxx"))),
 		},
 		{
 			name: "using source",
 			body: `{"$match": {"source": "xxx"}}`,
-			expectQuery: ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
+			expectQuery: ledgerstore.NewGetTransactionsQuery(ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
 				PITFilter: ledgerstore.PITFilter{
 					PIT: &now,
 				},
 			}).
-				WithQueryBuilder(query.Match("source", "xxx")),
+				WithQueryBuilder(query.Match("source", "xxx"))),
 		},
 		{
 			name: "using empty cursor",
 			queryParams: url.Values{
 				"cursor": []string{bunpaginate.EncodeCursor(ledgerstore.NewGetTransactionsQuery(ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{})))},
 			},
-			expectQuery: ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
+			expectQuery: ledgerstore.NewGetTransactionsQuery(ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
 				PITFilter: ledgerstore.PITFilter{},
-			}),
+			})),
 		},
 		{
 			name: "using invalid cursor",
@@ -682,29 +682,39 @@ func TestGetTransactions(t *testing.T) {
 			queryParams: url.Values{
 				"pageSize": []string{"1000000"},
 			},
-			expectQuery: ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
+			expectQuery: ledgerstore.NewGetTransactionsQuery(ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
 				PITFilter: ledgerstore.PITFilter{
 					PIT: &now,
 				},
 			}).
-				WithPageSize(v2.MaxPageSize),
+				WithPageSize(v2.MaxPageSize)),
 		},
 		{
 			name: "using cursor",
 			queryParams: url.Values{
 				"cursor": []string{"eyJwYWdlU2l6ZSI6MTUsImJvdHRvbSI6bnVsbCwiY29sdW1uIjoiaWQiLCJwYWdpbmF0aW9uSUQiOm51bGwsIm9yZGVyIjoxLCJmaWx0ZXJzIjp7InFiIjp7fSwicGFnZVNpemUiOjE1LCJvcHRpb25zIjp7InBpdCI6bnVsbCwidm9sdW1lcyI6ZmFsc2UsImVmZmVjdGl2ZVZvbHVtZXMiOmZhbHNlfX0sInJldmVyc2UiOmZhbHNlfQ"},
 			},
-			expectQuery: ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}),
+			expectQuery: ledgerstore.NewGetTransactionsQuery(ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{})),
 		},
 		{
 			name: "using $exists metadata filter",
 			body: `{"$exists": {"metadata": "foo"}}`,
-			expectQuery: ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
+			expectQuery: ledgerstore.NewGetTransactionsQuery(ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
 				PITFilter: ledgerstore.PITFilter{
 					PIT: &now,
 				},
 			}).
-				WithQueryBuilder(query.Exists("metadata", "foo")),
+				WithQueryBuilder(query.Exists("metadata", "foo"))),
+		},
+		{
+			name:        "paginate using effective order",
+			queryParams: map[string][]string{"order": {"effective"}},
+			expectQuery: ledgerstore.NewGetTransactionsQuery(ledgerstore.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{
+				PITFilter: ledgerstore.PITFilter{
+					PIT: &now,
+				},
+			})).
+				WithColumn("timestamp"),
 		},
 	}
 	for _, testCase := range testCases {
@@ -729,7 +739,7 @@ func TestGetTransactions(t *testing.T) {
 			backend, mockLedger := newTestingBackend(t, true)
 			if testCase.expectStatusCode < 300 && testCase.expectStatusCode >= 200 {
 				mockLedger.EXPECT().
-					GetTransactions(gomock.Any(), ledgerstore.NewGetTransactionsQuery(testCase.expectQuery)).
+					GetTransactions(gomock.Any(), testCase.expectQuery).
 					Return(&expectedCursor, nil)
 			}
 

diff --git a/components/ledger/internal/storage/ledgerstore/transactions.go b/components/ledger/internal/storage/ledgerstore/transactions.go
@@ -10,6 +10,8 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/formancehq/stack/libs/go-libs/pointer"
+
 	"github.com/formancehq/stack/libs/go-libs/time"
 
 	"github.com/formancehq/stack/libs/go-libs/bun/bunpaginate"
@@ -161,7 +163,7 @@ func (store *Store) buildTransactionQuery(p PITFilterWithVolumes, query *bun.Sel
 	if p.PIT != nil && !p.PIT.IsZero() {
 		query = query.
 			Where("timestamp <= ?", p.PIT).
-			ColumnExpr("distinct on(transactions.id) transactions.*").
+			ColumnExpr("transactions.*").
 			Column("transactions_metadata.metadata").
 			Join(fmt.Sprintf(`left join lateral (%s) as transactions_metadata on true`, selectMetadata.String())).
 			ColumnExpr(fmt.Sprintf("case when reverted_at is not null and reverted_at > '%s' then null else reverted_at end", p.PIT.Format(time.DateFormat)))
@@ -401,6 +403,13 @@ func (q GetTransactionsQuery) WithExpandEffectiveVolumes() GetTransactionsQuery
 	return q
 }
 
+func (q GetTransactionsQuery) WithColumn(column string) GetTransactionsQuery {
+	ret := pointer.For((bunpaginate.ColumnPaginatedQuery[PaginatedQueryOptions[PITFilterWithVolumes]])(q))
+	ret = ret.WithColumn(column)
+
+	return GetTransactionsQuery(*ret)
+}
+
 func NewGetTransactionsQuery(options PaginatedQueryOptions[PITFilterWithVolumes]) GetTransactionsQuery {
 	return GetTransactionsQuery{
 		PageSize: options.PageSize,

diff --git a/components/ledger/openapi.yaml b/components/ledger/openapi.yaml
@@ -1757,6 +1757,18 @@ paths:
           schema:
             type: string
             format: date-time
+        - name: order
+          in: query
+          required: false
+          schema:
+            type: string
+            enum:
+              - effective
+        - name: reverse
+          in: query
+          required: false
+          schema:
+            type: boolean
       requestBody:
         content:
           application/json:

diff --git a/components/ledger/openapi/v2.yaml b/components/ledger/openapi/v2.yaml
@@ -631,6 +631,18 @@ paths:
           schema:
             type: string
             format: date-time
+        - name: order
+          in: query
+          required: false
+          schema:
+            type: string
+            enum:
+              - effective
+        - name: reverse
+          in: query
+          required: false
+          schema:
+            type: boolean
       requestBody:
         content:
           application/json:

diff --git a/libs/go-libs/bun/bunpaginate/pagination.go b/libs/go-libs/bun/bunpaginate/pagination.go
@@ -64,6 +64,12 @@ func (a *ColumnPaginatedQuery[PAYLOAD]) WithPageSize(pageSize uint64) *ColumnPag
 	return a
 }
 
+func (a *ColumnPaginatedQuery[PAYLOAD]) WithColumn(column string) *ColumnPaginatedQuery[PAYLOAD] {
+	a.Column = column
+
+	return a
+}
+
 type OffsetPaginatedQuery[OPTIONS any] struct {
 	Offset   uint64  `json:"offset"`
 	Order    Order   `json:"order"`

diff --git a/libs/go-libs/bun/bunpaginate/pagination_column.go b/libs/go-libs/bun/bunpaginate/pagination_column.go
@@ -6,7 +6,9 @@ import (
 	"math/big"
 	"reflect"
 	"strings"
+	"time"
 
+	libtime "github.com/formancehq/stack/libs/go-libs/time"
 	"github.com/uptrace/bun"
 )
 
@@ -59,14 +61,29 @@ func UsingColumn[FILTERS any, ENTITY any](ctx context.Context,
 	}
 
 	var (
-		paginationIDs = make([]*BigInt, 0)
+		paginationIDs = make([]*big.Int, 0)
 	)
 	for _, t := range ret {
-		paginationID := reflect.ValueOf(t).
+		rawPaginationID := reflect.ValueOf(t).
 			Field(paginatedColumnIndex).
-			Interface().(*BigInt)
+			Interface()
+		var paginationID *big.Int
+		switch rawPaginationID := rawPaginationID.(type) {
+		case time.Time:
+			paginationID = big.NewInt(rawPaginationID.UTC().UnixMicro())
+		case libtime.Time:
+			paginationID = big.NewInt(rawPaginationID.UTC().UnixMicro())
+		case *BigInt:
+			paginationID = (*big.Int)(rawPaginationID)
+		case *big.Int:
+			paginationID = rawPaginationID
+		case int64:
+			paginationID = big.NewInt(rawPaginationID)
+		default:
+			panic(fmt.Sprintf("invalid paginationID, type %T not handled", rawPaginationID))
+		}
 		if query.Bottom == nil {
-			query.Bottom = (*big.Int)(paginationID)
+			query.Bottom = paginationID
 		}
 		paginationIDs = append(paginationIDs, paginationID)
 	}
@@ -90,13 +107,13 @@ func UsingColumn[FILTERS any, ENTITY any](ctx context.Context,
 
 		if hasMore {
 			cp := query
-			cp.PaginationID = (*big.Int)(paginationIDs[len(paginationIDs)-2])
+			cp.PaginationID = paginationIDs[len(paginationIDs)-2]
 			previous = &cp
 		}
 	} else {
 		if hasMore {
 			cp := query
-			cp.PaginationID = (*big.Int)(paginationIDs[len(paginationIDs)-1])
+			cp.PaginationID = paginationIDs[len(paginationIDs)-1]
 			next = &cp
 		}
 		if query.PaginationID != nil {

diff --git a/libs/go-libs/bun/bunpaginate/pagination_offset.go b/libs/go-libs/bun/bunpaginate/pagination_offset.go
@@ -67,13 +67,9 @@ func usingOffset[Q any, T any](ctx context.Context, sb *bun.SelectQuery, query O
 }
 
 func UsingOffset[Q any, T any](ctx context.Context, sb *bun.SelectQuery, query OffsetPaginatedQuery[Q], builders ...func(query *bun.SelectQuery) *bun.SelectQuery) (*Cursor[T], error) {
-
 	return usingOffset[Q, T](ctx, sb, query, true, builders...)
-
 }
 
 func UsingOffsetWithoutModel[Q any, T any](ctx context.Context, sb *bun.SelectQuery, query OffsetPaginatedQuery[Q], builders ...func(query *bun.SelectQuery) *bun.SelectQuery) (*Cursor[T], error) {
-
 	return usingOffset[Q, T](ctx, sb, query, false, builders...)
-
 }
diff --git a/releases/sdks/go/.speakeasy/gen.lock b/releases/sdks/go/.speakeasy/gen.lock
@@ -1,7 +1,7 @@
 lockVersion: 2.0.0
 id: 7eac0a45-60a2-40bb-9e85-26bd77ec2a6d
 management:
-  docChecksum: f349c78f846ac290e8a10eff3984042d
+  docChecksum: 8a9ef86467c00955c6ddee01c0a79bcb
   docVersion: v0.0.0
   speakeasyVersion: 1.292.0
   generationVersion: 2.332.4
@@ -673,6 +673,7 @@ generatedFiles:
   - docs/pkg/models/operations/v2listledgersresponse.md
   - docs/pkg/models/operations/v2listlogsrequest.md
   - docs/pkg/models/operations/v2listlogsresponse.md
+  - docs/pkg/models/operations/order.md
   - docs/pkg/models/operations/v2listtransactionsrequest.md
   - docs/pkg/models/operations/v2listtransactionsresponse.md
   - docs/pkg/models/operations/v2readstatsrequest.md

diff --git a/releases/sdks/go/docs/pkg/models/operations/order.md b/releases/sdks/go/docs/pkg/models/operations/order.md
@@ -0,0 +1,8 @@
+# Order
+
+
+## Values
+
+| Name             | Value            |
+| ---------------- | ---------------- |
+| `OrderEffective` | effective        |