feat(payments): push raw data in nats but not in benthos (#1012)

formancehq · Dec 13, 2023 · 8d57c67 · 8d57c67 · vercel · Dec 13, 2023
1 parent b1e0292
commit 8d57c67
Show file tree

Hide file tree

Showing 34 changed files with 36 additions and 33 deletions.
diff --git a/.../testdata/monopod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml b/.../testdata/monopod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/...trollers/stack/testdata/monopod-latest/results/configmaps--v1/search-benthos-streams.yaml b/...trollers/stack/testdata/monopod-latest/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/...ollers/stack/testdata/monopod-ledgerv1/results/configmaps--v1/search-benthos-streams.yaml b/...ollers/stack/testdata/monopod-ledgerv1/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/.../monopod-payments-after-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml b/.../monopod-payments-after-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/...monopod-payments-before-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml b/...monopod-payments-before-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/.../testdata/monopod-search-before-v0.7.0/results/configmaps--v1/search-benthos-streams.yaml b/.../testdata/monopod-search-before-v0.7.0/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/...trollers/stack/testdata/multipod-debug/results/configmaps--v1/search-benthos-streams.yaml b/...trollers/stack/testdata/multipod-debug/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/...controllers/stack/testdata/multipod-debug/results/deployments-apps-v1/search-benthos.yaml b/...controllers/stack/testdata/multipod-debug/results/deployments-apps-v1/search-benthos.yaml
@@ -20,7 +20,7 @@ spec:
     template:
         metadata:
             annotations:
-                stack.formance.cloud/volumes-hash: -UoS45tOcCl3jiSwzxW_3wpgEP5fZtADgfvq9c999bk=
+                stack.formance.cloud/volumes-hash: zAMPIto8AXFdiwdNiCVG3o8e_2plT6XlsIrrlxf0NcA=
             creationTimestamp: null
             labels:
                 app.kubernetes.io/name: search-benthos

diff --git a/...testdata/multipod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml b/...testdata/multipod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/...ck/testdata/multipod-disabled-one-service/results/deployments-apps-v1/search-benthos.yaml b/...ck/testdata/multipod-disabled-one-service/results/deployments-apps-v1/search-benthos.yaml
@@ -20,7 +20,7 @@ spec:
     template:
         metadata:
             annotations:
-                stack.formance.cloud/volumes-hash: -UoS45tOcCl3jiSwzxW_3wpgEP5fZtADgfvq9c999bk=
+                stack.formance.cloud/volumes-hash: zAMPIto8AXFdiwdNiCVG3o8e_2plT6XlsIrrlxf0NcA=
             creationTimestamp: null
             labels:
                 app.kubernetes.io/name: search-benthos

diff --git a/...testdata/multipod-latest-no-monitoring/results/configmaps--v1/search-benthos-streams.yaml b/...testdata/multipod-latest-no-monitoring/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/...ck/testdata/multipod-latest-no-monitoring/results/deployments-apps-v1/search-benthos.yaml b/...ck/testdata/multipod-latest-no-monitoring/results/deployments-apps-v1/search-benthos.yaml
@@ -20,7 +20,7 @@ spec:
     template:
         metadata:
             annotations:
-                stack.formance.cloud/volumes-hash: ZCAv3TDE8_leBkgtrdIjpiELW6zKYWYZ_4UZz3IQZtk=
+                stack.formance.cloud/volumes-hash: iXL4EcTjD7D2Zkq9Qb1D6Bctp8pGhkzkiwtjjBeKSnU=
             creationTimestamp: null
             labels:
                 app.kubernetes.io/name: search-benthos

diff --git a/...data/multipod-latest-with-check-scopes/results/configmaps--v1/search-benthos-streams.yaml b/...data/multipod-latest-with-check-scopes/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/...estdata/multipod-latest-with-check-scopes/results/deployments-apps-v1/search-benthos.yaml b/...estdata/multipod-latest-with-check-scopes/results/deployments-apps-v1/search-benthos.yaml
@@ -20,7 +20,7 @@ spec:
     template:
         metadata:
             annotations:
-                stack.formance.cloud/volumes-hash: -UoS45tOcCl3jiSwzxW_3wpgEP5fZtADgfvq9c999bk=
+                stack.formance.cloud/volumes-hash: zAMPIto8AXFdiwdNiCVG3o8e_2plT6XlsIrrlxf0NcA=
             creationTimestamp: null
             labels:
                 app.kubernetes.io/name: search-benthos

diff --git a/...a/multipod-latest-with-custom-registry/results/configmaps--v1/search-benthos-streams.yaml b/...a/multipod-latest-with-custom-registry/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/...data/multipod-latest-with-custom-registry/results/deployments-apps-v1/search-benthos.yaml b/...data/multipod-latest-with-custom-registry/results/deployments-apps-v1/search-benthos.yaml
@@ -20,7 +20,7 @@ spec:
     template:
         metadata:
             annotations:
-                stack.formance.cloud/volumes-hash: -UoS45tOcCl3jiSwzxW_3wpgEP5fZtADgfvq9c999bk=
+                stack.formance.cloud/volumes-hash: zAMPIto8AXFdiwdNiCVG3o8e_2plT6XlsIrrlxf0NcA=
             creationTimestamp: null
             labels:
                 app.kubernetes.io/name: search-benthos

diff --git a/.../multipod-latest-with-gateway-fallback/results/configmaps--v1/search-benthos-streams.yaml b/.../multipod-latest-with-gateway-fallback/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/...ata/multipod-latest-with-gateway-fallback/results/deployments-apps-v1/search-benthos.yaml b/...ata/multipod-latest-with-gateway-fallback/results/deployments-apps-v1/search-benthos.yaml
@@ -20,7 +20,7 @@ spec:
     template:
         metadata:
             annotations:
-                stack.formance.cloud/volumes-hash: -UoS45tOcCl3jiSwzxW_3wpgEP5fZtADgfvq9c999bk=
+                stack.formance.cloud/volumes-hash: zAMPIto8AXFdiwdNiCVG3o8e_2plT6XlsIrrlxf0NcA=
             creationTimestamp: null
             labels:
                 app.kubernetes.io/name: search-benthos

diff --git a/.../testdata/multipod-latest-with-secrets/results/configmaps--v1/search-benthos-streams.yaml b/.../testdata/multipod-latest-with-secrets/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/...ack/testdata/multipod-latest-with-secrets/results/deployments-apps-v1/search-benthos.yaml b/...ack/testdata/multipod-latest-with-secrets/results/deployments-apps-v1/search-benthos.yaml
@@ -20,7 +20,7 @@ spec:
     template:
         metadata:
             annotations:
-                stack.formance.cloud/volumes-hash: -UoS45tOcCl3jiSwzxW_3wpgEP5fZtADgfvq9c999bk=
+                stack.formance.cloud/volumes-hash: zAMPIto8AXFdiwdNiCVG3o8e_2plT6XlsIrrlxf0NcA=
             creationTimestamp: null
             labels:
                 app.kubernetes.io/name: search-benthos

diff --git a/...rollers/stack/testdata/multipod-latest/results/configmaps--v1/search-benthos-streams.yaml b/...rollers/stack/testdata/multipod-latest/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/...ontrollers/stack/testdata/multipod-latest/results/deployments-apps-v1/search-benthos.yaml b/...ontrollers/stack/testdata/multipod-latest/results/deployments-apps-v1/search-benthos.yaml
@@ -20,7 +20,7 @@ spec:
     template:
         metadata:
             annotations:
-                stack.formance.cloud/volumes-hash: -UoS45tOcCl3jiSwzxW_3wpgEP5fZtADgfvq9c999bk=
+                stack.formance.cloud/volumes-hash: zAMPIto8AXFdiwdNiCVG3o8e_2plT6XlsIrrlxf0NcA=
             creationTimestamp: null
             labels:
                 app.kubernetes.io/name: search-benthos

diff --git a/...testdata/multipod-orchestration-v0.2.0/results/configmaps--v1/search-benthos-streams.yaml b/...testdata/multipod-orchestration-v0.2.0/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/...ck/testdata/multipod-orchestration-v0.2.0/results/deployments-apps-v1/search-benthos.yaml b/...ck/testdata/multipod-orchestration-v0.2.0/results/deployments-apps-v1/search-benthos.yaml
@@ -20,7 +20,7 @@ spec:
     template:
         metadata:
             annotations:
-                stack.formance.cloud/volumes-hash: -UoS45tOcCl3jiSwzxW_3wpgEP5fZtADgfvq9c999bk=
+                stack.formance.cloud/volumes-hash: zAMPIto8AXFdiwdNiCVG3o8e_2plT6XlsIrrlxf0NcA=
             creationTimestamp: null
             labels:
                 app.kubernetes.io/name: search-benthos

diff --git a/...k/testdata/multipod-service-annotation/results/configmaps--v1/search-benthos-streams.yaml b/...k/testdata/multipod-service-annotation/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/...tack/testdata/multipod-service-annotation/results/deployments-apps-v1/search-benthos.yaml b/...tack/testdata/multipod-service-annotation/results/deployments-apps-v1/search-benthos.yaml
@@ -20,7 +20,7 @@ spec:
     template:
         metadata:
             annotations:
-                stack.formance.cloud/volumes-hash: ZCAv3TDE8_leBkgtrdIjpiELW6zKYWYZ_4UZz3IQZtk=
+                stack.formance.cloud/volumes-hash: iXL4EcTjD7D2Zkq9Qb1D6Bctp8pGhkzkiwtjjBeKSnU=
             creationTimestamp: null
             labels:
                 app.kubernetes.io/name: search-benthos

diff --git a/...ta/multipod-with-audit-log-unvalid-tag/results/configmaps--v1/search-benthos-streams.yaml b/...ta/multipod-with-audit-log-unvalid-tag/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/...tdata/multipod-with-audit-log-unvalid-tag/results/deployments-apps-v1/search-benthos.yaml b/...tdata/multipod-with-audit-log-unvalid-tag/results/deployments-apps-v1/search-benthos.yaml
@@ -20,7 +20,7 @@ spec:
     template:
         metadata:
             annotations:
-                stack.formance.cloud/volumes-hash: F6bbZbx0MQNCbgZUdSGeciveDQa7otqWM4t6cfklKHU=
+                stack.formance.cloud/volumes-hash: ohqyd8i40N1OhA-WN_ifFolSeJokKiFPJd4m-8X3ybA=
             creationTimestamp: null
             labels:
                 app.kubernetes.io/name: search-benthos

diff --git a/...stack/testdata/multipod-with-audit-log/results/configmaps--v1/search-benthos-streams.yaml b/...stack/testdata/multipod-with-audit-log/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/...rs/stack/testdata/multipod-with-audit-log/results/deployments-apps-v1/search-benthos.yaml b/...rs/stack/testdata/multipod-with-audit-log/results/deployments-apps-v1/search-benthos.yaml
@@ -20,7 +20,7 @@ spec:
     template:
         metadata:
             annotations:
-                stack.formance.cloud/volumes-hash: F6bbZbx0MQNCbgZUdSGeciveDQa7otqWM4t6cfklKHU=
+                stack.formance.cloud/volumes-hash: ohqyd8i40N1OhA-WN_ifFolSeJokKiFPJd4m-8X3ybA=
             creationTimestamp: null
             labels:
                 app.kubernetes.io/name: search-benthos

diff --git a/...ck/testdata/multipod-without-audit-log/results/configmaps--v1/search-benthos-streams.yaml b/...ck/testdata/multipod-without-audit-log/results/configmaps--v1/search-benthos-streams.yaml
@@ -151,7 +151,7 @@ data:
                 - bloblang: |
                       root = {
                           "document": {
-                            "data": this.payload,
+                            "data": this.payload.without("rawData"),
                             "indexed": {
                                 "id": this.payload.id,
                                 "reference": this.payload.reference,

diff --git a/...stack/testdata/multipod-without-audit-log/results/deployments-apps-v1/search-benthos.yaml b/...stack/testdata/multipod-without-audit-log/results/deployments-apps-v1/search-benthos.yaml
@@ -20,7 +20,7 @@ spec:
     template:
         metadata:
             annotations:
-                stack.formance.cloud/volumes-hash: ZCAv3TDE8_leBkgtrdIjpiELW6zKYWYZ_4UZz3IQZtk=
+                stack.formance.cloud/volumes-hash: iXL4EcTjD7D2Zkq9Qb1D6Bctp8pGhkzkiwtjjBeKSnU=
             creationTimestamp: null
             labels:
                 app.kubernetes.io/name: search-benthos

diff --git a/components/payments/internal/messages/payments.go b/components/payments/internal/messages/payments.go
@@ -1,6 +1,7 @@
 package messages
 
 import (
+	"encoding/json"
 	"math/big"
 	"time"
 
@@ -18,6 +19,7 @@ type paymentMessagePayload struct {
 	Status      models.PaymentStatus `json:"status"`
 	Scheme      models.PaymentScheme `json:"scheme"`
 	Asset       models.Asset         `json:"asset"`
+	RawData     json.RawMessage      `json:"rawData"`
 
 	// TODO: Remove 'initialAmount' once frontend has switched to 'amount
 	InitialAmount *big.Int          `json:"initialAmount"`
@@ -38,6 +40,7 @@ func NewEventSavedPayments(provider models.ConnectorProvider, payment *models.Pa
 		CreatedAt:     payment.CreatedAt,
 		ConnectorID:   payment.ConnectorID.String(),
 		Provider:      provider.String(),
+		RawData:       payment.RawData,
 		Metadata: func() map[string]string {
 			ret := make(map[string]string)
 			for _, m := range payment.Metadata {

diff --git a/ee/search/benthos/streams/payments/v0.0.0/payments_ingestion.yaml b/ee/search/benthos/streams/payments/v0.0.0/payments_ingestion.yaml
@@ -13,7 +13,7 @@ pipeline:
         - bloblang: |
               root = {
                   "document": {
-                    "data": this.payload,
+                    "data": this.payload.without("rawData"),
                     "indexed": {
                         "id": this.payload.id,
                         "reference": this.payload.reference,