feat: add first draft of reconciliation service (account based) (#957)

formancehq · Dec 6, 2023 · c57bfb1 · c57bfb1 · vercel · Dec 6, 2023
1 parent 4330cf4
commit c57bfb1
Show file tree

Hide file tree

Showing 49 changed files with 2,073 additions and 15 deletions.
diff --git a/components/operator/.earthly/configuration/templates/configuration.yaml b/components/operator/.earthly/configuration/templates/configuration.yaml
@@ -54,6 +54,8 @@ spec:
         host: postgres-postgresql.formance.svc.cluster.local
         port: 5432
         disableSSLMode: true
+    reconciliation:
+      debug: true
     payments:
       postgres:
         credentialsFromSecret: postgres

diff --git a/components/operator/.earthly/k8s-versions.yaml b/components/operator/.earthly/k8s-versions.yaml
@@ -10,6 +10,7 @@ spec:
   ledger: local
   orchestration: local
   payments: local
+  reconciliation: local
   search: local
   wallets: local
   webhooks: local

diff --git a/components/operator/apis/stack/v1beta3/configuration_types.go b/components/operator/apis/stack/v1beta3/configuration_types.go
@@ -31,14 +31,15 @@ import (
 // Fields order is important.
 // For example, auth must be defined later as other services create static auth clients which must be used by auth.
 type ConfigurationServicesSpec struct {
-	Control       ControlSpec       `json:"control,omitempty"`
-	Ledger        LedgerSpec        `json:"ledger,omitempty"`
-	Payments      PaymentsSpec      `json:"payments,omitempty"`
-	Webhooks      WebhooksSpec      `json:"webhooks,omitempty"`
-	Wallets       WalletsSpec       `json:"wallets,omitempty"`
-	Orchestration OrchestrationSpec `json:"orchestration,omitempty"`
-	Search        SearchSpec        `json:"search,omitempty"`
-	Auth          AuthSpec          `json:"auth,omitempty"`
+	Control        ControlSpec        `json:"control,omitempty"`
+	Ledger         LedgerSpec         `json:"ledger,omitempty"`
+	Payments       PaymentsSpec       `json:"payments,omitempty"`
+	Reconciliation ReconciliationSpec `json:"reconciliation,omitempty"`
+	Webhooks       WebhooksSpec       `json:"webhooks,omitempty"`
+	Wallets        WalletsSpec        `json:"wallets,omitempty"`
+	Orchestration  OrchestrationSpec  `json:"orchestration,omitempty"`
+	Search         SearchSpec         `json:"search,omitempty"`
+	Auth           AuthSpec           `json:"auth,omitempty"`
 
 	// +optional
 	Gateway  GatewaySpec  `json:"gateway,omitempty"`

diff --git a/components/operator/apis/stack/v1beta3/reconciliation_types.go b/components/operator/apis/stack/v1beta3/reconciliation_types.go
@@ -0,0 +1,9 @@
+package v1beta3
+
+type ReconciliationSpec struct {
+	CommonServiceProperties `json:",inline"`
+
+	// +optional
+	ResourceProperties *ResourceProperties     `json:"resourceProperties,omitempty"`
+	Annotations        AnnotationsServicesSpec `json:"annotations,omitempty"`
+}
diff --git a/components/operator/apis/stack/v1beta3/stack_types.go b/components/operator/apis/stack/v1beta3/stack_types.go
@@ -100,12 +100,13 @@ type StackStargateConfig struct {
 
 type StackServicesSpec struct {
 	// +optional
-	Ledger        StackServicePropertiesSpec `json:"ledger,omitempty"`
-	Orchestration StackServicePropertiesSpec `json:"orchestration,omitempty"`
-	Payments      StackServicePropertiesSpec `json:"payments,omitempty"`
-	Wallets       StackServicePropertiesSpec `json:"wallets,omitempty"`
-	Webhooks      StackServicePropertiesSpec `json:"webhooks,omitempty"`
-	Control       StackServicePropertiesSpec `json:"control,omitempty"`
+	Ledger         StackServicePropertiesSpec `json:"ledger,omitempty"`
+	Orchestration  StackServicePropertiesSpec `json:"orchestration,omitempty"`
+	Reconciliation StackServicePropertiesSpec `json:"reconciliation,omitempty"`
+	Payments       StackServicePropertiesSpec `json:"payments,omitempty"`
+	Wallets        StackServicePropertiesSpec `json:"wallets,omitempty"`
+	Webhooks       StackServicePropertiesSpec `json:"webhooks,omitempty"`
+	Control        StackServicePropertiesSpec `json:"control,omitempty"`
 }
 
 type StackServicePropertiesSpec struct {

diff --git a/components/operator/apis/stack/v1beta3/versions_types.go b/components/operator/apis/stack/v1beta3/versions_types.go
@@ -46,6 +46,8 @@ type VersionsSpec struct {
 	Gateway string `json:"gateway"`
 	// +optional
 	Stargate string `json:"stargate"`
+	// +optional
+	Reconciliation string `json:"reconciliation"`
 }
 
 // VersionsStatus defines the observed state of Versions

diff --git a/components/operator/apis/stack/v1beta3/zz_generated.deepcopy.go b/components/operator/apis/stack/v1beta3/zz_generated.deepcopy.go
diff --git a/components/operator/config/crd/bases/stack.formance.com_configurations.yaml b/components/operator/config/crd/bases/stack.formance.com_configurations.yaml
@@ -4573,6 +4573,39 @@ spec:
                     - encryptionKey
                     - postgres
                     type: object
+                  reconciliation:
+                    properties:
+                      annotations:
+                        properties:
+                          service:
+                            additionalProperties:
+                              type: string
+                            type: object
+                        type: object
+                      debug:
+                        type: boolean
+                      dev:
+                        type: boolean
+                      disabled:
+                        type: boolean
+                      resourceProperties:
+                        properties:
+                          limits:
+                            properties:
+                              cpu:
+                                type: string
+                              memory:
+                                type: string
+                            type: object
+                          request:
+                            properties:
+                              cpu:
+                                type: string
+                              memory:
+                                type: string
+                            type: object
+                        type: object
+                    type: object
                   search:
                     properties:
                       annotations:

diff --git a/components/operator/config/crd/bases/stack.formance.com_stacks.yaml b/components/operator/config/crd/bases/stack.formance.com_stacks.yaml
@@ -1950,6 +1950,11 @@ spec:
                       disabled:
                         type: boolean
                     type: object
+                  reconciliation:
+                    properties:
+                      disabled:
+                        type: boolean
+                    type: object
                   wallets:
                     properties:
                       disabled:

diff --git a/components/operator/config/crd/bases/stack.formance.com_versions.yaml b/components/operator/config/crd/bases/stack.formance.com_versions.yaml
@@ -94,6 +94,8 @@ spec:
                 type: string
               payments:
                 type: string
+              reconciliation:
+                type: string
               search:
                 type: string
               stargate:

diff --git a/...n/apiextensions.k8s.io_v1_customresourcedefinition_configurations.stack.formance.com.yaml b/...n/apiextensions.k8s.io_v1_customresourcedefinition_configurations.stack.formance.com.yaml
@@ -4584,6 +4584,39 @@ spec:
                     - encryptionKey
                     - postgres
                     type: object
+                  reconciliation:
+                    properties:
+                      annotations:
+                        properties:
+                          service:
+                            additionalProperties:
+                              type: string
+                            type: object
+                        type: object
+                      debug:
+                        type: boolean
+                      dev:
+                        type: boolean
+                      disabled:
+                        type: boolean
+                      resourceProperties:
+                        properties:
+                          limits:
+                            properties:
+                              cpu:
+                                type: string
+                              memory:
+                                type: string
+                            type: object
+                          request:
+                            properties:
+                              cpu:
+                                type: string
+                              memory:
+                                type: string
+                            type: object
+                        type: object
+                    type: object
                   search:
                     properties:
                       annotations:

diff --git a/...lates/gen/apiextensions.k8s.io_v1_customresourcedefinition_stacks.stack.formance.com.yaml b/...lates/gen/apiextensions.k8s.io_v1_customresourcedefinition_stacks.stack.formance.com.yaml
@@ -1961,6 +1961,11 @@ spec:
                       disabled:
                         type: boolean
                     type: object
+                  reconciliation:
+                    properties:
+                      disabled:
+                        type: boolean
+                    type: object
                   wallets:
                     properties:
                       disabled:

diff --git a/...tes/gen/apiextensions.k8s.io_v1_customresourcedefinition_versions.stack.formance.com.yaml b/...tes/gen/apiextensions.k8s.io_v1_customresourcedefinition_versions.stack.formance.com.yaml
@@ -95,6 +95,8 @@ spec:
                 type: string
               payments:
                 type: string
+              reconciliation:
+                type: string
               search:
                 type: string
               stargate:

diff --git a/components/operator/internal/controllers/stack/stack_reconciler.go b/components/operator/internal/controllers/stack/stack_reconciler.go
@@ -24,6 +24,7 @@ import (
 	_ "github.com/formancehq/operator/internal/modules/ledger"
 	_ "github.com/formancehq/operator/internal/modules/orchestration"
 	_ "github.com/formancehq/operator/internal/modules/payments"
+	_ "github.com/formancehq/operator/internal/modules/reconciliation"
 	_ "github.com/formancehq/operator/internal/modules/search"
 	_ "github.com/formancehq/operator/internal/modules/stargate"
 	_ "github.com/formancehq/operator/internal/modules/wallets"

diff --git a/components/operator/internal/modules/all/all.go b/components/operator/internal/modules/all/all.go
@@ -7,6 +7,7 @@ import (
 	_ "github.com/formancehq/operator/internal/modules/ledger"
 	_ "github.com/formancehq/operator/internal/modules/orchestration"
 	_ "github.com/formancehq/operator/internal/modules/payments"
+	_ "github.com/formancehq/operator/internal/modules/reconciliation"
 	_ "github.com/formancehq/operator/internal/modules/search"
 	_ "github.com/formancehq/operator/internal/modules/stargate"
 	_ "github.com/formancehq/operator/internal/modules/wallets"

diff --git a/components/operator/internal/modules/auth/handler.go b/components/operator/internal/modules/auth/handler.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/formancehq/operator/internal/modules/control"
 	"github.com/formancehq/operator/internal/modules/orchestration"
+	"github.com/formancehq/operator/internal/modules/reconciliation"
 	"github.com/formancehq/operator/internal/modules/wallets"
 
 	stackv1beta3 "github.com/formancehq/operator/apis/stack/v1beta3"
@@ -25,6 +26,7 @@ func (a module) DependsOn() []modules.Module {
 		control.Module,
 		orchestration.Module,
 		wallets.Module,
+		reconciliation.Module,
 	}
 }
 

diff --git a/components/operator/internal/modules/gateway/handler.go b/components/operator/internal/modules/gateway/handler.go
@@ -12,6 +12,7 @@ import (
 	"github.com/formancehq/operator/internal/modules/ledger"
 	"github.com/formancehq/operator/internal/modules/orchestration"
 	"github.com/formancehq/operator/internal/modules/payments"
+	"github.com/formancehq/operator/internal/modules/reconciliation"
 	"github.com/formancehq/operator/internal/modules/search"
 	"github.com/formancehq/operator/internal/modules/wallets"
 	"github.com/formancehq/operator/internal/modules/webhooks"
@@ -36,6 +37,7 @@ func (g module) DependsOn() []modules.Module {
 		control.Module,
 		ledger.Module,
 		orchestration.Module,
+		reconciliation.Module,
 		payments.Module,
 		search.Module,
 		wallets.Module,

diff --git a/components/operator/internal/modules/reconciliation/handler.go b/components/operator/internal/modules/reconciliation/handler.go
@@ -0,0 +1,64 @@
+package reconciliation
+
+import (
+	stackv1beta3 "github.com/formancehq/operator/apis/stack/v1beta3"
+	"github.com/formancehq/operator/internal/modules"
+)
+
+type module struct{}
+
+func (o module) Name() string {
+	return "reconciliation"
+}
+
+func (o module) IsEE() bool {
+	return true
+}
+
+func (o module) Versions() map[string]modules.Version {
+	return map[string]modules.Version{
+		"v0.0.0": {
+			Services: func(ctx modules.ReconciliationConfig) modules.Services {
+				return modules.Services{
+					{
+						InjectPostgresVariables: false,
+						AuthConfiguration: func(config modules.ReconciliationConfig) stackv1beta3.ClientConfiguration {
+							return stackv1beta3.NewClientConfiguration()
+						},
+						HasVersionEndpoint: true,
+						ListenEnvVar:       "LISTEN",
+						ExposeHTTP:         modules.DefaultExposeHTTP,
+						Annotations:        ctx.Configuration.Spec.Services.Reconciliation.Annotations.Service,
+						Container: func(resolveContext modules.ContainerResolutionConfiguration) modules.Container {
+							return modules.Container{
+								Env:   reconciliationEnvVars(resolveContext),
+								Image: modules.GetImage("reconciliation", resolveContext.Versions.Spec.Reconciliation),
+								Resources: modules.GetResourcesWithDefault(
+									resolveContext.Configuration.Spec.Services.Reconciliation.ResourceProperties,
+									modules.ResourceSizeSmall(),
+								),
+							}
+						},
+					},
+				}
+			},
+		},
+	}
+}
+
+func reconciliationEnvVars(resolveContext modules.ContainerResolutionConfiguration) modules.ContainerEnv {
+	env := modules.NewEnv().Append(
+		modules.Env("STACK_CLIENT_ID", resolveContext.Stack.Status.StaticAuthClients["reconciliation"].ID),
+		modules.Env("STACK_CLIENT_SECRET", resolveContext.Stack.Status.StaticAuthClients["reconciliation"].Secrets[0]),
+	)
+
+	return env
+}
+
+var Module = &module{}
+
+var _ modules.Module = Module
+
+func init() {
+	modules.Register(Module)
+}
diff --git a/components/payments/openapi.yaml b/components/payments/openapi.yaml
@@ -693,7 +693,7 @@ components:
       content:
         application/json:
           schema:
-            $ref: '#/components/schemas/PoolBalance'
+            $ref: '#/components/schemas/PoolBalancesResponse'
     TransferInitiations:
       description: OK
       content:

diff --git a/components/reconciliation/.gitignore b/components/reconciliation/.gitignore
@@ -0,0 +1,10 @@
+.DS_Store
+.idea
+vendor
+.cloud/ressources/.terraform
+.cloud/ressources/.terraform.lock.hcl
+/.cloud/helm/charts/
+coverage.out
+dist/
+.env
+payments