feat(operator): delete, enable, disable, backups

components/agent/Taskfile.yaml

@@ -18,3 +18,8 @@ tasks:
   helm:
     cmds:
       - echo "ok"
+
+  generate-client:
+    cmds:
+    - go generate
+

components/agent/internal/grpc/client.go

@@ -165,7 +165,7 @@ func (client *client) createStack(stack *generated.Stack) *v1beta3.Stack {
 	}
 }
 
-func (client *client) stackFusion(currentStack *v1beta3.Stack, createStack *v1beta3.Stack) *v1beta3.Stack {
+func (client *client) mergeStack(currentStack *v1beta3.Stack, createStack *v1beta3.Stack) *v1beta3.Stack {
 	createStack.SetResourceVersion(currentStack.GetResourceVersion())
 	createStack.Spec.Services = currentStack.Spec.Services
 	createStack.Spec.Seed = currentStack.Spec.Seed
@@ -259,13 +259,14 @@ func (client *client) Start(ctx context.Context) error {
 			if err := client.connectClient.SendMsg(&generated.Message{
 				Message: &generated.Message_StatusChanged{
 					StatusChanged: &generated.StatusChanged{
-						StackId: stack.Name,
-						Status:  status,
+						ClusterName: stack.Name,
+						Status:      status,
 					},
 				},
 			}); err != nil {
 				sharedlogging.FromContext(ctx).Errorf("Unable to send stack status to server: %s", err)
 			}
+
 		case msg := <-msgs:
 			switch msg := msg.Message.(type) {
 			// TODO: Implement UpdateOrCreate
@@ -285,32 +286,28 @@ func (client *client) Start(ctx context.Context) error {
 					continue
 				}
 
-				newStack := client.stackFusion(existingStack, createStack)
+				newStack := client.mergeStack(existingStack, createStack)
 				if _, err := client.k8sClient.Update(ctx, newStack); err != nil {
 					sharedlogging.FromContext(ctx).Errorf("Updating stack cluster side: %s", err)
 					continue
 				}
-				sharedlogging.FromContext(ctx).Infof("Stack %s updated", newStack.Name)
+
+				sharedlogging.FromContext(ctx).Infof("Stack %s updated cluster side", newStack.Name)
 
 			case *generated.Order_DeletedStack:
 				if err := client.k8sClient.Delete(ctx, msg.DeletedStack.ClusterName); err != nil {
-					sharedlogging.FromContext(ctx).Errorf("Deleting cluster side: %s", err)
 					if controllererrors.IsNotFound(err) {
-						if err := client.connectClient.SendMsg(&generated.Message{
-							Message: &generated.Message_StatusChanged{
-								StatusChanged: &generated.StatusChanged{
-									StackId: msg.DeletedStack.ClusterName,
-									Status:  generated.StackStatus_Deleted,
-								},
-							},
-						}); err != nil {
-							sharedlogging.FromContext(ctx).Errorf("Unable to send stack status to server: %s", err)
-							continue
-						}
+						sharedlogging.FromContext(ctx).Infof("Cannot delete not existing stack: %s", msg.DeletedStack.ClusterName)
+
+						continue
 					}
+					sharedlogging.FromContext(ctx).Errorf("Deleting cluster side: %s", err)
+					continue
 				}
 				sharedlogging.FromContext(ctx).Infof("Stack %s deleted", msg.DeletedStack.ClusterName)
 			case *generated.Order_DisabledStack:
+				sharedlogging.FromContext(ctx).Infof("Incomming DISABLING: %s", msg.DisabledStack.ClusterName)
+
 				existingStack, err := client.k8sClient.Get(ctx, msg.DisabledStack.ClusterName, metav1.GetOptions{})
 				if err != nil {
 					if controllererrors.IsNotFound(err) {
@@ -327,6 +324,7 @@ func (client *client) Start(ctx context.Context) error {
 				}
 				sharedlogging.FromContext(ctx).Infof("Stack %s disabled", msg.DisabledStack.ClusterName)
 			case *generated.Order_EnabledStack:
+				sharedlogging.FromContext(ctx).Infof("Incomming ENABLING: %s", msg.EnabledStack.ClusterName)
 				existingStack, err := client.k8sClient.Get(ctx, msg.EnabledStack.ClusterName, metav1.GetOptions{})
 				if err != nil {
 					if controllererrors.IsNotFound(err) {

components/agent/server.proto

@@ -1,5 +1,6 @@
 syntax = "proto3";
 
+import "google/protobuf/timestamp.proto";
 option go_package = "github.com/formancehq/membership/internal/grpc/generated";
 
 package server;
@@ -48,6 +49,7 @@ message Stack {
   repeated AuthClient staticClients = 4;
   StargateConfig stargateConfig = 5;
   bool disabled = 6;
+  google.protobuf.Timestamp deletedAt = 7;
 }
 
 enum StackStatus {
@@ -58,7 +60,7 @@ enum StackStatus {
 }
 
 message StatusChanged {
-  string stackId = 1;
+  string clusterName = 1;
   StackStatus status = 2;
 }
 

components/operator/.gitignore

@@ -91,3 +91,6 @@ ca.crt
 helm/operator/charts
 helm/templates/rbac/kustomization.yaml
 operator
+
+# helm build
+*.tgz

components/operator/Makefile

@@ -71,23 +71,20 @@ build: generate fmt vet ## Build manager binary.
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./main.go
 
-.PHONY: docker-build-local-prod
-docker-build-local-prod: ## Build docker image with the manager.
-	go build -o operator main.go
+.PHONY: docker-build-prod
+docker-build-prod: ## Build docker image with go releaser
+	goreleaser build --skip-validate --clean
+	mv ./dist/operator_linux_amd64_v1/operator ./
 	docker build -t ${IMG} -f ./build.Dockerfile ./  --no-cache
 
 .PHONY: docker-build
-docker-build: ## Build docker image with the manager.
+docker-build: ## Build docker image for development.
 	docker build -t ${IMG} -f ./Dockerfile ../../  --no-cache
 
-.PHONY: docker-push-local-prod
-docker-push-local-prod: ## Build docker image with the manager.
-	docker tag ${IMG} k3d-registry.host.k3d.internal:12345/${IMG}:dev-latest
-	docker push k3d-registry.host.k3d.internal:12345/${IMG}:dev-latest
-
 .PHONY: docker-push
 docker-push: ## Push docker image with the manager.
-	docker push ${IMG}
+	docker tag ${IMG} k3d-registry.host.k3d.internal:12345/${IMG}:dev-latest
+	docker push k3d-registry.host.k3d.internal:12345/${IMG}:dev-latest
 
 ##@ Deployment
 
@@ -105,7 +102,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
 
 .PHONY: deploy
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
-	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	cd config/manager && $(KUSTOMIZE) edit set image controller=k3d-registry.host.k3d.internal:12345/${IMG}:dev-latest
 	$(KUSTOMIZE) build config/default | kubectl apply -f -
 
 .PHONY: undeploy
@@ -125,27 +122,27 @@ helm-update: manifests generate
 .PHONY: helm-debug
 helm-debug: helm-update
 	helm template ./helm --debug
-	helm install --create-namespace --namespace formance-operator -f ./helm/values.yaml formance-operator ./helm --dry-run
+	helm install --create-namespace --namespace formance-system -f ./helm/values.yaml formance-operator ./helm --dry-run
 
 .PHONY: helm-install
 helm-install: helm-update
-	helm install --create-namespace --namespace formance-operator -f ./helm/values.yaml formance-operator ./helm
+	helm install --create-namespace --namespace formance-system -f ./helm/values.yaml formance-operator ./helm
 
 .PHONY: helm-local-install
 helm-local-install: helm-update
-	helm install --create-namespace --namespace formance-operator -f ./helm/values.yaml --set image.repository="${IMG}" formance-operator ./helm
+	helm install --create-namespace --namespace formance-system -f ./helm/values.yaml --set image.repository="k3d-registry.host.k3d.internal:12345/${IMG}" --set image.tag="dev-latest" formance-operator ./helm
 
 .PHONY: helm-uninstall
 helm-uninstall:
-	helm uninstall formance-operator --namespace formance-operator
+	helm uninstall formance-operator --namespace formance-system
 
 .PHONY: helm-upgrade
 helm-upgrade: helm-update
-	helm upgrade --namespace formance-operator -f ./helm/values.yaml formance-operator ./helm
+	helm upgrade --namespace formance-system -f ./helm/values.yaml formance-operator ./helm
 
 .PHONY: helm-local-upgrade
 helm-local-upgrade: helm-update
-	helm upgrade --namespace formance-operator -f ./helm/values.yaml --set image.repository="${IMG}" formance-operator ./helm
+	helm upgrade --namespace formance-system -f ./helm/values.yaml --set image.repository="k3d-registry.host.k3d.internal:12345/${IMG}" --set image.tag="dev-latest" formance-operator ./helm
 
 
 ##@ Build Dependencies

components/operator/README.md

@@ -1,15 +1,16 @@
-# formance-operator
+# Formance-operator
 
 This operator is in charge of deploying a full or partial Formance OSS Stack.
 It aims to simplify deployment and releases management of different parts of the Formance ecosystem.
 
 ## Getting Started
 
-You’ll need a Kubernetes cluster to run against.
+You'll need a Kubernetes cluster to run against.
 Scripts of this repository are using [K3D](https://k3d.io/v5.4.6/). You have to install it.
 Also, we use [Garden](https://docs.garden.io/) for management.
 
 ### Running on the cluster
+
 1. Create the cluster:
 
 ```sh
@@ -54,12 +55,21 @@ Add an entry for `host.k3d.internal` inside /etc/hosts file, pointing to 127.0.0
 Go to http://host.k3d.internal.
 Login with [email protected] / password
 
-### Tests
+### Push to local registry
 
-Run command :
-```sh
-make test
-```
+In order to be able to pull and push the image in the internal-registry named `k3d-registry.host.k3d.internal` 
+on fixed port `12345` defined in `garden/k3d.yaml` 
+
+
+Add an entry for `k3d-registry.host.k3d.internal` inside /etc/hosts file, pointing to 127.0.0.1.
+
+Then in order to build and publish your image
+    1. BUILD: `make docker-build`
+    2. PUSH: `make docker-push`
+    3. BUILD CRD: `make kustomize`
+    4. DEPLOY HELM:`make helm-local-install`
+    5. REDEPLOY HELM: `make helm-local-upgrade`
+
 
 
 ### Push to local registry
@@ -86,6 +96,17 @@ It uses [Controllers](https://kubernetes.io/docs/concepts/architecture/controlle
 which provides a reconcile function responsible for synchronizing resources until the desired state is reached on the cluster
 
 
+#### Create a stack
+
+```sh
+kubectl apply -f garden/example-v1beta3.yaml
+### Tests
+
+Run command :
+```sh
+make test
+```
+
 ## License
 
 Copyright 2022.
@@ -100,4 +121,4 @@ Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
-limitations under the License.
+limitations under the License.

components/operator/apis/stack/v1beta3/configuration_types.go

@@ -116,6 +116,7 @@ type ConfigurationSpec struct {
 	Broker   Broker                    `json:"broker"`
 	// +optional
 	Monitoring *MonitoringSpec `json:"monitoring,omitempty"`
+
 	// +optional
 	Ingress  IngressGlobalConfig `json:"ingress,omitempty"`
 	Temporal TemporalConfig      `json:"temporal"`

components/operator/apis/stack/v1beta3/postgres.go

@@ -12,6 +12,9 @@ type PostgresConfig struct {
 	Username string `json:"username"`
 	// +optional
 	Password string `json:"password"`
+
+	// +optional
+	Debug bool `json:"debug"`
 	// +optional
 	CredentialsFromSecret string `json:"credentialsFromSecret"`
 	// +optional

components/operator/config/crd/bases/stack.formance.com_configurations.yaml

@@ -4231,6 +4231,8 @@ spec:
                         properties:
                           credentialsFromSecret:
                             type: string
+                          debug:
+                            type: boolean
                           disableSSLMode:
                             type: boolean
                           host:
@@ -4402,6 +4404,8 @@ spec:
                         properties:
                           credentialsFromSecret:
                             type: string
+                          debug:
+                            type: boolean
                           disableSSLMode:
                             type: boolean
                           host:
@@ -4453,6 +4457,8 @@ spec:
                         properties:
                           credentialsFromSecret:
                             type: string
+                          debug:
+                            type: boolean
                           disableSSLMode:
                             type: boolean
                           host:
@@ -4506,6 +4512,8 @@ spec:
                         properties:
                           credentialsFromSecret:
                             type: string
+                          debug:
+                            type: boolean
                           disableSSLMode:
                             type: boolean
                           host:
@@ -4713,6 +4721,8 @@ spec:
                         properties:
                           credentialsFromSecret:
                             type: string
+                          debug:
+                            type: boolean
                           disableSSLMode:
                             type: boolean
                           host:

components/operator/garden.yml

@@ -8,24 +8,28 @@ spec:
   deployCommand:
     - sh
     - -c
-    - make docker-build-local-prod 
+    - make docker-build-prod
 ---
 apiVersion: garden.io/v0
 kind: Deploy
 description: operator pre-deploy push
 type: exec
 name: operator-push
+dependencies:
+  - deploy.operator-build
 spec:
   deployCommand:
     - sh
     - -c
-    - make docker-push-local-prod
+    - make docker-push
 ---
 apiVersion: garden.io/v0
 kind: Deploy
 description: operator pre-deploy helm update
 type: exec
 name: operator-helm-build
+dependencies:
+  - deploy.operator-push
 spec:
   deployCommand:
     - sh