From 0e3a4ca270c3f401dee7464d9947e9845a78c241 Mon Sep 17 00:00:00 2001 From: qj0r9j0vc2 Date: Sun, 30 Jun 2024 21:21:45 +0900 Subject: [PATCH 1/3] feat: Add svc creation logic for sentry mode If set sentry mode, private validator will be horcrux what is connected through horcrux-proxy. In horcrux-proxy, it has automatic discover service logic but cosmos-operator doesn't have automatic service creation logic. through this improvement, horcrux, horcrux-proxy, and cosmos fullnode will be connected automatically. --- internal/fullnode/service_builder.go | 54 ++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/internal/fullnode/service_builder.go b/internal/fullnode/service_builder.go index 6f880a0a..45a64173 100644 --- a/internal/fullnode/service_builder.go +++ b/internal/fullnode/service_builder.go @@ -30,7 +30,12 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service max = *v } maxExternal := lo.Clamp(max, 0, crd.Spec.Replicas) - p2ps := make([]diff.Resource[*corev1.Service], crd.Spec.Replicas) + svcs := make([]diff.Resource[*corev1.Service], func(nodeType cosmosv1.FullNodeType) int32 { + if nodeType == cosmosv1.Sentry { + return crd.Spec.Replicas * 2 + } + return crd.Spec.Replicas + }(crd.Spec.Type)) for i := int32(0); i < crd.Spec.Replicas; i++ { ordinal := i @@ -65,12 +70,51 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service svc.Spec.Type = corev1.ServiceTypeClusterIP } - p2ps[i] = diff.Adapt(&svc, i) + svcs[i] = diff.Adapt(&svc, i) } rpc := rpcService(crd) - return append(p2ps, diff.Adapt(rpc, len(p2ps))) + if crd.Spec.Type == cosmosv1.Sentry { + for i := int32(0); i < crd.Spec.Replicas; i++ { + ordinal := i + var svc corev1.Service + svc.Name = sentryServiceName(crd, ordinal) + svc.Namespace = crd.Namespace + svc.Kind = "Service" + svc.APIVersion = "v1" + + svc.Labels = defaultLabels(crd, + kube.InstanceLabel, instanceName(crd, ordinal), + kube.ComponentLabel, "sentry", + ) + svc.Annotations = map[string]string{} + + svc.Spec.Ports = []corev1.ServicePort{ + { + // https://github.com/strangelove-ventures/horcrux-proxy/blob/23a7d31806ce62481162a64adcca848fd879bc52/cmd/watcher.go#L162 + Name: "sentry-privval", + Protocol: corev1.ProtocolTCP, + Port: privvalPort, + TargetPort: intstr.FromString("privval"), + }, + } + svc.Spec.Selector = map[string]string{kube.InstanceLabel: instanceName(crd, ordinal)} + + preserveMergeInto(svc.Labels, crd.Spec.Service.P2PTemplate.Metadata.Labels) + preserveMergeInto(svc.Annotations, crd.Spec.Service.P2PTemplate.Metadata.Annotations) + svc.Spec.Type = corev1.ServiceTypeClusterIP + + // If you run pod as sentry mode, it'll be unhealthy cause of no API from app. + // But to run cosmos app, horcrux-proxy should connect to pod even pod's status is unhealthy. + // therefore, you should allow this option. + svc.Spec.PublishNotReadyAddresses = true + + svcs[i] = diff.Adapt(&svc, i) + } + } + + return append(svcs, diff.Adapt(rpc, len(svcs))) } func rpcService(crd *cosmosv1.CosmosFullNode) *corev1.Service { @@ -139,6 +183,10 @@ func p2pServiceName(crd *cosmosv1.CosmosFullNode, ordinal int32) string { return fmt.Sprintf("%s-p2p-%d", appName(crd), ordinal) } +func sentryServiceName(crd *cosmosv1.CosmosFullNode, ordinal int32) string { + return fmt.Sprintf("%s-sentry-%d", appName(crd), ordinal) +} + func rpcServiceName(crd *cosmosv1.CosmosFullNode) string { return fmt.Sprintf("%s-rpc", appName(crd)) } From 0745192675c94b1a8a296325017c8edf87362c7e Mon Sep 17 00:00:00 2001 From: vimystic <122659254+vimystic@users.noreply.github.com> Date: Mon, 16 Dec 2024 01:35:53 -0700 Subject: [PATCH 2/3] Updating component label Based on suggestion Co-authored-by: Leo Pang <34628052+allthatjazzleo@users.noreply.github.com> --- internal/fullnode/service_builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/fullnode/service_builder.go b/internal/fullnode/service_builder.go index 45a64173..23e5f63e 100644 --- a/internal/fullnode/service_builder.go +++ b/internal/fullnode/service_builder.go @@ -86,7 +86,7 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service svc.Labels = defaultLabels(crd, kube.InstanceLabel, instanceName(crd, ordinal), - kube.ComponentLabel, "sentry", + kube.ComponentLabel, "cosmos-sentry", ) svc.Annotations = map[string]string{} From fd65a4e11b313ba5432dd96e944cb69222dfbbe8 Mon Sep 17 00:00:00 2001 From: vimystic <122659254+vimystic@users.noreply.github.com> Date: Mon, 16 Dec 2024 01:36:55 -0700 Subject: [PATCH 3/3] Update p2p service name Based on suggestion Co-authored-by: Leo Pang <34628052+allthatjazzleo@users.noreply.github.com> --- internal/fullnode/service_builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/fullnode/service_builder.go b/internal/fullnode/service_builder.go index 23e5f63e..dabb3241 100644 --- a/internal/fullnode/service_builder.go +++ b/internal/fullnode/service_builder.go @@ -184,7 +184,7 @@ func p2pServiceName(crd *cosmosv1.CosmosFullNode, ordinal int32) string { } func sentryServiceName(crd *cosmosv1.CosmosFullNode, ordinal int32) string { - return fmt.Sprintf("%s-sentry-%d", appName(crd), ordinal) + return fmt.Sprintf("%s-privval-%d", appName(crd), ordinal) } func rpcServiceName(crd *cosmosv1.CosmosFullNode) string {