Skip to content

Commit

Permalink
feature: support multi service canary deployment (#61)
Browse files Browse the repository at this point in the history 
support multi service canary deployment

Signed-off-by: gang.liu <[email protected]>
  • Loading branch information
@izturn
izturn authored Jan 5, 2024
1 parent 5218cd7 commit eb414ff
Show file tree
Hide file tree
Showing 4 changed files with 156 additions and 65 deletions.
54 changes: 38 additions & 16 deletions pkg/mocks/plugin.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package mocks

import (
"github.com/argoproj-labs/rollouts-plugin-trafficrouter-contour/pkg/utils"

contourv1 "github.com/projectcontour/contour/apis/projectcontour/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
Expand All @@ -9,14 +11,19 @@ import (
const (
StableServiceName = "argo-rollouts-stable"
CanaryServiceName = "argo-rollouts-canary"
AddOnServiceName = "argo-rollouts-addon"

HTTPProxyName = "argo-rollouts"
ValidHTTPProxyName = "argo-rollouts-valid"
OutdatedHTTPProxyName = "argo-rollouts-outdated"
InvalidHTTPProxyName = "argo-rollouts-invalid"
FalseConditionHTTPProxyName = "argo-rollouts-false-condition"

HTTPProxyDesiredWeight = 20
// HTTPProxyAddOnWeight represents the add-ons services' weight in the total weight
HTTPProxyAddOnWeight = 20

// HTTPProxyCanaryWeightPercent represents the canary's weight for the canary deploment service (only)
HTTPProxyCanaryWeightPercent = 40
)

const (
Expand All @@ -34,27 +41,29 @@ func makeDetailedCondition(typ string, status contourv1.ConditionStatus) contour
}
}

func makeService(name string, weight int64) contourv1.Service {
return contourv1.Service{
Name: name,
Weight: weight,
func MakeName(origin string, appendPostfix ...bool) string {
if len(appendPostfix) == 0 || !appendPostfix[0] {
return origin
}
return origin + "-addon"
}
func MakeObjects() []runtime.Object {
httpProxy := newHTTPProxy(HTTPProxyName)
validHttpProxy := newHTTPProxy(ValidHTTPProxyName)

invalidHttpProxy := newHTTPProxy(InvalidHTTPProxyName)
func MakeObjects(appendPostfix bool, addonServices ...contourv1.Service) []runtime.Object {

httpProxy := newHTTPProxy(MakeName(HTTPProxyName, appendPostfix), addonServices...)
validHttpProxy := newHTTPProxy(MakeName(ValidHTTPProxyName, appendPostfix), addonServices...)

invalidHttpProxy := newHTTPProxy(MakeName(InvalidHTTPProxyName, appendPostfix), addonServices...)
invalidHttpProxy.Status = contourv1.HTTPProxyStatus{
Conditions: []contourv1.DetailedCondition{
makeDetailedCondition(contourv1.ConditionTypeServiceError, contourv1.ConditionTrue),
},
}

outdatedHttpProxy := newHTTPProxy(OutdatedHTTPProxyName)
outdatedHttpProxy := newHTTPProxy(MakeName(OutdatedHTTPProxyName, appendPostfix), addonServices...)
outdatedHttpProxy.Generation = httpProxyGeneration + 1

falseConditionHttpProxy := newHTTPProxy(FalseConditionHTTPProxyName)
falseConditionHttpProxy := newHTTPProxy(MakeName(FalseConditionHTTPProxyName, appendPostfix), addonServices...)
falseConditionHttpProxy.Status = contourv1.HTTPProxyStatus{
Conditions: []contourv1.DetailedCondition{
makeDetailedCondition(contourv1.ValidConditionType, contourv1.ConditionFalse),
Expand All @@ -71,7 +80,23 @@ func MakeObjects() []runtime.Object {
return objs
}

func newHTTPProxy(name string) *contourv1.HTTPProxy {
func mainServices(totalWeight int64) []contourv1.Service {
canaryWeight, stableWeight := utils.CalcWeight(totalWeight, HTTPProxyCanaryWeightPercent)
return []contourv1.Service{
utils.MakeService(StableServiceName, stableWeight),
utils.MakeService(CanaryServiceName, canaryWeight),
}
}
func newHTTPProxy(name string, addOnServices ...contourv1.Service) *contourv1.HTTPProxy {
totalWeight := int64(100)

for _, svc := range addOnServices {
totalWeight -= svc.Weight
}

services := mainServices(totalWeight)
services = append(services, addOnServices...)

return &contourv1.HTTPProxy{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Expand All @@ -81,10 +106,7 @@ func newHTTPProxy(name string) *contourv1.HTTPProxy {
Spec: contourv1.HTTPProxySpec{
Routes: []contourv1.Route{
{
Services: []contourv1.Service{
makeService(StableServiceName, 100-HTTPProxyDesiredWeight),
makeService(CanaryServiceName, HTTPProxyDesiredWeight),
},
Services: services,
},
},
},
Expand Down
58 changes: 40 additions & 18 deletions pkg/plugin/plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func (r *RpcPlugin) UpdateHash(rollout *v1alpha1.Rollout, canaryHash, stableHash
return pluginTypes.RpcError{}
}

func (r *RpcPlugin) SetWeight(rollout *v1alpha1.Rollout, desiredWeight int32, additionalDestinations []v1alpha1.WeightDestination) pluginTypes.RpcError {
func (r *RpcPlugin) SetWeight(rollout *v1alpha1.Rollout, canaryWeightPercent int32, additionalDestinations []v1alpha1.WeightDestination) pluginTypes.RpcError {
if err := validateRolloutParameters(rollout); err != nil {
return pluginTypes.RpcError{ErrorString: err.Error()}
}
Expand All @@ -72,7 +72,7 @@ func (r *RpcPlugin) SetWeight(rollout *v1alpha1.Rollout, desiredWeight int32, ad
for _, proxy := range ctr.HTTPProxies {
slog.Debug("updating httpproxy", slog.String("name", proxy))

if err := r.updateHTTPProxy(ctx, proxy, rollout, desiredWeight); err != nil {
if err := r.updateHTTPProxy(ctx, proxy, rollout, canaryWeightPercent); err != nil {
slog.Error("failed to update httpproxy", slog.String("name", proxy), slog.Any("err", err))
return pluginTypes.RpcError{ErrorString: err.Error()}
}
Expand All @@ -91,7 +91,7 @@ func (r *RpcPlugin) SetMirrorRoute(rollout *v1alpha1.Rollout, setMirrorRoute *v1
return pluginTypes.RpcError{}
}

func (r *RpcPlugin) VerifyWeight(rollout *v1alpha1.Rollout, desiredWeight int32, additionalDestinations []v1alpha1.WeightDestination) (pluginTypes.RpcVerified, pluginTypes.RpcError) {
func (r *RpcPlugin) VerifyWeight(rollout *v1alpha1.Rollout, canaryWeightPercent int32, additionalDestinations []v1alpha1.WeightDestination) (pluginTypes.RpcVerified, pluginTypes.RpcError) {
if err := validateRolloutParameters(rollout); err != nil {
return pluginTypes.NotVerified, pluginTypes.RpcError{ErrorString: err.Error()}
}
Expand All @@ -106,7 +106,7 @@ func (r *RpcPlugin) VerifyWeight(rollout *v1alpha1.Rollout, desiredWeight int32,
for _, proxy := range ctr.HTTPProxies {
slog.Debug("verifying httpproxy", slog.String("name", proxy))

verified, err := r.verifyHTTPProxy(ctx, proxy, rollout, desiredWeight)
verified, err := r.verifyHTTPProxy(ctx, proxy, rollout, canaryWeightPercent)
if err != nil {
slog.Error("failed to verify httpproxy", slog.String("name", proxy), slog.Any("err", err))
return pluginTypes.NotVerified, pluginTypes.RpcError{ErrorString: err.Error()}
Expand Down Expand Up @@ -142,21 +142,25 @@ func (r *RpcPlugin) getHTTPProxy(ctx context.Context, namespace string, name str
return &httpProxy, nil
}

func (r *RpcPlugin) updateHTTPProxy(ctx context.Context, httpProxyName string, rollout *v1alpha1.Rollout, desiredWeight int32) error {
func (r *RpcPlugin) updateHTTPProxy(
ctx context.Context,
httpProxyName string,
rollout *v1alpha1.Rollout,
canaryWeightPercent int32) error {

httpProxy, err := r.getHTTPProxy(ctx, rollout.Namespace, httpProxyName)
if err != nil {
return err
}

canarySvc, stableSvc, err := getCanaryAndStableServices(httpProxy, rollout)
canarySvc, stableSvc, totalWeight, err := getRouteServices(httpProxy, rollout)
if err != nil {
return err
}

slog.Debug("old weight", slog.Int64("canary", canarySvc.Weight), slog.Int64("stable", stableSvc.Weight))

canarySvc.Weight = int64(desiredWeight)
stableSvc.Weight = 100 - canarySvc.Weight
canarySvc.Weight, stableSvc.Weight = utils.CalcWeight(totalWeight, float32(canaryWeightPercent))

slog.Debug("new weight", slog.Int64("canary", canarySvc.Weight), slog.Int64("stable", stableSvc.Weight))

Expand All @@ -180,7 +184,12 @@ func (r *RpcPlugin) updateHTTPProxy(ctx context.Context, httpProxyName string, r
return nil
}

func (r *RpcPlugin) verifyHTTPProxy(ctx context.Context, httpProxyName string, rollout *v1alpha1.Rollout, desiredWeight int32) (bool, error) {
func (r *RpcPlugin) verifyHTTPProxy(
ctx context.Context,
httpProxyName string,
rollout *v1alpha1.Rollout,
canaryWeightPercent int32) (bool, error) {

httpProxy, err := r.getHTTPProxy(ctx, rollout.Namespace, httpProxyName)
if err != nil {
return false, err
Expand All @@ -200,22 +209,22 @@ func (r *RpcPlugin) verifyHTTPProxy(ctx context.Context, httpProxyName string, r
return false, nil
}

canarySvc, stableSvc, err := getCanaryAndStableServices(httpProxy, rollout)
canarySvc, stableSvc, totalWeight, err := getRouteServices(httpProxy, rollout)
if err != nil {
return false, err
}

canarySvcDesiredWeight := int64(desiredWeight)
stableSvcDesiredWeight := 100 - canarySvcDesiredWeight
if canarySvc.Weight != canarySvcDesiredWeight || stableSvc.Weight != stableSvcDesiredWeight {
slog.Debug(fmt.Sprintf("expected weights are canary=%d and stable=%d, but got canary=%d and stable=%d", canarySvcDesiredWeight, stableSvcDesiredWeight, canarySvc.Weight, stableSvc.Weight), slog.String("name", httpProxyName))
canaryWeight, stableWeight := utils.CalcWeight(totalWeight, float32(canaryWeightPercent))
if canarySvc.Weight != canaryWeight || stableSvc.Weight != stableWeight {
slog.Debug(fmt.Sprintf("expected weights are canary=%d and stable=%d, but got canary=%d and stable=%d", canaryWeight, stableWeight, canarySvc.Weight, stableSvc.Weight), slog.String("name", httpProxyName))
return false, nil
}

return true, nil
}

func getCanaryAndStableServices(httpProxy *contourv1.HTTPProxy, rollout *v1alpha1.Rollout) (*contourv1.Service, *contourv1.Service, error) {
func getRouteServices(httpProxy *contourv1.HTTPProxy, rollout *v1alpha1.Rollout) (
*contourv1.Service, *contourv1.Service, int64, error) {
canarySvcName := rollout.Spec.Strategy.Canary.CanaryService
stableSvcName := rollout.Spec.Strategy.Canary.StableService

Expand All @@ -226,15 +235,28 @@ func getCanaryAndStableServices(httpProxy *contourv1.HTTPProxy, rollout *v1alpha

canarySvc, err := getService(canarySvcName, svcMap)
if err != nil {
return nil, nil, err
return nil, nil, 0, err
}

stableSvc, err := getService(stableSvcName, svcMap)
if err != nil {
return nil, nil, err
return nil, nil, 0, err
}

otherWeight := int64(0)
for name, svc := range svcMap {
if name == stableSvcName || name == canarySvcName {
continue
}
otherWeight += svc.Weight
}

// the total weight must equals to 100
if otherWeight+canarySvc.Weight+stableSvc.Weight != 100 {
return nil, nil, 0, fmt.Errorf("the total weight must equals to 100")
}

return canarySvc, stableSvc, nil
return canarySvc, stableSvc, 100 - otherWeight, nil
}

func getContourTrafficRouting(rollout *v1alpha1.Rollout) (*ContourTrafficRouting, error) {
Expand Down
94 changes: 63 additions & 31 deletions pkg/plugin/plugin_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ import (
"testing"
"time"

"github.com/argoproj-labs/rollouts-plugin-trafficrouter-contour/pkg/mocks"
"github.com/argoproj-labs/rollouts-plugin-trafficrouter-contour/pkg/utils"

"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
rolloutsPlugin "github.com/argoproj/argo-rollouts/rollout/trafficrouting/plugin/rpc"
"github.com/argoproj/argo-rollouts/utils/plugin/types"
Expand All @@ -16,9 +19,6 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
fakeDynClient "k8s.io/client-go/dynamic/fake"

"github.com/argoproj-labs/rollouts-plugin-trafficrouter-contour/pkg/mocks"
"github.com/argoproj-labs/rollouts-plugin-trafficrouter-contour/pkg/utils"
)

var testHandshake = goPlugin.HandshakeConfig{
Expand All @@ -39,7 +39,16 @@ func TestRunSuccessfully(t *testing.T) {
}

_ = b.AddToScheme(s)
dynClient := fakeDynClient.NewSimpleDynamicClient(s, mocks.MakeObjects()...)

// no add-on service
objects := mocks.MakeObjects(false)

addOnSvc := utils.MakeService(mocks.AddOnServiceName, mocks.HTTPProxyAddOnWeight)

// with add-on service
objects = append(objects, mocks.MakeObjects(true, addOnSvc)...)

dynClient := fakeDynClient.NewSimpleDynamicClient(s, objects...)
rpcPluginImp := &RpcPlugin{
IsTest: true,
dynamicClient: dynClient,
Expand Down Expand Up @@ -107,43 +116,66 @@ func TestRunSuccessfully(t *testing.T) {
t.Fail()
}

t.Run("SetWeight", func(t *testing.T) {
rollout := newRollout(mocks.StableServiceName, mocks.CanaryServiceName, mocks.HTTPProxyName)
desiredWeight := int32(30)

if err := pluginInstance.SetWeight(rollout, desiredWeight, []v1alpha1.WeightDestination{}); err.HasError() {
t.Fail()
}
makeSetWeightTester := func(name string, totalWeight, canaryWeightPercent int32) func(t *testing.T) {
return func(t *testing.T) {
rollout := newRollout(mocks.StableServiceName, mocks.CanaryServiceName, name)

svcs := rpcPluginImp.UpdatedMockHTTPProxy.Spec.Routes[0].Services
if err := pluginInstance.SetWeight(rollout, canaryWeightPercent, []v1alpha1.WeightDestination{}); err.HasError() {
t.Fail()
}

if 100-desiredWeight != int32(svcs[0].Weight) {
t.Fail()
}
if desiredWeight != int32(svcs[1].Weight) {
t.Fail()
}
})
canaryWeight, stableWeight := utils.CalcWeight(int64(totalWeight), float32(canaryWeightPercent))

t.Run("VerifyWeight", func(t *testing.T) {
verifyWeight := func(httpProxyName string, desiredWeight int32, expected types.RpcVerified) {
rollout := newRollout(mocks.StableServiceName, mocks.CanaryServiceName, httpProxyName)
svcs := rpcPluginImp.UpdatedMockHTTPProxy.Spec.Routes[0].Services

actual, err := pluginInstance.VerifyWeight(rollout, desiredWeight, []v1alpha1.WeightDestination{})
if err.HasError() {
if stableWeight != svcs[0].Weight {
t.Fail()
}
if actual != expected {
if canaryWeight != svcs[1].Weight {
t.Fail()
}
}
}

makeVerifyWeightTester := func(canaryWeightPercent int32, appendPostfix ...bool) func(t *testing.T) {
return func(t *testing.T) {
verifyWeight := func(httpProxyName string, canaryWeightPercent int32, expected types.RpcVerified) {
rollout := newRollout(mocks.StableServiceName, mocks.CanaryServiceName, httpProxyName)

actual, err := pluginInstance.VerifyWeight(rollout, canaryWeightPercent, []v1alpha1.WeightDestination{})
if err.HasError() {
t.Fail()
}
if actual != expected {
t.Fail()
}
}

verifyWeight(mocks.MakeName(mocks.ValidHTTPProxyName, appendPostfix...), canaryWeightPercent, types.Verified)
verifyWeight(mocks.MakeName(mocks.ValidHTTPProxyName, appendPostfix...), canaryWeightPercent+10, types.NotVerified)
verifyWeight(mocks.MakeName(mocks.InvalidHTTPProxyName, appendPostfix...), canaryWeightPercent, types.NotVerified)
verifyWeight(mocks.MakeName(mocks.OutdatedHTTPProxyName, appendPostfix...), canaryWeightPercent, types.NotVerified)
verifyWeight(mocks.MakeName(mocks.FalseConditionHTTPProxyName, appendPostfix...), canaryWeightPercent, types.NotVerified)
}
}

verifyWeight(mocks.ValidHTTPProxyName, mocks.HTTPProxyDesiredWeight, types.Verified)
verifyWeight(mocks.ValidHTTPProxyName, mocks.HTTPProxyDesiredWeight+10, types.NotVerified)
verifyWeight(mocks.InvalidHTTPProxyName, mocks.HTTPProxyDesiredWeight, types.NotVerified)
verifyWeight(mocks.OutdatedHTTPProxyName, mocks.HTTPProxyDesiredWeight, types.NotVerified)
verifyWeight(mocks.FalseConditionHTTPProxyName, mocks.HTTPProxyDesiredWeight, types.NotVerified)
})
t.Run(
mocks.MakeName("SetWeight"),
makeSetWeightTester(
mocks.HTTPProxyName,
100,
mocks.HTTPProxyCanaryWeightPercent))
t.Run(
mocks.MakeName("SetWeight", true),
makeSetWeightTester(
mocks.MakeName(mocks.HTTPProxyName, true),
100-mocks.HTTPProxyAddOnWeight,
mocks.HTTPProxyCanaryWeightPercent))
t.Run("VerifyWeight",
makeVerifyWeightTester(mocks.HTTPProxyCanaryWeightPercent))
t.Run(
mocks.MakeName("VerifyWeight", true),
makeVerifyWeightTester(mocks.HTTPProxyCanaryWeightPercent, true))

// Canceling should cause an exit
cancel()
Expand Down
Loading

0 comments on commit eb414ff

Please sign in to comment.