From 48708b7dc552d21a941b4efe2c646910c79d5a00 Mon Sep 17 00:00:00 2001 From: Patrick Zhao Date: Wed, 12 Jun 2024 11:00:22 +0800 Subject: [PATCH 1/2] do not patch k8s resources if the yaml not change Signed-off-by: Patrick Zhao --- .../aslan/core/common/service/kube/apply.go | 76 +++++++++++++------ .../jobcontroller/job_deploy.go | 2 + .../core/environment/service/environment.go | 2 +- .../core/environment/service/share_env.go | 2 +- 4 files changed, 58 insertions(+), 24 deletions(-) diff --git a/pkg/microservice/aslan/core/common/service/kube/apply.go b/pkg/microservice/aslan/core/common/service/kube/apply.go index d13cb92685..f41e6f9802 100644 --- a/pkg/microservice/aslan/core/common/service/kube/apply.go +++ b/pkg/microservice/aslan/core/common/service/kube/apply.go @@ -277,7 +277,7 @@ func removeResources(currentItems, newItems []*unstructured.Unstructured, namesp } func ManifestToResource(manifest string) ([]*commonmodels.ServiceResource, error) { - unstructuredList, err := ManifestToUnstructured(manifest) + unstructuredList, _, err := ManifestToUnstructured(manifest) if err != nil { return nil, err } @@ -295,22 +295,33 @@ func UnstructuredToResources(unstructured []*unstructured.Unstructured) []*commo return ret } -func ManifestToUnstructured(manifest string) ([]*unstructured.Unstructured, error) { +type Resource struct { + mainfest string + unstructured *unstructured.Unstructured +} + +func ManifestToUnstructured(manifest string) ([]*unstructured.Unstructured, map[string]*Resource, error) { if len(manifest) == 0 { - return nil, nil + return nil, nil, nil } manifests := releaseutil.SplitManifests(manifest) errList := &multierror.Error{} - resources := make([]*unstructured.Unstructured, 0, len(manifests)) + resources := []*unstructured.Unstructured{} + resourceMap := make(map[string]*Resource) for _, item := range manifests { u, err := serializer.NewDecoder().YamlToUnstructured([]byte(item)) if err != nil { errList = multierror.Append(errList, err) continue } + GVKN := fmt.Sprintf("%s-%s", u.GetObjectKind().GroupVersionKind(), u.GetName()) + resourceMap[GVKN] = &Resource{ + mainfest: item, + unstructured: u, + } resources = append(resources, u) } - return resources, errList.ErrorOrNil() + return resources, resourceMap, errList.ErrorOrNil() } func CheckReleaseInstalledByOtherEnv(releaseNames sets.String, productInfo *commonmodels.Product) error { @@ -347,7 +358,7 @@ func CheckReleaseInstalledByOtherEnv(releaseNames sets.String, productInfo *comm } func CheckResourceAppliedByOtherEnv(serviceYaml string, productInfo *commonmodels.Product, serviceName string) error { - unstructuredRes, err := ManifestToUnstructured(serviceYaml) + unstructuredRes, _, err := ManifestToUnstructured(serviceYaml) if err != nil { return fmt.Errorf("failed to convert manifest to resource, error: %v", err) } @@ -399,25 +410,13 @@ func CheckResourceAppliedByOtherEnv(serviceYaml string, productInfo *commonmodel // CreateOrPatchResource create or patch resources defined in UpdateResourceYaml // `CurrentResourceYaml` will be used to determine if some resources will be deleted func CreateOrPatchResource(applyParam *ResourceApplyParam, log *zap.SugaredLogger) ([]*unstructured.Unstructured, error) { + var err error productInfo := applyParam.ProductInfo - namespace, productName, envName := productInfo.Namespace, productInfo.ProductName, productInfo.EnvName informer := applyParam.Informer kubeClient := applyParam.KubeClient istioClient := applyParam.IstioClient - curResources, err := ManifestToUnstructured(applyParam.CurrentResourceYaml) - if err != nil { - log.Errorf("Failed to convert currently deplyed resource yaml to Unstructured, manifest is\n%s\n, error: %v", applyParam.CurrentResourceYaml, err) - return nil, err - } - - resources, err := ManifestToUnstructured(applyParam.UpdateResourceYaml) - if err != nil { - log.Errorf("Failed to convert yaml to Unstructured, manifest is\n%s\n, error: %v", applyParam.UpdateResourceYaml, err) - return nil, err - } - clientSet, errGetClientSet := kubeclient.GetKubeClientSet(config.HubServerAddress(), productInfo.ClusterID) if errGetClientSet != nil { err = errors.WithMessagef(errGetClientSet, "failed to init k8s clientset") @@ -429,19 +428,49 @@ func CreateOrPatchResource(applyParam *ResourceApplyParam, log *zap.SugaredLogge return nil, err } + curResources, curResourceMap, err := ManifestToUnstructured(applyParam.CurrentResourceYaml) + if err != nil { + log.Errorf("Failed to convert currently deplyed resource yaml to Unstructured, manifest is\n%s\n, error: %v", applyParam.CurrentResourceYaml, err) + return nil, err + } + + updateResources, updateResourceMap, err := ManifestToUnstructured(applyParam.UpdateResourceYaml) + if err != nil { + log.Errorf("Failed to convert yaml to Unstructured, manifest is\n%s\n, error: %v", applyParam.UpdateResourceYaml, err) + return nil, err + } + if applyParam.Uninstall { if !commonutil.ServiceDeployed(applyParam.ServiceName, productInfo.ServiceDeployStrategy) { return nil, nil } - err = removeResources(curResources, resources, namespace, applyParam.WaitForUninstall, applyParam.KubeClient, clientSet, versionInfo, log) + err = removeResources(curResources, updateResources, namespace, applyParam.WaitForUninstall, applyParam.KubeClient, clientSet, versionInfo, log) if err != nil { return nil, errors.Wrapf(err, "failed to remove old resources") } return nil, nil } - err = removeResources(curResources, resources, namespace, applyParam.WaitForUninstall, applyParam.KubeClient, clientSet, versionInfo, log) + // calc resources to be removed and resources to be created or updated + removeRes := []*unstructured.Unstructured{} + unchangedResources := []*unstructured.Unstructured{} + for GVKN, cr := range curResourceMap { + r, ok := updateResourceMap[GVKN] + if !ok { + removeRes = append(removeRes, cr.unstructured) + } + if r.mainfest == cr.mainfest { + unchangedResources = append(unchangedResources, cr.unstructured) + delete(updateResourceMap, GVKN) + } + } + updateResources = []*unstructured.Unstructured{} + for _, r := range updateResourceMap { + updateResources = append(updateResources, r.unstructured) + } + + err = removeResources(removeRes, nil, namespace, applyParam.WaitForUninstall, applyParam.KubeClient, clientSet, versionInfo, log) if err != nil { return nil, errors.Wrapf(err, "failed to remove old resources") } @@ -456,7 +485,10 @@ func CreateOrPatchResource(applyParam *ResourceApplyParam, log *zap.SugaredLogge var res []*unstructured.Unstructured errList := &multierror.Error{} - for _, u := range resources { + for _, u := range unchangedResources { + res = append(res, u) + } + for _, u := range updateResources { switch u.GetKind() { case setting.Ingress: ls := MergeLabels(labels, u.GetLabels()) diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_deploy.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_deploy.go index 31f30ee161..5a97b28b3c 100644 --- a/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_deploy.go +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_deploy.go @@ -526,8 +526,10 @@ func (c *DeployJobCtl) getResourcesPodOwnerUIDImpl(strict bool) ([]commonmodels. func (c *DeployJobCtl) getResourcesPodOwnerUID() ([]commonmodels.Resource, error) { timeout := time.After(time.Second * 20) + var newResources []commonmodels.Resource var err error + for { if len(newResources) > 0 || err != nil { break diff --git a/pkg/microservice/aslan/core/environment/service/environment.go b/pkg/microservice/aslan/core/environment/service/environment.go index 785ac305cb..e6ec480a89 100644 --- a/pkg/microservice/aslan/core/environment/service/environment.go +++ b/pkg/microservice/aslan/core/environment/service/environment.go @@ -2241,7 +2241,7 @@ func deleteK8sProductServices(productInfo *commonmodels.Product, serviceNames [] continue } - unstructuredList, err := kube.ManifestToUnstructured(serviceRelatedYaml[name]) + unstructuredList, _, err := kube.ManifestToUnstructured(serviceRelatedYaml[name]) if err != nil { // Only record and do not block subsequent traversals. log.Errorf("failed to convert k8s manifest to unstructured list when deleting service: %s, err: %s", name, err) diff --git a/pkg/microservice/aslan/core/environment/service/share_env.go b/pkg/microservice/aslan/core/environment/service/share_env.go index 451d7d8286..cf2d78feb3 100644 --- a/pkg/microservice/aslan/core/environment/service/share_env.go +++ b/pkg/microservice/aslan/core/environment/service/share_env.go @@ -1023,7 +1023,7 @@ func parseK8SProjectServices(ctx context.Context, env *commonmodels.Product, ser if err != nil { return nil, fmt.Errorf("failed to render env service, err: %w", err) } - resources, err := kube.ManifestToUnstructured(yaml) + resources, _, err := kube.ManifestToUnstructured(yaml) if err != nil { return nil, fmt.Errorf("failed to convert yaml to Unstructured, manifest is\n%s\n, error: %v", yaml, err) } From 794aa8e432956cc068a8279a60dff71f53f9ac47 Mon Sep 17 00:00:00 2001 From: Patrick Zhao Date: Fri, 14 Jun 2024 10:11:40 +0800 Subject: [PATCH 2/2] fix k8s apply resource panic Signed-off-by: Patrick Zhao --- pkg/microservice/aslan/core/common/service/kube/apply.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/microservice/aslan/core/common/service/kube/apply.go b/pkg/microservice/aslan/core/common/service/kube/apply.go index f41e6f9802..53577bb4dd 100644 --- a/pkg/microservice/aslan/core/common/service/kube/apply.go +++ b/pkg/microservice/aslan/core/common/service/kube/apply.go @@ -459,10 +459,11 @@ func CreateOrPatchResource(applyParam *ResourceApplyParam, log *zap.SugaredLogge r, ok := updateResourceMap[GVKN] if !ok { removeRes = append(removeRes, cr.unstructured) - } - if r.mainfest == cr.mainfest { - unchangedResources = append(unchangedResources, cr.unstructured) - delete(updateResourceMap, GVKN) + } else { + if r.mainfest == cr.mainfest { + unchangedResources = append(unchangedResources, cr.unstructured) + delete(updateResourceMap, GVKN) + } } } updateResources = []*unstructured.Unstructured{}