Merge branch 'master' into feat/grpc-message-size-as-env

cmd/argocd/commands/root.go

@@ -75,11 +75,11 @@ func NewCommand() *cobra.Command {
 	command.PersistentFlags().StringVar(&clientOpts.GRPCWebRootPath, "grpc-web-root-path", config.GetFlag("grpc-web-root-path", ""), "Enables gRPC-web protocol. Useful if Argo CD server is behind proxy which does not support HTTP2. Set web root.")
 	command.PersistentFlags().StringVar(&cmdutil.LogFormat, "logformat", config.GetFlag("logformat", "text"), "Set the logging format. One of: text|json")
 	command.PersistentFlags().StringVar(&cmdutil.LogLevel, "loglevel", config.GetFlag("loglevel", "info"), "Set the logging level. One of: debug|info|warn|error")
-	command.PersistentFlags().StringSliceVarP(&clientOpts.Headers, "header", "H", []string{}, "Sets additional header to all requests made by Argo CD CLI. (Can be repeated multiple times to add multiple headers, also supports comma separated headers)")
+	command.PersistentFlags().StringSliceVarP(&clientOpts.Headers, "header", "H", config.GetStringSliceFlag("header", []string{}), "Sets additional header to all requests made by Argo CD CLI. (Can be repeated multiple times to add multiple headers, also supports comma separated headers)")
 	command.PersistentFlags().BoolVar(&clientOpts.PortForward, "port-forward", config.GetBoolFlag("port-forward"), "Connect to a random argocd-server port using port forwarding")
 	command.PersistentFlags().StringVar(&clientOpts.PortForwardNamespace, "port-forward-namespace", config.GetFlag("port-forward-namespace", ""), "Namespace name which should be used for port forwarding")
-	command.PersistentFlags().IntVar(&clientOpts.HttpRetryMax, "http-retry-max", 0, "Maximum number of retries to establish http connection to Argo CD server")
-	command.PersistentFlags().BoolVar(&clientOpts.Core, "core", false, "If set to true then CLI talks directly to Kubernetes instead of talking to Argo CD API server")
+	command.PersistentFlags().IntVar(&clientOpts.HttpRetryMax, "http-retry-max", config.GetIntFlag("http-retry-max", 0), "Maximum number of retries to establish http connection to Argo CD server")
+	command.PersistentFlags().BoolVar(&clientOpts.Core, "core", config.GetBoolFlag("core"), "If set to true then CLI talks directly to Kubernetes instead of talking to Argo CD API server")
 	command.PersistentFlags().StringVar(&clientOpts.ServerName, "server-name", env.StringFromEnv(common.EnvServerName, common.DefaultServerName), fmt.Sprintf("Name of the Argo CD API server; set this or the %s environment variable when the server's name label differs from the default, for example when installing via the Helm chart", common.EnvServerName))
 	command.PersistentFlags().StringVar(&clientOpts.AppControllerName, "controller-name", env.StringFromEnv(common.EnvAppControllerName, common.DefaultApplicationControllerName), fmt.Sprintf("Name of the Argo CD Application controller; set this or the %s environment variable when the controller's name label differs from the default, for example when installing via the Helm chart", common.EnvAppControllerName))
 	command.PersistentFlags().StringVar(&clientOpts.RedisHaProxyName, "redis-haproxy-name", env.StringFromEnv(common.EnvRedisHaProxyName, common.DefaultRedisHaProxyName), fmt.Sprintf("Name of the Redis HA Proxy; set this or the %s environment variable when the HA Proxy's name label differs from the default, for example when installing via the Helm chart", common.EnvRedisHaProxyName))

cmd/util/app_test.go

@@ -174,12 +174,12 @@ func (f *appOptionsFixture) SetFlag(key, value string) error {
 	return err
 }
 
-func (f *appOptionsFixture) SetFlagWithSourceIndex(key, value string, index int) error {
+func (f *appOptionsFixture) SetFlagWithSourcePosition(key, value string, sourcePosition int) error {
 	err := f.command.Flags().Set(key, value)
 	if err != nil {
 		return err
 	}
-	_ = SetAppSpecOptions(f.command.Flags(), f.spec, f.options, index)
+	_ = SetAppSpecOptions(f.command.Flags(), f.spec, f.options, sourcePosition)
 	return err
 }
 
@@ -251,34 +251,34 @@ func newMultiSourceAppOptionsFixture() *appOptionsFixture {
 
 func Test_setAppSpecOptionsMultiSourceApp(t *testing.T) {
 	f := newMultiSourceAppOptionsFixture()
-	index := 0
-	index1 := 1
-	index2 := 2
+	sourcePosition := 0
+	sourcePosition1 := 1
+	sourcePosition2 := 2
 	t.Run("SyncPolicy", func(t *testing.T) {
-		assert.NoError(t, f.SetFlagWithSourceIndex("sync-policy", "automated", index1))
+		assert.NoError(t, f.SetFlagWithSourcePosition("sync-policy", "automated", sourcePosition1))
 		assert.NotNil(t, f.spec.SyncPolicy.Automated)
 
 		f.spec.SyncPolicy = nil
-		assert.NoError(t, f.SetFlagWithSourceIndex("sync-policy", "automatic", index1))
+		assert.NoError(t, f.SetFlagWithSourcePosition("sync-policy", "automatic", sourcePosition1))
 		assert.NotNil(t, f.spec.SyncPolicy.Automated)
 	})
-	t.Run("Helm - Index 0", func(t *testing.T) {
-		assert.NoError(t, f.SetFlagWithSourceIndex("helm-version", "v2", index))
+	t.Run("Helm - SourcePosition 0", func(t *testing.T) {
+		assert.NoError(t, f.SetFlagWithSourcePosition("helm-version", "v2", sourcePosition))
 		assert.Equal(t, len(f.spec.GetSources()), 2)
-		assert.Equal(t, f.spec.GetSources()[index].Helm.Version, "v2")
+		assert.Equal(t, f.spec.GetSources()[sourcePosition].Helm.Version, "v2")
 	})
 	t.Run("Kustomize", func(t *testing.T) {
-		assert.NoError(t, f.SetFlagWithSourceIndex("kustomize-replica", "my-deployment=2", index1))
-		assert.Equal(t, f.spec.Sources[index1-1].Kustomize.Replicas, v1alpha1.KustomizeReplicas{{Name: "my-deployment", Count: intstr.FromInt(2)}})
-		assert.NoError(t, f.SetFlagWithSourceIndex("kustomize-replica", "my-deployment=4", index2))
-		assert.Equal(t, f.spec.Sources[index2-1].Kustomize.Replicas, v1alpha1.KustomizeReplicas{{Name: "my-deployment", Count: intstr.FromInt(4)}})
+		assert.NoError(t, f.SetFlagWithSourcePosition("kustomize-replica", "my-deployment=2", sourcePosition1))
+		assert.Equal(t, f.spec.Sources[sourcePosition1-1].Kustomize.Replicas, v1alpha1.KustomizeReplicas{{Name: "my-deployment", Count: intstr.FromInt(2)}})
+		assert.NoError(t, f.SetFlagWithSourcePosition("kustomize-replica", "my-deployment=4", sourcePosition2))
+		assert.Equal(t, f.spec.Sources[sourcePosition2-1].Kustomize.Replicas, v1alpha1.KustomizeReplicas{{Name: "my-deployment", Count: intstr.FromInt(4)}})
 	})
 	t.Run("Helm", func(t *testing.T) {
-		assert.NoError(t, f.SetFlagWithSourceIndex("helm-version", "v2", index1))
-		assert.NoError(t, f.SetFlagWithSourceIndex("helm-version", "v3", index2))
+		assert.NoError(t, f.SetFlagWithSourcePosition("helm-version", "v2", sourcePosition1))
+		assert.NoError(t, f.SetFlagWithSourcePosition("helm-version", "v3", sourcePosition2))
 		assert.Equal(t, len(f.spec.GetSources()), 2)
-		assert.Equal(t, f.spec.GetSources()[index1-1].Helm.Version, "v2")
-		assert.Equal(t, f.spec.GetSources()[index2-1].Helm.Version, "v3")
+		assert.Equal(t, f.spec.GetSources()[sourcePosition1-1].Helm.Version, "v2")
+		assert.Equal(t, f.spec.GetSources()[sourcePosition2-1].Helm.Version, "v3")
 	})
 }
 

controller/appcontroller_test.go

@@ -53,14 +53,15 @@ type namespacedResource struct {
 }
 
 type fakeData struct {
-	apps                   []runtime.Object
-	manifestResponse       *apiclient.ManifestResponse
-	manifestResponses      []*apiclient.ManifestResponse
-	managedLiveObjs        map[kube.ResourceKey]*unstructured.Unstructured
-	namespacedResources    map[kube.ResourceKey]namespacedResource
-	configMapData          map[string]string
-	metricsCacheExpiration time.Duration
-	applicationNamespaces  []string
+	apps                           []runtime.Object
+	manifestResponse               *apiclient.ManifestResponse
+	manifestResponses              []*apiclient.ManifestResponse
+	managedLiveObjs                map[kube.ResourceKey]*unstructured.Unstructured
+	namespacedResources            map[kube.ResourceKey]namespacedResource
+	configMapData                  map[string]string
+	metricsCacheExpiration         time.Duration
+	applicationNamespaces          []string
+	updateRevisionForPathsResponse *apiclient.UpdateRevisionForPathsResponse
 }
 
 type MockKubectl struct {
@@ -106,6 +107,8 @@ func newFakeController(data *fakeData, repoErr error) *ApplicationController {
 		}
 	}
 
+	mockRepoClient.On("UpdateRevisionForPaths", mock.Anything, mock.Anything).Return(data.updateRevisionForPathsResponse, nil)
+
 	mockRepoClientset := mockrepoclient.Clientset{RepoServerServiceClient: &mockRepoClient}
 
 	secret := corev1.Secret{

controller/state.go

@@ -33,6 +33,7 @@ import (
 	"github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
 	appclientset "github.com/argoproj/argo-cd/v2/pkg/client/clientset/versioned"
 	"github.com/argoproj/argo-cd/v2/reposerver/apiclient"
+	"github.com/argoproj/argo-cd/v2/util/app/path"
 	"github.com/argoproj/argo-cd/v2/util/argo"
 	argodiff "github.com/argoproj/argo-cd/v2/util/argo/diff"
 	appstatecache "github.com/argoproj/argo-cd/v2/util/cache/appstate"
@@ -194,6 +195,38 @@ func (m *appStateManager) GetRepoObjs(app *v1alpha1.Application, sources []v1alp
 			return nil, nil, fmt.Errorf("failed to get Kustomize options for source %d of %d: %w", i+1, len(sources), err)
 		}
 
+		syncedRevision := app.Status.Sync.Revision
+		if app.Spec.HasMultipleSources() {
+			if i < len(app.Status.Sync.Revisions) {
+				syncedRevision = app.Status.Sync.Revisions[i]
+			} else {
+				syncedRevision = ""
+			}
+		}
+
+		val, ok := app.Annotations[v1alpha1.AnnotationKeyManifestGeneratePaths]
+		if !source.IsHelm() && syncedRevision != "" && ok && val != "" {
+			// Validate the manifest-generate-path annotation to avoid generating manifests if it has not changed.
+			_, err = repoClient.UpdateRevisionForPaths(context.Background(), &apiclient.UpdateRevisionForPathsRequest{
+				Repo:               repo,
+				Revision:           revisions[i],
+				SyncedRevision:     syncedRevision,
+				Paths:              path.GetAppRefreshPaths(app),
+				AppLabelKey:        appLabelKey,
+				AppName:            app.InstanceName(m.namespace),
+				Namespace:          app.Spec.Destination.Namespace,
+				ApplicationSource:  &source,
+				KubeVersion:        serverVersion,
+				ApiVersions:        argo.APIResourcesToStrings(apiResources, true),
+				TrackingMethod:     string(argo.GetTrackingMethod(m.settingsMgr)),
+				RefSources:         refSources,
+				HasMultipleSources: app.Spec.HasMultipleSources(),
+			})
+			if err != nil {
+				return nil, nil, fmt.Errorf("failed to compare revisions for source %d of %d: %w", i+1, len(sources), err)
+			}
+		}
+
 		ts.AddCheckpoint("version_ms")
 		log.Debugf("Generating Manifest for source %s revision %s", source, revisions[i])
 		manifestInfo, err := repoClient.GenerateManifest(context.Background(), &apiclient.ManifestRequest{

controller/state_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
 	argoappv1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
 	"github.com/argoproj/argo-cd/v2/reposerver/apiclient"
+	mockrepoclient "github.com/argoproj/argo-cd/v2/reposerver/apiclient/mocks"
 	"github.com/argoproj/argo-cd/v2/test"
 	"github.com/argoproj/argo-cd/v2/util/argo"
 )
@@ -651,6 +652,37 @@ var defaultProj = argoappv1.AppProject{
 	},
 }
 
+// TestCompareAppStateWithManifestGeneratePath tests that it compares revisions when the manifest-generate-path annotation is set.
+func TestCompareAppStateWithManifestGeneratePath(t *testing.T) {
+	app := newFakeApp()
+	app.SetAnnotations(map[string]string{argoappv1.AnnotationKeyManifestGeneratePaths: "."})
+	app.Status.Sync = argoappv1.SyncStatus{
+		Revision: "abc123",
+		Status:   argoappv1.SyncStatusCodeSynced,
+	}
+
+	data := fakeData{
+		manifestResponse: &apiclient.ManifestResponse{
+			Manifests: []string{},
+			Namespace: test.FakeDestNamespace,
+			Server:    test.FakeClusterURL,
+			Revision:  "abc123",
+		},
+		updateRevisionForPathsResponse: &apiclient.UpdateRevisionForPathsResponse{},
+	}
+
+	ctrl := newFakeController(&data, nil)
+	revisions := make([]string, 0)
+	revisions = append(revisions, "abc123")
+	compRes, err := ctrl.appStateManager.CompareAppState(app, &defaultProj, revisions, app.Spec.GetSources(), false, false, nil, false)
+
+	assert.Nil(t, err)
+	assert.NotNil(t, compRes)
+	assert.Equal(t, argoappv1.SyncStatusCodeSynced, compRes.syncStatus.Status)
+	assert.Equal(t, "abc123", compRes.syncStatus.Revision)
+	ctrl.repoClientset.(*mockrepoclient.Clientset).RepoServerServiceClient.(*mockrepoclient.RepoServerServiceClient).AssertNumberOfCalls(t, "UpdateRevisionForPaths", 1)
+}
+
 func TestSetHealth(t *testing.T) {
 	app := newFakeApp()
 	deployment := kube.MustToUnstructured(&v1.Deployment{

docs/operator-manual/high_availability.md

@@ -178,17 +178,19 @@ If the manifest generation has no side effects then requests are processed in pa
   * **Multiple Kustomize applications in same repository with [parameter overrides](../user-guide/parameters.md):** sorry, no workaround for now.
 
 
-### Webhook and Manifest Paths Annotation
+### Manifest Paths Annotation
 
 Argo CD aggressively caches generated manifests and uses the repository commit SHA as a cache key. A new commit to the Git repository invalidates the cache for all applications configured in the repository.
 This can negatively affect repositories with multiple applications. You can use [webhooks](https://github.com/argoproj/argo-cd/blob/master/docs/operator-manual/webhook.md) and the `argocd.argoproj.io/manifest-generate-paths` Application CRD annotation to solve this problem and improve performance.
 
-The `argocd.argoproj.io/manifest-generate-paths` annotation contains a semicolon-separated list of paths within the Git repository that are used during manifest generation. The webhook compares paths specified in the annotation with the changed files specified in the webhook payload. If no modified files match the paths specified in `argocd.argoproj.io/manifest-generate-paths`, then the webhook will not trigger application reconciliation and the existing cache will be considered valid for the new commit.
+The `argocd.argoproj.io/manifest-generate-paths` annotation contains a semicolon-separated list of paths within the Git repository that are used during manifest generation. It will use the paths specified in the annotation to compare the last cached revision to the latest commit. If no modified files match the paths specified in `argocd.argoproj.io/manifest-generate-paths`, then it will not trigger application reconciliation and the existing cache will be considered valid for the new commit.
 
 Installations that use a different repository for each application are **not** subject to this behavior and will likely get no benefit from using these annotations.
 
+For webhooks, the comparison is done using the files specified in the webhook event payload instead.
+
 !!! note
-    Application manifest paths annotation support depends on the git provider used for the Application. It is currently only supported for GitHub, GitLab, and Gogs based repos.
+    Application manifest paths annotation support for webhooks depends on the git provider used for the Application. It is currently only supported for GitHub, GitLab, and Gogs based repos.
 
 * **Relative path** The annotation might contain a relative path. In this case the path is considered relative to the path specified in the application source:
 

docs/user-guide/commands/argocd_app.md

@@ -91,7 +91,7 @@ argocd app [flags]
 * [argocd app manifests](argocd_app_manifests.md)	 - Print manifests of an application
 * [argocd app patch](argocd_app_patch.md)	 - Patch application
 * [argocd app patch-resource](argocd_app_patch-resource.md)	 - Patch resource in an application
-* [argocd app remove-source](argocd_app_remove-source.md)	 - Remove a source from multiple sources application. Index starts with 1. Default value is -1.
+* [argocd app remove-source](argocd_app_remove-source.md)	 - Remove a source from multiple sources application. Counting starts with 1. Default value is -1.
 * [argocd app resources](argocd_app_resources.md)	 - List resource of application
 * [argocd app rollback](argocd_app_rollback.md)	 - Rollback application to a previous deployed version by History ID, omitted will Rollback to the previous version
 * [argocd app set](argocd_app_set.md)	 - Set application parameters

docs/user-guide/commands/argocd_app_diff.md

@@ -18,18 +18,18 @@ argocd app diff APPNAME [flags]
 ### Options
 
 ```
-  -N, --app-namespace string        Only render the difference in namespace
-      --exit-code                   Return non-zero exit code when there is a diff (default true)
-      --hard-refresh                Refresh application data as well as target manifests cache
-  -h, --help                        help for diff
-      --local string                Compare live app to a local manifests
-      --local-include stringArray   Used with --server-side-generate, specify patterns of filenames to send. Matching is based on filename and not path. (default [*.yaml,*.yml,*.json])
-      --local-repo-root string      Path to the repository root. Used together with --local allows setting the repository root (default "/")
-      --refresh                     Refresh application data when retrieving
-      --revision string             Compare live app to a particular revision
-      --revisions stringArray       Show manifests at specific revisions for the index of sources in source-indexes
-      --server-side-generate        Used with --local, this will send your manifests to the server for diffing
-      --source-indexes int64Slice   List of source indexes. Default is empty array. Indexes start at 1. (default [])
+  -N, --app-namespace string          Only render the difference in namespace
+      --exit-code                     Return non-zero exit code when there is a diff (default true)
+      --hard-refresh                  Refresh application data as well as target manifests cache
+  -h, --help                          help for diff
+      --local string                  Compare live app to a local manifests
+      --local-include stringArray     Used with --server-side-generate, specify patterns of filenames to send. Matching is based on filename and not path. (default [*.yaml,*.yml,*.json])
+      --local-repo-root string        Path to the repository root. Used together with --local allows setting the repository root (default "/")
+      --refresh                       Refresh application data when retrieving
+      --revision string               Compare live app to a particular revision
+      --revisions stringArray         Show manifests at specific revisions for source position in source-positions
+      --server-side-generate          Used with --local, this will send your manifests to the server for diffing
+      --source-positions int64Slice   List of source positions. Default is empty array. Counting start at 1. (default [])
 ```
 
 ### Options inherited from parent commands