From f70c232a5581597f1bf2844d5dd3c12a33f1d6fe Mon Sep 17 00:00:00 2001 From: John Pitman Date: Tue, 5 Nov 2024 13:49:05 -0500 Subject: [PATCH] updates based on PR feedback Signed-off-by: John Pitman --- .github/workflows/ci.yaml | 2 +- Makefile | 2 +- test/e2e2/README.md | 2 +- test/e2e2/basic_test.go | 73 ++++++++++++++++++-- test/e2e2/fixture/fixture.go | 4 +- test/e2e2/fixture/kubeclient.go | 50 ++++++++++++++ test/e2e2/fixture/kubeconfig.go | 15 ---- test/e2e2/fixture_test.go | 4 ++ test/e2e2/sync_test.go | 66 +++++++++++++++++- test/e2e2/test-env/start-agent-autonomous.sh | 5 +- test/e2e2/test-env/start-agent-managed.sh | 3 +- test/e2e2/test-env/start-principal.sh | 3 +- 12 files changed, 195 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6c988e0..ce5eacc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -205,7 +205,7 @@ jobs: make setup-e2e2 - name: Run the principal and agents run: | - make start-argocd-agent 2>&1 | sed -r "s/[[:cntrl:]]\[[0-9]{1,3}m//g" > /tmp/e2e-argocd-agent.log & + make start-e2e2 2>&1 | sed -r "s/[[:cntrl:]]\[[0-9]{1,3}m//g" > /tmp/e2e-argocd-agent.log & sleep 10 - name: Run the e2e tests run: | diff --git a/Makefile b/Makefile index d6c20fc..3306480 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ setup-e2e2: test/e2e2/test-env/setup-vcluster-env.sh create .PHONY: start-argocd-agent -start-argocd-agent: +start-e2e2: test/e2e2/test-env/gen-creds.sh goreman -f test/e2e2/test-env/Procfile start diff --git a/test/e2e2/README.md b/test/e2e2/README.md index 1ea7da2..036a8c1 100644 --- a/test/e2e2/README.md +++ b/test/e2e2/README.md @@ -27,7 +27,7 @@ make setup-e2e2 To run the principal and agents, execute the following command from the repository root: ```shell -make start-argocd-agent +make start-e2e2 ``` To run the tests, execute the following command from the repository root in a separate terminal instance: diff --git a/test/e2e2/basic_test.go b/test/e2e2/basic_test.go index 5235e34..f4bda8e 100644 --- a/test/e2e2/basic_test.go +++ b/test/e2e2/basic_test.go @@ -30,7 +30,7 @@ type BasicTestSuite struct { fixture.BaseSuite } -func (suite *BasicTestSuite) Test_Agent_Managed() { +func (suite *BasicTestSuite) Test_AgentManaged() { requires := suite.Require() // Create a managed application in the principal's cluster @@ -69,6 +69,37 @@ func (suite *BasicTestSuite) Test_Agent_Managed() { return err == nil }, 30*time.Second, 1*time.Second) + // Check that the .spec field of the managed-agent matches that of the + // principal + app = argoapp.Application{} + err = suite.PrincipalClient.Get(suite.Ctx, key, &app, metav1.GetOptions{}) + requires.NoError(err) + mapp := argoapp.Application{} + err = suite.ManagedAgentClient.Get(suite.Ctx, key, &mapp, metav1.GetOptions{}) + requires.NoError(err) + requires.Equal(&app.Spec, &mapp.Spec) + + // Modify the application on the principal and ensure the change is + // propagated to the managed-agent + err = suite.PrincipalClient.EnsureApplicationUpdate(suite.Ctx, key, func(app *argoapp.Application) error { + app.Spec.Info = []argoapp.Info{ + { + Name: "e2e", + Value: "test", + }, + } + return nil + }, metav1.UpdateOptions{}) + requires.NoError(err) + requires.Eventually(func() bool { + app := argoapp.Application{} + err := suite.ManagedAgentClient.Get(suite.Ctx, key, &app, metav1.GetOptions{}) + return err == nil && + len(app.Spec.Info) == 1 && + app.Spec.Info[0].Name == "e2e" && + app.Spec.Info[0].Value == "test" + }, 30*time.Second, 1*time.Second) + // Delete the app from the principal err = suite.PrincipalClient.Delete(suite.Ctx, &app, metav1.DeleteOptions{}) requires.NoError(err) @@ -81,7 +112,7 @@ func (suite *BasicTestSuite) Test_Agent_Managed() { }, 30*time.Second, 1*time.Second) } -func (suite *BasicTestSuite) Test_Agent_Autonomous() { +func (suite *BasicTestSuite) Test_AgentAutonomous() { requires := suite.Require() // Create an autonomous application on the autonomous-agent's cluster @@ -114,15 +145,47 @@ func (suite *BasicTestSuite) Test_Agent_Autonomous() { err := suite.AutonomousAgentClient.Create(suite.Ctx, &app, metav1.CreateOptions{}) requires.NoError(err) - key := types.NamespacedName{Name: app.Name, Namespace: "agent-autonomous"} + principalKey := types.NamespacedName{Name: app.Name, Namespace: "agent-autonomous"} + agentKey := fixture.ToNamespacedName(&app) // Ensure the app has been pushed to the principal requires.Eventually(func() bool { app := argoapp.Application{} - err := suite.PrincipalClient.Get(suite.Ctx, key, &app, metav1.GetOptions{}) + err := suite.PrincipalClient.Get(suite.Ctx, principalKey, &app, metav1.GetOptions{}) return err == nil }, 30*time.Second, 1*time.Second) + // Check that the .spec field of the principal matches that of the + // autonomous-agent + app = argoapp.Application{} + err = suite.AutonomousAgentClient.Get(suite.Ctx, agentKey, &app, metav1.GetOptions{}) + requires.NoError(err) + papp := argoapp.Application{} + err = suite.PrincipalClient.Get(suite.Ctx, principalKey, &papp, metav1.GetOptions{}) + requires.NoError(err) + requires.Equal(&app.Spec, &papp.Spec) + + // Modify the application on the autonomous-agent and ensure the change is + // propagated to the principal + err = suite.AutonomousAgentClient.EnsureApplicationUpdate(suite.Ctx, agentKey, func(app *argoapp.Application) error { + app.Spec.Info = []argoapp.Info{ + { + Name: "e2e", + Value: "test", + }, + } + return nil + }, metav1.UpdateOptions{}) + requires.NoError(err) + requires.Eventually(func() bool { + app := argoapp.Application{} + err := suite.PrincipalClient.Get(suite.Ctx, principalKey, &app, metav1.GetOptions{}) + return err == nil && + len(app.Spec.Info) == 1 && + app.Spec.Info[0].Name == "e2e" && + app.Spec.Info[0].Value == "test" + }, 30*time.Second, 1*time.Second) + // Delete the app from the autonomous-agent err = suite.AutonomousAgentClient.Delete(suite.Ctx, &app, metav1.DeleteOptions{}) requires.NoError(err) @@ -130,7 +193,7 @@ func (suite *BasicTestSuite) Test_Agent_Autonomous() { // Ensure the app has been deleted from the principal requires.Eventually(func() bool { app := argoapp.Application{} - err := suite.PrincipalClient.Get(suite.Ctx, key, &app, metav1.GetOptions{}) + err := suite.PrincipalClient.Get(suite.Ctx, principalKey, &app, metav1.GetOptions{}) return errors.IsNotFound(err) }, 30*time.Second, 1*time.Second) } diff --git a/test/e2e2/fixture/fixture.go b/test/e2e2/fixture/fixture.go index a7c005a..641180f 100644 --- a/test/e2e2/fixture/fixture.go +++ b/test/e2e2/fixture/fixture.go @@ -57,14 +57,14 @@ func (suite *BaseSuite) SetupSuite() { func (suite *BaseSuite) SetupTest() { err := CleanUp(suite.Ctx, suite.PrincipalClient, suite.ManagedAgentClient, suite.AutonomousAgentClient) - suite.Assert().Nil(err) + suite.Require().Nil(err) suite.T().Logf("Test begun at: %v", time.Now()) } func (suite *BaseSuite) TearDownTest() { suite.T().Logf("Test ended at: %v", time.Now()) err := CleanUp(suite.Ctx, suite.PrincipalClient, suite.ManagedAgentClient, suite.AutonomousAgentClient) - suite.Assert().Nil(err) + suite.Require().Nil(err) } func ensureDeletion(ctx context.Context, kclient KubeClient, app argoapp.Application) error { diff --git a/test/e2e2/fixture/kubeclient.go b/test/e2e2/fixture/kubeclient.go index 0b0c5da..2632d3e 100644 --- a/test/e2e2/fixture/kubeclient.go +++ b/test/e2e2/fixture/kubeclient.go @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package fixture provides a client interface similar to the one provided by +// the controller-runtime package, in order to avoid creating a dependency on +// the controller-runtime package. package fixture import ( @@ -25,6 +28,7 @@ import ( apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -37,11 +41,19 @@ import ( "k8s.io/client-go/restmapper" ) +// KubeObject represents a Kubernetes object. This allows the client interface +// to work seamlessly with any resource that implements both the metav1.Object +// and runtime.Object interfaces. This is similar to the controller-runtime's +// client.Object interface. type KubeObject interface { metav1.Object runtime.Object } +// KubeObjectList represents a Kubernetes object list. This allows the client +// interface to work seamlessly with any resource that implements both the +// metav1.ListInterface and runtime.Object interfaces. This is similar to the +// controller-runtime's client.ObjectList interface. type KubeObjectList interface { metav1.ListInterface runtime.Object @@ -108,6 +120,9 @@ func NewKubeClient(config *rest.Config) (KubeClient, error) { }, nil } +// Get returns the object with the specified key from the cluster. object must +// be a struct pointer so it can be updated with the result returned by the +// server. func (c KubeClient) Get(ctx context.Context, key types.NamespacedName, object KubeObject, options metav1.GetOptions) error { resource, err := c.resourceFor(object) if err != nil { @@ -131,6 +146,10 @@ func (c KubeClient) Get(ctx context.Context, key types.NamespacedName, object Ku return err } +// List returns a list of objects matching the criteria specified by the given +// list options from the given namespace. list must be a struct pointer so that +// the Items field in the list can be populated with the results returned by the +// server. func (c KubeClient) List(ctx context.Context, namespace string, list KubeObjectList, options metav1.ListOptions) error { resource, err := c.resourceFor(list) if err != nil { @@ -150,6 +169,8 @@ func (c KubeClient) List(ctx context.Context, namespace string, list KubeObjectL return err } +// Create creates the given object in the cluster. object must be a struct +// pointer so that it can be updated with the result returned by the server. func (c KubeClient) Create(ctx context.Context, object KubeObject, options metav1.CreateOptions) error { resource, err := c.resourceFor(object) if err != nil { @@ -183,6 +204,8 @@ func (c KubeClient) Create(ctx context.Context, object KubeObject, options metav return err } +// Update updates the given object in the cluster. object must be a struct +// pointer so that it can be updated with the result returned by the server. func (c KubeClient) Update(ctx context.Context, object KubeObject, options metav1.UpdateOptions) error { resource, err := c.resourceFor(object) if err != nil { @@ -206,6 +229,9 @@ func (c KubeClient) Update(ctx context.Context, object KubeObject, options metav return err } +// Patch patches the given object in the cluster using the JSONPatch patch type. +// object must be a struct pointer so that it can be updated with the result +// returned by the server. func (c KubeClient) Patch(ctx context.Context, object KubeObject, jsonPatch []interface{}, options metav1.PatchOptions) error { resource, err := c.resourceFor(object) if err != nil { @@ -230,6 +256,7 @@ func (c KubeClient) Patch(ctx context.Context, object KubeObject, jsonPatch []in return err } +// Delete deletes the given object from the server. func (c KubeClient) Delete(ctx context.Context, object KubeObject, options metav1.DeleteOptions) error { resource, err := c.resourceFor(object) if err != nil { @@ -269,3 +296,26 @@ func (c KubeClient) resourceFor(object runtime.Object) (schema.GroupVersionResou } return resource, nil } + +// EnsureApplicationUpdate ensures the argocd application with the given key is +// updated by retrying if there is a conflicting change. +func (c KubeClient) EnsureApplicationUpdate(ctx context.Context, key types.NamespacedName, modify func(*argoapp.Application) error, options metav1.UpdateOptions) error { + var err error + for { + var app argoapp.Application + err = c.Get(ctx, key, &app, metav1.GetOptions{}) + if err != nil { + return err + } + + err = modify(&app) + if err != nil { + return err + } + + err = c.Update(ctx, &app, options) + if !errors.IsConflict(err) { + return err + } + } +} diff --git a/test/e2e2/fixture/kubeconfig.go b/test/e2e2/fixture/kubeconfig.go index 6ec4386..60c9de8 100644 --- a/test/e2e2/fixture/kubeconfig.go +++ b/test/e2e2/fixture/kubeconfig.go @@ -39,20 +39,5 @@ func GetSystemKubeConfig(kcontext string) (*rest.Config, error) { return nil, err } - err = setRateLimitOnRestConfig(restConfig) - if err != nil { - return nil, err - } - return restConfig, nil } - -// setRateLimitOnRestConfig sets the QPS and Burst for the rest config -func setRateLimitOnRestConfig(restConfig *rest.Config) error { - if restConfig != nil { - // Prevent rate limiting of our requests - restConfig.QPS = 100 - restConfig.Burst = 250 - } - return nil -} diff --git a/test/e2e2/fixture_test.go b/test/e2e2/fixture_test.go index 54bb827..d41b7b1 100644 --- a/test/e2e2/fixture_test.go +++ b/test/e2e2/fixture_test.go @@ -34,6 +34,10 @@ import ( "k8s.io/client-go/restmapper" ) +// FixtureTestSuit is code used to experiment with and test the e2e fixture code +// itself. It doesn't test any of the project's components. In order to run, it +// requires the Argo CD CRDs (i.e. Application etc.) to be installed on the +// target cluster. It is currently commented out. type FixtureTestSuite struct { suite.Suite } diff --git a/test/e2e2/sync_test.go b/test/e2e2/sync_test.go index a8df89b..57de806 100644 --- a/test/e2e2/sync_test.go +++ b/test/e2e2/sync_test.go @@ -88,7 +88,7 @@ func (suite *SyncTestSuite) TearDownTest() { } -func (suite *SyncTestSuite) Test_Sync_Managed() { +func (suite *SyncTestSuite) Test_SyncManaged() { requires := suite.Require() // Create a managed application in the principal's cluster @@ -152,6 +152,37 @@ func (suite *SyncTestSuite) Test_Sync_Managed() { return err == nil && app.Status.Sync.Status == argoapp.SyncStatusCodeSynced }, 60*time.Second, 1*time.Second) + // Check that the .spec field of the managed-agent matches that of the + // principal + app = argoapp.Application{} + err = suite.PrincipalClient.Get(suite.Ctx, key, &app, metav1.GetOptions{}) + requires.NoError(err) + mapp := argoapp.Application{} + err = suite.ManagedAgentClient.Get(suite.Ctx, key, &mapp, metav1.GetOptions{}) + requires.NoError(err) + requires.Equal(&app.Spec, &mapp.Spec) + + // Modify the application on the principal and ensure the change is + // propagated to the managed-agent + err = suite.PrincipalClient.EnsureApplicationUpdate(suite.Ctx, key, func(app *argoapp.Application) error { + app.Spec.Info = []argoapp.Info{ + { + Name: "e2e", + Value: "test", + }, + } + return nil + }, metav1.UpdateOptions{}) + requires.NoError(err) + requires.Eventually(func() bool { + app := argoapp.Application{} + err := suite.ManagedAgentClient.Get(suite.Ctx, key, &app, metav1.GetOptions{}) + return err == nil && + len(app.Spec.Info) == 1 && + app.Spec.Info[0].Name == "e2e" && + app.Spec.Info[0].Value == "test" + }, 30*time.Second, 1*time.Second) + // Delete the app from the principal err = suite.PrincipalClient.Delete(suite.Ctx, &app, metav1.DeleteOptions{}) requires.NoError(err) @@ -164,7 +195,7 @@ func (suite *SyncTestSuite) Test_Sync_Managed() { }, 90*time.Second, 1*time.Second) } -func (suite *SyncTestSuite) Test_Sync_Autonomous() { +func (suite *SyncTestSuite) Test_SyncAutonomous() { requires := suite.Require() // Create an autonomous application on the autonomous-agent's cluster @@ -232,6 +263,37 @@ func (suite *SyncTestSuite) Test_Sync_Autonomous() { return err == nil && app.Status.Sync.Status == argoapp.SyncStatusCodeSynced }, 60*time.Second, 1*time.Second) + // Check that the .spec field of the principal matches that of the + // autonomous-agent + app = argoapp.Application{} + err = suite.AutonomousAgentClient.Get(suite.Ctx, agentKey, &app, metav1.GetOptions{}) + requires.NoError(err) + papp := argoapp.Application{} + err = suite.PrincipalClient.Get(suite.Ctx, principalKey, &papp, metav1.GetOptions{}) + requires.NoError(err) + requires.Equal(&app.Spec, &papp.Spec) + + // Modify the application on the autonomous-agent and ensure the change is + // propagated to the principal + err = suite.AutonomousAgentClient.EnsureApplicationUpdate(suite.Ctx, agentKey, func(app *argoapp.Application) error { + app.Spec.Info = []argoapp.Info{ + { + Name: "e2e", + Value: "test", + }, + } + return nil + }, metav1.UpdateOptions{}) + requires.NoError(err) + requires.Eventually(func() bool { + app := argoapp.Application{} + err := suite.PrincipalClient.Get(suite.Ctx, principalKey, &app, metav1.GetOptions{}) + return err == nil && + len(app.Spec.Info) == 1 && + app.Spec.Info[0].Name == "e2e" && + app.Spec.Info[0].Value == "test" + }, 30*time.Second, 1*time.Second) + // Delete the app from the autonomous-agent err = suite.AutonomousAgentClient.Delete(suite.Ctx, &app, metav1.DeleteOptions{}) requires.NoError(err) diff --git a/test/e2e2/test-env/start-agent-autonomous.sh b/test/e2e2/test-env/start-agent-autonomous.sh index faa01f9..c7f2aa0 100755 --- a/test/e2e2/test-env/start-agent-autonomous.sh +++ b/test/e2e2/test-env/start-agent-autonomous.sh @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -e -o pipefail +set -ex -o pipefail ARGS=$* if ! kubectl config get-contexts | tail -n +2 | awk '{ print $2 }' | grep -qE '^vcluster-agent-autonomous$'; then echo "kube context vcluster-agent-autonomous is not configured; missing setup?" >&2 exit 1 fi SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -test -f cmd/agent/main.go || (echo "Script should be run from argocd-agent's root path" >&2; exit 1) -go run ./cmd/agent/main.go --agent-mode autonomous --creds userpass:${SCRIPTPATH}/creds/creds.agent-autonomous --server-address 127.0.0.1 --server-port 8443 --insecure-tls --kubecontext vcluster-agent-autonomous --namespace argocd $ARGS +go run github.com/argoproj-labs/argocd-agent/cmd/agent --agent-mode autonomous --creds userpass:${SCRIPTPATH}/creds/creds.agent-autonomous --server-address 127.0.0.1 --server-port 8443 --insecure-tls --kubecontext vcluster-agent-autonomous --namespace argocd --log-level trace $ARGS diff --git a/test/e2e2/test-env/start-agent-managed.sh b/test/e2e2/test-env/start-agent-managed.sh index a710391..4f49491 100755 --- a/test/e2e2/test-env/start-agent-managed.sh +++ b/test/e2e2/test-env/start-agent-managed.sh @@ -20,5 +20,4 @@ if ! kubectl config get-contexts | tail -n +2 | awk '{ print $2 }' | grep -qE '^ exit 1 fi SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -test -f cmd/agent/main.go || (echo "Script should be run from argocd-agent's root path" >&2; exit 1) -go run ./cmd/agent/main.go --agent-mode managed --creds userpass:${SCRIPTPATH}/creds/creds.agent-managed --server-address 127.0.0.1 --server-port 8443 --insecure-tls --kubecontext vcluster-agent-managed --namespace agent-managed $ARGS +go run github.com/argoproj-labs/argocd-agent/cmd/agent --agent-mode managed --creds userpass:${SCRIPTPATH}/creds/creds.agent-managed --server-address 127.0.0.1 --server-port 8443 --insecure-tls --kubecontext vcluster-agent-managed --namespace agent-managed --log-level trace $ARGS diff --git a/test/e2e2/test-env/start-principal.sh b/test/e2e2/test-env/start-principal.sh index 8b0570b..9a0ab0d 100755 --- a/test/e2e2/test-env/start-principal.sh +++ b/test/e2e2/test-env/start-principal.sh @@ -20,5 +20,4 @@ if ! kubectl config get-contexts | tail -n +2 | awk '{ print $2 }' | grep -qE '^ exit 1 fi SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -test -f cmd/principal/main.go || (echo "Script should be run from argocd-agent's root path" >&2; exit 1) -go run ./cmd/principal --allowed-namespaces '*' --insecure-tls-generate --insecure-jwt-generate --kubecontext vcluster-control-plane --log-level trace --passwd ${SCRIPTPATH}/creds/users.control-plane $ARGS +go run github.com/argoproj-labs/argocd-agent/cmd/principal --allowed-namespaces '*' --insecure-tls-generate --insecure-jwt-generate --kubecontext vcluster-control-plane --log-level trace --passwd ${SCRIPTPATH}/creds/users.control-plane $ARGS