From f399ca63a772860b09e126db1d572a89084a21a9 Mon Sep 17 00:00:00 2001
From: Jonathan West <jonwest@redhat.com>
Date: Tue, 1 Oct 2024 14:34:18 -0400
Subject: [PATCH] Add timeout, fix Rollouts E2E script and Makefile bugs

Signed-off-by: Jonathan West <jonwest@redhat.com>
---
 .github/workflows/e2e_tests.yml          |  1 +
 .github/workflows/rollouts_e2e_tests.yml |  1 +
 Makefile                                 |  4 ++--
 hack/run-rollouts-manager-e2e-tests.sh   | 20 ++++++++++++++------
 4 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml
index 3313bb3..b369d8c 100644
--- a/.github/workflows/e2e_tests.yml
+++ b/.github/workflows/e2e_tests.yml
@@ -12,6 +12,7 @@ jobs:
 
   test-e2e:
     name: Run end-to-end tests
+    timeout-minutes: 90
     runs-on: ubuntu-latest
     strategy:
       matrix:
diff --git a/.github/workflows/rollouts_e2e_tests.yml b/.github/workflows/rollouts_e2e_tests.yml
index da552a6..c71200b 100644
--- a/.github/workflows/rollouts_e2e_tests.yml
+++ b/.github/workflows/rollouts_e2e_tests.yml
@@ -13,6 +13,7 @@ jobs:
   test-e2e:
     name: Run end-to-end tests from upstream Argo Rollouts repo
     runs-on: ubuntu-latest
+    timeout-minutes: 90
     strategy:
       matrix:
         k3s-version: [ v1.28.2 ]
diff --git a/Makefile b/Makefile
index c4e5cf3..5afc515 100644
--- a/Makefile
+++ b/Makefile
@@ -136,7 +136,7 @@ start-e2e-namespace-scoped-bg: ## Start the operator, to run the e2e tests
 
 .PHONY: start-e2e-cluster-scoped-bg 
 start-e2e-cluster-scoped-bg: ## Start the operator, to run the e2e tests
-	RUN_IN_BACKGROUND=true hack/start-rollouts-manager-for-e2e-tests.sh
+	RUN_IN_BACKGROUND=true  NAMESPACE_SCOPED_ARGO_ROLLOUTS=false hack/start-rollouts-manager-for-e2e-tests.sh
 
 
 .PHONY: test-e2e-namespace-scoped
@@ -145,7 +145,7 @@ test-e2e-namespace-scoped: ## Run operator e2e tests
 
 .PHONY: test-e2e-cluster-scoped
 test-e2e-cluster-scoped: ## Run operator e2e tests
-	hack/run-rollouts-manager-e2e-tests.sh
+	NAMESPACE_SCOPED_ARGO_ROLLOUTS=false hack/run-rollouts-manager-e2e-tests.sh
 
 .PHONY: start-test-e2e-all
 start-test-e2e-all: start-e2e-namespace-scoped-bg test-e2e-namespace-scoped start-e2e-cluster-scoped-bg test-e2e-cluster-scoped
diff --git a/hack/run-rollouts-manager-e2e-tests.sh b/hack/run-rollouts-manager-e2e-tests.sh
index 29f6845..154323d 100755
--- a/hack/run-rollouts-manager-e2e-tests.sh
+++ b/hack/run-rollouts-manager-e2e-tests.sh
@@ -157,7 +157,7 @@ sanity_test_metrics_data() {
 
   fi
 
-  if [[ "`expr $DELTA_SUCCESS_RECONCILES \> 250`" == "1" ]]; then
+  if [[ "`expr $DELTA_SUCCESS_RECONCILES \> 1200`" == "1" ]]; then
     # This value is arbitrary, and should be updated if at any point it becomes inaccurate (but first audit the test/code to make sure it is not an actual product/test issue, before increasing)
 
     echo "Number of Reconcile calls that returned success $DELTA_SUCCESS_RECONCILES was greater than the expected value"
@@ -181,11 +181,19 @@ if [ -f "/tmp/e2e-operator-run.log" ]; then
   # Grep the log for unexpected errors
   # - Ignore errors that are expected to occur
 
-  UNEXPECTED_ERRORS_FOUND_TEXT=`cat /tmp/e2e-operator-run.log | grep "ERROR" | grep -v "because it is being terminated" | grep -v "the object has been modified; please apply your changes to the latest version and try again" | grep -v "unable to fetch" | grep -v "StorageError"` | grep -v "client rate limiter Wait returned an error: context canceled"
-  UNEXPECTED_ERRORS_COUNT=`echo $UNEXPECTED_ERRORS_FOUND_TEXT | grep "ERROR" | wc -l`
+  set +u # allow undefined vars
+
+  UNEXPECTED_ERRORS_FOUND_TEXT=`cat /tmp/e2e-operator-run.log | grep "ERROR" | grep -v "because it is being terminated" | grep -v "the object has been modified; please apply your changes to the latest version and try again" | grep -v "unable to fetch" | grep -v "StorageError" | grep -v "client rate limiter Wait returned an error: context canceled"`
+
+  if [ "$UNEXPECTED_ERRORS_FOUND_TEXT" != "" ]; then
   
-  if [ "$UNEXPECTED_ERRORS_COUNT" != "0" ]; then
-      echo "Unexpected errors found: $UNEXPECTED_ERRORS_FOUND_TEXT"
-      exit 1
+    UNEXPECTED_ERRORS_COUNT=`echo $UNEXPECTED_ERRORS_FOUND_TEXT | grep "ERROR" | wc -l`
+    
+    if [ "$UNEXPECTED_ERRORS_COUNT" != "0" ]; then
+        echo "Unexpected errors found: $UNEXPECTED_ERRORS_FOUND_TEXT"
+        exit 1
+    fi  
   fi
+
+
 fi