WIP - operators-installer - manual incremental upgrades

* adds `automaticIntermediateManualUpgrades` option to allow the
approver to automatically approve ever required update between currently
installed CSV and target CSV

.github/workflows/install-integration-tests-operators-installer.yaml

@@ -0,0 +1,94 @@
+---
+# these integration tests need to be per operator since they don't do clean up
+#
+# NOTE: can't use chart-testing because `ct` does not allow for a fixed release so you can't run two different tests that affect the same resources
+
+name: Install Integration Test - operators-installer
+
+on:
+  pull_request:
+    paths:
+    - .github/**
+    - _test/charts-integration-tests/operators-installer/**
+    - charts/operators-installer/**
+
+# Declare default permissions as read only.
+permissions: read-all
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  install-test:
+    runs-on: ubuntu-latest
+    env:
+      # renovate: datasource=github-releases depName=helm/helm
+      HELM_VERSION: v3.15.0
+      # renovate: datasource=github-tags depName=python/cpython
+      PYTHON_VERSION: v3.12.3
+      # renovate: datasource=github-releases depName=kubernetes-sigs/kind
+      KIND_VERSION: v0.23.0
+      # renovate: datasource=github-releases depName=operator-framework/operator-lifecycle-manager
+      OLM_VERSION: v0.28.0
+    steps:
+    - name: Checkout 🛎️
+      uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
+      with:
+        fetch-depth: 0
+
+    - name: Setup Helm 🧰
+      uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4
+      with:
+        version: ${{ env.HELM_VERSION }}
+
+    - name: Setup Python 🐍
+      uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+
+    - name: Setup kind cluster 🧰
+      uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
+      with:
+        version: ${{ env.KIND_VERSION }}
+
+    # for helm charts we are testing that require installing operators 
+    - name: Setup kind cluster - Install OLM 🧰
+      run: |
+        curl -L https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/install.sh -o install.sh
+        chmod +x install.sh
+        ./install.sh ${OLM_VERSION}
+
+    # for helm charts we are testing that require ingress
+    - name: Setup kind cluster - Install ingress controller 🧰
+      run: |
+        helm repo add haproxy-ingress https://haproxy-ingress.github.io/charts
+        helm install haproxy-ingress haproxy-ingress/haproxy-ingress \
+            --create-namespace --namespace=ingress-controller \
+            --set controller.hostNetwork=true
+        kubectl apply -f - <<EOF
+        apiVersion: networking.k8s.io/v1
+        kind: IngressClass
+        metadata:
+          name: haproxy
+          annotations:
+            ingressclass.kubernetes.io/is-default-class: 'true'
+        spec:
+          controller: haproxy-ingress.github.io/controller
+        EOF
+
+    # NOTE: can't use chart-testing because `ct` does not allow for a fixed release so you can't run two different tests that affect the same resources
+    - name: Run integration tests 🧪
+      timeout-minutes: 30
+      run: |
+        helm upgrade --install operators-installer-integration-test charts/operators-installer \
+          --namespace operators-installer-integration-test \
+          --create-namespace \
+          --wait \
+          --values charts/operators-installer/_integration-tests/test-install-operator-0-automatic-intermediate-manual-upgrades-values.yaml \
+          --debug --timeout 10m0s
+        helm upgrade --install operators-installer-integration-test charts/operators-installer \
+          --namespace operators-installer-integration-test \
+          --wait \
+          --values charts/operators-installer/_integration-tests/test-install-operator-1-automatic-intermediate-manual-upgrades-values.yaml \
+          --debug --timeout 30m0s

.github/workflows/install-test.yaml → .github/workflows/install-unit-test.yaml

@@ -1,5 +1,5 @@
 ---
-name: Install Test
+name: Install Unit Tests
 
 on:
   pull_request:
@@ -93,8 +93,8 @@ jobs:
         kubectl create namespace openshift-operators
       if: steps.changed-charts.outputs.changed == 'true'
 
-    - name: Run Chart install tests 🧪
+    - name: Run unit tests 🧪
       timeout-minutes: 30
       run: |
-        ct install --target-branch main --debug --config _test/ct-config.yaml
+        ct install --target-branch main --debug --config _test/charts-integration-tests/ct-config --charts _test/
       if: steps.changed-charts.outputs.changed == 'true'

charts/operators-installer/Chart.yaml

@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 2.4.3
+version: 3.0.0
 
 home: https://github.com/redhat-cop/helm-charts
 

charts/operators-installer/README.md

@@ -28,25 +28,48 @@ For all of the Subscription parameters see
 | operators[].sourceNamespace                  |               | Yes       | [Subscription](https://docs.openshift.com/container-platform/4.14/rest_api/operatorhub_apis/subscription-operators-coreos-com-v1alpha1.html#spec) sourceNamespace.
 | operators[].config                           |               | No        | [Subscription](https://docs.openshift.com/container-platform/4.14/rest_api/operatorhub_apis/subscription-operators-coreos-com-v1alpha1.html#spec-config) config.
 | operators[].csv                              |               | Yes       | The CSV to install.
-| operators[].installPlanApproverRetries       | `10           | No        | Number of times to try to approve the InstallPlan. This may need to be increased for unpredictable reasons about some clusters taking longer to create InstallPlans.
-| operators[].installPlanApproverActiveDeadlineSeconds | `120` | No        | Total amount of time that can be spent waiting for InstallPlan to be approved. This may need to be increased for unpredictable reasons about some clusters taking longer to create InstallPlans.
+| operators[].installPlanApproverRetries       | `10`          | No        | Number of times to try to approve the InstallPlan. This may need to be increased for unpredictable reasons about some clusters taking longer to create InstallPlans.
+| operators[].installPlanApproverActiveDeadlineSeconds | None  | No        | Total amount of time that can be spent waiting for InstallPlan to be approved. If having issues with InstallPlans never finishing to install and thus approval job getting infinity stuck, set this to some reasonable number. But, keep in mind if `automaticIntermediateManualUpgrades` is `true` then it can take a while to increment through a bunch of intermediate installs on the way to the specified `csv` if an old `csv` is already installed.
 | operators[].installPlanVerifierRetries       | `10`          | No        | Number of times to check if the InstallPlan has actually been installed. This may need to increase of an operator takes a long time to install.
-| operators[].installPlanVerifierActiveDeadlineSeconds | `120` | No        | Total amount of time that can be spent waiting for InstallPlan to finish installing. This may need to increase of an operator takes a long time to install.
-| operators[].namespace                        | `.Release.Namespace` | No | Specify the namespace to install the operator into, which allows different operators to be installed into different namespaces from the same chart. If 
+| operators[].installPlanVerifierActiveDeadlineSeconds | None  | No        | Total amount of time that can be spent waiting for InstallPlan to finish installing. This may need to increase of an operator takes a long time to install. If having issues with InstallPlans never finishing to install and thus verify job getting infinity stuck, set this to some reasonable number. But, keep in mind if `automaticIntermediateManualUpgrades` is `true` then it can take a while to increment through a bunch of intermediate installs on the way to the specified `csv` if an old `csv` is already installed.
+| operators[].namespace                        | `.Release.Namespace` | No | Specify the namespace to install the operator into, which allows different operators to be installed into different namespaces from the same chart.
+| operators[].automaticIntermediateManualUpgrades | `false`     |           | If `true` and `installPlanApproval` is `Manual` and there are required intermediate upgrades between the currently installed `csv` and the specified `csv`, automatically inclemently install those intermediate versions until reaching the specified `csv`. If `false` and there are required intermediate upgrades then chart will fail.
+| operators[].automaticIntermediateManualUpgradesIncrementalInstallBackoffLimit   | `10` | No | When `automaticIntermediateManualUpgrades` is `true`, the number of retries for installing and then then verifying each incremental install 
+| operators[].automaticIntermediateManualUpgradesIncrementalInstallDelayIncrement | `5`  | No | When `automaticIntermediateManualUpgrades` is `true`, the delay increment to scale by attempts to wait between each retry of installing or  verifying each incremental install 
 | operatorGroups                               | `[]`          | No        | Optional list of configuration for OperatorGroups. If this is not supplied then it is assumed OperatorGroups are already in place in the selected `operators[].namespace`s.
 | operatorGroups[].name                        | `.Release.Namespace` | No | Name of the OperatorGroup & Namespace the OperatorGroup will be placed in.
 | operatorGroups[].createNamespace             | `false`       | No        | If `true` create the Namespace of the same name of the OperatorGroup. If `false` assumed the Namespace is already in place.
 | operatorGroups[].targetOwnNamespace          | `false`       | No        | If `true` add the OperatorGroup's Namespace as a `targetNamespaces`. If `true` then OperatorGroup will only work for Operators using `OwnNamespace` or `MultiNamespace` `installModes`. If blank and no `otherTargetNamespaces` specified then OperatorGroup will be configured to allow for operators using `installModes` `AllNamespaces`.
 | operatorGroups[].otherTargetNamespaces       | `[]`          | No        | List of additional Namespaces to target. If specified OperatorGroup will only work for operators using `SingleNamespace` or `MultiNamespace` `installModes` depending on value of `targetOwnNamespace`.
 | installPlanApproverAndVerifyJobsImage        | `registry.redhat.io/openshift4/ose-cli:v4.10` | Yes | Image to use for the InstallPlan Approver and Verify Jobs 
 | approveManualInstallPlanViaHook              | `true`        | No        | `true` to create (and clean up) manual InstallPlan approval resources as part of post-install,post-upgrade helm hook<br>`false` to create  manual InstallPlan approval resources as part of normal install<br><br>The hook method is nice to not have lingering resources needed for the manual InstallPlan approval but has the downside that no CustomResources using CustomResourceDefinitions installed by the operator can be used in the same chart because the operator InstallPlan wont be approved, and therefor the operator wont be installed, until the post-install,post-upgrade phase which means you will never get to that phase because your CustomResources wont be able to apply because the Operator isn't installed.<br><br>This is is ultimately a trade off between cleaning up these resources or being able to install and configure the operator in the same helm chart that has a dependency on this helm chart.
+| installRequiredPythonLibraries               | `true`        | No        | If `true`, install the required Python libraries (openshift-client, semver==2.13.0) dynamically from the given `pythonIndexURL` and `pythonExtraIndexURL` into the `installPlanApproverAndVerifyJobsImage` at run time
+| pythonIndexURL                               | https://pypi.org/simple/ | No | If `installRequiredPythonLibraries` is `true` then use this python index to pull required libraries
+| pythonExtraIndexURL                          | https://pypi.org/simple/ | No | If `installRequiredPythonLibraries` is `true` then use this python extra index to pull required library dependencies
 | commonLabels                                 | `{}`          | No        | Common labels to add to all chart created resources. Implements the same idea from Kustomize for this chart.
 | global.commonLabels                          | `{}`          | No        | Common labels coming from global values to add to all chart created resources. Implements the same idea from Kustomize for this chart.
 
 ## Warnings
 
-### Can not install / upgrade different operators in same namespace independently
+### Disconnected Use
+If wanting use this chart in a disconnected environment you need to either:
+
+#### Option 1: local python index
+Set the `pythonIndexURL` and `pythonExtraIndexURL` values to a local disconnected python index that minimally includes (and their dependencies):
+* openshift-client
+* semver==2.13.0
+
+#### Option 2: custom `installPlanApproverAndVerifyJobsImage` with required dependencies
+Build a custom container image with:
+* binary - `oc`
+* python lib - `openshift-client`
+* python lib - `semver==2.13.0`
 
+Suggestion is to build such an image on top of the latest `registry.redhat.io/openshift4/ose-cli` image
+
+Then provide that custom image to `installPlanApproverAndVerifyJobsImage` and set `installRequiredPythonLibraries` to false.
+
+### Can not install / upgrade different operators in same namespace independently
 As documented in [How can Operators be updated independently from each other?](https://access.redhat.com/solutions/6389681) when more then one operator install or update is pending in the same namespace the Operator Lifecycle Manager (OLM) will combine those installs/updates into a single InstallPlan and there is no way to separate them. Therefor if you use this helm chart in namespace ZZZ to install operator A at v1.0 and it has a pending update to v1.1 and then update the configuration to also install operator B at v42.0 in namespace ZZZ the ZZZ v42.0 InstallPlan and the A v1.1 InstallPlan will get merged (by OLM) and this helm chart will then approve that InstallPlan as it will match on the ZZZ v42.0 pending install, which will incidentally install the A v1.1 update.
 
 There is no way for this or any helm chart, automation, or even click ops to prevent this, as documented in [How can Operators be updated independently from each other?](https://access.redhat.com/solutions/6389681) this is currently considered "a feature of OLM".

charts/operators-installer/_integration-tests/test-install-operator-0-automatic-intermediate-manual-upgrades-values.yaml

@@ -0,0 +1,24 @@
+approveManualInstallPlanViaHook: true
+
+#installPlanApproverAndVerifyJobsImage: quay.io/openshift/origin-cli:4.15
+
+operatorGroups:
+- name: argocd-operator
+  createNamespace: true
+  targetOwnNamespace: true
+  otherTargetNamespaces:
+
+operators:
+- name: argocd-operator
+  channel: alpha
+  csv: argocd-operator.v0.6.0
+  installPlanApproval: Manual
+  source: operatorhubio-catalog
+  sourceNamespace: olm
+  namespace: argocd-operator
+  installPlanVerifierActiveDeadlineSeconds: 1200
+  automaticIntermediateManualUpgrades: true
+  config:
+    env:
+    - name: DISABLE_DEFAULT_ARGOCD_INSTANCE
+      value: "true"

charts/operators-installer/_integration-tests/test-install-operator-1-automatic-intermediate-manual-upgrades-values.yaml

@@ -0,0 +1,24 @@
+approveManualInstallPlanViaHook: true
+
+#installPlanApproverAndVerifyJobsImage: quay.io/openshift/origin-cli:4.15
+
+operatorGroups:
+- name: argocd-operator
+  createNamespace: true
+  targetOwnNamespace: true
+  otherTargetNamespaces:
+
+operators:
+- name: argocd-operator
+  channel: alpha
+  csv: argocd-operator.v0.10.1
+  installPlanApproval: Manual
+  source: operatorhubio-catalog
+  sourceNamespace: olm
+  namespace: argocd-operator
+  installPlanVerifierActiveDeadlineSeconds: 1200
+  automaticIntermediateManualUpgrades: true
+  config:
+    env:
+    - name: DISABLE_DEFAULT_ARGOCD_INSTANCE
+      value: "true"

charts/operators-installer/_scripts/installplan-approver.py

@@ -4,63 +4,59 @@
 import sys
 import installplan_utils
 
-namespace_name = os.environ["NAMESPACE"]
-subscription_name = os.environ["SUBSCRIPTION"]
-subscription_csv = os.environ["SUBSCRIPTION_CSV"]
+NAMESPACE_NAME = os.getenv("NAMESPACE") or installplan_utils.error_and_exit(
+    "env is missing expected value: NAMESPACE", 2
+)
+SUBSCRIPTION_NAME = os.getenv("SUBSCRIPTION") or installplan_utils.error_and_exit(
+    "env is missing expected value: SUBSCRIPTION", 2
+)
+CSV = os.getenv("CSV") or installplan_utils.error_and_exit(
+    "env is missing expected value: CSV", 2
+)
 
 print()
-print("******************************")
-print("* START InstallPlan Approver *")
-print("******************************")
+print("********************************************************************")
+print("* START InstallPlan approver")
+print(f"*\t- NAMESPACE_NAME: {NAMESPACE_NAME}")
+print(f"*\t- SUBSCRIPTION_NAME: {SUBSCRIPTION_NAME}")
+print(f"*\t- CSV: {CSV}")
+print("********************************************************************")
 
 # find the subscription uid
 print()
-print(f"Get Subscription ({subscription_name}) UID")
-subscription_uid = installplan_utils.get_subscription_uid(subscription_name)
-print(f"Subscription ({subscription_name}) UID: {subscription_uid}")
+print(f"Get Subscription ({SUBSCRIPTION_NAME}) UID")
+subscription_uid = installplan_utils.get_subscription_uid(SUBSCRIPTION_NAME)
+print(f"\t- Subscription ({SUBSCRIPTION_NAME}) UID: {subscription_uid}")
 
 # if found subscription uid find InstallPlan for given CSV with owner of the given subscription
 # else error
 if subscription_uid:
     # find the InstallPlan that has expected owner subscription id and expected target CSV name
     # NOTE: if more then one InstallPlan matches, choose the first one
     print(
-        f"Get InstallPlan for CSV ({subscription_csv}) with Subscription (${subscription_uid}) owner"
+        f"Find InstallPlan in Namespace ({NAMESPACE_NAME}) for CSV ({CSV}) with Subscription (${subscription_uid}) owner"
     )
-    target_install_plan = installplan_utils.get_installplan(
-        namespace_name, subscription_csv, subscription_uid
+    target_installplan = installplan_utils.get_installplan(
+        NAMESPACE_NAME, CSV, subscription_uid
     )
-    # if found target InstallPlan, approve it
+
+    # if found target InstallPlan, approve it, and success exit
     # else fail
-    if target_install_plan:
-        target_install_plan_name = target_install_plan.model.metadata.name
-        print(
-            f"InstallPlan for CSV ({subscription_csv}) with Subscription ({subscription_name}) ({subscription_uid}): {target_install_plan.model.metadata.name}"
-        )
-        print()
-        print(
-            f"Current InstallPlan ({target_install_plan_name}) approval state: {target_install_plan.model.spec.approved}"
-        )
-        # if already approved, just notify
-        # else approve
-        if target_install_plan.model.spec.approved:
-            print(f"InstallPlan ({target_install_plan_name}) is already approved")
-        else:
-            print(f"Approving InstallPlan ({target_install_plan_name})")
-            target_install_plan.model.spec.approved = True
-            target_install_plan.apply()
-            print(f"Approved InstallPlan ({target_install_plan_name})")
+    if target_installplan:
+        print(f"\t- Found InstallPlan: {target_installplan.model.metadata.name}")
+        installplan_utils.approve_installplan(target_installplan)
         sys.exit(0)
     else:
         print()
         print(
-            f"Could not find InstallPlan for CSV ${subscription_csv}) with Subscription ({subscription_name}) ({subscription_uid}) owner."
-            + "\nThis can happen if InstallPlan isn't created yet. Try again."
+            f"ERROR: Could not find next InstallPlan to reach CSV ${CSV}) with Subscription ({SUBSCRIPTION_NAME}) ({subscription_uid}) owner."
+            + "\nThis can happen if InstallPlan isn't created yet or no valid upgrade path between current CSV and target CSV."
+            + "\nTry again."
         )
         sys.exit(1)
 else:
     print()
     print(
-        "Failed to get Subscription ({subscription_name}) UID. This really shouldn't happen."
+        f"ERROR: Failed to get Subscription ({SUBSCRIPTION_NAME}) UID. This really shouldn't happen."
     )
     sys.exit(1)