Add v3.4 smoke tests and basic HA cluster upgrade test

.github/workflows/reusable_smoke_tests.yml

@@ -160,7 +160,7 @@ jobs:
             -e MEMGRAPH_LAST_DOCKERHUB_IMAGE="$MEMGRAPH_LAST_DOCKERHUB_IMAGE" \
             -e MEMGRAPH_NEXT_DOCKERHUB_IMAGE="$MEMGRAPH_NEXT_DOCKERHUB_IMAGE" \
             "$CONTAINER_NAME" \
-            bash -c "cd /mage/smoke-release-testing && ./test.bash"
+            bash -c "cd /mage/smoke-release-testing && ./test_single_mage.bash"
 
       - name: Clean up inner Docker state
         if: always()

smoke-release-testing/experiment.bash

@@ -10,8 +10,8 @@ echo "Waiting for memgraph to initialize..."
 wait_for_memgraph $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
 echo "Memgraph is up and running!"
 
-source ./mgconsole/regex.bash
-test_regex $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
+source ./mgconsole/vector_search.bash
+test_vector_search $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
 
 # NOTE: Test what's the exit status of the script by using `echo $?`:
 #   * if it's == 0 -> all good

smoke-release-testing/k8s/run.bash

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 source "$SCRIPT_DIR/../utils.bash"
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
@@ -16,31 +16,165 @@ export PATH="$(go env GOPATH)/bin:$PATH"
 #   * It takes some time to delete all PVs, check with `kubectl get pv`.
 #   * If you want more details or helm dry run just append `--debug` of
 #   `--dry-run`.
+BOLT_SERVER="localhost:10000" # Just tmp value -> each coordinator should have a different value.
+# E.g. if kubectl port-foward is used, the configured host values should be passed as `bolt_server`.
+
+setup_coordinator() {
+  local i=$1
+  echo "ADD COORDINATOR $i WITH CONFIG {\"bolt_server\": \"$BOLT_SERVER\", \"management_server\":  \"memgraph-coordinator-$i.default.svc.cluster.local:10000\", \"coordinator_server\":  \"memgraph-coordinator-$i.default.svc.cluster.local:12000\"};" | $MEMGRAPH_CONSOLE_BINARY --port 17687
+  echo "coordinator $i DONE"
+}
+setup_replica() {
+  local i=$1
+  echo "REGISTER INSTANCE instance_$i WITH CONFIG {\"bolt_server\": \"$BOLT_SERVER\", \"management_server\": \"memgraph-data-$i.default.svc.cluster.local:10000\", \"replication_server\": \"memgraph-data-$i.default.svc.cluster.local:20000\"};" | $MEMGRAPH_CONSOLE_BINARY --port 17687
+  echo "replica $i DONE"
+}
+setup_main() {
+  local i=$1
+  echo "SET INSTANCE instance_$i TO MAIN;" | $MEMGRAPH_CONSOLE_BINARY --port 17687
+  echo "main DONE"
+}
+setup_cluster() {
+  with_kubectl_portforward memgraph-coordinator-1-0 17687:7687 'wait_for_memgraph_coordinator localhost 17687 5' -- \
+    'setup_coordinator 1' \
+    'setup_coordinator 2' \
+    'setup_coordinator 3' \
+    'setup_replica 0' \
+    'setup_main 0'
+}
+
+execute_query_against_main() {
+  query="$1"
+  with_kubectl_portforward memgraph-coordinator-1-0 17687:7687 'wait_for_memgraph_coordinator localhost 17687 5' -- \
+    "MAIN_INSTANCE=\$(echo \"SHOW INSTANCES;\" | $MEMGRAPH_CONSOLE_BINARY --port 17687 --output-format=csv | python3 $SCRIPT_DIR/../validator.py get_main_parser)" \
+    "echo \"NOTE: MAIN instance is \$MAIN_INSTANCE\"" \
+    "echo \"MG_MAIN=\$MAIN_INSTANCE\" > $SCRIPT_DIR/mg_main.out" # Couldn't get export to move the info -> used file instead.
+  source $SCRIPT_DIR/mg_main.out
+  # NOTE: Waiting for MAIN is required because sometimes all instances are up, but MAIN is not yet fully configured.
+  with_kubectl_portforward "$MG_MAIN-0" 17687:7687 'wait_for_memgraph localhost 17687 5' -- \
+    "wait_for_memgraph_main localhost 17687 10" \
+    "echo \"$query\" | $MEMGRAPH_CONSOLE_BINARY --port 17687"
+}
+
+validate_nodes_against_main() {
+  expected=$1
+  with_kubectl_portforward memgraph-coordinator-1-0 17687:7687 'wait_for_memgraph_coordinator localhost 17687 5' -- \
+    "MAIN_INSTANCE=\$(echo \"SHOW INSTANCES;\" | $MEMGRAPH_CONSOLE_BINARY --port 17687 --output-format=csv | python3 $SCRIPT_DIR/../validator.py get_main_parser)" \
+    "echo \"NOTE: MAIN instance is \$MAIN_INSTANCE\"" \
+    "echo \"MG_MAIN=\$MAIN_INSTANCE\" > $SCRIPT_DIR/mg_main.out" # Couldn't get export to move the info -> used file instead.
+  source $SCRIPT_DIR/mg_main.out
+  with_kubectl_portforward "$MG_MAIN-0" 17687:7687 'wait_for_memgraph localhost 17687 5' -- \
+    "echo \"MATCH (n) RETURN n;\" | $MEMGRAPH_CONSOLE_BINARY --port 17687 --output-format=csv | python3 $SCRIPT_DIR/../validator.py validate_number_of_results -e $expected"
+  echo "validate_nodes_against_main PASSED"
+}
 
 test_k8s_single() {
   echo "Test k8s single memgraph instance using image: $MEMGRAPH_NEXT_DOCKERHUB_IMAGE"
   kind load docker-image $MEMGRAPH_NEXT_DOCKERHUB_IMAGE -n smoke-release-testing
   MEMGRAPH_NEXT_DOCKERHUB_TAG="${MEMGRAPH_NEXT_DOCKERHUB_IMAGE##*:}"
   helm install memgraph-single-smoke memgraph/memgraph \
-    -f "$SCRIPT_DIR/values-single.yaml" --set "image.tag=$MEMGRAPH_NEXT_DOCKERHUB_TAG"
+    -f "$SCRIPT_DIR/values-single.yaml" \
+    --set "image.tag=$MEMGRAPH_NEXT_DOCKERHUB_TAG"
   kubectl wait --for=condition=Ready pod/memgraph-single-smoke-0 --timeout=120s
-  kubectl port-forward memgraph-single-smoke-0 17687:7687 &
-  PF_PID=$!
-  wait_for_memgraph localhost 17687
-  echo "CREATE ();" | $MEMGRAPH_CONSOLE_BINARY --port 17687
-  echo "MATCH (n) RETURN n;" | $MEMGRAPH_CONSOLE_BINARY --port 17687
-  kill $PF_PID
-  wait $PF_PID 2>/dev/null || true
+
+  with_kubectl_portforward memgraph-single-smoke-0 17687:7687 "wait_for_memgraph localhost 17687 5" -- \
+    "echo \"CREATE ();\" | $MEMGRAPH_CONSOLE_BINARY --port 17687" \
+    "echo \"MATCH (n) RETURN n;\" | $MEMGRAPH_CONSOLE_BINARY --port 17687"
+
   helm uninstall memgraph-single-smoke
 }
 
-test_k8s_ha() {
-  helm install myhadb memgraph/memgraph-high-availability \
+helm_install_myhadb() {
+  chart_path="$1"
+  image_tag="$2"
+  helm install myhadb $chart_path \
     --set env.MEMGRAPH_ENTERPRISE_LICENSE=$MEMGRAPH_ENTERPRISE_LICENSE,env.MEMGRAPH_ORGANIZATION_NAME=$MEMGRAPH_ORGANIZATION_NAME \
-    -f $SCRIPT_DIR/values-ha.yaml
-  # TODO(gitbuda): Setup cluster commands + routing + test query.
+    -f "$SCRIPT_DIR/values-ha.yaml" \
+    --set "image.tag=$image_tag"
+}
+
+test_k8s_help() {
+  echo "usage: test_k8s_ha LAST|NEXT [-p|--chart-path PATH]"
+  echo "                             [-s|--skip-cluster-setup] [-u|--skip-helm-uninstall] [-c|--skip-cleanup]"
+  echo "                             [-n|--expected-nodes-no]"
+  echo "                             [-h|--help]"
+  exit 1
+}
+
+cleanup_k8s_all() {
+  # NOTE: An attempt to cleanup any leftovers from kubectl port-forward...
+  kill -9 $(pgrep kubectl) || true
+  if helm status myhadb > /dev/null 2>&1; then
+    helm uninstall myhadb
+  fi
+  if helm status myhadb > /dev/null 2>&1; then
+    helm uninstall memgraph-single-smoke
+  fi
+  kubectl delete pvc --all
+}
+
+test_k8s_ha() {
+  if [ "$#" -lt 1 ]; then
+    test_k8s_help
+  fi
+  WHICH="$1"
+  WHICH_TMP="MEMGRAPH_${WHICH}_DOCKERHUB_IMAGE"
+  WHICH_IMAGE="${!WHICH_TMP}"
+  MEMGRAPH_DOCKERHUB_TAG="${WHICH_IMAGE##*:}"
+  shift
+  CHART_PATH="memgraph/memgraph-high-availability"
+  SKIP_CLUSTER_SETUP=false
+  SKIP_CLEANUP=false
+  SKIP_HELM_UNINSTALL=false
+  EXPECTED_NODES_COUNT=1
+  while [ "$#" -gt 0 ]; do
+    case $1 in
+      -p|--chart-path)          CHART_PATH="$2";           shift 2 ;;
+      -s|--skip-cluster-setup)  SKIP_CLUSTER_SETUP=true;   shift ;;
+      -u|--skip-helm-uninstall) SKIP_HELM_UNINSTALL=true;  shift ;;
+      -c|--skip-cleanup)        SKIP_CLEANUP=true;         shift ;;
+      -n|--expected-nodes-no)   EXPECTED_NODES_COUNT="$2"; shift 2 ;;
+      -h|--help)                test_k8s_help;             ;;
+      *)                        shift;                     break ;;
+    esac
+  done
+  echo "Test k8s HA memgraph cluster using image:"
+  echo "  * image: $WHICH_IMAGE"
+  echo "  * tag: $MEMGRAPH_DOCKERHUB_TAG"
+  echo "  * chart: $CHART_PATH"
+  echo "  * expected nodes number: $EXPECTED_NODES_COUNT"
+  echo "  * skip cluster setup: $SKIP_CLUSTER_SETUP"
+  echo "  * skip helm uninstall: $SKIP_HELM_UNINSTALL"
+  echo "  * skip cleanup: $SKIP_CLEANUP"
+
+  kind load docker-image $WHICH_IMAGE -n smoke-release-testing
+  helm_install_myhadb $CHART_PATH $MEMGRAPH_DOCKERHUB_TAG
+  sleep 1 # NOTE: Sometimes there is an Error from Server -> pod XYZ not found...
+  kubectl wait --for=condition=Ready pod -l role=coordinator --timeout=120s
+  kubectl wait --for=condition=Ready pod -l role=data --timeout=120s
+
+  if [ "$SKIP_CLUSTER_SETUP" = false ]; then
+    setup_cluster
+  fi
+  execute_query_against_main "SHOW VERSION;"
+  execute_query_against_main "CREATE ();"
+  validate_nodes_against_main $EXPECTED_NODES_COUNT
+  if [ "$SKIP_HELM_UNINSTALL" = false ]; then
+    helm uninstall myhadb
+  fi
+  if [ "$SKIP_CLEANUP" = false ]; then
+    kubectl delete pvc --all
+  fi
 }
 
 if [ "${BASH_SOURCE[0]}" -ef "$0" ]; then
-  test_k8s_single
+  # NOTE: Developing workflow: download+load required images and defined MEMGRAPH_NEXT_DOCKERHUB_IMAGE.
+  echo "Running $0 directly..."
+
+  # test_k8s_single
+  # test_k8s_ha LAST -c # Skip cleanup because we want to recover from existing PVCs.
+  # test_k8s_ha NEXT -s # Skip cluster setup because that should be recovered from PVCs.
+
+  # How to inject local version of the helm chart because we want to test any local fixes upfront.
+  # test_k8s_ha NEXT ~/Workspace/code/memgraph/helm-charts/charts/memgraph-high-availability
 fi

smoke-release-testing/k8s/values-ha.yaml

@@ -2,7 +2,7 @@ image:
   repository: memgraph/memgraph
   # It is a bad practice to set the image tag name to latest as it can trigger automatic upgrade of the charts
   # With some of the pullPolicy values. Please consider fixing the tag to a specific Memgraph version
-  tag: 3.2.1
+  tag: 3.4.0
   pullPolicy: IfNotPresent
 
 env:

smoke-release-testing/k8s/values-single.yaml

@@ -1,5 +1,5 @@
 image:
-  repository: memgraph/memgraph-mage
+  repository: memgraph/memgraph
   # Overrides the image tag whose default is v{{ .Chart.AppVersion }}
   # It is a bad practice to set the image tag name to latest as it can trigger automatic upgrade of the charts
   # With some of the pullPolicy values. Please consider fixing the tag to a specific Memgraph version

smoke-release-testing/mgconsole/indices.bash

@@ -11,3 +11,14 @@ test_composite_indices() {
   echo "CREATE (:Label {prop1:0, prop2: 1});" | $MEMGRAPH_CONSOLE_BINARY --host $__host --port $__port
   echo "EXPLAIN MATCH (n:Label {prop1:0, prop2: 1}) RETURN n;" | $MEMGRAPH_CONSOLE_BINARY --host $__host --port $__port | grep -E "ScanAllByLabelProperties \(n :Label \{prop1, prop2\}\)"
 }
+
+test_nested_indices() {
+  __host="$1"
+  __port="$2"
+  echo "FEATURE: Nested Indices"
+
+  echo "CREATE INDEX ON :Project(delivery.status.due_date);" | $MEMGRAPH_CONSOLE_BINARY --host $__host --port $__port
+  echo "CREATE (:Project {delivery: {status: {due_date: date('2025-06-04'), milestone: 'v3.14'}}});" | $MEMGRAPH_CONSOLE_BINARY --host $__host --port $__port
+  echo "EXPLAIN MATCH (proj:Project) WHERE proj.delivery.status.due_date = date('2025-06-04') RETURN *;" | $MEMGRAPH_CONSOLE_BINARY --host $__host --port $__port | grep -E "ScanAllByLabelProperties"
+
+}

smoke-release-testing/mgconsole/vector_search.bash

@@ -8,6 +8,8 @@ test_vector_search() {
   echo "FEATURE: Vector Search"
 
   echo "CREATE VECTOR INDEX vsi ON :Label(embedding) WITH CONFIG {\"dimension\":2, \"capacity\": 10};" | $MEMGRAPH_CONSOLE_BINARY --host $__host --port $__port
+  echo "CREATE VECTOR EDGE INDEX etvsi ON :EdgeType(embedding) WITH CONFIG {\"dimension\": 256, \"capacity\": 1000};" | $MEMGRAPH_CONSOLE_BINARY --host $__host --port $__port
+
   echo "CALL vector_search.show_index_info() YIELD * RETURN *;" | $MEMGRAPH_CONSOLE_BINARY --host $__host --port $__port
   echo "SHOW VECTOR INDEX INFO;" | $MEMGRAPH_CONSOLE_BINARY --host $__host --port $__port
 }

smoke-release-testing/test_ha_memgraph.bash

@@ -0,0 +1,12 @@
+#!/bin/bash -e
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPT_DIR/utils.bash"
+
+source $SCRIPT_DIR/k8s/run.bash
+cleanup_k8s_all
+
+test_k8s_single
+
+# Test the upgrade scenario. PVCs are preserved between the two runs.
+test_k8s_ha LAST -n 1 -c # Skip cleanup because we want to recover from existing PVCs.
+test_k8s_ha NEXT -n 2 -s # Skip cluster setup because that should be recovered from PVCs.

smoke-release-testing/test.bash → smoke-release-testing/test_single_mage.bash

@@ -2,11 +2,8 @@
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 source "$SCRIPT_DIR/utils.bash"
 
-# TODO(gitbuda): Measure the total execution time, it should be under ~10s.
-# TODO(gitbuda): Test docker compose.
-
 # NOTE: 1st arg is how to pull LAST image, 2nd arg is how to pull NEXT image.
-spinup_and_cleanup_memgraph_dockers DockerHub RC
+spinup_and_cleanup_memgraph_dockers Dockerhub RC
 echo "Waiting for memgraph to initialize..."
 wait_for_memgraph $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_LAST_DATA_BOLT_PORT
 wait_for_memgraph $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
@@ -44,11 +41,8 @@ test_edge_type_operations $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
 test_composite_indices $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
 test_monitoring $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
 test_multi_tenancy $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
+test_nested_indices $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
 test_or_expression_for_labels $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
 # NOTE: If the testing container is NOT restarted, all the auth test have to
 # come after all tests that assume there are no users.
 test_impersonate_user $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
-
-# k8s is a special case, because it requires extra setup.
-source $SCRIPT_DIR/k8s/run.bash
-test_k8s_single