_shared-run-test.yaml

#name: Run assertoor tests on kurtosis with kubernetes backend

on:
  workflow_call:
    inputs:
      id:
        type: string
        description: "The test identifier."
        required: true
      name:
        type: string
        description: "The test name."
        required: true
      pairs:
        type: string
        description: "The client pairs to use (format: consensus1-execution1,consensus2-execution2,...)."
        required: true
      worker:
        type: string
        description: "The worker to use (default: ubuntu-latest)."
        default: "ubuntu-latest"
      backend:
        type: string
        description: "The backend type to use (docker, kubernetes)."
        default: "docker"
      kubeCluster:
        type: string
        description: "The name of the cluster to run the test on."
        required: true
      kubeStorageClass:
        type: string
        description: "The kubernetes storage class to run the kurtosis engine with."
        required: true
      clients:
        type: string
        description: "The clients config to use (default: clients/latest.yaml)."
        default: "clients/latest.yaml"
      kurtosis:
        type: string
        description: "The kurtosis network config to use (default: kurtosis-config/default.yaml)."
        default: "kurtosis-config/default.yaml"
      kurtosis_version:
        type: string
        description: "The latest version of the kurtosis cli (used as caching key)."
        default: ""
      kurtosis_branch:
        type: string
        description: "The branch name for the kurtosis ethereum package to use (default: )."
        default: ""
      assertoor_tests:
        type: string
        description: "The list of assertoor test files to run."
        required: true
      send_notification:
        type: string
        description: "Send discord notification on test failure (default: false)."
        default: "false"
      use_chatgpt:
        type: string
        description: "Use ChatGPT to summarize error (default: false)."
        default: "false"
    secrets:
      KUBECONFIG:
        description: 'Kubernetes config'
        required: true
      DISCORD_HOOK:
        description: 'Discord hook'
        required: false
      CHATGPT_KEY:
        description: 'ChatGPT Key'
        required: false

jobs:
  run_test:
    name: "Run ${{ inputs.id }}"
    runs-on: ${{ fromJson(inputs.worker) }}
    timeout-minutes: 1440
    steps:
    - name: Checkout Repository
      uses: actions/checkout@v4
    - name: Cache Docker images.
      uses: ScribeMD/docker-cache@0.5.0
      with:
        key: kurtosis-docker-${{ runner.os }}-${{ inputs.kurtosis_version }}
        read-only: true
    - name: "Install shell dependencies"
      shell: bash
      run: |
        if [ -z "$(which yq)" ]; then
          sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq
        fi

        if [ -z "$(which envsubst)" ]; then
          sudo apt-get update
          sudo apt-get install gettext-base
        fi

        if [ -z "$(which gh)" ]; then
          sudo mkdir -p -m 755 /etc/apt/keyrings
          wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
          sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
          echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
          sudo apt update
          sudo apt install gh -y
        fi
    - name: "Generate kurtosis config with client pairs: ${{ inputs.pairs }}"
      shell: bash
      id: clients
      run: |
        clients_file="${{ inputs.clients }}"
        kurtosis_file="${{ inputs.kurtosis }}"
        pairs="${{ inputs.pairs }}"

        mkdir -p ./temp
        touch ./temp/participants.yaml

        echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT

        if [ ! -z "$pairs" ]; then
          resolve_image() {
            image="$1"

            if [ ! -z "$(echo "$1" | grep -E '^.*::.*$')" ]; then
              repository=$(echo "$1" | sed 's/^\(.*\)::\(.*\)$/\1/')
              tag_pattern=$(echo "$1" | sed 's/^\(.*\)::\(.*\)$/\2/')
              while read tag; do
                if [ ! -z "$(echo "$tag" | grep -E $tag_pattern)" ]; then
                  image="${repository}:${tag}"
                  break
                fi
              done <<< $(curl --silent "https://hub.docker.com/v2/repositories/${repository}/tags?page_size=1000" | jq -r ".results[].name")
            fi

            echo "$image"
          }

          echo "participants:" >> ./temp/participants.yaml
          touch ./temp/clients_summary.txt

          client_idx=0
          for pair in $(echo $pairs | tr "," "\n")
          do
            client_idx=$(expr $client_idx + 1)
            if [ "$pair" == "random" ]; then
              cl_type="random"
              el_type="random"
            else
              cl_type=$(echo "$pair" | sed 's/^\(.*\)-\(.*\)$/\1/')
              el_type=$(echo "$pair" | sed 's/^\(.*\)-\(.*\)$/\2/')
            fi
            
            if [[ "$cl_type" == *"/"* ]]; then
              vc_type=$(echo "$cl_type" | sed 's/^\(.*\)\/\(.*\)$/\2/')
              cl_type=$(echo "$cl_type" | sed 's/^\(.*\)\/\(.*\)$/\1/')
            else
              vc_type=""
            fi

            if [ "$cl_type" == "random" ]; then
              cl_type=$(cat $clients_file | yq -r ".consensus | keys" | shuf | tr -d ' -' | head -n 1)
            fi
            if [ "$vc_type" == "random" ]; then
              vc_type=$(cat $clients_file | yq -r ".consensus | keys" | shuf | tr -d ' -' | head -n 1)
            fi
            if [ "$el_type" == "random" ]; then
              el_type=$(cat $clients_file | yq -r ".execution | keys" | shuf | tr -d ' -' | head -n 1)
            fi

            cl_image="$(cat $clients_file | yq ".consensus.${cl_type}.image")"
            if [ "$cl_image" == "null" ]; then
              echo "Unknown CL client type: $cl_type"
              exit 1
            fi
            cl_image=$(resolve_image "$cl_image")

            if [ -z "$vc_type" ]; then
              vc_image=""
            else
              vc_image="$(cat $clients_file | yq ".consensus.${vc_type}.vc_image // .consensus.${vc_type}.image")"
              if [ "$vc_image" == "null" ]; then
                echo "Unknown VC client type: $vc_type"
                exit 1
              fi
              vc_image=$(resolve_image "$vc_image")
            fi

            el_image="$(cat $clients_file | yq ".execution.${el_type}.image")"
            if [ "$el_image" == "null" ]; then
              echo "Unknown EL client type: $el_type"
              exit 1
            fi
            el_image=$(resolve_image "$el_image")

            echo "  - el_type: $el_type" >> ./temp/participants.yaml
            echo "    el_image: $el_image" >> ./temp/participants.yaml
            el_extra_params="$(cat $clients_file | yq ".execution.${el_type}.params")"
            if [ "$el_extra_params" != "null" ]; then
              echo "    el_extra_params: $el_extra_params" >> ./temp/participants.yaml
            fi

            echo "    cl_type: $cl_type" >> ./temp/participants.yaml
            echo "    cl_image: $cl_image" >> ./temp/participants.yaml
            cl_extra_params="$(cat $clients_file | yq ".consensus.${cl_type}.params")"
            if [ "$cl_extra_params" != "null" ]; then
              echo "    cl_extra_params: $cl_extra_params" >> ./temp/participants.yaml
            fi

            if [ ! -z "$vc_type" ]; then
              echo "    vc_type: $vc_type" >> ./temp/participants.yaml
              echo "    vc_image: $vc_image" >> ./temp/participants.yaml
              vc_extra_params="$(cat $clients_file | yq ".consensus.${cl_type}.vc_params")"
              if [ "$vc_extra_params" != "null" ]; then
                echo "    vc_extra_params: $vc_extra_params" >> ./temp/participants.yaml
              fi
            fi

            echo "    count: 1" >> ./temp/participants.yaml

            echo "Client $client_idx"
            echo "CL: $cl_type  ($cl_image)"
            if [ ! -z "$vc_type" ]; then
              echo "VC: $vc_type  ($vc_image)"
            fi
            echo "EL: $el_type  ($el_image)"

            participant="${el_type}-${cl_type}"
            if [ ! -z "$vc_type" ]; then
              participant="${participant}-${vc_type}"
            fi
            echo "pair $client_idx: $participant" >> ./temp/clients_summary.txt
          done
        fi

        export PUBURL="https://raw.githubusercontent.com/${{ github.repository }}/${{ github.sha }}"

        cp ./temp/participants.yaml ./temp/test-network.yaml
        cat $kurtosis_file | envsubst >> ./temp/test-network.yaml

        kurtosis_cfg="$(cat ./temp/test-network.yaml | yq '.assertoor_params.tests=[]')"
        tests_json=$(
        cat <<"EOF"
        ${{ inputs.assertoor_tests }}
        EOF
        )
        test_index=0
        while read assertoorTest; do
          kurtosis_cfg="$(echo "$kurtosis_cfg" | yq ".assertoor_params.tests[$test_index]=\"$PUBURL/$assertoorTest\"")"
          test_index=$(expr $test_index + 1)
        done <<< $(echo "$tests_json" | jq -r '.[]')
        echo "$kurtosis_cfg" > ./temp/test-network.yaml

        echo ""
        cat ./temp/test-network.yaml

    - name: Run kurtosis testnet
      id: testnet
      uses: ethpandaops/kurtosis-assertoor-github-action@v1
      with:
        kurtosis_extra_args: "--non-blocking-tasks --verbosity DETAILED"
        kurtosis_backend: ${{ inputs.backend }}
        kubernetes_config: ${{ secrets.KUBECONFIG }}
        kubernetes_cluster: ${{ inputs.kubeCluster }}
        kubernetes_storage_class: ${{ inputs.kubeStorageClass }}
        ethereum_package_branch: ${{ inputs.kurtosis_branch }}
        ethereum_package_args: "./temp/test-network.yaml"
        enclave_name: "assertoor-${{ github.run_id }}-${{ inputs.id }}"
        await_assertoor_tests: "false"
        enclave_dump: "false"
    
    - name: Await assertoor test result
      id: test_result
      uses: ethpandaops/assertoor-github-action@v1
      if: ${{ fromJson(steps.testnet.outputs.services).assertoor.http.url != '' }}
      with:
        kurtosis_enclave_name: "assertoor-${{ github.run_id }}-${{ inputs.id }}"
        assertoor_api_url: ${{ fromJson(steps.testnet.outputs.services).assertoor.http.url }}

    - name: Generate enclave dump
      if: ${{ steps.test_result.outputs.result != '' }}
      id: enclave_dump
      shell: bash
      run: |
        enclave_name="assertoor-${{ github.run_id }}-${{ inputs.id }}"
        tempdir="${{ runner.temp }}/$enclave_name"
        mkdir -p $tempdir/dump
        cp ./temp/test-network.yaml $tempdir/dump/kurtosis-params.yaml
        cd $tempdir/dump
        echo "dump_dir=$tempdir/dump" >> $GITHUB_OUTPUT

        kurtosis enclave dump $enclave_name

    - name: Generate failure summary via ChatGPT (on failure)
      if: ${{ steps.test_result.outputs.result == 'failure' && inputs.use_chatgpt != 'false' }}
      id: chatgpt
      shell: bash
      run: |
        test_status=$(
        cat <<"EOF"
        ${{ steps.test_result.outputs.test_overview }}
        EOF
        )
        failed_test_status=$(
        cat <<"EOF"
        ${{ steps.test_result.outputs.failed_test_details }}
        EOF
        )

        chatgpt_config=$(
        cat <<"EOF"
        ${{ inputs.use_chatgpt }}
        EOF
        )

        chatgpt_url="https://api.openai.com/v1/chat/completions"
        chatgpt_model="gpt-4-turbo-preview"
        chatgpt_extra_cfg="{}"

        if [ "$chatgpt_config" != "true" ]; then
          chatgpt_url=$(echo "$chatgpt_config" | jq -r ".url // \"$chatgpt_url\"")
          chatgpt_model=$(echo "$chatgpt_config" | jq -r ".model // \"$chatgpt_model\"")
          chatgpt_extra_cfg=$(echo "$chatgpt_config" | jq -c ".extra_cfg // {}")
        fi

        touch ./temp/chatgpt_query.txt
        echo "You will receive the test status & the last 200 lines of logs from the ethereum testnet testing tool assertoor." >> ./temp/chatgpt_query.txt
        echo "The status & logs you get are from failed tests. Please find the error, which clients caused it and try summarizing what went wrong for a first technical insight." >> ./temp/chatgpt_query.txt
        echo "Assume that assertoor and the test are working fine, so it is probably one or multiple clients causing problems." >> ./temp/chatgpt_query.txt
        echo "Your summary should not exceed 500 characters. Use easy language with technical details like client names. Avoid listing block numbers or timestamps." >> ./temp/chatgpt_query.txt
        echo "" >> ./temp/chatgpt_query.txt

        echo "Client Pairs: (Format: <execution layer client>-<consensus layer client>[-<validator client>], validator client may be left out)" >> ./temp/chatgpt_query.txt
        cat ./temp/clients_summary.txt >> ./temp/chatgpt_query.txt
        echo "" >> ./temp/chatgpt_query.txt

        echo "$test_status" >> ./temp/chatgpt_query.txt
        echo "" >> ./temp/chatgpt_query.txt
        echo "Failed Test Task Status:" >> ./temp/chatgpt_query.txt
        echo "$failed_test_status" >> ./temp/chatgpt_query.txt
        echo "" >> ./temp/chatgpt_query.txt

        tempdir="${{ steps.enclave_dump.outputs.dump_dir }}"
        tail -n 200 $tempdir/assertoor-*/assertoor--*/output.log >> ./temp/chatgpt_query.txt

        query_text="$(cat ./temp/chatgpt_query.txt)"
        cp ./temp/chatgpt_query.txt $tempdir/chatgpt_query.txt

        export QUERY_TEXT=$(echo "$query_text" | jq -R -s '.')
        export CHATGPT_MODEL=$chatgpt_model
        query_json=$(
        cat <<"EOF" | envsubst | jq -c
        {
          "model": "$CHATGPT_MODEL",
          "messages": [
            {
              "role": "user",
              "content": $QUERY_TEXT
            }
          ]
        }
        EOF
        )

        query_json=$(echo "$query_json" | jq -c ".+=${chatgpt_extra_cfg}")

        result=$(curl $chatgpt_url \
          -H "Content-Type: application/json" \
          -H "Authorization: Bearer $CHATGPT_KEY" \
          -d "$query_json"
        )

        echo "ChatGPT Summary:"
        summary="$(echo $result | jq -r ".choices[0].message.content")"
        if [ "$summary" == "null" ]; then
          echo "ChatGPT failed to generate a summary"
          echo $result | jq
        else
          echo "$summary" > ./temp/chatgpt_summary.txt
        fi

        echo "summary<<EOF" >> $GITHUB_OUTPUT
        echo $result | jq -r ".choices[0].message.content" >> $GITHUB_OUTPUT
        echo "EOF" >> $GITHUB_OUTPUT

      env:
        CHATGPT_KEY: ${{ secrets.CHATGPT_KEY }}

    - name: Upload dump artifact
      if: ${{ steps.enclave_dump.outputs.dump_dir != '' }}
      uses: actions/upload-artifact@v4
      with:
        name: "enclave-dump-assertoor-${{ github.run_id }}-${{ inputs.id }}"
        path: "${{ steps.enclave_dump.outputs.dump_dir }}"

    - name: Generate github summary (on failure)
      if: ${{ (failure() || steps.test_result.outputs.result == 'failure') }}
      shell: bash
      run: |
        echo "# Test Failed: ${{ inputs.id }}" >> $GITHUB_STEP_SUMMARY
        echo "" >> $GITHUB_STEP_SUMMARY
        echo "## Test Details" >> $GITHUB_STEP_SUMMARY
        echo "- **Test ID:** ${{ inputs.id }}" >> $GITHUB_STEP_SUMMARY
        echo "- **Name:** ${{ inputs.name }}" >> $GITHUB_STEP_SUMMARY
        echo "- **Status:** ${{ steps.test_result.outputs.result }}" >> $GITHUB_STEP_SUMMARY
        
        start_time="${{ steps.clients.outputs.start_time }}"
        end_time="$(date +%s)"
        runtime="$(expr $end_time - $start_time)"
        echo "- **Run Time:** $(printf '%02dh:%02dm:%02ds\n' $((runtime/3600)) $((runtime%3600/60)) $((runtime%60)))" >> $GITHUB_STEP_SUMMARY
        
        echo "" >> $GITHUB_STEP_SUMMARY
        echo "## Client Configuration" >> $GITHUB_STEP_SUMMARY
        cat ./temp/clients_summary.txt >> $GITHUB_STEP_SUMMARY
        echo "" >> $GITHUB_STEP_SUMMARY
        
        echo "## Failed Test" >> $GITHUB_STEP_SUMMARY
        if [ -z "${{ steps.test_result.outputs.result }}" ]; then
          echo "kurtosis testnet failed during startup" >> $GITHUB_STEP_SUMMARY
        else
          test_status=$(
          cat <<"EOF"
        ${{ steps.test_result.outputs.test_overview }}
        EOF
          )
          echo '```' >> $GITHUB_STEP_SUMMARY
          echo "$test_status" | grep "failure" >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
        fi

        chatgpt_summary=$(
        cat <<"EOF"
        ${{ steps.chatgpt.outputs.summary }}
        EOF
        )
        if ! [ -z "$chatgpt_summary" ]; then
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "## Summary (ChatGPT)" >> $GITHUB_STEP_SUMMARY
          echo "$chatgpt_summary" >> $GITHUB_STEP_SUMMARY
        fi

        echo "" >> $GITHUB_STEP_SUMMARY
        echo "See job logs and artifacts for more details about this failure." >> $GITHUB_STEP_SUMMARY

    - name: Send discord notification (on failure)
      if: ${{ (failure() || steps.test_result.outputs.result == 'failure') && inputs.send_notification == 'true' }}
      shell: bash
      run: |
        if [ -z "$DISCORD_HOOK" ]; then
          echo "discord hook missing"
          exit 1
        fi

        touch ./temp/notification_text.txt
        echo "**Test ID:** ${{ inputs.id }}" >> ./temp/notification_text.txt
        echo "**Name:** ${{ inputs.name }}" >> ./temp/notification_text.txt
        echo "**Status:** ${{ steps.test_result.outputs.result }}" >> ./temp/notification_text.txt

        start_time="${{ steps.clients.outputs.start_time }}"
        end_time="$(date +%s)"
        runtime="$(expr $end_time - $start_time)"
        echo "**Run Time:** $(printf '%02dh:%02dm:%02ds\n' $((runtime/3600)) $((runtime%3600/60)) $((runtime%60)))" >> ./temp/notification_text.txt

        echo "**Clients:**" >> ./temp/notification_text.txt
        cat ./temp/clients_summary.txt >> ./temp/notification_text.txt
        echo "" >> ./temp/notification_text.txt
        
        echo "**Failed Test:**" >> ./temp/notification_text.txt
        if [ -z "${{ steps.test_result.outputs.result }}" ]; then
          echo "kurtosis testnet failed during startup" >> ./temp/notification_text.txt
        else
        test_status=$(
        cat <<"EOF"
        ${{ steps.test_result.outputs.test_overview }}
        EOF
        )
          echo '```ansi'  >> ./temp/notification_text.txt
          echo "$test_status" | grep "failure" >> ./temp/notification_text.txt
          echo '```'  >> ./temp/notification_text.txt
        fi

        chatgpt_summary=$(
        cat <<"EOF"
        ${{ steps.chatgpt.outputs.summary }}
        EOF
        )
        if ! [ -z "$chatgpt_summary" ]; then
          echo "**Summary: (ChatGPT)**" >> ./temp/notification_text.txt
          echo "$chatgpt_summary" >> ./temp/notification_text.txt
        fi

        echo "Resolve job_id..."
        jobs=$(gh api "repos/${{ github.repository }}/actions/runs/${{ github.run_id}}/attempts/${{ github.run_attempt }}/jobs")
        job_id=$(echo $jobs | jq -r '.jobs[] | select(.runner_name=="${{ runner.name }}") | .id')
        job_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id}}/job/${job_id}"

        echo "Generate discord message..."
        msgjson="{}"
        msgjson=$(echo "$msgjson" | jq -c ".title|=$(echo -n "Assertoor Test Failed: ${{ inputs.id }}" | jq -R -s '.')")
        msgjson=$(echo "$msgjson" | jq -c ".color|=15083560")
        msgjson=$(echo "$msgjson" | jq -c ".url|=$(echo -n "$job_url" | jq -R -s '.')")
        msgjson=$(echo "$msgjson" | jq -c ".description|=$(cat ./temp/notification_text.txt | jq -R -s '.')")

        hookjson="{}"
        hookjson=$(echo "$hookjson" | jq -c ".username|=$(echo -n "Assertoor Test Notification" | jq -R -s '.')")
        hookjson=$(echo "$hookjson" | jq -c ".avatar_url|=$(echo -n "https://raw.githubusercontent.com/ethpandaops/assertoor/master/.github/resources/assertoor.png" | jq -R -s '.')")
        hookjson=$(echo "$hookjson" | jq -c ".embeds|=[${msgjson}]")
        echo $hookjson | jq

        echo "Send discord message..."
        curl -X POST -H 'Content-Type: application/json' -d "$hookjson" $DISCORD_HOOK
      env:
        DISCORD_HOOK: ${{ secrets.DISCORD_HOOK }}
        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

    - name: Return test result
      shell: bash
      run: |
        test_result="${{ steps.test_result.outputs.result }}"
        test_status=$(
        cat <<"EOF"
        ${{ steps.test_result.outputs.test_overview }}
        EOF
        )
        failed_test_status=$(
        cat <<"EOF"
        ${{ steps.test_result.outputs.failed_test_details }}
        EOF
        )
        chatgpt_summary=$(
        cat <<"EOF"
        ${{ steps.chatgpt.outputs.summary }}
        EOF
        )

        echo "Test Result:  $test_result"
        echo "$test_status"

        if ! [ "$test_result" == "success" ]; then
          echo ""
          echo "Failed Test Task Status:"
          echo "$failed_test_status"

          if ! [ -z "$chatgpt_summary" ]; then
            echo ""
            echo "Failure summary (ChatGPT):"
            echo "$chatgpt_summary"
          fi


          echo ""
          echo "See 'Await assertoor test result' task or run artifact for detailed logs about this failure!"
          echo ""

          exit 1 # fail action
        fi