ci.yml

name: CI

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

jobs:

  test:
    name: Test suite
    # List of supported runners:
    # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners
    runs-on: ubuntu-22.04

    env:
      COMPOSE_PROJECT_NAME: docker-elk

    steps:
      - uses: actions/checkout@v4

      #####################################################
      #                                                   #
      # Install all dependencies required by test suites. #
      #                                                   #
      #####################################################

      - name: Pre-build container images
        run: >-
          docker compose
          -f docker-compose.yml
          -f extensions/fleet/fleet-compose.yml
          -f extensions/fleet/agent-apmserver-compose.yml
          -f extensions/metricbeat/metricbeat-compose.yml
          -f extensions/filebeat/filebeat-compose.yml
          -f extensions/heartbeat/heartbeat-compose.yml
          -f extensions/enterprise-search/enterprise-search-compose.yml
          build

      ########################################################
      #                                                      #
      # Ensure §"Initial setup" of the README remains valid. #
      #                                                      #
      ########################################################

      - name: Set password of every built-in user to 'testpasswd'
        run: >-
          sed -i
          -e 's/\(ELASTIC_PASSWORD=\)'\''changeme'\''/\1testpasswd/g'
          -e 's/\(LOGSTASH_INTERNAL_PASSWORD=\)'\''changeme'\''/\1testpasswd/g'
          -e 's/\(KIBANA_SYSTEM_PASSWORD=\)'\''changeme'\''/\1testpasswd/g'
          -e 's/\(METRICBEAT_INTERNAL_PASSWORD=\)'\'\''/\1testpasswd/g'
          -e 's/\(FILEBEAT_INTERNAL_PASSWORD=\)'\'\''/\1testpasswd/g'
          -e 's/\(HEARTBEAT_INTERNAL_PASSWORD=\)'\'\''/\1testpasswd/g'
          -e 's/\(MONITORING_INTERNAL_PASSWORD=\)'\'\''/\1testpasswd/g'
          -e 's/\(BEATS_SYSTEM_PASSWORD=\)'\'\''/\1testpasswd/g'
          .env

      - name:  Set up users and roles
        run: docker compose up setup

      #############################
      #                           #
      # Test core and extensions. #
      #                           #
      #############################

      # Elasticsearch's high disk watermark gets regularly exceeded on GitHub Actions runners.
      # https://www.elastic.co/guide/en/elasticsearch/reference/8.10/fix-watermark-errors.html
      - name: Disable Elasticsearch disk allocation decider
        run: .github/workflows/scripts/disable-disk-alloc-decider.sh

      #
      # Core components: Elasticsearch, Logstash, Kibana
      #

      - name: Execute core test suite
        run: |
          docker compose up -d
          .github/workflows/scripts/run-tests-core.sh
          # next steps don't need Logstash
          docker compose stop logstash

      #
      # Fleet
      #

      - name: Execute Fleet test suite
        run: |
          docker compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml -f extensions/fleet/agent-apmserver-compose.yml up --remove-orphans -d fleet-server apm-server
          .github/workflows/scripts/run-tests-fleet.sh

      #
      # Metricbeat
      #

      - name: Execute Metricbeat test suite
        run: |
          docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml up --remove-orphans -d metricbeat
          .github/workflows/scripts/run-tests-metricbeat.sh

      #
      # Filebeat
      #

      - name: Execute Filebeat test suite
        run: |
          docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml up --remove-orphans -d filebeat
          .github/workflows/scripts/run-tests-filebeat.sh

      #
      # Heartbeat
      #

      - name: Execute Heartbeat test suite
        run: |
          docker compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml up --remove-orphans -d heartbeat
          .github/workflows/scripts/run-tests-heartbeat.sh

      #
      # Enterprise Search
      #

      - name: Execute Enterprise Search test suite
        run: |

          # Set mandatory Elasticsearch settings

          sed -i '$ a xpack.security.authc.api_key.enabled: true' elasticsearch/config/elasticsearch.yml

          # Restart Elasticsearch for changes to take effect

          docker compose restart elasticsearch

          # Run Enterprise Search and execute tests

          sed -i 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml

          docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up --remove-orphans -d enterprise-search
          .github/workflows/scripts/run-tests-enterprise-search.sh

          # Revert changes to Elasticsearch configuration

          sed -i '/xpack.security.authc.api_key.enabled: true/d' elasticsearch/config/elasticsearch.yml
          docker compose restart elasticsearch

      - name: Collect troubleshooting data
        id: debug-data
        if: failure()
        run: |
          declare debug_data_dir="$(mktemp -d)"

          docker compose \
          -f docker-compose.yml \
          -f extensions/fleet/fleet-compose.yml \
          -f extensions/fleet/agent-apmserver-compose.yml \
          -f extensions/metricbeat/metricbeat-compose.yml \
          -f extensions/filebeat/filebeat-compose.yml \
          -f extensions/heartbeat/heartbeat-compose.yml \
          -f extensions/enterprise-search/enterprise-search-compose.yml \
          ps >"$debug_data_dir"/docker_ps.log

          docker compose \
          -f docker-compose.yml \
          -f extensions/fleet/fleet-compose.yml \
          -f extensions/fleet/agent-apmserver-compose.yml \
          -f extensions/metricbeat/metricbeat-compose.yml \
          -f extensions/filebeat/filebeat-compose.yml \
          -f extensions/heartbeat/heartbeat-compose.yml \
          -f extensions/enterprise-search/enterprise-search-compose.yml \
          logs >"$debug_data_dir"/docker_logs.log

          echo "path=${debug_data_dir}" >>"$GITHUB_OUTPUT"

      - name: Upload collected troubleshooting data
        if: always() && steps.debug-data.outputs.path
        uses: actions/upload-artifact@v4
        with:
          name: debug-data
          path: ${{ steps.debug-data.outputs.path }}/*.*

        ##############
        #            #
        # Tear down. #
        #            #
        ##############

      - name: Terminate all components
        if: always()
        run: >-
          docker compose
          -f docker-compose.yml
          -f extensions/fleet/fleet-compose.yml
          -f extensions/fleet/agent-apmserver-compose.yml
          -f extensions/metricbeat/metricbeat-compose.yml
          -f extensions/filebeat/filebeat-compose.yml
          -f extensions/heartbeat/heartbeat-compose.yml
          -f extensions/enterprise-search/enterprise-search-compose.yml
          down -v