Migrate Jenkins jobs to Buildkite

.ci/Dockerfile → .buildkite/Dockerfile

@@ -2,9 +2,12 @@ ARG PYTHON_VERSION=3.8
 FROM python:${PYTHON_VERSION}
 
 WORKDIR /code/enterprise-search-python
-COPY . .
+
+COPY dev-requirements.txt .
 
 RUN python -m pip install \
     --disable-pip-version-check \
     --no-cache-dir \
     -r dev-requirements.txt
+
+COPY . .

.ci/certs/README.md → .buildkite/certs/README.md

@@ -8,7 +8,7 @@ The `ca.crt` and `ca.key` can be used to generate any other certificates that ma
 for CI. Perhaps the easiest way to do so is using 
 [`elasticsearch-certutil`](https://www.elastic.co/guide/en/elasticsearch/reference/current/certutil.html)
 
-Using the elasticsearch docker container, run the following from the `.ci/certs` directory
+Using the elasticsearch docker container, run the following from the `.buildkite/certs` directory
 
 ```sh
 docker run \

.ci/functions/cleanup.sh → .buildkite/functions/cleanup.sh

@@ -2,7 +2,7 @@
 #
 # Shared cleanup routines between different steps
 #
-# Please source .ci/functions/imports.sh as a whole not just this file
+# Please source .buildkite/functions/imports.sh as a whole not just this file
 #
 # Version 1.0.0
 # - Initial version after refactor

.ci/functions/wait-for-container.sh → .buildkite/functions/wait-for-container.sh

@@ -2,7 +2,7 @@
 #
 # Exposes a routine scripts can call to wait for a container if that container set up a health command
 #
-# Please source .ci/functions/imports.sh as a whole not just this file
+# Please source .buildkite/functions/imports.sh as a whole not just this file
 #
 # Version 1.0.1
 # - Initial version after refactor

.buildkite/pipeline.yml

@@ -1,3 +1,32 @@
+---
 steps:
-  - label: ":python: Greetings :elastic-enterprise-search:"
-    command: "echo 'Hello, world!'"
+  - label: ":python: :elastic-enterprise-search:"
+    agents:
+      provider: gcp
+    matrix:
+      setup:
+        python:
+          - "3.7"
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+        stack_version:
+          - "8.3.0-SNAPSHOT"
+    env:
+      PYTHON_VERSION: "{{ matrix.python }}"
+      STACK_VERSION: "{{ matrix.stack_version }}"
+    command: ./.buildkite/run-tests
+    artifact_paths: "*-junit.xml"
+  - wait: ~
+    continue_on_failure: true
+  - label: ":junit: Test results"
+    agents:
+      provider: "gcp"
+      image: family/core-ubuntu-2204
+    plugins:
+      - junit-annotate#v2.4.1:
+          artifacts: "junit/*-junit.xml"
+          job-uuid-file-pattern: "(.*)-junit.xml"
+          fail-build-on-error: true
+          failure-format: file

.ci/run-elasticsearch.sh → .buildkite/run-elasticsearch.sh

@@ -26,12 +26,16 @@ script_path=$(dirname $(realpath -s $0))
 source $script_path/functions/imports.sh
 set -euo pipefail
 
-echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on seperate terminals \033[0m"
+echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on separate terminals \033[0m"
 cleanup_node $es_node_name
 
 master_node_name=${es_node_name}
 cluster_name=${moniker}${suffix}
 
+if [[ "$BUILDKITE" == "true" ]]; then
+  sudo sysctl -w vm.max_map_count=262144
+fi
+
 declare -a volumes
 environment=($(cat <<-END
   --env ELASTIC_PASSWORD=$elastic_password

.ci/run-enterprise-search.sh → .buildkite/run-enterprise-search.sh

@@ -7,7 +7,7 @@
 
 # Version 1.1.0
 # - Initial version of the run-app-search.sh script
-# - Refactored .ci version
+# - Refactored .buildkite version
 
 script_path=$(dirname $(realpath -s $0))
 source $script_path/functions/imports.sh

.ci/run-repository.sh → .buildkite/run-repository.sh

@@ -38,21 +38,21 @@ echo -e "\033[32;1mSUCCESS:\033[0m successfully started the ${STACK_VERSION} sta
 echo -e "\033[34;1mINFO:\033[0m STACK_VERSION ${STACK_VERSION}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m PYTHON_VERSION ${PYTHON_VERSION}\033[0m"
 
-echo -e "\033[1m>>>>> Build [elastic/enterprise-search-python container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+echo ":docker: :python: :elastic-enterprise-search: Build elastic/enterprise-search-python image"
 
 docker build \
-       --file .ci/Dockerfile \
-       --tag elastic/enterprise-search-python \
-       --build-arg PYTHON_VERSION=${PYTHON_VERSION} \
-       .
+  --file .buildkite/Dockerfile \
+  --tag elastic/enterprise-search-python \
+  --build-arg PYTHON_VERSION="$PYTHON_VERSION" \
+  .
 
-echo -e "\033[1m>>>>> Run [elastic/enterprise-search-python container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+echo ":docker: :python: :elastic-enterprise-search: Run elastic/enterprise-search-python container"
 
-mkdir -p junit
+mkdir -p "$(pwd)/junit"
 docker run \
   --network ${network_name} \
   --name enterprise-search-python \
   --rm \
-  --volume `pwd`:/code/enterprise-search-python \
+  -v "$(pwd)/junit:/junit" \
   elastic/enterprise-search-python \
-  nox -s test-${PYTHON_VERSION}
+  bash -c "nox -s test-$PYTHON_VERSION; [ -f ./junit/$BUILDKITE_JOB_ID-junit.xml ] && mv ./junit/$BUILDKITE_JOB_ID-junit.xml /junit || echo 'No JUnit artifact found'"

.ci/run-tests → .buildkite/run-tests

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 #
 # Version 1.1
-# - Moved to .ci folder and seperated out `run-repository.sh`
+# - Moved to .buildkite folder and separated out `run-repository.sh`
 # - Add `$RUNSCRIPTS` env var for running Elasticsearch dependent products
 
 # Default environment variables
@@ -13,17 +13,17 @@ script_path=$(dirname $(realpath -s $0))
 source $script_path/functions/imports.sh
 set -euo pipefail
 
-echo -e "\033[1m>>>>> Start [$STACK_VERSION container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
-DETACH=true bash .ci/run-elasticsearch.sh
+echo "--- :python: :elastic-enterprise-search: Start $STACK_VERSION container"
+DETACH=true bash .buildkite/run-elasticsearch.sh
 
 if [[ -n "$RUNSCRIPTS" ]]; then
   for RUNSCRIPT in ${RUNSCRIPTS//,/ } ; do
-    echo -e "\033[1m>>>>> Running run-$RUNSCRIPT.sh >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+    echo -e "--- Running run-$RUNSCRIPT.sh"
     CONTAINER_NAME=${RUNSCRIPT} \
       DETACH=true \
-      bash .ci/run-${RUNSCRIPT}.sh
+      bash .buildkite/run-${RUNSCRIPT}.sh
   done
 fi
 
-echo -e "\033[1m>>>>> Repository specific tests >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
-bash .ci/run-repository.sh
+echo -e "--- Run repository specific tests"
+bash .buildkite/run-repository.sh

.ci/make.sh

@@ -115,7 +115,7 @@ esac
 echo -e "\033[34;1mINFO: building $product container\033[0m"
 
 docker build \
-  --file $repo/.ci/Dockerfile \
+  --file $repo/.buildkite/Dockerfile \
   --tag ${product} \
   .
 

CONTRIBUTING.md

@@ -49,7 +49,7 @@
 
 ## Running Integration Tests
 
-Run the full integration test suite with `$ .ci/run-tests`.
+Run the full integration test suite with `$ .buildkite/run-tests`.
 
 There are several environment variabels that control integration tests:
 

catalog-info.yaml

@@ -0,0 +1,44 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/catalog-info.json
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: enterprise-search-python
+spec:
+  type: library
+  owner: group:clients-team
+  lifecycle: production
+
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: enterprise-search-python-test
+  description: Test enterprise-search-python client
+spec:
+  type: buildkite-pipeline
+  owner: group:clients-team
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: enterprise-search-python-test
+    spec:
+      repository: elastic/enterprise-search-python
+      pipeline_file: .buildkite/pipeline.yaml
+      teams:
+        clients-team:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY
+      provider_settings:
+        build_pull_requests: true
+        build_branches: true
+      cancel_intermediate_builds: true
+      cancel_intermediate_builds_branch_filter: '!main'
+      schedules:
+        main_semi_daily:
+          branch: 'main'
+          cronline: '@daily'