diff --git a/.buildkite/dockerfiles-pipeline.yml b/.buildkite/dockerfiles-pipeline.yml new file mode 100644 index 000000000..e8c8135de --- /dev/null +++ b/.buildkite/dockerfiles-pipeline.yml @@ -0,0 +1,124 @@ +definitions: + steps: + - step: &test-agents + agents: + provider: "gcp" + machineType: "n1-standard-8" + useVault: true + image: family/enterprise-search-ubuntu-2204-connectors-py + retries: &retries + retry: + automatic: + - exit_status: -1 # Connection to the Agent was lost + signal_reason: none + limit: 2 + - exit_status: 255 # Timeout + signal_reason: none + limit: 2 + - exit_status: 2 # Flaky test + signal_reason: none + limit: 2 + +steps: + - group: ":truck: Building, Testing and Scanning Dockerfile and Dockerfile.ftest" + key: "build_test_scan_group" + if: "(build.branch == \"main\")" + steps: + # ---- + # Dockerfile build and tests on amd64 + # ---- + - label: "Build amd64 image from Dockerfile" + agents: + provider: aws + instanceType: m6i.xlarge + imagePrefix: ci-amazonlinux-2 + env: + ARCHITECTURE: "amd64" + DOCKERFILE_PATH: "Dockerfile" + DOCKER_IMAGE_NAME: "docker.elastic.co/ci-agent-images/elastic-connectors-dockerfile" + DOCKER_ARTIFACT_KEY: "elastic-connectors-dockerfile" + command: ".buildkite/publish/build-docker.sh" + key: "build_dockerfile_image_amd64" + artifact_paths: ".artifacts/*.tar.gz" + - label: "Test amd64 image built from Dockerfile" + agents: + provider: aws + instanceType: m6i.xlarge + imagePrefix: ci-amazonlinux-2 + env: + ARCHITECTURE: "amd64" + DOCKERFILE_PATH: "Dockerfile" + DOCKER_IMAGE_NAME: "docker.elastic.co/ci-agent-images/elastic-connectors-dockerfile" + DOCKER_ARTIFACT_KEY: "elastic-connectors-dockerfile" + depends_on: "build_dockerfile_image_amd64" + key: "test_dockerfile_image_amd64" + commands: + - "mkdir -p .artifacts" + - buildkite-agent artifact download '.artifacts/*.tar.gz*' .artifacts/ --step build_dockerfile_image_amd64 + - ".buildkite/publish/test-docker.sh" + + # ---- + # Dockerfile.ftest build and tests on amd64 + # ---- + - label: "Build amd64 image from Dockerfile.ftest" + agents: + provider: aws + instanceType: m6i.xlarge + imagePrefix: ci-amazonlinux-2 + env: + ARCHITECTURE: "amd64" + DOCKERFILE_PATH: "Dockerfile.ftest" + DOCKER_IMAGE_NAME: "docker.elastic.co/ci-agent-images/elastic-connectors-dockerfile-ftest" + DOCKER_ARTIFACT_KEY: "elastic-connectors-dockerfile-ftest" + command: ".buildkite/publish/build-docker.sh" + key: "build_dockerfile_ftest_image_amd64" + artifact_paths: ".artifacts/*.tar.gz" + - label: "Test amd64 image built from Dockerfile.ftest" + agents: + provider: aws + instanceType: m6i.xlarge + imagePrefix: ci-amazonlinux-2 + env: + ARCHITECTURE: "amd64" + DOCKERFILE_PATH: "Dockerfile.ftest" + DOCKER_IMAGE_NAME: "docker.elastic.co/ci-agent-images/elastic-connectors-dockerfile-ftest" + DOCKER_ARTIFACT_KEY: "elastic-connectors-dockerfile-ftest" + depends_on: "build_dockerfile_ftest_image_amd64" + key: "test_dockerfile_ftest_image_amd64" + commands: + - "mkdir -p .artifacts" + - buildkite-agent artifact download '.artifacts/*.tar.gz*' .artifacts/ --step build_dockerfile_ftest_image_amd64 + - ".buildkite/publish/test-docker.sh" + + + # ---- + # Vulnerability scanning on Dockerfile and Dockerfile.ftest built images + # ---- + - label: "Trivy Scan amd64 Dockerfile Artifacts" + timeout_in_minutes: 10 + depends_on: + - test_dockerfile_image_amd64 + key: "trivy-scan-dockerfile-image" + agents: + provider: k8s + image: "docker.elastic.co/ci-agent-images/trivy:latest" + command: |- + mkdir -p .artifacts + buildkite-agent artifact download '.artifacts/*.tar.gz*' .artifacts/ --step build_dockerfile_image_amd64 + trivy --version + env | grep TRIVY + find .artifacts -type f -name '*.tar.gz*' -exec trivy image --quiet --input {} \; + - label: "Trivy Scan amd64 Dockerfile.ftest Artifacts" + timeout_in_minutes: 10 + depends_on: + - test_dockerfile_ftest_image_amd64 + key: "trivy-scan-dockerfile-ftest-image" + agents: + provider: k8s + image: "docker.elastic.co/ci-agent-images/trivy:latest" + command: |- + mkdir -p release + buildkite-agent artifact download '.artifacts/*.tar.gz*' .artifacts/ --step build_dockerfile_ftest_image_amd64 + trivy --version + env | grep TRIVY + find .artifacts -type f -name '*.tar.gz*' -exec trivy image --quiet --input {} \; diff --git a/Dockerfile.ftest b/Dockerfile.ftest index bc1d556e4..0e5e0de4d 100644 --- a/Dockerfile.ftest +++ b/Dockerfile.ftest @@ -8,5 +8,4 @@ COPY --chown=nonroot:nonroot . /app USER nonroot WORKDIR /app -RUN make clean install -RUN .venv/bin/pip install -r requirements/ftest.txt +RUN make clean install && .venv/bin/pip install -r requirements/ftest.txt diff --git a/catalog-info.yaml b/catalog-info.yaml index 3416d91b0..64c6a6057 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -177,6 +177,39 @@ spec: search-extract-and-transform: {} search-productivity-team: {} +# Dockerfiles nightly images build pipeline +--- +apiVersion: "backstage.io/v1alpha1" +kind: "Resource" +metadata: + name: "connectors-dockerfiles-nightly" + description: "Nightly Connectors Dockerfiles Build" +spec: + type: "buildkite-pipeline" + owner: "group:search-extract-and-transform" + system: "buildkite" + implementation: + apiVersion: "buildkite.elastic.dev/v1" + kind: "Pipeline" + metadata: + name: "connectors-dockerfiles-nightly" + description: "Nightly Connectors Dockerfiles Build" + spec: + pipeline_file: ".buildkite/dockerfiles-pipeline.yml" + provider_settings: + trigger_mode: "none" + repository: "elastic/connectors" + schedules: + Daily main: + branch: main + cronline: '@daily' + message: "Runs daily `main` Dockerfiles image builds" + teams: + everyone: + access_level: "READ_ONLY" + search-extract-and-transform: {} + search-productivity-team: {} + ######## # Docker image build and publish - manual release ########