Skip to content

Commit

Permalink
add buildkite pipeline to build test and scan images
Browse files Browse the repository at this point in the history
  • Loading branch information
kostasb committed Dec 27, 2024
1 parent 02b9176 commit d6c5b45
Show file tree
Hide file tree
Showing 3 changed files with 158 additions and 2 deletions.
124 changes: 124 additions & 0 deletions .buildkite/dockerfiles-pipeline.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
definitions:
steps:
- step: &test-agents
agents:
provider: "gcp"
machineType: "n1-standard-8"
useVault: true
image: family/enterprise-search-ubuntu-2204-connectors-py
retries: &retries
retry:
automatic:
- exit_status: -1 # Connection to the Agent was lost
signal_reason: none
limit: 2
- exit_status: 255 # Timeout
signal_reason: none
limit: 2
- exit_status: 2 # Flaky test
signal_reason: none
limit: 2

steps:
- group: ":truck: Building, Testing and Scanning Dockerfile and Dockerfile.ftest"
key: "build_test_scan_group"
if: "(build.branch == \"main\")"
steps:
# ----
# Dockerfile build and tests on amd64
# ----
- label: "Build amd64 image from Dockerfile"
agents:
provider: aws
instanceType: m6i.xlarge
imagePrefix: ci-amazonlinux-2
env:
ARCHITECTURE: "amd64"
DOCKERFILE_PATH: "Dockerfile"
DOCKER_IMAGE_NAME: "docker.elastic.co/ci-agent-images/elastic-connectors-dockerfile"
DOCKER_ARTIFACT_KEY: "elastic-connectors-dockerfile"
command: ".buildkite/publish/build-docker.sh"
key: "build_dockerfile_image_amd64"
artifact_paths: ".artifacts/*.tar.gz"
- label: "Test amd64 image built from Dockerfile"
agents:
provider: aws
instanceType: m6i.xlarge
imagePrefix: ci-amazonlinux-2
env:
ARCHITECTURE: "amd64"
DOCKERFILE_PATH: "Dockerfile"
DOCKER_IMAGE_NAME: "docker.elastic.co/ci-agent-images/elastic-connectors-dockerfile"
DOCKER_ARTIFACT_KEY: "elastic-connectors-dockerfile"
depends_on: "build_dockerfile_image_amd64"
key: "test_dockerfile_image_amd64"
commands:
- "mkdir -p .artifacts"
- buildkite-agent artifact download '.artifacts/*.tar.gz*' .artifacts/ --step build_dockerfile_image_amd64
- ".buildkite/publish/test-docker.sh"

# ----
# Dockerfile.ftest build and tests on amd64
# ----
- label: "Build amd64 image from Dockerfile.ftest"
agents:
provider: aws
instanceType: m6i.xlarge
imagePrefix: ci-amazonlinux-2
env:
ARCHITECTURE: "amd64"
DOCKERFILE_PATH: "Dockerfile.ftest"
DOCKER_IMAGE_NAME: "docker.elastic.co/ci-agent-images/elastic-connectors-dockerfile-ftest"
DOCKER_ARTIFACT_KEY: "elastic-connectors-dockerfile-ftest"
command: ".buildkite/publish/build-docker.sh"
key: "build_dockerfile_ftest_image_amd64"
artifact_paths: ".artifacts/*.tar.gz"
- label: "Test amd64 image built from Dockerfile.ftest"
agents:
provider: aws
instanceType: m6i.xlarge
imagePrefix: ci-amazonlinux-2
env:
ARCHITECTURE: "amd64"
DOCKERFILE_PATH: "Dockerfile.ftest"
DOCKER_IMAGE_NAME: "docker.elastic.co/ci-agent-images/elastic-connectors-dockerfile-ftest"
DOCKER_ARTIFACT_KEY: "elastic-connectors-dockerfile-ftest"
depends_on: "build_dockerfile_ftest_image_amd64"
key: "test_dockerfile_ftest_image_amd64"
commands:
- "mkdir -p .artifacts"
- buildkite-agent artifact download '.artifacts/*.tar.gz*' .artifacts/ --step build_dockerfile_ftest_image_amd64
- ".buildkite/publish/test-docker.sh"


# ----
# Vulnerability scanning on Dockerfile and Dockerfile.ftest built images
# ----
- label: "Trivy Scan amd64 Dockerfile Artifacts"
timeout_in_minutes: 10
depends_on:
- test_dockerfile_image_amd64
key: "trivy-scan-dockerfile-image"
agents:
provider: k8s
image: "docker.elastic.co/ci-agent-images/trivy:latest"
command: |-
mkdir -p .artifacts
buildkite-agent artifact download '.artifacts/*.tar.gz*' .artifacts/ --step build_dockerfile_image_amd64
trivy --version
env | grep TRIVY
find .artifacts -type f -name '*.tar.gz*' -exec trivy image --quiet --input {} \;
- label: "Trivy Scan amd64 Dockerfile.ftest Artifacts"
timeout_in_minutes: 10
depends_on:
- test_dockerfile_ftest_image_amd64
key: "trivy-scan-dockerfile-ftest-image"
agents:
provider: k8s
image: "docker.elastic.co/ci-agent-images/trivy:latest"
command: |-
mkdir -p release
buildkite-agent artifact download '.artifacts/*.tar.gz*' .artifacts/ --step build_dockerfile_ftest_image_amd64
trivy --version
env | grep TRIVY
find .artifacts -type f -name '*.tar.gz*' -exec trivy image --quiet --input {} \;
3 changes: 1 addition & 2 deletions Dockerfile.ftest
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,4 @@ COPY --chown=nonroot:nonroot . /app

USER nonroot
WORKDIR /app
RUN make clean install
RUN .venv/bin/pip install -r requirements/ftest.txt
RUN make clean install && .venv/bin/pip install -r requirements/ftest.txt
33 changes: 33 additions & 0 deletions catalog-info.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,39 @@ spec:
search-extract-and-transform: {}
search-productivity-team: {}

# Dockerfiles nightly images build pipeline
---
apiVersion: "backstage.io/v1alpha1"
kind: "Resource"
metadata:
name: "connectors-dockerfiles-nightly"
description: "Nightly Connectors Dockerfiles Build"
spec:
type: "buildkite-pipeline"
owner: "group:search-extract-and-transform"
system: "buildkite"
implementation:
apiVersion: "buildkite.elastic.dev/v1"
kind: "Pipeline"
metadata:
name: "connectors-dockerfiles-nightly"
description: "Nightly Connectors Dockerfiles Build"
spec:
pipeline_file: ".buildkite/dockerfiles-pipeline.yml"
provider_settings:
trigger_mode: "none"
repository: "elastic/connectors"
schedules:
Daily main:
branch: main
cronline: '@daily'
message: "Runs daily `main` Dockerfiles image builds"
teams:
everyone:
access_level: "READ_ONLY"
search-extract-and-transform: {}
search-productivity-team: {}

########
# Docker image build and publish - manual release
########
Expand Down

0 comments on commit d6c5b45

Please sign in to comment.