ci: Split integration tests into separate jobs

Signed-off-by: Brian Goff <[email protected]>
Azure · Dec 21, 2024 · 2033d8b · 2033d8b
1 parent fb56bc9
commit 2033d8b
Show file tree

Hide file tree

Showing 2 changed files with 79 additions and 35 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -60,9 +60,26 @@ jobs:
             exit 1
           fi
 
-
   integration:
     runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        suite:
+          - Mariner2
+          - Azlinux3
+          - Bookworm
+          - Bullseye
+          - Bionic
+          - Focal
+          - Jammy
+          - Noble
+          - Windows
+          - other
+        include:
+          - suite: other
+            skip: Mariner2|Azlinux3|Bookworm|Bullseye|Bionic|Focal|Jammy|Noble|Windows
+
     # TODO: support diff/merge
     # Right now this is handled by the e2e suite, but we can migrate that here.
     steps:
@@ -105,13 +122,31 @@ jobs:
         with:
           driver-opts: |
             network=host
-            env.OTLP_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318
+            env.OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318
             env.OTEL_SERVICE_NAME=buildkitd
 
       - name: download deps
         run: go mod download
+      - name: pre-cache worker image
+        run: |
+          if [ ! "${TEST_SUITE}" = "other" ]; then
+            go run ./cmd/ci-build-workers ${TEST_SUITE}
+          fi
+        env:
+          TEST_SUITE: ${{ matrix.suite }}
       - name: Run integration tests
-        run: go test -v -json --parallel 128 ./test | go run ./cmd/test2json2gha --slow 5s
+        run: |
+          set -ex
+          if [ -n "${TEST_SUITE}" ] && [ ! "${TEST_SUITE}" = "other" ]; then
+            run="-run=${TEST_SUITE}"
+          fi
+          if [ -n "${TEST_SKIP}" ]; then
+            skip="-skip=${TEST_SKIP}"
+          fi
+          go test -timeout=30m -v -json ${run} ${skip} ./test | go run ./cmd/test2json2gha --slow 120s
+        env:
+          TEST_SUITE: ${{ matrix.suite }}
+          TEST_SKIP: ${{ matrix.skip }}
       - name: dump logs
         if: failure()
         run: sudo journalctl -u docker
@@ -127,7 +162,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: integration-test-reports
+          name: integration-test-reports-${{matrix.suite}}
           path: /tmp/reports/*
           retention-days: 1
 
@@ -148,8 +183,6 @@ jobs:
           cache: false
       - name: download deps
         run: go mod download
-      - name: precache worker images
-        run: go run ./cmd/ci-build-workers
       - name: Run unit tests
         run: go test -v --test.short --json ./... | go run ./cmd/test2json2gha
 

diff --git a/cmd/ci-build-workers/main.go b/cmd/ci-build-workers/main.go
@@ -3,10 +3,12 @@ package main
 import (
 	"bytes"
 	"context"
+	"flag"
 	"fmt"
 	"os"
 	"os/signal"
 	"path"
+	"strings"
 	"sync"
 	"syscall"
 	"time"
@@ -27,6 +29,22 @@ import (
 )
 
 func main() {
+	flag.Parse()
+	targets := flag.Args()
+	if len(targets) == 0 {
+		targets = []string{
+			"mariner2",
+			"azlinux3",
+			"bionic",
+			"focal",
+			"jammy",
+			"noble",
+			"bullseye",
+			"bookworm",
+			"windowscross",
+		}
+	}
+
 	buildx := testenv.New()
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -57,13 +75,13 @@ func main() {
 		panic(err)
 	}
 
-	if err := do(ctx, client); err != nil {
+	if err := do(ctx, client, targets); err != nil {
 		fmt.Fprintln(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 }
 
-func do(ctx context.Context, c *client.Client) (retErr error) {
+func do(ctx context.Context, c *client.Client, targets []string) (retErr error) {
 	ch := make(chan *client.SolveStatus)
 	so := client.SolveOpt{}
 
@@ -99,39 +117,32 @@ func do(ctx context.Context, c *client.Client) (retErr error) {
 		}
 	}()
 
-	_, err = c.Build(ctx, so, "", build, ch)
+	_, err = c.Build(ctx, so, "", build(targets), ch)
 	return err
 }
 
-func build(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) {
-	targets := []string{
-		"mariner2",
-		"azlinux3",
-		"bionic",
-		"focal",
-		"jammy",
-		"noble",
-		"bullseye",
-		"bookworm",
-		"windowscross",
-	}
+func build(targets []string) gwclient.BuildFunc {
+	return func(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) {
+		eg, ctx := errgroup.WithContext(ctx)
 
-	eg, ctx := errgroup.WithContext(ctx)
+		bctx, err := llb.Scratch().File(llb.Mkfile("Dockerfile", 0o644, []byte("null"))).Marshal(ctx)
+		if err != nil {
+			return nil, errors.Wrap(err, "creating build context")
+		}
+		for _, t := range targets {
+			t := strings.ToLower(t)
+			if t == "windows" {
+				t = "windowscross"
+			}
+			eg.Go(func() error {
+				f := buildWorker(t, bctx.ToPB())
+				_, err := f(ctx, client)
+				return errors.Wrap(err, t)
+			})
+		}
 
-	bctx, err := llb.Scratch().File(llb.Mkfile("Dockerfile", 0o644, []byte("null"))).Marshal(ctx)
-	if err != nil {
-		return nil, errors.Wrap(err, "creating build context")
+		return gwclient.NewResult(), eg.Wait()
 	}
-	for _, t := range targets {
-		t := t
-		eg.Go(func() error {
-			f := buildWorker(t, bctx.ToPB())
-			_, err := f(ctx, client)
-			return errors.Wrap(err, t)
-		})
-	}
-
-	return gwclient.NewResult(), eg.Wait()
 }
 
 func buildWorker(t string, bctx *pb.Definition) gwclient.BuildFunc {